From 864cbb0fa0ce1f5ce6a41a73d2c4ad78bbd89adf Mon Sep 17 00:00:00 2001 From: Miguel Cruz Date: Wed, 15 Jul 2026 19:02:34 -0400 Subject: [PATCH] [AGC/Vulkan] Extend PS5 runtime and rendering compatibility (#216) * [Core] Add POSIX native execution and PS5 SELF support Extend the native backend, guest TLS, fixed-address memory, and loader paths needed by PS5 titles on Windows, Linux, and macOS. Keep workstation GC so high-core-count hosts do not reserve over fixed guest image bases. * [HLE] Expand PS5 service and media compatibility Add the kernel, threading, save-data, networking, audio, video-codec, font, dialog, and service exports required by newer PS5 software. Preserve every SysAbi NID currently registered by main while adding the compatibility surface used by ASTRO BOT. * [AGC/Vulkan] Extend Gen5 shader and presentation support Expand PM4 handling, Gen5 shader translation, MRT and packed export support, guest image tracking, depth initialization, texture aliasing, and Vulkan presentation. Add the performance overlay and address-filtered diagnostics used to validate ASTRO BOT with original shaders. * [Core] Align static TLS reservation across hosts * [Pad] Align primary user ID with UserService * [Gpu] Preserve runtime scalar buffers across renderer seam * [AGC] Restore omitted command helper exports * [Vulkan] Reuse primary views for promoted MRT targets * [Vulkan] Preserve scratch storage bindings in compute dispatches --- docs/bink2-bridge.md | 46 + native/bink2-bridge/sharpemu_bink2_bridge.c | 51 + src/SharpEmu.CLI/Program.cs | 29 + src/SharpEmu.CLI/SharpEmu.CLI.csproj | 6 +- src/SharpEmu.Core/Cpu/CpuDispatcher.cs | 113 +- .../DirectExecutionBackend.Diagnostics.cs | 242 +- .../DirectExecutionBackend.Exceptions.cs | 430 +- .../Native/DirectExecutionBackend.Imports.cs | 1682 ++-- .../DirectExecutionBackend.NativeWorker.cs | 68 +- .../DirectExecutionBackend.PosixSignals.cs | 47 +- .../Cpu/Native/DirectExecutionBackend.cs | 3282 +++++-- .../Cpu/Native/PosixHostStubs.cs | 425 + src/SharpEmu.Core/Cpu/Native/StubManager.cs | 37 +- src/SharpEmu.Core/Loader/ProgramHeader.cs | 2 + src/SharpEmu.Core/Loader/SelfImage.cs | 15 +- src/SharpEmu.Core/Loader/SelfLoader.cs | 621 +- .../Memory/PhysicalVirtualMemory.cs | 161 +- src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs | 311 +- src/SharpEmu.HLE/CpuContext.cs | 6 + src/SharpEmu.HLE/GuestImageWriteTracker.cs | 613 ++ src/SharpEmu.HLE/GuestThreadExecution.cs | 114 +- src/SharpEmu.HLE/GuestTlsTemplate.cs | 442 + src/SharpEmu.HLE/HleDataSymbols.cs | 4 +- src/SharpEmu.HLE/HostMainThread.cs | 11 +- src/SharpEmu.HLE/HostMemory.cs | 498 + src/SharpEmu.Libs/Agc/AgcExports.cs | 7588 ++++++++++++--- .../Agc/AgcPrimaryRegisterDefaults.cs | 143 + .../Agc/AgcShaderCompilerHooks.cs | 3 + src/SharpEmu.Libs/Agc/GnmTiling.cs | 486 + src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs | 195 +- src/SharpEmu.Libs/Ajm/AjmExports.cs | 24 +- src/SharpEmu.Libs/Ampr/AmprExports.cs | 90 +- src/SharpEmu.Libs/Audio/AjmExports.cs | 104 + src/SharpEmu.Libs/Audio/AudioOut2Exports.cs | 253 +- src/SharpEmu.Libs/Audio/AudioOutExports.cs | 45 +- src/SharpEmu.Libs/Audio/AudioPcmConversion.cs | 11 +- src/SharpEmu.Libs/Audio/FmodCompatExports.cs | 47 + src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs | 1265 ++- src/SharpEmu.Libs/Bink/Bink2MovieBridge.cs | 403 + src/SharpEmu.Libs/Codec/CodecExports.cs | 104 + .../CommonDialog/ErrorDialogExports.cs | 106 + src/SharpEmu.Libs/Fiber/FiberExports.cs | 561 +- src/SharpEmu.Libs/Font/FontExports.cs | 293 + src/SharpEmu.Libs/Gpu/GuestGpuTypes.cs | 57 +- src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs | 67 +- .../Gpu/Vulkan/VulkanGuestGpuBackend.cs | 130 +- src/SharpEmu.Libs/HostTiming.cs | 82 + src/SharpEmu.Libs/Json/JsonExports.cs | 561 +- .../Kernel/KernelEventFlagCompatExports.cs | 379 +- .../Kernel/KernelEventQueueCompatExports.cs | 87 +- .../Kernel/KernelExceptionCompatExports.cs | 71 + src/SharpEmu.Libs/Kernel/KernelExports.cs | 111 +- .../Kernel/KernelFileExtendedExports.cs | 662 ++ .../Kernel/KernelMemoryCompatExports.cs | 2026 ++-- .../Kernel/KernelModuleRegistry.cs | 123 + .../Kernel/KernelPthreadCompatExports.cs | 1137 ++- .../KernelPthreadExtendedCompatExports.cs | 161 +- .../Kernel/KernelRuntimeCompatExports.cs | 781 +- .../Kernel/KernelSemaphoreCompatExports.cs | 635 +- src/SharpEmu.Libs/LibcStdioExports.cs | 18 +- src/SharpEmu.Libs/Network/NetExports.cs | 306 +- src/SharpEmu.Libs/Ngs2/Ngs2Exports.cs | 273 +- src/SharpEmu.Libs/Np/NpManagerExports.cs | 108 +- src/SharpEmu.Libs/Pad/HostWindowInput.cs | 5 + src/SharpEmu.Libs/Pad/ImeExports.cs | 41 + src/SharpEmu.Libs/Pad/MouseExports.cs | 100 + src/SharpEmu.Libs/Pad/PadExports.cs | 6 +- src/SharpEmu.Libs/SaveData/SaveDataExports.cs | 297 +- src/SharpEmu.Libs/Stubs/GameServiceStubs.cs | 62 + .../SystemService/SystemServiceExports.cs | 52 + .../UserService/UserServiceExports.cs | 238 +- src/SharpEmu.Libs/VideoOut/PerfOverlay.cs | 363 + src/SharpEmu.Libs/VideoOut/VideoOutExports.cs | 952 +- .../VideoOut/VulkanVideoPresenter.cs | 8124 ++++++++++++++--- .../Gen5SpirvTranslator.Alu.cs | 1050 ++- .../Gen5SpirvTranslator.cs | 3239 ++++++- .../SpirvFixedShaders.cs | 96 + .../SpirvModuleBuilder.cs | 1 + src/SharpEmu.ShaderCompiler/Gen5ShaderIr.cs | 88 +- .../Gen5ShaderMetadataReader.cs | 48 +- .../Gen5ShaderScalarEvaluator.cs | 1215 ++- .../Gen5ShaderTranslator.cs | 564 +- .../Gfx10UnifiedFormat.cs | 138 + .../Tls/GuestTlsTemplateTests.cs | 32 + tools/SharpEmu.Tools.ShaderDump/Program.cs | 12 +- 85 files changed, 36299 insertions(+), 9176 deletions(-) create mode 100644 docs/bink2-bridge.md create mode 100644 native/bink2-bridge/sharpemu_bink2_bridge.c create mode 100644 src/SharpEmu.Core/Cpu/Native/PosixHostStubs.cs create mode 100644 src/SharpEmu.HLE/GuestImageWriteTracker.cs create mode 100644 src/SharpEmu.HLE/GuestTlsTemplate.cs create mode 100644 src/SharpEmu.HLE/HostMemory.cs create mode 100644 src/SharpEmu.Libs/Agc/AgcPrimaryRegisterDefaults.cs create mode 100644 src/SharpEmu.Libs/Agc/GnmTiling.cs create mode 100644 src/SharpEmu.Libs/Audio/AjmExports.cs create mode 100644 src/SharpEmu.Libs/Audio/FmodCompatExports.cs create mode 100644 src/SharpEmu.Libs/Bink/Bink2MovieBridge.cs create mode 100644 src/SharpEmu.Libs/Codec/CodecExports.cs create mode 100644 src/SharpEmu.Libs/CommonDialog/ErrorDialogExports.cs create mode 100644 src/SharpEmu.Libs/Font/FontExports.cs create mode 100644 src/SharpEmu.Libs/HostTiming.cs create mode 100644 src/SharpEmu.Libs/Kernel/KernelFileExtendedExports.cs create mode 100644 src/SharpEmu.Libs/Pad/ImeExports.cs create mode 100644 src/SharpEmu.Libs/Pad/MouseExports.cs create mode 100644 src/SharpEmu.Libs/Stubs/GameServiceStubs.cs create mode 100644 src/SharpEmu.Libs/VideoOut/PerfOverlay.cs create mode 100644 src/SharpEmu.ShaderCompiler/Gfx10UnifiedFormat.cs create mode 100644 tests/SharpEmu.Libs.Tests/Tls/GuestTlsTemplateTests.cs diff --git a/docs/bink2-bridge.md b/docs/bink2-bridge.md new file mode 100644 index 0000000..6ea35a5 --- /dev/null +++ b/docs/bink2-bridge.md @@ -0,0 +1,46 @@ + + +# Bink 2 bridge + +Demon's Souls plays Bink 2 (.bk2) files through a Bink implementation linked +directly into eboot.bin. It does not use libSceVideodec, therefore an HLE video +decoder cannot observe or replace those frames. + +SharpEmu observes successful guest .bk2 opens and, when a Bink bridge is +available, presents its decoded BGRA frames at the normal guest-flip boundary. +This preserves the game's own timing and lets the host Vulkan presenter display +the movie without trying to execute the PS5-specific Bink GPU decode path. + +Without an adapter, Bink movies are skipped by default: their open call returns +not-found so games that mark cinematics as optional progress to their next +state instead of waiting on an empty Bink GPU texture. + +Set SHARPEMU_BINK_MODE=dummy to retain the open and show a built-in, +non-decoded placeholder frame. This requires no SDK, but is a visual diagnostic +only; it does not decode the movie or alter its game logic. Set +SHARPEMU_BINK_MODE=native to force native bridge mode. + +## Supplying the adapter + +Bink 2 is proprietary. Obtain a compatible Mac Bink 2 SDK from RAD Game Tools, +then compile sharpemu_bink2_bridge.c against the SDK's bink.h and Mac library. +The adapter deliberately contains only a three-function C ABI so the managed +emulator never depends on RAD's private binary ABI. + +Place the resulting libsharpemu_bink2_bridge.dylib next to the SharpEmu +executable, or point to it explicitly: + + SHARPEMU_BINK2_BRIDGE=/absolute/path/libsharpemu_bink2_bridge.dylib \ + ./SharpEmu /path/to/eboot.bin + +The expected exports are sharpemu_bink2_open_utf8, +sharpemu_bink2_decode_next_bgra, and sharpemu_bink2_close. The supplied +adapter opens one movie, exposes BGRA pixels, and advances after each decoded +frame. The managed side validates dimensions and retains ownership of the +destination buffer. + +If the bridge is absent in native mode, SharpEmu logs one informational line +and retains the existing guest rendering path. diff --git a/native/bink2-bridge/sharpemu_bink2_bridge.c b/native/bink2-bridge/sharpemu_bink2_bridge.c new file mode 100644 index 0000000..a2db193 --- /dev/null +++ b/native/bink2-bridge/sharpemu_bink2_bridge.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2026 SharpEmu Emulator Project + * SPDX-License-Identifier: GPL-2.0-or-later + * + * Build this small adapter with a licensed RAD Bink 2 SDK. The SDK and its + * headers are not distributed by SharpEmu. See docs/bink2-bridge.md. + */ + +#include +#include "bink.h" + +typedef struct sharpemu_bink2_info { + uint32_t width; + uint32_t height; + uint32_t frames_per_second_numerator; + uint32_t frames_per_second_denominator; +} sharpemu_bink2_info; + +int sharpemu_bink2_open_utf8(const char *path, HBINK *movie, sharpemu_bink2_info *info) { + HBINK bink; + if (!path || !movie || !info) return 0; + + bink = BinkOpen(path, 0); + if (!bink) return 0; + + *movie = bink; + info->width = bink->Width; + info->height = bink->Height; + info->frames_per_second_numerator = bink->FrameRate; + info->frames_per_second_denominator = bink->FrameRateDiv; + return 1; +} + +int sharpemu_bink2_decode_next_bgra(HBINK movie, uint8_t *destination, + uint32_t stride, uint32_t destination_bytes) { + uint64_t needed; + if (!movie || !destination || stride < movie->Width * 4) return 0; + needed = (uint64_t)stride * movie->Height; + if (needed > destination_bytes) return 0; + + /* Async Bink I/O has not filled the next frame yet; retry on the next host present. */ + if (BinkWait(movie)) return 0; + BinkDoFrame(movie); + BinkCopyToBuffer(movie, destination, stride, movie->Height, 0, 0, BINKSURFACE32RA); + BinkNextFrame(movie); + return 1; +} + +void sharpemu_bink2_close(HBINK movie) { + if (movie) BinkClose(movie); +} diff --git a/src/SharpEmu.CLI/Program.cs b/src/SharpEmu.CLI/Program.cs index 26e56b3..c0dd2bb 100644 --- a/src/SharpEmu.CLI/Program.cs +++ b/src/SharpEmu.CLI/Program.cs @@ -45,6 +45,11 @@ internal static partial class Program [STAThread] private static int Main(string[] args) { + // Avoid blocking full collections while guest and render threads are + // running, and establish the GC mode before the runtime reserves the + // fixed guest address-space window. + System.Runtime.GCSettings.LatencyMode = System.Runtime.GCLatencyMode.SustainedLowLatency; + try { return Run(args); @@ -84,6 +89,7 @@ internal static partial class Program { if (OperatingSystem.IsMacOS()) { + ConfigureMoltenVkDefaults(); PreloadMacVulkanLoader(); } @@ -151,6 +157,26 @@ internal static partial class Program return false; } + /// + /// Applies MoltenVK performance defaults before the Vulkan loader is + /// loaded. Existing user-provided values always take precedence. + /// + private static void ConfigureMoltenVkDefaults() + { + try + { + _ = MacSetEnv("MVK_CONFIG_SYNCHRONOUS_QUEUE_SUBMITS", "0", 0); + _ = MacSetEnv("MVK_CONFIG_SHOULD_MAXIMIZE_CONCURRENT_COMPILATION", "1", 0); + _ = MacSetEnv("MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS", "1", 0); + _ = MacSetEnv("MVK_CONFIG_RESUME_LOST_DEVICE", "1", 0); + } + catch (Exception exception) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Failed to set MoltenVK defaults: {exception.Message}"); + } + } + /// /// Makes a Vulkan loader visible to GLFW's dlopen("libvulkan.1.dylib"). /// Homebrew's Vulkan libraries are arm64-only and cannot load into this @@ -1319,4 +1345,7 @@ internal static partial class Program uint creationDisposition, uint flagsAndAttributes, nint templateFile); + + [DllImport("libSystem", EntryPoint = "setenv")] + private static extern int MacSetEnv(string name, string value, int overwrite); } diff --git a/src/SharpEmu.CLI/SharpEmu.CLI.csproj b/src/SharpEmu.CLI/SharpEmu.CLI.csproj index 219d12f..5b9ddc8 100644 --- a/src/SharpEmu.CLI/SharpEmu.CLI.csproj +++ b/src/SharpEmu.CLI/SharpEmu.CLI.csproj @@ -25,7 +25,11 @@ SPDX-License-Identifier: GPL-2.0-or-later true enable true - true + + false true true diff --git a/src/SharpEmu.Core/Cpu/CpuDispatcher.cs b/src/SharpEmu.Core/Cpu/CpuDispatcher.cs index 28e1cb1..02fad8c 100644 --- a/src/SharpEmu.Core/Cpu/CpuDispatcher.cs +++ b/src/SharpEmu.Core/Cpu/CpuDispatcher.cs @@ -7,15 +7,11 @@ using SharpEmu.Core.Cpu.Native; using SharpEmu.Core.Loader; using SharpEmu.Core.Memory; using SharpEmu.HLE; -using SharpEmu.HLE.Host; -using SharpEmu.Logging; namespace SharpEmu.Core.Cpu; public sealed class CpuDispatcher : ICpuDispatcher, IDisposable { - private static readonly SharpEmuLogger Log = SharpEmuLog.For("Dispatcher"); - private enum EntryFrameKind { ProcessEntry, @@ -31,9 +27,9 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable private static readonly ulong TlsBaseAddress = OperatingSystem.IsWindows() ? 0x7FFE_0000_0000UL : 0x6FFE_0000_0000UL; private const ulong TlsSize = 0x0001_0000UL; // The static TLS blocks live at negative offsets from the TCB (FreeBSD - // amd64 variant II); libc.prx alone reaches beyond -0x1700, so give the - // prefix a full 64KB on POSIX. Windows keeps its historical 4KB prefix. - private static readonly ulong TlsPrefixSize = OperatingSystem.IsWindows() ? 0x0000_1000UL : 0x0001_0000UL; + // amd64 variant II). Keep every host in sync with GuestTlsTemplate's + // startup reservation; PS5 modules routinely reach beyond one host page. + private const ulong TlsPrefixSize = GuestTlsTemplate.StartupStaticTlsReservation; private static readonly ulong BootstrapStubBaseAddress = OperatingSystem.IsWindows() ? 0x7FFD_F000_0000UL : 0x6FFD_F000_0000UL; private static readonly ulong BootstrapPayloadBaseAddress = OperatingSystem.IsWindows() ? 0x7FFD_E000_0000UL : 0x6FFD_E000_0000UL; private static readonly ulong DynlibFallbackStubBaseAddress = OperatingSystem.IsWindows() ? 0x7FFD_D000_0000UL : 0x6FFD_D000_0000UL; @@ -49,19 +45,16 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable ]; private readonly IVirtualMemory _virtualMemory; private readonly IModuleManager _moduleManager; - private readonly IHostPlatform? _hostPlatform; private INativeCpuBackend? _nativeCpuBackend; public CpuDispatcher( IVirtualMemory virtualMemory, IModuleManager moduleManager, - INativeCpuBackend? nativeCpuBackend = null, - IHostPlatform? hostPlatform = null) + INativeCpuBackend? nativeCpuBackend = null) { _virtualMemory = virtualMemory ?? throw new ArgumentNullException(nameof(virtualMemory)); _moduleManager = moduleManager ?? throw new ArgumentNullException(nameof(moduleManager)); _nativeCpuBackend = nativeCpuBackend; - _hostPlatform = hostPlatform; } public ulong? LastEntryPoint { get; private set; } @@ -94,8 +87,8 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable string processImageName = "eboot.bin", CpuExecutionOptions executionOptions = default) { - Log.Debug("=== DispatchEntry START ==="); - Log.Debug($"entryPoint=0x{entryPoint:X16}, generation={generation}"); + Console.Error.WriteLine("[DISPATCHER] === DispatchEntry START ==="); + Console.Error.WriteLine($"[DISPATCHER] entryPoint=0x{entryPoint:X16}, generation={generation}"); try { @@ -103,7 +96,8 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable } catch (Exception ex) { - Log.Critical($"FATAL EXCEPTION in DispatchEntry: {ex.GetType().Name}: {ex.Message}", ex); + Console.Error.WriteLine($"[DISPATCHER] FATAL EXCEPTION in DispatchEntry: {ex.GetType().Name}: {ex.Message}"); + Console.Error.WriteLine($"[DISPATCHER] Stack trace: {ex.StackTrace}"); throw; } } @@ -116,8 +110,8 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable string moduleName = "module", CpuExecutionOptions executionOptions = default) { - Log.Debug("=== DispatchModuleInitializer START ==="); - Log.Debug($"moduleInit=0x{entryPoint:X16}, generation={generation}, module={moduleName}"); + Console.Error.WriteLine("[DISPATCHER] === DispatchModuleInitializer START ==="); + Console.Error.WriteLine($"[DISPATCHER] moduleInit=0x{entryPoint:X16}, generation={generation}, module={moduleName}"); try { @@ -132,7 +126,8 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable } catch (Exception ex) { - Log.Critical($"FATAL EXCEPTION in DispatchModuleInitializer: {ex.GetType().Name}: {ex.Message}", ex); + Console.Error.WriteLine($"[DISPATCHER] FATAL EXCEPTION in DispatchModuleInitializer: {ex.GetType().Name}: {ex.Message}"); + Console.Error.WriteLine($"[DISPATCHER] Stack trace: {ex.StackTrace}"); throw; } } @@ -146,7 +141,7 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable CpuExecutionOptions executionOptions = default, EntryFrameKind frameKind = EntryFrameKind.ProcessEntry) { - Log.Debug("DispatchEntryCore STARTING..."); + Console.Error.WriteLine("[DISPATCHER] DispatchEntryCore STARTING..."); LastEntryPoint = entryPoint; LastTrapInfo = null; @@ -277,7 +272,7 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable entryFrameDiagnostic, Environment.NewLine, "CpuEngine: native-only"); - _nativeCpuBackend ??= new DirectExecutionBackend(_moduleManager, _hostPlatform); + _nativeCpuBackend ??= new DirectExecutionBackend(_moduleManager); if (_nativeCpuBackend.TryExecute( context, entryPoint, @@ -318,7 +313,7 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable LastMilestoneLog, Environment.NewLine, $"CpuEngine native-only failed: {backendError}"); - Log.Error($"Native backend FAILED: {backendError}"); + Console.Error.WriteLine($"[DISPATCHER] Native backend FAILED: {backendError}"); return FailEarly( OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_IMPLEMENTED, CpuExitReason.NativeBackendUnavailable); @@ -377,11 +372,19 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable private static bool InitializeTls(CpuContext context, ulong tlsBase) { - return context.TryWriteUInt64(tlsBase - 0xF0, 0) && - context.TryWriteUInt64(tlsBase + 0x00, tlsBase) && - context.TryWriteUInt64(tlsBase + 0x10, tlsBase) && - context.TryWriteUInt64(tlsBase + 0x28, 0xC0DEC0DECAFEBABEUL) && - context.TryWriteUInt64(tlsBase + 0x60, tlsBase); + if (!context.TryWriteUInt64(tlsBase - 0xF0, 0) || + !context.TryWriteUInt64(tlsBase + 0x00, tlsBase) || + !context.TryWriteUInt64(tlsBase + 0x10, tlsBase) || + !context.TryWriteUInt64(tlsBase + 0x28, 0xC0DEC0DECAFEBA00UL) || + !context.TryWriteUInt64(tlsBase + 0x60, tlsBase)) + { + return false; + } + + // Seed the static TLS block below the thread pointer with the main + // module's initialized thread-locals (variant II layout). + SharpEmu.HLE.GuestTlsTemplate.SeedThreadBlock(context, tlsBase); + return true; } private static bool InitializeGuestFrameChainSentinel(CpuContext context) @@ -407,35 +410,54 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable ulong programExitHandlerAddress) { var imageName = string.IsNullOrWhiteSpace(processImageName) ? "eboot.bin" : processImageName; - var encodedNameLength = Encoding.UTF8.GetByteCount(imageName); - Span argv0Buffer = encodedNameLength + 1 <= 512 - ? stackalloc byte[encodedNameLength + 1] - : new byte[encodedNameLength + 1]; - if (Encoding.UTF8.GetBytes(imageName.AsSpan(), argv0Buffer) != encodedNameLength) + var arguments = new List(3) { imageName }; + var configuredArguments = Environment.GetEnvironmentVariable("SHARPEMU_GUEST_ARGS"); + if (!string.IsNullOrWhiteSpace(configuredArguments)) { - return false; + // The PS5 entry-parameter ABI exposes three inline argv pointers. + // Two compatibility arguments are therefore safe without changing + // the fixed 0x20-byte structure expected by existing titles. + var compatibilityArguments = configuredArguments.Split( + (char[]?)null, + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + arguments.AddRange(compatibilityArguments.Take(2)); } - argv0Buffer[encodedNameLength] = 0; var cursor = context[CpuRegister.Rsp]; - - var argv0Address = AlignDown(cursor - (ulong)argv0Buffer.Length, 16); - if (!context.Memory.TryWrite(argv0Address, argv0Buffer)) + var argumentAddresses = new ulong[arguments.Count]; + for (var index = arguments.Count - 1; index >= 0; index--) { - return false; + var encoded = Encoding.UTF8.GetBytes(arguments[index] + '\0'); + cursor = AlignDown(cursor - (ulong)encoded.Length, 16); + if (!context.Memory.TryWrite(cursor, encoded)) + { + return false; + } + + argumentAddresses[index] = cursor; } const ulong entryParamsSize = 0x20; - var entryParamsAddress = AlignDown(argv0Address - entryParamsSize, 16); - if (!context.TryWriteUInt32(entryParamsAddress + 0x00, 1) || - !context.TryWriteUInt32(entryParamsAddress + 0x04, 0) || - !context.TryWriteUInt64(entryParamsAddress + 0x08, argv0Address) || - !context.TryWriteUInt64(entryParamsAddress + 0x10, 0) || - !context.TryWriteUInt64(entryParamsAddress + 0x18, 0)) + var entryParamsAddress = AlignDown(cursor - entryParamsSize, 16); + if (!TryWriteUInt32(context, entryParamsAddress + 0x00, (uint)arguments.Count) || + !TryWriteUInt32(context, entryParamsAddress + 0x04, 0) || + !context.TryWriteUInt64(entryParamsAddress + 0x08, argumentAddresses[0]) || + !context.TryWriteUInt64( + entryParamsAddress + 0x10, + argumentAddresses.Length > 1 ? argumentAddresses[1] : 0) || + !context.TryWriteUInt64( + entryParamsAddress + 0x18, + argumentAddresses.Length > 2 ? argumentAddresses[2] : 0)) { return false; } + if (arguments.Count > 1) + { + Console.Error.WriteLine( + $"[DISPATCHER] Guest arguments: {string.Join(' ', arguments.Skip(1))}"); + } + var entryStackPointer = entryParamsAddress - sizeof(ulong); if (!context.TryWriteUInt64(entryStackPointer, 0)) { @@ -468,6 +490,13 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable return value & ~(alignment - 1); } + private static bool TryWriteUInt32(CpuContext context, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return context.Memory.TryWrite(address, buffer); + } + private static string BuildEntryFrameDiagnostic( ulong entryPoint, CpuContext context, diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Diagnostics.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Diagnostics.cs index 2bc8b5e..c741472 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Diagnostics.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Diagnostics.cs @@ -7,8 +7,8 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; using System.Threading; +using SharpEmu.Core.Cpu.Disasm; using SharpEmu.HLE; -using SharpEmu.HLE.Host; using SharpEmu.Logging; namespace SharpEmu.Core.Cpu.Native; @@ -17,6 +17,55 @@ public sealed partial class DirectExecutionBackend { private static readonly ConcurrentDictionary _knownExecutablePages = new(); + private static readonly bool _perfHleHistogram = + string.Equals(System.Environment.GetEnvironmentVariable("SHARPEMU_PERF_HLE"), "1", System.StringComparison.Ordinal); + private static readonly System.Collections.Concurrent.ConcurrentDictionary _perfHleCounts = new(); + private static long _perfHleTotal; + private static long _perfHleDispatchTicks; + + private static void RecordPerfHleDispatchTime(long ticks) + { + var total = System.Threading.Interlocked.Add(ref _perfHleDispatchTicks, ticks); + var calls = System.Threading.Interlocked.Read(ref _perfHleTotal); + if (calls > 0 && calls % 500000 == 0) + { + var avgUs = (double)total / System.Diagnostics.Stopwatch.Frequency * 1_000_000.0 / calls; + System.Console.Error.WriteLine($"[PERF][HLE] managed_dispatch_avg={avgUs:F3}us total_managed_s={(double)total / System.Diagnostics.Stopwatch.Frequency:F2}"); + } + } + + private static readonly bool _perfHleNoDict = + string.Equals(System.Environment.GetEnvironmentVariable("SHARPEMU_PERF_HLE_NODICT"), "1", System.StringComparison.Ordinal); + + private static void RecordPerfHleCall(string name) + { + var total = System.Threading.Interlocked.Increment(ref _perfHleTotal); + if (!_perfHleNoDict) + { + _perfHleCounts.AddOrUpdate(name, 1, static (_, v) => v + 1); + } + + if (total % 500000 == 0 && !_perfHleNoDict) + { + // Snapshot via foreach (a safe moving enumerator) before sorting. + // LINQ over a ConcurrentDictionary uses ICollection.CopyTo, which + // throws ArgumentException if another thread adds a key between the + // Count read and the copy — that exception was being swallowed into + // a CPU_TRAP return and crashing the guest. + var snapshot = new System.Collections.Generic.List>(_perfHleCounts.Count + 16); + foreach (var kvp in _perfHleCounts) + { + snapshot.Add(kvp); + } + + var top = snapshot + .OrderByDescending(kvp => kvp.Value) + .Take(20) + .Select(kvp => $"{kvp.Key}={kvp.Value}"); + System.Console.Error.WriteLine($"[PERF][HLE] total={total} top: {string.Join(", ", top)}"); + } + } + private void RecordRecentImportTrace( long dispatchIndex, string nid, @@ -25,101 +74,40 @@ public sealed partial class DirectExecutionBackend ulong arg1, ulong arg2) { - _recentImportTrace[_recentImportTraceWriteIndex] = new RecentImportTraceEntry( + var trace = _recentImportTrace; + trace[_recentImportTraceWriteIndex] = new RecentImportTraceEntry( dispatchIndex, nid, returnRip, arg0, arg1, - arg2); - _recentImportTraceWriteIndex = (_recentImportTraceWriteIndex + 1) % _recentImportTrace.Length; - if (_recentImportTraceCount < _recentImportTrace.Length) + arg2, + GuestThreadExecution.CurrentGuestThreadHandle, + Environment.CurrentManagedThreadId); + _recentImportTraceWriteIndex = (_recentImportTraceWriteIndex + 1) % trace.Length; + if (_recentImportTraceCount < trace.Length) { _recentImportTraceCount++; } } - private void RecordDeferredBootstrapTrace( - long dispatchIndex, - ulong op, - ulong symbolPointer, - ulong outputPointer, - ulong returnRip) - { - lock (_deferredBootstrapTraceGate) - { - _deferredBootstrapTrace[_deferredBootstrapTraceWriteIndex] = new DeferredBootstrapTraceEntry( - dispatchIndex, - op, - symbolPointer, - outputPointer, - returnRip); - _deferredBootstrapTraceWriteIndex = - (_deferredBootstrapTraceWriteIndex + 1) % _deferredBootstrapTrace.Length; - if (_deferredBootstrapTraceCount < _deferredBootstrapTrace.Length) - { - _deferredBootstrapTraceCount++; - } - } - } - - private void DrainDeferredBootstrapTraces() - { - if (!_logBootstrap) - { - return; - } - - DeferredBootstrapTraceEntry[] pending; - lock (_deferredBootstrapTraceGate) - { - if (_deferredBootstrapTraceCount == 0) - { - return; - } - - pending = new DeferredBootstrapTraceEntry[_deferredBootstrapTraceCount]; - var readIndex = (_deferredBootstrapTraceWriteIndex - _deferredBootstrapTraceCount + - _deferredBootstrapTrace.Length) % _deferredBootstrapTrace.Length; - for (var i = 0; i < _deferredBootstrapTraceCount; i++) - { - pending[i] = _deferredBootstrapTrace[(readIndex + i) % _deferredBootstrapTrace.Length]; - } - - _deferredBootstrapTraceCount = 0; - } - - foreach (var entry in pending) - { - var symbolText = ""; - if (TryReadAsciiZ(entry.SymbolPointer, 256, out var sym)) - { - symbolText = sym; - } - - Console.Error.WriteLine( - $"[LOADER][TRACE] bootstrap_call#{entry.DispatchIndex}: op=0x{entry.Op:X16} " + - $"sym_ptr=0x{entry.SymbolPointer:X16} sym='{symbolText}' " + - $"out_ptr=0x{entry.OutputPointer:X16} ret=0x{entry.ReturnRip:X16}"); - } - } - private void DumpRecentImportTrace() { - if (_recentImportTraceCount == 0) + var trace = _recentImportTrace; + if (trace is null || _recentImportTraceCount == 0) { return; } - Log.Info($" Recent import calls ({_recentImportTraceCount}):"); - int num = (_recentImportTraceWriteIndex - _recentImportTraceCount + _recentImportTrace.Length) % _recentImportTrace.Length; + Log.Info($" Recent import calls for managed={Environment.CurrentManagedThreadId} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} ({_recentImportTraceCount}):"); + int num = (_recentImportTraceWriteIndex - _recentImportTraceCount + trace.Length) % trace.Length; for (int i = 0; i < _recentImportTraceCount; i++) { - int num2 = (num + i) % _recentImportTrace.Length; - var entry = _recentImportTrace[num2]; + int num2 = (num + i) % trace.Length; + var entry = trace[num2]; if (!string.IsNullOrEmpty(entry.Nid)) { Log.Info( - $" #{entry.DispatchIndex} nid={entry.Nid} ret=0x{entry.ReturnRip:X16} " + + $" #{entry.DispatchIndex} managed={entry.ManagedThreadId} guest=0x{entry.GuestThreadHandle:X16} nid={entry.Nid} ret=0x{entry.ReturnRip:X16} " + $"rdi=0x{entry.Arg0:X16} rsi=0x{entry.Arg1:X16} rdx=0x{entry.Arg2:X16}"); } } @@ -135,9 +123,8 @@ public sealed partial class DirectExecutionBackend int num2 = 0; List list = new List(16); ulong num3 = scanStart; - var hostMemory = ResolveDiagnosticsHostMemory(); - HostRegionInfo lpBuffer; - while (num3 < scanEnd && hostMemory.Query(num3, out lpBuffer)) + MEMORY_BASIC_INFORMATION64 lpBuffer; + while (num3 < scanEnd && VirtualQuery((void*)num3, out lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) != 0) { ulong baseAddress = lpBuffer.BaseAddress; ulong num4 = baseAddress + lpBuffer.RegionSize; @@ -147,7 +134,7 @@ public sealed partial class DirectExecutionBackend } ulong value = Math.Max(num3, baseAddress); ulong num5 = Math.Min(num4, scanEnd); - if (lpBuffer.State == HostRegionState.Committed && IsReadableProtection(lpBuffer.RawProtection) && !IsExecutableProtection(lpBuffer.RawProtection)) + if (lpBuffer.State == 4096 && IsReadableProtection(lpBuffer.Protect) && !IsExecutableProtection(lpBuffer.Protect)) { ulong num6 = AlignUp(value, 8uL); for (ulong num7 = num6; num7 + 8 <= num5; num7 += 8) @@ -186,6 +173,50 @@ public sealed partial class DirectExecutionBackend { return; } + const int preludeSize = 192; + Span prelude = stackalloc byte[preludeSize]; + if (returnRip >= preludeSize && cpuContext.Memory.TryRead(returnRip - preludeSize, prelude)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] Import#{dispatchIndex} pre-return bytes @0x{returnRip - preludeSize:X16}: " + + BitConverter.ToString(prelude.ToArray()).Replace("-", " ")); + + List? bestCallChain = null; + var preludeAddress = returnRip - preludeSize; + for (var startOffset = 0; startOffset < preludeSize; startOffset++) + { + var cursor = preludeAddress + (ulong)startOffset; + var candidate = new List(); + while (cursor < returnRip && candidate.Count < 96 && + IcedDecoder.TryReadGuestBytes(cpuContext.Memory, cursor, 15, out var instructionBytes) && + IcedDecoder.TryDecode(cursor, instructionBytes, out var instruction) && + instruction.Length > 0 && + cursor + (ulong)instruction.Length <= returnRip) + { + candidate.Add(instruction); + cursor += (ulong)instruction.Length; + } + + if (cursor == returnRip && + candidate.Count > 0 && + string.Equals(candidate[^1].Mnemonic, "Call", StringComparison.OrdinalIgnoreCase) && + (bestCallChain is null || candidate.Count > bestCallChain.Count)) + { + bestCallChain = candidate; + } + } + + if (bestCallChain is not null) + { + Console.Error.WriteLine($"[LOADER][TRACE] Import#{dispatchIndex} pre-return disassembly:"); + foreach (var instruction in bestCallChain.TakeLast(32)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] 0x{instruction.Rip:X16}: {instruction.Text} " + + $"bytes={IcedDecoder.FormatBytes(instruction.Bytes)}"); + } + } + } Span destination = stackalloc byte[128]; if (!cpuContext.Memory.TryRead(returnRip, destination)) { @@ -237,15 +268,14 @@ public sealed partial class DirectExecutionBackend ulong callRip = returnRip + (ulong)i; ulong target = unchecked((ulong)((long)(callRip + 5) + rel32)); Log.Debug($"Import#{dispatchIndex} near-call @{callRip:X16}: target=0x{target:X16}"); - var importEntries = _importEntries; - for (int importIndex = 0; importIndex < importEntries.Length; importIndex++) + for (int importIndex = 0; importIndex < _importEntries.Length; importIndex++) { - if (importEntries[importIndex].Address != target) + if (_importEntries[importIndex].Address != target) { continue; } - string nid = importEntries[importIndex].Nid; + string nid = _importEntries[importIndex].Nid; if (_moduleManager.TryGetExport(nid, out var export)) { Log.Debug( @@ -272,14 +302,14 @@ public sealed partial class DirectExecutionBackend { Log.Debug( $"Import#{dispatchIndex} near-call PLT slot: [0x{slot:X16}] = 0x{slotTarget:X16}"); - for (int importIndex = 0; importIndex < importEntries.Length; importIndex++) + for (int importIndex = 0; importIndex < _importEntries.Length; importIndex++) { - if (importEntries[importIndex].Address != slotTarget) + if (_importEntries[importIndex].Address != slotTarget) { continue; } - string nid = importEntries[importIndex].Nid; + string nid = _importEntries[importIndex].Nid; if (_moduleManager.TryGetExport(nid, out var export)) { Log.Debug( @@ -303,6 +333,28 @@ public sealed partial class DirectExecutionBackend return value == 65534 || value == 4294967294u || value == 18446744073709551614uL; } + private static ulong ParseOptionalHexAddress(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return 0; + } + + var text = value.Trim(); + if (text.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + text = text[2..]; + } + + return ulong.TryParse( + text, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) + ? address + : 0; + } + private static bool IsPlausibleReturnAddress(ulong address) { return address >= 12884901888L && address < 17592186044416L && !IsUnresolvedSentinel(address); @@ -352,7 +404,7 @@ public sealed partial class DirectExecutionBackend { return false; } - if (!ResolveDiagnosticsHostMemory().Query(address, out var lpBuffer)) + if (VirtualQuery((void*)address, out var lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { return false; } @@ -361,7 +413,7 @@ public sealed partial class DirectExecutionBackend { return false; } - if (lpBuffer.State != HostRegionState.Committed || !IsReadableProtection(lpBuffer.RawProtection)) + if (lpBuffer.State != 4096 || !IsReadableProtection(lpBuffer.Protect)) { return false; } @@ -393,12 +445,12 @@ public sealed partial class DirectExecutionBackend return true; } - if (!ResolveDiagnosticsHostMemory().Query(address, out var lpBuffer)) + if (VirtualQuery((void*)address, out var lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { return false; } - var executable = lpBuffer.State == HostRegionState.Committed && IsExecutableProtection(lpBuffer.RawProtection); + var executable = lpBuffer.State == 4096 && IsExecutableProtection(lpBuffer.Protect); if (executable) { _knownExecutablePages.TryAdd(pageAddress, 0); @@ -417,14 +469,6 @@ public sealed partial class DirectExecutionBackend return (value + num) & ~num; } - // Diagnostics helpers are static (reachable from static handler paths), so - // they use the platform injected into the backend active on this thread and - // fall back to the process-wide singleton only when no run is bound. - private static IHostMemory ResolveDiagnosticsHostMemory() - { - return _activeExecutionBackend?._hostMemory ?? HostPlatform.Current.Memory; - } - private static bool IsReadableProtection(uint protect) { if ((protect & 0x100) != 0 || (protect & 1) != 0) diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Exceptions.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Exceptions.cs index 7c96cab..9ea5199 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Exceptions.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Exceptions.cs @@ -9,9 +9,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Threading; using SharpEmu.Core.Cpu.Disasm; -using SharpEmu.Core.Cpu.Native.Windows; using SharpEmu.HLE; -using SharpEmu.HLE.Host; namespace SharpEmu.Core.Cpu.Native; @@ -19,6 +17,8 @@ public sealed partial class DirectExecutionBackend { private const ulong LazyCommitWindowBytes = 0x0200_0000UL; private static int _lazyCommitTraceCount; + private static int _guestAllocatorHoleRecoveries; + private static int _auxiliaryThreadExecuteFaultRecoveries; private unsafe void SetupExceptionHandler() { @@ -30,12 +30,12 @@ public sealed partial class DirectExecutionBackend if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_RAW_HANDLER"), "1", StringComparison.Ordinal)) { - _rawExceptionHandlerStub = _faultHandling.CreateHandlerThunk(RawVectoredHandlerPtrManaged, _hostRspSlotTlsIndex, _tlsGetValueAddress); + _rawExceptionHandlerStub = CreateExceptionHandlerTrampoline(RawVectoredHandlerPtrManaged); if (_rawExceptionHandlerStub == 0) { throw new InvalidOperationException("Failed to create raw exception handler trampoline"); } - _rawExceptionHandler = _faultHandling.AddFirstChanceHandler(_rawExceptionHandlerStub); + _rawExceptionHandler = (nint)AddVectoredExceptionHandler(1u, _rawExceptionHandlerStub); Console.Error.WriteLine($"[LOADER][INFO] Raw exception handler installed: 0x{_rawExceptionHandler:X16}"); } else @@ -45,22 +45,22 @@ public sealed partial class DirectExecutionBackend _handlerDelegate = VectoredHandler; _handlerHandle = GCHandle.Alloc(_handlerDelegate); - _exceptionHandlerStub = _faultHandling.CreateHandlerThunk(Marshal.GetFunctionPointerForDelegate(_handlerDelegate), _hostRspSlotTlsIndex, _tlsGetValueAddress); + _exceptionHandlerStub = CreateExceptionHandlerTrampoline(Marshal.GetFunctionPointerForDelegate(_handlerDelegate)); if (_exceptionHandlerStub == 0) { throw new InvalidOperationException("Failed to create exception handler trampoline"); } - _exceptionHandler = _faultHandling.AddFirstChanceHandler(_exceptionHandlerStub); + _exceptionHandler = (nint)AddVectoredExceptionHandler(1u, _exceptionHandlerStub); Console.Error.WriteLine($"[LOADER][INFO] Exception handler installed: 0x{_exceptionHandler:X16}"); _unhandledFilterDelegate = UnhandledExceptionFilter; _unhandledFilterHandle = GCHandle.Alloc(_unhandledFilterDelegate); - _unhandledFilterStub = _faultHandling.CreateHandlerThunk(Marshal.GetFunctionPointerForDelegate(_unhandledFilterDelegate), _hostRspSlotTlsIndex, _tlsGetValueAddress); + _unhandledFilterStub = CreateExceptionHandlerTrampoline(Marshal.GetFunctionPointerForDelegate(_unhandledFilterDelegate)); if (_unhandledFilterStub == 0) { throw new InvalidOperationException("Failed to create unhandled exception filter trampoline"); } - _faultHandling.SetUnhandledFilter(_unhandledFilterStub); + SetUnhandledExceptionFilter(_unhandledFilterStub); } private unsafe int UnhandledExceptionFilter(void* exceptionInfo) @@ -68,8 +68,8 @@ public sealed partial class DirectExecutionBackend try { EXCEPTION_RECORD* exceptionRecord = ((EXCEPTION_POINTERS*)exceptionInfo)->ExceptionRecord; - ulong rip = ReadCtxU64(((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord, CTX_RIP); - ulong rsp = ReadCtxU64(((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord, CTX_RSP); + ulong rip = ReadCtxU64(((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord, 248); + ulong rsp = ReadCtxU64(((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord, 152); Console.Error.WriteLine("[LOADER][FATAL] Unhandled exception filter fired."); Console.Error.WriteLine($"[LOADER][FATAL] Code: 0x{exceptionRecord->ExceptionCode:X8}"); Console.Error.WriteLine($"[LOADER][FATAL] Exception Address: 0x{(ulong)(nint)exceptionRecord->ExceptionAddress:X16}"); @@ -108,19 +108,23 @@ public sealed partial class DirectExecutionBackend return 0; } - ulong rip = ReadCtxU64(contextRecord, CTX_RIP); - ulong rsp = ReadCtxU64(contextRecord, CTX_RSP); - - // Thread-mode probe: a hardware exception raised while this thread is inside - // the managed import gateway means the VEH->managed reentry happened from - // cooperative GC mode — a ReversePInvokeBadTransition candidate. - if (LogThreadMode && _threadModeGatewayDepth > 0) + ulong rip = ReadCtxU64(contextRecord, 248); + ulong rsp = ReadCtxU64(contextRecord, 152); + if (TryRecoverGuestInt41(exceptionCode, contextRecord, rip)) { - TraceThreadMode( - $"veh_in_gateway code=0x{exceptionCode:X8} rip=0x{rip:X16} gateway_depth={_threadModeGatewayDepth}"); + return -1; + } + if (TryRecoverAuxiliaryThreadExecuteFault(exceptionRecord, contextRecord, rip)) + { + return -1; } - if (exceptionCode == WindowsFaultCodes.AccessViolation && TryHandleLazyCommittedPage(exceptionRecord, rip, rsp)) + if (exceptionCode == 3221225477u && TryHandleLazyCommittedPage(exceptionRecord, rip, rsp)) + { + return -1; + } + if (exceptionCode == 3221225477u && + TryRecoverGuestAllocatorHole(exceptionRecord, contextRecord, rip)) { return -1; } @@ -135,10 +139,10 @@ public sealed partial class DirectExecutionBackend switch (exceptionCode) { - case WindowsFaultCodes.AccessViolation: + case 3221225477u: LogAccessViolationTrace(exceptionAddress, exceptionRecord); break; - case WindowsFaultCodes.FastFail: + case 3221226505u: { ulong p0 = exceptionRecord->NumberParameters >= 1 ? (*exceptionRecord->ExceptionInformation) : 0; ulong p1 = exceptionRecord->NumberParameters >= 2 ? exceptionRecord->ExceptionInformation[1] : 0; @@ -148,27 +152,56 @@ public sealed partial class DirectExecutionBackend } } - ulong rax = ReadCtxU64(contextRecord, CTX_RAX); - ulong rbx = ReadCtxU64(contextRecord, CTX_RBX); - ulong rcx = ReadCtxU64(contextRecord, CTX_RCX); - ulong rdx = ReadCtxU64(contextRecord, CTX_RDX); - ulong rsi = ReadCtxU64(contextRecord, CTX_RSI); - ulong rdi = ReadCtxU64(contextRecord, CTX_RDI); - ulong rbp = ReadCtxU64(contextRecord, CTX_RBP); - ulong r8 = ReadCtxU64(contextRecord, CTX_R8); - ulong r9 = ReadCtxU64(contextRecord, CTX_R9); - ulong r10 = ReadCtxU64(contextRecord, CTX_R10); - ulong r11 = ReadCtxU64(contextRecord, CTX_R11); - ulong r12 = ReadCtxU64(contextRecord, CTX_R12); - ulong r13 = ReadCtxU64(contextRecord, CTX_R13); - ulong r14 = ReadCtxU64(contextRecord, CTX_R14); - ulong r15 = ReadCtxU64(contextRecord, CTX_R15); + ulong rax = ReadCtxU64(contextRecord, 120); + ulong rbx = ReadCtxU64(contextRecord, 144); + ulong rcx = ReadCtxU64(contextRecord, 128); + ulong rdx = ReadCtxU64(contextRecord, 136); + ulong rsi = ReadCtxU64(contextRecord, 168); + ulong rdi = ReadCtxU64(contextRecord, 176); + ulong rbp = ReadCtxU64(contextRecord, 160); + ulong r8 = ReadCtxU64(contextRecord, 184); + ulong r9 = ReadCtxU64(contextRecord, 192); + ulong r10 = ReadCtxU64(contextRecord, 200); + ulong r11 = ReadCtxU64(contextRecord, 208); + ulong r12 = ReadCtxU64(contextRecord, 216); + ulong r13 = ReadCtxU64(contextRecord, 224); + ulong r14 = ReadCtxU64(contextRecord, 232); + ulong r15 = ReadCtxU64(contextRecord, 240); Console.Error.WriteLine("[LOADER][INFO] ========================================="); Console.Error.WriteLine("[LOADER][INFO] NATIVE EXCEPTION CAUGHT!"); Console.Error.WriteLine($"[LOADER][INFO] Code: 0x{exceptionCode:X8}"); Console.Error.WriteLine($"[LOADER][INFO] Exception Address: 0x{exceptionAddress:X16}"); Console.Error.WriteLine($"[LOADER][INFO] RIP: 0x{rip:X16}"); + Console.Error.WriteLine( + $"[LOADER][INFO] Host thread: managed={Environment.CurrentManagedThreadId} " + + $"name='{Thread.CurrentThread.Name ?? ""}'"); + if (_activeGuestThreadState is { } activeGuestThread) + { + Console.Error.WriteLine( + $"[LOADER][INFO] Guest thread: handle=0x{activeGuestThread.ThreadHandle:X16} " + + $"name='{activeGuestThread.Name}' state={activeGuestThread.State} " + + $"last_import={activeGuestThread.LastImportNid ?? ""} " + + $"last_ret=0x{activeGuestThread.LastReturnRip:X16}"); + Console.Error.WriteLine( + $"[LOADER][INFO] Last import registers: " + + $"rax=0x{Volatile.Read(ref activeGuestThread.LastImportRax):X16} " + + $"result_valid={Volatile.Read(ref activeGuestThread.LastImportResultValid) != 0} " + + $"rdi=0x{activeGuestThread.LastImportRdi:X16} " + + $"rsi=0x{activeGuestThread.LastImportRsi:X16} " + + $"rdx=0x{activeGuestThread.LastImportRdx:X16} " + + $"rcx=0x{activeGuestThread.LastImportRcx:X16} " + + $"r8=0x{activeGuestThread.LastImportR8:X16} " + + $"r9=0x{activeGuestThread.LastImportR9:X16}"); + Console.Error.WriteLine( + $"[LOADER][INFO] Last import stack args: " + + $"0=0x{activeGuestThread.LastImportStack0:X16} " + + $"1=0x{activeGuestThread.LastImportStack1:X16} " + + $"2=0x{activeGuestThread.LastImportStack2:X16} " + + $"3=0x{activeGuestThread.LastImportStack3:X16} " + + $"4=0x{activeGuestThread.LastImportStack4:X16} " + + $"5=0x{activeGuestThread.LastImportStack5:X16}"); + } if (TryFormatNearestRuntimeSymbol(rip, out string symbol)) { Console.Error.WriteLine("[LOADER][INFO] RIP symbol: " + symbol); @@ -193,7 +226,7 @@ public sealed partial class DirectExecutionBackend ulong accessType = 0; ulong target = 0; - if (exceptionCode == WindowsFaultCodes.AccessViolation && exceptionRecord->NumberParameters >= 2) + if (exceptionCode == 3221225477u && exceptionRecord->NumberParameters >= 2) { accessType = *exceptionRecord->ExceptionInformation; target = exceptionRecord->ExceptionInformation[1]; @@ -206,9 +239,9 @@ public sealed partial class DirectExecutionBackend }; Console.Error.WriteLine("[LOADER][INFO] AV access: " + accessText); Console.Error.WriteLine($"[LOADER][INFO] AV target: 0x{target:X16}"); - if (_hostMemory.Query(target, out var mbi)) + if (VirtualQuery((void*)target, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) != 0) { - Console.Error.WriteLine($"[LOADER][INFO] AV target region: base=0x{mbi.BaseAddress:X16} size=0x{mbi.RegionSize:X16} state=0x{mbi.RawState:X08} protect=0x{mbi.RawProtection:X08}"); + Console.Error.WriteLine($"[LOADER][INFO] AV target region: base=0x{mbi.BaseAddress:X16} size=0x{mbi.RegionSize:X16} state=0x{mbi.State:X08} protect=0x{mbi.Protect:X08}"); } } @@ -225,13 +258,46 @@ public sealed partial class DirectExecutionBackend Console.Error.WriteLine($"[LOADER][INFO] [rsp+0x{i * 8:X2}] @0x{stackAddr:X16} = 0x{value:X16}"); } + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DUMP_FAULT_STACK_WINDOW"), + "1", + StringComparison.Ordinal)) + { + Console.Error.WriteLine("[LOADER][INFO] Full fault stack window (RSP-0x300..RSP+0x100):"); + var windowStart = rsp >= 0x300 ? rsp - 0x300 : 0; + for (var stackAddr = windowStart; stackAddr < rsp + 0x100; stackAddr += 8) + { + if (!TryReadHostQword(stackAddr, out var value)) + { + continue; + } + + var relative = unchecked((long)(stackAddr - rsp)); + var relativeText = relative >= 0 + ? $"+0x{relative:X}" + : $"-0x{-relative:X}"; + var symbolText = TryFormatNearestRuntimeSymbol(value, out var stackSymbol) + ? $" [{stackSymbol}]" + : string.Empty; + Console.Error.WriteLine( + $"[LOADER][INFO] [rsp{relativeText}] " + + $"@0x{stackAddr:X16} = 0x{value:X16}{symbolText}"); + } + } + + DumpPointerWindow("fault-register-rbx", rbx, 0x60); + DumpPointerWindow("fault-register-rsi", rsi, 0x60); + DumpPointerWindow("fault-register-rdi", rdi, 0x60); + DumpPointerWindow("fault-register-r13", r13, 0x60); + DumpPointerWindow("fault-register-r14", r14, 0x60); + try { Console.Error.WriteLine("[LOADER][INFO] Frame chain (RBP walk):"); ulong frame = rbp; for (int i = 0; i < 12; i++) { - if (frame < 140733193388032L || frame > 140737488355327L) + if (frame < 0x10000) { break; } @@ -256,7 +322,7 @@ public sealed partial class DirectExecutionBackend switch (exceptionCode) { - case WindowsFaultCodes.AccessViolation: + case 3221225477u: Console.Error.WriteLine("[LOADER][ERROR] Type: Access Violation"); Console.Error.WriteLine("[LOADER][ERROR] This usually means:"); Console.Error.WriteLine("[LOADER][ERROR] - Guest code called an unmapped import"); @@ -290,22 +356,57 @@ public sealed partial class DirectExecutionBackend { Console.Error.WriteLine("[LOADER][INFO] Code window [RIP-0x20..]: " + BitConverter.ToString(window).Replace("-", " ")); } + for (var stackIndex = 0; stackIndex < 16; stackIndex++) + { + byte[] stackSlot = new byte[8]; + if (!TryReadHostBytes(rsp + (ulong)(stackIndex * 8), stackSlot)) + { + continue; + } + var candidate = BitConverter.ToUInt64(stackSlot); + if (candidate < _entryPoint || candidate >= _entryPoint + 0x10000000 || candidate < 24) + { + continue; + } + byte[] callSiteWindow = new byte[48]; + if (TryReadHostBytes(candidate - 24, callSiteWindow)) + { + Console.Error.WriteLine( + $"[LOADER][INFO] Stack guest-code candidate [rsp+0x{stackIndex * 8:X2}]=0x{candidate:X16}, bytes [-0x18..]: " + + BitConverter.ToString(callSiteWindow).Replace("-", " ")); + } + } } else { Console.Error.WriteLine("[LOADER][ERROR] Could not read code at RIP"); } DumpRecentImportTrace(); - DumpGuestDisasmDiagnostics(rip, rbp); + DumpGuestDisasmDiagnostics(rip, rbp, rsp); + DumpGuestRegisterWindowDiagnostics( + rax, rbx, rcx, rdx, rsi, rdi, rbp, rsp, + r8, r9, r10, r11, r12, r13, r14, r15); DumpGuestReferenceDiagnostics(); DumpGuestPointerWindowDiagnostics(); break; - case WindowsFaultCodes.Breakpoint: + case 2147483651u: Console.Error.WriteLine("[LOADER][WARNING] Type: Breakpoint (int3)"); Console.Error.WriteLine("[LOADER][WARNING] Unexpected breakpoint in direct-bridge mode"); break; - case WindowsFaultCodes.IllegalInstruction: + case 1073741845u: + Console.Error.WriteLine("[LOADER][ERROR] Type: Abort (SIGABRT)"); + DumpRecentImportTrace(); + DumpGuestDisasmDiagnostics(rip, rbp, rsp); + break; + case 3221225501u: Console.Error.WriteLine("[LOADER][INFO] Type: Illegal Instruction"); + byte[] illegalCode = new byte[16]; + if (TryReadHostBytes(rip, illegalCode)) + { + Console.Error.WriteLine("[LOADER][INFO] Code at RIP: " + BitConverter.ToString(illegalCode).Replace("-", " ")); + } + DumpRecentImportTrace(); + DumpGuestDisasmDiagnostics(rip, rbp, rsp); break; } @@ -319,6 +420,108 @@ public sealed partial class DirectExecutionBackend } } + private unsafe bool TryRecoverAuxiliaryThreadExecuteFault( + EXCEPTION_RECORD* exceptionRecord, + void* contextRecord, + ulong rip) + { + if (exceptionRecord->ExceptionCode != 3221225477u || + rip >= 0x0000000800000000UL || + _activeGuestThreadState is not { Name: "tbb_thead" } activeThread) + { + return false; + } + + var hostExit = ActiveEntryReturnSentinelRip; + if (hostExit < 0x10000) + { + hostExit = unchecked((ulong)_guestReturnStub); + } + if (hostExit < 0x10000) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Could not recover auxiliary TBB execute fault: target=0x{rip:X16} " + + $"active_exit=0x{ActiveEntryReturnSentinelRip:X16} guest_return_stub=0x{unchecked((ulong)_guestReturnStub):X16}"); + return false; + } + + _ = TryPatchActiveGuestReturnSlot(hostExit); + WriteCtxU64(contextRecord, 120, 0); + WriteCtxU64(contextRecord, 248, hostExit); + var recovery = Interlocked.Increment(ref _auxiliaryThreadExecuteFaultRecoveries); + Console.Error.WriteLine( + $"[LOADER][WARN] Recovered auxiliary TBB execute fault #{recovery}: " + + $"thread=0x{activeThread.ThreadHandle:X16} target=0x{rip:X16} -> host_exit=0x{hostExit:X16}"); + return true; + } + + private unsafe bool TryRecoverGuestInt41(uint exceptionCode, void* contextRecord, ulong rip) + { + if (!_ignoreGuestInt41 || exceptionCode != 3221225477u || rip < 0x10000) + { + return false; + } + + byte[] opcode = new byte[2]; + if (!TryReadHostBytes(rip, opcode) || opcode[0] != 0xCD || opcode[1] != 0x41) + { + return false; + } + + var count = Interlocked.Increment(ref _ignoredGuestInt41Count); + WriteCtxU64(contextRecord, 248, rip + 2); + if (count <= 16 || count % 65536 == 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Ignored guest int 0x41 trap #{count} at 0x{rip:X16} (SHARPEMU_IGNORE_INT41=1)"); + Console.Error.Flush(); + } + return true; + } + + private unsafe static bool TryRecoverGuestAllocatorHole( + EXCEPTION_RECORD* exceptionRecord, + void* contextRecord, + ulong rip) + { + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_GUEST_ALLOCATOR_HOLE_RECOVERY"), + "1", + StringComparison.Ordinal) || + exceptionRecord->NumberParameters < 2 || + exceptionRecord->ExceptionInformation[0] != 0 || + exceptionRecord->ExceptionInformation[1] != 8 || + ReadCtxU64(contextRecord, CTX_RDI) != 0 || + rip < 0x10000) + { + return false; + } + + // Demon's Souls occasionally leaves an empty payload in a locked pool + // tree node. The allocator dereferences payload+8 before reaching its + // existing empty-pool fallback. Match the instruction stream instead of + // a title-specific absolute address, then resume at that fallback so the + // lock is released and the allocator can try its next backing pool. + const ulong allocatorHoleSignature = 0x634CFF568D08778BUL; + if (*(ulong*)rip != allocatorHoleSignature || *((byte*)rip + 8) != 0xF2) + { + return false; + } + + const ulong emptyPoolFallbackDelta = 0x8E; + WriteCtxU64(contextRecord, CTX_RIP, rip + emptyPoolFallbackDelta); + var recovery = Interlocked.Increment(ref _guestAllocatorHoleRecoveries); + if (recovery <= 16 || (recovery & (recovery - 1)) == 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Guest allocator empty-node adapter recovery #{recovery}: " + + $"rip=0x{rip:X16} -> 0x{rip + emptyPoolFallbackDelta:X16}"); + Console.Error.Flush(); + } + + return true; + } + private static bool IsBenignHostDebugException(uint exceptionCode) { return exceptionCode is DBG_PRINTEXCEPTION_C or DBG_PRINTEXCEPTION_WIDE_C or MS_VC_THREADNAME_EXCEPTION; @@ -337,8 +540,8 @@ public sealed partial class DirectExecutionBackend EXCEPTION_POINTERS* pointers = (EXCEPTION_POINTERS*)exceptionInfo; EXCEPTION_RECORD* record = pointers->ExceptionRecord; void* contextRecord = pointers->ContextRecord; - ulong rip = contextRecord != null ? ReadCtxU64(contextRecord, CTX_RIP) : 0; - ulong rsp = contextRecord != null ? ReadCtxU64(contextRecord, CTX_RSP) : 0; + ulong rip = contextRecord != null ? ReadCtxU64(contextRecord, 248) : 0; + ulong rsp = contextRecord != null ? ReadCtxU64(contextRecord, 152) : 0; ulong accessType = record->NumberParameters >= 1 ? *record->ExceptionInformation : 0; ulong target = record->NumberParameters >= 2 ? record->ExceptionInformation[1] : 0; Console.Error.WriteLine( @@ -398,7 +601,7 @@ public sealed partial class DirectExecutionBackend } } - private void DumpGuestDisasmDiagnostics(ulong rip, ulong rbp) + private void DumpGuestDisasmDiagnostics(ulong rip, ulong rbp, ulong rsp) { if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_DISASM"), "1", StringComparison.Ordinal)) { @@ -410,12 +613,20 @@ public sealed partial class DirectExecutionBackend DumpGuestInstructionStream("fault-prelude", rip - 0x20, 24); } + // Optimized guest code frequently omits frame pointers. The return + // address at RSP is then more useful than an RBP walk and identifies the + // exact call site that supplied the faulting arguments. + if (TryReadHostQword(rsp, out var stackReturn) && stackReturn >= 0x60) + { + DumpGuestInstructionStream("stack-return-prelude", stackReturn - 0x60, 40); + } + try { ulong frame = rbp; for (int i = 0; i < 3; i++) { - if (frame < 140733193388032L || frame > 140737488355327L) + if (frame < 0x10000) { break; } @@ -484,7 +695,7 @@ public sealed partial class DirectExecutionBackend ulong address = scanBase; while (address < scanEnd) { - if (!_hostMemory.Query(address, out var mbi)) + if (VirtualQuery((void*)address, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { break; } @@ -496,9 +707,9 @@ public sealed partial class DirectExecutionBackend break; } - if (mbi.State == HostRegionState.Committed && - IsReadableProtection(mbi.RawProtection) && - IsExecutableProtection(mbi.RawProtection)) + if (mbi.State == MEM_COMMIT && + IsReadableProtection(mbi.Protect) && + IsExecutableProtection(mbi.Protect)) { ScanExecutableRegionForTargetReferences(regionBase, regionEnd, targetList, hitCounts, maxHitsPerTarget); } @@ -559,6 +770,55 @@ public sealed partial class DirectExecutionBackend } } + private void DumpGuestRegisterWindowDiagnostics( + ulong rax, + ulong rbx, + ulong rcx, + ulong rdx, + ulong rsi, + ulong rdi, + ulong rbp, + ulong rsp, + ulong r8, + ulong r9, + ulong r10, + ulong r11, + ulong r12, + ulong r13, + ulong r14, + ulong r15) + { + if (!string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_REGISTER_WINDOWS"), + "1", + StringComparison.Ordinal)) + { + return; + } + + // A register can be the only surviving reference to the object or + // argument array that caused a native guest fault. Capture a compact + // window while the process is alive so the post-mortem log can + // distinguish an absent object from a partially initialized one. + var registers = new (string Name, ulong Value)[] + { + ("rax", rax), ("rbx", rbx), ("rcx", rcx), ("rdx", rdx), + ("rsi", rsi), ("rdi", rdi), ("rbp", rbp), ("rsp", rsp), + ("r8", r8), ("r9", r9), ("r10", r10), ("r11", r11), + ("r12", r12), ("r13", r13), ("r14", r14), ("r15", r15), + }; + var seen = new HashSet(); + foreach (var (name, value) in registers) + { + if (value < 0x10000 || !seen.Add(value)) + { + continue; + } + + DumpPointerWindow($"register-{name}", value, 0x80); + } + } + private void ScanExecutableRegionForTargetReferences( ulong regionBase, ulong regionEnd, @@ -803,13 +1063,13 @@ public sealed partial class DirectExecutionBackend return false; } - if (!_hostMemory.Query(address, out var mbi)) + if (VirtualQuery((void*)address, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { return false; } ulong regionEnd = mbi.BaseAddress + mbi.RegionSize; - if (mbi.State != HostRegionState.Committed || !IsReadableProtection(mbi.RawProtection) || regionEnd <= address || address > regionEnd - 8) + if (mbi.State != MEM_COMMIT || !IsReadableProtection(mbi.Protect) || regionEnd <= address || address > regionEnd - 8) { return false; } @@ -848,7 +1108,7 @@ public sealed partial class DirectExecutionBackend } } - private unsafe bool TryReadHostBytes(ulong address, byte[] buffer) + private unsafe static bool TryReadHostBytes(ulong address, byte[] buffer) { if (address < 65536) { @@ -861,9 +1121,9 @@ public sealed partial class DirectExecutionBackend ulong end = address + (ulong)buffer.Length; for (ulong page = address & 0xFFFFFFFFFFFFF000uL; page < end; page += 4096) { - if (!_hostMemory.Query(page, out var mbi) || - mbi.State != HostRegionState.Committed || - !IsReadableProtection(mbi.RawProtection)) + if (VirtualQuery((void*)page, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0 || + mbi.State != MEM_COMMIT || + !IsReadableProtection(mbi.Protect)) { return false; } @@ -976,25 +1236,25 @@ public sealed partial class DirectExecutionBackend { return false; } - if (!_hostMemory.Query(faultAddress, out var mbi)) + if (VirtualQuery((void*)faultAddress, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { return false; } ulong pageBase = faultAddress & 0xFFFFFFFFFFFFF000uL; - uint commitProtect = ResolveLazyCommitProtection(accessType, mbi.RawAllocationProtection); + uint commitProtect = ResolveLazyCommitProtection(accessType, mbi.AllocationProtect); int traceIndex = Interlocked.Increment(ref _lazyCommitTraceCount); bool traceLazyCommit = ShouldTraceLazyCommit(traceIndex); if (traceLazyCommit) { - Console.Error.WriteLine($"[LOADER][TRACE] lazy-query#{traceIndex}: fault=0x{faultAddress:X16} owner={owner} rip=0x{rip:X16} rsp=0x{rsp:X16} state=0x{mbi.RawState:X08} base=0x{mbi.BaseAddress:X16} size=0x{mbi.RegionSize:X16} alloc=0x{mbi.RawAllocationProtection:X08} prot=0x{mbi.RawProtection:X08}"); + Console.Error.WriteLine($"[LOADER][TRACE] lazy-query#{traceIndex}: fault=0x{faultAddress:X16} owner={owner} rip=0x{rip:X16} rsp=0x{rsp:X16} state=0x{mbi.State:X08} base=0x{mbi.BaseAddress:X16} size=0x{mbi.RegionSize:X16} alloc=0x{mbi.AllocationProtect:X08} prot=0x{mbi.Protect:X08}"); } - if (mbi.State == HostRegionState.Committed && IsAccessCompatible(accessType, mbi.RawProtection)) + if (mbi.State == 4096 && IsAccessCompatible(accessType, mbi.Protect)) { if (traceLazyCommit) { - Console.Error.WriteLine($"[LOADER][TRACE] lazy-commit-race#{traceIndex}: fault=0x{faultAddress:X16} protect=0x{mbi.RawProtection:X08}"); + Console.Error.WriteLine($"[LOADER][TRACE] lazy-commit-race#{traceIndex}: fault=0x{faultAddress:X16} protect=0x{mbi.Protect:X08}"); } return true; } @@ -1003,10 +1263,10 @@ public sealed partial class DirectExecutionBackend ulong committedBase = 0; ulong committedSize = 0; - if (mbi.State == HostRegionState.Free) + if (mbi.State == 65536) { if (TryGetLazyCommitWindow(faultAddress, mbi.BaseAddress, mbi.RegionSize, out var windowBase, out var windowSize) && - TryReserveThenCommit(_hostMemory, windowBase, windowSize, windowBase, windowSize, commitProtect)) + TryReserveThenCommit(windowBase, windowSize, windowBase, windowSize, commitProtect)) { committed = true; committedBase = windowBase; @@ -1015,7 +1275,7 @@ public sealed partial class DirectExecutionBackend else { ulong largeBase = faultAddress & 0xFFFFFFFFFFE00000uL; - if (TryReserveThenCommit(_hostMemory, largeBase, 2097152uL, largeBase, 2097152uL, commitProtect)) + if (TryReserveThenCommit(largeBase, 2097152uL, largeBase, 2097152uL, commitProtect)) { committed = true; committedBase = largeBase; @@ -1026,13 +1286,13 @@ public sealed partial class DirectExecutionBackend if (!committed) { ulong region64kBase = faultAddress & 0xFFFFFFFFFFFF0000uL; - if (TryReserveThenCommit(_hostMemory, region64kBase, 65536uL, region64kBase, 65536uL, commitProtect)) + if (TryReserveThenCommit(region64kBase, 65536uL, region64kBase, 65536uL, commitProtect)) { committed = true; committedBase = region64kBase; committedSize = 65536uL; } - else if (TryReserveThenCommit(_hostMemory, pageBase, 4096uL, pageBase, 4096uL, commitProtect)) + else if (TryReserveThenCommit(pageBase, 4096uL, pageBase, 4096uL, commitProtect)) { committed = true; committedBase = pageBase; @@ -1045,7 +1305,7 @@ public sealed partial class DirectExecutionBackend return false; } - TryCommitRange(_hostMemory, pageBase + 4096, 4096uL, commitProtect); + TryCommitRange(pageBase + 4096, 4096uL, commitProtect); if (traceLazyCommit) { Console.Error.WriteLine($"[LOADER][TRACE] lazy-reserve-commit#{traceIndex}: addr=0x{committedBase:X16} size=0x{committedSize:X16} access={accessType} protect=0x{commitProtect:X8}"); @@ -1053,13 +1313,13 @@ public sealed partial class DirectExecutionBackend return true; } - if (mbi.State != HostRegionState.Reserved) + if (mbi.State != 8192) { return false; } if (TryGetLazyCommitWindow(faultAddress, mbi.BaseAddress, mbi.RegionSize, out var commitWindowBase, out var commitWindowSize) && - TryCommitRange(_hostMemory, commitWindowBase, commitWindowSize, commitProtect)) + TryCommitRange(commitWindowBase, commitWindowSize, commitProtect)) { committed = true; committedBase = commitWindowBase; @@ -1068,7 +1328,7 @@ public sealed partial class DirectExecutionBackend else { ulong largeCommitBase = faultAddress & 0xFFFFFFFFFFE00000uL; - if (TryCommitRange(_hostMemory, largeCommitBase, 2097152uL, commitProtect)) + if (TryCommitRange(largeCommitBase, 2097152uL, commitProtect)) { committed = true; committedBase = largeCommitBase; @@ -1079,19 +1339,19 @@ public sealed partial class DirectExecutionBackend if (!committed) { ulong region64kBase = faultAddress & 0xFFFFFFFFFFFF0000uL; - if (TryCommitRange(_hostMemory, region64kBase, 65536uL, commitProtect)) + if (TryCommitRange(region64kBase, 65536uL, commitProtect)) { committed = true; committedBase = region64kBase; committedSize = 65536uL; } - else if (TryCommitRange(_hostMemory, pageBase, 8192uL, commitProtect)) + else if (TryCommitRange(pageBase, 8192uL, commitProtect)) { committed = true; committedBase = pageBase; committedSize = 8192uL; } - else if (TryCommitRange(_hostMemory, pageBase, 4096uL, commitProtect)) + else if (TryCommitRange(pageBase, 4096uL, commitProtect)) { committed = true; committedBase = pageBase; @@ -1104,7 +1364,7 @@ public sealed partial class DirectExecutionBackend return false; } - TryCommitRange(_hostMemory, pageBase + 4096, 4096uL, commitProtect); + TryCommitRange(pageBase + 4096, 4096uL, commitProtect); if (traceLazyCommit) { Console.Error.WriteLine($"[LOADER][TRACE] lazy-commit#{traceIndex}: addr=0x{committedBase:X16} size=0x{committedSize:X16} access={accessType} protect=0x{commitProtect:X8}"); @@ -1145,33 +1405,31 @@ public sealed partial class DirectExecutionBackend return true; } - // The commit protection is one of the two raw values ResolveLazyCommitProtection - // produces (0x40 RWX / 0x04 RW); the enum mapping reproduces those exactly. - static bool TryCommitRange(IHostMemory hostMemory, ulong baseAddress, ulong length, uint protection) + static unsafe bool TryCommitRange(ulong baseAddress, ulong length, uint protection) { if (length == 0) { return false; } - return hostMemory.Commit(baseAddress, length, protection == 64u ? HostPageProtection.ReadWriteExecute : HostPageProtection.ReadWrite); + return VirtualAlloc((void*)baseAddress, (nuint)length, 4096u, protection) != null; } - static bool TryReserveRange(IHostMemory hostMemory, ulong baseAddress, ulong length) + static unsafe bool TryReserveRange(ulong baseAddress, ulong length) { if (length == 0) { return false; } - return hostMemory.Reserve(baseAddress, length, HostPageProtection.ReadWrite) != 0; + return VirtualAlloc((void*)baseAddress, (nuint)length, 8192u, 4u) != null; } - static bool TryReserveThenCommit(IHostMemory hostMemory, ulong reserveAddress, ulong reserveSize, ulong commitAddress, ulong commitSize, uint protection) + static bool TryReserveThenCommit(ulong reserveAddress, ulong reserveSize, ulong commitAddress, ulong commitSize, uint protection) { - if (!TryReserveRange(hostMemory, reserveAddress, reserveSize)) + if (!TryReserveRange(reserveAddress, reserveSize)) { return false; } - return TryCommitRange(hostMemory, commitAddress, commitSize, protection); + return TryCommitRange(commitAddress, commitSize, protection); } static bool IsAccessCompatible(ulong accessType, uint protection) diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs index 74b5d56..e691704 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs @@ -2,36 +2,43 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; using System.Threading; using SharpEmu.Core.Cpu; -using SharpEmu.Core.Cpu.Native.Windows; -using SharpEmu.Core.Loader; using SharpEmu.HLE; -using SharpEmu.HLE.Host; +using SharpEmu.Libs.Kernel; namespace SharpEmu.Core.Cpu.Native; public sealed partial class DirectExecutionBackend { + // The native import trampoline keeps the original guest GPR stack layout at + // argPackPtr and stores volatile SysV-only state immediately below it. This + // lets the managed gateway observe AL (the variadic vector-argument count) + // and all eight vector argument registers without changing the return-slot + // offsets used by the guest scheduler. + private const int ImportSavedRaxOffset = -176; + private const int ImportSavedR10Offset = -168; + private const int ImportSavedR11Offset = -160; + private const int ImportSavedMxcsrOffset = -152; + private const int ImportSavedFpuControlOffset = -148; + private const int ImportSavedXmmOffset = -128; + private const int ImportVectorRegisterCount = 8; + private const ulong StackCheckGuardValue = 0xC0DEC0DECAFEBA00UL; + private static long _canaryReturnRecoveries; + private readonly object _importResultLogSampleGate = new(); private readonly Dictionary _importResultLogSamples = new(StringComparer.Ordinal); + private int _il2CppExceptionDiagnosticCount; private static ulong ImportDispatchGatewayManaged(nint backendHandle, int importIndex, nint argPackPtr) { - if (LogThreadMode) - { - _threadModeGatewayCalls++; - _threadModeGatewayDepth++; - if (!_threadModeGatewayFirstLogged) - { - _threadModeGatewayFirstLogged = true; - TraceThreadMode($"gateway_first import={importIndex} total={_threadModeGatewayCalls}"); - } - } try { if (!(GCHandle.FromIntPtr(backendHandle).Target is DirectExecutionBackend directExecutionBackend)) @@ -41,6 +48,14 @@ public sealed partial class DirectExecutionBackend return 18446744071562199042uL; } + if (_perfHleHistogram) + { + var startTicks = System.Diagnostics.Stopwatch.GetTimestamp(); + var r = directExecutionBackend.DispatchImport(importIndex, argPackPtr); + RecordPerfHleDispatchTime(System.Diagnostics.Stopwatch.GetTimestamp() - startTicks); + return r; + } + return directExecutionBackend.DispatchImport(importIndex, argPackPtr); } catch (Exception ex) @@ -49,13 +64,6 @@ public sealed partial class DirectExecutionBackend $"[LOADER][ERROR] ImportDispatchGatewayManaged exception: {ex.GetType().Name}: {ex.Message}"); return 18446744071562199298uL; } - finally - { - if (LogThreadMode) - { - _threadModeGatewayDepth--; - } - } } private unsafe static int RawVectoredHandlerManaged(void* exceptionInfo) @@ -71,35 +79,68 @@ public sealed partial class DirectExecutionBackend private unsafe static int TryRecoverUnresolvedSentinel(void* exceptionInfo) { EXCEPTION_RECORD* exceptionRecord = ((EXCEPTION_POINTERS*)exceptionInfo)->ExceptionRecord; - if (exceptionRecord->ExceptionCode != WindowsFaultCodes.AccessViolation) + if (exceptionRecord->ExceptionCode != 3221225477u) { return 0; } void* contextRecord = ((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord; - ulong value = ReadCtxU64(contextRecord, CTX_RIP); + ulong value = ReadCtxU64(contextRecord, 248); ulong value2 = (ulong)exceptionRecord->ExceptionAddress; + if (value == StackCheckGuardValue && TryRecoverCanaryReturn(contextRecord)) + { + return -1; + } if (!IsUnresolvedSentinel(value) && !IsUnresolvedSentinel(value2)) { return 0; } - ulong rsp = ReadCtxU64(contextRecord, CTX_RSP); - WriteCtxU64(contextRecord, CTX_RAX, 0uL); + ulong rsp = ReadCtxU64(contextRecord, 152); + WriteCtxU64(contextRecord, 120, 0uL); if (TryGetPlausibleReturnFromStack(rsp, out var returnRip, out var nextRsp)) { - WriteCtxU64(contextRecord, CTX_RSP, nextRsp); - WriteCtxU64(contextRecord, CTX_RIP, returnRip); + WriteCtxU64(contextRecord, 152, nextRsp); + WriteCtxU64(contextRecord, 248, returnRip); Interlocked.Increment(ref _rawSentinelRecoveries); - if (LogThreadMode) - { - TraceThreadMode( - $"sentinel_recover rip=0x{value:X16} -> 0x{returnRip:X16} rsp=0x{rsp:X16} -> 0x{nextRsp:X16} " + - $"gateway_depth={_threadModeGatewayDepth}"); - } return -1; } return 0; } + private unsafe static bool TryRecoverCanaryReturn(void* contextRecord) + { + var rsp = ReadCtxU64(contextRecord, CTX_RSP); + var interruptedReturn = ReadCtxU64(contextRecord, CTX_RBP); + var interruptedFrame = rsp + 0x18; + if (!IsLikelyReturnAddress(interruptedReturn) || + rsp < sizeof(ulong) || + !TryReadStackU64(interruptedFrame, out var callerRbp) || + !TryReadStackU64(rsp + 0x20, out var callerReturn) || + callerRbp <= rsp || callerRbp - rsp > 0x10000 || + !IsLikelyReturnAddress(callerReturn)) + { + return false; + } + + // The guest unwind reached this callback return one stack slot late: the + // final pop loaded the interrupted return into rbp and ret consumed the + // stack guard. Resume at that return with the outer frame pointer rebuilt + // from the still-intact caller frame on the stack. + WriteCtxU64(contextRecord, CTX_RBP, interruptedFrame); + WriteCtxU64(contextRecord, CTX_RSP, rsp - sizeof(ulong)); + WriteCtxU64(contextRecord, CTX_RIP, interruptedReturn); + var recoveryCount = Interlocked.Increment(ref _canaryReturnRecoveries); + if (recoveryCount <= 4 || recoveryCount % 1000 == 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Recovered malformed canary return #{recoveryCount}: " + + $"resume=0x{interruptedReturn:X16} rsp=0x{rsp - sizeof(ulong):X16} " + + $"rbp=0x{interruptedFrame:X16} caller_rbp=0x{callerRbp:X16} " + + $"caller=0x{callerReturn:X16}"); + Console.Error.Flush(); + } + return true; + } + private unsafe ulong DispatchImport(int importIndex, nint argPackPtr) { long num = NextImportDispatchIndex(); @@ -113,32 +154,30 @@ public sealed partial class DirectExecutionBackend LastError = "Import dispatch called without active CPU context"; return 18446744071562199298uL; } - var importEntries = _importEntries; - if ((uint)importIndex >= (uint)importEntries.Length) + if ((uint)importIndex >= (uint)_importEntries.Length) { LastError = $"Import dispatch index out of range: {importIndex}"; return 18446744071562199042uL; } - ImportStubEntry importStubEntry = importEntries[importIndex]; + ImportStubEntry importStubEntry = _importEntries[importIndex]; + if (_perfHleHistogram) + { + RecordPerfHleCall(importStubEntry.Export?.Name ?? importStubEntry.Nid); + } int num2 = Volatile.Read(in _rawSentinelRecoveries); if (num2 != _lastReportedRawSentinelRecoveries) { Console.Error.WriteLine($"[LOADER][TRACE] Raw sentinel recoveries: {num2} (last import index={importIndex})"); _lastReportedRawSentinelRecoveries = num2; } - // Leaf imports take a scalar-only fast path that reads its own operands and - // intentionally bypasses the SysV variadic XMM marshalling below. This is safe - // only while the leaf set contains no float-variadic or float-returning function. - // The constraint and the audited list live on IsLeafImport — do not add a - // function that consumes xmm0-7 args or returns in xmm0 to the leaf set; - // such functions must stay on the full gateway path below. - if (IsLeafImport(importStubEntry.Nid) && + if (importStubEntry.IsLeaf && TryDispatchLeafImport(cpuContext, importStubEntry, argPackPtr, num, out var leafResult)) { return leafResult; } cpuContext.Rip = importStubEntry.Address; + LoadImportVolatileArguments(cpuContext, argPackPtr); cpuContext[CpuRegister.Rdi] = *(ulong*)argPackPtr; cpuContext[CpuRegister.Rsi] = *(ulong*)(argPackPtr + 8); cpuContext[CpuRegister.Rdx] = *(ulong*)(argPackPtr + 16); @@ -151,26 +190,7 @@ public sealed partial class DirectExecutionBackend cpuContext[CpuRegister.R13] = *(ulong*)(argPackPtr + 72); cpuContext[CpuRegister.R14] = *(ulong*)(argPackPtr + 80); cpuContext[CpuRegister.R15] = *(ulong*)(argPackPtr + 88); - // The trampoline spills the SysV variadic XMM save area (xmm0..xmm7) into the - // 0x80 bytes immediately below the GP argpack, so variadic float args (printf - // %f, and powf/logf inputs) reach the handler. xmm{i} is at argPackPtr-0x80+i*16. - for (var xmmIndex = 0; xmmIndex < 8; xmmIndex++) - { - var xmmSlot = argPackPtr - 0x80 + (xmmIndex * 16); - cpuContext.SetXmmRegister( - xmmIndex, - *(ulong*)xmmSlot, - *(ulong*)(xmmSlot + 8)); - } cpuContext[CpuRegister.Rsp] = (ulong)argPackPtr + 96uL; - if (importStubEntry.Kind == ImportStubKind.BootstrapBridge) - { - NormalizeKernelDynlibDlsymArguments(cpuContext, out _, out _); - *(ulong*)argPackPtr = cpuContext[CpuRegister.Rdi]; - *(ulong*)(argPackPtr + 8) = cpuContext[CpuRegister.Rsi]; - *(ulong*)(argPackPtr + 16) = cpuContext[CpuRegister.Rdx]; - } - ulong value = cpuContext[CpuRegister.Rdi]; ulong value2 = cpuContext[CpuRegister.Rsi]; ulong num3 = cpuContext[CpuRegister.Rdx]; @@ -184,6 +204,25 @@ public sealed partial class DirectExecutionBackend ulong value7 = cpuContext[CpuRegister.R14]; ulong value8 = cpuContext[CpuRegister.R15]; ulong num7 = *(ulong*)(argPackPtr + 96); + var importStackPointer = (ulong)argPackPtr + 96; + var probeTarget = (_probeImportReturnAddress != 0 && num7 == _probeImportReturnAddress) || + (string.Equals(importStubEntry.Nid, "2Z+PpY6CaJg", StringComparison.Ordinal) && + importStackPointer >= 0x00006FFFAC1FF000UL && + importStackPointer < 0x00006FFFAC200000UL); + if (probeTarget && + Interlocked.Increment(ref _probeImportReturnAddressCount) <= 2048) + { + var frameValue = TryReadStackU64(value4, out var savedRbp) ? savedRbp : 0; + var frameReturn = TryReadStackU64(value4 + sizeof(ulong), out var savedReturn) + ? savedReturn + : 0; + Console.Error.WriteLine( + $"[LOADER][TRACE] import-return-address-probe " + + $"thread=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} " + + $"nid={importStubEntry.Nid} ret=0x{num7:X16} " + + $"rsp=0x{(ulong)argPackPtr + 96:X16} rbp=0x{value4:X16} " + + $"saved_rbp=0x{frameValue:X16} saved_ret=0x{frameReturn:X16}"); + } var isGuestWorker = GuestThreadExecution.IsGuestThread; if (!IsLikelyReturnAddress(num7)) { @@ -199,11 +238,54 @@ public sealed partial class DirectExecutionBackend } } } + // Diagnostic compatibility escape hatch for a guest stack-protector + // failure whose noreturn call is immediately followed by UD2. Returning + // normally from the HLE export would execute that UD2; redirect this one + // well-known compiler epilogue back through its register/stack unwind. + // Keep the byte-pattern check strict so the opt-in cannot guess at an + // unrelated function layout. + if (string.Equals(importStubEntry.Nid, "Ou3iL1abvng", StringComparison.Ordinal) && + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_IGNORE_STACK_CHK"), + "1", + StringComparison.Ordinal) && + num7 >= 0x20) + { + var returnCode = (byte*)num7; + if (returnCode[0] == 0x0F && returnCode[1] == 0x0B && + returnCode[-22] == 0x75 && returnCode[-21] == 0x0F && + returnCode[-20] == 0x48 && returnCode[-19] == 0x83 && + returnCode[-18] == 0xC4) + { + var recoveredReturn = num7 - 20; + *(ulong*)(argPackPtr + 96) = recoveredReturn; + cpuContext[CpuRegister.Rax] = 0; + Console.Error.WriteLine( + $"[LOADER][WARN] Recovered guest stack-check epilogue " + + $"ret=0x{num7:X16} -> 0x{recoveredReturn:X16}"); + return 0; + } + } if (_activeGuestThreadState is { } activeGuestThreadState) { Interlocked.Increment(ref activeGuestThreadState.ImportCount); - Volatile.Write(ref activeGuestThreadState.LastImportNid, importStubEntry.Nid); + activeGuestThreadState.LastImportRdi = value; + activeGuestThreadState.LastImportRsi = value2; + activeGuestThreadState.LastImportRdx = num3; + activeGuestThreadState.LastImportRcx = num4; + activeGuestThreadState.LastImportR8 = num5; + activeGuestThreadState.LastImportR9 = num6; + activeGuestThreadState.LastImportStack0 = ReadImportStackArgument(argPackPtr, 0); + activeGuestThreadState.LastImportStack1 = ReadImportStackArgument(argPackPtr, 1); + activeGuestThreadState.LastImportStack2 = ReadImportStackArgument(argPackPtr, 2); + activeGuestThreadState.LastImportStack3 = ReadImportStackArgument(argPackPtr, 3); + activeGuestThreadState.LastImportStack4 = ReadImportStackArgument(argPackPtr, 4); + activeGuestThreadState.LastImportStack5 = ReadImportStackArgument(argPackPtr, 5); + Volatile.Write(ref activeGuestThreadState.LastImportResultValid, 0); Volatile.Write(ref activeGuestThreadState.LastReturnRip, num7); + // Publish the NID last so readers cannot pair a new import name with + // the preceding import's argument snapshot. + Volatile.Write(ref activeGuestThreadState.LastImportNid, importStubEntry.Nid); } if (_logStrlenBursts) { @@ -212,7 +294,8 @@ public sealed partial class DirectExecutionBackend } if (!string.IsNullOrWhiteSpace(_probeImportReturn) && (string.Equals(_probeImportReturn, "*", StringComparison.Ordinal) || - string.Equals(_probeImportReturn, importStubEntry.Nid, StringComparison.Ordinal))) + string.Equals(_probeImportReturn, importStubEntry.Nid, StringComparison.Ordinal)) && + Interlocked.Increment(ref _probeImportReturnAddressCount) <= 8) { ProbeReturnRip(num7, num); } @@ -221,55 +304,37 @@ public sealed partial class DirectExecutionBackend TraceGuestContext( $"import dispatch={num} nid={importStubEntry.Nid} ret=0x{num7:X16} managed={Environment.CurrentManagedThreadId} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16} active={HasActiveExecutionThread}"); } + if (_logBootstrap && string.Equals(importStubEntry.Nid, RuntimeStubNids.BootstrapBridge, StringComparison.Ordinal)) + { + string symbolText = ""; + if (TryReadAsciiZ(value2, 256, out var sym)) + { + symbolText = sym; + } + Console.Error.WriteLine( + $"[LOADER][TRACE] bootstrap_call#{num}: op=0x{value:X16} sym_ptr=0x{value2:X16} sym='{symbolText}' out_ptr=0x{num3:X16} ret=0x{num7:X16}"); + } if (!isGuestWorker && !ActiveForcedGuestExit && - ShouldForceGuestExitOnImportLoop(importStubEntry.Nid, num7, num, value, value2) && + ShouldForceGuestExitOnImportLoop(in importStubEntry, num7, num, value, value2) && TryForceGuestExitToHostStub(argPackPtr, num, num7, importStubEntry.Nid)) { cpuContext[CpuRegister.Rax] = 1uL; return 1uL; } - // Backend teardown in progress: unwind this worker to the host instead of - // dispatching. RunGuestThread parks it as Blocked with no wake handler, its - // host thread exits, and RequestGuestThreadTeardown can finish before any - // executable memory is freed. - if (isGuestWorker && - _guestTeardownRequested && - TryYieldGuestThreadToHostStub(argPackPtr, num, num7, importStubEntry.Nid, "backend teardown")) - { - cpuContext[CpuRegister.Rax] = 0uL; - return 0uL; - } - if (_hostShutdownRequested) - { - if (isGuestWorker && - TryYieldGuestThreadToHostStub(argPackPtr, num, num7, importStubEntry.Nid, "host shutdown")) - { - cpuContext[CpuRegister.Rax] = 0uL; - return 0uL; - } - - if (!isGuestWorker && - TryAbortGuestForHostShutdown(argPackPtr, num, num7)) - { - cpuContext[CpuRegister.Rax] = 1uL; - return 1uL; - } - } - bool flag0 = ShouldSuppressStrlenTrace(importStubEntry.Nid); + bool flag0 = importStubEntry.SuppressStrlenTrace; bool flag = num7 >= 2156221920u && num7 <= 2156225024u; bool flag2 = num7 >= 2156351360u && num7 <= 2156352080u; bool flag3 = num >= 1020 && num <= 1040; bool flag4 = !string.IsNullOrWhiteSpace(_importFilter); bool flag5 = false; ExportedFunction? matchedExport = importStubEntry.Export; - var traceFlags = importStubEntry.TraceFlags; bool periodicTrace = num <= 128 || (num >= 240 && num <= 400) || (num >= 900 && num <= 1300) || num % 100000 == 0L || - ((traceFlags & ImportStubTraceFlags.PeriodicEvery1000) != 0 && (num <= 256 || num % 1000 == 0L)) || - ((traceFlags & ImportStubTraceFlags.PeriodicEvery128) != 0 && (num <= 256 || num % 128 == 0)) || + (importStubEntry.Nid == "tsvEmnenz48" && (num <= 256 || num % 1000 == 0L)) || + (importStubEntry.Nid == "rTXw65xmLIA" && (num <= 256 || num % 128 == 0)) || flag || flag2 || flag3; @@ -330,21 +395,33 @@ public sealed partial class DirectExecutionBackend cpuContext[CpuRegister.Rsi], cpuContext[CpuRegister.Rdx]); } - if ((traceFlags & ImportStubTraceFlags.Memset) != 0 && num <= 256) + if (importStubEntry.Nid == "8zTFvBIAIN8" && num <= 256) { Console.Error.WriteLine($"[LOADER][TRACE] memset#{num}: dst=0x{cpuContext[CpuRegister.Rdi]:X16} val=0x{cpuContext[CpuRegister.Rsi] & 0xFF:X2} len=0x{cpuContext[CpuRegister.Rdx]:X16} ret=0x{num7:X16}"); } - if ((traceFlags & ImportStubTraceFlags.CxaAtexit) != 0 && num <= 64) + if (importStubEntry.Nid == "tsvEmnenz48" && num <= 64) { Console.Error.WriteLine($"[LOADER][TRACE] __cxa_atexit#{num}: func=0x{cpuContext[CpuRegister.Rdi]:X16} arg=0x{cpuContext[CpuRegister.Rsi]:X16} dso=0x{cpuContext[CpuRegister.Rdx]:X16} ret=0x{num7:X16}"); } - if ((traceFlags & ImportStubTraceFlags.RawArgs) != 0) + if (importStubEntry.Nid == "bzQExy189ZI" || importStubEntry.Nid == "8G2LB+A3rzg") { Console.Error.WriteLine($"[LOADER][TRACE] {importStubEntry.Nid}#{num}: rdi=0x{cpuContext[CpuRegister.Rdi]:X16} rsi=0x{cpuContext[CpuRegister.Rsi]:X16} rdx=0x{cpuContext[CpuRegister.Rdx]:X16} ret=0x{num7:X16}"); } if (flag6 || flag || flag2 || flag3) { Console.Error.WriteLine($"[LOADER][TRACE] ImportCtx#{num}: nid={importStubEntry.Nid} ret=0x{num7:X16} rdi=0x{cpuContext[CpuRegister.Rdi]:X16} rsi=0x{cpuContext[CpuRegister.Rsi]:X16} rdx=0x{cpuContext[CpuRegister.Rdx]:X16} rcx=0x{cpuContext[CpuRegister.Rcx]:X16}"); + if (flag6) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] ImportArgs#{num}: " + + $"r8=0x{cpuContext[CpuRegister.R8]:X16} r9=0x{cpuContext[CpuRegister.R9]:X16} " + + $"stack0=0x{ReadImportStackArgument(argPackPtr, 0):X16} " + + $"stack1=0x{ReadImportStackArgument(argPackPtr, 1):X16} " + + $"stack2=0x{ReadImportStackArgument(argPackPtr, 2):X16} " + + $"stack3=0x{ReadImportStackArgument(argPackPtr, 3):X16} " + + $"stack4=0x{ReadImportStackArgument(argPackPtr, 4):X16} " + + $"stack5=0x{ReadImportStackArgument(argPackPtr, 5):X16}"); + } Console.Error.WriteLine($"[LOADER][TRACE] ImportNV#{num}: rbx=0x{value3:X16} rbp=0x{value4:X16} r12=0x{value5:X16} r13=0x{value6:X16} r14=0x{value7:X16} r15=0x{value8:X16}"); if (flag3) { @@ -368,17 +445,30 @@ public sealed partial class DirectExecutionBackend Console.Error.Flush(); } } - if ((traceFlags & ImportStubTraceFlags.StackChkFail) != 0) + if (importStubEntry.Nid == "Ou3iL1abvng") { if (_logStackCheck) { - var savedGuardAddress = value4 >= 0x10 ? value4 - 0x10 : 0; - var guardKnown = TryReadUInt64Compat(value3, out var guardValue); - var savedKnown = TryReadUInt64Compat(savedGuardAddress, out var savedGuardValue); + var rbxGuardKnown = TryReadUInt64Compat(value3, out var rbxGuardValue); + var r12GuardKnown = TryReadUInt64Compat(value5, out var r12GuardValue); Console.Error.WriteLine( - $"[LOADER][TRACE] stack_chk_diag#{num}: ret=0x{num7:X16} guard_ptr=0x{value3:X16} " + - $"guard={(guardKnown ? $"0x{guardValue:X16}" : "?")} saved@0x{savedGuardAddress:X16}={(savedKnown ? $"0x{savedGuardValue:X16}" : "?")} " + + $"[LOADER][TRACE] stack_chk_diag#{num}: ret=0x{num7:X16} " + + $"rbx=0x{value3:X16}:{(rbxGuardKnown ? $"0x{rbxGuardValue:X16}" : "?")} " + + $"r12=0x{value5:X16}:{(r12GuardKnown ? $"0x{r12GuardValue:X16}" : "?")} " + $"rbp=0x{value4:X16} rsp=0x{((ulong)argPackPtr + 96uL):X16}"); + + // Stack-protector layouts vary by compiler and function. Emit the + // bounded caller-frame tail instead of assuming a fixed canary + // offset; this also exposes an adjacent ABI output buffer overwrite. + for (var frameOffset = 0x10; frameOffset <= 0x80; frameOffset += sizeof(ulong)) + { + var slotAddress = value4 >= (ulong)frameOffset ? value4 - (ulong)frameOffset : 0; + if (slotAddress != 0 && TryReadUInt64Compat(slotAddress, out var slotValue)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] stack_chk_frame#{num}: [rbp-0x{frameOffset:X}]=0x{slotValue:X16}"); + } + } } try { @@ -400,20 +490,16 @@ public sealed partial class DirectExecutionBackend ActiveGuestReturnSlotAddress); try { - if (importStubEntry.Kind == ImportStubKind.BootstrapBridge) + if (string.Equals(importStubEntry.Nid, RuntimeStubNids.BootstrapBridge, StringComparison.Ordinal)) { - if (_logBootstrap) - { - RecordDeferredBootstrapTrace(num, value, value2, num3, num7); - } - orbisGen2Result = DispatchBootstrapBridge(); } - else if (importStubEntry.Kind == ImportStubKind.KernelDynlibDlsym) + else if (string.Equals(importStubEntry.Nid, RuntimeStubNids.KernelDynlibDlsym, StringComparison.Ordinal) || + string.Equals(importStubEntry.Nid, "LwG8g3niqwA", StringComparison.Ordinal)) { orbisGen2Result = DispatchKernelDynlibDlsym(); } - else if (importStubEntry.Kind == ImportStubKind.Il2CppApiLookupSymbol) + else if (string.Equals(importStubEntry.Nid, "r8mvOaWdi28", StringComparison.Ordinal)) { orbisGen2Result = DispatchIl2CppApiLookupSymbol(); } @@ -428,26 +514,21 @@ public sealed partial class DirectExecutionBackend } orbisGen2Result = (OrbisGen2Result)returnValue; } - else if (importStubEntry.Export is { } mismatchedExport) - { - dispatchResolved = true; - orbisGen2Result = OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_IMPLEMENTED; - cpuContext[CpuRegister.Rax] = unchecked((ulong)(int)orbisGen2Result); - Console.Error.WriteLine( - $"[LOADER][WARN] Import#{num} not implemented for generation {cpuContext.TargetGeneration}: " + - $"nid={importStubEntry.Nid} targets={mismatchedExport.Target} ret=0x{num7:X16}"); - } - else - { - dispatchResolved = false; - orbisGen2Result = OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; - cpuContext[CpuRegister.Rax] = unchecked((ulong)(int)orbisGen2Result); - } + else + { + dispatchResolved = false; + orbisGen2Result = OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + cpuContext[CpuRegister.Rax] = unchecked((ulong)(int)orbisGen2Result); + } } finally { GuestThreadExecution.RestoreImportCallFrame(previousImportCallFrame); } + DeliverPendingGuestExceptionAtSafePoint( + cpuContext, + CaptureImportBoundaryContinuation(cpuContext, argPackPtr, num7)); + StoreImportVectorReturn(cpuContext, argPackPtr); if (dispatchResolved && orbisGen2Result == OrbisGen2Result.ORBIS_GEN2_OK && string.Equals(importStubEntry.Nid, "BohYr-F7-is", StringComparison.Ordinal)) @@ -457,6 +538,15 @@ public sealed partial class DirectExecutionBackend if (!dispatchResolved) { LastError = "Missing HLE export for NID: " + importStubEntry.Nid; + if (string.Equals(importStubEntry.Nid, "cfwBSQyr5Ys", StringComparison.Ordinal) && + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_IL2CPP_EXCEPTION"), + "1", + StringComparison.Ordinal) && + Interlocked.Increment(ref _il2CppExceptionDiagnosticCount) <= 4) + { + DumpIl2CppExceptionDiagnostic(cpuContext, value, num7); + } Console.Error.WriteLine( $"[LOADER][WARN] Import#{num} unresolved: nid={importStubEntry.Nid} ret=0x{num7:X16} " + $"rdi=0x{value:X16} rsi=0x{value2:X16} rdx=0x{num3:X16} rcx=0x{num4:X16} r8=0x{num5:X16} r9=0x{num6:X16}"); @@ -516,6 +606,11 @@ public sealed partial class DirectExecutionBackend $"fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16}"); } + if (_activeGuestThreadState is { } transferGuestThreadState) + { + Volatile.Write(ref transferGuestThreadState.LastImportRax, transferTarget.Rax); + Volatile.Write(ref transferGuestThreadState.LastImportResultValid, 1); + } return unchecked((ulong)transferFrame); } if (GuestThreadExecution.TryConsumeCurrentEntryExit(out var exitValue, out var exitReason)) @@ -559,15 +654,31 @@ public sealed partial class DirectExecutionBackend Console.Error.Flush(); } } - // Publish the handler's XMM0 back into the argpack's xmm0 save slot; the - // trampoline epilogue reloads it into the guest's XMM0, delivering float/double - // return values (powf/logf/wcstod). Harmless for int/pointer returns (XMM is - // volatile across a SysV call, so the guest never relies on a preserved XMM0). - cpuContext.GetXmmRegister(0, out var returnXmm0Low, out var returnXmm0High); - *(ulong*)(argPackPtr - 0x80) = returnXmm0Low; - *(ulong*)(argPackPtr - 0x80 + 8) = returnXmm0High; - DrainDeferredBootstrapTraces(); - return cpuContext[CpuRegister.Rax]; + var guestReturnValue = cpuContext[CpuRegister.Rax]; + if (probeTarget) + { + ulong finalReturnSlot; + try + { + finalReturnSlot = *(ulong*)(argPackPtr + 96); + } + catch + { + finalReturnSlot = 0; + } + Console.Error.WriteLine( + $"[LOADER][TRACE] import-return-address-probe-exit " + + $"thread=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} " + + $"nid={importStubEntry.Nid} original=0x{num7:X16} " + + $"final=0x{finalReturnSlot:X16} rsp=0x{(ulong)argPackPtr + 96:X16} " + + $"yield={ActiveGuestThreadYieldRequested}"); + } + if (_activeGuestThreadState is { } completedGuestThreadState) + { + Volatile.Write(ref completedGuestThreadState.LastImportRax, guestReturnValue); + Volatile.Write(ref completedGuestThreadState.LastImportResultValid, 1); + } + return guestReturnValue; } catch (Exception ex) { @@ -575,11 +686,539 @@ public sealed partial class DirectExecutionBackend Console.Error.WriteLine($"[LOADER][ERROR] {LastError}"); Console.Error.WriteLine($"[LOADER][ERROR] {ex.StackTrace}"); cpuContext[CpuRegister.Rax] = 18446744071562199298uL; - DrainDeferredBootstrapTraces(); - return 18446744071562199298uL; + if (_activeGuestThreadState is { } failedGuestThreadState) + { + Volatile.Write(ref failedGuestThreadState.LastImportRax, cpuContext[CpuRegister.Rax]); + Volatile.Write(ref failedGuestThreadState.LastImportResultValid, 1); + } + return cpuContext[CpuRegister.Rax]; } } + private static void DumpIl2CppExceptionDiagnostic( + CpuContext cpuContext, + ulong wrapperAddress, + ulong returnAddress) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.wrapper=0x{wrapperAddress:X16} " + + $"ret=0x{returnAddress:X16}"); + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.registers " + + $"rbx=0x{cpuContext[CpuRegister.Rbx]:X16} " + + $"r12=0x{cpuContext[CpuRegister.R12]:X16} " + + $"r13=0x{cpuContext[CpuRegister.R13]:X16} " + + $"r14=0x{cpuContext[CpuRegister.R14]:X16} " + + $"r15=0x{cpuContext[CpuRegister.R15]:X16}"); + if (GuestThreadExecution.TryGetCurrentImportCallFrame(out var frame)) + { + DumpGuestCodePointers(cpuContext, "stack", frame.ResumeRsp, 0x1000); + } + DumpGuestFramePointerChain(cpuContext, cpuContext[CpuRegister.Rbp]); + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_PS5_USER_SLOTS"), + "1", + StringComparison.Ordinal)) + { + for (var slot = 0; slot < 4; slot++) + { + DumpGuestQwords( + cpuContext, + $"ps5_user_slot[{slot}]", + 0x0000000801A73110 + (ulong)(slot * 0x51C8), + 0x60); + } + } + + if (!TryReadGuestU64(cpuContext, wrapperAddress, out var exceptionAddress)) + { + Console.Error.WriteLine("[LOADER][TRACE] il2cpp_exception.wrapper unreadable"); + return; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.object=0x{exceptionAddress:X16}"); + DumpGuestQwords(cpuContext, "exception", exceptionAddress, 0x90); + + // Il2CppException begins with Il2CppObject (klass, monitor), followed by + // trace_ips, inner_ex and message. Decode both the documented message + // slot and nearby object pointers because Unity revisions have appended + // fields without changing the wrapper itself. + for (var offset = 0; offset <= 0x80; offset += 8) + { + if (!TryReadGuestU64(cpuContext, exceptionAddress + (ulong)offset, out var candidate) || + candidate < 0x10000) + { + continue; + } + + if (TryReadIl2CppString(cpuContext, candidate, out var text)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.string+0x{offset:X2}=" + + $"'{text}'"); + } + } + + if (TryReadGuestU64(cpuContext, exceptionAddress + 0x38, out var traceIpsAddress)) + { + DumpIl2CppPointerArray(cpuContext, "trace_ips", traceIpsAddress); + } + + if (TryReadGuestU64(cpuContext, exceptionAddress, out var klassAddress)) + { + DumpGuestQwords(cpuContext, "exception_class", klassAddress, 0x100); + for (var offset = 0; offset <= 0xF8; offset += 8) + { + if (!TryReadGuestU64(cpuContext, klassAddress + (ulong)offset, out var candidate) || + candidate < 0x10000) + { + continue; + } + + if (TryReadGuestCString(cpuContext, candidate, out var text)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.class_string+0x{offset:X2}=" + + $"'{text}'"); + } + } + } + } + + private static void DumpGuestCodePointers( + CpuContext cpuContext, + string label, + ulong address, + int byteCount) + { + var buffer = new byte[byteCount]; + if (!cpuContext.Memory.TryRead(address, buffer)) + { + return; + } + + var logged = 0; + for (var offset = 0; offset <= buffer.Length - 8 && logged < 128; offset += 8) + { + var candidate = BinaryPrimitives.ReadUInt64LittleEndian(buffer.AsSpan(offset, 8)); + if (candidate < 0x0000000800000000 || candidate >= 0x0000001000000000) + { + continue; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}+0x{offset:X3}=0x{candidate:X16}"); + logged++; + } + } + + private static void DumpGuestFramePointerChain(CpuContext cpuContext, ulong framePointer) + { + for (var depth = 0; depth < 64 && framePointer >= 0x10000; depth++) + { + if (!TryReadGuestU64(cpuContext, framePointer, out var nextFrame) || + !TryReadGuestU64(cpuContext, framePointer + 8, out var returnRip)) + { + break; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.frame[{depth}] " + + $"rbp=0x{framePointer:X16} ret=0x{returnRip:X16}"); + if (framePointer >= 0x40) + { + DumpGuestQwords( + cpuContext, + $"frame[{depth}]_locals", + framePointer - 0x40, + 0x60); + if (depth <= 10 && + TryReadGuestU64(cpuContext, framePointer - 0x20, out var savedRbx) && + savedRbx >= 0x0000000100000000 && + savedRbx < 0x0000000800000000) + { + DumpGuestQwords( + cpuContext, + $"frame[{depth}]_saved_rbx_object", + savedRbx, + 0x50); + if (depth is >= 4 and <= 12) + { + DumpIl2CppObjectGraph( + cpuContext, + $"frame[{depth}]_saved_rbx", + savedRbx); + } + } + DumpIl2CppStringsInRange( + cpuContext, + $"frame[{depth}]", + framePointer >= 0x100 ? framePointer - 0x100 : 0, + 0x140); + DumpIl2CppObjectsInRange( + cpuContext, + $"frame[{depth}]", + framePointer >= 0x100 ? framePointer - 0x100 : 0, + 0x140); + } + if (nextFrame <= framePointer || nextFrame - framePointer > 0x100000) + { + break; + } + + framePointer = nextFrame; + } + } + + private static void DumpIl2CppObjectsInRange( + CpuContext cpuContext, + string label, + ulong address, + int byteCount) + { + var buffer = new byte[byteCount]; + if (address == 0 || !cpuContext.Memory.TryRead(address, buffer)) + { + return; + } + + for (var offset = 0; offset <= buffer.Length - 8; offset += 8) + { + var candidate = BinaryPrimitives.ReadUInt64LittleEndian(buffer.AsSpan(offset, 8)); + if (candidate < 0x0000000100000000 || + candidate >= 0x0000000800000000 || + !TryReadGuestU64(cpuContext, candidate, out var klass) || + klass < 0x0000000100000000 || + klass >= 0x0000000800000000 || + !TryReadGuestU64(cpuContext, klass + 0x10, out var nameAddress) || + !TryReadGuestCString(cpuContext, nameAddress, out var name)) + { + continue; + } + + var nameSpace = string.Empty; + if (TryReadGuestU64(cpuContext, klass + 0x18, out var namespaceAddress)) + { + _ = TryReadGuestCString(cpuContext, namespaceAddress, out nameSpace); + } + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_object@{offset:X3}=" + + $"0x{candidate:X16} {nameSpace}.{name}"); + if (string.Equals(name, "ControllerMap_Editor", StringComparison.Ordinal)) + { + DumpGuestQwords(cpuContext, $"{label}_controller_map", candidate, 0x40); + for (var fieldOffset = 0x20; fieldOffset <= 0x28; fieldOffset += 8) + { + if (TryReadGuestU64(cpuContext, candidate + (ulong)fieldOffset, out var stringAddress) && + TryReadIl2CppString(cpuContext, stringAddress, out var fieldText)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_controller_map+0x{fieldOffset:X2}=" + + $"'{fieldText}'"); + } + } + } + } + } + + private static void DumpIl2CppStringsInRange( + CpuContext cpuContext, + string label, + ulong address, + int byteCount) + { + var buffer = new byte[byteCount]; + if (address == 0 || !cpuContext.Memory.TryRead(address, buffer)) + { + return; + } + + for (var offset = 0; offset <= buffer.Length - 8; offset += 8) + { + var candidate = BinaryPrimitives.ReadUInt64LittleEndian(buffer.AsSpan(offset, 8)); + if (candidate < 0x0000000100000000 || + candidate >= 0x0000000800000000 || + !TryReadIl2CppString(cpuContext, candidate, out var text) || + text.Length == 0) + { + continue; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_string@{offset:X3}=" + + $"0x{candidate:X16} '{text}'"); + } + } + + private static void DumpIl2CppObjectGraph( + CpuContext cpuContext, + string label, + ulong objectAddress) + { + if (!TryReadGuestU64(cpuContext, objectAddress, out var klassAddress)) + { + return; + } + + DumpGuestQwords(cpuContext, $"{label}_class", klassAddress, 0x400); + for (var offset = 0; offset <= 0xF8; offset += 8) + { + if (TryReadGuestU64(cpuContext, klassAddress + (ulong)offset, out var candidate) && + candidate >= 0x10000 && + TryReadGuestCString(cpuContext, candidate, out var text)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_class_string+0x{offset:X2}='{text}'"); + } + } + for (var offset = 0x10; offset <= 0x48; offset += 8) + { + if (!TryReadGuestU64(cpuContext, objectAddress + (ulong)offset, out var candidate) || + candidate < 0x0000000100000000 || + candidate >= 0x0000000800000000) + { + continue; + } + + DumpGuestQwords( + cpuContext, + $"{label}_field+0x{offset:X2}", + candidate, + 0x80); + if (TryReadGuestU64(cpuContext, candidate, out var candidateClass)) + { + DumpGuestQwords( + cpuContext, + $"{label}_field+0x{offset:X2}_class", + candidateClass, + 0x400); + for (var classOffset = 0; classOffset <= 0xF8; classOffset += 8) + { + if (TryReadGuestU64( + cpuContext, + candidateClass + (ulong)classOffset, + out var classCandidate) && + classCandidate >= 0x10000 && + TryReadGuestCString(cpuContext, classCandidate, out var text)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_field+0x{offset:X2}" + + $"_class_string+0x{classOffset:X2}='{text}'"); + } + } + } + } + } + + private static void DumpIl2CppPointerArray( + CpuContext cpuContext, + string label, + ulong address) + { + if (address < 0x10000 || + !TryReadGuestU64(cpuContext, address + 0x18, out var rawLength)) + { + return; + } + + var length = (int)Math.Min(rawLength, 128); + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}=0x{address:X16} " + + $"length={rawLength}"); + for (var index = 0; index < length; index++) + { + if (!TryReadGuestU64(cpuContext, address + 0x20 + (ulong)(index * 8), out var value)) + { + break; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}[{index}]=0x{value:X16}"); + } + } + + private static void DumpGuestQwords( + CpuContext cpuContext, + string label, + ulong address, + int byteCount) + { + var buffer = new byte[byteCount]; + if (!cpuContext.Memory.TryRead(address, buffer)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}=unreadable@0x{address:X16}"); + return; + } + + for (var offset = 0; offset < buffer.Length; offset += 32) + { + var values = new string[Math.Min(4, (buffer.Length - offset) / 8)]; + for (var i = 0; i < values.Length; i++) + { + values[i] = $"{BinaryPrimitives.ReadUInt64LittleEndian(buffer.AsSpan(offset + i * 8, 8)):X16}"; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}+0x{offset:X2}: " + + string.Join(" ", values)); + } + } + + private static bool TryReadGuestU64(CpuContext cpuContext, ulong address, out ulong value) + { + Span buffer = stackalloc byte[8]; + if (!cpuContext.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(buffer); + return true; + } + + private static bool TryReadIl2CppString( + CpuContext cpuContext, + ulong address, + out string text) + { + text = string.Empty; + Span header = stackalloc byte[20]; + if (!cpuContext.Memory.TryRead(address, header)) + { + return false; + } + + var length = BinaryPrimitives.ReadInt32LittleEndian(header[16..]); + if (length <= 0 || length > 2048) + { + return false; + } + + var bytes = new byte[length * 2]; + if (!cpuContext.Memory.TryRead(address + 20, bytes)) + { + return false; + } + + text = SanitizeDiagnosticText(Encoding.Unicode.GetString(bytes)); + return text.Length != 0; + } + + private static bool TryReadGuestCString( + CpuContext cpuContext, + ulong address, + out string text) + { + text = string.Empty; + var buffer = new byte[256]; + if (!cpuContext.Memory.TryRead(address, buffer)) + { + return false; + } + + var length = Array.IndexOf(buffer, (byte)0); + if (length <= 0) + { + return false; + } + + for (var i = 0; i < length; i++) + { + if (buffer[i] is < 0x20 or > 0x7E) + { + return false; + } + } + + text = Encoding.UTF8.GetString(buffer, 0, length); + return true; + } + + private static string SanitizeDiagnosticText(string text) + { + var builder = new StringBuilder(Math.Min(text.Length, 512)); + foreach (var character in text) + { + if (builder.Length >= 512) + { + break; + } + + builder.Append(char.IsControl(character) ? ' ' : character); + } + + return builder.ToString().Trim(); + } + + private unsafe static void LoadImportVolatileArguments(CpuContext cpuContext, nint argPackPtr) + { + cpuContext[CpuRegister.Rax] = *(ulong*)(argPackPtr + ImportSavedRaxOffset); + cpuContext[CpuRegister.R10] = *(ulong*)(argPackPtr + ImportSavedR10Offset); + cpuContext[CpuRegister.R11] = *(ulong*)(argPackPtr + ImportSavedR11Offset); + cpuContext.Mxcsr = *(uint*)(argPackPtr + ImportSavedMxcsrOffset); + cpuContext.FpuControlWord = *(ushort*)(argPackPtr + ImportSavedFpuControlOffset); + for (var registerIndex = 0; registerIndex < ImportVectorRegisterCount; registerIndex++) + { + var registerAddress = argPackPtr + ImportSavedXmmOffset + (registerIndex * 16); + cpuContext.SetXmmRegister( + registerIndex, + *(ulong*)registerAddress, + *(ulong*)(registerAddress + 8)); + } + } + + private unsafe static void StoreImportVectorReturn(CpuContext cpuContext, nint argPackPtr) + { + // AMD64 returns scalar/vector floating-point values in XMM0 and may use + // XMM1 for the second eightbyte of a classified aggregate. + for (var registerIndex = 0; registerIndex < 2; registerIndex++) + { + cpuContext.GetXmmRegister(registerIndex, out var low, out var high); + var registerAddress = argPackPtr + ImportSavedXmmOffset + (registerIndex * 16); + *(ulong*)registerAddress = low; + *(ulong*)(registerAddress + 8) = high; + } + } + + private static ulong ReadImportStackArgument(nint argPackPtr, int index) + { + var address = checked((ulong)argPackPtr + 104UL + (ulong)index * sizeof(ulong)); + return TryReadHostQword(address, out var value) ? value : 0; + } + + private static GuestCpuContinuation CaptureImportBoundaryContinuation( + CpuContext context, + nint argPackPtr, + ulong returnRip) => + new( + Rip: returnRip, + Rsp: (ulong)argPackPtr + 104UL, + ReturnSlotAddress: (ulong)argPackPtr + 96UL, + Rflags: context.Rflags, + FsBase: context.FsBase, + GsBase: context.GsBase, + Rax: context[CpuRegister.Rax], + Rcx: context[CpuRegister.Rcx], + Rdx: context[CpuRegister.Rdx], + Rbx: context[CpuRegister.Rbx], + Rbp: context[CpuRegister.Rbp], + Rsi: context[CpuRegister.Rsi], + Rdi: context[CpuRegister.Rdi], + R8: context[CpuRegister.R8], + R9: context[CpuRegister.R9], + R10: context[CpuRegister.R10], + R11: context[CpuRegister.R11], + R12: context[CpuRegister.R12], + R13: context[CpuRegister.R13], + R14: context[CpuRegister.R14], + R15: context[CpuRegister.R15], + FpuControlWord: context.FpuControlWord, + Mxcsr: context.Mxcsr, + RestoreFullFpuState: false); + private unsafe bool TryDispatchLeafImport( CpuContext cpuContext, ImportStubEntry importStubEntry, @@ -596,7 +1235,20 @@ public sealed partial class DirectExecutionBackend var arg0 = *(ulong*)argPackPtr; var returnRip = *(ulong*)(argPackPtr + 96); + var leafStackPointer = (ulong)argPackPtr + 96UL; + var probeLeafReturn = _logAllImports && + string.Equals(importStubEntry.Nid, "2Z+PpY6CaJg", StringComparison.Ordinal) && + leafStackPointer >= 0x00006FFFAC1FF000UL && + leafStackPointer < 0x00006FFFAC200000UL; + if (probeLeafReturn) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] leaf-return-probe-enter nid={importStubEntry.Nid} " + + $"ret=0x{returnRip:X16} rsp=0x{leafStackPointer:X16} " + + $"active_slot=0x{ActiveGuestReturnSlotAddress:X16}"); + } cpuContext.Rip = importStubEntry.Address; + LoadImportVolatileArguments(cpuContext, argPackPtr); cpuContext[CpuRegister.Rdi] = arg0; cpuContext[CpuRegister.Rsi] = *(ulong*)(argPackPtr + 8); cpuContext[CpuRegister.Rdx] = *(ulong*)(argPackPtr + 16); @@ -614,8 +1266,21 @@ public sealed partial class DirectExecutionBackend if (_activeGuestThreadState is { } activeGuestThreadState) { Interlocked.Increment(ref activeGuestThreadState.ImportCount); - Volatile.Write(ref activeGuestThreadState.LastImportNid, importStubEntry.Nid); + activeGuestThreadState.LastImportRdi = arg0; + activeGuestThreadState.LastImportRsi = *(ulong*)(argPackPtr + 8); + activeGuestThreadState.LastImportRdx = *(ulong*)(argPackPtr + 16); + activeGuestThreadState.LastImportRcx = *(ulong*)(argPackPtr + 24); + activeGuestThreadState.LastImportR8 = *(ulong*)(argPackPtr + 32); + activeGuestThreadState.LastImportR9 = *(ulong*)(argPackPtr + 40); + activeGuestThreadState.LastImportStack0 = ReadImportStackArgument(argPackPtr, 0); + activeGuestThreadState.LastImportStack1 = ReadImportStackArgument(argPackPtr, 1); + activeGuestThreadState.LastImportStack2 = ReadImportStackArgument(argPackPtr, 2); + activeGuestThreadState.LastImportStack3 = ReadImportStackArgument(argPackPtr, 3); + activeGuestThreadState.LastImportStack4 = ReadImportStackArgument(argPackPtr, 4); + activeGuestThreadState.LastImportStack5 = ReadImportStackArgument(argPackPtr, 5); + Volatile.Write(ref activeGuestThreadState.LastImportResultValid, 0); Volatile.Write(ref activeGuestThreadState.LastReturnRip, returnRip); + Volatile.Write(ref activeGuestThreadState.LastImportNid, importStubEntry.Nid); } if (dispatchIndex % 100000 == 0) { @@ -656,6 +1321,10 @@ public sealed partial class DirectExecutionBackend GuestThreadExecution.RestoreImportCallFrame(previousImportCallFrame); } } + DeliverPendingGuestExceptionAtSafePoint( + cpuContext, + CaptureImportBoundaryContinuation(cpuContext, argPackPtr, returnRip)); + StoreImportVectorReturn(cpuContext, argPackPtr); if (returnValue != (int)OrbisGen2Result.ORBIS_GEN2_OK) { @@ -671,13 +1340,15 @@ public sealed partial class DirectExecutionBackend } } - if (GuestThreadExecution.TryConsumeCurrentThreadBlock( + var consumedThreadBlock = GuestThreadExecution.TryConsumeCurrentThreadBlock( out var blockReason, out var blockContinuation, out var hasBlockContinuation, out var blockWakeKey, - out var blockWaiter, - out var blockDeadlineTimestamp) && + out var blockResumeHandler, + out var blockWakeHandler, + out var blockDeadlineTimestamp); + if (consumedThreadBlock && TryYieldGuestThreadToHostStub(argPackPtr, dispatchIndex, returnRip, importStubEntry.Nid, blockReason)) { if (hasBlockContinuation) @@ -686,24 +1357,31 @@ public sealed partial class DirectExecutionBackend GuestThreadExecution.CurrentGuestThreadHandle, blockContinuation, blockWakeKey, - blockWaiter, + blockResumeHandler, + blockWakeHandler, blockDeadlineTimestamp); } cpuContext[CpuRegister.Rax] = 0uL; } + if (probeLeafReturn) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] leaf-return-probe-exit nid={importStubEntry.Nid} " + + $"original=0x{returnRip:X16} final=0x{*(ulong*)(argPackPtr + 96):X16} " + + $"rsp=0x{leafStackPointer:X16} active_slot=0x{ActiveGuestReturnSlotAddress:X16} " + + $"block={consumedThreadBlock} yield={ActiveGuestThreadYieldRequested}"); + } result = cpuContext[CpuRegister.Rax]; + if (_activeGuestThreadState is { } completedGuestThreadState) + { + Volatile.Write(ref completedGuestThreadState.LastImportRax, result); + Volatile.Write(ref completedGuestThreadState.LastImportResultValid, 1); + } return true; } - // Subset of the leaf set that additionally skips the import-call-frame - // bookkeeping. The same scalar-only (no XMM args, no XMM return) constraint - // as IsLeafImport applies — see the note there before adding entries. - // NOTE: this filter is only consulted after IsLeafImport accepts the NID; - // entries listed here but not in IsLeafImport (scePadRead, scePadOpen, - // sceUserServiceGetEvent, sceUserServiceGetPlatformPrivacySetting, - // pthread_mutex_trylock) currently take the full gateway path. private static bool IsNoBlockLeafImport(string nid) => nid is "8aI7R7WaOlc" or // sceAmprCommandBufferConstructor @@ -719,19 +1397,21 @@ public sealed partial class DirectExecutionBackend "C+IEj+BsAFM" or // sceAmprMeasureCommandSizeWriteAddressOnCompletion "tZDDEo2tE5k" or // sceAmprCommandBufferGetSize "GnxKOHEawhk" or // sceAmprCommandBufferGetCurrentOffset + "gzndltBEzWc" or // sceAmprCommandBufferGetNumCommands "H896Pt-yB4I" or // sceAmprCommandBufferWriteKernelEventQueue_04_00 "sJXyWHjP-F8" or // sceAmprCommandBufferWriteAddressOnCompletion + "mPpPxv5CZt4" or // sceSystemServiceGetHdrToneMapLuminance + "1FZBKy8HeNU" or // sceVideoOutGetVblankStatus "ASoW5WE-UPo" or // sceKernelAprSubmitCommandBufferAndGetResult "rqwFKI4PAiM" or // sceKernelAprWaitCommandBuffer "eE4Szl8sil8" or // sceKernelAprSubmitCommandBuffer "qvMUCyyaCSI" or // sceKernelAprSubmitCommandBufferAndGetId "Q2V+iqvjgC0" or // vsnprintf - "AV6ipCNa4Rw" or // strcasecmp "q1cHNfGycLI" or // scePadRead "xk0AcarP3V4" or // scePadOpen "yH17Q6NWtVg" or // sceUserServiceGetEvent "D-CzAxQL0XI" or // sceUserServiceGetPlatformPrivacySetting - "K-jXhbt2gn4"; // pthread_mutex_trylock (scePthreadMutexTrylock is upoVrzMHFeE) + "K-jXhbt2gn4"; // scePthreadMutexTrylock private bool ShouldLogImportResult(string nid, OrbisGen2Result result) { @@ -750,12 +1430,12 @@ public sealed partial class DirectExecutionBackend var expectedEqueueTimeout = string.Equals(nid, "fzyMKs9kim0", StringComparison.Ordinal) && result == OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; - var expectedEventFlagTimeout = - string.Equals(nid, "JTvBflhYazQ", StringComparison.Ordinal) && - result == OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; var expectedMutexTrylockBusy = string.Equals(nid, "K-jXhbt2gn4", StringComparison.Ordinal) && result == OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; + var expectedNetAcceptWouldBlock = + string.Equals(nid, "PIWqhn9oSxc", StringComparison.Ordinal) && + resultValue == unchecked((int)0x80410123); var expectedUserServiceNoEvent = string.Equals(nid, "yH17Q6NWtVg", StringComparison.Ordinal) && resultValue == unchecked((int)0x80960007); @@ -765,8 +1445,8 @@ public sealed partial class DirectExecutionBackend if (!expectedFileProbeMiss && !expectedTimedWaitTimeout && !expectedEqueueTimeout && - !expectedEventFlagTimeout && !expectedMutexTrylockBusy && + !expectedNetAcceptWouldBlock && !expectedUserServiceNoEvent && !expectedPrivacyInvalidParameter) { @@ -802,45 +1482,25 @@ public sealed partial class DirectExecutionBackend "1G3lF1Gg1k8" or // sceKernelOpen "gEpBkcwxUjw"; // sceKernelAprResolveFilepathsToIdsAndFileSizes - // LEAF-IMPORT REGISTRATION — scalar-only constraint. - // - // TryDispatchLeafImport is a fast path that never copies the trampoline's XMM - // save area into CpuContext and never publishes an XMM0 return back to the - // guest (see DispatchImport). Every NID listed here must therefore satisfy BOTH: - // 1. no float/double parameters (nothing passed in xmm0..xmm7), and - // 2. no float/double return value (nothing returned in xmm0). - // Integer/pointer arguments and returns only. va_list-based functions - // (vsnprintf) are safe: SysV va_list floats are read from the caller-built - // reg_save_area in guest memory, not from the callee's incoming XMM registers. - // - // If a function that consumes XMM arguments or returns in xmm0 (e.g. powf, - // logf, wcstod, sceVideoOutColorSettingsSetGamma_) is ever added here, its - // float arguments will silently arrive stale and its return value will be - // dropped. Such functions must stay on the full gateway path — do not list them. - // - // Audited 2026-07-11 (PR #59): every NID below maps to a registered - // SysAbiExport whose handler neither reads nor writes CpuContext XMM state - // and whose known guest signature is integer/pointer only. private bool IsLeafImport(string nid) { - if (nid == "1jfXLRVzisc") // sceKernelUsleep + if (nid == "1jfXLRVzisc") { return !_logUsleep; } - // Mutex lock uses this block-capable leaf path. Keep it out of the no-block subset. + // These leaf operations cannot park the current guest thread. Unlocks may + // wake another cooperative thread, but the scheduler drain is independent + // of the caller's import-boundary bookkeeping. return nid is - "9UK1vLZQft4" or // scePthreadMutexLock - "7H0iTOciTLo" or // pthread_mutex_lock "tn3VlD0hG60" or // scePthreadMutexUnlock "2Z+PpY6CaJg" or // pthread_mutex_unlock - "EgmLo6EWgso" or // pthread_rwlock_unlock - "+L98PIbGttk" or // scePthreadRwlockUnlock - "q1cHNfGycLI" or // scePadRead + "EgmLo6EWgso" or // scePthreadRwlockUnlock + "+L98PIbGttk" or // pthread_rwlock_unlock "8aI7R7WaOlc" or // sceAmprCommandBufferConstructor "zgXifHT9ErY" or // sceVideoOutIsFlipPending "V++UgBtQhn0" or // sceAgcGetDataPacketPayloadAddress - "qj7QZpgr9Uw" or // sceAgcUnknownQj7QZpgr9Uw (unknown NID; observed scalar: command-buffer pointer in, packet address out) + "qj7QZpgr9Uw" or // Gen5 graphics type-2 packet "LtTouSCZjHM" or // sceAgcCbNop "k3GhuSNmBLU" or // sceAgcCbDispatch "UZbQjYAwwXM" or // sceAgcCbSetShRegistersDirect @@ -862,25 +1522,34 @@ public sealed partial class DirectExecutionBackend "H7uZqCoNuWk" or // sceAgcDcbPopMarker "IxYiarKlXxM" or // sceAgcDmaDataPatchSetDstAddressOrOffset "3KDcnM3lrcU" or // sceAgcWaitRegMemPatchAddress + "n485EBnIWmk" or // sceAgcWaitRegMemPatchCompareFunction + "7nOoijNPvEU" or // sceAgcWaitRegMemPatchReference + "hXAnLgDHCoI" or // sceAgcWaitRegMemPatchMask "0fWWK5uG9rQ" or // sceAgcQueueEndOfPipeActionPatchAddress - "a8uLzYY--tM" or // sceAmprAprCommandBufferConstructor - "Qs1xtplKo0U" or // sceAmprAprCommandBufferDestructor - "GuchCTefuZw" or // sceAmprCommandBufferDestructor - "N-FSPA4S3nI" or // sceAmprCommandBufferSetBuffer - "baQO9ez2gL4" or // sceAmprCommandBufferReset - "ULvXMDz56po" or // sceAmprCommandBufferClearBuffer - "mQ16-QdKv7k" or // sceAmprAprCommandBufferReadFile - "vWU-odnS+fU" or // sceAmprMeasureCommandSizeReadFile - "sSAUCCU1dv4" or // sceAmprMeasureCommandSizeWriteKernelEventQueue_04_00 - "C+IEj+BsAFM" or // sceAmprMeasureCommandSizeWriteAddressOnCompletion - "tZDDEo2tE5k" or // sceAmprCommandBufferGetSize - "GnxKOHEawhk" or // sceAmprCommandBufferGetCurrentOffset - "H896Pt-yB4I" or // sceAmprCommandBufferWriteKernelEventQueue_04_00 - "sJXyWHjP-F8" or // sceAmprCommandBufferWriteAddressOnCompletion - "ASoW5WE-UPo" or // sceKernelAprSubmitCommandBufferAndGetResult - "rqwFKI4PAiM" or // sceKernelAprWaitCommandBuffer - "eE4Szl8sil8" or // sceKernelAprSubmitCommandBuffer - "qvMUCyyaCSI" or // sceKernelAprSubmitCommandBufferAndGetId + "J8YCgfKAMQs" or // sceAgcQueueEndOfPipeActionPatchGcrCntl + "MlEw1feXcjg" or // sceAgcQueueEndOfPipeActionPatchData + "T9fjQIINoeE" or // sceAgcQueueEndOfPipeActionPatchType + "a8uLzYY--tM" or + "Qs1xtplKo0U" or + "GuchCTefuZw" or + "N-FSPA4S3nI" or + "baQO9ez2gL4" or + "ULvXMDz56po" or + "mQ16-QdKv7k" or + "vWU-odnS+fU" or + "sSAUCCU1dv4" or + "C+IEj+BsAFM" or + "tZDDEo2tE5k" or + "GnxKOHEawhk" or + "gzndltBEzWc" or + "H896Pt-yB4I" or + "sJXyWHjP-F8" or + "mPpPxv5CZt4" or + "1FZBKy8HeNU" or + "ASoW5WE-UPo" or + "rqwFKI4PAiM" or + "eE4Szl8sil8" or + "qvMUCyyaCSI" or "Vo5V8KAwCmk" or // sceSystemServiceHideSplashScreen "TywrFKCoLGY" or // sceSaveDataInitialize3 "dyIhnXq-0SM" or // sceSaveDataDirNameSearch @@ -895,11 +1564,10 @@ public sealed partial class DirectExecutionBackend "Q2V+iqvjgC0" or // vsnprintf "j4ViWNHEgww" or // strlen "5jNubw4vlAA" or // strnlen - "LHMrG7e8G78" or // wcsmisc + "LHMrG7e8G78" or // wcslen "WkkeywLJcgU" or // wcslen "Ovb2dSJOAuE" or // strcmp "aesyjrHVWy4" or // strncmp - "AV6ipCNa4Rw" or // strcasecmp "pNtJdE3x49E" or // wcscmp "fV2xHER+bKE" or // wcscoll "E8wCoUEbfzk" or // wcsncmp @@ -920,12 +1588,7 @@ public sealed partial class DirectExecutionBackend "vz+pg2zdopI" or // sceKernelGetEventUserData "mJ7aghmgvfc" or // sceKernelGetEventId "23CPPI1tyBY" or // sceKernelGetEventFilter - "kwGyyjohI50" or // sceKernelGetEventData - // _Getpctype is a per-character ctype table lookup; the embedded mcpp preprocessor - // re-fetches the table pointer on every isdigit()/isalpha() check, so classifying a - // large input legitimately calls it hundreds of thousands of times back-to-back - - // real, finite work that would otherwise trip the loop guard. - "sUP1hBaouOw"; // _Getpctype + "kwGyyjohI50"; // sceKernelGetEventData } private long NextImportDispatchIndex() @@ -989,31 +1652,6 @@ public sealed partial class DirectExecutionBackend return true; } - private unsafe bool TryAbortGuestForHostShutdown(nint argPackPtr, long dispatchIndex, ulong returnRip) - { - ulong hostExit = ActiveEntryReturnSentinelRip; - if (hostExit < 65536 || !TryPatchActiveGuestReturnSlot(hostExit)) - { - return false; - } - try - { - *(ulong*)(argPackPtr + 96) = hostExit; - } - catch - { - return false; - } - ActiveForcedGuestExit = true; - if (string.IsNullOrWhiteSpace(LastError)) - { - LastError = "Host shutdown requested."; - } - Console.Error.WriteLine( - $"[LOADER][INFO] Guest unwind for host shutdown at import#{dispatchIndex} ret=0x{returnRip:X16} -> host_exit=0x{hostExit:X16}"); - return true; - } - private unsafe bool TryCompleteGuestEntryToHostStub(nint argPackPtr, long dispatchIndex, ulong returnRip, string nid, string reason, ulong value) { ulong hostExit = ActiveEntryReturnSentinelRip; @@ -1068,7 +1706,7 @@ public sealed partial class DirectExecutionBackend ActiveCpuContext.TryWriteUInt64(returnSlotAddress, hostExit); } - private bool ShouldForceGuestExitOnImportLoop(string nid, ulong returnRip, long dispatchIndex, ulong arg0, ulong arg1) + private bool ShouldForceGuestExitOnImportLoop(in ImportStubEntry entry, ulong returnRip, long dispatchIndex, ulong arg0, ulong arg1) { if (dispatchIndex < 1200) { @@ -1078,18 +1716,18 @@ public sealed partial class DirectExecutionBackend { return false; } - if (IsImportLoopGuardBoundary(nid)) + if (entry.IsLoopGuardBoundary) { ResetImportLoopPattern(); return false; } - if (!_importNidHashCache.TryGetValue(nid, out var value)) - { - value = StableHash64(nid); - _importNidHashCache[nid] = value; - } + var value = entry.NidHash; RecordImportLoopSignature(value, returnRip, BuildImportLoopSignature(value, returnRip, arg0, arg1)); - if ((dispatchIndex & 0x3F) != 0) + // The O(period x repeats) pattern scan is a boot/hang watchdog, not a + // steady-state feature; sampling every 256th dispatch keeps its cost + // off the hot path while still tripping within a couple of thousand + // dispatches of a genuine import loop. + if ((dispatchIndex & 0xFF) != 0) { return false; } @@ -1120,15 +1758,12 @@ public sealed partial class DirectExecutionBackend } private static bool IsImportLoopGuardBoundary(string nid) => - nid switch - { - "1jfXLRVzisc" => true, // sceKernelUsleep - "QcteRwbsnV0" => true, // usleep - "n88vx3C5nW8" => true, // gettimeofday - "Zxa0VhQVTsk" => true, // sceKernelWaitSema - "T72hz6ffq08" => true, // scePthreadYield - _ => false - }; + nid is + "1jfXLRVzisc" or // sceKernelUsleep + "WKAXJ4XBPQ4" or // scePthreadCondWait + "BmMjYxmew1w" or // scePthreadCondTimedwait + "Op8TBGY5KHg" or // pthread_cond_wait + "27bAgiJmOh0"; // pthread_cond_timedwait private void ResetImportLoopPattern() { @@ -1368,483 +2003,66 @@ public sealed partial class DirectExecutionBackend { return OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - - NormalizeKernelDynlibDlsymArguments(cpuContext, out var symbolNameAddress, out var outputAddress); - try + ulong symbolNameAddress = cpuContext[CpuRegister.Rsi]; + ulong outputAddress = cpuContext[CpuRegister.Rdx]; + if (!TryReadAsciiZ(symbolNameAddress, 512, out var symbolName)) { - if (!TryReadAsciiZ(symbolNameAddress, 512, out var symbolName) || - !TryResolveDlsymGuestAddress(symbolName, out var resolvedAddress) || - outputAddress == 0 || - !TryWriteUInt64Compat(outputAddress, resolvedAddress)) - { - return CompleteKernelDynlibDlsymFailure(cpuContext, outputAddress); - } + cpuContext[CpuRegister.Rax] = 18446744073709551615uL; + return OrbisGen2Result.ORBIS_GEN2_OK; } - catch + var moduleHandle = unchecked((int)cpuContext[CpuRegister.Rdi]); + if (!TryResolveModuleSymbolAddress(moduleHandle, symbolName, out var resolvedAddress) && + !TryResolveRuntimeSymbolAddress(symbolName, out resolvedAddress) && + !TryResolveRuntimeSymbolAddress(ComputePsNid(symbolName), out resolvedAddress) && + !TryResolveRuntimeSymbolAlias(symbolName, out resolvedAddress)) { - return CompleteKernelDynlibDlsymFailure(cpuContext, outputAddress); + Console.Error.WriteLine( + $"[LOADER][WARN] sceKernelDlsym failed: handle=0x{cpuContext[CpuRegister.Rdi]:X} symbol='{symbolName}'"); + cpuContext[CpuRegister.Rax] = 18446744073709551615uL; + return OrbisGen2Result.ORBIS_GEN2_OK; + } + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_DLSYM"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] sceKernelDlsym: handle=0x{moduleHandle:X} symbol='{symbolName}' -> 0x{resolvedAddress:X16}"); + } + if (outputAddress == 0L || !TryWriteUInt64Compat(outputAddress, resolvedAddress)) + { + cpuContext[CpuRegister.Rax] = 18446744073709551615uL; + return OrbisGen2Result.ORBIS_GEN2_OK; } - cpuContext[CpuRegister.Rax] = 0uL; return OrbisGen2Result.ORBIS_GEN2_OK; } - private static void NormalizeKernelDynlibDlsymArguments( - CpuContext cpuContext, - out ulong symbolNameAddress, - out ulong outputAddress) + private static bool TryResolveModuleSymbolAddress(int moduleHandle, string symbolName, out ulong address) { - var handle = cpuContext[CpuRegister.Rdi]; - symbolNameAddress = cpuContext[CpuRegister.Rsi]; - outputAddress = cpuContext[CpuRegister.Rdx]; - - // Standalone bootstrap loaders sometimes call through the bridge with - // (symbol_ptr, handle, out) while sceKernelDlsym is (handle, symbol_ptr, out). - // Heuristic only: valid when RSI looks like a small handle and RDI is a guest pointer. - if (symbolNameAddress < 0x10000 && - IsPlausibleDynlibSymbolPointer(handle)) - { - symbolNameAddress = handle; - handle = cpuContext[CpuRegister.Rsi]; - cpuContext[CpuRegister.Rdi] = handle; - cpuContext[CpuRegister.Rsi] = symbolNameAddress; - } - } - - private static bool IsPlausibleDynlibSymbolPointer(ulong address) - { - return address >= 0x10000 && address < 0x0000_8000_0000_0000UL; - } - - private OrbisGen2Result CompleteKernelDynlibDlsymFailure(CpuContext cpuContext, ulong outputAddress) - { - if (outputAddress != 0) - { - _ = TryWriteUInt64Compat(outputAddress, 0); - } - - cpuContext[CpuRegister.Rax] = ulong.MaxValue; - return OrbisGen2Result.ORBIS_GEN2_OK; - } - - private void ResetLazyDlsymStubState() - { - lock (_lazyDlsymStubGate) - { - _lazyDlsymStubCache.Clear(); - _lazyImportStubPoolMapped = false; - _lazyImportStubPoolBase = 0; - _lazyImportStubNextSlot = 0; - _lazyImportStubPoolLimit = 0; - } - } - - private bool TryResolveDlsymGuestAddress(string symbolName, out ulong guestAddress) - { - guestAddress = 0; - if (string.IsNullOrWhiteSpace(symbolName)) - { - return false; - } - - // Tier 0: ELF runtime symbols and aliases. - if (TryResolveRuntimeSymbolAddress(symbolName, out guestAddress) || - TryResolveRuntimeSymbolAlias(symbolName, out guestAddress)) + if (KernelModuleRegistry.TryResolveModuleSymbol(moduleHandle, symbolName, out address)) { return true; } - // Tier 1: existing import stub entries (duplicate prevention). - if (TryFindImportStubGuestAddress(symbolName, out guestAddress)) - { - return true; - } - - var hasAerolibSymbol = Aerolib.Instance.TryGetByExportName(symbolName, out var hleSymbol); - if (hasAerolibSymbol) - { - if (HleDataSymbols.TryGetAddress(hleSymbol.Nid, out _)) - { - return false; - } - - if (TryResolveRuntimeSymbolAddress(hleSymbol.Nid, out guestAddress) || - TryResolveRuntimeSymbolAddress(hleSymbol.ExportName, out guestAddress) || - TryFindImportStubGuestAddress(hleSymbol.Nid, out guestAddress) || - TryFindImportStubGuestAddress(hleSymbol.ExportName, out guestAddress)) - { - return true; - } - } - else if (Aerolib.Instance.TryGetByNid(symbolName, out hleSymbol)) - { - if (HleDataSymbols.TryGetAddress(hleSymbol.Nid, out _)) - { - return false; - } - - if (TryFindImportStubGuestAddress(hleSymbol.Nid, out guestAddress) || - TryFindImportStubGuestAddress(hleSymbol.ExportName, out guestAddress)) - { - return true; - } - - hasAerolibSymbol = true; - } - - // Tier 3: lazy materialization for callable HLE symbols missing from ELF imports. - if (!TryResolveLazyDispatchTarget(symbolName, hasAerolibSymbol, in hleSymbol, out var dispatchNid, out var export)) - { - return false; - } - - return TryGetOrCreateLazyImportStub( - dispatchNid, - symbolName, - hasAerolibSymbol ? hleSymbol : null, - export, - out guestAddress); + var nid = ComputePsNid(symbolName); + return KernelModuleRegistry.TryResolveModuleSymbol(moduleHandle, nid, out address); } - private bool TryFindImportStubGuestAddress(string identifier, out ulong guestAddress) + private static string ComputePsNid(string symbolName) { - guestAddress = 0; - if (string.IsNullOrWhiteSpace(identifier)) - { - return false; - } - - var importEntries = _importEntries; - for (var i = 0; i < importEntries.Length; i++) - { - var entry = importEntries[i]; - if (!ImportStubEntryMatchesIdentifier(entry, identifier)) - { - continue; - } - - if (entry.Address >= 0x10000) - { - guestAddress = entry.Address; - return true; - } - } - - return false; - } - - private static bool ImportStubEntryMatchesIdentifier(in ImportStubEntry entry, string identifier) - { - if (string.Equals(entry.Nid, identifier, StringComparison.Ordinal)) - { - return true; - } - - if (entry.Export is not { } export) - { - return false; - } - - return string.Equals(export.Name, identifier, StringComparison.Ordinal) || - string.Equals(export.Nid, identifier, StringComparison.Ordinal); - } - - private static bool IsKernelDynlibDlsymIdentifier(string identifier) - { - return string.Equals(identifier, "sceKernelDlsym", StringComparison.Ordinal) || - string.Equals(identifier, KernelDynlibDlsymAerolibNid, StringComparison.Ordinal) || - string.Equals(identifier, RuntimeStubNids.KernelDynlibDlsym, StringComparison.Ordinal); - } - - private bool TryResolveLazyDispatchTarget( - string symbolName, - bool hasAerolibSymbol, - in SysAbiSymbol hleSymbol, - out string dispatchNid, - out ExportedFunction? export) - { - export = null; - dispatchNid = string.Empty; - - if (IsKernelDynlibDlsymIdentifier(symbolName)) - { - dispatchNid = KernelDynlibDlsymAerolibNid; - _ = _moduleManager.TryGetExport(dispatchNid, out export); - return true; - } - - if (!hasAerolibSymbol) - { - return false; - } - - if (HleDataSymbols.TryGetAddress(hleSymbol.Nid, out _)) - { - return false; - } - - if (_moduleManager.TryGetExport(hleSymbol.Nid, out export)) - { - dispatchNid = hleSymbol.Nid; - return true; - } - - return false; - } - - private bool TryGetOrCreateLazyImportStub( - string dispatchNid, - string symbolName, - SysAbiSymbol? hleSymbol, - ExportedFunction? export, - out ulong guestAddress) - { - guestAddress = 0; - if (string.IsNullOrWhiteSpace(dispatchNid)) - { - return false; - } - - lock (_lazyDlsymStubGate) - { - if (TryGetCachedLazyDlsymAddress(dispatchNid, symbolName, out guestAddress)) - { - return true; - } - - if (!EnsureLazyImportStubPoolMapped()) - { - LogLazyImportStubMaterializationFailure( - "import stub region unresolved", - dispatchNid, - symbolName); - return false; - } - - if (!TryAllocateLazyImportStubSlot(out guestAddress)) - { - LogLazyImportStubMaterializationFailure( - "lazy import stub pool exhausted", - dispatchNid, - symbolName); - return false; - } - - var previousEntries = _importEntries; - var importIndex = previousEntries.Length; - var newEntries = new ImportStubEntry[importIndex + 1]; - if (importIndex > 0) - { - Array.Copy(previousEntries, newEntries, importIndex); - } - - newEntries[importIndex] = new ImportStubEntry(guestAddress, dispatchNid, export); - - var hostTrampoline = CreateImportHandlerTrampoline(importIndex); - if (hostTrampoline == 0 || - !PatchImportStub((nint)(long)guestAddress, hostTrampoline)) - { - guestAddress = 0; - LogLazyImportStubMaterializationFailure( - "failed to patch lazy import stub trampoline", - dispatchNid, - symbolName); - return false; - } - - _importEntries = newEntries; - - RegisterDlsymRuntimeSymbolKeys(symbolName, dispatchNid, hleSymbol, guestAddress); - return true; - } - } - - private bool TryGetCachedLazyDlsymAddress(string dispatchNid, string symbolName, out ulong guestAddress) - { - if (_lazyDlsymStubCache.TryGetValue(dispatchNid, out guestAddress) || - _lazyDlsymStubCache.TryGetValue(symbolName, out guestAddress)) - { - return guestAddress >= 0x10000; - } - - return false; - } - - private void RegisterDlsymRuntimeSymbolKeys( - string symbolName, - string dispatchNid, - SysAbiSymbol? hleSymbol, - ulong guestAddress) - { - RegisterDlsymRuntimeSymbolKey(symbolName, guestAddress); - RegisterDlsymRuntimeSymbolKey(dispatchNid, guestAddress); - - if (IsKernelDynlibDlsymIdentifier(symbolName) || IsKernelDynlibDlsymIdentifier(dispatchNid)) - { - RegisterDlsymRuntimeSymbolKey(RuntimeStubNids.KernelDynlibDlsym, guestAddress); - RegisterDlsymRuntimeSymbolKey(KernelDynlibDlsymAerolibNid, guestAddress); - RegisterDlsymRuntimeSymbolKey("sceKernelDlsym", guestAddress); - } - - if (hleSymbol.HasValue) - { - RegisterDlsymRuntimeSymbolKey(hleSymbol.Value.Nid, guestAddress); - RegisterDlsymRuntimeSymbolKey(hleSymbol.Value.ExportName, guestAddress); - } - } - - private void RegisterDlsymRuntimeSymbolKey(string key, ulong guestAddress) - { - if (string.IsNullOrWhiteSpace(key) || !IsRuntimeSymbolAddressUsable(guestAddress)) - { - return; - } - - _runtimeSymbolsByName[key] = guestAddress; - _lazyDlsymStubCache[key] = guestAddress; - } - - private void LogLazyImportStubMaterializationFailure(string reason, string dispatchNid, string symbolName) - { - Console.Error.WriteLine( - $"[LOADER][WARN] Lazy import stub materialization failed ({reason}): " + - $"nid={dispatchNid} symbol='{symbolName}'"); - } - - private unsafe bool TryResolveImportStubRegionBounds( - ImportStubEntry[] importEntries, - out ulong regionBase, - out ulong regionLimit) - { - regionBase = 0; - regionLimit = 0; - - for (var candidateIndex = 0; candidateIndex < 64; candidateIndex++) - { - var candidateBase = ImportStubRegionCanonicalBase - - (ulong)candidateIndex * ImportStubRegionAddressStride; - if (!_hostMemory.Query(candidateBase, out var memoryInfo) || - memoryInfo.RegionSize == 0 || - memoryInfo.State != HostRegionState.Committed) - { - continue; - } - - var candidateLimit = candidateBase + memoryInfo.RegionSize; - var hasStub = false; - for (var i = 0; i < importEntries.Length; i++) - { - var entryAddress = importEntries[i].Address; - if (entryAddress < candidateBase || entryAddress >= candidateLimit) - { - continue; - } - - if ((entryAddress - candidateBase) % LazyImportStubSlotSize != 0) - { - continue; - } - - hasStub = true; - break; - } - - if (!hasStub) - { - continue; - } - - regionBase = candidateBase; - regionLimit = candidateLimit; - return true; - } - - ulong maxStubEnd = 0; - for (var i = 0; i < importEntries.Length; i++) - { - var entryAddress = importEntries[i].Address; - if (entryAddress < ImportStubRegionCanonicalBase) - { - continue; - } - - if ((entryAddress - ImportStubRegionCanonicalBase) % LazyImportStubSlotSize != 0) - { - continue; - } - - var entryEnd = entryAddress + LazyImportStubSlotSize; - if (entryEnd > maxStubEnd) - { - maxStubEnd = entryEnd; - regionBase = ImportStubRegionCanonicalBase; - } - } - - if (regionBase == 0 || maxStubEnd <= regionBase) - { - return false; - } - - var spanBytes = maxStubEnd - regionBase; - regionLimit = regionBase + AlignUp(spanBytes, ImportStubRegionPageSize); - return regionLimit > regionBase; - } - - private bool EnsureLazyImportStubPoolMapped() - { - if (_lazyImportStubPoolMapped && _lazyImportStubPoolBase != 0) - { - return true; - } - - var importEntries = _importEntries; - if (!TryResolveImportStubRegionBounds(importEntries, out var importStubRegionBase, out var importStubRegionLimit)) - { - return false; - } - - ulong nextSlot = importStubRegionBase; - for (var i = 0; i < importEntries.Length; i++) - { - var entryAddress = importEntries[i].Address; - if (entryAddress < importStubRegionBase || entryAddress >= importStubRegionLimit) - { - continue; - } - - var entryEnd = entryAddress + LazyImportStubSlotSize; - if (entryEnd > nextSlot) - { - nextSlot = entryEnd; - } - } - - if (nextSlot >= importStubRegionLimit) - { - return false; - } - - _lazyImportStubPoolBase = importStubRegionBase; - _lazyImportStubNextSlot = nextSlot; - _lazyImportStubPoolLimit = importStubRegionLimit; - _lazyImportStubPoolMapped = true; - return true; - } - - private bool TryAllocateLazyImportStubSlot(out ulong guestAddress) - { - guestAddress = 0; - if (!_lazyImportStubPoolMapped || - _lazyImportStubNextSlot < _lazyImportStubPoolBase || - _lazyImportStubNextSlot + LazyImportStubSlotSize > _lazyImportStubPoolLimit) - { - return false; - } - - guestAddress = _lazyImportStubNextSlot; - _lazyImportStubNextSlot += LazyImportStubSlotSize; - return true; + ReadOnlySpan salt = + [ + 0x51, 0x8D, 0x64, 0xA6, 0x35, 0xDE, 0xD8, 0xC1, + 0xE6, 0xB0, 0x39, 0xB1, 0xC3, 0xE5, 0x52, 0x30, + ]; + var nameBytes = Encoding.UTF8.GetBytes(symbolName); + var input = new byte[nameBytes.Length + salt.Length]; + nameBytes.CopyTo(input, 0); + salt.CopyTo(input.AsSpan(nameBytes.Length)); + Span digest = stackalloc byte[20]; + SHA1.HashData(input, digest); + var value = BinaryPrimitives.ReadUInt64LittleEndian(digest); + Span bigEndianValue = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64BigEndian(bigEndianValue, value); + return Convert.ToBase64String(bigEndianValue).TrimEnd('=').Replace('/', '-'); } private bool TryResolveRuntimeSymbolAlias(string symbolName, out ulong address) @@ -1911,20 +2129,29 @@ public sealed partial class DirectExecutionBackend return OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + ulong bridgeHandle = cpuContext[CpuRegister.Rdi]; + ulong symbolNameAddress = cpuContext[CpuRegister.Rsi]; ulong outputAddress = cpuContext[CpuRegister.Rdx]; - try + _ = TryReadAsciiZ(symbolNameAddress, 512, out var symbolName); + + OrbisGen2Result result = DispatchKernelDynlibDlsym(); + if (result != OrbisGen2Result.ORBIS_GEN2_OK) { - OrbisGen2Result result = DispatchKernelDynlibDlsym(); - if (result != OrbisGen2Result.ORBIS_GEN2_OK) - { - return result; - } + return result; } - catch + if (_logBootstrap) { - return CompleteKernelDynlibDlsymFailure(cpuContext, outputAddress); + Console.Error.WriteLine( + $"[LOADER][TRACE] bootstrap_dispatch: handle=0x{bridgeHandle:X16} symbol='{symbolName}' out=0x{outputAddress:X16} rax=0x{cpuContext[CpuRegister.Rax]:X16}"); } + if (cpuContext[CpuRegister.Rax] == 0uL) + { + return OrbisGen2Result.ORBIS_GEN2_OK; + } + + Console.Error.WriteLine( + $"[LOADER][WARN] bootstrap_bridge unresolved: handle=0x{bridgeHandle:X} symbol='{symbolName}' out=0x{outputAddress:X16}"); return OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1966,15 +2193,14 @@ public sealed partial class DirectExecutionBackend return false; } - // Reads stay byte-by-byte through TryReadByteCompat (its Marshal.ReadByte - // fallback must probe exactly up to the terminator), but the bytes land in a - // stack buffer instead of a List + ToArray per symbol resolution. const int StackBufferLength = 512; - byte[]? rented = maxLength > StackBufferLength ? System.Buffers.ArrayPool.Shared.Rent(maxLength) : null; + byte[]? rented = maxLength > StackBufferLength + ? System.Buffers.ArrayPool.Shared.Rent(maxLength) + : null; Span buffer = rented is null ? stackalloc byte[StackBufferLength] : rented; try { - for (int i = 0; i < maxLength; i++) + for (var i = 0; i < maxLength; i++) { if (!TryReadByteCompat(address + (ulong)i, buffer.Slice(i, 1))) { @@ -2077,7 +2303,7 @@ public sealed partial class DirectExecutionBackend uint flNewProtect = default(uint); try { - if (Marshal.ReadByte(num2) != 232 || !_hostMemory.Protect((ulong)(void*)num, 5u, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (Marshal.ReadByte(num2) != 232 || !VirtualProtect((void*)num, 5u, 64u, &flNewProtect)) { return; } @@ -2085,7 +2311,7 @@ public sealed partial class DirectExecutionBackend { Marshal.WriteByte(num2 + i, 144); } - _hostMemory.FlushInstructionCache((ulong)(void*)num, 5u); + FlushInstructionCache(GetCurrentProcess(), (void*)num, 5u); _patchedEa020eLookupCall = true; Console.Error.WriteLine($"[LOADER][WARNING] Import#{dispatchIndex}: patched hash-lookup call at 0x{num:X16} -> NOP*5"); } @@ -2096,7 +2322,7 @@ public sealed partial class DirectExecutionBackend { if (flNewProtect != 0) { - _hostMemory.ProtectRaw((ulong)(void*)num, 5u, flNewProtect, out flNewProtect); + VirtualProtect((void*)num, 5u, flNewProtect, &flNewProtect); } } } diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.NativeWorker.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.NativeWorker.cs index 43e8414..38a468d 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.NativeWorker.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.NativeWorker.cs @@ -6,8 +6,6 @@ using System.Collections.Generic; using System.Runtime.InteropServices; using System.Threading; using SharpEmu.HLE; -using SharpEmu.HLE.Host; -using SharpEmu.HLE.Host.Posix; namespace SharpEmu.Core.Cpu.Native; @@ -37,6 +35,20 @@ public sealed partial class DirectExecutionBackend private bool _nativeWorkersDisposed; private int _nativeWorkerCreationFailedLogged; + private const uint StackSizeParamIsAReservation = 0x00010000u; + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern nint CreateThread( + nint lpThreadAttributes, + nuint dwStackSize, + nint lpStartAddress, + nint lpParameter, + uint dwCreationFlags, + out uint lpThreadId); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern uint WaitForSingleObject(nint hHandle, uint dwMilliseconds); + // Runs an emitted guest entry stub. Preferred path is a pooled native worker // thread; falls back to the historical inline calli (guest frames above this // thread's managed frames) when workers are disabled or unavailable. @@ -49,7 +61,7 @@ public sealed partial class DirectExecutionBackend var worker = RentNativeGuestExecutor(); if (worker is null) { - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, (nint)hostRspSlot); + TlsSetValue(_hostRspSlotTlsIndex, (nint)hostRspSlot); return CallNativeEntry(entryStub); } try @@ -80,7 +92,10 @@ public sealed partial class DirectExecutionBackend private NativeGuestExecutor? RentNativeGuestExecutor() { - if (NativeGuestWorkersDisabled) + // NativeGuestExecutor emits a Win32 wait loop and creates it with + // kernel32!CreateThread. POSIX hosts use the established inline entry + // path until the worker loop has a pthread/eventfd implementation. + if (!OperatingSystem.IsWindows() || NativeGuestWorkersDisabled) { return null; } @@ -234,7 +249,7 @@ public sealed partial class DirectExecutionBackend public static NativeGuestExecutor? TryCreate(DirectExecutionBackend backend) { - if (!EnsureHostRuntimeExports(backend._hostSymbols)) + if (!EnsureKernel32Exports()) { return null; } @@ -247,27 +262,32 @@ public sealed partial class DirectExecutionBackend return executor; } - private static bool EnsureHostRuntimeExports(IHostSymbolResolver symbols) + private static bool EnsureKernel32Exports() { if (_exitThreadAddress != 0) { return _waitForSingleObjectAddress != 0 && _setEventAddress != 0; } - _waitForSingleObjectAddress = symbols.GetAddress(HostRuntimeFunction.WaitForSingleObject); - _setEventAddress = symbols.GetAddress(HostRuntimeFunction.SetEvent); - _exitThreadAddress = symbols.GetAddress(HostRuntimeFunction.ExitThread); + nint kernel32 = GetModuleHandle("kernel32.dll"); + if (kernel32 == 0) + { + return false; + } + _waitForSingleObjectAddress = GetProcAddress(kernel32, "WaitForSingleObject"); + _setEventAddress = GetProcAddress(kernel32, "SetEvent"); + _exitThreadAddress = GetProcAddress(kernel32, "ExitThread"); return _waitForSingleObjectAddress != 0 && _setEventAddress != 0 && _exitThreadAddress != 0; } private bool Initialize() { _selfHandle = GCHandle.Alloc(this); - _controlBlock = (void*)_backend._hostMemory.Allocate(0, 4096u, HostPageProtection.ReadWrite); + _controlBlock = VirtualAlloc(null, 4096u, 12288u, 4u); if (_controlBlock == null) { return false; } - _loopStub = (void*)_backend._hostMemory.Allocate(0, LoopStubSize, HostPageProtection.ReadWriteExecute); + _loopStub = VirtualAlloc(null, LoopStubSize, 12288u, 64u); if (_loopStub == null) { return false; @@ -375,15 +395,17 @@ public sealed partial class DirectExecutionBackend *(int*)(code + skipJump) = skipEntryOffset - (skipJump + sizeof(int)); uint oldProtect = 0; - if (!_backend._hostMemory.Protect((ulong)_loopStub, LoopStubSize, HostPageProtection.ReadExecute, out oldProtect)) + if (!VirtualProtect(_loopStub, LoopStubSize, 32u, &oldProtect)) { return false; } - _backend._hostMemory.FlushInstructionCache((ulong)_loopStub, LoopStubSize); - _threadHandle = _backend._hostThreading.CreateNativeThread( - (nint)_loopStub, + FlushInstructionCache(GetCurrentProcess(), _loopStub, LoopStubSize); + _threadHandle = CreateThread( 0, WorkerStackReservation, + (nint)_loopStub, + 0, + StackSizeParamIsAReservation, out _nativeThreadId); if (_threadHandle == 0) { @@ -511,7 +533,7 @@ public sealed partial class DirectExecutionBackend _prevYieldRequested = _activeGuestThreadYieldRequested; _prevYieldReason = _activeGuestThreadYieldReason; _prevState = _activeGuestThreadState; - _prevHostRspSlot = backend._hostThreading.GetTlsValue(backend._hostRspSlotTlsIndex); + _prevHostRspSlot = TlsGetValue(backend._hostRspSlotTlsIndex); _prevGuestThreadHandle = GuestThreadExecution.EnterGuestThread(_runGuestThreadHandle); _entered = true; _activeExecutionBackend = backend; @@ -523,11 +545,11 @@ public sealed partial class DirectExecutionBackend _activeGuestThreadYieldReason = null; _activeGuestThreadState = _runState; backend.BindTlsBase(_runContext!); - backend._hostThreading.SetTlsValue(backend._hostRspSlotTlsIndex, _runHostRspSlot); + TlsSetValue(backend._hostRspSlotTlsIndex, _runHostRspSlot); if (_runState is { } state) { _prevHostThreadId = Volatile.Read(ref state.HostThreadId); - Volatile.Write(ref state.HostThreadId, unchecked((int)backend._hostThreading.CurrentThreadId)); + Volatile.Write(ref state.HostThreadId, unchecked((int)GetCurrentThreadId())); } if (_runAffinityMask != 0) { @@ -557,7 +579,7 @@ public sealed partial class DirectExecutionBackend { Volatile.Write(ref state.HostThreadId, _prevHostThreadId); } - _backend._hostThreading.SetTlsValue(_backend._hostRspSlotTlsIndex, _prevHostRspSlot); + TlsSetValue(_backend._hostRspSlotTlsIndex, _prevHostRspSlot); GuestThreadExecution.RestoreGuestThread(_prevGuestThreadHandle); _activeExecutionBackend = _prevBackend; _activeCpuContext = _prevContext; @@ -594,8 +616,8 @@ public sealed partial class DirectExecutionBackend var exited = _threadHandle == 0; if (_threadHandle != 0) { - exited = _backend._hostThreading.WaitForThreadExit(_threadHandle, 1000u); - _backend._hostThreading.CloseThreadHandle(_threadHandle); + exited = WaitForSingleObject(_threadHandle, 1000u) == 0u; + CloseHandle(_threadHandle); _threadHandle = 0; } if (!exited) @@ -609,12 +631,12 @@ public sealed partial class DirectExecutionBackend } if (_loopStub != null) { - _backend._hostMemory.Free((ulong)_loopStub); + VirtualFree(_loopStub, 0u, 32768u); _loopStub = null; } if (_controlBlock != null) { - _backend._hostMemory.Free((ulong)_controlBlock); + VirtualFree(_controlBlock, 0u, 32768u); _controlBlock = null; } if (_selfHandle.IsAllocated) diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.PosixSignals.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.PosixSignals.cs index d9dbf82..a91b3f1 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.PosixSignals.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.PosixSignals.cs @@ -4,7 +4,7 @@ using System; using System.Runtime.InteropServices; using System.Threading; -using SharpEmu.Core.Cpu.Native.Windows; +using SharpEmu.HLE; namespace SharpEmu.Core.Cpu.Native; @@ -21,6 +21,8 @@ public sealed unsafe partial class DirectExecutionBackend // runtime keeps turning its own faults into managed exceptions. private const int PosixSigIll = 4; + private const int PosixSigTrap = 5; + private const int PosixSigAbort = 6; private const int PosixSigSegv = 11; private static readonly int PosixSigBus = OperatingSystem.IsMacOS() ? 10 : 7; @@ -62,6 +64,9 @@ public sealed unsafe partial class DirectExecutionBackend private static bool _posixSignalWarmup; private static readonly nint[] _posixPreviousActions = new nint[32]; private static int _posixSignalTraceCount; + private static long _perfSignalCount; + private static readonly bool _perfSignalCounter = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_PERF_MEM"), "1", StringComparison.Ordinal); [ThreadStatic] private static int _posixSignalHandlerDepth; @@ -88,10 +93,13 @@ public sealed unsafe partial class DirectExecutionBackend } WarmUpPosixSignalPath(); + SharpEmu.HLE.GuestImageWriteTracker.WarmUp(); if (!InstallPosixSignalHandler(PosixSigSegv) || !InstallPosixSignalHandler(PosixSigBus) || - !InstallPosixSignalHandler(PosixSigIll)) + !InstallPosixSignalHandler(PosixSigIll) || + !InstallPosixSignalHandler(PosixSigTrap) || + !InstallPosixSignalHandler(PosixSigAbort)) { throw new InvalidOperationException("Failed to install POSIX fault signal handlers"); } @@ -132,8 +140,8 @@ public sealed unsafe partial class DirectExecutionBackend // action and sigaction(0, ...) fails with EINVAL). EXCEPTION_RECORD record = default; record.ExceptionCode = DBG_PRINTEXCEPTION_C; - byte* contextRecord = stackalloc byte[Win64ContextOffsets.Size]; - new Span(contextRecord, Win64ContextOffsets.Size).Clear(); + byte* contextRecord = stackalloc byte[Win64ContextSize]; + new Span(contextRecord, Win64ContextSize).Clear(); EXCEPTION_POINTERS pointers; pointers.ExceptionRecord = &record; pointers.ContextRecord = contextRecord; @@ -190,8 +198,27 @@ public sealed unsafe partial class DirectExecutionBackend } _posixSignalHandlerDepth++; + if (_perfSignalCounter) + { + var n = Interlocked.Increment(ref _perfSignalCount); + if (n % 100000 == 0) + { + Console.Error.WriteLine($"[PERF][MEM] posix_faults={n}"); + } + } try { + // Guest-image write tracking runs first: it only needs the fault + // address (safe for host and guest threads alike) and must resume + // the faulting write immediately after restoring write access. + if (signal != PosixSigIll && + siginfo != 0 && + SharpEmu.HLE.GuestImageWriteTracker.TryHandleWriteFault( + *(ulong*)((byte*)siginfo + PosixSigInfoAddressOffset))) + { + return; + } + if (TryHandlePosixFault(signal, siginfo, ucontext)) { return; @@ -217,8 +244,8 @@ public sealed unsafe partial class DirectExecutionBackend return false; } - byte* contextRecord = stackalloc byte[Win64ContextOffsets.Size]; - new Span(contextRecord, Win64ContextOffsets.Size).Clear(); + byte* contextRecord = stackalloc byte[Win64ContextSize]; + new Span(contextRecord, Win64ContextSize).Clear(); int[] offsets = PosixRegisterOffsets; for (int i = 0; i < offsets.Length; i++) { @@ -231,6 +258,14 @@ public sealed unsafe partial class DirectExecutionBackend { record.ExceptionCode = 3221225501u; } + else if (signal == PosixSigTrap) + { + record.ExceptionCode = 2147483651u; + } + else if (signal == PosixSigAbort) + { + record.ExceptionCode = 1073741845u; + } else { ulong faultAddress = GetPosixFaultAddress(siginfo, registers); diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs index d8d27a1..df70098 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs @@ -2,52 +2,39 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System; -using System.Collections.Concurrent; +using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; using System.Threading; using SharpEmu.Core.Cpu; -using SharpEmu.Core.Cpu.Native.Windows; using SharpEmu.Core.Loader; using SharpEmu.Core.Memory; using SharpEmu.HLE; -using SharpEmu.HLE.Host; -using SharpEmu.HLE.Host.Posix; -using SharpEmu.Logging; namespace SharpEmu.Core.Cpu.Native; public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, IGuestThreadScheduler, IDisposable { - private static readonly SharpEmuLogger Log = SharpEmuLog.For("Native"); + private static readonly SharpEmu.Logging.SharpEmuLogger Log = SharpEmu.Logging.SharpEmuLog.For("Native"); + + private static readonly bool LogThreadMode = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_THREAD_MODE"), "1", StringComparison.Ordinal); + + private static void TraceThreadMode(string message) + { + Console.Error.WriteLine( + $"[THREADMODE] {message} tid={GetCurrentThreadId()} managed={Environment.CurrentManagedThreadId}"); + Console.Error.Flush(); + } + private const int ImportLoopHistoryLength = 2048; private const int ImportLoopWideDiversityWindow = 768; private const int DefaultImportLoopGuardSeconds = 5; - private enum ImportStubKind : byte - { - Normal = 0, - BootstrapBridge = 1, - KernelDynlibDlsym = 2, - Il2CppApiLookupSymbol = 3, - } - - [Flags] - private enum ImportStubTraceFlags : byte - { - None = 0, - Memset = 1 << 0, - CxaAtexit = 1 << 1, - RawArgs = 1 << 2, - StackChkFail = 1 << 3, - PeriodicEvery1000 = 1 << 4, - PeriodicEvery128 = 1 << 5, - } - private readonly struct ImportStubEntry { public ulong Address { get; } @@ -56,36 +43,34 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I public ExportedFunction? Export { get; } - public ImportStubKind Kind { get; } + // Precomputed per-import classification: DispatchImport runs for + // every guest HLE call, so per-call string pattern matches and NID + // hashing are hoisted to stub-setup time. + public bool IsLeaf { get; } - public ImportStubTraceFlags TraceFlags { get; } + public bool SuppressStrlenTrace { get; } - public ImportStubEntry(ulong address, string nid, ExportedFunction? export) + public bool IsLoopGuardBoundary { get; } + + public ulong NidHash { get; } + + public ImportStubEntry( + ulong address, + string nid, + ExportedFunction? export, + bool isLeaf, + bool suppressStrlenTrace, + bool isLoopGuardBoundary, + ulong nidHash) { Address = address; Nid = nid; Export = export; - Kind = ClassifyKind(nid); - TraceFlags = ClassifyTraceFlags(nid); + IsLeaf = isLeaf; + SuppressStrlenTrace = suppressStrlenTrace; + IsLoopGuardBoundary = isLoopGuardBoundary; + NidHash = nidHash; } - - private static ImportStubKind ClassifyKind(string nid) => nid switch - { - RuntimeStubNids.BootstrapBridge => ImportStubKind.BootstrapBridge, - RuntimeStubNids.KernelDynlibDlsym or "LwG8g3niqwA" => ImportStubKind.KernelDynlibDlsym, - "r8mvOaWdi28" => ImportStubKind.Il2CppApiLookupSymbol, - _ => ImportStubKind.Normal, - }; - - private static ImportStubTraceFlags ClassifyTraceFlags(string nid) => nid switch - { - "8zTFvBIAIN8" => ImportStubTraceFlags.Memset, - "tsvEmnenz48" => ImportStubTraceFlags.CxaAtexit | ImportStubTraceFlags.PeriodicEvery1000, - "bzQExy189ZI" or "8G2LB+A3rzg" => ImportStubTraceFlags.RawArgs, - "Ou3iL1abvng" => ImportStubTraceFlags.StackChkFail, - "rTXw65xmLIA" => ImportStubTraceFlags.PeriodicEvery128, - _ => ImportStubTraceFlags.None, - }; } private readonly record struct RecentImportTraceEntry( @@ -94,14 +79,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ulong ReturnRip, ulong Arg0, ulong Arg1, - ulong Arg2); - - private readonly record struct DeferredBootstrapTraceEntry( - long DispatchIndex, - ulong Op, - ulong SymbolPointer, - ulong OutputPointer, - ulong ReturnRip); + ulong Arg2, + ulong GuestThreadHandle, + int ManagedThreadId); #pragma warning disable CS0649 private struct EXCEPTION_POINTERS @@ -129,6 +109,29 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private delegate int ExceptionHandlerDelegate(void* exceptionInfo); +#pragma warning disable CS0649 + private struct MEMORY_BASIC_INFORMATION64 + { + public ulong BaseAddress; + + public ulong AllocationBase; + + public uint AllocationProtect; + + public uint __alignment1; + + public ulong RegionSize; + + public uint State; + + public uint Protect; + + public uint Type; + + public uint __alignment2; + } +#pragma warning restore CS0649 + private const ulong SYSTEM_RESERVED = 34359738368uL; private const ulong CODE_BASE_OFFSET = 4294967296uL; @@ -139,29 +142,32 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private const ulong GuestImageScanEnd = 36507222016uL; - // The 0x7FFx window is Windows-specific; dyld and Rosetta reserve that - // range on macOS, so POSIX guest threads use the lower 0x6FFx window. - // The POSIX stack base sits a further 1GB down: the import-stub region - // descends from 0x7000_0000_0000 on the same 16MB grid and reaches - // 0x6FFF_C000_0000 at its 64-module limit, which would otherwise consume - // the top stack slots (on Windows the two bands are ~15TB apart). - private static readonly ulong GuestThreadStackBaseAddress = - OperatingSystem.IsWindows() ? 0x7FFF_E000_0000UL : 0x6FFF_A000_0000UL; + // See CpuDispatcher: the 0x7FFx window is Windows-only; POSIX hosts + // (dyld shared cache, Rosetta 2 runtime) use 0x6FFx instead. + private static readonly ulong GuestThreadStackBaseAddress = OperatingSystem.IsWindows() ? 0x7FFF_E000_0000UL : 0x6FFF_E000_0000UL; - private static readonly ulong GuestThreadTlsBaseAddress = - OperatingSystem.IsWindows() ? 0x7FFE_0000_0000UL : 0x6FFE_0000_0000UL; + private static readonly ulong GuestThreadTlsBaseAddress = OperatingSystem.IsWindows() ? 0x7FFE_0000_0000UL : 0x6FFE_0000_0000UL; private const ulong GuestThreadStackSize = 0x0020_0000UL; private const ulong GuestThreadTlsSize = 0x0001_0000UL; // Matches CpuDispatcher.TlsPrefixSize: static TLS blocks sit below the - // TCB, and libc.prx already reaches beyond -0x1700 on POSIX. - private static readonly ulong GuestThreadTlsPrefixSize = OperatingSystem.IsWindows() ? 0x0000_1000UL : 0x0001_0000UL; + // thread pointer and PS5 modules can reach beyond one host page. + private const ulong GuestThreadTlsPrefixSize = GuestTlsTemplate.StartupStaticTlsReservation; private const ulong GuestThreadRegionStride = 0x0100_0000UL; - private const int GuestThreadRegionSlotCount = 256; + // Unity titles routinely create more than 64 workers once native plugins, + // lighting, streaming, and audio are active at the same time. Keep a broad + // deterministic address window for their stack and TLS regions. + private const int GuestThreadRegionSlots = 1024; + + [ThreadStatic] + private static List<(IVirtualMemory Memory, ulong Base)>? _nestedGuestCallbackStacks; + + [ThreadStatic] + private static int _nestedGuestCallbackDepth; private const uint PAGE_EXECUTE_READWRITE = 64u; @@ -262,7 +268,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private readonly List _importHandlerTrampolines = new List(); - private const int GuestContextTransferFrameQwords = 15; + private const int GuestContextTransferFrameQwords = 20; private readonly object _guestContextTransferStubGate = new(); @@ -278,31 +284,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private KeyValuePair[] _runtimeSymbolsByAddress = Array.Empty>(); - private readonly ConcurrentDictionary _runtimeSymbolsByName = - new(StringComparer.Ordinal); - - // Keep in sync with SelfLoader import-stub mapping constants. - private const ulong ImportStubRegionCanonicalBase = 0x0000_7000_0000_0000UL; - - private const ulong ImportStubRegionAddressStride = 0x0000_0000_0100_0000UL; - - private const ulong LazyImportStubSlotSize = 0x10; - - private const ulong ImportStubRegionPageSize = 0x1000UL; - - private const string KernelDynlibDlsymAerolibNid = "LwG8g3niqwA"; - - private readonly object _lazyDlsymStubGate = new(); - - private readonly Dictionary _lazyDlsymStubCache = new(StringComparer.Ordinal); - - private ulong _lazyImportStubPoolBase; - - private ulong _lazyImportStubNextSlot; - - private ulong _lazyImportStubPoolLimit; - - private bool _lazyImportStubPoolMapped; + private readonly Dictionary _runtimeSymbolsByName = new Dictionary(StringComparer.Ordinal); private readonly RecentImportTraceEntry[] _recentImportTrace = new RecentImportTraceEntry[64]; @@ -310,14 +292,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private int _recentImportTraceWriteIndex; - private readonly DeferredBootstrapTraceEntry[] _deferredBootstrapTrace = new DeferredBootstrapTraceEntry[32]; - - private int _deferredBootstrapTraceCount; - - private int _deferredBootstrapTraceWriteIndex; - - private readonly object _deferredBootstrapTraceGate = new(); - private readonly string[] _distinctImportNidHistory = new string[128]; private int _distinctImportNidHistoryCount; @@ -336,6 +310,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private bool _logGuestContext; + private bool _ignoreGuestInt41; + + private int _ignoredGuestInt41Count; + private bool _logGuestThreads; private bool _logUsleep; @@ -352,6 +330,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private string? _probeImportReturn; + private ulong _probeImportReturnAddress; + + private long _probeImportReturnAddressCount; + private string? _importFilter; private bool _disableImportLoopGuard; @@ -398,7 +380,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private long _importLoopPatternStartTimestamp; - private readonly Dictionary _importNidHashCache = new Dictionary(StringComparer.Ordinal); private enum GuestThreadRunState { @@ -427,11 +408,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I public string Name { get; init; } = string.Empty; - public int Priority { get; init; } + public int Priority { get; set; } - public ulong AffinityMask { get; init; } + public ulong AffinityMask { get; set; } - public CpuContext Context { get; init; } = null!; + public CpuContext Context { get; set; } = null!; + + public bool IsExternalExecutor { get; init; } + + public ulong StackBase { get; init; } + + public ulong StackSize { get; init; } + + public ulong ExceptionStackBase { get; set; } public GuestThreadRunState State { get; set; } @@ -448,6 +437,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I // Stays set through the wake transition; Resume() consumes it when the thread pumps. public IGuestThreadBlockWaiter? BlockWaiter { get; set; } + public Func? BlockResumeHandler { get; set; } + + public Func? BlockWakeHandler { get; set; } + public long BlockDeadlineTimestamp { get; set; } public long ImportCount; @@ -456,13 +449,55 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I public ulong LastReturnRip; + // Busy guest workers overwrite the global recent-import ring. Preserve + // the most recent complete SysV call frame per guest thread so native + // fault diagnostics can identify the exact preceding HLE invocation. + public ulong LastImportRdi; + public ulong LastImportRsi; + public ulong LastImportRdx; + public ulong LastImportRcx; + public ulong LastImportR8; + public ulong LastImportR9; + public ulong LastImportStack0; + public ulong LastImportStack1; + public ulong LastImportStack2; + public ulong LastImportStack3; + public ulong LastImportStack4; + public ulong LastImportStack5; + public ulong LastImportRax; + public int LastImportResultValid; + public Thread? HostThread { get; set; } public int HostThreadId; + // State may become Ready as soon as another guest thread satisfies this + // thread's wait, while the host executor that yielded it is still + // unwinding. Keep executor ownership separate from State so a Ready + // continuation cannot be claimed concurrently with that unwind. + public bool ExecutorActive { get; set; } + + public bool ExceptionDeliveryActive { get; set; } + + public long ExecutorClaimDeferrals { get; set; } + public GuestContinuationRunner? ContinuationRunner { get; set; } + + public GuestExecutionRunner? ExecutionRunner { get; set; } } + private sealed class ExternalGuestThreadState + { + public CpuContext Context { get; set; } = null!; + + public ulong ExceptionStackBase { get; set; } + } + + private readonly record struct PendingGuestException( + ulong Handler, + int ExceptionType, + ulong ExceptionStackBase); + private sealed class GuestContinuationRunner : IDisposable { private readonly ulong _guestThreadHandle; @@ -501,10 +536,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private void ThreadMain() { var previousGuestThreadHandle = GuestThreadExecution.EnterGuestThread(_guestThreadHandle); - if (LogThreadMode) - { - TraceThreadMode($"runner_start guest=0x{_guestThreadHandle:X16}"); - } try { while (true) @@ -515,11 +546,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return; } - if (LogThreadMode) - { - _threadModeCycleId = Interlocked.Increment(ref _threadModeCycleCounter); - TraceThreadMode($"runner_run guest=0x{_guestThreadHandle:X16}"); - } try { _work?.Invoke(); @@ -532,10 +558,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - if (LogThreadMode) - { - TraceThreadMode($"runner_stop guest=0x{_guestThreadHandle:X16}"); - } GuestThreadExecution.RestoreGuestThread(previousGuestThreadHandle); } } @@ -553,6 +575,86 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } } + // A guest pthread may block and resume hundreds of times per second. Creating + // a new host Thread for every resume is especially costly for audio workers + // and eventually floods macOS with short-lived FMOD threads. Keep one dormant + // host executor per guest pthread and signal it for each runnable slice. + private sealed class GuestExecutionRunner : IDisposable + { + private readonly object _gate = new(); + private readonly AutoResetEvent _workAvailable = new(false); + private readonly Thread _thread; + private Action? _work; + private volatile bool _stopping; + + public GuestExecutionRunner(ulong guestThreadHandle, string name, ThreadPriority priority) + { + _thread = new Thread(() => ThreadMain(guestThreadHandle)) + { + IsBackground = true, + Name = $"SharpEmu-{name}", + Priority = priority, + }; + _thread.Start(); + } + + public void Schedule(Action work) + { + lock (_gate) + { + if (_stopping) + { + return; + } + if (_work is not null) + { + throw new InvalidOperationException("Guest execution runner already has pending work."); + } + _work = work; + _workAvailable.Set(); + } + } + + private void ThreadMain(ulong guestThreadHandle) + { + var previousGuestThreadHandle = GuestThreadExecution.EnterGuestThread(guestThreadHandle); + try + { + while (true) + { + _workAvailable.WaitOne(); + if (_stopping) + { + return; + } + + Action? work; + lock (_gate) + { + work = _work; + _work = null; + } + work?.Invoke(); + } + } + finally + { + GuestThreadExecution.RestoreGuestThread(previousGuestThreadHandle); + } + } + + public void Dispose() + { + _stopping = true; + _workAvailable.Set(); + if (!ReferenceEquals(Thread.CurrentThread, _thread)) + { + _thread.Join(500); + } + _workAvailable.Dispose(); + } + } + private readonly record struct LazyCommitRange(ulong BaseAddress, ulong Size); private readonly object _guestThreadGate = new object(); @@ -591,22 +693,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private readonly Queue _readyGuestThreads = new Queue(); - // Once set, guest worker threads are unwound to the host at their next import - // dispatch and Pump refuses to start new ones. This must happen before the - // runtime frees trampolines or guest memory: workers that keep running native - // guest code during teardown execute freed pages (observed as an execute-AV in - // a MEM_FREE module region plus a CLR "UnmanagedCallersOnly" fatal from a Pump - // thread entering a freed stub). - private volatile bool _guestTeardownRequested; - - private volatile bool _hostShutdownRequested; - - private static volatile DirectExecutionBackend? _activeSessionBackend; - private int _readyGuestThreadCount; private readonly Dictionary _guestThreads = new Dictionary(); + private readonly Dictionary _externalGuestThreads = new Dictionary(); + + [ThreadStatic] + private static ulong _currentExternalGuestThreadHandle; + + private readonly Dictionary _pendingGuestExceptions = new Dictionary(); + + private readonly HashSet _activeGuestExceptionDeliveries = new HashSet(); + private int _guestThreadPumpDepth; private bool _guestThreadYieldRequested; @@ -631,20 +730,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private Thread? _stallWatchdogThread; + private volatile bool _readyDispatchStop; + + private Thread? _readyDispatchThread; + private GCHandle _selfHandle; private nint _selfHandlePtr; - private readonly IHostPlatform _hostPlatform; - - private readonly IHostThreading _hostThreading; - - private readonly IHostSymbolResolver _hostSymbols; - - private readonly IHostMemory _hostMemory; - - private readonly IHostFaultHandling _faultHandling; - private const int MinTlsPatchInstructionBytes = 9; private delegate ulong ImportGatewayDelegate(nint backendHandle, int importIndex, nint argPackPtr); @@ -653,17 +746,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private static readonly RawExceptionHandlerDelegate RawVectoredHandlerDelegateInstance = RawVectoredHandlerManaged; private static readonly RawExceptionHandlerDelegate RawUnhandledFilterDelegateInstance = RawUnhandledFilterManaged; - private static readonly nint ImportGatewayPtr = ResolveImportGatewayPtr(); + private static readonly nint ImportGatewayPtr = ResolveWin64CallbackPtr( + Marshal.GetFunctionPointerForDelegate(ImportGatewayDelegateInstance)); - // Emitted import trampolines use the Win64 ABI. Managed callbacks use the - // host ABI, so POSIX needs a small Win64-to-SysV register-shuffling thunk. - private static nint ResolveImportGatewayPtr() - { - var managedPtr = Marshal.GetFunctionPointerForDelegate(ImportGatewayDelegateInstance); - return OperatingSystem.IsWindows() - ? managedPtr - : PosixHostStubs.CreateWin64ToSysVThunk(managedPtr); - } + // Emitted trampolines call managed callbacks with the Win64 ABI. On + // Windows the runtime already compiles them that way; on POSIX .NET they + // are SysV, so route through a Win64->SysV thunk. + private static nint ResolveWin64CallbackPtr(nint sysvPtr) => + OperatingSystem.IsWindows() ? sysvPtr : PosixHostStubs.CreateWin64ToSysVThunk(sysvPtr); private static readonly nint RawVectoredHandlerPtrManaged = Marshal.GetFunctionPointerForDelegate(RawVectoredHandlerDelegateInstance); @@ -671,41 +761,41 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private static readonly nint RawUnhandledFilterPtrManaged = Marshal.GetFunctionPointerForDelegate(RawUnhandledFilterDelegateInstance); - private const int CTX_MXCSR = Win64ContextOffsets.Mxcsr; + private const int CTX_MXCSR = 52; - private const int CTX_RAX = Win64ContextOffsets.Rax; + private const int CTX_RAX = 120; - private const int CTX_RCX = Win64ContextOffsets.Rcx; + private const int CTX_RCX = 128; - private const int CTX_RDX = Win64ContextOffsets.Rdx; + private const int CTX_RDX = 136; - private const int CTX_RBX = Win64ContextOffsets.Rbx; + private const int CTX_RBX = 144; - private const int CTX_RSP = Win64ContextOffsets.Rsp; + private const int CTX_RSP = 152; - private const int CTX_RBP = Win64ContextOffsets.Rbp; + private const int CTX_RBP = 160; - private const int CTX_RSI = Win64ContextOffsets.Rsi; + private const int CTX_RSI = 168; - private const int CTX_RDI = Win64ContextOffsets.Rdi; + private const int CTX_RDI = 176; - private const int CTX_R8 = Win64ContextOffsets.R8; + private const int CTX_R8 = 184; - private const int CTX_R9 = Win64ContextOffsets.R9; + private const int CTX_R9 = 192; - private const int CTX_R10 = Win64ContextOffsets.R10; + private const int CTX_R10 = 200; - private const int CTX_R11 = Win64ContextOffsets.R11; + private const int CTX_R11 = 208; - private const int CTX_R12 = Win64ContextOffsets.R12; + private const int CTX_R12 = 216; - private const int CTX_R13 = Win64ContextOffsets.R13; + private const int CTX_R13 = 224; - private const int CTX_R14 = Win64ContextOffsets.R14; + private const int CTX_R14 = 232; - private const int CTX_R15 = Win64ContextOffsets.R15; + private const int CTX_R15 = 240; - private const int CTX_RIP = Win64ContextOffsets.Rip; + private const int CTX_RIP = 248; private ExceptionHandlerDelegate? _handlerDelegate; @@ -720,37 +810,13 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private static int _nestedVehTraceCount; - // SHARPEMU_LOG_THREAD_MODE=1 — GC thread-mode corruption investigation. Traces - // every managed<->guest transition per guest-thread run cycle so the last event - // before a ReversePInvokeBadTransition FailFast identifies the corrupting path. - private static readonly bool LogThreadMode = - string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_THREAD_MODE"), "1", StringComparison.Ordinal); + private const uint MEM_COMMIT = 4096u; - private static long _threadModeCycleCounter; + private const uint MEM_RESERVE = 8192u; - [ThreadStatic] - private static long _threadModeCycleId; + private const uint MEM_FREE = 65536u; - [ThreadStatic] - private static int _threadModeGatewayDepth; - - [ThreadStatic] - private static long _threadModeGatewayCalls; - - [ThreadStatic] - private static bool _threadModeGatewayFirstLogged; - - private static void TraceThreadMode(string message) - { - // Prefer the platform injected into the backend bound to this thread; - // the ambient is set on every guest/import/VEH transition this tracer - // observes. The singleton fallback only covers pre-run traces, where a - // constructed backend implies the platform resolved successfully. - var threading = _activeExecutionBackend?._hostThreading ?? HostPlatform.Current.Threading; - Console.Error.WriteLine( - $"[THREADMODE] {message} cycle={_threadModeCycleId} tid={threading.CurrentThreadId} managed={Environment.CurrentManagedThreadId}"); - Console.Error.Flush(); - } + private const uint MEM_RELEASE = 32768u; private const uint PAGE_EXECUTE = 16u; @@ -770,6 +836,16 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private const uint HostXmmSaveAreaSize = 0xA0u; + private const uint ContextAmd64ControlInteger = 0x00100003u; + + private const uint ThreadGetContext = 0x0008u; + + private const uint ThreadSuspendResume = 0x0002u; + + private const int Win64ContextSize = 0x4D0; + + private const int Win64ContextFlagsOffset = 0x30; + private readonly record struct HostThreadContextSnapshot( bool IsValid, ulong Rip, @@ -792,17 +868,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe static int CallNativeEntry(void* entry) { var nativeEntry = (delegate* unmanaged[Cdecl])entry; - if (!LogThreadMode) - { - return nativeEntry(); - } - - _threadModeGatewayFirstLogged = false; - var dispatchesBefore = _threadModeGatewayCalls; - TraceThreadMode($"native_enter entry=0x{(ulong)entry:X16}"); - var result = nativeEntry(); - TraceThreadMode($"native_exit result=0x{result:X8} dispatches={_threadModeGatewayCalls - dispatchesBefore}"); - return result; + return nativeEntry(); } private unsafe static void WriteCtxU64(void* contextRecord, int offset, ulong value) @@ -845,8 +911,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private bool ActiveForcedGuestExit { - get => _hostShutdownRequested || - (HasActiveExecutionThread ? _activeForcedGuestExit : _forcedGuestExit); + get => HasActiveExecutionThread ? _activeForcedGuestExit : _forcedGuestExit; set { if (HasActiveExecutionThread) @@ -910,41 +975,47 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _activeGuestThreadYieldReason = previousYieldReason; } - public unsafe DirectExecutionBackend(IModuleManager moduleManager, IHostPlatform? hostPlatform = null, IHostFaultHandling? faultHandling = null) + public unsafe DirectExecutionBackend(IModuleManager moduleManager) { _moduleManager = moduleManager ?? throw new ArgumentNullException("moduleManager"); - _hostPlatform = hostPlatform ?? HostPlatform.Current; - _hostThreading = _hostPlatform.Threading; - _hostSymbols = _hostPlatform.Symbols; - _hostMemory = _hostPlatform.Memory; - _faultHandling = faultHandling ?? (OperatingSystem.IsWindows() - ? new WindowsFaultHandling(_hostMemory) - : NullHostFaultHandling.Instance); _selfHandle = GCHandle.Alloc(this); _selfHandlePtr = GCHandle.ToIntPtr(_selfHandle); - _guestTlsBaseTlsIndex = _hostThreading.AllocateTlsSlot(); - _hostRspSlotTlsIndex = _hostThreading.AllocateTlsSlot(); + _guestTlsBaseTlsIndex = TlsAlloc(); + _hostRspSlotTlsIndex = TlsAlloc(); if (_guestTlsBaseTlsIndex == uint.MaxValue || _hostRspSlotTlsIndex == uint.MaxValue) { throw new OutOfMemoryException("Failed to allocate native TLS slots"); } - _tlsGetValueAddress = _hostSymbols.GetAddress(HostRuntimeFunction.TlsGetValue); - if (_tlsGetValueAddress == 0) + if (OperatingSystem.IsWindows()) { - throw new InvalidOperationException("Failed to resolve kernel32!TlsGetValue"); + nint kernel32 = GetModuleHandle("kernel32.dll"); + _tlsGetValueAddress = kernel32 != 0 ? GetProcAddress(kernel32, "TlsGetValue") : 0; + if (_tlsGetValueAddress == 0) + { + throw new InvalidOperationException("Failed to resolve kernel32!TlsGetValue"); + } + _queryPerformanceCounterAddress = kernel32 != 0 ? GetProcAddress(kernel32, "QueryPerformanceCounter") : 0; + if (_queryPerformanceCounterAddress == 0) + { + throw new InvalidOperationException("Failed to resolve kernel32!QueryPerformanceCounter"); + } + _switchToThreadAddress = kernel32 != 0 ? GetProcAddress(kernel32, "SwitchToThread") : 0; + _sleepAddress = kernel32 != 0 ? GetProcAddress(kernel32, "Sleep") : 0; + if (_switchToThreadAddress == 0 || _sleepAddress == 0) + { + throw new InvalidOperationException("Failed to resolve kernel32 thread timing functions"); + } } - _queryPerformanceCounterAddress = _hostSymbols.GetAddress(HostRuntimeFunction.QueryPerformanceCounter); - if (_queryPerformanceCounterAddress == 0) + else { - throw new InvalidOperationException("Failed to resolve kernel32!QueryPerformanceCounter"); + // Win64-ABI-compatible helper stubs so the emitted call sites do + // not need to change per platform. + _tlsGetValueAddress = PosixHostStubs.TlsGetValueStubAddress; + _queryPerformanceCounterAddress = PosixHostStubs.QueryPerformanceCounterStubAddress; + _switchToThreadAddress = PosixHostStubs.SwitchToThreadStubAddress; + _sleepAddress = PosixHostStubs.SleepStubAddress; } - _switchToThreadAddress = _hostSymbols.GetAddress(HostRuntimeFunction.SwitchToThread); - _sleepAddress = _hostSymbols.GetAddress(HostRuntimeFunction.Sleep); - if (_switchToThreadAddress == 0 || _sleepAddress == 0) - { - throw new InvalidOperationException("Failed to resolve kernel32 thread timing functions"); - } - _tlsBaseAddress = (nint)_hostMemory.Allocate(0, 4096u, HostPageProtection.ReadWrite); + _tlsBaseAddress = (nint)VirtualAlloc(null, 4096u, 12288u, 4u); if (_tlsBaseAddress == 0) { throw new OutOfMemoryException("Failed to allocate TLS base"); @@ -952,7 +1023,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _ownedTlsBaseAddress = _tlsBaseAddress; _ownsTlsBaseAddress = true; SeedTlsLayout(_tlsBaseAddress); - _hostRspSlotStorage = (nint)_hostMemory.Allocate(0, 4096u, HostPageProtection.ReadWrite); + _hostRspSlotStorage = (nint)VirtualAlloc(null, 4096u, 12288u, 4u); if (_hostRspSlotStorage == 0) { throw new OutOfMemoryException("Failed to allocate host stack slot storage"); @@ -981,14 +1052,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I result = OrbisGen2Result.ORBIS_GEN2_OK; LastError = null; InitializeRuntimeSymbolIndex(runtimeSymbols); - ResetLazyDlsymStubState(); _recentImportTraceCount = 0; _recentImportTraceWriteIndex = 0; - lock (_deferredBootstrapTraceGate) - { - _deferredBootstrapTraceCount = 0; - _deferredBootstrapTraceWriteIndex = 0; - } _distinctImportNidHistoryCount = 0; _distinctImportNidHistoryWriteIndex = 0; _lastDistinctImportNid = string.Empty; @@ -998,6 +1063,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _logStrlenBursts = _logStrlenImports || string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_STRLEN_BURSTS"), "1", StringComparison.Ordinal); _logGuestContext = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_CONTEXT"), "1", StringComparison.Ordinal); + _ignoreGuestInt41 = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_IGNORE_INT41"), "1", StringComparison.Ordinal); + _ignoredGuestInt41Count = 0; _logGuestThreads = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_THREADS"), "1", StringComparison.Ordinal); _logUsleep = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_USLEEP"), "1", StringComparison.Ordinal); _logBootstrap = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_BOOTSTRAP"), "1", StringComparison.Ordinal); @@ -1006,6 +1073,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _logImportRecent = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_IMPORT_RECENT"), "1", StringComparison.Ordinal); _logStackCheck = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_STACK_CHK"), "1", StringComparison.Ordinal); _probeImportReturn = Environment.GetEnvironmentVariable("SHARPEMU_PROBE_IMPORT_RET"); + _probeImportReturnAddress = ParseOptionalHexAddress( + Environment.GetEnvironmentVariable("SHARPEMU_PROBE_IMPORT_RET_ADDRESS")); + _probeImportReturnAddressCount = 0; _importFilter = Environment.GetEnvironmentVariable("SHARPEMU_LOG_IMPORT_FILTER"); _disableImportLoopGuard = string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_IMPORT_LOOP_GUARD"), @@ -1014,15 +1084,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _importLoopGuardSeconds = GetImportLoopGuardSeconds(); _entryReturnSentinelRip = 0uL; _forcedGuestExit = false; - _hostShutdownRequested = false; - _guestTeardownRequested = false; - _activeSessionBackend = this; HostSessionControl.SetShutdownHandler(RequestHostShutdown); _importLoopSignatureCount = 0; _importLoopSignatureWriteIndex = 0; _importLoopPatternHits = 0; _importLoopPatternStartTimestamp = 0; - _importNidHashCache.Clear(); lock (_importResultLogSampleGate) { _importResultLogSamples.Clear(); @@ -1035,6 +1101,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _contextualUnresolvedReturnSites.Clear(); _stallWatchdogTriggered = 0; _stallWatchdogStop = false; + _readyDispatchStop = false; _patchedEa020eLookupCall = false; MarkExecutionProgress(); BindTlsBase(context); @@ -1066,11 +1133,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I finally { HostSessionControl.SetShutdownHandler(null); - if (ReferenceEquals(_activeSessionBackend, this)) - { - _activeSessionBackend = null; - } - DrainDeferredBootstrapTraces(); GuestThreadExecution.Scheduler = previousGuestThreadScheduler; Console.Error.WriteLine("[LOADER][INFO] === Execute END (LastError: " + (LastError ?? "null") + ") ==="); } @@ -1078,9 +1140,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I internal void RequestHostShutdown(string reason) { - _hostShutdownRequested = true; _forcedGuestExit = true; - _guestTeardownRequested = true; LastError = string.IsNullOrWhiteSpace(reason) ? "Host shutdown requested." : $"Host shutdown requested: {reason}"; @@ -1099,7 +1159,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I foreach (var (num4, text2) in importStubs) { _ = _moduleManager.TryGetExport(text2, out var resolvedExport); - _importEntries[num] = new ImportStubEntry(num4, text2, resolvedExport); + _importEntries[num] = new ImportStubEntry( + num4, + text2, + resolvedExport, + IsLeafImport(text2), + ShouldSuppressStrlenTrace(text2), + IsImportLoopGuardBoundary(text2), + StableHash64(text2)); if ((num4 >= 34393242112L && num4 <= 34393242624L) || (num4 >= 34393258496L && num4 <= 34393259008L)) { if (resolvedExport is not null) @@ -1113,7 +1180,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } if (TryResolveDirectImportTarget(text2, out var targetAddress, out var resolvedSymbol) && !hashSet.Contains(targetAddress)) { - Console.Error.WriteLine($"[LOADER][DEBUG] SetupImportStubs: Direct bridge for {text2} -> 0x{targetAddress:X16}"); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] SetupImportStubs: Direct bridge for {text2} -> 0x{targetAddress:X16}"); + } if (!PatchImportStub((nint)(long)num4, (nint)(long)targetAddress)) { LastError = $"Failed to patch direct import stub at 0x{num4:X16}"; @@ -1146,7 +1216,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I LastError = "Failed to create import trampoline for NID " + text2; return false; } - Console.Error.WriteLine($"[LOADER][DEBUG] SetupImportStubs: Trampoline for {text2} -> 0x{num5:X16}"); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] SetupImportStubs: Trampoline for {text2} -> 0x{num5:X16}"); + } if (!PatchImportStub((nint)num4, num5)) { LastError = $"Failed to patch import stub at 0x{num4:X16}"; @@ -1333,72 +1406,21 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I 0x75, 0xF5, 0xC3, ], - // memcpy: guarded native copy. The first "rep movsb" intrinsic had no bounds/null - // checking and crashed with a read at -1 right after a null-dst memset recovery in - // the NGS2 audio streaming code path, so it was temporarily pulled in favor of the - // safe C# HLE memcpy. That detour costs a full import dispatch per call - far too - // slow for a function this hot - so this stub keeps the native leaf path and adds - // the same guards as memset below: it silently returns dst without copying when dst - // or src is null/low-page (< 0x10000) or outside canonical user space, or when len - // is 0 or absurd (> 512MB). "Q3VBxCXhUHs" => [ - 0x48, 0x89, 0xF8, // mov rax, rdi (return dst) - 0x48, 0x81, 0xFF, 0x00, 0x00, 0x01, 0x00, // cmp rdi, 0x10000 - 0x72, 0x31, // jb done - 0x49, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, // mov r8, 0x800000000000 - 0x4C, 0x39, 0xC7, // cmp rdi, r8 - 0x73, 0x22, // jae done - 0x48, 0x81, 0xFE, 0x00, 0x00, 0x01, 0x00, // cmp rsi, 0x10000 - 0x72, 0x19, // jb done - 0x4C, 0x39, 0xC6, // cmp rsi, r8 - 0x73, 0x14, // jae done - 0x48, 0x81, 0xFA, 0x00, 0x00, 0x00, 0x20, // cmp rdx, 0x20000000 - 0x77, 0x0B, // ja done - 0x48, 0x85, 0xD2, // test rdx, rdx - 0x74, 0x06, // jz done - 0x48, 0x89, 0xD1, // mov rcx, rdx - 0xFC, // cld - 0xF3, 0xA4, // rep movsb - 0xC3, // done: ret - ], - // memset: guarded native fill. An earlier unguarded version crashed with a write AV - // at address 0 (NGS2 audio streaming init memsets a never-populated buffer field), - // so this one mirrors the HLE guards and silently returns dst without writing when - // dst is null/low-page (< 0x10000), dst is outside canonical user space, or len is - // absurd (> 512MB, e.g. the 0x27060035 / sign-extended values NGS2 passes). Routing - // memset through the HLE trampoline instead is not viable: parse/streaming loops - // issue hundreds of thousands of small memsets back-to-back, which crawls at - // dispatch speed and looks like a repeating-import hang to the loop guard. - // _sigprocmask: the HLE handler (KernelRuntimeCompatExports.Sigprocmask) is a pure - // no-op returning 0 that never writes oldset, so this is behavior-identical. The - // game's bundled libc queries the mask (set=NULL) once per iteration in its font/ - // parse loops - hundreds of thousands of back-to-back calls that both crawl at - // dispatch speed and read as a repeating-import hang to the loop guard. - "6xVpy0Fdq+I" => - [ - 0x31, 0xC0, // xor eax, eax - 0xC3, // ret + 0x48, 0x89, 0xF8, + 0x48, 0x89, 0xD1, + 0xF3, 0xA4, + 0xC3, ], "8zTFvBIAIN8" => [ - 0x48, 0x89, 0xF8, // mov rax, rdi (return dst) - 0x48, 0x81, 0xFF, 0x00, 0x00, 0x01, 0x00, // cmp rdi, 0x10000 - 0x72, 0x2B, // jb done - 0x49, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, // mov r8, 0x800000000000 - 0x4C, 0x39, 0xC7, // cmp rdi, r8 - 0x73, 0x1C, // jae done - 0x48, 0x81, 0xFA, 0x00, 0x00, 0x00, 0x20, // cmp rdx, 0x20000000 - 0x77, 0x13, // ja done - 0x48, 0x85, 0xD2, // test rdx, rdx - 0x74, 0x0E, // jz done - 0x48, 0x89, 0xD1, // mov rcx, rdx - 0x49, 0x89, 0xF9, // mov r9, rdi - 0x89, 0xF0, // mov eax, esi - 0xFC, // cld - 0xF3, 0xAA, // rep stosb - 0x4C, 0x89, 0xC8, // mov rax, r9 - 0xC3, // done: ret + 0x49, 0x89, 0xF8, + 0x48, 0x89, 0xF0, + 0x48, 0x89, 0xD1, + 0xF3, 0xAA, + 0x4C, 0x89, 0xC0, + 0xC3, ], _ => default, }; @@ -1409,7 +1431,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } const uint intrinsicAllocationSize = 128u; - void* memory = (void*)_hostMemory.Allocate(0, intrinsicAllocationSize, HostPageProtection.ReadWriteExecute); + void* memory = VirtualAlloc(null, intrinsicAllocationSize, 12288u, 64u); if (memory == null) { address = 0; @@ -1427,14 +1449,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I *(nint*)((byte*)memory + 64) = _sleepAddress; } uint oldProtect = 0; - if (!_hostMemory.Protect((ulong)memory, intrinsicAllocationSize, HostPageProtection.ReadExecute, out oldProtect)) + if (!VirtualProtect(memory, intrinsicAllocationSize, 32u, &oldProtect)) { - _hostMemory.Free((ulong)memory); + VirtualFree(memory, 0u, 32768u); address = 0; return false; } - _hostMemory.FlushInstructionCache((ulong)memory, (nuint)code.Length); + FlushInstructionCache(GetCurrentProcess(), memory, (nuint)code.Length); address = (nint)memory; _importHandlerTrampolines.Add(address); return true; @@ -1457,7 +1479,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { if (IsKernelLibrary(export.LibraryName)) { - Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} ({export.LibraryName}:{export.Name}) -> HLE (kernel library)"); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} ({export.LibraryName}:{export.Name}) -> HLE (kernel library)"); + } return false; } if (!IsLibcLibrary(export.LibraryName) || !PreferLleForLibcExport(export.Name)) @@ -1482,13 +1507,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return false; } - Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} not in HLE table, checking runtime symbols..."); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} not in HLE table, checking runtime symbols..."); + } if (TryResolveRuntimeSymbolAddress(nid, out var directValue) && IsDirectImportTargetUsable(directValue)) { targetAddress = directValue; resolvedSymbol = nid; - Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} -> runtime symbol 0x{targetAddress:X16}"); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} -> runtime symbol 0x{targetAddress:X16}"); + } return true; } @@ -1617,25 +1648,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { return exportName switch { + "memcpy" or "memmove" or - // memset/memcpy excluded: the raw LLE routines have no null/bounds guard and crash - // with an access violation on bad pointers (observed hit during Quake's CL_Init, - // where a still-unidentified upstream bug calls memcpy/memset with a null - // destination). Both are instead served by the guarded native intrinsics in - // TryCreateNativeImportIntrinsic, which fail safely without leaving the leaf path. - "memcmp" or - // _Getpctype must come from the game's own Dinkumware libc when one is bundled: - // it returns a pointer to that CRT's ctype bitmask table, whose bit layout - // (_DI=0x20, _SP=0x04, _BB=0x80, ...) differs from the MSVC-style table the HLE - // fallback used to serve. Serving the wrong layout made the bundled printf engine - // render every Sys_Error message as an empty string (isdigit misfired during - // %-directive parsing) and made mcpp drop 'a'-'f' from identifiers ("texture" -> - // "txtur", the 0x80 bit reads as _BB/control there). _Getptolower/_Getptoupper - // already resolve to the bundled module because no HLE export shadows them; this - // keeps _Getpctype consistent with them. It is a pure accessor returning a pointer - // to a const table, so it is also the cheapest possible LLE call - important - // because parsers hit it once per input character. - "_Getpctype" => true, + "memset" or + "memcmp" => true, _ => false, }; } @@ -1695,7 +1711,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I context.FsBase = (ulong)num; context.GsBase = (ulong)num; SeedTlsLayout(num); - _hostThreading.SetTlsValue(_guestTlsBaseTlsIndex, num); + TlsSetValue(_guestTlsBaseTlsIndex, num); } } @@ -1719,7 +1735,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } uint oldProtect = default; - if (!_hostMemory.Protect((ulong)(void*)_tlsHandlerAddress, 16u, HostPageProtection.ReadWriteExecute, out oldProtect)) + if (!VirtualProtect((void*)_tlsHandlerAddress, 16u, 64u, &oldProtect)) { return; } @@ -1730,8 +1746,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)_tlsHandlerAddress, 16u, oldProtect, out oldProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)_tlsHandlerAddress, 16u); + VirtualProtect((void*)_tlsHandlerAddress, 16u, oldProtect, &oldProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)_tlsHandlerAddress, 16u); } } @@ -1749,6 +1765,13 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I error = $"invalid guest context transfer target rip=0x{target.Rip:X16} rsp=0x{target.Rsp:X16}"; return false; } + if (target.Rsp < sizeof(ulong) || + ActiveCpuContext is not { } activeContext || + !activeContext.TryWriteUInt64(target.Rsp - sizeof(ulong), target.Rip)) + { + error = $"guest context transfer slot is not writable at 0x{target.Rsp - sizeof(ulong):X16}"; + return false; + } transferStub = GetOrCreateGuestContextTransferStub(); if (transferStub == 0) @@ -1776,10 +1799,15 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I frame[8] = target.Rdi; frame[9] = target.R8; frame[10] = target.R9; - frame[11] = target.R12; - frame[12] = target.R13; - frame[13] = target.R14; - frame[14] = target.R15; + frame[11] = target.R10; + frame[12] = target.R11; + frame[13] = target.R12; + frame[14] = target.R13; + frame[15] = target.R14; + frame[16] = target.R15; + frame[17] = target.Mxcsr == 0 ? 0x1F80u : target.Mxcsr; + frame[18] = target.FpuControlWord == 0 ? 0x037Fu : target.FpuControlWord; + frame[19] = target.RestoreFullFpuState ? 1u : 0u; return true; } @@ -1797,8 +1825,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return _guestContextTransferStub; } - const uint stubSize = 128; - var code = (byte*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); + const uint stubSize = 256; + var code = (byte*)VirtualAlloc(null, stubSize, 12288u, 64u); if (code == null) { return 0; @@ -1813,10 +1841,47 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I Emit((byte)(0x40 | ((register & 7) << 3) | 0x03)); Emit(displacement); } + void EmitLoadFromR11Disp32(int register, int displacement) + { + Emit((byte)(0x49 | (register >= 8 ? 0x04 : 0x00))); + Emit(0x8B); + Emit((byte)(0x80 | ((register & 7) << 3) | 0x03)); + *(int*)(code + offset) = displacement; + offset += sizeof(int); + } Emit(0x49); Emit(0x89); Emit(0xC3); // mov r11, rax - EmitLoadFromR11(10, 0); // target RIP - EmitLoadFromR11(4, 8); // rsp + // A new >=3.50 fiber receives the SDK-defined MXCSR verbatim. A + // resumed fiber follows _sceFiberLongJmp: preserve status bits 0-5 + // while restoring the saved control bits. + Emit(0x49); Emit(0x83); Emit(0xBB); // cmp qword [r11+152],0 + *(int*)(code + offset) = 152; offset += sizeof(int); Emit(0x00); + Emit(0x0F); Emit(0x84); // je merge_status + var mergeBranch = offset; offset += sizeof(int); + Emit(0x41); Emit(0x0F); Emit(0xAE); Emit(0x93); // ldmxcsr [r11+136] + *(int*)(code + offset) = 136; offset += sizeof(int); + Emit(0xE9); // jmp mxcsr_done + var doneBranch = offset; offset += sizeof(int); + var mergeLabel = offset; + Emit(0x48); Emit(0x83); Emit(0xEC); Emit(0x08); // sub rsp,8 + Emit(0x0F); Emit(0xAE); Emit(0x1C); Emit(0x24); // stmxcsr [rsp] + Emit(0x41); Emit(0x8B); Emit(0x83); // mov eax,[r11+136] + *(int*)(code + offset) = 136; offset += sizeof(int); + Emit(0x25); *(uint*)(code + offset) = 0xFFFFFFC0u; offset += sizeof(uint); // and eax,~0x3f + Emit(0x8B); Emit(0x0C); Emit(0x24); // mov ecx,[rsp] + Emit(0x83); Emit(0xE1); Emit(0x3F); // and ecx,0x3f + Emit(0x09); Emit(0xC8); // or eax,ecx + Emit(0x89); Emit(0x04); Emit(0x24); // mov [rsp],eax + Emit(0x0F); Emit(0xAE); Emit(0x14); Emit(0x24); // ldmxcsr [rsp] + Emit(0x48); Emit(0x83); Emit(0xC4); Emit(0x08); // add rsp,8 + var mxcsrDoneLabel = offset; + *(int*)(code + mergeBranch) = mergeLabel - (mergeBranch + sizeof(int)); + *(int*)(code + doneBranch) = mxcsrDoneLabel - (doneBranch + sizeof(int)); + Emit(0x41); Emit(0xD9); Emit(0xAB); // fldcw [r11+144] + *(int*)(code + offset) = 144; offset += sizeof(int); + + EmitLoadFromR11(4, 8); // resume rsp + Emit(0x48); Emit(0x83); Emit(0xEC); Emit(0x08); // point at transfer return slot EmitLoadFromR11(1, 24); // rcx EmitLoadFromR11(2, 32); // rdx EmitLoadFromR11(3, 40); // rbx @@ -1825,21 +1890,24 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I EmitLoadFromR11(7, 64); // rdi EmitLoadFromR11(8, 72); // r8 EmitLoadFromR11(9, 80); // r9 - EmitLoadFromR11(12, 88); // r12 - EmitLoadFromR11(13, 96); // r13 - EmitLoadFromR11(14, 104); // r14 - EmitLoadFromR11(15, 112); // r15 + EmitLoadFromR11(10, 88); // r10 + EmitLoadFromR11(12, 104); // r12 + EmitLoadFromR11(13, 112); // r13 + EmitLoadFromR11(14, 120); // r14 + EmitLoadFromR11Disp32(15, 128); // r15 EmitLoadFromR11(0, 16); // rax - Emit(0x41); Emit(0xFF); Emit(0xE2); // jmp r10 + EmitLoadFromR11(11, 96); // r11 (last: frame pointer) + Emit(0xC3); // ret through [resume_rsp-8] + Debug.Assert(offset <= stubSize, "Guest context transfer stub exceeded its allocation."); uint oldProtect = 0; - if (!_hostMemory.Protect((ulong)code, stubSize, HostPageProtection.ReadExecute, out oldProtect)) + if (!VirtualProtect(code, stubSize, 32u, &oldProtect)) { - _hostMemory.Free((ulong)code); + VirtualFree(code, 0u, 32768u); return 0; } - _hostMemory.FlushInstructionCache((ulong)code, stubSize); + FlushInstructionCache(GetCurrentProcess(), code, stubSize); Volatile.Write(ref _guestContextTransferStub, (nint)code); return (nint)code; } @@ -1847,7 +1915,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe nint CreateImportHandlerTrampoline(int importIndex) { - void* ptr = (void*)_hostMemory.Allocate(0, 256u, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, 512u, 12288u, 64u); if (ptr == null) { return 0; @@ -1875,21 +1943,41 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[num++] = 82; ptr2[num++] = 86; ptr2[num++] = 87; - // sub rsp, 0x80 — reserve 8*16 bytes for the SysV variadic XMM save area - ptr2[num++] = 0x48; ptr2[num++] = 0x81; ptr2[num++] = 0xEC; - ptr2[num++] = 0x80; ptr2[num++] = 0x00; ptr2[num++] = 0x00; ptr2[num++] = 0x00; - // movdqu [rsp + i*0x10], xmm{i} for i = 0..7 (F3 0F 7F /r, SIB=0x24 base=rsp, disp8) - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x44; ptr2[num++] = 0x24; ptr2[num++] = 0x00; // xmm0 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x4C; ptr2[num++] = 0x24; ptr2[num++] = 0x10; // xmm1 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x54; ptr2[num++] = 0x24; ptr2[num++] = 0x20; // xmm2 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x5C; ptr2[num++] = 0x24; ptr2[num++] = 0x30; // xmm3 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x64; ptr2[num++] = 0x24; ptr2[num++] = 0x40; // xmm4 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x6C; ptr2[num++] = 0x24; ptr2[num++] = 0x50; // xmm5 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x74; ptr2[num++] = 0x24; ptr2[num++] = 0x60; // xmm6 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x7C; ptr2[num++] = 0x24; ptr2[num++] = 0x70; // xmm7 - // lea r12, [rsp + 0x80] — r12 = argpack base (the 12 pushed GP regs), past the XMM area - ptr2[num++] = 0x4C; ptr2[num++] = 0x8D; ptr2[num++] = 0xA4; ptr2[num++] = 0x24; - ptr2[num++] = 0x80; ptr2[num++] = 0x00; ptr2[num++] = 0x00; ptr2[num++] = 0x00; + // Preserve incoming guest RAX/AL and XMM0-XMM7 below the existing + // GPR argument pack. The original pack still begins with RDI and its + // return address remains at +0x60. + ptr2[num++] = 0x48; + ptr2[num++] = 0x81; + ptr2[num++] = 0xEC; + *(uint*)(ptr2 + num) = 0xB0; + num += 4; + ptr2[num++] = 0x48; + ptr2[num++] = 0x89; + ptr2[num++] = 0x04; + ptr2[num++] = 0x24; + // Preserve the remaining volatile guest machine context before any + // host call is made. libSceFiber's setjmp/longjmp contract includes + // R10/R11 and the x87/MXCSR control state. + ptr2[num++] = 0x4C; ptr2[num++] = 0x89; ptr2[num++] = 0x54; ptr2[num++] = 0x24; ptr2[num++] = 0x08; // mov [rsp+8],r10 + ptr2[num++] = 0x4C; ptr2[num++] = 0x89; ptr2[num++] = 0x5C; ptr2[num++] = 0x24; ptr2[num++] = 0x10; // mov [rsp+16],r11 + ptr2[num++] = 0x0F; ptr2[num++] = 0xAE; ptr2[num++] = 0x5C; ptr2[num++] = 0x24; ptr2[num++] = 0x18; // stmxcsr [rsp+24] + ptr2[num++] = 0xD9; ptr2[num++] = 0x7C; ptr2[num++] = 0x24; ptr2[num++] = 0x1C; // fnstcw [rsp+28] + for (var xmm = 0; xmm < 8; xmm++) + { + ptr2[num++] = 0xF3; + ptr2[num++] = 0x0F; + ptr2[num++] = 0x7F; + ptr2[num++] = (byte)(0x84 | (xmm << 3)); + ptr2[num++] = 0x24; + *(uint*)(ptr2 + num) = (uint)(0x30 + (xmm * 0x10)); + num += 4; + } + ptr2[num++] = 0x4C; + ptr2[num++] = 0x8D; + ptr2[num++] = 0xA4; + ptr2[num++] = 0x24; + *(uint*)(ptr2 + num) = 0xB0; + num += 4; ptr2[num++] = 72; ptr2[num++] = 131; ptr2[num++] = 236; @@ -1907,6 +1995,12 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[num++] = 131; ptr2[num++] = 196; ptr2[num++] = 40; + ptr2[num++] = 0x4C; + ptr2[num++] = 0x8D; + ptr2[num++] = 0xA4; + ptr2[num++] = 0x24; + *(uint*)(ptr2 + num) = 0xB0; + num += 4; ptr2[num++] = 73; ptr2[num++] = 137; ptr2[num++] = 195; @@ -1916,7 +2010,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[num++] = 72; ptr2[num++] = 131; ptr2[num++] = 236; - ptr2[num++] = 40; + ptr2[num++] = 56; + ptr2[num++] = 0x4C; ptr2[num++] = 0x89; ptr2[num++] = 0x64; ptr2[num++] = 0x24; ptr2[num++] = 0x28; // mov [rsp+0x28],r12 ptr2[num++] = 72; ptr2[num++] = 185; *(long*)(ptr2 + num) = _selfHandlePtr; @@ -1933,15 +2028,24 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I num += 8; ptr2[num++] = byte.MaxValue; ptr2[num++] = 208; + ptr2[num++] = 0x4C; ptr2[num++] = 0x8B; ptr2[num++] = 0x64; ptr2[num++] = 0x24; ptr2[num++] = 0x28; // mov r12,[rsp+0x28] ptr2[num++] = 72; ptr2[num++] = 131; ptr2[num++] = 196; - ptr2[num++] = 40; - // movdqu xmm0, [r12 - 0x80] — reload the return XMM0 the gateway wrote into the - // argpack's xmm0 save slot (float/double returns: powf/logf/wcstod). SysV/Win64 - // XMM regs are volatile across calls, so an unconditional reload is ABI-safe. - ptr2[num++] = 0xF3; ptr2[num++] = 0x41; ptr2[num++] = 0x0F; ptr2[num++] = 0x6F; - ptr2[num++] = 0x84; ptr2[num++] = 0x24; ptr2[num++] = 0x80; ptr2[num++] = 0xFF; ptr2[num++] = 0xFF; ptr2[num++] = 0xFF; + ptr2[num++] = 56; + // Materialize SysV vector return registers written by the managed HLE + // gateway before restoring the guest stack. + for (var xmm = 0; xmm < 2; xmm++) + { + ptr2[num++] = 0xF3; + ptr2[num++] = 0x41; + ptr2[num++] = 0x0F; + ptr2[num++] = 0x6F; + ptr2[num++] = (byte)(0x84 | (xmm << 3)); + ptr2[num++] = 0x24; + *(int*)(ptr2 + num) = -0x80 + (xmm * 0x10); + num += 4; + } ptr2[num++] = 76; ptr2[num++] = 137; ptr2[num++] = 228; @@ -1964,14 +2068,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[num++] = 65; ptr2[num++] = 95; ptr2[num++] = 195; - uint num2 = default(uint); - if (!_hostMemory.Protect((ulong)ptr, 256u, HostPageProtection.ReadExecute, out num2)) - { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for import dispatch stub at 0x{(nint)ptr:X16}"); - return 0; - } - _hostMemory.FlushInstructionCache((ulong)ptr, 256u); - return (nint)ptr; + Debug.Assert(num <= 512, "Import handler trampoline exceeded its allocation."); + uint num2 = default(uint); + VirtualProtect(ptr, 512u, 32u, &num2); + FlushInstructionCache(GetCurrentProcess(), ptr, 512u); + return (nint)ptr; } catch { @@ -1982,7 +2083,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe bool PatchImportStub(nint address, nint trampoline) { uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)address, 16u, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)address, 16u, 64u, &flNewProtect)) { Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for import stub at 0x{address:X16}"); return false; @@ -2002,8 +2103,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)address, 16u, flNewProtect, out flNewProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)address, 16u); + VirtualProtect((void*)address, 16u, flNewProtect, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, 16u); } } @@ -2013,7 +2114,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { if (importHandlerTrampoline != 0) { - _hostMemory.Free((ulong)importHandlerTrampoline); + VirtualFree((void*)importHandlerTrampoline, 0u, 32768u); } } _importHandlerTrampolines.Clear(); @@ -2024,7 +2125,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _tlsHandlerAddress = (nint)TryAllocateNearEntry(TlsHandlerRegionSize); if (_tlsHandlerAddress == 0) { - _tlsHandlerAddress = (nint)_hostMemory.Allocate(0, TlsHandlerRegionSize, HostPageProtection.ReadWriteExecute); + _tlsHandlerAddress = (nint)VirtualAlloc(null, TlsHandlerRegionSize, 12288u, 64u); } if (_tlsHandlerAddress == 0) { @@ -2080,18 +2181,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I tlsHandlerAddress[num++] = 0xC3; // ret _tlsPatchStubOffset = (num + 15) & ~15; uint num2 = default(uint); - if (!_hostMemory.Protect((ulong)(void*)_tlsHandlerAddress, TlsHandlerRegionSize, HostPageProtection.ReadExecute, out num2)) - { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for TLS handler at 0x{_tlsHandlerAddress:X16}"); - return; - } - _hostMemory.FlushInstructionCache((ulong)(void*)_tlsHandlerAddress, TlsHandlerRegionSize); + VirtualProtect((void*)_tlsHandlerAddress, TlsHandlerRegionSize, 32u, &num2); + FlushInstructionCache(GetCurrentProcess(), (void*)_tlsHandlerAddress, TlsHandlerRegionSize); Console.Error.WriteLine($"[LOADER][INFO] TLS handler at 0x{_tlsHandlerAddress:X16}"); } - private unsafe nint CreateUnresolvedReturnStub() + private unsafe static nint CreateUnresolvedReturnStub() { - void* ptr = (void*)_hostMemory.Allocate(0, 4096u, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, 4096u, 12288u, 64u); if (ptr == null) { return 0; @@ -2105,19 +2202,15 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[i] = 144; } uint num = default(uint); - if (!_hostMemory.Protect((ulong)ptr, 4096u, HostPageProtection.ReadExecute, out num)) - { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for unresolved return stub at 0x{(nint)ptr:X16}"); - return 0; - } - _hostMemory.FlushInstructionCache((ulong)ptr, 16u); + VirtualProtect(ptr, 4096u, 32u, &num); + FlushInstructionCache(GetCurrentProcess(), ptr, 16u); return (nint)ptr; } private unsafe nint CreateGuestReturnStub() { const uint stubSize = 256u; - void* ptr = (void*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, stubSize, 12288u, 4u); if (ptr == null) { return 0; @@ -2125,16 +2218,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I byte* code = (byte*)ptr; int offset = 0; - // TlsGetValue returns its TLS pointer in RAX. Preserve the guest return value - // above the 32-byte Windows shadow space while keeping the call site aligned. - EmitByte(code, ref offset, 0x48); // sub rsp, 0x30 + EmitByte(code, ref offset, 0x48); // sub rsp, 0x20 EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xEC); - EmitByte(code, ref offset, 0x30); - EmitByte(code, ref offset, 0x48); // mov [rsp+0x20], rax - EmitByte(code, ref offset, 0x89); - EmitByte(code, ref offset, 0x44); - EmitByte(code, ref offset, 0x24); EmitByte(code, ref offset, 0x20); EmitByte(code, ref offset, 0xB9); // mov ecx, tlsIndex EmitUInt32(code, ref offset, _hostRspSlotTlsIndex); @@ -2144,21 +2230,13 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I offset += sizeof(ulong); EmitByte(code, ref offset, 0xFF); // call rax EmitByte(code, ref offset, 0xD0); - EmitByte(code, ref offset, 0x49); // mov r11, rax - EmitByte(code, ref offset, 0x89); - EmitByte(code, ref offset, 0xC3); - EmitByte(code, ref offset, 0x48); // mov rax, [rsp+0x20] - EmitByte(code, ref offset, 0x8B); - EmitByte(code, ref offset, 0x44); - EmitByte(code, ref offset, 0x24); - EmitByte(code, ref offset, 0x20); - EmitByte(code, ref offset, 0x48); // add rsp, 0x30 + EmitByte(code, ref offset, 0x48); // add rsp, 0x20 EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xC4); - EmitByte(code, ref offset, 0x30); - EmitByte(code, ref offset, 0x49); // mov rsp, [r11] + EmitByte(code, ref offset, 0x20); + EmitByte(code, ref offset, 0x48); // mov rsp, [rax] EmitByte(code, ref offset, 0x8B); - EmitByte(code, ref offset, 0x23); + EmitByte(code, ref offset, 0x20); EmitHostNonvolatileXmmRestore(code, ref offset); EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x5F); EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x5E); @@ -2171,12 +2249,104 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I EmitByte(code, ref offset, 0xC3); uint oldProtect = default; - if (!_hostMemory.Protect((ulong)ptr, stubSize, HostPageProtection.ReadExecute, out oldProtect)) + if (!VirtualProtect(ptr, stubSize, 32u, &oldProtect)) { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for guest return stub at 0x{(nint)ptr:X16}"); + VirtualFree(ptr, 0u, 32768u); return 0; } - _hostMemory.FlushInstructionCache((ulong)ptr, (nuint)offset); + FlushInstructionCache(GetCurrentProcess(), ptr, (nuint)offset); + return (nint)ptr; + } + + private unsafe nint CreateExceptionHandlerTrampoline(nint managedHandler) + { + const uint stubSize = 256u; + void* ptr = VirtualAlloc(null, stubSize, 12288u, 64u); + if (ptr == null) + { + return 0; + } + + byte* code = (byte*)ptr; + int offset = 0; + EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x54); // push r12 + EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x55); // push r13 + EmitByte(code, ref offset, 0x49); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xE4); // mov r12, rsp + EmitByte(code, ref offset, 0x49); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xCD); // mov r13, rcx + EmitByte(code, ref offset, 0x65); EmitByte(code, ref offset, 0x48); // mov rax, gs:[8] + EmitByte(code, ref offset, 0x8B); EmitByte(code, ref offset, 0x04); EmitByte(code, ref offset, 0x25); + EmitUInt32(code, ref offset, 8u); + EmitByte(code, ref offset, 0x49); EmitByte(code, ref offset, 0x39); EmitByte(code, ref offset, 0xC4); // cmp r12, rax + EmitByte(code, ref offset, 0x0F); EmitByte(code, ref offset, 0x83); // jae guestStack + int aboveStackJump = offset; + EmitUInt32(code, ref offset, 0u); + EmitByte(code, ref offset, 0x65); EmitByte(code, ref offset, 0x48); // mov rax, gs:[0x10] + EmitByte(code, ref offset, 0x8B); EmitByte(code, ref offset, 0x04); EmitByte(code, ref offset, 0x25); + EmitUInt32(code, ref offset, 0x10u); + EmitByte(code, ref offset, 0x49); EmitByte(code, ref offset, 0x39); EmitByte(code, ref offset, 0xC4); // cmp r12, rax + EmitByte(code, ref offset, 0x0F); EmitByte(code, ref offset, 0x82); // jb guestStack + int belowStackJump = offset; + EmitUInt32(code, ref offset, 0u); + + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xEC); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xE9); // mov rcx, r13 + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0xB8); + *(nint*)(code + offset) = managedHandler; + offset += sizeof(nint); + EmitByte(code, ref offset, 0xFF); EmitByte(code, ref offset, 0xD0); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xC4); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0xE9); + int hostRestoreJump = offset; + EmitUInt32(code, ref offset, 0u); + + int guestStackOffset = offset; + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xEC); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0xB9); + EmitUInt32(code, ref offset, _hostRspSlotTlsIndex); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0xB8); + *(nint*)(code + offset) = _tlsGetValueAddress; + offset += sizeof(nint); + EmitByte(code, ref offset, 0xFF); EmitByte(code, ref offset, 0xD0); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xC4); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x85); EmitByte(code, ref offset, 0xC0); // test rax, rax + EmitByte(code, ref offset, 0x0F); EmitByte(code, ref offset, 0x84); + int missingTlsJump = offset; + EmitUInt32(code, ref offset, 0u); + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x8B); EmitByte(code, ref offset, 0x18); // mov r11, [rax] + EmitByte(code, ref offset, 0x4D); EmitByte(code, ref offset, 0x85); EmitByte(code, ref offset, 0xDB); // test r11, r11 + EmitByte(code, ref offset, 0x0F); EmitByte(code, ref offset, 0x84); + int missingHostStackJump = offset; + EmitUInt32(code, ref offset, 0u); + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xDC); // mov rsp, r11 + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xEC); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xE9); // mov rcx, r13 + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0xB8); + *(nint*)(code + offset) = managedHandler; + offset += sizeof(nint); + EmitByte(code, ref offset, 0xFF); EmitByte(code, ref offset, 0xD0); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xC4); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0xE9); + int guestRestoreJump = offset; + EmitUInt32(code, ref offset, 0u); + + int passThroughOffset = offset; + EmitByte(code, ref offset, 0x31); EmitByte(code, ref offset, 0xC0); // xor eax, eax + int restoreOffset = offset; + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xE4); // mov rsp, r12 + EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x5D); + EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x5C); + EmitByte(code, ref offset, 0xC3); + + *(int*)(code + aboveStackJump) = guestStackOffset - (aboveStackJump + sizeof(int)); + *(int*)(code + belowStackJump) = guestStackOffset - (belowStackJump + sizeof(int)); + *(int*)(code + hostRestoreJump) = restoreOffset - (hostRestoreJump + sizeof(int)); + *(int*)(code + missingTlsJump) = passThroughOffset - (missingTlsJump + sizeof(int)); + *(int*)(code + missingHostStackJump) = passThroughOffset - (missingHostStackJump + sizeof(int)); + *(int*)(code + guestRestoreJump) = restoreOffset - (guestRestoreJump + sizeof(int)); + + uint oldProtect = default; + VirtualProtect(ptr, stubSize, 32u, &oldProtect); + FlushInstructionCache(GetCurrentProcess(), ptr, (nuint)offset); return (nint)ptr; } @@ -2198,7 +2368,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return null; } - private unsafe bool TryAllocAt(ulong baseAddress, long signedDelta, nuint size, out void* memory) + private unsafe static bool TryAllocAt(ulong baseAddress, long signedDelta, nuint size, out void* memory) { memory = null; ulong num; @@ -2219,7 +2389,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } num = baseAddress - num2; } - void* ptr = (void*)_hostMemory.Allocate(num, size, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc((void*)num, size, 12288u, 64u); if (ptr == null) { return false; @@ -2230,15 +2400,18 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe void PatchTlsPatterns() { - const ulong MaxScanBytes = 33554432uL; + // Large Gen5 executables can keep valid code well past the first 32 MiB. + // Astro Bot, for example, has an FS:[0] TLS load near +0x70A0000. + const ulong MaxScanBytes = 134217728uL; ulong num = _entryPoint; ulong num2 = num + MaxScanBytes; int num3 = 0; int num4 = 0; int num9 = 0; + int sse4aPatchCount = 0; while (num < num2) { - if (!_hostMemory.Query(num, out var lpBuffer) || lpBuffer.RegionSize == 0) + if (VirtualQuery((void*)num, out var lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0 || lpBuffer.RegionSize == 0) { num += 4096uL; continue; @@ -2249,8 +2422,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { num6 = num2; } - uint num7 = lpBuffer.RawProtection & 0xFF; - bool flag = lpBuffer.State == HostRegionState.Committed && (lpBuffer.RawProtection & PAGE_GUARD) == 0 && num7 != PAGE_NOACCESS; + uint num7 = lpBuffer.Protect & 0xFF; + bool flag = lpBuffer.State == 4096 && (lpBuffer.Protect & PAGE_GUARD) == 0 && num7 != PAGE_NOACCESS; bool flag2 = num7 == PAGE_EXECUTE || num7 == 32 || num7 == 64 || num7 == PAGE_EXECUTE_WRITECOPY; if (flag && flag2 && num6 > num5 + MinTlsPatchInstructionBytes) { @@ -2268,6 +2441,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { num9++; } + else if (remainingBytes >= 12 && TryPatchSse4aExtrqBlend(address, ptr + i)) + { + sse4aPatchCount++; + } else if (TryPatchStackCanaryInstruction(address, ptr + i)) { num4++; @@ -2276,7 +2453,51 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } num = num6 > num ? num6 : num + 4096uL; } - Console.Error.WriteLine($"[LOADER][INFO] Patched {num3} TLS loads, {num9} TLS stores, {num4} stack-canary accesses"); + Console.Error.WriteLine($"[LOADER][INFO] Patched {num3} TLS loads, {num9} TLS stores, {num4} stack-canary accesses, {sse4aPatchCount} SSE4a EXTRQ blends"); + } + + private unsafe bool TryPatchSse4aExtrqBlend(nint address, byte* source) + { + // Rosetta does not implement AMD SSE4a EXTRQ. This exact sequence masks + // xmm2 to its low 40 bits, then copies the resulting second dword into + // xmm0. PEXTRB/PINSRD provides the same observable result in 12 bytes: + // extract source byte 4 and insert the zero-extended value into lane 1. + ReadOnlySpan pattern = + [ + 0x66, 0x0F, 0x78, 0xC2, 0x28, 0x00, + 0xC4, 0xE3, 0x79, 0x02, 0xC2, 0x02, + ]; + for (var i = 0; i < pattern.Length; i++) + { + if (source[i] != pattern[i]) + { + return false; + } + } + + ReadOnlySpan replacement = + [ + 0x66, 0x0F, 0x3A, 0x14, 0xD0, 0x04, + 0x66, 0x0F, 0x3A, 0x22, 0xC0, 0x01, + ]; + uint oldProtect = 0; + if (!VirtualProtect((void*)address, (nuint)replacement.Length, 64u, &oldProtect)) + { + return false; + } + try + { + for (var i = 0; i < replacement.Length; i++) + { + ((byte*)address)[i] = replacement[i]; + } + } + finally + { + VirtualProtect((void*)address, (nuint)replacement.Length, oldProtect, &oldProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, (nuint)replacement.Length); + } + return true; } private unsafe bool IsPatternMatch(byte* ptr, byte[] pattern) @@ -2335,7 +2556,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } byte b5 = (byte)(0xC0 | ((num4 & 7) << 3) | (num4 & 7)); uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)address, (nuint)num2, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)address, (nuint)num2, 64u, &flNewProtect)) { return false; } @@ -2351,8 +2572,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)address, (nuint)num2, flNewProtect, out flNewProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)address, (nuint)num2); + VirtualProtect((void*)address, (nuint)num2, flNewProtect, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, (nuint)num2); } return true; } @@ -2419,7 +2640,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe bool PatchTlsLoadInstruction(nint address, int instructionLength, int destinationRegister) { uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)address, (nuint)instructionLength, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)address, (nuint)instructionLength, 64u, &flNewProtect)) { return false; } @@ -2453,8 +2674,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)address, (nuint)instructionLength, flNewProtect, out flNewProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)address, (nuint)instructionLength); + VirtualProtect((void*)address, (nuint)instructionLength, flNewProtect, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, (nuint)instructionLength); } } @@ -2506,12 +2727,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr[num2++] = 144; } uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)num, 32u, HostPageProtection.ReadExecute, out flNewProtect)) - { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for TLS store helper at 0x{num:X16}"); - return 0; - } - _hostMemory.FlushInstructionCache((ulong)(void*)num, 32u); + VirtualProtect((void*)num, 32u, 32u, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)num, 32u); return num; } @@ -2530,7 +2747,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I nint result = _tlsHandlerAddress + _tlsPatchStubOffset; _tlsPatchStubOffset += num; uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)result, (nuint)num, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)result, (nuint)num, 64u, &flNewProtect)) { return 0; } @@ -2544,7 +2761,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return false; } uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)address, (nuint)instructionLength, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)address, (nuint)instructionLength, 64u, &flNewProtect)) { return false; } @@ -2565,17 +2782,17 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)address, (nuint)instructionLength, flNewProtect, out flNewProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)address, (nuint)instructionLength); + VirtualProtect((void*)address, (nuint)instructionLength, flNewProtect, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, (nuint)instructionLength); } return true; } private unsafe void TryPreReservePrtAperture(ulong baseAddress, ulong size) { - if (_hostMemory.Query(baseAddress, out var lpBuffer) && lpBuffer.State != HostRegionState.Free) + if (VirtualQuery((void*)baseAddress, out var lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) != 0 && lpBuffer.State != 65536) { - Console.Error.WriteLine($"[LOADER][INFO] PRT aperture at 0x{baseAddress:X16} already in use (state=0x{lpBuffer.RawState:X}), will use lazy-commit"); + Console.Error.WriteLine($"[LOADER][INFO] PRT aperture at 0x{baseAddress:X16} already in use (state=0x{lpBuffer.State:X}), will use lazy-commit"); return; } ulong num = baseAddress; @@ -2587,7 +2804,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { ulong val = num2 - num; num5 = (nuint)Math.Min(2097152uL, val); - if (_hostMemory.Reserve(num, num5, HostPageProtection.ReadWrite) != 0) + void* ptr = VirtualAlloc((void*)num, num5, 8192u, 4u); + if (ptr != null) { num3++; } @@ -2609,7 +2827,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I int num8 = 0; for (; num6 < num7; num6 += 2097152) { - if (_hostMemory.Commit(num6, 2097152u, HostPageProtection.ReadWrite)) + void* ptr2 = VirtualAlloc((void*)num6, 2097152u, 4096u, 4u); + if (ptr2 != null) { num8++; } @@ -2706,11 +2925,44 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I $"entry=0x{thread.EntryPoint:X16} arg=0x{thread.Argument:X16} priority={thread.Priority} " + $"host_priority={MapGuestThreadPriority(thread.Priority)} affinity=0x{thread.AffinityMask:X}"); Pump(creatorContext, "pthread_create"); + // Pump is suppressed while another cooperative dispatch is active. The + // background dispatcher would eventually observe this thread, but an + // immediate authoritative drain avoids making thread creation depend on + // the approximate ready-count polling hint. + DispatchReadyGuestThreads(); return true; } public bool SupportsGuestContextTransfer => true; + public void RegisterGuestThreadContext(ulong threadHandle, CpuContext context) + { + if (threadHandle == 0) + { + return; + } + + lock (_guestThreadGate) + { + _currentExternalGuestThreadHandle = threadHandle; + if (_guestThreads.ContainsKey(threadHandle)) + { + return; + } + + if (_externalGuestThreads.TryGetValue(threadHandle, out var existing)) + { + existing.Context = context; + return; + } + + _externalGuestThreads[threadHandle] = new ExternalGuestThreadState + { + Context = context, + }; + } + } + public bool TryJoinThread( CpuContext callerContext, ulong threadHandle, @@ -2767,21 +3019,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I if (hostThread is not null && !ReferenceEquals(hostThread, Thread.CurrentThread)) { - // The handle is published before the host thread starts. - if ((hostThread.ThreadState & System.Threading.ThreadState.Unstarted) != 0) - { - Thread.Sleep(1); - continue; - } - - try - { - hostThread.Join(joinPollMilliseconds); - } - catch (ThreadStateException) - { - Thread.Sleep(joinPollMilliseconds); - } + hostThread.Join(1); } else { @@ -2801,10 +3039,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I public void Pump(CpuContext callerContext, string reason) { _ = callerContext; - if (_guestTeardownRequested) - { - return; - } var runSynchronously = string.Equals(reason, "entry_return", StringComparison.Ordinal); WakeExpiredBlockedGuestThreads(); if (Volatile.Read(ref _readyGuestThreadCount) == 0) @@ -2822,17 +3056,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I GuestThreadState? thread = null; using (LockGate("Pump.dequeue")) { - while (_readyGuestThreads.Count > 0) - { - var candidate = _readyGuestThreads.Dequeue(); - Interlocked.Decrement(ref _readyGuestThreadCount); - if (candidate.State == GuestThreadRunState.Ready) - { - thread = candidate; - thread.State = GuestThreadRunState.Running; - break; - } - } + _ = TryClaimReadyGuestThreadLocked(out thread); } if (thread == null) { @@ -2845,17 +3069,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I continue; } - var hostThread = new Thread(() => RunGuestThread(thread, reason)) - { - IsBackground = true, - Name = $"SharpEmu-{thread.Name}", - Priority = MapGuestThreadPriority(thread.Priority), - }; - using (LockGate("Pump.bind_host")) - { - thread.HostThread = hostThread; - } - hostThread.Start(); + ScheduleGuestThreadExecution(thread, reason); } } finally @@ -2964,7 +3178,49 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I thread.HasBlockedContinuation = true; thread.BlockWakeKey = wakeKey; thread.BlockWaiter = waiter; + thread.BlockResumeHandler = null; + thread.BlockWakeHandler = null; thread.BlockDeadlineTimestamp = blockDeadlineTimestamp; + TraceFocusedContinuation( + "register", + guestThreadHandle, + continuation, + wakeKey); + } + } + + private void RegisterBlockedGuestThreadContinuation( + ulong guestThreadHandle, + GuestCpuContinuation continuation, + string wakeKey, + Func? resumeHandler, + Func? wakeHandler, + long blockDeadlineTimestamp) + { + if (guestThreadHandle == 0 || continuation.Rip < 65536 || continuation.Rsp == 0) + { + return; + } + + using (LockGate("RegisterBlockedContinuation")) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out var thread)) + { + return; + } + + thread.BlockedContinuation = continuation; + thread.HasBlockedContinuation = true; + thread.BlockWakeKey = wakeKey; + thread.BlockWaiter = null; + thread.BlockResumeHandler = resumeHandler; + thread.BlockWakeHandler = wakeHandler; + thread.BlockDeadlineTimestamp = blockDeadlineTimestamp; + TraceFocusedContinuation( + "register", + guestThreadHandle, + continuation, + wakeKey); } } @@ -3079,6 +3335,32 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I string reason, out string? error) { + return TryCallGuestFunction( + callerContext, + entryPoint, + arg0, + arg1, + 0, + stackAddress, + stackSize, + reason, + out _, + out error); + } + + public bool TryCallGuestFunction( + CpuContext callerContext, + ulong entryPoint, + ulong arg0, + ulong arg1, + ulong arg2, + ulong stackAddress, + ulong stackSize, + string reason, + out ulong returnValue, + out string? error) + { + returnValue = 0; error = null; if (entryPoint < 65536) { @@ -3093,6 +3375,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ulong callbackStackBase; ulong callbackStackSize; + var usesCachedCallbackStack = false; if (stackAddress != 0 && stackSize >= 0x100) { callbackStackBase = stackAddress; @@ -3100,10 +3383,37 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } else { - if (!TryMapGuestThreadRegion(virtualMemory, GuestThreadStackBaseAddress, GuestThreadStackSize, ProgramHeaderFlags.Read | ProgramHeaderFlags.Write, out callbackStackBase, out error)) + var callbackDepth = _nestedGuestCallbackDepth; + _nestedGuestCallbackStacks ??= []; + if (callbackDepth < _nestedGuestCallbackStacks.Count && + ReferenceEquals(_nestedGuestCallbackStacks[callbackDepth].Memory, virtualMemory)) { - return false; + callbackStackBase = _nestedGuestCallbackStacks[callbackDepth].Base; } + else + { + if (!TryMapGuestThreadRegion( + virtualMemory, + GuestThreadStackBaseAddress, + GuestThreadStackSize, + ProgramHeaderFlags.Read | ProgramHeaderFlags.Write, + out callbackStackBase, + out error)) + { + return false; + } + + if (callbackDepth < _nestedGuestCallbackStacks.Count) + { + _nestedGuestCallbackStacks[callbackDepth] = (virtualMemory, callbackStackBase); + } + else + { + _nestedGuestCallbackStacks.Add((virtualMemory, callbackStackBase)); + } + } + + usesCachedCallbackStack = true; callbackStackSize = GuestThreadStackSize; } @@ -3119,7 +3429,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I context[CpuRegister.Rsp] = AlignDown(callbackStackBase + callbackStackSize, 16) - sizeof(ulong); context[CpuRegister.Rdi] = arg0; context[CpuRegister.Rsi] = arg1; - context[CpuRegister.Rdx] = 0; + context[CpuRegister.Rdx] = arg2; context[CpuRegister.Rcx] = 0; context[CpuRegister.R8] = 0; context[CpuRegister.R9] = 0; @@ -3128,26 +3438,169 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I error = "failed to initialize guest callback stack"; return false; } + if (usesCachedCallbackStack) + { + _nestedGuestCallbackDepth++; + } var previousLastError = LastError; try { LastError = null; var exitReason = ExecuteGuestThreadEntry(context, entryPoint, reason, out var callbackReason); + if (exitReason == GuestNativeCallExitReason.Blocked && + !ResumeBlockedNestedGuestCallback(context, reason, ref exitReason, ref callbackReason)) + { + error = callbackReason ?? LastError ?? "guest callback could not resume after blocking"; + return false; + } if (exitReason is GuestNativeCallExitReason.Exception or GuestNativeCallExitReason.ForcedExit) { error = callbackReason ?? LastError ?? "guest callback failed"; return false; } + returnValue = context[CpuRegister.Rax]; return true; } finally { + if (usesCachedCallbackStack) + { + _nestedGuestCallbackDepth--; + } LastError = previousLastError; } } + /// + /// Completes a nested guest callback which blocked in an HLE import. The + /// outer guest entry is still executing managed HLE code, so returning a + /// successful callback result here would abandon the callback continuation + /// and let a noreturn operation such as pthread_exit unwind through live + /// libc cleanup state. Temporarily expose the owning guest thread as blocked, + /// let the normal scheduler wake it, and resume the callback continuation on + /// this executor until it either returns or fails. + /// + private bool ResumeBlockedNestedGuestCallback( + CpuContext callbackContext, + string reason, + ref GuestNativeCallExitReason exitReason, + ref string? callbackReason) + { + var guestThreadHandle = GuestThreadExecution.CurrentGuestThreadHandle; + if (guestThreadHandle == 0) + { + callbackReason = $"nested guest callback '{reason}' blocked without a schedulable guest thread"; + exitReason = GuestNativeCallExitReason.Exception; + return false; + } + + while (exitReason == GuestNativeCallExitReason.Blocked && !ActiveForcedGuestExit) + { + GuestThreadState? owner; + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out owner) || + !owner.HasBlockedContinuation) + { + callbackReason = + $"nested guest callback '{reason}' blocked without a captured continuation"; + exitReason = GuestNativeCallExitReason.Exception; + return false; + } + + owner.State = GuestThreadRunState.Blocked; + owner.BlockReason = callbackReason ?? reason; + if (owner.BlockWakeHandler is not null && owner.BlockWakeHandler()) + { + owner.State = GuestThreadRunState.Ready; + owner.BlockReason = null; + owner.BlockWakeHandler = null; + owner.BlockDeadlineTimestamp = 0; + } + } + if (_logGuestThreads) + { + Console.Error.WriteLine( + $"[LOADER][INFO] nested_callback.block name='{owner!.Name}' callback='{reason}' " + + $"wake={owner.BlockWakeKey ?? "none"} continuation=0x{owner.BlockedContinuation.Rip:X16}"); + } + + GuestCpuContinuation continuation = default; + Func? resumeHandler = null; + while (!ActiveForcedGuestExit) + { + WakeExpiredBlockedGuestThreads(); + var ready = false; + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out owner)) + { + callbackReason = + $"nested guest callback '{reason}' lost its owning guest thread"; + exitReason = GuestNativeCallExitReason.Exception; + return false; + } + + if (owner.State == GuestThreadRunState.Ready && owner.HasBlockedContinuation) + { + continuation = owner.BlockedContinuation; + owner.BlockedContinuation = default; + owner.HasBlockedContinuation = false; + owner.BlockWakeKey = null; + resumeHandler = owner.BlockResumeHandler; + owner.BlockResumeHandler = null; + owner.BlockWakeHandler = null; + owner.BlockDeadlineTimestamp = 0; + owner.BlockReason = null; + owner.State = GuestThreadRunState.Running; + ready = true; + } + } + + if (ready) + { + break; + } + + Thread.Sleep(1); + } + + if (ActiveForcedGuestExit) + { + callbackReason = LastError ?? $"nested guest callback '{reason}' was forced to exit"; + exitReason = GuestNativeCallExitReason.ForcedExit; + return false; + } + + if (resumeHandler is not null) + { + continuation = continuation with { Rax = unchecked((ulong)(long)resumeHandler()) }; + } + if (_logGuestThreads) + { + Console.Error.WriteLine( + $"[LOADER][INFO] nested_callback.resume thread=0x{guestThreadHandle:X16} callback='{reason}' " + + $"continuation=0x{continuation.Rip:X16}"); + } + + exitReason = ExecuteBlockedGuestThreadContinuation( + callbackContext, + continuation, + reason, + out callbackReason); + } + + if (exitReason == GuestNativeCallExitReason.Blocked && ActiveForcedGuestExit) + { + callbackReason = LastError ?? $"nested guest callback '{reason}' was forced to exit"; + exitReason = GuestNativeCallExitReason.ForcedExit; + } + + return exitReason == GuestNativeCallExitReason.Returned; + } + public bool TryCallGuestContinuation( CpuContext callerContext, GuestCpuContinuation continuation, @@ -3174,6 +3627,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I Rflags = continuation.Rflags == 0 ? 0x202UL : continuation.Rflags, FsBase = callerContext.FsBase != 0 ? callerContext.FsBase : (continuation.FsBase != 0 ? continuation.FsBase : fallbackTlsBase), GsBase = callerContext.GsBase != 0 ? callerContext.GsBase : (continuation.GsBase != 0 ? continuation.GsBase : fallbackTlsBase), + FpuControlWord = continuation.FpuControlWord == 0 ? (ushort)0x037F : continuation.FpuControlWord, + Mxcsr = continuation.Mxcsr == 0 ? 0x1F80u : continuation.Mxcsr, }; context[CpuRegister.Rax] = continuation.Rax; @@ -3185,6 +3640,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I context[CpuRegister.Rdi] = continuation.Rdi; context[CpuRegister.R8] = continuation.R8; context[CpuRegister.R9] = continuation.R9; + context[CpuRegister.R10] = continuation.R10; + context[CpuRegister.R11] = continuation.R11; context[CpuRegister.R12] = continuation.R12; context[CpuRegister.R13] = continuation.R13; context[CpuRegister.R14] = continuation.R14; @@ -3302,6 +3759,512 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return true; } + public bool TryRaiseGuestException( + CpuContext callerContext, + ulong threadHandle, + ulong handler, + int exceptionType, + out string? error) + { + error = null; + var logGuestExceptions = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_EXCEPTIONS"), + "1", + StringComparison.Ordinal); + if (threadHandle == 0 || handler < 65536 || exceptionType is < 0 or >= 128) + { + error = "invalid guest exception delivery request"; + return false; + } + + GuestThreadState target; + GuestThreadRunState savedState; + bool savedExecutorActive; + string? savedBlockReason; + bool savedHasBlockedContinuation; + GuestCpuContinuation savedBlockedContinuation; + string? savedBlockWakeKey; + Func? savedBlockResumeHandler; + Func? savedBlockWakeHandler; + long savedBlockDeadlineTimestamp; + ulong exceptionStackBase; + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(threadHandle, out target!)) + { + if (!_externalGuestThreads.TryGetValue(threadHandle, out var external)) + { + error = $"unknown guest exception target 0x{threadHandle:X16}"; + return false; + } + + if (external.ExceptionStackBase == 0) + { + string? mapError = null; + if (!TryGetVirtualMemory(external.Context, out var virtualMemory) || + !TryMapGuestThreadRegion( + virtualMemory, + GuestThreadStackBaseAddress, + GuestThreadStackSize, + ProgramHeaderFlags.Read | ProgramHeaderFlags.Write, + out var stackBase, + out mapError)) + { + error = mapError ?? "external guest context has no virtual memory"; + return false; + } + + external.ExceptionStackBase = stackBase; + } + + if (_pendingGuestExceptions.ContainsKey(threadHandle)) + { + return true; + } + if (_activeGuestExceptionDeliveries.Contains(threadHandle)) + { + // Preserve one signal raised while the previous handler is + // unwinding. Unity can begin its next stop-the-world cycle in + // that window; treating the new raise as part of the old delivery + // strands the collector waiting for an acknowledgement. + _pendingGuestExceptions[threadHandle] = new PendingGuestException( + handler, + exceptionType, + external.ExceptionStackBase); + return true; + } + + // A primary/external executor is already running guest code on its + // own host thread. Running its signal handler concurrently on a new + // managed thread corrupts the worker's control state. Queue the + // request and let that exact executor consume it at its next HLE + // boundary, where the original guest thread is safely paused. + _pendingGuestExceptions[threadHandle] = new PendingGuestException( + handler, + exceptionType, + external.ExceptionStackBase); + if (logGuestExceptions) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.queued " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} mode=external"); + } + return true; + } + + if (target.State is GuestThreadRunState.Exited or GuestThreadRunState.Faulted) + { + error = $"guest exception target 0x{threadHandle:X16} is no longer running"; + return false; + } + + if (target.ExceptionStackBase == 0) + { + string? mapError = null; + if (!TryGetVirtualMemory(target.Context, out var virtualMemory) || + !TryMapGuestThreadRegion( + virtualMemory, + GuestThreadStackBaseAddress, + GuestThreadStackSize, + ProgramHeaderFlags.Read | ProgramHeaderFlags.Write, + out var mappedExceptionStack, + out mapError)) + { + error = mapError ?? "guest thread context has no virtual memory"; + return false; + } + + target.ExceptionStackBase = mappedExceptionStack; + } + exceptionStackBase = target.ExceptionStackBase; + + if (target.State != GuestThreadRunState.Blocked || target.ExecutorActive) + { + if (_pendingGuestExceptions.ContainsKey(threadHandle) || + _activeGuestExceptionDeliveries.Contains(threadHandle)) + { + return true; + } + if (target.ExceptionDeliveryActive) + { + _pendingGuestExceptions[threadHandle] = new PendingGuestException( + handler, + exceptionType, + exceptionStackBase); + return true; + } + + _pendingGuestExceptions[threadHandle] = new PendingGuestException( + handler, + exceptionType, + exceptionStackBase); + if (logGuestExceptions) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.queued " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} " + + $"mode=scheduled state={target.State} executor={target.ExecutorActive}"); + } + return true; + } + + // A parked cooperative thread has no active executor, so its saved + // continuation can be handled immediately on a temporary executor. + if (target.ExceptionDeliveryActive) + { + return true; + } + + savedState = target.State; + savedExecutorActive = target.ExecutorActive; + savedBlockReason = target.BlockReason; + savedHasBlockedContinuation = target.HasBlockedContinuation; + savedBlockedContinuation = target.BlockedContinuation; + savedBlockWakeKey = target.BlockWakeKey; + savedBlockResumeHandler = target.BlockResumeHandler; + savedBlockWakeHandler = target.BlockWakeHandler; + savedBlockDeadlineTimestamp = target.BlockDeadlineTimestamp; + + target.State = GuestThreadRunState.Running; + target.ExecutorActive = true; + target.ExceptionDeliveryActive = true; + target.BlockReason = null; + target.HasBlockedContinuation = false; + target.BlockedContinuation = default; + target.BlockWakeKey = null; + target.BlockResumeHandler = null; + target.BlockWakeHandler = null; + target.BlockDeadlineTimestamp = 0; + } + + const ulong exceptionContextSize = 0x500; + const ulong callbackStackOffset = 0x1000; + const ulong callbackStackSize = 0xF000; + var exceptionContextAddress = exceptionStackBase + 0x100; + var guestExceptionCallback = 0UL; + if (handler >= 0x210) + { + _ = target.Context.TryReadUInt64(handler - 0x210 + 0xC020, out guestExceptionCallback); + } + if (!TryWriteGuestExceptionContext( + target.Context, + exceptionContextAddress, + savedHasBlockedContinuation ? savedBlockedContinuation : default, + exceptionContextSize)) + { + lock (_guestThreadGate) + { + RestoreInterruptedGuestThread(); + } + error = "failed to write guest exception context"; + return false; + } + + if (logGuestExceptions) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.delivery_enter " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} mode=parked " + + $"rip=0x{savedBlockedContinuation.Rip:X16} rsp=0x{savedBlockedContinuation.Rsp:X16} " + + $"rbp=0x{savedBlockedContinuation.Rbp:X16} rbx=0x{savedBlockedContinuation.Rbx:X16} " + + $"r12=0x{savedBlockedContinuation.R12:X16} r13=0x{savedBlockedContinuation.R13:X16} " + + $"r14=0x{savedBlockedContinuation.R14:X16} r15=0x{savedBlockedContinuation.R15:X16} " + + $"stack=0x{exceptionStackBase:X16} callback=0x{guestExceptionCallback:X16}"); + } + + void RestoreInterruptedGuestThread() + { + target.State = savedState; + target.ExecutorActive = savedExecutorActive; + target.ExceptionDeliveryActive = false; + target.BlockReason = savedBlockReason; + target.HasBlockedContinuation = savedHasBlockedContinuation; + target.BlockedContinuation = savedBlockedContinuation; + target.BlockWakeKey = savedBlockWakeKey; + target.BlockResumeHandler = savedBlockResumeHandler; + target.BlockWakeHandler = savedBlockWakeHandler; + target.BlockDeadlineTimestamp = savedBlockDeadlineTimestamp; + + // A condition/event wake can arrive while the parked thread is + // temporarily marked Running for signal delivery. WakeBlockedThreads + // cannot claim it in that state, so re-check the restored wait before + // releasing scheduler ownership. Without this handoff a completed + // pthread wait remains parked forever after a GC suspension races it. + if (target.State == GuestThreadRunState.Blocked && + target.HasBlockedContinuation && + target.BlockWakeHandler is not null && + target.BlockWakeHandler()) + { + target.State = GuestThreadRunState.Ready; + target.BlockReason = null; + target.BlockWakeHandler = null; + target.BlockDeadlineTimestamp = 0; + _readyGuestThreads.Enqueue(target); + Interlocked.Increment(ref _readyGuestThreadCount); + } + } + + void DeliverException() + { + var previousGuestThreadHandle = GuestThreadExecution.EnterGuestThread(threadHandle); + var previousGuestThreadState = _activeGuestThreadState; + _activeGuestThreadState = target; + var deliverySucceeded = false; + string? deliveryError = null; + var deliveryStarted = Stopwatch.GetTimestamp(); + try + { + deliverySucceeded = TryCallGuestFunction( + target.Context, + handler, + unchecked((ulong)exceptionType), + exceptionContextAddress, + exceptionStackBase + callbackStackOffset, + callbackStackSize, + $"kernel exception 0x{exceptionType:X2}", + out deliveryError); + if (!deliverySucceeded) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] Guest exception delivery failed: " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} " + + $"error={deliveryError ?? "unknown"}"); + } + } + finally + { + PendingGuestException? followUp = null; + if (logGuestExceptions) + { + var recordAddress = FindGuestExceptionThreadRecord( + target.Context, + guestExceptionCallback, + threadHandle); + var recordedStack = 0UL; + var registeredStackBound = 0UL; + if (recordAddress != 0) + { + _ = target.Context.TryReadUInt64(recordAddress + 0x100, out registeredStackBound); + _ = target.Context.TryReadUInt64(recordAddress + 0x18, out recordedStack); + } + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.delivery_exit " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} " + + $"success={deliverySucceeded} error={deliveryError ?? "none"} " + + $"elapsed_ms={Stopwatch.GetElapsedTime(deliveryStarted).TotalMilliseconds:F3} " + + $"record=0x{recordAddress:X16} stack_bound=0x{registeredStackBound:X16} " + + $"recorded_rsp=0x{recordedStack:X16}"); + } + _activeGuestThreadState = previousGuestThreadState; + GuestThreadExecution.RestoreGuestThread(previousGuestThreadHandle); + lock (_guestThreadGate) + { + RestoreInterruptedGuestThread(); + if (target.State == GuestThreadRunState.Blocked && + !target.ExecutorActive && + _pendingGuestExceptions.Remove(threadHandle, out var queued)) + { + followUp = queued; + } + } + if (followUp is { } pendingFollowUp && + !TryRaiseGuestException( + target.Context, + threadHandle, + pendingFollowUp.Handler, + pendingFollowUp.ExceptionType, + out var followUpError)) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] Guest exception follow-up delivery failed: " + + $"target=0x{threadHandle:X16} type=0x{pendingFollowUp.ExceptionType:X2} " + + $"error={followUpError ?? "unknown"}"); + } + } + } + + // A real sceKernelRaiseException interrupts the target pthread and runs + // its signal handler on that same native thread. Parked cooperative + // guest threads have no active call frame to interrupt, but their + // persistent execution runner is idle and preserves the native-thread + // identity/TLS that Unity's stop-the-world collector registered. Using + // an unrelated temporary host thread makes the suspension acknowledge + // appear valid while publishing roots from the wrong native execution + // context, which lets live IL2CPP delegates be reclaimed. + GuestExecutionRunner deliveryRunner; + lock (_guestThreadGate) + { + deliveryRunner = target.ExecutionRunner ??= new GuestExecutionRunner( + target.ThreadHandle, + target.Name, + MapGuestThreadPriority(target.Priority)); + } + deliveryRunner.Schedule(DeliverException); + return true; + } + + private static ulong FindGuestExceptionThreadRecord( + CpuContext context, + ulong callback, + ulong threadHandle) + { + if (callback < 65536) + { + return 0; + } + + // Unity's suspend callback uses a 256-bucket table at this fixed + // image-relative offset from the callback entry. Each node stores the + // pthread handle at +0x08 and the next pointer at +0x00. + var tableAddress = callback + 0x102E8B0; + for (var bucket = 0; bucket < 256; bucket++) + { + if (!context.TryReadUInt64(tableAddress + unchecked((ulong)bucket * 8), out var node)) + { + return 0; + } + + for (var depth = 0; node >= 65536 && depth < 1024; depth++) + { + if (!context.TryReadUInt64(node + 0x08, out var registeredThread)) + { + break; + } + if (registeredThread == threadHandle) + { + return node; + } + if (!context.TryReadUInt64(node, out node)) + { + break; + } + } + } + + return 0; + } + + private void DeliverPendingGuestExceptionAtSafePoint( + CpuContext currentContext, + GuestCpuContinuation interruptedContinuation) + { + var threadHandle = GuestThreadExecution.CurrentGuestThreadHandle; + if (threadHandle == 0) + { + threadHandle = _currentExternalGuestThreadHandle; + } + PendingGuestException pending; + lock (_guestThreadGate) + { + if (threadHandle == 0) + { + return; + } + + if (!_pendingGuestExceptions.Remove(threadHandle, out pending)) + { + return; + } + + _activeGuestExceptionDeliveries.Add(threadHandle); + } + + const ulong exceptionContextSize = 0x500; + const ulong callbackStackOffset = 0x1000; + const ulong callbackStackSize = 0xF000; + var exceptionContextAddress = pending.ExceptionStackBase + 0x100; + try + { + if (!TryWriteGuestExceptionContext( + currentContext, + exceptionContextAddress, + interruptedContinuation, + exceptionContextSize)) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] Guest exception safe-point context write failed: " + + $"target=0x{threadHandle:X16} type=0x{pending.ExceptionType:X2}"); + return; + } + + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_EXCEPTIONS"), + "1", + StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.safe_point_enter " + + $"target=0x{threadHandle:X16} type=0x{pending.ExceptionType:X2} " + + $"rip=0x{interruptedContinuation.Rip:X16}"); + } + + if (!TryCallGuestFunction( + currentContext, + pending.Handler, + unchecked((ulong)pending.ExceptionType), + exceptionContextAddress, + pending.ExceptionStackBase + callbackStackOffset, + callbackStackSize, + $"kernel exception 0x{pending.ExceptionType:X2} safe point", + out var callbackError)) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] Guest exception safe-point delivery failed: " + + $"target=0x{threadHandle:X16} type=0x{pending.ExceptionType:X2} " + + $"error={callbackError ?? "unknown"}"); + } + } + finally + { + lock (_guestThreadGate) + { + _activeGuestExceptionDeliveries.Remove(threadHandle); + } + } + } + + private static bool TryWriteGuestExceptionContext( + CpuContext context, + ulong address, + GuestCpuContinuation continuation, + ulong size) + { + var bytes = new byte[checked((int)size)]; + void Write64(int offset, ulong value) => + BinaryPrimitives.WriteUInt64LittleEndian(bytes.AsSpan(offset, sizeof(ulong)), value); + + var hasContinuation = continuation.Rip >= 65536 && continuation.Rsp != 0; + // Orbis ucontext_t has a 0x10-byte signal mask and 0x30 bytes of + // private fields before its amd64 mcontext. These offsets match the + // platform ABI used by libScePs5Util and Unity's Boehm GC. Supplying a + // bare mcontext here makes the collector miss live register roots. + const int mcontext = 0x40; + Write64(mcontext + 0x08, hasContinuation ? continuation.Rdi : context[CpuRegister.Rdi]); + Write64(mcontext + 0x10, hasContinuation ? continuation.Rsi : context[CpuRegister.Rsi]); + Write64(mcontext + 0x18, hasContinuation ? continuation.Rdx : context[CpuRegister.Rdx]); + Write64(mcontext + 0x20, hasContinuation ? continuation.Rcx : context[CpuRegister.Rcx]); + Write64(mcontext + 0x28, hasContinuation ? continuation.R8 : context[CpuRegister.R8]); + Write64(mcontext + 0x30, hasContinuation ? continuation.R9 : context[CpuRegister.R9]); + Write64(mcontext + 0x38, hasContinuation ? continuation.Rax : context[CpuRegister.Rax]); + Write64(mcontext + 0x40, hasContinuation ? continuation.Rbx : context[CpuRegister.Rbx]); + Write64(mcontext + 0x48, hasContinuation ? continuation.Rbp : context[CpuRegister.Rbp]); + Write64(mcontext + 0x50, hasContinuation ? continuation.R10 : context[CpuRegister.R10]); + Write64(mcontext + 0x58, hasContinuation ? continuation.R11 : context[CpuRegister.R11]); + Write64(mcontext + 0x60, hasContinuation ? continuation.R12 : context[CpuRegister.R12]); + Write64(mcontext + 0x68, hasContinuation ? continuation.R13 : context[CpuRegister.R13]); + Write64(mcontext + 0x70, hasContinuation ? continuation.R14 : context[CpuRegister.R14]); + Write64(mcontext + 0x78, hasContinuation ? continuation.R15 : context[CpuRegister.R15]); + var rip = hasContinuation ? continuation.Rip : context.Rip; + var rsp = hasContinuation ? continuation.Rsp : context[CpuRegister.Rsp]; + Write64(mcontext + 0xA0, rip); + Write64(mcontext + 0xB0, hasContinuation ? continuation.Rflags : 0); + Write64(mcontext + 0xB8, rsp); + Write64(mcontext + 0xC8, 0x480); // sizeof(Orbis mcontext_t) + Write64(mcontext + 0x440, hasContinuation ? continuation.FsBase : context.FsBase); + Write64(mcontext + 0x448, hasContinuation ? continuation.GsBase : context.GsBase); + return context.Memory.TryWrite(address, bytes); + } + private void TraceGuestContext(string message) { if (_logGuestContext) @@ -3333,61 +4296,35 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I continuationThread.Join(); } - /// - /// Parks every guest worker thread before the caller starts freeing executable - /// memory. Workers unwind to the host at their next import dispatch (see the - /// teardown check in DispatchImport); this waits for their host threads to - /// finish within . Returns false when at least one - /// worker is still running — the caller must then leak its executable stubs - /// rather than free memory a live thread may still execute. - /// - private bool RequestGuestThreadTeardown(int timeoutMs) - { - _guestTeardownRequested = true; - Thread[] hostThreads; - using (LockGate("RequestGuestThreadTeardown")) - { - _readyGuestThreads.Clear(); - Interlocked.Exchange(ref _readyGuestThreadCount, 0); - hostThreads = _guestThreads.Values - .Select(static thread => thread.HostThread) - .Where(static host => host is not null && host != Thread.CurrentThread && host.IsAlive) - .Cast() - .ToArray(); - } - - var deadline = Environment.TickCount64 + timeoutMs; - var allStopped = true; - foreach (var host in hostThreads) - { - var remaining = (int)Math.Max(1L, deadline - Environment.TickCount64); - if (!host.Join(remaining) && host.IsAlive) - { - allStopped = false; - Console.Error.WriteLine( - $"[LOADER][WARN] Guest worker host thread '{host.Name}' still running after teardown wait."); - } - } - - return allStopped; - } - private void ClearGuestThreads() { - GuestContinuationRunner[] runners; - using (LockGate("ClearGuestThreads")) + GuestContinuationRunner[] continuationRunners; + GuestExecutionRunner[] executionRunners; + lock (_guestThreadGate) { - runners = _guestThreads.Values + continuationRunners = _guestThreads.Values .Select(static thread => thread.ContinuationRunner) .Where(static runner => runner is not null) .Cast() .ToArray(); + executionRunners = _guestThreads.Values + .Select(static thread => thread.ExecutionRunner) + .Where(static runner => runner is not null) + .Cast() + .ToArray(); _readyGuestThreads.Clear(); Interlocked.Exchange(ref _readyGuestThreadCount, 0); _guestThreads.Clear(); + _externalGuestThreads.Clear(); + _pendingGuestExceptions.Clear(); + _activeGuestExceptionDeliveries.Clear(); } - foreach (var runner in runners) + foreach (var runner in continuationRunners) + { + runner.Dispose(); + } + foreach (var runner in executionRunners) { runner.Dispose(); } @@ -3440,6 +4377,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I Priority = request.Priority, AffinityMask = request.AffinityMask, Context = context, + StackBase = stackBase, + StackSize = GuestThreadStackSize, State = GuestThreadRunState.Ready, }; error = null; @@ -3471,7 +4410,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I out ulong mappedBase, out string? error) { - for (int i = 0; i < GuestThreadRegionSlotCount; i++) + for (int i = 0; i < GuestThreadRegionSlots; i++) { var candidateBase = baseAddress - ((ulong)i * GuestThreadRegionStride); if (!IsGuestThreadRegionFree(virtualMemory, candidateBase, size)) @@ -3505,7 +4444,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I out ulong tlsBase, out string? error) { - for (int i = 0; i < GuestThreadRegionSlotCount; i++) + for (int i = 0; i < GuestThreadRegionSlots; i++) { var candidateBase = GuestThreadTlsBaseAddress - ((ulong)i * GuestThreadRegionStride); var mappedBase = candidateBase - GuestThreadTlsPrefixSize; @@ -3571,11 +4510,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private static bool InitializeGuestThreadTls(CpuContext context, ulong tlsBase, ulong threadHandle) { - return context.TryWriteUInt64(tlsBase - 0xF0, 0) && - context.TryWriteUInt64(tlsBase + 0x00, tlsBase) && - context.TryWriteUInt64(tlsBase + 0x10, threadHandle) && - context.TryWriteUInt64(tlsBase + 0x28, 0xC0DEC0DECAFEBABEUL) && - context.TryWriteUInt64(tlsBase + 0x60, tlsBase); + if (!context.TryWriteUInt64(tlsBase - 0xF0, 0) || + !context.TryWriteUInt64(tlsBase + 0x00, tlsBase) || + !context.TryWriteUInt64(tlsBase + 0x10, threadHandle) || + !context.TryWriteUInt64(tlsBase + 0x28, 0xC0DEC0DECAFEBA00UL) || + !context.TryWriteUInt64(tlsBase + 0x60, tlsBase)) + { + return false; + } + + // Seed initialized thread-locals into the static TLS block below the + // thread pointer so per-thread TLS matches the main thread. + SharpEmu.HLE.GuestTlsTemplate.SeedThreadBlock(context, tlsBase); + return true; } private static ThreadPriority MapGuestThreadPriority(int priority) @@ -3600,7 +4547,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return; } - if (!_hostThreading.TrySetCurrentThreadAffinity((nuint)hostAffinityMask) && _logGuestThreads) + if (SetThreadAffinityMask(GetCurrentThread(), (nuint)hostAffinityMask) == 0 && _logGuestThreads) { Console.Error.WriteLine( $"[LOADER][WARN] Failed to set guest thread affinity guest=0x{guestAffinityMask:X} " + @@ -3643,21 +4590,70 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return hostAffinityMask; } + public bool TrySetGuestThreadPriority(ulong guestThreadHandle, int guestPriority) + { + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out var thread)) + { + return false; + } + + thread.Priority = guestPriority; + var host = thread.HostThread; + if (host is not null && host.IsAlive) + { + try + { + host.Priority = MapGuestThreadPriority(guestPriority); + } + catch (Exception exception) when (exception is ThreadStateException or InvalidOperationException) + { + // The thread may have exited between the alive check and + // the assignment; the stored priority still takes effect + // if it is ever restarted. + } + } + + return true; + } + } + + public bool TrySetGuestThreadAffinity(ulong guestThreadHandle, ulong affinityMask) + { + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out var thread)) + { + return false; + } + + thread.AffinityMask = affinityMask; + // A running thread applies its own affinity via + // ApplyGuestThreadAffinity; cross-thread affinity is not portable, + // so the new mask takes effect on the thread's next scheduling. + return true; + } + } + private void RunGuestThread(GuestThreadState thread, string reason) { + lock (_guestThreadGate) + { + if (!thread.ExecutorActive) + { + throw new InvalidOperationException( + $"Guest thread '{thread.Name}' started without scheduler executor ownership."); + } + + thread.HostThread = Thread.CurrentThread; + } var previousLastError = LastError; var previousGuestThreadHandle = GuestThreadExecution.EnterGuestThread(thread.ThreadHandle); var previousGuestThreadState = _activeGuestThreadState; ApplyGuestThreadAffinity(thread.AffinityMask); - Volatile.Write(ref thread.HostThreadId, unchecked((int)_hostThreading.CurrentThreadId)); + Volatile.Write(ref thread.HostThreadId, unchecked((int)GetCurrentThreadId())); _activeGuestThreadState = thread; - if (LogThreadMode) - { - _threadModeCycleId = Interlocked.Increment(ref _threadModeCycleCounter); - TraceThreadMode( - $"cycle_start name='{thread.Name}' guest=0x{thread.ThreadHandle:X16} reason={reason} " + - $"rsp_slot=0x{(ulong)_hostThreading.GetTlsValue(_hostRspSlotTlsIndex):X}"); - } try { LastError = null; @@ -3701,6 +4697,12 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I case GuestNativeCallExitReason.Returned: thread.ExitValue = thread.Context[CpuRegister.Rax]; thread.State = GuestThreadRunState.Exited; + if (_logGuestThreads) + Console.Error.WriteLine( + $"[LOADER][INFO] Guest thread exited: name='{thread.Name}' " + + $"exitValue=0x{thread.ExitValue:X16} imports={Interlocked.Read(ref thread.ImportCount)} " + + $"lastNid={Volatile.Read(ref thread.LastImportNid) ?? "none"} " + + $"entry=0x{thread.EntryPoint:X16} ret=0x{Volatile.Read(ref thread.LastReturnRip):X16}"); break; case GuestNativeCallExitReason.Blocked: thread.State = GuestThreadRunState.Blocked; @@ -3730,17 +4732,18 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - if (LogThreadMode) - { - TraceThreadMode( - $"cycle_end name='{thread.Name}' state={thread.State} " + - $"imports={Interlocked.Read(ref thread.ImportCount)} " + - $"rsp_slot=0x{(ulong)_hostThreading.GetTlsValue(_hostRspSlotTlsIndex):X}"); - } _activeGuestThreadState = previousGuestThreadState; Volatile.Write(ref thread.HostThreadId, 0); GuestThreadExecution.RestoreGuestThread(previousGuestThreadHandle); LastError = previousLastError; + lock (_guestThreadGate) + { + if (ReferenceEquals(thread.HostThread, Thread.CurrentThread)) + { + thread.HostThread = null; + } + thread.ExecutorActive = false; + } } } @@ -3750,6 +4753,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I string name, out string? reason) { + TraceFocusedContinuation( + "execute", + GuestThreadExecution.CurrentGuestThreadHandle, + continuation, + name); ApplyGuestContinuation(context, continuation); return ExecuteGuestContinuationEntry( context, @@ -3759,6 +4767,30 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I out reason); } + private static void TraceFocusedContinuation( + string operation, + ulong threadHandle, + GuestCpuContinuation continuation, + string detail) + { + if (!string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_FOCUSED_CONTINUATION"), + "1", + StringComparison.Ordinal) || + continuation.Rsp < 0x00006FFFAC000000UL || + continuation.Rsp >= 0x00006FFFAC200000UL) + { + return; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] focused_continuation.{operation} " + + $"thread=0x{threadHandle:X16} rip=0x{continuation.Rip:X16} " + + $"rsp=0x{continuation.Rsp:X16} slot=0x{continuation.ReturnSlotAddress:X16} " + + $"rbp=0x{continuation.Rbp:X16} rbx=0x{continuation.Rbx:X16} " + + $"detail={detail}"); + } + private static void ApplyGuestContinuation(CpuContext context, GuestCpuContinuation continuation) { context.Rip = continuation.Rip; @@ -3781,11 +4813,17 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I context[CpuRegister.Rdi] = continuation.Rdi; context[CpuRegister.R8] = continuation.R8; context[CpuRegister.R9] = continuation.R9; + context[CpuRegister.R10] = continuation.R10; + context[CpuRegister.R11] = continuation.R11; context[CpuRegister.R12] = continuation.R12; context[CpuRegister.R13] = continuation.R13; context[CpuRegister.R14] = continuation.R14; context[CpuRegister.R15] = continuation.R15; context[CpuRegister.Rsp] = continuation.Rsp; + context.FpuControlWord = continuation.FpuControlWord == 0 + ? (ushort)0x037F + : continuation.FpuControlWord; + context.Mxcsr = continuation.Mxcsr == 0 ? 0x1F80u : continuation.Mxcsr; } private unsafe GuestNativeCallExitReason ExecuteGuestThreadEntry(CpuContext context, ulong entryPoint, string name, out string? reason) @@ -3797,12 +4835,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return GuestNativeCallExitReason.Exception; } const uint stubSize = 512u; - void* ptr = (void*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, stubSize, 12288u, 4u); if (ptr == null) { reason = "failed to allocate executable memory for guest thread stub"; return GuestNativeCallExitReason.Exception; } + void* hostRspStorage = NativeMemory.Alloc((nuint)sizeof(ulong)); + if (hostRspStorage == null) + { + VirtualFree(ptr, 0u, 32768u); + reason = "failed to allocate writable host-RSP storage for guest thread stub"; + return GuestNativeCallExitReason.Exception; + } var previousActiveBackend = _activeExecutionBackend; var previousActiveContext = _activeCpuContext; var previousSentinel = _activeEntryReturnSentinelRip; @@ -3810,285 +4855,144 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I var previousForcedExit = _activeForcedGuestExit; var previousYieldRequested = _activeGuestThreadYieldRequested; var previousYieldReason = _activeGuestThreadYieldReason; - nint previousHostRspSlotValue = _hostThreading.GetTlsValue(_hostRspSlotTlsIndex); - if (LogThreadMode) - { - TraceThreadMode( - $"entry_setup name='{name}' entry=0x{entryPoint:X16} stub=0x{(ulong)ptr:X16} " + - $"guest_rsp=0x{context[CpuRegister.Rsp]:X16} rsp_slot_prev=0x{(ulong)previousHostRspSlotValue:X}"); - } - try - { - _activeExecutionBackend = this; - _activeCpuContext = context; - _activeEntryReturnSentinelRip = 0; - _activeGuestReturnSlotAddress = 0; - _activeForcedGuestExit = false; - _activeGuestThreadYieldRequested = false; - _activeGuestThreadYieldReason = null; - BindTlsBase(context); - byte* ptr2 = (byte*)ptr; - ulong hostRspSlot = (ulong)ptr + stubSize - 16uL; - int offset = 0; - ptr2[offset++] = 83; - ptr2[offset++] = 85; - ptr2[offset++] = 87; - ptr2[offset++] = 86; - ptr2[offset++] = 65; - ptr2[offset++] = 84; - ptr2[offset++] = 65; - ptr2[offset++] = 85; - ptr2[offset++] = 65; - ptr2[offset++] = 86; - ptr2[offset++] = 65; - ptr2[offset++] = 87; - EmitHostNonvolatileXmmSave(ptr2, ref offset); - ptr2[offset++] = 73; - ptr2[offset++] = 186; - *(ulong*)(ptr2 + offset) = hostRspSlot; - offset += 8; - ptr2[offset++] = 73; - ptr2[offset++] = 137; - ptr2[offset++] = 34; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rsp]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 196; - ptr2[offset++] = 72; - ptr2[offset++] = 131; - ptr2[offset++] = 236; - ptr2[offset++] = 8; - ptr2[offset++] = 72; - ptr2[offset++] = 189; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rbp]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rdi]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 199; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rsi]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 198; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rdx]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 194; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rcx]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 193; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = entryPoint; - offset += 8; - ptr2[offset++] = byte.MaxValue; - ptr2[offset++] = 208; - int sentinelOffset = offset + 4; - ptr2[offset++] = 72; - ptr2[offset++] = 131; - ptr2[offset++] = 196; - ptr2[offset++] = 8; - ptr2[offset++] = 73; - ptr2[offset++] = 186; - *(ulong*)(ptr2 + offset) = hostRspSlot; - offset += 8; - ptr2[offset++] = 73; - ptr2[offset++] = 139; - ptr2[offset++] = 34; - EmitHostNonvolatileXmmRestore(ptr2, ref offset); - ptr2[offset++] = 65; - ptr2[offset++] = 95; - ptr2[offset++] = 65; - ptr2[offset++] = 94; - ptr2[offset++] = 65; - ptr2[offset++] = 93; - ptr2[offset++] = 65; - ptr2[offset++] = 92; - ptr2[offset++] = 94; - ptr2[offset++] = 95; - ptr2[offset++] = 93; - ptr2[offset++] = 91; - ptr2[offset++] = 195; - ulong sentinel = (ulong)ptr + (ulong)sentinelOffset; - ActiveEntryReturnSentinelRip = (ulong)_guestReturnStub; - _activeGuestReturnSlotAddress = context[CpuRegister.Rsp] - 16uL; - if (!context.TryWriteUInt64(context[CpuRegister.Rsp], sentinel)) - { - reason = $"failed to patch guest thread return sentinel at 0x{context[CpuRegister.Rsp]:X16}"; - return GuestNativeCallExitReason.Exception; - } - uint oldProtect = default(uint); - if (!_hostMemory.Protect((ulong)ptr, stubSize, HostPageProtection.ReadWriteExecute, out oldProtect)) - { - reason = $"VirtualProtect failed for guest thread entry stub at 0x{(nint)ptr:X16}"; - return GuestNativeCallExitReason.Exception; - } - _hostMemory.FlushInstructionCache((ulong)ptr, stubSize); - ActiveGuestThreadYieldRequested = false; - ActiveGuestThreadYieldReason = null; - var nativeReturn = RunGuestEntryStub(ptr, hostRspSlot); - if (ActiveGuestThreadYieldRequested) - { - reason = ActiveGuestThreadYieldReason ?? "guest thread blocked"; - return GuestNativeCallExitReason.Blocked; - } - if (ActiveForcedGuestExit) - { - reason = LastError ?? "guest thread forced exit"; - return GuestNativeCallExitReason.ForcedExit; - } - reason = $"returned 0x{nativeReturn:X8}"; - return GuestNativeCallExitReason.Returned; - } - catch (AccessViolationException ex) - { - reason = "access violation: " + ex.Message; - return GuestNativeCallExitReason.Exception; - } - catch (Exception ex) - { - reason = ex.GetType().Name + ": " + ex.Message; - return GuestNativeCallExitReason.Exception; - } - finally - { - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); - RestoreActiveExecutionThread( - previousActiveBackend, - previousActiveContext, - previousSentinel, - previousReturnSlotAddress, - previousForcedExit, - previousYieldRequested, - previousYieldReason); - _hostMemory.Free((ulong)ptr); - } -} - - private unsafe GuestNativeCallExitReason ExecuteGuestContinuationEntry( - CpuContext context, - ulong entryPoint, - ulong returnSlotAddress, - string name, - out string? reason) - { - reason = null; - if (context[CpuRegister.Rsp] == 0) - { - reason = "guest thread stack pointer is zero"; - return GuestNativeCallExitReason.Exception; - } - const uint stubSize = 512u; - void* ptr = (void*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); - if (ptr == null) - { - reason = "failed to allocate executable memory for guest thread stub"; - return GuestNativeCallExitReason.Exception; - } - var previousActiveBackend = _activeExecutionBackend; - var previousActiveContext = _activeCpuContext; - var previousSentinel = _activeEntryReturnSentinelRip; - var previousReturnSlotAddress = _activeGuestReturnSlotAddress; - var previousForcedExit = _activeForcedGuestExit; - var previousYieldRequested = _activeGuestThreadYieldRequested; - var previousYieldReason = _activeGuestThreadYieldReason; - nint previousHostRspSlotValue = _hostThreading.GetTlsValue(_hostRspSlotTlsIndex); - if (LogThreadMode) - { - TraceThreadMode( - $"continuation_setup name='{name}' resume=0x{entryPoint:X16} stub=0x{(ulong)ptr:X16} " + - $"guest_rsp=0x{context[CpuRegister.Rsp]:X16} rsp_slot_prev=0x{(ulong)previousHostRspSlotValue:X}"); - } + nint previousHostRspSlotValue = TlsGetValue(_hostRspSlotTlsIndex); try { _activeExecutionBackend = this; _activeCpuContext = context; _activeEntryReturnSentinelRip = 0; - _activeGuestReturnSlotAddress = returnSlotAddress; + _activeGuestReturnSlotAddress = 0; _activeForcedGuestExit = false; _activeGuestThreadYieldRequested = false; _activeGuestThreadYieldReason = null; BindTlsBase(context); byte* ptr2 = (byte*)ptr; - ulong hostRspSlot = (ulong)ptr + stubSize - 16uL; + // Rosetta does not reliably permit a generated x86 thunk to write data + // in the same page from which it is currently executing, even when the + // mapping reports PAGE_EXECUTE_READWRITE. Keep mutable transition state + // in a separate writable allocation. + ulong hostRspSlot = (ulong)hostRspStorage; int offset = 0; - - void Emit(byte value) => ptr2[offset++] = value; - void EmitU64(ulong value) - { - *(ulong*)(ptr2 + offset) = value; - offset += sizeof(ulong); - } - void EmitMovR64Imm(byte rex, byte opcode, ulong value) - { - Emit(rex); - Emit(opcode); - EmitU64(value); - } - - Emit(0x53); // push rbx - Emit(0x55); // push rbp - Emit(0x57); // push rdi - Emit(0x56); // push rsi - Emit(0x41); Emit(0x54); // push r12 - Emit(0x41); Emit(0x55); // push r13 - Emit(0x41); Emit(0x56); // push r14 - Emit(0x41); Emit(0x57); // push r15 + ptr2[offset++] = 83; + ptr2[offset++] = 85; + ptr2[offset++] = 87; + ptr2[offset++] = 86; + ptr2[offset++] = 65; + ptr2[offset++] = 84; + ptr2[offset++] = 65; + ptr2[offset++] = 85; + ptr2[offset++] = 65; + ptr2[offset++] = 86; + ptr2[offset++] = 65; + ptr2[offset++] = 87; EmitHostNonvolatileXmmSave(ptr2, ref offset); - EmitMovR64Imm(0x49, 0xBA, hostRspSlot); // mov r10, hostRspSlot - Emit(0x49); Emit(0x89); Emit(0x22); // mov [r10], rsp - EmitMovR64Imm(0x48, 0xB8, context[CpuRegister.Rsp]); // mov rax, guest rsp - Emit(0x48); Emit(0x89); Emit(0xC4); // mov rsp, rax - EmitMovR64Imm(0x48, 0xBB, context[CpuRegister.Rbx]); // mov rbx, imm64 - EmitMovR64Imm(0x48, 0xBD, context[CpuRegister.Rbp]); // mov rbp, imm64 - EmitMovR64Imm(0x48, 0xBF, context[CpuRegister.Rdi]); // mov rdi, imm64 - EmitMovR64Imm(0x48, 0xBE, context[CpuRegister.Rsi]); // mov rsi, imm64 - EmitMovR64Imm(0x48, 0xBA, context[CpuRegister.Rdx]); // mov rdx, imm64 - EmitMovR64Imm(0x48, 0xB9, context[CpuRegister.Rcx]); // mov rcx, imm64 - EmitMovR64Imm(0x49, 0xB8, context[CpuRegister.R8]); // mov r8, imm64 - EmitMovR64Imm(0x49, 0xB9, context[CpuRegister.R9]); // mov r9, imm64 - EmitMovR64Imm(0x49, 0xBC, context[CpuRegister.R12]); // mov r12, imm64 - EmitMovR64Imm(0x49, 0xBD, context[CpuRegister.R13]); // mov r13, imm64 - EmitMovR64Imm(0x49, 0xBE, context[CpuRegister.R14]); // mov r14, imm64 - EmitMovR64Imm(0x49, 0xBF, context[CpuRegister.R15]); // mov r15, imm64 - EmitMovR64Imm(0x48, 0xB8, context[CpuRegister.Rax]); // mov rax, imm64 - EmitMovR64Imm(0x49, 0xBB, entryPoint); // mov r11, entryPoint - Emit(0x41); Emit(0xFF); Emit(0xE3); // jmp r11 + ptr2[offset++] = 73; + ptr2[offset++] = 186; + *(ulong*)(ptr2 + offset) = hostRspSlot; + offset += 8; + ptr2[offset++] = 73; + ptr2[offset++] = 137; + ptr2[offset++] = 34; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rsp]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 196; + ptr2[offset++] = 72; + ptr2[offset++] = 131; + ptr2[offset++] = 236; + ptr2[offset++] = 8; + ptr2[offset++] = 72; + ptr2[offset++] = 189; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rbp]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rdi]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 199; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rsi]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 198; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rdx]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 194; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rcx]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 193; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = entryPoint; + offset += 8; + ptr2[offset++] = byte.MaxValue; + ptr2[offset++] = 208; + int sentinelOffset = offset + 4; + ptr2[offset++] = 72; + ptr2[offset++] = 131; + ptr2[offset++] = 196; + ptr2[offset++] = 8; + ptr2[offset++] = 73; + ptr2[offset++] = 186; + *(ulong*)(ptr2 + offset) = hostRspSlot; + offset += 8; + ptr2[offset++] = 73; + ptr2[offset++] = 139; + ptr2[offset++] = 34; + EmitHostNonvolatileXmmRestore(ptr2, ref offset); + ptr2[offset++] = 65; + ptr2[offset++] = 95; + ptr2[offset++] = 65; + ptr2[offset++] = 94; + ptr2[offset++] = 65; + ptr2[offset++] = 93; + ptr2[offset++] = 65; + ptr2[offset++] = 92; + ptr2[offset++] = 94; + ptr2[offset++] = 95; + ptr2[offset++] = 93; + ptr2[offset++] = 91; + ptr2[offset++] = 195; + ulong sentinel = (ulong)ptr + (ulong)sentinelOffset; ActiveEntryReturnSentinelRip = (ulong)_guestReturnStub; - if (returnSlotAddress == 0 || !context.TryWriteUInt64(returnSlotAddress, (ulong)_guestReturnStub)) + _activeGuestReturnSlotAddress = context[CpuRegister.Rsp] - 16uL; + if (!context.TryWriteUInt64(context[CpuRegister.Rsp], sentinel)) { - reason = $"failed to patch guest continuation return slot at 0x{returnSlotAddress:X16}"; + reason = $"failed to patch guest thread return sentinel at 0x{context[CpuRegister.Rsp]:X16}"; return GuestNativeCallExitReason.Exception; } uint oldProtect = default(uint); - if (!_hostMemory.Protect((ulong)ptr, stubSize, HostPageProtection.ReadWriteExecute, out oldProtect)) + if (!VirtualProtect(ptr, stubSize, 32u, &oldProtect)) { - reason = $"VirtualProtect failed for guest continuation stub at 0x{(nint)ptr:X16}"; + reason = "failed to seal guest thread stub execute-read"; + return GuestNativeCallExitReason.Exception; + } + FlushInstructionCache(GetCurrentProcess(), ptr, stubSize); + if (!TlsSetValue(_hostRspSlotTlsIndex, (nint)hostRspSlot)) + { + reason = "failed to bind host-RSP storage for guest thread stub"; return GuestNativeCallExitReason.Exception; } - _hostMemory.FlushInstructionCache((ulong)ptr, stubSize); ActiveGuestThreadYieldRequested = false; ActiveGuestThreadYieldReason = null; - try { - var nativeReturn = RunGuestEntryStub(ptr, hostRspSlot); + var nativeReturn = CallNativeEntry(ptr); if (ActiveGuestThreadYieldRequested) { reason = ActiveGuestThreadYieldReason ?? "guest thread blocked"; @@ -4115,7 +5019,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); + TlsSetValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); RestoreActiveExecutionThread( previousActiveBackend, previousActiveContext, @@ -4124,7 +5028,176 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I previousForcedExit, previousYieldRequested, previousYieldReason); - _hostMemory.Free((ulong)ptr); + NativeMemory.Free(hostRspStorage); + VirtualFree(ptr, 0u, 32768u); + } + } + + private unsafe GuestNativeCallExitReason ExecuteGuestContinuationEntry( + CpuContext context, + ulong entryPoint, + ulong returnSlotAddress, + string name, + out string? reason) + { + reason = null; + if (context[CpuRegister.Rsp] == 0) + { + reason = "guest thread stack pointer is zero"; + return GuestNativeCallExitReason.Exception; + } + const uint stubSize = 512u; + void* ptr = VirtualAlloc(null, stubSize, 12288u, 4u); + if (ptr == null) + { + reason = "failed to allocate executable memory for guest thread stub"; + return GuestNativeCallExitReason.Exception; + } + void* hostRspStorage = NativeMemory.Alloc((nuint)sizeof(ulong)); + if (hostRspStorage == null) + { + VirtualFree(ptr, 0u, 32768u); + reason = "failed to allocate writable host-RSP storage for guest continuation stub"; + return GuestNativeCallExitReason.Exception; + } + var previousActiveBackend = _activeExecutionBackend; + var previousActiveContext = _activeCpuContext; + var previousSentinel = _activeEntryReturnSentinelRip; + var previousReturnSlotAddress = _activeGuestReturnSlotAddress; + var previousForcedExit = _activeForcedGuestExit; + var previousYieldRequested = _activeGuestThreadYieldRequested; + var previousYieldReason = _activeGuestThreadYieldReason; + nint previousHostRspSlotValue = TlsGetValue(_hostRspSlotTlsIndex); + try + { + _activeExecutionBackend = this; + _activeCpuContext = context; + _activeEntryReturnSentinelRip = 0; + _activeGuestReturnSlotAddress = returnSlotAddress; + _activeForcedGuestExit = false; + _activeGuestThreadYieldRequested = false; + _activeGuestThreadYieldReason = null; + BindTlsBase(context); + byte* ptr2 = (byte*)ptr; + ulong hostRspSlot = (ulong)hostRspStorage; + int offset = 0; + + void Emit(byte value) => ptr2[offset++] = value; + void EmitU64(ulong value) + { + *(ulong*)(ptr2 + offset) = value; + offset += sizeof(ulong); + } + void EmitMovR64Imm(byte rex, byte opcode, ulong value) + { + Emit(rex); + Emit(opcode); + EmitU64(value); + } + + Emit(0x53); // push rbx + Emit(0x55); // push rbp + Emit(0x57); // push rdi + Emit(0x56); // push rsi + Emit(0x41); Emit(0x54); // push r12 + Emit(0x41); Emit(0x55); // push r13 + Emit(0x41); Emit(0x56); // push r14 + Emit(0x41); Emit(0x57); // push r15 + EmitHostNonvolatileXmmSave(ptr2, ref offset); + // Restore the fiber's floating-point control environment before + // abandoning the host stack. This path is used when a blocked guest + // continuation migrates to another managed worker. + Emit(0x48); Emit(0x83); Emit(0xEC); Emit(0x08); // sub rsp,8 + Emit(0xC7); Emit(0x04); Emit(0x24); // mov dword [rsp],imm32 + *(uint*)(ptr2 + offset) = context.Mxcsr; offset += sizeof(uint); + Emit(0x0F); Emit(0xAE); Emit(0x14); Emit(0x24); // ldmxcsr [rsp] + Emit(0x66); Emit(0xC7); Emit(0x04); Emit(0x24); // mov word [rsp],imm16 + *(ushort*)(ptr2 + offset) = context.FpuControlWord; offset += sizeof(ushort); + Emit(0xD9); Emit(0x2C); Emit(0x24); // fldcw [rsp] + Emit(0x48); Emit(0x83); Emit(0xC4); Emit(0x08); // add rsp,8 + EmitMovR64Imm(0x49, 0xBA, hostRspSlot); // mov r10, hostRspSlot + Emit(0x49); Emit(0x89); Emit(0x22); // mov [r10], rsp + EmitMovR64Imm(0x48, 0xB8, context[CpuRegister.Rsp]); // mov rax, guest rsp + Emit(0x48); Emit(0x89); Emit(0xC4); // mov rsp, rax + Emit(0x48); Emit(0x83); Emit(0xEC); Emit(0x08); // reserve transfer slot + EmitMovR64Imm(0x48, 0xB8, entryPoint); // mov rax, entryPoint + Emit(0x48); Emit(0x89); Emit(0x04); Emit(0x24); // mov [rsp],rax + EmitMovR64Imm(0x48, 0xBB, context[CpuRegister.Rbx]); // mov rbx, imm64 + EmitMovR64Imm(0x48, 0xBD, context[CpuRegister.Rbp]); // mov rbp, imm64 + EmitMovR64Imm(0x48, 0xBF, context[CpuRegister.Rdi]); // mov rdi, imm64 + EmitMovR64Imm(0x48, 0xBE, context[CpuRegister.Rsi]); // mov rsi, imm64 + EmitMovR64Imm(0x48, 0xBA, context[CpuRegister.Rdx]); // mov rdx, imm64 + EmitMovR64Imm(0x48, 0xB9, context[CpuRegister.Rcx]); // mov rcx, imm64 + EmitMovR64Imm(0x49, 0xB8, context[CpuRegister.R8]); // mov r8, imm64 + EmitMovR64Imm(0x49, 0xB9, context[CpuRegister.R9]); // mov r9, imm64 + EmitMovR64Imm(0x49, 0xBA, context[CpuRegister.R10]); // mov r10, imm64 + EmitMovR64Imm(0x49, 0xBC, context[CpuRegister.R12]); // mov r12, imm64 + EmitMovR64Imm(0x49, 0xBD, context[CpuRegister.R13]); // mov r13, imm64 + EmitMovR64Imm(0x49, 0xBE, context[CpuRegister.R14]); // mov r14, imm64 + EmitMovR64Imm(0x49, 0xBF, context[CpuRegister.R15]); // mov r15, imm64 + EmitMovR64Imm(0x49, 0xBB, context[CpuRegister.R11]); // mov r11, imm64 + EmitMovR64Imm(0x48, 0xB8, context[CpuRegister.Rax]); // mov rax, imm64 + Emit(0xC3); // ret through the synthetic transfer slot + ActiveEntryReturnSentinelRip = (ulong)_guestReturnStub; + if (returnSlotAddress == 0 || !context.TryWriteUInt64(returnSlotAddress, (ulong)_guestReturnStub)) + { + reason = $"failed to patch guest continuation return slot at 0x{returnSlotAddress:X16}"; + return GuestNativeCallExitReason.Exception; + } + uint oldProtect = default(uint); + if (!VirtualProtect(ptr, stubSize, 32u, &oldProtect)) + { + reason = "failed to seal guest continuation stub execute-read"; + return GuestNativeCallExitReason.Exception; + } + FlushInstructionCache(GetCurrentProcess(), ptr, stubSize); + if (!TlsSetValue(_hostRspSlotTlsIndex, (nint)hostRspSlot)) + { + reason = "failed to bind host-RSP storage for guest continuation stub"; + return GuestNativeCallExitReason.Exception; + } + ActiveGuestThreadYieldRequested = false; + ActiveGuestThreadYieldReason = null; + try + { + var nativeReturn = CallNativeEntry(ptr); + if (ActiveGuestThreadYieldRequested) + { + reason = ActiveGuestThreadYieldReason ?? "guest thread blocked"; + return GuestNativeCallExitReason.Blocked; + } + if (ActiveForcedGuestExit) + { + reason = LastError ?? "guest thread forced exit"; + return GuestNativeCallExitReason.ForcedExit; + } + reason = $"returned 0x{nativeReturn:X8}"; + return GuestNativeCallExitReason.Returned; + } + catch (AccessViolationException ex) + { + reason = "access violation: " + ex.Message; + return GuestNativeCallExitReason.Exception; + } + catch (Exception ex) + { + reason = ex.GetType().Name + ": " + ex.Message; + return GuestNativeCallExitReason.Exception; + } + } + finally + { + TlsSetValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); + RestoreActiveExecutionThread( + previousActiveBackend, + previousActiveContext, + previousSentinel, + previousReturnSlotAddress, + previousForcedExit, + previousYieldRequested, + previousYieldReason); + NativeMemory.Free(hostRspStorage); + VirtualFree(ptr, 0u, 32768u); } } @@ -4208,13 +5281,21 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } Console.Error.WriteLine($"[LOADER][INFO] StackTop: 0x{num:X16}"); const uint stubSize = 512u; - void* ptr = (void*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, stubSize, 12288u, 4u); if (ptr == null) { LastError = "Failed to allocate executable memory for stub"; result = OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; return false; } + void* hostRspStorage = NativeMemory.Alloc((nuint)sizeof(ulong)); + if (hostRspStorage == null) + { + VirtualFree(ptr, 0u, 32768u); + LastError = "Failed to allocate writable host-RSP storage for stub"; + result = OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + return false; + } var previousActiveBackend = _activeExecutionBackend; var previousActiveContext = _activeCpuContext; var previousSentinel = _activeEntryReturnSentinelRip; @@ -4222,7 +5303,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I var previousForcedExit = _activeForcedGuestExit; var previousYieldRequested = _activeGuestThreadYieldRequested; var previousYieldReason = _activeGuestThreadYieldReason; - nint previousHostRspSlotValue = _hostThreading.GetTlsValue(_hostRspSlotTlsIndex); + nint previousHostRspSlotValue = TlsGetValue(_hostRspSlotTlsIndex); try { _activeExecutionBackend = this; @@ -4234,7 +5315,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _activeGuestThreadYieldReason = null; BindTlsBase(context); byte* ptr2 = (byte*)ptr; - ulong num2 = (ulong)ptr + stubSize - 16uL; + ulong num2 = (ulong)hostRspStorage; int num3 = 0; ptr2[num3++] = 83; ptr2[num3++] = 85; @@ -4341,18 +5422,23 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return false; } uint num5 = default(uint); - if (!_hostMemory.Protect((ulong)ptr, stubSize, HostPageProtection.ReadWriteExecute, out num5)) + if (!VirtualProtect(ptr, stubSize, 32u, &num5)) { - LastError = $"VirtualProtect failed for guest entry stub at 0x{(nint)ptr:X16}"; + LastError = "Failed to seal native entry stub execute-read"; result = OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; return false; } - _hostMemory.FlushInstructionCache((ulong)ptr, stubSize); + FlushInstructionCache(GetCurrentProcess(), ptr, stubSize); if (_hostRspSlotStorage != 0) { *(ulong*)_hostRspSlotStorage = num2; } - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, (nint)num2); + if (!TlsSetValue(_hostRspSlotTlsIndex, (nint)num2)) + { + LastError = "Failed to bind host-RSP storage for native entry stub"; + result = OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + return false; + } if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_SENTINEL_PROBE"), "1", StringComparison.Ordinal)) { Console.Error.WriteLine("[LOADER][INFO] Running unresolved sentinel probe..."); @@ -4361,10 +5447,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } Console.Error.WriteLine("[LOADER][INFO] Calling guest entry..."); StartStallWatchdog(); + StartReadyThreadDispatcher(); int num6 = -1; try { - num6 = RunGuestEntryStub(ptr, num2); + num6 = CallNativeEntry(ptr); Console.Error.WriteLine($"[LOADER][INFO] Guest returned: {num6}"); PumpUntilGuestThreadsIdle(context, "entry_return"); } @@ -4388,12 +5475,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I result = OrbisGen2Result.ORBIS_GEN2_ERROR_CPU_TRAP; if (string.IsNullOrEmpty(LastError)) { - LastError = _hostShutdownRequested - ? "Host shutdown requested." - : "Detected repeating import loop and forced guest unwind to host."; + LastError = "Detected repeating import loop and forced guest unwind to host."; } Console.Error.WriteLine("[LOADER][ERROR] " + LastError); - RequestGuestThreadTeardown(3000); return false; } if (num6 == 0) @@ -4408,14 +5492,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I LastError = $"Guest entry point returned non-zero: {num6}"; } Console.Error.WriteLine("[LOADER][ERROR] " + LastError); - RequestGuestThreadTeardown(3000); return false; } finally { + StopReadyThreadDispatcher(); StopStallWatchdog(); ActiveEntryReturnSentinelRip = 0uL; - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); + TlsSetValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); if (_hostRspSlotStorage != 0) { *(long*)_hostRspSlotStorage = 0L; @@ -4428,7 +5512,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I previousForcedExit, previousYieldRequested, previousYieldReason); - _hostMemory.Free((ulong)ptr); + NativeMemory.Free(hostRspStorage); + VirtualFree(ptr, 0u, 32768u); } } @@ -4447,17 +5532,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return 20; } - // SHARPEMU_PERIODIC_SNAPSHOT_SECONDS=N: dump the stall snapshot every N - // seconds regardless of progress, for diagnosing soft stalls where imports - // keep flowing but the game stops advancing. - private static int GetPeriodicSnapshotSeconds() - { - if (int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_PERIODIC_SNAPSHOT_SECONDS"), out var result)) - { - return Math.Max(0, result); - } - return 0; - } private void StartStallWatchdog() { @@ -4488,8 +5562,12 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I dispatcherThread.Start(); long num = (long)((double)stallWatchdogSeconds * Stopwatch.Frequency); - var periodicSnapshotTicks = (long)((double)GetPeriodicSnapshotSeconds() * Stopwatch.Frequency); - var lastPeriodicSnapshot = Stopwatch.GetTimestamp(); + int periodicSnapshotSeconds = + int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_PERIODIC_SNAPSHOT_SECONDS"), out var pss) + ? Math.Max(0, pss) + : 0; + long periodicSnapshotTicks = (long)((double)periodicSnapshotSeconds * Stopwatch.Frequency); + long lastPeriodicSnapshot = Stopwatch.GetTimestamp(); _stallWatchdogThread = new Thread(new ThreadStart(delegate { while (!_stallWatchdogStop) @@ -4503,53 +5581,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I Stopwatch.GetTimestamp() - lastPeriodicSnapshot >= periodicSnapshotTicks) { lastPeriodicSnapshot = Stopwatch.GetTimestamp(); - var gateOwnerSite = _gateOwnerSite; - var gateOwnerTid = Volatile.Read(ref _gateOwnerManagedThreadId); - var gateHeldMs = gateOwnerSite is null - ? 0.0 - : Stopwatch.GetElapsedTime(Volatile.Read(ref _gateAcquireTimestamp)).TotalMilliseconds; - var snapshotText = new System.Text.StringBuilder(); - snapshotText.AppendLine( - $"[LOADER][DIAG] Periodic snapshot: gate_owner={gateOwnerSite ?? "none"} " + - $"gate_tid={gateOwnerTid} gate_held_ms={gateHeldMs:0}"); - // Never touch the gate here: the periodic snapshot must keep - // reporting even (especially) when the gate is wedged. - // Dump guest threads without the lock; tolerate torn reads. - try - { - foreach (var thread in _guestThreads.Values) - { - snapshotText.AppendLine( - $"[LOADER][DIAG] gateless guest-thread: handle=0x{thread.ThreadHandle:X16} name='{thread.Name}' " + - $"state={thread.State} imports={Interlocked.Read(ref thread.ImportCount)} " + - $"nid={Volatile.Read(ref thread.LastImportNid) ?? "none"} ret=0x{Volatile.Read(ref thread.LastReturnRip):X16} " + - $"block={thread.BlockReason ?? "none"} wake={thread.BlockWakeKey ?? "none"}"); - } - } - catch (Exception snapshotError) - { - snapshotText.AppendLine($"[LOADER][DIAG] gateless snapshot failed: {snapshotError.Message}"); - } - - // Console can be wedged by whatever is being diagnosed, so - // write to a side file when one is configured and fall back - // to stderr otherwise. - var snapshotPath = Environment.GetEnvironmentVariable("SHARPEMU_PERIODIC_SNAPSHOT_FILE"); - if (!string.IsNullOrWhiteSpace(snapshotPath)) - { - try - { - System.IO.File.AppendAllText(snapshotPath, snapshotText.ToString()); - } - catch - { - } - } - else - { - Console.Error.Write(snapshotText.ToString()); - Console.Error.Flush(); - } + Console.Error.WriteLine("[LOADER][ERROR] --- periodic snapshot ---"); + LogStallWatchdogSnapshot(); + Console.Error.Flush(); } long num2 = Stopwatch.GetTimestamp() - Volatile.Read(ref _lastProgressTimestamp); if (num2 < num) @@ -4558,12 +5592,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } if (HasReadyGuestThread()) { - if (_cpuContext is { } watchdogContext) - { - Pump(watchdogContext, "watchdog"); - } Console.Error.WriteLine( - $"[LOADER][WARN] No import progress for {stallWatchdogSeconds}s, but a guest thread is ready; continuing."); + $"[LOADER][WARN] No import progress for {stallWatchdogSeconds}s, but a guest thread is ready; dispatcher will resume it."); LogStallWatchdogSnapshot(); Console.Error.Flush(); MarkExecutionProgress(); @@ -4666,6 +5696,178 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _stallWatchdogThread = null; } + // A guest thread only gets dispatched to a native thread when some running + // guest thread calls Pump (which happens inside blocking HLE primitives: + // waits, usleep, pthread_create, entry_return). That leaves a starvation + // hole: a guest thread that spins on a non-blocking HLE call (e.g. + // sceAudioOutOutput) never pumps, so any thread that was made Ready — for + // example a job worker woken by sceKernelSetEventFlag — sits in the ready + // queue forever. Import progress keeps advancing (the spin), so the stall + // watchdog never fires either, and the whole game deadlocks with 0 draws. + // + // This background dispatcher closes the hole: it drains the ready queue on + // a short interval regardless of whether any guest thread pumps. It is + // deliberately self-contained (it does not touch Pump or the pump-depth + // guard) so it cannot alter the existing cooperative dispatch path. + private void StartReadyThreadDispatcher() + { + if (_readyDispatchThread != null) + { + return; + } + var logSnapshots = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_THREAD_SNAPSHOTS"), + "1", + StringComparison.Ordinal); + var nextSnapshotTimestamp = Stopwatch.GetTimestamp() + Stopwatch.Frequency; + _readyDispatchStop = false; + _readyDispatchThread = new Thread(new ThreadStart(delegate + { + while (!_readyDispatchStop) + { + Thread.Sleep(1); + if (_readyDispatchStop) + { + break; + } + // The count is a fast diagnostic hint, while the queue/state pair under + // _guestThreadGate is authoritative. Always attempt a locked drain so a + // stale hint cannot strand a runnable continuation. + DispatchReadyGuestThreads(); + if (logSnapshots && Stopwatch.GetTimestamp() >= nextSnapshotTimestamp) + { + lock (_guestThreadGate) + { + foreach (var thread in _guestThreads.Values) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_thread.snapshot " + + $"handle=0x{thread.ThreadHandle:X16} name='{thread.Name}' " + + $"state={thread.State} executor={thread.ExecutorActive} " + + $"imports={Interlocked.Read(ref thread.ImportCount)} " + + $"nid={Volatile.Read(ref thread.LastImportNid) ?? "none"} " + + $"ret=0x{Volatile.Read(ref thread.LastReturnRip):X16} " + + $"block={thread.BlockReason ?? "none"} " + + $"wake={thread.BlockWakeKey ?? "none"} " + + $"host_managed={thread.HostThread?.ManagedThreadId ?? 0} " + + $"host_tid={Volatile.Read(ref thread.HostThreadId)}"); + } + } + nextSnapshotTimestamp = Stopwatch.GetTimestamp() + Stopwatch.Frequency; + } + } + })) + { + IsBackground = true, + Name = "SharpEmu-ReadyDispatch", + }; + _readyDispatchThread.Start(); + } + + private void StopReadyThreadDispatcher() + { + _readyDispatchStop = true; + Thread? readyDispatchThread = _readyDispatchThread; + if (readyDispatchThread == null) + { + return; + } + if (!ReferenceEquals(Thread.CurrentThread, readyDispatchThread)) + { + try + { + readyDispatchThread.Join(300); + } + catch + { + } + } + _readyDispatchThread = null; + } + + // Dequeue every currently-ready guest thread and start a native thread for + // each, mirroring Pump's dispatch step. Dequeue and the Ready->Running + // transition happen under _guestThreadGate, so this races safely with a + // concurrent Pump: each ready thread is claimed once (the State check skips + // any that another dispatcher already took). + private void DispatchReadyGuestThreads() + { + while (true) + { + GuestThreadState? thread = null; + lock (_guestThreadGate) + { + _ = TryClaimReadyGuestThreadLocked(out thread); + } + + if (thread == null) + { + return; + } + + ScheduleGuestThreadExecution(thread, "ready-dispatch"); + } + } + + private void ScheduleGuestThreadExecution(GuestThreadState thread, string reason) + { + GuestExecutionRunner runner; + lock (_guestThreadGate) + { + runner = thread.ExecutionRunner ??= new GuestExecutionRunner( + thread.ThreadHandle, + thread.Name, + MapGuestThreadPriority(thread.Priority)); + } + runner.Schedule(() => RunGuestThread(thread, reason)); + } + + // Caller must hold _guestThreadGate. A guest wait can be satisfied before + // RunGuestThread has finished restoring its host/TLS state, so Ready alone + // is not sufficient to authorize another executor. ExecutorActive is the + // scheduler's authoritative single-owner token and covers both asynchronous + // host threads and synchronous Pump("entry_return") execution. + private bool TryClaimReadyGuestThreadLocked(out GuestThreadState? thread) + { + thread = null; + var candidatesToInspect = _readyGuestThreads.Count; + for (var index = 0; index < candidatesToInspect; index++) + { + var candidate = _readyGuestThreads.Dequeue(); + Interlocked.Decrement(ref _readyGuestThreadCount); + if (candidate.State != GuestThreadRunState.Ready) + { + continue; + } + + if (candidate.ExecutorActive) + { + _readyGuestThreads.Enqueue(candidate); + Interlocked.Increment(ref _readyGuestThreadCount); + candidate.ExecutorClaimDeferrals++; + if (_logGuestThreads && + (candidate.ExecutorClaimDeferrals <= 4 || + (candidate.ExecutorClaimDeferrals & (candidate.ExecutorClaimDeferrals - 1)) == 0)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_threads.defer_active_executor " + + $"handle=0x{candidate.ThreadHandle:X16} name='{candidate.Name}' " + + $"host_managed={candidate.HostThread?.ManagedThreadId ?? 0} " + + $"host_tid={Volatile.Read(ref candidate.HostThreadId)} " + + $"deferrals={candidate.ExecutorClaimDeferrals}"); + } + continue; + } + + candidate.ExecutorActive = true; + candidate.State = GuestThreadRunState.Running; + thread = candidate; + return true; + } + + return false; + } + private void LogStallWatchdogSnapshot() { try @@ -4678,14 +5880,13 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ulong rsp = cpuContext[CpuRegister.Rsp]; Console.Error.WriteLine($"[LOADER][ERROR] Stall snapshot: rip=0x{cpuContext.Rip:X16} rsp=0x{rsp:X16} rbp=0x{cpuContext[CpuRegister.Rbp]:X16} rax=0x{cpuContext[CpuRegister.Rax]:X16} rbx=0x{cpuContext[CpuRegister.Rbx]:X16} rcx=0x{cpuContext[CpuRegister.Rcx]:X16} rdx=0x{cpuContext[CpuRegister.Rdx]:X16} rsi=0x{cpuContext[CpuRegister.Rsi]:X16} rdi=0x{cpuContext[CpuRegister.Rdi]:X16}"); ulong num = cpuContext.Rip & 0xFFFFFFFFFFFFFFF0uL; - var importEntries = _importEntries; - for (int i = 0; i < importEntries.Length; i++) + for (int i = 0; i < _importEntries.Length; i++) { - if (importEntries[i].Address != num) + if (_importEntries[i].Address != num) { continue; } - string text = importEntries[i].Nid; + string text = _importEntries[i].Nid; if (_moduleManager.TryGetExport(text, out ExportedFunction export)) { Console.Error.WriteLine($"[LOADER][ERROR] Stall import-stub: rip=0x{num:X16} nid={text} -> {export.LibraryName}:{export.Name}"); @@ -4734,7 +5935,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I $"[LOADER][ERROR] Stall guest-thread: handle=0x{thread.ThreadHandle:X16} name='{thread.Name}' " + $"state={thread.State} imports={Interlocked.Read(ref thread.ImportCount)} " + $"nid={Volatile.Read(ref thread.LastImportNid) ?? "none"} ret=0x{Volatile.Read(ref thread.LastReturnRip):X16} " + - $"block={thread.BlockReason ?? "none"}{hostContextText}"); + $"rdi=0x{Volatile.Read(ref thread.LastImportRdi):X16} rsi=0x{Volatile.Read(ref thread.LastImportRsi):X16} " + + $"rdx=0x{Volatile.Read(ref thread.LastImportRdx):X16} block={thread.BlockReason ?? "none"}{hostContextText}"); logged++; if (logged >= 48 && threads.Length > logged) { @@ -4749,46 +5951,163 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } } - private bool TryCaptureHostThreadContext(int hostThreadId, out HostThreadContextSnapshot snapshot) + private unsafe static bool TryCaptureHostThreadContext(int hostThreadId, out HostThreadContextSnapshot snapshot) { snapshot = default; - if (hostThreadId == 0 || unchecked((uint)hostThreadId) == _hostThreading.CurrentThreadId) + if (hostThreadId == 0 || unchecked((uint)hostThreadId) == GetCurrentThreadId()) { return false; } - if (!_hostThreading.TryCaptureThreadRegisters(unchecked((uint)hostThreadId), out var registers)) + var threadHandle = OpenThread(ThreadGetContext | ThreadSuspendResume, false, unchecked((uint)hostThreadId)); + if (threadHandle == 0) { return false; } - snapshot = new HostThreadContextSnapshot( - true, - registers.Rip, - registers.Rsp, - registers.Rbp, - registers.Rax, - registers.Rbx, - registers.Rcx, - registers.Rdx); - return true; + void* contextRecord = null; + var suspended = false; + try + { + if (SuspendThread(threadHandle) == uint.MaxValue) + { + return false; + } + + suspended = true; + contextRecord = NativeMemory.AllocZeroed((nuint)Win64ContextSize); + WriteCtxU32(contextRecord, Win64ContextFlagsOffset, ContextAmd64ControlInteger); + if (!GetThreadContext(threadHandle, contextRecord)) + { + return false; + } + + snapshot = new HostThreadContextSnapshot( + true, + ReadCtxU64(contextRecord, 248), + ReadCtxU64(contextRecord, 152), + ReadCtxU64(contextRecord, 160), + ReadCtxU64(contextRecord, 120), + ReadCtxU64(contextRecord, 144), + ReadCtxU64(contextRecord, 128), + ReadCtxU64(contextRecord, 136)); + return true; + } + finally + { + if (contextRecord != null) + { + NativeMemory.Free(contextRecord); + } + if (suspended) + { + _ = ResumeThread(threadHandle); + } + _ = CloseHandle(threadHandle); + } } + + private static uint TlsAlloc() => + OperatingSystem.IsWindows() ? Win32TlsAlloc() : PosixHostStubs.TlsAlloc(); + + private static bool TlsFree(uint dwTlsIndex) => + OperatingSystem.IsWindows() ? Win32TlsFree(dwTlsIndex) : PosixHostStubs.TlsFree(dwTlsIndex); + + private static bool TlsSetValue(uint dwTlsIndex, nint lpTlsValue) => + OperatingSystem.IsWindows() ? Win32TlsSetValue(dwTlsIndex, lpTlsValue) : PosixHostStubs.TlsSetValue(dwTlsIndex, lpTlsValue); + + private static nint TlsGetValue(uint dwTlsIndex) => + OperatingSystem.IsWindows() ? Win32TlsGetValue(dwTlsIndex) : PosixHostStubs.TlsGetValue(dwTlsIndex); + + private unsafe static void* AddVectoredExceptionHandler(uint first, IntPtr handler) => + OperatingSystem.IsWindows() ? Win32AddVectoredExceptionHandler(first, handler) : null; + + private unsafe static uint RemoveVectoredExceptionHandler(void* handle) => + OperatingSystem.IsWindows() ? Win32RemoveVectoredExceptionHandler(handle) : 0u; + + private static IntPtr SetUnhandledExceptionFilter(IntPtr lpTopLevelExceptionFilter) => + OperatingSystem.IsWindows() ? Win32SetUnhandledExceptionFilter(lpTopLevelExceptionFilter) : IntPtr.Zero; + + private static uint GetCurrentThreadId() => + OperatingSystem.IsWindows() ? Win32GetCurrentThreadId() : PosixHostStubs.GetCurrentThreadId(); + + private static nint GetCurrentThread() => + OperatingSystem.IsWindows() ? Win32GetCurrentThread() : 0; + + private static nuint SetThreadAffinityMask(nint hThread, nuint dwThreadAffinityMask) => + OperatingSystem.IsWindows() ? Win32SetThreadAffinityMask(hThread, dwThreadAffinityMask) : 1; + + private static nint OpenThread(uint dwDesiredAccess, bool bInheritHandle, uint dwThreadId) => + OperatingSystem.IsWindows() ? Win32OpenThread(dwDesiredAccess, bInheritHandle, dwThreadId) : 0; + + private static uint SuspendThread(nint hThread) => + OperatingSystem.IsWindows() ? Win32SuspendThread(hThread) : uint.MaxValue; + + private static uint ResumeThread(nint hThread) => + OperatingSystem.IsWindows() ? Win32ResumeThread(hThread) : uint.MaxValue; + + private unsafe static bool GetThreadContext(nint hThread, void* lpContext) => + OperatingSystem.IsWindows() && Win32GetThreadContext(hThread, lpContext); + + private static bool CloseHandle(nint hObject) => + OperatingSystem.IsWindows() && Win32CloseHandle(hObject); + + [DllImport("kernel32.dll", EntryPoint = "TlsAlloc")] + private static extern uint Win32TlsAlloc(); + + [DllImport("kernel32.dll", EntryPoint = "TlsFree")] + private static extern bool Win32TlsFree(uint dwTlsIndex); + + [DllImport("kernel32.dll", EntryPoint = "TlsSetValue")] + private static extern bool Win32TlsSetValue(uint dwTlsIndex, nint lpTlsValue); + + [DllImport("kernel32.dll", EntryPoint = "TlsGetValue")] + private static extern nint Win32TlsGetValue(uint dwTlsIndex); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] + private static extern nint GetModuleHandle(string lpModuleName); + + [DllImport("kernel32.dll", CharSet = CharSet.Ansi)] + private static extern nint GetProcAddress(nint hModule, string procName); + + [DllImport("kernel32.dll", EntryPoint = "AddVectoredExceptionHandler")] + private unsafe static extern void* Win32AddVectoredExceptionHandler(uint first, IntPtr handler); + + [DllImport("kernel32.dll", EntryPoint = "RemoveVectoredExceptionHandler")] + private unsafe static extern uint Win32RemoveVectoredExceptionHandler(void* handle); + + [DllImport("kernel32.dll", EntryPoint = "SetUnhandledExceptionFilter")] + private static extern IntPtr Win32SetUnhandledExceptionFilter(IntPtr lpTopLevelExceptionFilter); + + [DllImport("kernel32.dll", EntryPoint = "GetCurrentThreadId")] + private static extern uint Win32GetCurrentThreadId(); + + [DllImport("kernel32.dll", EntryPoint = "GetCurrentThread")] + private static extern nint Win32GetCurrentThread(); + + [DllImport("kernel32.dll", EntryPoint = "SetThreadAffinityMask", SetLastError = true)] + private static extern nuint Win32SetThreadAffinityMask(nint hThread, nuint dwThreadAffinityMask); + + [DllImport("kernel32.dll", EntryPoint = "OpenThread", SetLastError = true)] + private static extern nint Win32OpenThread(uint dwDesiredAccess, [MarshalAs(UnmanagedType.Bool)] bool bInheritHandle, uint dwThreadId); + + [DllImport("kernel32.dll", EntryPoint = "SuspendThread", SetLastError = true)] + private static extern uint Win32SuspendThread(nint hThread); + + [DllImport("kernel32.dll", EntryPoint = "ResumeThread", SetLastError = true)] + private static extern uint Win32ResumeThread(nint hThread); + + [DllImport("kernel32.dll", EntryPoint = "GetThreadContext", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private unsafe static extern bool Win32GetThreadContext(nint hThread, void* lpContext); + + [DllImport("kernel32.dll", EntryPoint = "CloseHandle", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Win32CloseHandle(nint hObject); + public unsafe void Dispose() { - if (!RequestGuestThreadTeardown(2000)) - { - // A guest worker is still executing native code; freeing the trampolines, - // exception-handler stubs, or GC handles under it turns process exit into - // an execute-AV / CLR fatal. Leak them — the process is going away anyway. - Console.Error.WriteLine( - "[LOADER][WARN] Skipping executable stub teardown: guest worker threads are still running."); - return; - } - // Native guest workers park idle once every guest thread has unwound; stop - // them before any executable stub or TLS index they reference is freed. - DisposeNativeGuestExecutors(); - if (ReferenceEquals(_posixSignalBackend, this)) { // The signal handlers stay installed (they chain to the previous @@ -4799,33 +6118,32 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ClearImportHandlerTrampolines(); _importEntries = Array.Empty(); _runtimeSymbolsByName.Clear(); - ResetLazyDlsymStubState(); - _importNidHashCache.Clear(); + StopReadyThreadDispatcher(); StopStallWatchdog(); if (_exceptionHandler != 0) { - _faultHandling.RemoveHandler(_exceptionHandler); + RemoveVectoredExceptionHandler((void*)_exceptionHandler); _exceptionHandler = 0; } if (_rawExceptionHandler != 0) { - _faultHandling.RemoveHandler(_rawExceptionHandler); + RemoveVectoredExceptionHandler((void*)_rawExceptionHandler); _rawExceptionHandler = 0; } if (_rawExceptionHandlerStub != 0) { - _faultHandling.FreeThunk(_rawExceptionHandlerStub); + VirtualFree((void*)_rawExceptionHandlerStub, 0u, 32768u); _rawExceptionHandlerStub = 0; } if (_exceptionHandlerStub != 0) { - _faultHandling.FreeThunk(_exceptionHandlerStub); + VirtualFree((void*)_exceptionHandlerStub, 0u, 32768u); _exceptionHandlerStub = 0; } if (_unhandledFilterStub != 0) { - _faultHandling.SetUnhandledFilter(0); - _faultHandling.FreeThunk(_unhandledFilterStub); + SetUnhandledExceptionFilter(0); + VirtualFree((void*)_unhandledFilterStub, 0u, 32768u); _unhandledFilterStub = 0; } if (_handlerHandle.IsAllocated) @@ -4843,7 +6161,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } if (_ownedTlsBaseAddress != 0) { - _hostMemory.Free((ulong)_ownedTlsBaseAddress); + VirtualFree((void*)_ownedTlsBaseAddress, 0u, 32768u); _ownedTlsBaseAddress = 0; } _tlsBaseAddress = 0; @@ -4854,44 +6172,44 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { if (num3 != 0) { - _hostMemory.Free((ulong)num3); + VirtualFree((void*)num3, 0u, 32768u); } } _tlsModuleBases.Clear(); } if (_tlsHandlerAddress != 0) { - _hostMemory.Free((ulong)_tlsHandlerAddress); + VirtualFree((void*)_tlsHandlerAddress, 0u, 32768u); _tlsHandlerAddress = 0; } if (_hostRspSlotStorage != 0) { - _hostMemory.Free((ulong)_hostRspSlotStorage); + VirtualFree((void*)_hostRspSlotStorage, 0u, 32768u); _hostRspSlotStorage = 0; } if (_guestTlsBaseTlsIndex != uint.MaxValue) { - _hostThreading.FreeTlsSlot(_guestTlsBaseTlsIndex); + TlsFree(_guestTlsBaseTlsIndex); _guestTlsBaseTlsIndex = uint.MaxValue; } if (_hostRspSlotTlsIndex != uint.MaxValue) { - _hostThreading.FreeTlsSlot(_hostRspSlotTlsIndex); + TlsFree(_hostRspSlotTlsIndex); _hostRspSlotTlsIndex = uint.MaxValue; } if (_unresolvedReturnStub != 0) { - _hostMemory.Free((ulong)_unresolvedReturnStub); + VirtualFree((void*)_unresolvedReturnStub, 0u, 32768u); _unresolvedReturnStub = 0; } if (_guestReturnStub != 0) { - _hostMemory.Free((ulong)_guestReturnStub); + VirtualFree((void*)_guestReturnStub, 0u, 32768u); _guestReturnStub = 0; } if (_guestContextTransferStub != 0) { - _hostMemory.Free((ulong)_guestContextTransferStub); + VirtualFree((void*)_guestContextTransferStub, 0u, 32768u); _guestContextTransferStub = 0; } foreach (var frame in _guestContextTransferFrames.Values) @@ -4904,19 +6222,59 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _guestContextTransferFrames.Dispose(); if (_lowIndexedTableScratch != 0) { - _hostMemory.Free((ulong)_lowIndexedTableScratch); + VirtualFree((void*)_lowIndexedTableScratch, 0u, 32768u); _lowIndexedTableScratch = 0; } if (_stackGuardCompareScratch != 0) { - _hostMemory.Free((ulong)_stackGuardCompareScratch); + VirtualFree((void*)_stackGuardCompareScratch, 0u, 32768u); _stackGuardCompareScratch = 0; } if (_nullObjectStoreScratch != 0) { - _hostMemory.Free((ulong)_nullObjectStoreScratch); + VirtualFree((void*)_nullObjectStoreScratch, 0u, 32768u); _nullObjectStoreScratch = 0; } Volatile.Write(ref _globalUnresolvedReturnStub, 0uL); } + + private unsafe static void* VirtualAlloc(void* lpAddress, nuint dwSize, uint flAllocationType, uint flProtect) => + HostMemory.Alloc(lpAddress, dwSize, flAllocationType, flProtect); + + private unsafe static bool VirtualFree(void* lpAddress, nuint dwSize, uint dwFreeType) => + HostMemory.Free(lpAddress, dwSize, dwFreeType); + + private unsafe static bool VirtualProtect(void* lpAddress, nuint dwSize, uint flNewProtect, uint* lpflOldProtect) + { + var success = HostMemory.Protect(lpAddress, dwSize, flNewProtect, out var oldProtect); + if (lpflOldProtect != null) + { + *lpflOldProtect = oldProtect; + } + + return success; + } + + private unsafe static void* GetCurrentProcess() => null; + + private unsafe static bool FlushInstructionCache(void* hProcess, void* lpBaseAddress, nuint dwSize) + { + _ = hProcess; + HostMemory.FlushInstructionCache(lpBaseAddress, dwSize); + return true; + } + + private unsafe static nuint VirtualQuery(void* lpAddress, out MEMORY_BASIC_INFORMATION64 lpBuffer, nuint dwLength) + { + _ = dwLength; + var result = HostMemory.Query(lpAddress, out var info); + lpBuffer = default; + lpBuffer.BaseAddress = info.BaseAddress; + lpBuffer.AllocationBase = info.AllocationBase; + lpBuffer.AllocationProtect = info.AllocationProtect; + lpBuffer.RegionSize = info.RegionSize; + lpBuffer.State = info.State; + lpBuffer.Protect = info.Protect; + return result; + } } diff --git a/src/SharpEmu.Core/Cpu/Native/PosixHostStubs.cs b/src/SharpEmu.Core/Cpu/Native/PosixHostStubs.cs new file mode 100644 index 0000000..43219b9 --- /dev/null +++ b/src/SharpEmu.Core/Cpu/Native/PosixHostStubs.cs @@ -0,0 +1,425 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using SharpEmu.HLE; + +namespace SharpEmu.Core.Cpu.Native; + +/// +/// POSIX replacements for the kernel32 helpers the native backend embeds in +/// emitted x86-64 code. Every stub exposed here follows the Win64 calling +/// convention the emitted call sites were written for (first argument in +/// ECX, result in RAX, Win64 non-volatile registers preserved), so the +/// emission code stays identical across platforms. +/// +internal static unsafe class PosixHostStubs +{ + private static readonly object Gate = new(); + private static bool _initialized; + private static nint _tlsGetValueStub; + private static nint _queryPerformanceCounterStub; + private static nint _switchToThreadStub; + private static nint _sleepStub; + + public static nint TlsGetValueStubAddress + { + get { EnsureInitialized(); return _tlsGetValueStub; } + } + + public static nint QueryPerformanceCounterStubAddress + { + get { EnsureInitialized(); return _queryPerformanceCounterStub; } + } + + public static nint SwitchToThreadStubAddress + { + get { EnsureInitialized(); return _switchToThreadStub; } + } + + public static nint SleepStubAddress + { + get { EnsureInitialized(); return _sleepStub; } + } + + public static nint CreateWorkerEvent() + { + if (OperatingSystem.IsMacOS()) + { + return dispatch_semaphore_create(0); + } + + var semaphore = Marshal.AllocHGlobal(64); + if (sem_init(semaphore, 0, 0) != 0) + { + Marshal.FreeHGlobal(semaphore); + return 0; + } + + return semaphore; + } + + public static bool SignalWorkerEvent(nint handle) + { + if (OperatingSystem.IsMacOS()) + { + _ = dispatch_semaphore_signal(handle); + return true; + } + + return sem_post(handle) == 0; + } + + public static bool WaitWorkerEvent(nint handle, int timeoutMilliseconds) + { + if (OperatingSystem.IsMacOS()) + { + if (timeoutMilliseconds < 0) + { + return dispatch_semaphore_wait(handle, ulong.MaxValue) == 0; + } + + var deadline = dispatch_time(0, timeoutMilliseconds * 1_000_000L); + return dispatch_semaphore_wait(handle, deadline) == 0; + } + + if (timeoutMilliseconds < 0) + { + while (sem_wait(handle) != 0) + { + } + + return true; + } + + var deadlineTicks = Environment.TickCount64 + timeoutMilliseconds; + while (sem_trywait(handle) != 0) + { + if (Environment.TickCount64 >= deadlineTicks) + { + return false; + } + + Thread.Sleep(1); + } + + return true; + } + + public static void DestroyWorkerEvent(nint handle) + { + if (handle == 0) + { + return; + } + + if (OperatingSystem.IsMacOS()) + { + dispatch_release(handle); + return; + } + + _ = sem_destroy(handle); + Marshal.FreeHGlobal(handle); + } + + /// Allocates a pthread TLS key, mirroring kernel32!TlsAlloc. + public static uint TlsAlloc() + { + if (OperatingSystem.IsMacOS()) + { + nuint key; + return pthread_key_create_mac(&key, 0) == 0 ? (uint)key : uint.MaxValue; + } + + uint key32; + return pthread_key_create_linux(&key32, 0) == 0 ? key32 : uint.MaxValue; + } + + public static bool TlsFree(uint key) + { + return OperatingSystem.IsMacOS() + ? pthread_key_delete_mac((nuint)key) == 0 + : pthread_key_delete_linux(key) == 0; + } + + public static bool TlsSetValue(uint key, nint value) + { + return OperatingSystem.IsMacOS() + ? pthread_setspecific_mac((nuint)key, value) == 0 + : pthread_setspecific_linux(key, value) == 0; + } + + public static nint TlsGetValue(uint key) + { + return OperatingSystem.IsMacOS() + ? pthread_getspecific_mac((nuint)key) + : pthread_getspecific_linux(key); + } + + /// Stable numeric id of the calling thread (kernel32!GetCurrentThreadId). + public static uint GetCurrentThreadId() + { + if (OperatingSystem.IsMacOS()) + { + ulong tid; + return pthread_threadid_np(0, &tid) == 0 ? unchecked((uint)tid) : 0u; + } + + return unchecked((uint)gettid()); + } + + /// + /// Wraps a managed callback (compiled for the SysV ABI on POSIX .NET) in a + /// thunk that accepts up to four integer arguments in the Win64 ABI the + /// emitted x86-64 call sites use. Win64 passes args in rcx/rdx/r8/r9 and + /// treats rdi/rsi as non-volatile; SysV expects rdi/rsi/rdx/rcx and + /// clobbers them, so the thunk saves rdi/rsi, shuffles the registers, keeps + /// the stack 16-byte aligned for the call, and forwards the rax result. + /// + public static nint CreateWin64ToSysVThunk(nint sysvTarget) + { + var page = (byte*)HostMemory.Alloc( + null, + 4096, + HostMemory.MEM_COMMIT | HostMemory.MEM_RESERVE, + HostMemory.PAGE_EXECUTE_READWRITE); + if (page == null) + { + throw new OutOfMemoryException("Failed to allocate Win64->SysV thunk page"); + } + + var offset = 0; + Emit(page, ref offset, 0x57); // push rdi + Emit(page, ref offset, 0x56); // push rsi + Emit(page, ref offset, 0x48, 0x89, 0xCF); // mov rdi, rcx + Emit(page, ref offset, 0x48, 0x89, 0xD6); // mov rsi, rdx + Emit(page, ref offset, 0x4C, 0x89, 0xC2); // mov rdx, r8 + Emit(page, ref offset, 0x4C, 0x89, 0xC9); // mov rcx, r9 + Emit(page, ref offset, 0x48, 0x83, 0xEC, 0x08); // sub rsp, 8 (realign to 16) + EmitMovRaxImm64(page, ref offset, sysvTarget); // mov rax, target + Emit(page, ref offset, 0xFF, 0xD0); // call rax + Emit(page, ref offset, 0x48, 0x83, 0xC4, 0x08); // add rsp, 8 + Emit(page, ref offset, 0x5E); // pop rsi + Emit(page, ref offset, 0x5F); // pop rdi + Emit(page, ref offset, 0xC3); // ret + + if (!HostMemory.Protect(page, 4096, HostMemory.PAGE_EXECUTE_READ, out _)) + { + throw new InvalidOperationException("Failed to protect Win64->SysV thunk page"); + } + + HostMemory.FlushInstructionCache(page, (nuint)offset); + return (nint)page; + } + + private static void EnsureInitialized() + { + if (_initialized) + { + return; + } + + lock (Gate) + { + if (_initialized) + { + return; + } + + BuildStubs(); + _initialized = true; + } + } + + private static void BuildStubs() + { + var page = (byte*)HostMemory.Alloc( + null, + 4096, + HostMemory.MEM_COMMIT | HostMemory.MEM_RESERVE, + HostMemory.PAGE_EXECUTE_READWRITE); + if (page == null) + { + throw new OutOfMemoryException("Failed to allocate POSIX host helper stub page"); + } + + var offset = 0; + _tlsGetValueStub = EmitTlsGetValue(page, ref offset); + _queryPerformanceCounterStub = EmitQueryPerformanceCounter(page, ref offset); + _switchToThreadStub = EmitSwitchToThread(page, ref offset); + _sleepStub = EmitSleep(page, ref offset); + + if (!HostMemory.Protect(page, 4096, HostMemory.PAGE_EXECUTE_READ, out _)) + { + throw new InvalidOperationException("Failed to protect POSIX host helper stub page"); + } + + HostMemory.FlushInstructionCache(page, (nuint)offset); + } + + private static nint EmitTlsGetValue(byte* page, ref int offset) + { + var start = (nint)(page + offset); + if (OperatingSystem.IsMacOS()) + { + // On macOS x86-64 pthread keys index the gs-based thread specific + // data array directly, so TlsGetValue(index in ecx) collapses to a + // single load that clobbers nothing but RAX. + Emit(page, ref offset, 0x89, 0xC8); // mov eax, ecx + Emit(page, ref offset, 0x65, 0x48, 0x8B, 0x04, 0xC5, 0, 0, 0, 0); // mov rax, gs:[rax*8] + Emit(page, ref offset, 0xC3); // ret + return start; + } + + // Linux: call pthread_getspecific, preserving the registers that are + // volatile in SysV but non-volatile in Win64 (rsi, rdi). + var pthreadGetSpecific = ResolveLibcExport("pthread_getspecific"); + Emit(page, ref offset, 0x56); // push rsi + Emit(page, ref offset, 0x57); // push rdi + Emit(page, ref offset, 0x48, 0x83, 0xEC, 0x08); // sub rsp, 8 + Emit(page, ref offset, 0x89, 0xCF); // mov edi, ecx + EmitMovRaxImm64(page, ref offset, pthreadGetSpecific); // mov rax, imm64 + Emit(page, ref offset, 0xFF, 0xD0); // call rax + Emit(page, ref offset, 0x48, 0x83, 0xC4, 0x08); // add rsp, 8 + Emit(page, ref offset, 0x5F); // pop rdi + Emit(page, ref offset, 0x5E); // pop rsi + Emit(page, ref offset, 0xC3); // ret + return start; + } + + private static nint EmitQueryPerformanceCounter(byte* page, ref int offset) + { + // BOOL QueryPerformanceCounter(LARGE_INTEGER* out in rcx): the emitted + // consumers only need a monotonically increasing counter, which rdtsc + // provides without leaving Win64-safe registers. + var start = (nint)(page + offset); + Emit(page, ref offset, 0x0F, 0x31); // rdtsc + Emit(page, ref offset, 0x48, 0xC1, 0xE2, 0x20); // shl rdx, 32 + Emit(page, ref offset, 0x48, 0x09, 0xD0); // or rax, rdx + Emit(page, ref offset, 0x48, 0x89, 0x01); // mov [rcx], rax + Emit(page, ref offset, 0xB8, 0x01, 0x00, 0x00, 0x00); // mov eax, 1 + Emit(page, ref offset, 0xC3); // ret + return start; + } + + private static nint EmitSwitchToThread(byte* page, ref int offset) + { + var schedYield = ResolveLibcExport("sched_yield"); + var start = (nint)(page + offset); + Emit(page, ref offset, 0x56); // push rsi + Emit(page, ref offset, 0x57); // push rdi + Emit(page, ref offset, 0x48, 0x83, 0xEC, 0x08); // sub rsp, 8 + EmitMovRaxImm64(page, ref offset, schedYield); // mov rax, imm64 + Emit(page, ref offset, 0xFF, 0xD0); // call rax + Emit(page, ref offset, 0x48, 0x83, 0xC4, 0x08); // add rsp, 8 + Emit(page, ref offset, 0x5F); // pop rdi + Emit(page, ref offset, 0x5E); // pop rsi + Emit(page, ref offset, 0xB8, 0x01, 0x00, 0x00, 0x00); // mov eax, 1 + Emit(page, ref offset, 0xC3); // ret + return start; + } + + private static nint EmitSleep(byte* page, ref int offset) + { + // void Sleep(DWORD milliseconds in ecx) -> usleep(microseconds in edi). + var usleep = ResolveLibcExport("usleep"); + var start = (nint)(page + offset); + Emit(page, ref offset, 0x56); // push rsi + Emit(page, ref offset, 0x57); // push rdi + Emit(page, ref offset, 0x48, 0x83, 0xEC, 0x08); // sub rsp, 8 + Emit(page, ref offset, 0x89, 0xCF); // mov edi, ecx + Emit(page, ref offset, 0x81, 0xFF, 0xFF, 0x0F, 0x00, 0x00); // cmp edi, 0xFFF + Emit(page, ref offset, 0x76, 0x05); // jbe +5 + Emit(page, ref offset, 0xBF, 0xFF, 0x0F, 0x00, 0x00); // mov edi, 0xFFF (cap at ~4s) + Emit(page, ref offset, 0x69, 0xFF, 0xE8, 0x03, 0x00, 0x00); // imul edi, edi, 1000 + EmitMovRaxImm64(page, ref offset, usleep); // mov rax, imm64 + Emit(page, ref offset, 0xFF, 0xD0); // call rax + Emit(page, ref offset, 0x48, 0x83, 0xC4, 0x08); // add rsp, 8 + Emit(page, ref offset, 0x5F); // pop rdi + Emit(page, ref offset, 0x5E); // pop rsi + Emit(page, ref offset, 0xC3); // ret + return start; + } + + private static nint ResolveLibcExport(string name) + { + var libc = NativeLibrary.Load(OperatingSystem.IsMacOS() ? "libSystem.dylib" : "libc.so.6"); + return NativeLibrary.GetExport(libc, name); + } + + private static void Emit(byte* page, ref int offset, params byte[] bytes) + { + foreach (var value in bytes) + { + page[offset++] = value; + } + } + + private static void EmitMovRaxImm64(byte* page, ref int offset, nint value) + { + Emit(page, ref offset, 0x48, 0xB8); + *(long*)(page + offset) = value; + offset += sizeof(long); + } + + [DllImport("libc", EntryPoint = "pthread_key_create", SetLastError = true)] + private static extern int pthread_key_create_mac(nuint* key, nint destructor); + + [DllImport("libc", EntryPoint = "pthread_key_create", SetLastError = true)] + private static extern int pthread_key_create_linux(uint* key, nint destructor); + + [DllImport("libc", EntryPoint = "pthread_key_delete")] + private static extern int pthread_key_delete_mac(nuint key); + + [DllImport("libc", EntryPoint = "pthread_key_delete")] + private static extern int pthread_key_delete_linux(uint key); + + [DllImport("libc", EntryPoint = "pthread_setspecific")] + private static extern int pthread_setspecific_mac(nuint key, nint value); + + [DllImport("libc", EntryPoint = "pthread_setspecific")] + private static extern int pthread_setspecific_linux(uint key, nint value); + + [DllImport("libc", EntryPoint = "pthread_getspecific")] + private static extern nint pthread_getspecific_mac(nuint key); + + [DllImport("libc", EntryPoint = "pthread_getspecific")] + private static extern nint pthread_getspecific_linux(uint key); + + [DllImport("libc")] + private static extern int pthread_threadid_np(nint thread, ulong* threadId); + + [DllImport("libc")] + private static extern int gettid(); + + [DllImport("libc")] + private static extern nint dispatch_semaphore_create(long value); + + [DllImport("libc")] + private static extern nint dispatch_semaphore_signal(nint semaphore); + + [DllImport("libc")] + private static extern nint dispatch_semaphore_wait(nint semaphore, ulong timeout); + + [DllImport("libc")] + private static extern ulong dispatch_time(ulong when, long deltaNanoseconds); + + [DllImport("libc")] + private static extern void dispatch_release(nint handle); + + [DllImport("libc")] + private static extern int sem_init(nint semaphore, int shared, uint value); + + [DllImport("libc")] + private static extern int sem_post(nint semaphore); + + [DllImport("libc")] + private static extern int sem_wait(nint semaphore); + + [DllImport("libc")] + private static extern int sem_trywait(nint semaphore); + + [DllImport("libc")] + private static extern int sem_destroy(nint semaphore); +} diff --git a/src/SharpEmu.Core/Cpu/Native/StubManager.cs b/src/SharpEmu.Core/Cpu/Native/StubManager.cs index 37a2453..ffe3b79 100644 --- a/src/SharpEmu.Core/Cpu/Native/StubManager.cs +++ b/src/SharpEmu.Core/Cpu/Native/StubManager.cs @@ -3,7 +3,6 @@ using System.Runtime.InteropServices; using SharpEmu.HLE; -using SharpEmu.HLE.Host; namespace SharpEmu.Core.Cpu.Native; @@ -12,15 +11,17 @@ public sealed unsafe class StubManager : IDisposable private readonly List _allocatedStubs = new(); private readonly Dictionary _importHandlers = new(); private readonly Dictionary _stubAddresses = new(); - private readonly IHostMemory _hostMemory; private byte* _pltMemory; private int _pltOffset; private const int PltMemorySize = 1024 * 1024; // 1MB for stubs - public StubManager(IHostMemory? hostMemory = null) + public StubManager() { - _hostMemory = hostMemory ?? HostPlatform.Current.Memory; - _pltMemory = (byte*)_hostMemory.Allocate(0, PltMemorySize, HostPageProtection.ReadWriteExecute); + _pltMemory = (byte*)VirtualAlloc( + null, + (nuint)PltMemorySize, + AllocationType.Reserve | AllocationType.Commit, + MemoryProtection.ExecuteReadWrite); if (_pltMemory == null) { @@ -184,7 +185,7 @@ public sealed unsafe class StubManager : IDisposable { if (_pltMemory != null) { - _hostMemory.Free((ulong)_pltMemory); + VirtualFree(_pltMemory, 0, FreeType.Release); _pltMemory = null; } @@ -193,5 +194,29 @@ public sealed unsafe class StubManager : IDisposable _stubAddresses.Clear(); } + private static void* VirtualAlloc(void* lpAddress, nuint dwSize, AllocationType flAllocationType, MemoryProtection flProtect) => + HostMemory.Alloc(lpAddress, dwSize, (uint)flAllocationType, (uint)flProtect); + + private static bool VirtualFree(void* lpAddress, nuint dwSize, FreeType dwFreeType) => + HostMemory.Free(lpAddress, dwSize, (uint)dwFreeType); + + [Flags] + private enum AllocationType : uint + { + Commit = 0x1000, + Reserve = 0x2000, + } + + [Flags] + private enum MemoryProtection : uint + { + ExecuteReadWrite = 0x40, + } + + private enum FreeType : uint + { + Release = 0x8000, + } + public delegate void ImportHandler(CpuContext context); } diff --git a/src/SharpEmu.Core/Loader/ProgramHeader.cs b/src/SharpEmu.Core/Loader/ProgramHeader.cs index aa03d1c..400da70 100644 --- a/src/SharpEmu.Core/Loader/ProgramHeader.cs +++ b/src/SharpEmu.Core/Loader/ProgramHeader.cs @@ -48,6 +48,8 @@ public enum ProgramHeaderType : uint Tls = 7, + GnuEhFrame = 0x6474E550, + SceRela = 0x60000000, SceProcParam = 0x61000001, diff --git a/src/SharpEmu.Core/Loader/SelfImage.cs b/src/SharpEmu.Core/Loader/SelfImage.cs index 2f86c53..0919835 100644 --- a/src/SharpEmu.Core/Loader/SelfImage.cs +++ b/src/SharpEmu.Core/Loader/SelfImage.cs @@ -24,7 +24,10 @@ public sealed class SelfImage ulong procParamAddress = 0, string? title = null, string? titleId = null, - string? version = null) + string? version = null, + uint tlsModuleId = 0, + ulong tlsMemorySize = 0, + ulong tlsStaticOffset = 0) { ArgumentNullException.ThrowIfNull(programHeaders); ArgumentNullException.ThrowIfNull(mappedRegions); @@ -44,6 +47,9 @@ public sealed class SelfImage Title = title; TitleId = titleId; Version = version; + TlsModuleId = tlsModuleId; + TlsMemorySize = tlsMemorySize; + TlsStaticOffset = tlsStaticOffset; } public bool IsSelf { get; } @@ -75,4 +81,11 @@ public sealed class SelfImage public string? TitleId { get; } public string? Version { get; } + + public uint TlsModuleId { get; } + + public ulong TlsMemorySize { get; } + + /// Variant II distance from the thread pointer to this module's static TLS base. + public ulong TlsStaticOffset { get; } } diff --git a/src/SharpEmu.Core/Loader/SelfLoader.cs b/src/SharpEmu.Core/Loader/SelfLoader.cs index 7d30245..e804dc9 100644 --- a/src/SharpEmu.Core/Loader/SelfLoader.cs +++ b/src/SharpEmu.Core/Loader/SelfLoader.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System.Buffers.Binary; +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -10,16 +11,12 @@ using SharpEmu.Core; using SharpEmu.Core.Cpu; using SharpEmu.Core.Memory; using SharpEmu.HLE; -using SharpEmu.Logging; namespace SharpEmu.Core.Loader; public sealed class SelfLoader : ISelfLoader { - private static readonly SharpEmuLogger Log = SharpEmuLog.For("Loader"); - private const uint ElfMagic = 0x7F454C46; - private const uint Ps4SelfMagic = 0x4F153D1D; - private const uint Ps5SelfMagic = 0x5414F5EE; + private const uint SelfMagic = 0x4F153D1D; private const ulong SelfSegmentFlag = 0x800; private const int PageSize = 0x1000; private const ulong ImportStubBaseAddress = 0x0000_7000_0000_0000UL; @@ -58,11 +55,22 @@ public sealed class SelfLoader : ISelfLoader private const long DtSceSymTab = 0x61000039; private const long DtSceSymTabSize = 0x6100003F; + private const uint RelocationTypeNone = 0; private const uint RelocationTypeAbsolute64 = 1; + private const uint RelocationTypePc32 = 2; + private const uint RelocationTypePlt32 = 4; private const uint RelocationTypeGlobalData = 6; private const uint RelocationTypeJumpSlot = 7; private const uint RelocationTypeRelative = 8; - private const uint RelocationTypeTlsModuleId = 16; + private const uint RelocationTypeUnsigned32 = 10; + private const uint RelocationTypeSigned32 = 11; + private const uint RelocationTypeTlsModuleId = 16; // R_X86_64_DTPMOD64 + private const uint RelocationTypeTlsDtpOff64 = 17; // R_X86_64_DTPOFF64 + private const uint RelocationTypeTlsTpOff64 = 18; // R_X86_64_TPOFF64 + private const uint RelocationTypePc64 = 24; + private const uint RelocationTypeSize32 = 32; + private const uint RelocationTypeSize64 = 33; + private const uint RelocationTypeRelative64 = 38; private const ulong Ps5MainImageBase = 0x0000000800000000UL; private const ulong Ps4MainImageBase = 0x0000000000400000UL; private const ulong Ps5ModuleSearchStart = 0x0000000804000000UL; @@ -88,6 +96,11 @@ public sealed class SelfLoader : ISelfLoader private static readonly int SelfSegmentSize = Unsafe.SizeOf(); private static readonly int ProgramHeaderSize = Unsafe.SizeOf(); + static SelfLoader() + { + RunRelocationSelfChecks(); + } + public SelfImage Load(ReadOnlySpan imageData, IVirtualMemory virtualMemory) { return Load(imageData, virtualMemory, fs: null, mountRoot: null); @@ -156,16 +169,22 @@ public sealed class SelfLoader : ISelfLoader { virtualMemory.Clear(); _nextTlsModuleId = 1; + GuestTlsTemplate.Reset(); } - var tlsModuleId = _nextTlsModuleId == 0 ? 1u : _nextTlsModuleId; - Log.Debug($"TLS load_start clear={clearVirtualMemory} next={_nextTlsModuleId} assigned={tlsModuleId}"); - var loadContext = ParseLayout(imageData); var elfHeader = ReadUnmanaged(imageData, loadContext.ElfOffset); ValidateElfHeader(elfHeader); var programHeaders = ParseProgramHeaders(imageData, loadContext, elfHeader); + var hasTlsSegment = TryGetProgramHeader(programHeaders, ProgramHeaderType.Tls, out var processTlsHeader, out _) && + processTlsHeader.MemorySize != 0; + var tlsModuleId = hasTlsSegment + ? (_nextTlsModuleId == 0 ? 1u : _nextTlsModuleId) + : 0u; + Console.Error.WriteLine( + $"[LOADER][TLS] load_start clear={clearVirtualMemory} next={_nextTlsModuleId} " + + $"assigned={tlsModuleId} has_pt_tls={hasTlsSegment}"); var totalImageSize = CalculateTotalImageSize(programHeaders); Console.WriteLine($"Total image size needed: 0x{totalImageSize:X} ({totalImageSize} bytes)"); @@ -198,6 +217,13 @@ public sealed class SelfLoader : ISelfLoader } MapLoadSegments(imageData, loadContext, programHeaders, virtualMemory, imageBase); + // Register every module before relocations so DTPMOD/DTPOFF/TPOFF use + // the module's real PT_TLS identity and Variant II static offset. + var tlsInfo = RegisterModuleTlsTemplate( + programHeaders, + virtualMemory, + imageBase, + tlsModuleId); var importStubs = ResolveAndPatchImportStubs( imageData, loadContext, @@ -255,11 +281,11 @@ public sealed class SelfLoader : ISelfLoader Console.WriteLine($"[LOADER] PH[{i}]: type={ph.HeaderType}, vaddr=0x{ph.VirtualAddress:X16} -> 0x{ph.VirtualAddress + imageBase:X16}, memsz=0x{ph.MemorySize:X}"); } - if (_nextTlsModuleId == tlsModuleId && _nextTlsModuleId < uint.MaxValue) + if (tlsModuleId != 0 && _nextTlsModuleId == tlsModuleId && _nextTlsModuleId < uint.MaxValue) { _nextTlsModuleId++; } - Log.Debug($"TLS load_done assigned={tlsModuleId} next={_nextTlsModuleId}"); + Console.Error.WriteLine($"[LOADER][TLS] load_done assigned={tlsModuleId} next={_nextTlsModuleId}"); return new SelfImage( loadContext.IsSelf, @@ -276,7 +302,10 @@ public sealed class SelfLoader : ISelfLoader procParamAddress, applicationInfo.Title, applicationInfo.TitleId, - applicationInfo.Version); + applicationInfo.Version, + tlsModuleId, + tlsInfo.MemorySize, + tlsInfo.StaticOffset); } private static (string? Title, string? TitleId, string? Version) TryLoadParamJson( @@ -325,8 +354,7 @@ public sealed class SelfLoader : ISelfLoader throw new InvalidDataException("Input image is too small to contain an ELF header."); } - var magic = BinaryPrimitives.ReadUInt32BigEndian(imageData[..sizeof(uint)]); - if (magic is Ps4SelfMagic or Ps5SelfMagic) + if (imageData.Length >= sizeof(uint) && BinaryPrimitives.ReadUInt32BigEndian(imageData[..sizeof(uint)]) == SelfMagic) { var selfHeader = ReadUnmanaged(imageData, 0); if (!selfHeader.HasKnownLayout || selfHeader.Unknown != 0x22) @@ -348,10 +376,19 @@ public sealed class SelfLoader : ISelfLoader return new LoadContext(IsSelf: true, elfOffset, selfHeader.FileSize, segments); } - if (magic != ElfMagic) + // Not a recognized (fake-signed) SELF. Only a bare, decrypted ELF is + // acceptable here; anything else — most commonly a still-encrypted + // retail eboot — must be reported clearly rather than failing later + // with an opaque "not a valid ELF header" message. + const uint ElfMagicBigEndian = 0x7F454C46; // "\x7fELF" + var leadingWord = BinaryPrimitives.ReadUInt32BigEndian(imageData[..sizeof(uint)]); + if (leadingWord != ElfMagicBigEndian) { throw new InvalidDataException( - $"Unsupported executable signature 0x{magic:X8}"); + $"Image is neither a decrypted ELF nor a recognized fake-signed SELF " + + $"(leading bytes 0x{leadingWord:X8}). This is almost certainly a still-encrypted " + + $"retail eboot — SharpEmu has no decryption keys and requires a decrypted / " + + $"fake-signed (fSELF) image."); } return new LoadContext(IsSelf: false, ElfOffset: 0, SelfFileSize: 0, Array.Empty()); @@ -414,15 +451,15 @@ public sealed class SelfLoader : ISelfLoader var virtualAddress = header.VirtualAddress + imageBase; - Log.Debug($"Segment {index}: VAddr=0x{virtualAddress:X16}, FileSize=0x{header.FileSize:X}, MemSize=0x{header.MemorySize:X}, Align=0x{header.Alignment:X}"); + Console.Error.WriteLine($"[LOADER] Segment {index}: VAddr=0x{virtualAddress:X16}, FileSize=0x{header.FileSize:X}, MemSize=0x{header.MemorySize:X}, Align=0x{header.Alignment:X}"); if (header.Alignment > 1) { var vaddrMod = virtualAddress % header.Alignment; var offsetMod = header.Offset % header.Alignment; if (vaddrMod != offsetMod) { - Log.Warning( - $"Segment {index} ELF alignment mismatch! " + + Console.Error.WriteLine( + $"[LOADER] WARNING: Segment {index} ELF alignment mismatch! " + $"VAddr=0x{virtualAddress:X}, Offset=0x{header.Offset:X}, Align=0x{header.Alignment:X}, " + $"VAddr%Align=0x{vaddrMod:X}, Offset%Align=0x{offsetMod:X}"); } @@ -465,6 +502,43 @@ public sealed class SelfLoader : ISelfLoader return 0; } + private static ModuleTlsInfo RegisterModuleTlsTemplate( + IReadOnlyList programHeaders, + IVirtualMemory virtualMemory, + ulong imageBase, + uint tlsModuleId) + { + if (!TryGetProgramHeader(programHeaders, ProgramHeaderType.Tls, out var tlsHeader, out _) || + tlsHeader.MemorySize == 0) + { + return default; + } + + // tdata (initialized) bytes come from the mapped segment; tbss is the + // implicitly-zero remainder up to MemorySize. + var fileSize = (int)Math.Min(tlsHeader.FileSize, tlsHeader.MemorySize); + var initImage = fileSize > 0 ? new byte[fileSize] : []; + if (fileSize > 0 && + !virtualMemory.TryRead(imageBase + tlsHeader.VirtualAddress, initImage)) + { + Console.Error.WriteLine( + $"[LOADER][TLS] Failed to read TLS init image at 0x{imageBase + tlsHeader.VirtualAddress:X}; seeding zeros."); + initImage = []; + } + + var staticOffset = GuestTlsTemplate.RegisterModule( + tlsModuleId, + initImage, + tlsHeader.MemorySize, + tlsHeader.Alignment, + tlsHeader.VirtualAddress); + Console.Error.WriteLine( + $"[LOADER][TLS] Module {tlsModuleId} TLS template: memsz=0x{tlsHeader.MemorySize:X} " + + $"filesz=0x{tlsHeader.FileSize:X} align=0x{tlsHeader.Alignment:X} " + + $"static_offset=0x{staticOffset:X} total_static=0x{GuestTlsTemplate.StaticTlsSize:X}"); + return new ModuleTlsInfo(tlsHeader.MemorySize, staticOffset); + } + private static IReadOnlyDictionary ResolveAndPatchImportStubs( ReadOnlySpan imageData, LoadContext loadContext, @@ -544,12 +618,17 @@ public sealed class SelfLoader : ISelfLoader uint maxSymbolIndex = 0; foreach (var relocation in relocations) { + if (relocation.Type == RelocationTypeNone) + { + continue; + } + if (!IsSupportedRelocationType(relocation.Type)) { continue; } - if (relocation.Type is RelocationTypeRelative or RelocationTypeTlsModuleId) + if (relocation.Type is RelocationTypeNone or RelocationTypeRelative or RelocationTypeRelative64 or RelocationTypeTlsModuleId) { continue; } @@ -633,13 +712,16 @@ public sealed class SelfLoader : ISelfLoader importedRelocations = BuildImportedRelocations(descriptors); - var stubsByAddress = CreateImportStubMapping(virtualMemory, orderedImportNids); + var stubImportNids = orderedImportNids + .Where(nid => ShouldCreateImportStub(nid, descriptors, moduleManager)) + .ToArray(); + var stubsByAddress = CreateImportStubMapping(virtualMemory, stubImportNids); Console.WriteLine($"[LOADER] Created {stubsByAddress.Count} import stubs"); - int printCount = Math.Min(10, orderedImportNids.Count); + int printCount = Math.Min(10, stubImportNids.Length); for (int i = 0; i < printCount; i++) { - var nid = orderedImportNids[i]; + var nid = stubImportNids[i]; var addr = stubsByAddress.First(x => x.Value == nid).Key; } @@ -663,45 +745,65 @@ public sealed class SelfLoader : ISelfLoader foreach (var descriptor in descriptors) { - ulong targetValue; + ulong symbolValue; if (descriptor.ImportNid is null) { - targetValue = AddSigned(descriptor.SymbolValue, descriptor.Addend); + symbolValue = descriptor.SymbolValue; } else { - if (!addressesByNid.TryGetValue(descriptor.ImportNid, out var stubAddress)) + if (addressesByNid.TryGetValue(descriptor.ImportNid, out var stubAddress)) { - throw new InvalidOperationException($"Import stub not found for NID '{descriptor.ImportNid}'."); + symbolValue = stubAddress; } - - targetValue = AddSigned(stubAddress, descriptor.Addend); - } - - if (targetValue < 0x1000) - { - if (descriptor.ValueKind == RelocationValueKind.TlsModuleId) + else if (descriptor.IsWeak) { - Log.Debug( - $"Patching DTPMOD64 at 0x{descriptor.TargetAddress:X} with module id 0x{targetValue:X}"); + // ELF unresolved weak definitions use S=0. They must not + // receive a trap import stub, which would turn a permitted + // null test into a call to an unresolved-import handler. + symbolValue = 0; } else { - Log.Warning($"!!! CRITICAL !!! Patching address 0x{descriptor.TargetAddress:X} with INVALID value 0x{targetValue:X} for NID {descriptor.ImportNid ?? "(null)"}"); - Log.Warning($" SymbolValue=0x{descriptor.SymbolValue:X}, Addend=0x{descriptor.Addend:X}, StubAddress=0x{(addressesByNid.TryGetValue(descriptor.ImportNid ?? "", out var sa) ? sa : 0):X}"); + throw new InvalidOperationException($"Import stub not found for NID '{descriptor.ImportNid}'."); } } - if (!TryWriteUInt64(virtualMemory, descriptor.TargetAddress, targetValue)) + var targetValue = ComputeRelocationValue(descriptor, symbolValue); + + if (targetValue < 0x1000 && descriptor.ValueKind is + RelocationValueKind.TlsOffset or + RelocationValueKind.PcRelative or + RelocationValueKind.SymbolSize) { - throw new InvalidDataException($"Failed to patch relocation at 0x{descriptor.TargetAddress:X16}."); + // A TLS offset (TPOFF64/DTPOFF64) is a signed displacement, not a + // mapped address, so a small or negative value here is expected. + } + else if (targetValue < 0x1000 && !descriptor.IsWeak) + { + if (descriptor.ValueKind == RelocationValueKind.TlsModuleId) + { + Console.Error.WriteLine( + $"[LOADER][TLS] Patching DTPMOD64 at 0x{descriptor.TargetAddress:X} with module id 0x{targetValue:X}"); + } + else + { + Console.Error.WriteLine($"[LOADER] !!! CRITICAL !!! Patching address 0x{descriptor.TargetAddress:X} with INVALID value 0x{targetValue:X} for NID {descriptor.ImportNid ?? "(null)"}"); + Console.Error.WriteLine($"[LOADER] SymbolValue=0x{descriptor.SymbolValue:X}, Addend=0x{descriptor.Addend:X}, StubAddress=0x{(addressesByNid.TryGetValue(descriptor.ImportNid ?? "", out var sa) ? sa : 0):X}"); + } + } + + if (!TryWriteRelocationValue(virtualMemory, descriptor, targetValue, out var writeError)) + { + throw new InvalidDataException( + $"Failed to patch relocation at 0x{descriptor.TargetAddress:X16}: {writeError}"); } if (descriptor.TargetAddress >= 0x00000008030FC300UL && descriptor.TargetAddress <= 0x00000008030FC3F0UL) { - Log.Debug( - $"[RELOC] target=0x{descriptor.TargetAddress:X16} value=0x{targetValue:X16} addend=0x{descriptor.Addend:X} nid={(descriptor.ImportNid ?? "")}"); + Console.Error.WriteLine( + $"[LOADER][RELOC] target=0x{descriptor.TargetAddress:X16} value=0x{targetValue:X16} addend=0x{descriptor.Addend:X} nid={(descriptor.ImportNid ?? "")}"); } } @@ -794,29 +896,36 @@ public sealed class SelfLoader : ISelfLoader { if (IsFocusRelocationOffset(relocation.Offset, imageBase)) { - Log.Debug( - $"[FOCUS][SCAN] off=0x{relocation.Offset:X16} type={relocation.Type} sym={relocation.SymbolIndex} addend=0x{relocation.Addend:X}"); + Console.Error.WriteLine( + $"[LOADER][FOCUS][SCAN] off=0x{relocation.Offset:X16} type={relocation.Type} sym={relocation.SymbolIndex} addend=0x{relocation.Addend:X}"); } if (!IsSupportedRelocationType(relocation.Type)) { - if (IsFocusRelocationOffset(relocation.Offset, imageBase)) + // Surface unsupported relocation types loudly instead of + // skipping silently: TLS-relative (17/18), COPY (5), and + // IRELATIVE/ifunc (37) leave their targets unrelocated, which + // manifests later as reads of zero or calls into address 0. + ReportUnsupportedRelocation(relocation.Type, relocation.Offset, imageBase); + if (relocation.Type is 5 or 37) { - Log.Debug($"[FOCUS][SKIP] unsupported type={relocation.Type}"); + throw new NotSupportedException( + $"Relocation type {relocation.Type} requires deferred runtime-linker processing."); } continue; } - if (!TryResolveMappedAddress(virtualMemory, relocation.Offset, imageBase, sizeof(ulong), out var targetAddress)) + var relocationWriteSize = GetRelocationWriteSize(relocation.Type); + if (!TryResolveMappedAddress(virtualMemory, relocation.Offset, imageBase, relocationWriteSize, out var targetAddress)) { if (IsFocusRelocationOffset(relocation.Offset, imageBase)) { - Log.Debug("[FOCUS][SKIP] target address not mapped"); + Console.Error.WriteLine("[LOADER][FOCUS][SKIP] target address not mapped"); } continue; } - if (relocation.Type == RelocationTypeRelative) + if (relocation.Type is RelocationTypeRelative or RelocationTypeRelative64) { descriptors.Add(new RelocationDescriptor( targetAddress, @@ -830,7 +939,12 @@ public sealed class SelfLoader : ISelfLoader if (relocation.Type == RelocationTypeTlsModuleId) { - var dtpmodValue = tlsModuleId == 0 ? 1u : tlsModuleId; + if (tlsModuleId == 0) + { + throw new InvalidDataException( + $"R_X86_64_DTPMOD64 at 0x{targetAddress:X16} references a module without PT_TLS."); + } + var dtpmodValue = tlsModuleId; descriptors.Add(new RelocationDescriptor( targetAddress, 0, @@ -851,50 +965,100 @@ public sealed class SelfLoader : ISelfLoader { if (targetAddress >= FocusRelocGuestStart && targetAddress <= FocusRelocGuestEnd) { - Log.Debug($"[FOCUS][SKIP] symbol read failed index={symbolIndex}"); + Console.Error.WriteLine($"[LOADER][FOCUS][SKIP] symbol read failed index={symbolIndex}"); } continue; } - var addend = relocation.Type is RelocationTypeGlobalData or RelocationTypeJumpSlot ? 0 : relocation.Addend; - var symbolBind = GetSymbolBind(symbol.Info); - if (symbolBind == SymbolBindLocal) + if (relocation.Type is RelocationTypeTlsDtpOff64 or RelocationTypeTlsTpOff64) { - var symbolAddress = ResolveMappedAddressOrFallback(virtualMemory, symbol.Value, imageBase); - if (symbolAddress == 0) + // Variant II static TLS: the module block sits at + // [tp - blockSize, tp). DTPOFF64 is the module-relative offset + // (st_value + addend); TPOFF64 is that offset expressed relative + // to the thread pointer, i.e. minus the aligned block size. + var tlsSymbolOffset = AddSigned(symbol.Value, relocation.Addend); + if (!GuestTlsTemplate.TryGetStaticOffset(tlsModuleId, out var moduleStaticOffset)) { - Log.Warning( - $"Skipping local relocation with invalid symbol value 0x{symbol.Value:X} " + - $"at target 0x{targetAddress:X16}, type={relocation.Type}, sym={symbolIndex}"); + Console.Error.WriteLine( + $"[LOADER][TLS] Missing PT_TLS registration for module {tlsModuleId}; " + + $"cannot apply relocation {relocation.Type} at 0x{targetAddress:X16}."); + continue; } - + var tlsValue = relocation.Type == RelocationTypeTlsTpOff64 + ? unchecked(tlsSymbolOffset - moduleStaticOffset) + : tlsSymbolOffset; descriptors.Add(new RelocationDescriptor( targetAddress, - addend, + 0, null, - symbolAddress, - RelocationValueKind.Pointer, + tlsValue, + RelocationValueKind.TlsOffset, IsDataImport: false)); continue; } + var symbolBind = GetSymbolBind(symbol.Info); + if (symbolIndex == 0) + { + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, + targetAddress, + symbol, + symbolAddress: 0, + importNid: null, + isWeak: false)); + continue; + } + + if (symbolBind == SymbolBindLocal) + { + var symbolAddress = relocation.Type is RelocationTypeSize32 or RelocationTypeSize64 + ? 0 + : ResolveMappedAddressOrFallback(virtualMemory, symbol.Value, imageBase); + if (symbolAddress == 0) + { + if (relocation.Type is not (RelocationTypeSize32 or RelocationTypeSize64)) + { + Console.Error.WriteLine( + $"[LOADER] Skipping local relocation with invalid symbol value 0x{symbol.Value:X} " + + $"at target 0x{targetAddress:X16}, type={relocation.Type}, sym={symbolIndex}"); + continue; + } + } + + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, + targetAddress, + symbol, + symbolAddress, + importNid: null, + isWeak: false)); + continue; + } + if (symbol.Value != 0) { - var symbolAddress = ResolveMappedAddressOrFallback(virtualMemory, symbol.Value, imageBase); + var symbolAddress = relocation.Type is RelocationTypeSize32 or RelocationTypeSize64 + ? 0 + : ResolveMappedAddressOrFallback(virtualMemory, symbol.Value, imageBase); if (symbolAddress == 0) { - Log.Warning( - $"Skipping relocation with invalid symbol value 0x{symbol.Value:X} " + - $"at target 0x{targetAddress:X16}, type={relocation.Type}, sym={symbolIndex}"); + if (relocation.Type is not (RelocationTypeSize32 or RelocationTypeSize64)) + { + Console.Error.WriteLine( + $"[LOADER] Skipping relocation with invalid symbol value 0x{symbol.Value:X} " + + $"at target 0x{targetAddress:X16}, type={relocation.Type}, sym={symbolIndex}"); + continue; + } } - descriptors.Add(new RelocationDescriptor( + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, targetAddress, - addend, - null, + symbol, symbolAddress, - RelocationValueKind.Pointer, - IsDataImport: false)); + importNid: null, + isWeak: false)); continue; } @@ -902,16 +1066,26 @@ public sealed class SelfLoader : ISelfLoader { if (targetAddress >= FocusRelocGuestStart && targetAddress <= FocusRelocGuestEnd) { - Log.Debug($"[FOCUS][SKIP] bind={symbolBind} not importable"); + Console.Error.WriteLine($"[LOADER][FOCUS][SKIP] bind={symbolBind} not importable"); } continue; } if (!TryReadNullTerminatedAscii(stringTable, symbol.NameOffset, out var symbolName)) { + if (symbolBind == SymbolBindWeak) + { + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, + targetAddress, + symbol, + symbolAddress: 0, + importNid: null, + isWeak: true)); + } if (targetAddress >= FocusRelocGuestStart && targetAddress <= FocusRelocGuestEnd) { - Log.Debug($"[FOCUS][SKIP] symbol name read failed offset={symbol.NameOffset}"); + Console.Error.WriteLine($"[LOADER][FOCUS][SKIP] symbol name read failed offset={symbol.NameOffset}"); } continue; } @@ -919,6 +1093,16 @@ public sealed class SelfLoader : ISelfLoader var nid = ExtractNid(symbolName); if (string.IsNullOrWhiteSpace(nid)) { + if (symbolBind == SymbolBindWeak) + { + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, + targetAddress, + symbol, + symbolAddress: 0, + importNid: null, + isWeak: true)); + } continue; } @@ -927,16 +1111,74 @@ public sealed class SelfLoader : ISelfLoader orderedImportNids.Add(nid); } - descriptors.Add(new RelocationDescriptor( + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, targetAddress, - addend, - nid, - 0, - RelocationValueKind.Pointer, - IsDataImport: GetSymbolType(symbol.Info) == SymbolTypeObject)); + symbol, + symbolAddress: 0, + importNid: nid, + isWeak: symbolBind == SymbolBindWeak)); } } + private static RelocationDescriptor CreateSymbolRelocationDescriptor( + ElfRelocation relocation, + ulong targetAddress, + ElfSymbol symbol, + ulong symbolAddress, + string? importNid, + bool isWeak) + { + var valueKind = relocation.Type switch + { + RelocationTypePc32 or RelocationTypePlt32 or RelocationTypePc64 => RelocationValueKind.PcRelative, + RelocationTypeSize32 or RelocationTypeSize64 => RelocationValueKind.SymbolSize, + _ => RelocationValueKind.Pointer, + }; + var writeKind = relocation.Type switch + { + RelocationTypePc32 or RelocationTypePlt32 or RelocationTypeSigned32 => RelocationWriteKind.Int32, + RelocationTypeUnsigned32 or RelocationTypeSize32 => RelocationWriteKind.UInt32, + _ => RelocationWriteKind.UInt64, + }; + var symbolValue = valueKind == RelocationValueKind.SymbolSize + ? symbol.Size + : symbolAddress; + var addend = relocation.Type is RelocationTypeGlobalData or RelocationTypeJumpSlot + ? 0 + : relocation.Addend; + return new RelocationDescriptor( + targetAddress, + addend, + importNid, + symbolValue, + valueKind, + IsDataImport: GetSymbolType(symbol.Info) == SymbolTypeObject, + writeKind, + isWeak); + } + + private static bool ShouldCreateImportStub( + string nid, + IReadOnlyList descriptors, + IModuleManager? moduleManager) + { + for (var i = 0; i < descriptors.Count; i++) + { + var descriptor = descriptors[i]; + if (!string.Equals(descriptor.ImportNid, nid, StringComparison.Ordinal)) + { + continue; + } + if (!descriptor.IsWeak || moduleManager?.TryGetExport(nid, out _) == true) + { + return true; + } + } + + return false; + } + private static void RegisterRuntimeSymbolsAndHooks( ReadOnlySpan imageData, LoadContext loadContext, @@ -959,8 +1201,8 @@ public sealed class SelfLoader : ISelfLoader if (sectionSymbols > 0 || dynamicSymbols > 0) { - Log.Info( - $"Runtime symbol index populated: section={sectionSymbols}, dynamic={dynamicSymbols}, total={runtimeSymbols.Count}"); + Console.Error.WriteLine( + $"[LOADER] Runtime symbol index populated: section={sectionSymbols}, dynamic={dynamicSymbols}, total={runtimeSymbols.Count}"); } } @@ -1034,8 +1276,8 @@ public sealed class SelfLoader : ISelfLoader if (preInitializers.Count != 0 || initializers.Count != 0) { - Log.Info( - $"Initializers discovered: preinit={preInitializers.Count}, init={initializers.Count}"); + Console.Error.WriteLine( + $"[LOADER] Initializers discovered: preinit={preInitializers.Count}, init={initializers.Count}"); } } @@ -1418,8 +1660,8 @@ public sealed class SelfLoader : ISelfLoader var requiredBytes = checked((ulong)orderedImportNids.Count * ImportStubSlotSize); var mapSize = AlignUp(Math.Max(requiredBytes, (ulong)PageSize), PageSize); - Log.Debug( - $"CreateImportStubMapping: nids={orderedImportNids.Count}, required=0x{requiredBytes:X}, map_size=0x{mapSize:X}"); + Console.Error.WriteLine( + $"[LOADER] CreateImportStubMapping: nids={orderedImportNids.Count}, required=0x{requiredBytes:X}, map_size=0x{mapSize:X}"); if (mapSize > int.MaxValue) { throw new NotSupportedException("Import stub mapping exceeds 2 GB and is not supported."); @@ -1691,11 +1933,51 @@ public sealed class SelfLoader : ISelfLoader private static bool IsSupportedRelocationType(uint relocationType) { return relocationType is + RelocationTypeNone or RelocationTypeAbsolute64 or + RelocationTypePc32 or + RelocationTypePlt32 or RelocationTypeGlobalData or RelocationTypeJumpSlot or RelocationTypeRelative or - RelocationTypeTlsModuleId; + RelocationTypeUnsigned32 or + RelocationTypeSigned32 or + RelocationTypeTlsModuleId or + RelocationTypeTlsDtpOff64 or + RelocationTypeTlsTpOff64 or + RelocationTypePc64 or + RelocationTypeSize32 or + RelocationTypeSize64 or + RelocationTypeRelative64; + } + + private static readonly HashSet _reportedUnsupportedRelocationTypes = new(); + + private static void ReportUnsupportedRelocation(uint relocationType, ulong offset, ulong imageBase) + { + // Report each distinct unsupported type once to keep the log useful. + lock (_reportedUnsupportedRelocationTypes) + { + if (!_reportedUnsupportedRelocationTypes.Add(relocationType)) + { + if (IsFocusRelocationOffset(offset, imageBase)) + { + Console.Error.WriteLine($"[LOADER][FOCUS][SKIP] unsupported type={relocationType}"); + } + + return; + } + } + + var name = relocationType switch + { + 5 => "R_X86_64_COPY", + 37 => "R_X86_64_IRELATIVE (ifunc)", + _ => "unknown", + }; + Console.Error.WriteLine( + $"[LOADER][ERROR] Unsupported relocation type {relocationType} ({name}) rejected " + + $"(first at off=0x{offset:X16}); COPY requires dependency symbol storage and IRELATIVE requires resolver execution."); } private static ulong DetermineRequestedImageBase( @@ -1851,16 +2133,16 @@ public sealed class SelfLoader : ISelfLoader tableBytes = GC.AllocateUninitializedArray((int)size); var guestAddr = location + imageBase; - Log.Debug($"TryLoadTableBytes: trying guest address 0x{guestAddr:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: trying guest address 0x{guestAddr:X}"); if (virtualMemory.TryRead(guestAddr, tableBytes)) { - Log.Debug($"TryLoadTableBytes: loaded from guest memory at 0x{guestAddr:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: loaded from guest memory at 0x{guestAddr:X}"); return true; } if (virtualMemory.TryRead(location, tableBytes)) { - Log.Debug($"TryLoadTableBytes: loaded from absolute guest address 0x{location:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: loaded from absolute guest address 0x{location:X}"); return true; } @@ -1868,11 +2150,11 @@ public sealed class SelfLoader : ISelfLoader { var slice = elfData.Slice((int)location, (int)size); tableBytes = slice.ToArray(); - Log.Debug($"TryLoadTableBytes: loaded from elfData as file offset at 0x{location:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: loaded from elfData as file offset at 0x{location:X}"); return true; } - Log.Warning($"TryLoadTableBytes: FAILED for location 0x{location:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: FAILED for location 0x{location:X}"); tableBytes = Array.Empty(); return false; } @@ -1929,17 +2211,17 @@ public sealed class SelfLoader : ISelfLoader private static ulong ResolveMappedAddressOrFallback(IVirtualMemory virtualMemory, ulong address, ulong imageBase) { - Log.Debug($"ResolveMappedAddressOrFallback addr=0x{address:X} imageBase=0x{imageBase:X16}"); + Console.Error.WriteLine($"[LOADER][TEST] ResolveMappedAddressOrFallback addr=0x{address:X} imageBase=0x{imageBase:X16}"); if (address == 0) { - Log.Debug("-> return 0 (null)"); + Console.Error.WriteLine("[LOADER][TEST] -> return 0 (null)"); return 0; } if (TryResolveMappedAddress(virtualMemory, address, imageBase, 1, out var resolved)) { - Log.Debug($"-> resolved raw 0x{resolved:X16}"); + Console.Error.WriteLine($"[LOADER][TEST] -> resolved raw 0x{resolved:X16}"); return resolved; } @@ -1948,18 +2230,18 @@ public sealed class SelfLoader : ISelfLoader var rebased = address + imageBase; if (TryResolveMappedAddress(virtualMemory, rebased, imageBase, 1, out var resolvedRebased)) { - Log.Debug($"-> resolved rebased 0x{resolvedRebased:X16}"); + Console.Error.WriteLine($"[LOADER][TEST] -> resolved rebased 0x{resolvedRebased:X16}"); return resolvedRebased; } } if (address < 0x10000) { - Log.Debug($"-> reject small 0x{address:X}"); + Console.Error.WriteLine($"[LOADER][TEST] -> reject small 0x{address:X}"); return 0; } - Log.Debug($"-> fallback raw 0x{address:X}"); + Console.Error.WriteLine($"[LOADER][TEST] -> fallback raw 0x{address:X}"); return address; } @@ -2049,6 +2331,106 @@ public sealed class SelfLoader : ISelfLoader return virtualMemory.TryWrite(address, buffer); } + private static ulong ComputeRelocationValue(RelocationDescriptor descriptor, ulong resolvedSymbolValue) + { + var baseValue = descriptor.ValueKind == RelocationValueKind.SymbolSize + ? descriptor.SymbolValue + : resolvedSymbolValue; + var value = AddSigned(baseValue, descriptor.Addend); + return descriptor.ValueKind == RelocationValueKind.PcRelative + ? unchecked(value - descriptor.TargetAddress) + : value; + } + + private static bool TryWriteRelocationValue( + IVirtualMemory virtualMemory, + RelocationDescriptor descriptor, + ulong value, + out string? error) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + var length = sizeof(ulong); + switch (descriptor.WriteKind) + { + case RelocationWriteKind.UInt64: + BinaryPrimitives.WriteUInt64LittleEndian(buffer, value); + break; + + case RelocationWriteKind.UInt32: + if (value > uint.MaxValue) + { + error = $"value 0x{value:X16} overflows an unsigned 32-bit relocation"; + return false; + } + BinaryPrimitives.WriteUInt32LittleEndian(buffer, (uint)value); + length = sizeof(uint); + break; + + case RelocationWriteKind.Int32: + var signedValue = unchecked((long)value); + if (signedValue is < int.MinValue or > int.MaxValue) + { + error = $"value {signedValue} overflows a signed 32-bit relocation"; + return false; + } + BinaryPrimitives.WriteInt32LittleEndian(buffer, (int)signedValue); + length = sizeof(int); + break; + + default: + error = $"unknown relocation write kind {descriptor.WriteKind}"; + return false; + } + + error = virtualMemory.TryWrite(descriptor.TargetAddress, buffer[..length]) + ? null + : "target memory is not writable"; + return error is null; + } + + private static int GetRelocationWriteSize(uint relocationType) + { + return relocationType is + RelocationTypePc32 or + RelocationTypePlt32 or + RelocationTypeUnsigned32 or + RelocationTypeSigned32 or + RelocationTypeSize32 + ? sizeof(uint) + : sizeof(ulong); + } + + [Conditional("DEBUG")] + private static void RunRelocationSelfChecks() + { + var pc32 = new RelocationDescriptor( + TargetAddress: 0x1000, + Addend: -4, + ImportNid: null, + SymbolValue: 0x1800, + RelocationValueKind.PcRelative, + IsDataImport: false, + RelocationWriteKind.Int32); + Debug.Assert( + unchecked((long)ComputeRelocationValue(pc32, pc32.SymbolValue)) == 0x7FC, + "R_X86_64_PC32 did not apply S + A - P."); + + var weak = new RelocationDescriptor( + TargetAddress: 0x2000, + Addend: 7, + ImportNid: "weak", + SymbolValue: 0, + RelocationValueKind.Pointer, + IsDataImport: false, + IsWeak: true); + Debug.Assert( + ComputeRelocationValue(weak, resolvedSymbolValue: 0) == 7, + "An unresolved weak relocation did not use S=0."); + Debug.Assert( + !ShouldCreateImportStub("weak", [weak], moduleManager: null), + "An unresolved weak symbol incorrectly received a trap import stub."); + } + private static ulong AlignUp(ulong value, ulong alignment) { var mask = alignment - 1; @@ -2257,6 +2639,16 @@ public sealed class SelfLoader : ISelfLoader { throw new InvalidDataException($"Unsupported ELF program header entry size: {header.ProgramHeaderEntrySize}."); } + + // The CPU backend executes guest instructions natively, so a non + // x86-64 image can only fail deep in execution with an opaque SIGILL. + // Catch it here with an actionable message. (EM_X86_64 == 62.) + const ushort ElfMachineX86_64 = 62; + if (header.Machine != ElfMachineX86_64) + { + throw new InvalidDataException( + $"Unsupported ELF machine type 0x{header.Machine:X4}; SharpEmu only runs x86-64 (EM_X86_64) images."); + } } private static unsafe T ReadUnmanaged(ReadOnlySpan source, int offset) @@ -2337,10 +2729,25 @@ public sealed class SelfLoader : ISelfLoader public uint Type => (uint)(Info & uint.MaxValue); } + private readonly record struct ModuleTlsInfo(ulong MemorySize, ulong StaticOffset); + private enum RelocationValueKind : byte { Pointer = 0, - TlsModuleId = 1 + TlsModuleId = 1, + // A pre-computed TLS offset written verbatim (TPOFF64/DTPOFF64). Unlike + // Pointer it is a signed displacement, not a mapped address, so it is + // patched as-is without the low-address validity warning. + TlsOffset = 2, + PcRelative = 3, + SymbolSize = 4, + } + + private enum RelocationWriteKind : byte + { + UInt64 = 0, + UInt32 = 1, + Int32 = 2, } private readonly record struct RelocationDescriptor( @@ -2349,7 +2756,9 @@ public sealed class SelfLoader : ISelfLoader string? ImportNid, ulong SymbolValue, RelocationValueKind ValueKind, - bool IsDataImport); + bool IsDataImport, + RelocationWriteKind WriteKind = RelocationWriteKind.UInt64, + bool IsWeak = false); private enum SelfSegmentResolveStatus { @@ -2389,14 +2798,10 @@ public sealed class SelfLoader : ISelfLoader public ulong FileSize => _fileSize; public bool HasKnownLayout => - ((_ident0 == 0x4F && - _ident1 == 0x15 && - _ident2 == 0x3D && - _ident3 == 0x1D) || - (_ident0 == 0x54 && - _ident1 == 0x14 && - _ident2 == 0xF5 && - _ident3 == 0xEE)) && + _ident0 == 0x4F && + _ident1 == 0x15 && + _ident2 == 0x3D && + _ident3 == 0x1D && _ident4 == 0x00 && _ident5 == 0x01 && _ident6 == 0x01 && diff --git a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs index 7add040..f069d8f 100644 --- a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs +++ b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs @@ -48,7 +48,107 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA public PhysicalVirtualMemory(IHostMemory? hostMemory = null) { - _hostMemory = hostMemory ?? HostPlatform.Current.Memory; + _hostMemory = hostMemory ?? CrossPlatformHostMemory.Instance; + } + + private sealed class CrossPlatformHostMemory : IHostMemory + { + public static readonly CrossPlatformHostMemory Instance = new(); + + public ulong Allocate(ulong desiredAddress, ulong size, HostPageProtection protection) => + unchecked((ulong)HostMemory.Alloc( + (void*)desiredAddress, + (nuint)size, + HostMemory.MEM_RESERVE | HostMemory.MEM_COMMIT, + ToRawProtection(protection))); + + public ulong Reserve(ulong desiredAddress, ulong size, HostPageProtection protection) => + unchecked((ulong)HostMemory.Alloc( + (void*)desiredAddress, + (nuint)size, + HostMemory.MEM_RESERVE, + ToRawProtection(protection))); + + public bool Commit(ulong address, ulong size, HostPageProtection protection) => + HostMemory.Alloc( + (void*)address, + (nuint)size, + HostMemory.MEM_COMMIT, + ToRawProtection(protection)) != null; + + public bool Free(ulong address) => + HostMemory.Free((void*)address, 0, HostMemory.MEM_RELEASE); + + public bool Protect( + ulong address, + ulong size, + HostPageProtection protection, + out uint rawOldProtection) => + HostMemory.Protect( + (void*)address, + (nuint)size, + ToRawProtection(protection), + out rawOldProtection); + + public bool ProtectRaw( + ulong address, + ulong size, + uint rawProtection, + out uint rawOldProtection) => + HostMemory.Protect((void*)address, (nuint)size, rawProtection, out rawOldProtection); + + public bool Query(ulong address, out HostRegionInfo info) + { + if (HostMemory.Query((void*)address, out var raw) == 0) + { + info = default; + return false; + } + + var state = raw.State switch + { + HostMemory.MEM_FREE_STATE => HostRegionState.Free, + HostMemory.MEM_RESERVE => HostRegionState.Reserved, + _ => HostRegionState.Committed, + }; + + info = new HostRegionInfo( + raw.BaseAddress, + raw.AllocationBase, + raw.RegionSize, + state, + raw.State, + FromRawProtection(raw.Protect), + raw.Protect, + raw.AllocationProtect); + return true; + } + + public void FlushInstructionCache(ulong address, ulong size) => + HostMemory.FlushInstructionCache((void*)address, (nuint)size); + + private static uint ToRawProtection(HostPageProtection protection) => protection switch + { + HostPageProtection.NoAccess => HostMemory.PAGE_NOACCESS, + HostPageProtection.ReadOnly => HostMemory.PAGE_READONLY, + HostPageProtection.ReadWrite => HostMemory.PAGE_READWRITE, + HostPageProtection.Execute => HostMemory.PAGE_EXECUTE, + HostPageProtection.ReadExecute => HostMemory.PAGE_EXECUTE_READ, + HostPageProtection.ReadWriteExecute => HostMemory.PAGE_EXECUTE_READWRITE, + HostPageProtection.ExecuteWriteCopy => 0x80, + _ => HostMemory.PAGE_NOACCESS, + }; + + private static HostPageProtection FromRawProtection(uint protection) => protection switch + { + HostMemory.PAGE_READONLY => HostPageProtection.ReadOnly, + HostMemory.PAGE_READWRITE => HostPageProtection.ReadWrite, + HostMemory.PAGE_EXECUTE => HostPageProtection.Execute, + HostMemory.PAGE_EXECUTE_READ => HostPageProtection.ReadExecute, + HostMemory.PAGE_EXECUTE_READWRITE => HostPageProtection.ReadWriteExecute, + 0x80 => HostPageProtection.ExecuteWriteCopy, + _ => HostPageProtection.NoAccess, + }; } public bool TryAllocateAtExact(ulong desiredAddress, ulong size, bool executable, out ulong actualAddress) @@ -249,42 +349,11 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA var requestedCursor = AlignUp(desiredAddress, effectiveAlignment); var cursor = GetAllocationSearchCursor(desiredAddress, requestedCursor, effectiveAlignment, executable); - // Under Rosetta 2 the kernel can ignore placement hints for whole - // windows, so page-stepped exact probes are pathological on macOS. - // Linux must keep using the exact-address search below: PS5 resource - // descriptors cannot represent ordinary 0x7F... host mappings. Linux - // HostMemory uses MAP_FIXED_NOREPLACE, making those low-address probes - // safe without clobbering existing host mappings. - if (OperatingSystem.IsMacOS()) + // POSIX treats the requested address as a hint, and Rosetta may + // relocate whole guest windows. Over-allocate once so the returned + // host range always contains an aligned guest-visible start. + if (!OperatingSystem.IsWindows()) { - // Prefer the requested low address. Besides matching the guest - // address model, this keeps the allocation representable by every - // PS5 GPU descriptor (the strictest ones carry 40 address bits). - try - { - var exactAddress = AllocateAt( - cursor, - alignedSize, - executable, - allowAlternative: false); - if (exactAddress == cursor) - { - actualAddress = exactAddress; - UpdateAllocationSearchCursor( - desiredAddress, - effectiveAlignment, - executable, - exactAddress + alignedSize); - return true; - } - } - catch - { - } - - // Over-allocate by the alignment so a kernel-chosen placement - // always contains an aligned start; the unused head/tail stays - // part of the tracked region and is simply never handed out. var reserveSize = effectiveAlignment > PageSize ? alignedSize + effectiveAlignment : alignedSize; @@ -294,10 +363,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA if (posixAddress != 0) { var alignedBase = AlignUp(posixAddress, effectiveAlignment); - const ulong gpuAddressLimit = 1UL << 40; - if (alignedBase < gpuAddressLimit && - alignedSize <= gpuAddressLimit - alignedBase && - alignedBase + alignedSize <= posixAddress + reserveSize) + if (alignedBase + alignedSize <= posixAddress + reserveSize) { actualAddress = alignedBase; UpdateAllocationSearchCursor(desiredAddress, effectiveAlignment, executable, alignedBase + alignedSize); @@ -327,19 +393,10 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA continue; } - try - { - actualAddress = AllocateAt(cursor, alignedSize, executable, allowAlternative: false); - if (actualAddress == cursor) - { - UpdateAllocationSearchCursor(desiredAddress, effectiveAlignment, executable, actualAddress + alignedSize); - return true; - } - - actualAddress = 0; - } - catch + if (TryAllocateAtExact(cursor, alignedSize, executable, out actualAddress)) { + UpdateAllocationSearchCursor(desiredAddress, effectiveAlignment, executable, actualAddress + alignedSize); + return true; } cursor = AlignUp(cursor + effectiveAlignment, effectiveAlignment); diff --git a/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs b/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs index beeaf7f..b929248 100644 --- a/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs +++ b/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs @@ -7,25 +7,21 @@ using SharpEmu.Core.Cpu.Disasm; using SharpEmu.Core.Loader; using SharpEmu.Core.Memory; using SharpEmu.HLE; -using SharpEmu.HLE.Host; using SharpEmu.Libs.VideoOut; using SharpEmu.Libs.Kernel; using SharpEmu.Libs.AppContent; using SharpEmu.Libs.SaveData; -using SharpEmu.Logging; +using SharpEmu.Libs.Fiber; using System.Buffers.Binary; using System.Collections.Generic; using System.Linq; using System.Text; -using System.IO; namespace SharpEmu.Core.Runtime; public sealed class SharpEmuRuntime : ISharpEmuRuntime { - private static readonly SharpEmuLogger Log = SharpEmuLog.For("Runtime"); - - private readonly record struct LoadedModuleImage(string Path, SelfImage Image); + private readonly record struct LoadedModuleImage(string Path, SelfImage Image, int Handle, bool StartAtBoot); private static readonly HashSet PreloadSkipModules = new(StringComparer.OrdinalIgnoreCase) { @@ -89,19 +85,14 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime moduleManager.RegisterExports(SharpEmu.Generated.SysAbiExportRegistry.CreateExports(Generation.Gen4 | Generation.Gen5)); moduleManager.Freeze(); - // Resolve the host platform once at the composition root; on unsupported - // OSes this throws PlatformNotSupportedException with a clear message - // instead of failing on the first native call. - var hostPlatform = HostPlatform.Current; - - var virtualMemory = new PhysicalVirtualMemory(hostPlatform.Memory); + var virtualMemory = new PhysicalVirtualMemory(); var fileSystem = new PhysicalFileSystem(); return new SharpEmuRuntime( new SelfLoader(), virtualMemory, - new CpuDispatcher(virtualMemory, moduleManager, hostPlatform: hostPlatform), + new CpuDispatcher(virtualMemory, moduleManager), moduleManager, Aerolib.Instance, cpuExecutionOptions, @@ -139,20 +130,20 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime { var normalizedEbootPath = Path.GetFullPath(ebootPath); using var app0Binding = BindApp0Root(normalizedEbootPath); - Log.Info($"Loading: {ebootPath}"); + Console.Error.WriteLine($"[RUNTIME] Loading: {ebootPath}"); LastExecutionDiagnostics = null; LastExecutionTrace = null; LastSessionSummary = null; LastBasicBlockTrace = null; LastMilestoneLog = null; + FiberExports.ResetRuntimeState(); KernelModuleRegistry.Reset(); var image = LoadImage(normalizedEbootPath); - VideoOutExports.ConfigureApplicationInfo(image.Title, image.TitleId, image.Version, BuildInfo.CommitSha); + VideoOutExports.ConfigureApplicationInfo(image.Title, image.TitleId, image.Version); SaveDataExports.ConfigureApplicationInfo(image.TitleId); - LogAppBundleInfo(normalizedEbootPath, image); - RegisterLoadedModule(normalizedEbootPath, image, isMain: true, isSystemModule: false); + _ = RegisterLoadedModule(normalizedEbootPath, image, isMain: true, isSystemModule: false); KernelRuntimeCompatExports.ConfigureProcessProcParamAddress(image.ProcParamAddress); - Log.Info($"Entry: 0x{image.EntryPoint:X16}"); + Console.Error.WriteLine($"[RUNTIME] Entry: 0x{image.EntryPoint:X16}"); var generation = image.ElfHeader.AbiVersion == 2 ? Generation.Gen5 : Generation.Gen4; var activeImportStubs = new Dictionary(image.ImportStubs); var activeRuntimeSymbols = new Dictionary(image.RuntimeSymbols, StringComparer.Ordinal); @@ -175,7 +166,7 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime processImageName); if (initializerResult is { } failedInitializerResult) { - Log.Error($"Initializer dispatch failed: {failedInitializerResult}"); + Console.Error.WriteLine($"[RUNTIME] Initializer dispatch failed: {failedInitializerResult}"); LastExecutionTrace = _cpuDispatcher.LastImportResolutionTrace; LastMilestoneLog = _cpuDispatcher.LastMilestoneLog; LastSessionSummary = BuildSessionSummary(_cpuDispatcher.LastSessionSummary); @@ -183,8 +174,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return failedInitializerResult; } - Log.Info($"Dispatching, gen: {generation}"); - Log.Debug($"About to call DispatchEntry with entryPoint=0x{image.EntryPoint:X16}"); + Console.Error.WriteLine($"[RUNTIME] Dispatching, gen: {generation}"); + Console.Error.WriteLine($"[RUNTIME] About to call DispatchEntry with entryPoint=0x{image.EntryPoint:X16}"); var result = _cpuDispatcher.DispatchEntry( image.EntryPoint, @@ -194,8 +185,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime processImageName, _cpuExecutionOptions); - Log.Info($"DispatchEntry returned: {result}"); - Log.Info($"Dispatch result: {result}"); + Console.Error.WriteLine($"[RUNTIME] DispatchEntry returned: {result}"); + Console.Error.WriteLine($"[RUNTIME] Dispatch result: {result}"); LastExecutionTrace = _cpuDispatcher.LastImportResolutionTrace; LastMilestoneLog = _cpuDispatcher.LastMilestoneLog; LastSessionSummary = BuildSessionSummary(_cpuDispatcher.LastSessionSummary); @@ -366,66 +357,6 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return result; } - private static void LogAppBundleInfo(string ebootPath, SelfImage image) - { - var executableName = Path.GetFileName(ebootPath); - if (string.IsNullOrWhiteSpace(executableName)) - { - executableName = "eboot.bin"; - } - - var displayName = string.IsNullOrWhiteSpace(image.Title) ? "(unknown)" : image.Title!.Trim(); - var titleId = string.IsNullOrWhiteSpace(image.TitleId) ? "(unknown)" : image.TitleId!.Trim(); - var version = string.IsNullOrWhiteSpace(image.Version) ? "(unknown)" : image.Version!.Trim(); - var contentId = ResolveContentId(Path.GetDirectoryName(ebootPath)) ?? "(unknown)"; - - var builder = new StringBuilder(); - builder.AppendLine("App bundle info:"); - builder.AppendLine($"- Display name: {displayName}"); - builder.AppendLine($"- Version: {version}"); - builder.AppendLine($"- Title ID: {titleId}"); - builder.AppendLine($"- Content ID: {contentId}"); - builder.AppendLine($"- Executable: {executableName}"); - builder.Append("- Platform: PlayStation 5"); - Log.Info(builder.ToString()); - } - - private static string? ResolveContentId(string? bundleRoot) - { - if (string.IsNullOrEmpty(bundleRoot)) - { - return null; - } - - foreach (var candidate in new[] - { - Path.Combine(bundleRoot, "sce_sys", "param.json"), - Path.Combine(bundleRoot, "param.json"), - }) - { - if (!File.Exists(candidate)) - { - continue; - } - - try - { - using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllBytes(candidate)); - if (doc.RootElement.TryGetProperty("contentId", out var contentId)) - { - return contentId.GetString(); - } - } - catch (Exception ex) when (ex is IOException or System.Text.Json.JsonException) - { - } - - break; - } - - return null; - } - private static App0BindingScope? BindApp0Root(string normalizedEbootPath) { const string app0VariableName = "SHARPEMU_APP0_DIR"; @@ -440,6 +371,20 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return null; } + // Dump tools commonly place decrypted executables in an app0/decrypted + // sidecar while leaving Unity data in the parent app0 directory. Keep + // loading code/modules beside the decrypted eboot, but mount /app0 at + // the content-bearing parent so boot.config, metadata and assets resolve. + if (string.Equals(Path.GetFileName(app0Root), "decrypted", StringComparison.OrdinalIgnoreCase)) + { + var parentRoot = Path.GetDirectoryName(app0Root); + if (!string.IsNullOrWhiteSpace(parentRoot) && + Directory.Exists(Path.Combine(parentRoot, "Media"))) + { + app0Root = parentRoot; + } + } + Environment.SetEnvironmentVariable(app0VariableName, app0Root); return new App0BindingScope(app0VariableName); } @@ -485,6 +430,56 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return null; } + private bool TryGetEhFrameInfo( + SelfImage image, + ulong imageSize, + out ulong ehFrameHeaderAddress, + out ulong ehFrameAddress, + out ulong ehFrameSize) + { + ehFrameHeaderAddress = 0; + ehFrameAddress = 0; + ehFrameSize = 0; + var imageBase = image.EntryPoint >= image.ElfHeader.EntryPoint + ? image.EntryPoint - image.ElfHeader.EntryPoint + : 0UL; + for (var i = 0; i < image.ProgramHeaders.Count; i++) + { + var header = image.ProgramHeaders[i]; + if (header.HeaderType != ProgramHeaderType.GnuEhFrame || header.MemorySize < 8) + { + continue; + } + + var headerAddress = imageBase + header.VirtualAddress; + Span ehHeader = stackalloc byte[8]; + if (!_virtualMemory.TryRead(headerAddress, ehHeader) || + ehHeader[0] != 1 || + ehHeader[1] != 0x1B) + { + continue; + } + + var relativeOffset = BinaryPrimitives.ReadInt32LittleEndian(ehHeader[4..]); + ehFrameHeaderAddress = headerAddress; + ehFrameAddress = unchecked((ulong)((long)headerAddress + 4 + relativeOffset)); + if (ehFrameAddress < 0x10000) + { + continue; + } + + var relativeEhFrameAddress = ehFrameAddress - imageBase; + ehFrameSize = header.VirtualAddress > relativeEhFrameAddress + ? header.VirtualAddress - relativeEhFrameAddress + : imageSize > header.VirtualAddress + ? imageSize - header.VirtualAddress + : 0; + return true; + } + + return false; + } + private OrbisGen2Result? RunPreloadedModuleInitializers( IReadOnlyList loadedModuleImages, Generation generation, @@ -494,20 +489,30 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime for (var i = 0; i < loadedModuleImages.Count; i++) { var loadedModule = loadedModuleImages[i]; + if (!loadedModule.StartAtBoot) + { + continue; + } + var initEntryPoint = loadedModule.Image.InitFunctionEntryPoint; if (initEntryPoint < 0x10000) { continue; } + if (!KernelModuleRegistry.TryBeginModuleStart(loadedModule.Handle, out _)) + { + continue; + } + var moduleName = Path.GetFileName(loadedModule.Path); if (string.IsNullOrWhiteSpace(moduleName)) { moduleName = $"module#{i}"; } - Log.Info( - $"Starting module {moduleName}: dt_init=0x{initEntryPoint:X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Starting module {moduleName}: dt_init=0x{initEntryPoint:X16}"); var result = _cpuDispatcher.DispatchModuleInitializer( initEntryPoint, @@ -516,10 +521,13 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime activeRuntimeSymbols, moduleName, _cpuExecutionOptions); + KernelModuleRegistry.CompleteModuleStart( + loadedModule.Handle, + result == OrbisGen2Result.ORBIS_GEN2_OK); if (result != OrbisGen2Result.ORBIS_GEN2_OK) { - Log.Error( - $"Module start failed: {moduleName} -> {result}"); + Console.Error.WriteLine( + $"[RUNTIME] Module start failed: {moduleName} -> {result}"); return result; } } @@ -540,8 +548,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return null; } - Log.Info( - $"Running initializers for {label}: preinit={image.PreInitializerFunctions.Count}, init={image.InitializerFunctions.Count}"); + Console.Error.WriteLine( + $"[RUNTIME] Running initializers for {label}: preinit={image.PreInitializerFunctions.Count}, init={image.InitializerFunctions.Count}"); var result = RunInitializerList( $"{label}:preinit", @@ -580,8 +588,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime continue; } - Log.Debug( - $" Initializer {label}[{i}] -> 0x{initializerAddress:X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Initializer {label}[{i}] -> 0x{initializerAddress:X16}"); var result = _cpuDispatcher.DispatchEntry( initializerAddress, @@ -613,12 +621,17 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime var moduleDirectories = new[] { - Path.Combine(ebootDirectory, "sce_module"), - Path.Combine(ebootDirectory, "sce_modules"), - Path.Combine(ebootDirectory, "Media", "Modules"), + (Path: Path.Combine(ebootDirectory, "sce_module"), StartAtBoot: true), + (Path: Path.Combine(ebootDirectory, "sce_modules"), StartAtBoot: true), + (Path: Path.Combine(ebootDirectory, "Media", "Modules"), StartAtBoot: true), + // Unity native plugins are loaded later through sceKernelLoadStartModule. Map + // them up front so the HLE loader can return a real module handle and dlsym + // can resolve their exports, but defer DT_INIT until the guest requests them. + (Path: Path.Combine(ebootDirectory, "Media", "Plugins"), StartAtBoot: false), } - .Distinct(StringComparer.OrdinalIgnoreCase) - .Where(Directory.Exists) + .GroupBy(entry => entry.Path, StringComparer.OrdinalIgnoreCase) + .Select(group => group.First()) + .Where(entry => Directory.Exists(entry.Path)) .ToArray(); if (moduleDirectories.Length == 0) @@ -628,28 +641,30 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime var allModulePaths = moduleDirectories .SelectMany(directory => Directory - .EnumerateFiles(directory) - .OrderBy(path => path, StringComparer.OrdinalIgnoreCase)) - .Where(path => + .EnumerateFiles(directory.Path) + .OrderBy(path => path, StringComparer.OrdinalIgnoreCase) + .Select(path => (Path: path, directory.StartAtBoot))) + .Where(entry => { - var extension = Path.GetExtension(path); + var extension = Path.GetExtension(entry.Path); return string.Equals(extension, ".prx", StringComparison.OrdinalIgnoreCase) || string.Equals(extension, ".sprx", StringComparison.OrdinalIgnoreCase); }) - .Distinct(StringComparer.OrdinalIgnoreCase) + .GroupBy(entry => entry.Path, StringComparer.OrdinalIgnoreCase) + .Select(group => group.First()) .ToArray(); var modulePaths = allModulePaths - .Where(ShouldPreloadModule) + .Where(entry => ShouldPreloadModule(entry.Path)) .ToArray(); var skippedModules = allModulePaths - .Where(path => !ShouldPreloadModule(path)) - .Select(Path.GetFileName) + .Where(entry => !ShouldPreloadModule(entry.Path)) + .Select(entry => Path.GetFileName(entry.Path)) .Where(name => !string.IsNullOrWhiteSpace(name)) .ToArray(); if (skippedModules.Length > 0) { - Log.Info($"Skipping {skippedModules.Length} core module(s): {string.Join(", ", skippedModules)}"); + Console.Error.WriteLine($"[RUNTIME] Skipping {skippedModules.Length} core module(s): {string.Join(", ", skippedModules)}"); } if (modulePaths.Length == 0) @@ -657,14 +672,15 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return loadedImages; } - Log.Debug($"Module search directories: {string.Join(", ", moduleDirectories)}"); - Log.Info($"Loading {modulePaths.Length} module(s)..."); + Console.Error.WriteLine($"[RUNTIME] Module search directories: {string.Join(", ", moduleDirectories.Select(entry => entry.Path))}"); + Console.Error.WriteLine($"[RUNTIME] Loading {modulePaths.Length} module(s)..."); var loadedModules = 0; var failedModules = 0; var mergedImportCount = 0; var mergedSymbolCount = 0; - foreach (var modulePath in modulePaths) + foreach (var moduleEntry in modulePaths) { + var modulePath = moduleEntry.Path; try { var fileInfo = new FileInfo(modulePath); @@ -689,25 +705,52 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime mergedImportCount += MergeImportStubs(importStubs, moduleImage.ImportStubs, modulePath); mergedSymbolCount += MergeRuntimeSymbols(runtimeSymbols, moduleImage.RuntimeSymbols); - RegisterLoadedModule(modulePath, moduleImage, isMain: false, isSystemModule: false); - loadedImages.Add(new LoadedModuleImage(modulePath, moduleImage)); + InstallNativePluginCompatibilityHooks(importStubs, moduleImage, modulePath); + var moduleHandle = RegisterLoadedModule(modulePath, moduleImage, isMain: false, isSystemModule: false); + var moduleName = Path.GetFileName(modulePath); + var startAtBoot = moduleEntry.StartAtBoot || + string.Equals(moduleName, "libfmod.prx", StringComparison.OrdinalIgnoreCase) || + string.Equals(moduleName, "libfmodstudio.prx", StringComparison.OrdinalIgnoreCase); + loadedImages.Add(new LoadedModuleImage(modulePath, moduleImage, moduleHandle, startAtBoot)); loadedModules++; - Log.Info( - $"Loaded module {Path.GetFileName(modulePath)}: entry=0x{moduleImage.EntryPoint:X16}, imports={moduleImage.ImportStubs.Count}, symbols={moduleImage.RuntimeSymbols.Count}"); + Console.Error.WriteLine( + $"[RUNTIME] Loaded module {Path.GetFileName(modulePath)}: entry=0x{moduleImage.EntryPoint:X16}, imports={moduleImage.ImportStubs.Count}, symbols={moduleImage.RuntimeSymbols.Count}"); } catch (Exception ex) { failedModules++; - Log.Error($"Module load failed: {modulePath} ({ex.GetType().Name}: {ex.Message})", ex); + Console.Error.WriteLine($"[RUNTIME] Module load failed: {modulePath} ({ex.GetType().Name}: {ex.Message})"); } } - Log.Info( - $"Module preload summary: loaded={loadedModules}, failed={failedModules}, merged_imports={mergedImportCount}, merged_symbols={mergedSymbolCount}"); + Console.Error.WriteLine( + $"[RUNTIME] Module preload summary: loaded={loadedModules}, failed={failedModules}, merged_imports={mergedImportCount}, merged_symbols={mergedSymbolCount}"); return loadedImages; } + private static void InstallNativePluginCompatibilityHooks( + IDictionary importStubs, + SelfImage moduleImage, + string modulePath) + { + if (!string.Equals(Path.GetFileName(modulePath), "libfmod.prx", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + ReadOnlySpan compatibilityNids = ["uPLTdl3psGk"]; + foreach (var nid in compatibilityNids) + { + if (moduleImage.RuntimeSymbols.TryGetValue(nid, out var address) && address >= 0x10000) + { + importStubs[address] = nid; + Console.Error.WriteLine( + $"[RUNTIME] Installed FMOD compatibility hook: {nid} -> 0x{address:X16}"); + } + } + } + private void RebindImportedDataSymbols( SelfImage mainImage, IReadOnlyList loadedModuleImages, @@ -724,8 +767,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime if (rebound != 0 || unresolved != 0) { - Log.Info( - $"Imported data rebind: rebound={rebound}, unresolved={unresolved}"); + Console.Error.WriteLine( + $"[RUNTIME] Imported data rebind: rebound={rebound}, unresolved={unresolved}"); } } @@ -757,8 +800,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime { if (logRebind) { - Log.Warning( - $"Imported data unresolved: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} addend=0x{unchecked((ulong)relocation.Addend):X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Imported data unresolved: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} addend=0x{unchecked((ulong)relocation.Addend):X16}"); } unresolved++; @@ -770,8 +813,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime { if (logRebind) { - Log.Error( - $"Imported data write-failed: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} value=0x{reboundValue:X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Imported data write-failed: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} value=0x{reboundValue:X16}"); } unresolved++; @@ -780,8 +823,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime if (logRebind) { - Log.Debug( - $"Imported data rebound: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} value=0x{reboundValue:X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Imported data rebound: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} value=0x{reboundValue:X16}"); } rebound++; @@ -817,8 +860,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime { if (!string.Equals(existingNid, nid, StringComparison.Ordinal)) { - Log.Warning( - $"Import stub conflict at 0x{address:X16}: keep={existingNid}, skip={nid} ({Path.GetFileName(modulePath)})"); + Console.Error.WriteLine( + $"[RUNTIME] Import stub conflict at 0x{address:X16}: keep={existingNid}, skip={nid} ({Path.GetFileName(modulePath)})"); } continue; @@ -912,7 +955,7 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return !PreloadSkipModules.Contains(fileName); } - private static void RegisterLoadedModule(string modulePath, SelfImage image, bool isMain, bool isSystemModule) + private int RegisterLoadedModule(string modulePath, SelfImage image, bool isMain, bool isSystemModule) { if (!TryComputeImageRange(image, out var baseAddress, out var size)) { @@ -920,15 +963,27 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime size = 0; } + _ = TryGetEhFrameInfo( + image, + size, + out var ehFrameHeaderAddress, + out var ehFrameAddress, + out var ehFrameSize); var handle = KernelModuleRegistry.RegisterModule( modulePath, baseAddress, size, image.EntryPoint, + image.InitFunctionEntryPoint, + ehFrameHeaderAddress, + ehFrameAddress, + ehFrameSize, isMain, isSystemModule); - Log.Info( - $"Registered module handle={handle} name={Path.GetFileName(modulePath)} base=0x{baseAddress:X16} size=0x{size:X16}"); + KernelModuleRegistry.RegisterModuleSymbols(handle, image.RuntimeSymbols); + Console.Error.WriteLine( + $"[RUNTIME] Registered module handle={handle} name={Path.GetFileName(modulePath)} base=0x{baseAddress:X16} size=0x{size:X16}"); + return handle; } private static bool TryComputeImageRange(SelfImage image, out ulong baseAddress, out ulong size) diff --git a/src/SharpEmu.HLE/CpuContext.cs b/src/SharpEmu.HLE/CpuContext.cs index adb3768..66c24d6 100644 --- a/src/SharpEmu.HLE/CpuContext.cs +++ b/src/SharpEmu.HLE/CpuContext.cs @@ -26,6 +26,12 @@ public sealed class CpuContext(ICpuMemory memory, Generation generation) public ulong GsBase { get; set; } + /// x87 control word observed at the current guest boundary. + public ushort FpuControlWord { get; set; } = 0x037F; + + /// MXCSR observed at the current guest boundary. + public uint Mxcsr { get; set; } = 0x1F80; + public ulong this[CpuRegister register] { get => _registers[(int)register]; diff --git a/src/SharpEmu.HLE/GuestImageWriteTracker.cs b/src/SharpEmu.HLE/GuestImageWriteTracker.cs new file mode 100644 index 0000000..079bd09 --- /dev/null +++ b/src/SharpEmu.HLE/GuestImageWriteTracker.cs @@ -0,0 +1,613 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Globalization; +using System.Runtime.InteropServices; + +namespace SharpEmu.HLE; + +/// +/// Detects guest CPU writes into memory that backs a host GPU image. On PS5 +/// render targets alias unified memory, so games freely mix CPU writes and GPU +/// draws on the same surface (Chowdren titles memset their fog layers every +/// frame). Host GPU images are separate storage, so the video backend needs to +/// know when the guest CPU rewrote a surface to re-upload it. Ranges are +/// write-protected; the first write faults, the fault handler restores write +/// access and marks the range dirty, and the video backend consumes the dirty +/// flag once per flip and re-arms protection after re-uploading. +/// +public static unsafe class GuestImageWriteTracker +{ + private const int ProtRead = 0x1; + private const int ProtWrite = 0x2; + private const int ClockMonotonicRaw = 4; + + private sealed class TrackedRange + { + public ulong Address; + public ulong ByteCount; + public ulong Start; + public ulong End; + public int Dirty; + public int Armed; + public int FirstCpuWriteSeen; + public int PendingFirstCpuWrite; + public bool TraceLifetime; + public long SourceSequence; + public long FirstCpuWriteTraceSequence; + public long FirstCpuWriteTimestampNanoseconds; + public ulong FirstCpuWriteAddress; + public ulong FirstCpuWritePage; + public string Source = "unspecified"; + } + + [StructLayout(LayoutKind.Sequential)] + private struct Timespec + { + public long Seconds; + public long Nanoseconds; + } + + private static readonly object _gate = new(); + private static readonly Dictionary _rangesByAddress = new(); + + // Snapshot array read lock-free from the signal handler; rebuilt on every + // mutation under the gate. Signal handlers must not take managed locks. + private static TrackedRange[] _rangeSnapshot = []; + + private static readonly bool _enabled = !OperatingSystem.IsWindows() && + Environment.GetEnvironmentVariable("SHARPEMU_GUEST_IMAGE_CPU_SYNC") != "0"; + private static readonly (bool Wildcard, ulong[] Addresses) _lifetimeTraceFilter = + ParseAddressList(Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_ADDRS")); + private static readonly (bool Wildcard, string[] Sources) _lifetimeSourceTraceFilter = + ParseSourceList(Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_MEMORY_LIFETIME")); + private static readonly bool _lifetimeTraceEnabled = + _lifetimeTraceFilter.Wildcard || + _lifetimeTraceFilter.Addresses.Length != 0 || + _lifetimeSourceTraceFilter.Wildcard || + _lifetimeSourceTraceFilter.Sources.Length != 0; + private static readonly long _lifetimeTraceEpochNanoseconds = + _enabled && _lifetimeTraceEnabled ? GetMonotonicNanoseconds() : 0; + private static long _lifetimeTraceSequence; + + [DllImport("libc", EntryPoint = "mprotect", SetLastError = true)] + private static extern int Mprotect(nint address, nuint length, int protection); + + [DllImport("libc", EntryPoint = "clock_gettime", SetLastError = false)] + private static extern int ClockGetTime(int clockId, Timespec* time); + + public static bool Enabled => _enabled; + + /// + /// Exercises the fault-handling path once outside signal context so every + /// branch is JIT-compiled (and, under Rosetta 2, translated) before a real + /// fault arrives — a cold signal path is silently never entered there. + /// + public static void WarmUp() + { + if (!_enabled) + { + return; + } + + var scratch = NativeMemory.AllocZeroed(4096); + try + { + // Warm the timestamp P/Invoke used by the signal-safe scalar + // capture path before a real protected-page write reaches it. + _ = GetMonotonicNanoseconds(); + var address = (ulong)scratch; + Track(address, 4096); + _ = TryHandleWriteFault(address); + _ = ConsumeDirty(address); + Untrack(address); + } + finally + { + NativeMemory.Free(scratch); + } + } + + /// Registers a range and arms write protection on it. + public static void Track( + ulong address, + ulong byteCount, + long sourceSequence = 0, + string source = "unspecified") + { + if (!_enabled || address == 0 || byteCount == 0) + { + return; + } + + var (start, length) = PageAlign(address, byteCount); + lock (_gate) + { + _rangesByAddress.TryGetValue(address, out var range); + if (range is not null && + (range.Start != start || + range.End != start + length || + range.ByteCount != byteCount)) + { + // Never resize an object that is still reachable from the + // signal handler's lock-free snapshot. Retire it and publish + // a fresh immutable range. + DisarmLocked(range, "replace-range"); + _rangesByAddress.Remove(address); + range = null; + } + + if (range is null) + { + range = new TrackedRange + { + Address = address, + ByteCount = byteCount, + Start = start, + End = start + length, + TraceLifetime = + ShouldTraceRange(start, start + length) || ShouldTraceSource(source), + SourceSequence = sourceSequence, + Source = source, + }; + _rangesByAddress[address] = range; + RebuildSnapshotLocked(); + } + else + { + FlushPendingFirstCpuWrite(range); + } + + range.SourceSequence = sourceSequence; + range.Source = source; + range.TraceLifetime = + ShouldTraceRange(range.Start, range.End) || ShouldTraceSource(source); + ArmLocked(range, "arm"); + } + } + + public static void Untrack(ulong address) + { + if (!_enabled) + { + return; + } + + lock (_gate) + { + if (_rangesByAddress.TryGetValue(address, out var range)) + { + DisarmLocked(range, "untrack"); + _rangesByAddress.Remove(address); + RebuildSnapshotLocked(); + } + } + } + + /// + /// Returns true when the guest CPU wrote the range since the last call, + /// clearing the flag. The caller re-arms via after it + /// finished reading the guest bytes. + /// + public static bool ConsumeDirty(ulong address) + { + if (!_enabled) + { + return false; + } + + lock (_gate) + { + if (!_rangesByAddress.TryGetValue(address, out var range)) + { + return false; + } + + FlushPendingFirstCpuWrite(range); + return Interlocked.Exchange(ref range.Dirty, 0) != 0; + } + } + + /// + /// Non-consuming variant of : reports whether + /// the range has been written since it was last re-armed, leaving the + /// flag for the owner that evicts and re-uploads. + /// + public static bool PeekDirty(ulong address) + { + if (!_enabled) + { + return false; + } + + lock (_gate) + { + if (!_rangesByAddress.TryGetValue(address, out var range)) + { + return false; + } + + FlushPendingFirstCpuWrite(range); + return Volatile.Read(ref range.Dirty) != 0; + } + } + + public static void Rearm(ulong address) + { + if (!_enabled) + { + return; + } + + lock (_gate) + { + if (_rangesByAddress.TryGetValue(address, out var range)) + { + ArmLocked(range, "rearm"); + } + } + } + + /// + /// Prepares pages touched by a managed HLE memory write. Native guest + /// stores fault and enter through the + /// POSIX signal bridge, but a managed Buffer.MemoryCopy into a protected + /// page is surfaced by the runtime as a fatal AccessViolation instead of + /// a resumable guest fault. Visit every page in the write span up front so + /// all overlapping texture owners are dirtied and made writable. + /// + public static void NotifyManagedWrite(ulong address, ulong byteCount) + { + if (!_enabled || address == 0 || byteCount == 0) + { + return; + } + + var end = address > ulong.MaxValue - byteCount + ? ulong.MaxValue + : address + byteCount; + var candidate = address; + while (candidate < end) + { + _ = TryHandleWriteFault(candidate); + var nextPage = (candidate & ~0xFFFUL) + 0x1000UL; + if (nextPage <= candidate) + { + break; + } + candidate = nextPage; + } + } + + /// + /// Flushes scalar first-write records captured by the POSIX signal handler. + /// Call only from ordinary managed execution, never from signal context. + /// + public static void FlushPendingDiagnostics() + { + if (!_enabled || !_lifetimeTraceEnabled) + { + return; + } + + lock (_gate) + { + foreach (var range in _rangesByAddress.Values) + { + FlushPendingFirstCpuWrite(range); + } + } + } + + /// + /// Signal-handler entry: if the fault address lies in a tracked, armed + /// range, restore write access, mark the range dirty, and return true so + /// the faulting write can be retried. Must not allocate or lock. + /// + public static bool TryHandleWriteFault(ulong faultAddress) + { + if (!_enabled || faultAddress == 0) + { + return false; + } + + var ranges = Volatile.Read(ref _rangeSnapshot); + var writableStart = ulong.MaxValue; + var writableEnd = 0UL; + for (var index = 0; index < ranges.Length; index++) + { + var range = ranges[index]; + if (faultAddress < range.Start || faultAddress >= range.End) + { + continue; + } + + writableStart = Math.Min(writableStart, range.Start); + writableEnd = Math.Max(writableEnd, range.End); + } + + if (writableStart == ulong.MaxValue) + { + return false; + } + + // Ranges are page-aligned and may overlap (font atlases and other + // suballocations commonly share pages). Unprotecting one range also + // makes every overlapping tracked page writable. Expand to the full + // transitive overlap and dirty/disarm every owner, otherwise only the + // first dictionary entry observes the write and the others retain a + // stale cached texture indefinitely. + var expanded = true; + while (expanded) + { + expanded = false; + for (var index = 0; index < ranges.Length; index++) + { + var range = ranges[index]; + if (range.Start >= writableEnd || range.End <= writableStart) + { + continue; + } + + var start = Math.Min(writableStart, range.Start); + var end = Math.Max(writableEnd, range.End); + if (start != writableStart || end != writableEnd) + { + writableStart = start; + writableEnd = end; + expanded = true; + } + } + } + + var needsUnprotect = false; + for (var index = 0; index < ranges.Length; index++) + { + var range = ranges[index]; + if (range.Start < writableEnd && range.End > writableStart && + Volatile.Read(ref range.Armed) != 0) + { + needsUnprotect = true; + break; + } + } + + if (needsUnprotect && + Mprotect( + (nint)writableStart, + (nuint)(writableEnd - writableStart), + ProtRead | ProtWrite) != 0) + { + return false; + } + + for (var index = 0; index < ranges.Length; index++) + { + var range = ranges[index]; + if (range.Start >= writableEnd || range.End <= writableStart) + { + continue; + } + + var wasArmed = Interlocked.Exchange(ref range.Armed, 0) != 0; + if (wasArmed && + range.TraceLifetime && + Interlocked.CompareExchange(ref range.FirstCpuWriteSeen, 1, 0) == 0) + { + // Signal context: capture preallocated scalar fields only. + // Formatting and I/O are deferred to a locked safe path. + range.FirstCpuWriteTraceSequence = + Interlocked.Increment(ref _lifetimeTraceSequence); + range.FirstCpuWriteTimestampNanoseconds = GetMonotonicNanoseconds(); + range.FirstCpuWriteAddress = faultAddress; + range.FirstCpuWritePage = faultAddress & ~0xFFFUL; + Volatile.Write(ref range.PendingFirstCpuWrite, 1); + Volatile.Write(ref range.FirstCpuWriteSeen, 2); + } + + Volatile.Write(ref range.Dirty, 1); + } + + return true; + } + + private static void ArmLocked(TrackedRange range, string operation) + { + FlushPendingFirstCpuWrite(range); + if (Interlocked.Exchange(ref range.Armed, 1) == 1) + { + return; + } + + // A new publication/rearm starts a new first-write lifetime. + Volatile.Write(ref range.FirstCpuWriteSeen, 0); + var failed = Mprotect( + (nint)range.Start, + (nuint)(range.End - range.Start), + ProtRead) != 0; + if (failed) + { + Volatile.Write(ref range.Armed, 0); + } + + if (range.TraceLifetime) + { + TraceLifetime( + range, + failed ? $"{operation}-failed-errno-{Marshal.GetLastPInvokeError()}" : operation); + } + } + + private static void DisarmLocked(TrackedRange range, string operation) + { + FlushPendingFirstCpuWrite(range); + var wasArmed = Interlocked.Exchange(ref range.Armed, 0) == 1; + if (wasArmed) + { + _ = Mprotect( + (nint)range.Start, + (nuint)(range.End - range.Start), + ProtRead | ProtWrite); + } + + if (range.TraceLifetime) + { + TraceLifetime(range, wasArmed ? operation : $"{operation}-already-disarmed"); + } + } + + private static void RebuildSnapshotLocked() + { + _rangeSnapshot = _rangesByAddress.Values.ToArray(); + } + + private static (ulong Start, ulong Length) PageAlign(ulong address, ulong byteCount) + { + const ulong pageMask = 0xFFFUL; + var start = address & ~pageMask; + var end = (address + byteCount + pageMask) & ~pageMask; + return (start, end - start); + } + + private static bool ShouldTraceRange(ulong start, ulong end) + { + if (_lifetimeTraceFilter.Wildcard) + { + return true; + } + + var addresses = _lifetimeTraceFilter.Addresses; + for (var index = 0; index < addresses.Length; index++) + { + if (addresses[index] >= start && addresses[index] < end) + { + return true; + } + } + + return false; + } + + private static (bool Wildcard, ulong[] Addresses) ParseAddressList(string? addresses) + { + if (string.IsNullOrWhiteSpace(addresses)) + { + return (false, []); + } + + var parsedAddresses = new List(); + foreach (var token in addresses.Split( + [',', ';', ' ', '\t'], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (token == "*") + { + return (true, []); + } + + var span = token.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + if (ulong.TryParse(span, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var parsed)) + { + parsedAddresses.Add(parsed); + } + } + + return (false, parsedAddresses.ToArray()); + } + + private static bool ShouldTraceSource(string source) + { + if (_lifetimeSourceTraceFilter.Wildcard) + { + return true; + } + + return Array.IndexOf(_lifetimeSourceTraceFilter.Sources, source) >= 0; + } + + private static (bool Wildcard, string[] Sources) ParseSourceList(string? sources) + { + if (string.IsNullOrWhiteSpace(sources)) + { + return (false, []); + } + + var parsedSources = new List(); + foreach (var token in sources.Split( + [',', ';'], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (token == "*") + { + return (true, []); + } + + parsedSources.Add(token); + } + + return (false, parsedSources.ToArray()); + } + + private static void FlushPendingFirstCpuWrite(TrackedRange range) + { + var spin = new SpinWait(); + while (Volatile.Read(ref range.FirstCpuWriteSeen) == 1) + { + spin.SpinOnce(); + } + + if (!range.TraceLifetime || Interlocked.Exchange(ref range.PendingFirstCpuWrite, 0) == 0) + { + return; + } + + TraceLifetime( + range, + "first-cpu-write-disarm", + range.FirstCpuWriteAddress, + range.FirstCpuWritePage, + range.FirstCpuWriteTraceSequence, + range.FirstCpuWriteTimestampNanoseconds); + } + + private static void TraceLifetime( + TrackedRange range, + string operation, + ulong faultAddress = 0, + ulong faultPage = 0, + long traceSequence = 0, + long timestampNanoseconds = 0) + { + if (traceSequence == 0) + { + traceSequence = Interlocked.Increment(ref _lifetimeTraceSequence); + } + + if (timestampNanoseconds == 0) + { + timestampNanoseconds = GetMonotonicNanoseconds(); + } + + var elapsedMilliseconds = + (timestampNanoseconds - _lifetimeTraceEpochNanoseconds) / 1_000_000.0; + Console.Error.WriteLine( + $"[WT][LIFETIME] seq={traceSequence} t_ms={elapsedMilliseconds:F3} " + + $"event={operation} source_seq={range.SourceSequence} source='{range.Source}' " + + $"requested=0x{range.Address:X16}+0x{range.ByteCount:X} " + + $"range=0x{range.Start:X16}..0x{range.End:X16} " + + $"fault=0x{faultAddress:X16} page=0x{faultPage:X16}"); + } + + private static long GetMonotonicNanoseconds() + { + Timespec time; + return ClockGetTime(ClockMonotonicRaw, &time) == 0 + ? unchecked((time.Seconds * 1_000_000_000L) + time.Nanoseconds) + : 0; + } +} diff --git a/src/SharpEmu.HLE/GuestThreadExecution.cs b/src/SharpEmu.HLE/GuestThreadExecution.cs index 7a0c5ab..90d6f08 100644 --- a/src/SharpEmu.HLE/GuestThreadExecution.cs +++ b/src/SharpEmu.HLE/GuestThreadExecution.cs @@ -41,6 +41,14 @@ public interface IGuestThreadScheduler { bool SupportsGuestContextTransfer { get; } + /// + /// Associates a pthread identity created on the primary guest executor + /// with its live CPU context. Primary execution does not pass through + /// TryStartThread, but kernel exception delivery must still be able to + /// target it. + /// + void RegisterGuestThreadContext(ulong threadHandle, CpuContext context); + bool TryStartThread(CpuContext creatorContext, GuestThreadStartRequest request, out string? error); bool TryJoinThread( @@ -53,6 +61,19 @@ public interface IGuestThreadScheduler int WakeBlockedThreads(string wakeKey, int maxCount = int.MaxValue); + /// + /// Applies a new guest scheduling priority to a live thread, mapping it + /// onto the host thread if one is running. Returns false when the thread + /// handle is unknown. + /// + bool TrySetGuestThreadPriority(ulong guestThreadHandle, int guestPriority); + + /// + /// Records a new affinity mask for a guest thread and re-applies it to + /// the host thread where the platform supports it. + /// + bool TrySetGuestThreadAffinity(ulong guestThreadHandle, ulong affinityMask); + IReadOnlyList SnapshotThreads(); bool TryCallGuestFunction( @@ -65,11 +86,36 @@ public interface IGuestThreadScheduler string reason, out string? error); + bool TryCallGuestFunction( + CpuContext callerContext, + ulong entryPoint, + ulong arg0, + ulong arg1, + ulong arg2, + ulong stackAddress, + ulong stackSize, + string reason, + out ulong returnValue, + out string? error); + bool TryCallGuestContinuation( CpuContext callerContext, GuestCpuContinuation continuation, string reason, out string? error); + + /// + /// Asynchronously invokes an installed kernel exception handler as the + /// target guest thread. This is used by IL2CPP's stop-the-world collector: + /// the handler acknowledges suspension and may remain blocked until the + /// collecting thread resumes it. + /// + bool TryRaiseGuestException( + CpuContext callerContext, + ulong threadHandle, + ulong handler, + int exceptionType, + out string? error); } public readonly record struct GuestImportCallFrame( @@ -94,13 +140,34 @@ public readonly record struct GuestCpuContinuation( ulong Rdi, ulong R8, ulong R9, + ulong R10, + ulong R11, ulong R12, ulong R13, ulong R14, - ulong R15); + ulong R15, + ushort FpuControlWord, + uint Mxcsr, + bool RestoreFullFpuState); public static class GuestThreadExecution { + private sealed class DelegateGuestThreadBlockWaiter : IGuestThreadBlockWaiter + { + private readonly Func _resume; + private readonly Func _tryWake; + + public DelegateGuestThreadBlockWaiter(Func resume, Func tryWake) + { + _resume = resume; + _tryWake = tryWake; + } + + public int Resume() => _resume(); + + public bool TryWake() => _tryWake(); + } + [ThreadStatic] private static ulong _currentGuestThreadHandle; @@ -246,6 +313,23 @@ public static class GuestThreadExecution return true; } + // Compatibility bridge for exports that still describe blocked work as a + // resume/wake delegate pair. New hot paths should provide an + // IGuestThreadBlockWaiter directly to avoid allocating closures. + public static bool RequestCurrentThreadBlock( + CpuContext? context, + string reason, + string? wakeKey, + Func resumeHandler, + Func wakeHandler, + long blockDeadlineTimestamp = 0) => + RequestCurrentThreadBlock( + context, + reason, + wakeKey, + new DelegateGuestThreadBlockWaiter(resumeHandler, wakeHandler), + blockDeadlineTimestamp); + public static bool TryConsumeCurrentThreadBlock(out string reason) { return TryConsumeCurrentThreadBlock(out reason, out _, out _); @@ -314,6 +398,27 @@ public static class GuestThreadExecution return true; } + public static bool TryConsumeCurrentThreadBlock( + out string reason, + out GuestCpuContinuation continuation, + out bool hasContinuation, + out string wakeKey, + out Func? resumeHandler, + out Func? wakeHandler, + out long blockDeadlineTimestamp) + { + var consumed = TryConsumeCurrentThreadBlock( + out reason, + out continuation, + out hasContinuation, + out wakeKey, + out var waiter, + out blockDeadlineTimestamp); + resumeHandler = waiter is null ? null : waiter.Resume; + wakeHandler = waiter is null ? null : waiter.TryWake; + return consumed; + } + public static long ComputeDeadlineTimestamp(TimeSpan timeout) { if (timeout <= TimeSpan.Zero) @@ -360,10 +465,15 @@ public static class GuestThreadExecution context[CpuRegister.Rdi], context[CpuRegister.R8], context[CpuRegister.R9], + context[CpuRegister.R10], + context[CpuRegister.R11], context[CpuRegister.R12], context[CpuRegister.R13], context[CpuRegister.R14], - context[CpuRegister.R15]); + context[CpuRegister.R15], + context.FpuControlWord, + context.Mxcsr, + RestoreFullFpuState: false); return true; } diff --git a/src/SharpEmu.HLE/GuestTlsTemplate.cs b/src/SharpEmu.HLE/GuestTlsTemplate.cs new file mode 100644 index 0000000..65cf835 --- /dev/null +++ b/src/SharpEmu.HLE/GuestTlsTemplate.cs @@ -0,0 +1,442 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using System.Diagnostics; + +namespace SharpEmu.HLE; + +/// +/// Process-wide registry of ELF PT_TLS templates and per-thread dynamic thread +/// vectors. AMD64 uses TLS Variant II: startup/static module blocks precede the +/// thread pointer, while modules introduced after a thread was initialized are +/// represented by separately allocated DTV entries. +/// +public static class GuestTlsTemplate +{ + // Must match CpuDispatcher/DirectExecutionBackend's mapped prefix. PS5 + // modules can require more than one host page of Variant II static TLS; + // Dreaming Sarah's startup image, for example, reaches 0x1870 bytes. + public const ulong StartupStaticTlsReservation = 0x10000UL; + private static readonly object _gate = new(); + private static readonly SortedDictionary _modules = new(); + private static readonly Dictionary _threadDtvs = new(); + private static ulong _staticTlsSize; + private static ulong _maximumAlignment = 1; + private static ulong _generation; + + static GuestTlsTemplate() + { + RunTlsLayoutSelfChecks(); + } + + private sealed class ModuleTemplate + { + public required ulong ModuleId { get; init; } + public required byte[] InitImage { get; init; } + public required ulong MemorySize { get; init; } + public required ulong Alignment { get; init; } + public required ulong AlignmentBias { get; init; } + public required ulong StaticOffset { get; init; } + } + + private sealed class ThreadDtv + { + public ulong Generation { get; set; } + public Dictionary Entries { get; } = new(); + public nint DtvAllocationBase { get; set; } + public ulong DtvAddress { get; set; } + } + + private sealed class DtvEntry + { + public required ulong Address { get; init; } + public nint AllocationBase { get; init; } + } + + /// Main executable's initialized PT_TLS bytes. + public static byte[] InitImage + { + get + { + lock (_gate) + { + return _modules.TryGetValue(1, out var module) + ? (byte[])module.InitImage.Clone() + : []; + } + } + } + + /// Aligned size of the main executable's static TLS block. + public static ulong BlockSize + { + get + { + lock (_gate) + { + return _modules.TryGetValue(1, out var module) + ? module.StaticOffset + : 0; + } + } + } + + /// Main executable's PT_TLS alignment. + public static ulong Alignment + { + get + { + lock (_gate) + { + return _modules.TryGetValue(1, out var module) + ? module.Alignment + : 1; + } + } + } + + /// Total Variant II static TLS span required below the TCB. + public static ulong StaticTlsSize + { + get { lock (_gate) { return _staticTlsSize; } } + } + + /// Largest PT_TLS alignment required by the registered modules. + public static ulong MaximumAlignment + { + get { lock (_gate) { return _maximumAlignment; } } + } + + /// Current DTV generation, incremented for every registered module. + public static ulong Generation + { + get { lock (_gate) { return _generation; } } + } + + /// + /// Backwards-compatible main-module registration entry point. + /// + public static void Set(ReadOnlySpan initImage, ulong memorySize, ulong alignment) + { + Reset(); + RegisterModule(1, initImage, memorySize, alignment, alignmentBias: 0); + } + + /// + /// Registers one module's PT_TLS template and allocates its Variant II + /// static offset. Modules must be registered in loader module-ID order. + /// + public static ulong RegisterModule( + ulong moduleId, + ReadOnlySpan initImage, + ulong memorySize, + ulong alignment, + ulong alignmentBias = 0) + { + if (moduleId == 0) + { + throw new ArgumentOutOfRangeException(nameof(moduleId)); + } + + var normalizedAlignment = alignment == 0 ? 1 : alignment; + if ((normalizedAlignment & (normalizedAlignment - 1)) != 0) + { + throw new InvalidDataException($"PT_TLS alignment 0x{alignment:X} is not a power of two."); + } + if (memorySize > int.MaxValue || (ulong)initImage.Length > memorySize) + { + throw new InvalidDataException("PT_TLS template size is invalid or exceeds the supported process limit."); + } + + lock (_gate) + { + if (_modules.TryGetValue(moduleId, out var existing)) + { + if (existing.MemorySize != memorySize || + existing.Alignment != normalizedAlignment || + existing.AlignmentBias != alignmentBias || + !existing.InitImage.AsSpan().SequenceEqual(initImage)) + { + throw new InvalidOperationException($"TLS module {moduleId} was registered with a different template."); + } + + return existing.StaticOffset; + } + + // FreeBSD/AMD64 Variant II: choose the smallest offset satisfying + // offset - previous >= size and (-offset) % align == p_vaddr % align. + var staticOffset = CalculateStaticOffset( + _staticTlsSize, + memorySize, + normalizedAlignment, + alignmentBias); + if (staticOffset > StartupStaticTlsReservation) + { + throw new InvalidOperationException( + $"Static TLS requires 0x{staticOffset:X} bytes, but startup maps only " + + $"0x{StartupStaticTlsReservation:X} bytes below the thread pointer."); + } + _modules.Add(moduleId, new ModuleTemplate + { + ModuleId = moduleId, + InitImage = initImage.ToArray(), + MemorySize = memorySize, + Alignment = normalizedAlignment, + AlignmentBias = alignmentBias, + StaticOffset = staticOffset, + }); + _staticTlsSize = staticOffset; + _maximumAlignment = Math.Max(_maximumAlignment, normalizedAlignment); + _generation++; + return staticOffset; + } + } + + /// Returns the static TP-relative block offset for a module. + public static bool TryGetStaticOffset(ulong moduleId, out ulong staticOffset) + { + lock (_gate) + { + if (_modules.TryGetValue(moduleId, out var module)) + { + staticOffset = module.StaticOffset; + return true; + } + } + + staticOffset = 0; + return false; + } + + /// Clears all module templates and frees dynamic DTV blocks. + public static void Reset() + { + lock (_gate) + { + foreach (var dtv in _threadDtvs.Values) + { + FreeDynamicEntries(dtv); + } + + _threadDtvs.Clear(); + _modules.Clear(); + _staticTlsSize = 0; + _maximumAlignment = 1; + _generation++; + } + } + + /// + /// Initializes every currently registered startup TLS block for a thread + /// and records their addresses in that thread's DTV. + /// + public static void SeedThreadBlock(CpuContext context, ulong threadPointer) + { + ArgumentNullException.ThrowIfNull(context); + if (threadPointer == 0) + { + return; + } + + lock (_gate) + { + if (_threadDtvs.Remove(threadPointer, out var oldDtv)) + { + FreeDynamicEntries(oldDtv); + } + + var dtv = new ThreadDtv(); + _threadDtvs.Add(threadPointer, dtv); + foreach (var module in _modules.Values) + { + dtv.Entries[module.ModuleId] = CreateStaticOrDynamicEntry(context, threadPointer, module); + } + RebuildGuestDtv(context, threadPointer, dtv); + } + } + + /// + /// Implements the DTV lookup used by __tls_get_addr. Unknown module + /// IDs and offsets outside the module's TLS image are rejected with zero. + /// + public static ulong ResolveAddress(CpuContext context, ulong moduleId, ulong offset) + { + ArgumentNullException.ThrowIfNull(context); + if (context.FsBase == 0 || moduleId == 0) + { + return 0; + } + + lock (_gate) + { + if (!_modules.TryGetValue(moduleId, out var module) || offset >= module.MemorySize) + { + return 0; + } + + if (!_threadDtvs.TryGetValue(context.FsBase, out var dtv)) + { + dtv = new ThreadDtv(); + _threadDtvs.Add(context.FsBase, dtv); + foreach (var startupModule in _modules.Values) + { + dtv.Entries[startupModule.ModuleId] = CreateStaticOrDynamicEntry(context, context.FsBase, startupModule); + } + RebuildGuestDtv(context, context.FsBase, dtv); + } + + var entryAdded = false; + if (!dtv.Entries.TryGetValue(moduleId, out var entry)) + { + // This module appeared after the thread's startup TLS layout was + // seeded. Model rtld's lazy DTV allocation instead of assuming + // unused space exists below the already-live thread pointer. + entry = CreateDynamicEntry(module); + dtv.Entries.Add(moduleId, entry); + entryAdded = true; + } + + if (entryAdded || dtv.Generation != _generation) + { + RebuildGuestDtv(context, context.FsBase, dtv); + } + + return checked(entry.Address + offset); + } + } + + private static DtvEntry CreateStaticOrDynamicEntry( + CpuContext context, + ulong threadPointer, + ModuleTemplate module) + { + if (threadPointer >= module.StaticOffset) + { + var address = threadPointer - module.StaticOffset; + var zeroImage = module.MemorySize == 0 ? [] : new byte[(int)module.MemorySize]; + if ((zeroImage.Length == 0 || context.Memory.TryWrite(address, zeroImage)) && + (module.InitImage.Length == 0 || context.Memory.TryWrite(address, module.InitImage))) + { + return new DtvEntry { Address = address }; + } + } + + return CreateDynamicEntry(module); + } + + private static DtvEntry CreateDynamicEntry(ModuleTemplate module) + { + var size = Math.Max(1UL, module.MemorySize); + var allocationSize = checked(size + module.Alignment - 1); + if (allocationSize > int.MaxValue) + { + throw new OutOfMemoryException("TLS module allocation exceeds the supported host allocation size."); + } + + var allocationBase = Marshal.AllocHGlobal((int)allocationSize); + var alignedAddress = AlignUp(unchecked((ulong)allocationBase), module.Alignment); + Marshal.Copy(new byte[(int)size], 0, unchecked((nint)alignedAddress), (int)size); + if (module.InitImage.Length != 0) + { + Marshal.Copy(module.InitImage, 0, unchecked((nint)alignedAddress), module.InitImage.Length); + } + + return new DtvEntry + { + Address = alignedAddress, + AllocationBase = allocationBase, + }; + } + + private static void FreeDynamicEntries(ThreadDtv dtv) + { + foreach (var entry in dtv.Entries.Values) + { + if (entry.AllocationBase != 0) + { + Marshal.FreeHGlobal(entry.AllocationBase); + } + } + if (dtv.DtvAllocationBase != 0) + { + Marshal.FreeHGlobal(dtv.DtvAllocationBase); + dtv.DtvAllocationBase = 0; + dtv.DtvAddress = 0; + } + } + + private static void RebuildGuestDtv(CpuContext context, ulong threadPointer, ThreadDtv dtv) + { + var maximumModuleId = _modules.Count == 0 ? 0UL : _modules.Keys.Max(); + var byteSize = checked(2UL * sizeof(ulong) + maximumModuleId * sizeof(ulong)); + if (byteSize > int.MaxValue) + { + throw new OutOfMemoryException("Guest DTV exceeds the supported host allocation size."); + } + + var newAllocation = Marshal.AllocHGlobal((int)Math.Max((ulong)sizeof(ulong) * 2, byteSize)); + var newAddress = unchecked((ulong)newAllocation); + Marshal.Copy(new byte[(int)Math.Max((ulong)sizeof(ulong) * 2, byteSize)], 0, newAllocation, (int)Math.Max((ulong)sizeof(ulong) * 2, byteSize)); + Marshal.WriteInt64(newAllocation, 0, unchecked((long)_generation)); + Marshal.WriteInt64(newAllocation, sizeof(ulong), unchecked((long)maximumModuleId)); + foreach (var (moduleId, entry) in dtv.Entries) + { + var slotOffset = checked((int)(2UL * sizeof(ulong) + (moduleId - 1) * sizeof(ulong))); + Marshal.WriteInt64(newAllocation, slotOffset, unchecked((long)entry.Address)); + } + + if (!context.TryWriteUInt64(threadPointer + sizeof(ulong), newAddress)) + { + Marshal.FreeHGlobal(newAllocation); + throw new InvalidOperationException("Failed to install the guest DTV pointer in the thread control block."); + } + + if (dtv.DtvAllocationBase != 0) + { + Marshal.FreeHGlobal(dtv.DtvAllocationBase); + } + dtv.DtvAllocationBase = newAllocation; + dtv.DtvAddress = newAddress; + dtv.Generation = _generation; + } + + private static ulong AlignUp(ulong value, ulong alignment) + { + var mask = alignment - 1; + return checked((value + mask) & ~mask); + } + + private static ulong CalculateStaticOffset( + ulong previousOffset, + ulong size, + ulong alignment, + ulong alignmentBias) + { + var result = checked(previousOffset + size + alignment - 1); + return result - ((result + alignmentBias) & (alignment - 1)); + } + + [Conditional("DEBUG")] + private static void RunTlsLayoutSelfChecks() + { + var firstOffset = RegisterModule(1, [0x11, 0x22], 0x20, 0x10, 0); + var secondOffset = RegisterModule(2, [0x7A], 0x18, 0x20, 8); + Debug.Assert(firstOffset == 0x20, "First Variant II TLS offset is incorrect."); + Debug.Assert(secondOffset >= firstOffset + 0x18, "TLS modules overlap in the static layout."); + Debug.Assert((unchecked(0UL - secondOffset) & 0x1F) == 8, "PT_TLS virtual-address congruence was not preserved."); + + var lateEntry = CreateDynamicEntry(_modules[2]); + try + { + Debug.Assert(lateEntry.AllocationBase != 0, "A late TLS module did not receive a dynamic DTV block."); + Debug.Assert(Marshal.ReadByte(unchecked((nint)lateEntry.Address)) == 0x7A, "Late-module tdata was not initialized."); + Debug.Assert(Marshal.ReadByte(unchecked((nint)lateEntry.Address + 1)) == 0, "Late-module tbss was not zero initialized."); + } + finally + { + Marshal.FreeHGlobal(lateEntry.AllocationBase); + Reset(); + } + } +} diff --git a/src/SharpEmu.HLE/HleDataSymbols.cs b/src/SharpEmu.HLE/HleDataSymbols.cs index aa0bad0..12d1e3f 100644 --- a/src/SharpEmu.HLE/HleDataSymbols.cs +++ b/src/SharpEmu.HLE/HleDataSymbols.cs @@ -13,7 +13,9 @@ public static class HleDataSymbols private const string LibcNeedFlagNid = "P330P3dFF68"; private const string LibcInternalNeedFlagNid = "ZT4ODD2Ts9o"; private const int ProgNameMaxBytes = 511; - private const ulong StackChkGuardValue = 0xC0DEC0DECAFEBABEUL; + // Terminator canaries reserve the low byte as NUL. Keep the process data + // symbol and every per-thread TLS copy byte-for-byte identical. + private const ulong StackChkGuardValue = 0xC0DEC0DECAFEBA00UL; private static readonly object _gate = new(); private static readonly nint _stackChkGuardAddress = Allocate(sizeof(ulong) * 2); diff --git a/src/SharpEmu.HLE/HostMainThread.cs b/src/SharpEmu.HLE/HostMainThread.cs index 78f27ca..bcf94fd 100644 --- a/src/SharpEmu.HLE/HostMainThread.cs +++ b/src/SharpEmu.HLE/HostMainThread.cs @@ -6,12 +6,11 @@ using System.Collections.Concurrent; namespace SharpEmu.HLE; /// -/// Runs work on the real process main thread. GLFW windowing must live on -/// that thread on macOS (AppKit) and Linux (X11's single event queue), so the -/// CLI moves emulation onto a worker thread, parks the main thread in -/// , and the video presenter posts its window loop here. On -/// Windows stays false and the window keeps its own -/// thread. +/// Runs work on the real process main thread. macOS only allows AppKit (and +/// therefore GLFW windowing) on that thread, so the CLI moves emulation onto +/// a worker thread, parks the main thread in , and the +/// video presenter posts its window loop here. On other platforms +/// stays false and nothing changes. /// public static class HostMainThread { diff --git a/src/SharpEmu.HLE/HostMemory.cs b/src/SharpEmu.HLE/HostMemory.cs new file mode 100644 index 0000000..6049852 --- /dev/null +++ b/src/SharpEmu.HLE/HostMemory.cs @@ -0,0 +1,498 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; + +namespace SharpEmu.HLE; + +/// +/// Cross-platform host virtual memory API with Win32 semantics. +/// On Windows this forwards directly to kernel32. On POSIX systems it is +/// implemented over mmap/mprotect/munmap with a shadow region table that +/// answers VirtualQuery-style questions and tracks page protections. +/// POSIX anonymous mappings are demand-paged by the kernel, so Win32 +/// "reserve-only" regions are mapped as committed memory directly and +/// commit requests become protection changes. +/// +public static unsafe class HostMemory +{ + public const uint MEM_COMMIT = 0x1000; + public const uint MEM_RESERVE = 0x2000; + public const uint MEM_RELEASE = 0x8000; + public const uint MEM_FREE_STATE = 0x10000; + public const uint MEM_PRIVATE = 0x20000; + + public const uint PAGE_NOACCESS = 0x01; + public const uint PAGE_READONLY = 0x02; + public const uint PAGE_READWRITE = 0x04; + public const uint PAGE_EXECUTE = 0x10; + public const uint PAGE_EXECUTE_READ = 0x20; + public const uint PAGE_EXECUTE_READWRITE = 0x40; + + private const ulong PageSize = 0x1000; + + /// Win32 MEMORY_BASIC_INFORMATION (64-bit) layout. + public struct BasicInfo + { + public ulong BaseAddress; + public ulong AllocationBase; + public uint AllocationProtect; + public uint Alignment1; + public ulong RegionSize; + public uint State; + public uint Protect; + public uint Type; + public uint Alignment2; + } + + public static void* Alloc(void* address, nuint size, uint allocationType, uint protect) + { + if (OperatingSystem.IsWindows()) + { + return Win32VirtualAlloc(address, size, allocationType, protect); + } + + return Posix.Alloc(address, size, allocationType, protect); + } + + public static bool Free(void* address, nuint size, uint freeType) + { + if (OperatingSystem.IsWindows()) + { + return Win32VirtualFree(address, size, freeType); + } + + return Posix.Free(address, size, freeType); + } + + public static bool Protect(void* address, nuint size, uint newProtect, out uint oldProtect) + { + if (OperatingSystem.IsWindows()) + { + return Win32VirtualProtect(address, size, newProtect, out oldProtect); + } + + return Posix.Protect(address, size, newProtect, out oldProtect); + } + + public static nuint Query(void* address, out BasicInfo info) + { + if (OperatingSystem.IsWindows()) + { + return Win32VirtualQuery(address, out info, (nuint)sizeof(BasicInfo)); + } + + return Posix.Query(address, out info); + } + + public static void FlushInstructionCache(void* address, nuint size) + { + if (OperatingSystem.IsWindows()) + { + Win32FlushInstructionCache(Win32GetCurrentProcess(), address, size); + return; + } + + // The emulator only executes x86-64 guest code, so a non-Windows host + // is either x86-64 (including Rosetta 2 translation) with a coherent + // instruction cache, or would need sys_icache_invalidate for a future + // arm64 recompiler. Nothing to do today. + } + + [DllImport("kernel32.dll", EntryPoint = "VirtualAlloc", SetLastError = true)] + private static extern void* Win32VirtualAlloc(void* lpAddress, nuint dwSize, uint flAllocationType, uint flProtect); + + [DllImport("kernel32.dll", EntryPoint = "VirtualFree", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Win32VirtualFree(void* lpAddress, nuint dwSize, uint dwFreeType); + + [DllImport("kernel32.dll", EntryPoint = "VirtualProtect", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Win32VirtualProtect(void* lpAddress, nuint dwSize, uint flNewProtect, out uint lpflOldProtect); + + [DllImport("kernel32.dll", EntryPoint = "VirtualQuery")] + private static extern nuint Win32VirtualQuery(void* lpAddress, out BasicInfo lpBuffer, nuint dwLength); + + [DllImport("kernel32.dll", EntryPoint = "GetCurrentProcess")] + private static extern void* Win32GetCurrentProcess(); + + [DllImport("kernel32.dll", EntryPoint = "FlushInstructionCache")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Win32FlushInstructionCache(void* hProcess, void* lpBaseAddress, nuint dwSize); + + private static class Posix + { + private const int PROT_NONE = 0x0; + private const int PROT_READ = 0x1; + private const int PROT_WRITE = 0x2; + private const int PROT_EXEC = 0x4; + + private const int MAP_PRIVATE = 0x02; + private const int MAP_FIXED = 0x10; + private static readonly int MAP_ANON = OperatingSystem.IsMacOS() ? 0x1000 : 0x20; + private static readonly int MAP_NORESERVE = OperatingSystem.IsMacOS() ? 0 : 0x4000; + + // Linux-only: fail instead of clobbering an existing mapping. + private const int MAP_FIXED_NOREPLACE = 0x100000; + + private static readonly nint MAP_FAILED = -1; + + private static readonly object Gate = new(); + private static readonly SortedList Regions = new(); + + private sealed class Region + { + public ulong Base; + public ulong Size; + public uint DefaultProtect; + public Dictionary? PageProtects; + + public ulong End => Base + Size; + + public uint ProtectAt(ulong pageAddress) + { + if (PageProtects is not null && PageProtects.TryGetValue(pageAddress, out var overriden)) + { + return overriden; + } + + return DefaultProtect; + } + } + + public static void* Alloc(void* address, nuint size, uint allocationType, uint protect) + { + if (size == 0) + { + return null; + } + + var alignedSize = AlignUp((ulong)size, PageSize); + + lock (Gate) + { + if (allocationType == MEM_COMMIT && address != null && + TryFindRegionLocked((ulong)address, out var existing)) + { + // Note: MEM_RESERVE requests that overlap an existing + // region must fail like Win32 does; only a pure commit + // may target pages inside a tracked mapping. + // Commit inside an existing mapping: the pages are already + // backed (demand paged), so only apply the protection. + var start = AlignDown((ulong)address, PageSize); + var end = Math.Min(existing.End, AlignUp((ulong)address + alignedSize, PageSize)); + if (end <= start) + { + return null; + } + + if (mprotect((nint)start, (nuint)(end - start), ToPosixProtect(protect)) != 0) + { + return null; + } + + SetProtectRangeLocked(existing, start, end - start, protect); + return address; + } + + if ((allocationType & MEM_RESERVE) == 0) + { + // MEM_COMMIT alone outside any known region is invalid here. + return null; + } + + var posixProtect = ToPosixProtect(protect); + var flags = MAP_PRIVATE | MAP_ANON; + if ((allocationType & MEM_COMMIT) == 0) + { + // Reserve-only: keep the requested protection so the region + // is usable without a separate commit step, but tell the + // kernel not to account swap for it where supported. + flags |= MAP_NORESERVE; + } + + nint result; + if (address != null) + { + // Win32 maps at exactly the requested address or fails + // without touching existing mappings. Fail up front on + // any overlap we track, then place the mapping: Linux + // gets MAP_FIXED_NOREPLACE (fails cleanly on host + // mappings too). Darwin lacks NOREPLACE and plain + // MAP_FIXED would silently clobber untracked host + // memory (dyld, the runtime's JIT heap, Rosetta), so + // pass the address as a hint instead -- the kernel + // honors it when the range is free and relocates the + // mapping otherwise, which we treat as failure. + if (OverlapsTrackedRegionLocked((ulong)address, alignedSize)) + { + Trace($"exact overlap: addr=0x{(ulong)address:X16} size=0x{alignedSize:X}"); + return null; + } + + var exactFlags = OperatingSystem.IsMacOS() ? flags : flags | MAP_FIXED_NOREPLACE; + result = mmap((nint)address, (nuint)alignedSize, posixProtect, exactFlags, -1, 0); + if (result == MAP_FAILED || (ulong)result != (ulong)address) + { + Trace($"exact mmap failed: addr=0x{(ulong)address:X16} got=0x{(ulong)result:X16} size=0x{alignedSize:X} errno={Marshal.GetLastPInvokeError()}"); + if (result != MAP_FAILED) + { + munmap(result, (nuint)alignedSize); + } + + return null; + } + } + else + { + result = mmap(0, (nuint)alignedSize, posixProtect, flags, -1, 0); + if (result == MAP_FAILED) + { + Trace($"mmap failed: size=0x{alignedSize:X} errno={Marshal.GetLastPInvokeError()}"); + return null; + } + } + + Regions[(ulong)result] = new Region + { + Base = (ulong)result, + Size = alignedSize, + DefaultProtect = protect + }; + + return (void*)result; + } + } + + public static bool Free(void* address, nuint size, uint freeType) + { + _ = size; + _ = freeType; + + lock (Gate) + { + if (!Regions.TryGetValue((ulong)address, out var region)) + { + return false; + } + + Regions.Remove((ulong)address); + return munmap((nint)address, (nuint)region.Size) == 0; + } + } + + public static bool Protect(void* address, nuint size, uint newProtect, out uint oldProtect) + { + oldProtect = PAGE_NOACCESS; + if (size == 0) + { + return false; + } + + var start = AlignDown((ulong)address, PageSize); + var end = AlignUp((ulong)address + size, PageSize); + + lock (Gate) + { + if (!TryFindRegionLocked(start, out var region) || end > region.End) + { + return false; + } + + oldProtect = region.ProtectAt(start); + if (mprotect((nint)start, (nuint)(end - start), ToPosixProtect(newProtect)) != 0) + { + return false; + } + + SetProtectRangeLocked(region, start, end - start, newProtect); + return true; + } + } + + public static nuint Query(void* address, out BasicInfo info) + { + info = default; + var pageAddress = AlignDown((ulong)address, PageSize); + + lock (Gate) + { + if (TryFindRegionLocked(pageAddress, out var region)) + { + // Win32 VirtualQuery reports a run of pages sharing the + // same protection, so stop the run where it changes. Do + // not walk the whole mapping page-by-page here: PS5 GPU + // apertures can span hundreds of GiB, while protection + // overrides are sparse. A tiny read inside such a mapping + // otherwise performs tens of millions of dictionary + // lookups before it can return. + var pageProtects = region.PageProtects; + uint protect; + ulong runEnd; + if (pageProtects is null || pageProtects.Count == 0) + { + protect = region.DefaultProtect; + runEnd = region.End; + } + else if (pageProtects.TryGetValue(pageAddress, out protect)) + { + runEnd = pageAddress + PageSize; + while (runEnd < region.End && + pageProtects.TryGetValue(runEnd, out var nextProtect) && + nextProtect == protect) + { + runEnd += PageSize; + } + } + else + { + protect = region.DefaultProtect; + runEnd = region.End; + foreach (var overrideAddress in pageProtects.Keys) + { + if (overrideAddress > pageAddress && overrideAddress < runEnd) + { + runEnd = overrideAddress; + } + } + } + + info.BaseAddress = pageAddress; + info.AllocationBase = region.Base; + info.AllocationProtect = region.DefaultProtect; + info.RegionSize = runEnd - pageAddress; + info.State = MEM_COMMIT; + info.Protect = protect; + info.Type = MEM_PRIVATE; + return (nuint)sizeof(BasicInfo); + } + + // Untracked host memory (runtime heaps, stacks, libraries) is + // reported as a free block reaching to the next tracked region + // so scanning callers keep advancing. + var nextBase = ulong.MaxValue; + foreach (var regionBase in Regions.Keys) + { + if (regionBase > pageAddress) + { + nextBase = regionBase; + break; + } + } + + info.BaseAddress = pageAddress; + info.AllocationBase = 0; + info.AllocationProtect = PAGE_NOACCESS; + info.RegionSize = (nextBase == ulong.MaxValue ? pageAddress + PageSize : nextBase) - pageAddress; + info.State = MEM_FREE_STATE; + info.Protect = PAGE_NOACCESS; + info.Type = 0; + return (nuint)sizeof(BasicInfo); + } + } + + private static bool OverlapsTrackedRegionLocked(ulong start, ulong size) + { + var end = start + size; + foreach (var region in Regions.Values) + { + if (region.Base < end && start < region.End) + { + return true; + } + } + + return false; + } + + private static bool TryFindRegionLocked(ulong address, out Region region) + { + region = null!; + var keys = Regions.Keys; + var low = 0; + var high = keys.Count - 1; + Region? candidate = null; + while (low <= high) + { + var middle = low + ((high - low) >> 1); + var entry = Regions.Values[middle]; + if (entry.Base <= address) + { + candidate = entry; + low = middle + 1; + } + else + { + high = middle - 1; + } + } + + if (candidate is null || address >= candidate.End) + { + return false; + } + + region = candidate; + return true; + } + + private static void SetProtectRangeLocked(Region region, ulong start, ulong size, uint protect) + { + if (start == region.Base && size >= region.Size) + { + region.DefaultProtect = protect; + region.PageProtects = null; + return; + } + + region.PageProtects ??= new Dictionary(); + var end = start + size; + for (var pageAddress = start; pageAddress < end; pageAddress += PageSize) + { + if (protect == region.DefaultProtect) + { + region.PageProtects.Remove(pageAddress); + } + else + { + region.PageProtects[pageAddress] = protect; + } + } + } + + private static int ToPosixProtect(uint win32Protect) + { + return win32Protect switch + { + PAGE_NOACCESS => PROT_NONE, + PAGE_READONLY => PROT_READ, + PAGE_READWRITE => PROT_READ | PROT_WRITE, + PAGE_EXECUTE => PROT_READ | PROT_EXEC, + PAGE_EXECUTE_READ => PROT_READ | PROT_EXEC, + PAGE_EXECUTE_READWRITE => PROT_READ | PROT_WRITE | PROT_EXEC, + _ => PROT_READ | PROT_WRITE + }; + } + + private static void Trace(string message) + { + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_VMEM"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine($"[HOSTMEM] {message}"); + } + } + + private static ulong AlignDown(ulong value, ulong alignment) => value & ~(alignment - 1); + + private static ulong AlignUp(ulong value, ulong alignment) => checked((value + alignment - 1) & ~(alignment - 1)); + + [DllImport("libc", SetLastError = true)] + private static extern nint mmap(nint addr, nuint length, int prot, int flags, int fd, long offset); + + [DllImport("libc", SetLastError = true)] + private static extern int munmap(nint addr, nuint length); + + [DllImport("libc", SetLastError = true)] + private static extern int mprotect(nint addr, nuint length, int prot); + } +} diff --git a/src/SharpEmu.Libs/Agc/AgcExports.cs b/src/SharpEmu.Libs/Agc/AgcExports.cs index 853bf26..c7c8fc7 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.cs @@ -12,8 +12,19 @@ using System.Runtime.CompilerServices; namespace SharpEmu.Libs.Agc; -public static class AgcExports +public static partial class AgcExports { +#if DEBUG + static AgcExports() + { + ValidateWriteDataControlDecoders(); + ValidateDispatchInitiators(); + ValidateSubmittedQueueAndReleaseMemDecoders(); + ValidateAcquireMemAndQueueResetDecoders(); + ValidateDepthTargetDecoder(); + } +#endif + private const uint ShaderFileHeader = 0x34333231; private const uint ShaderVersion = 0x18; private const uint ItNop = 0x10; @@ -34,6 +45,7 @@ public static class AgcExports private const uint ItWaitRegMem = 0x3C; private const uint ItIndirectBuffer = 0x3F; private const uint ItEventWrite = 0x46; + private const uint ItReleaseMem = 0x49; private const uint ItDmaData = 0x50; private const uint ItSetContextReg = 0x69; private const uint ItSetShReg = 0x76; @@ -56,6 +68,8 @@ public static class AgcExports private const uint RFlip = 0x17; private const uint RReleaseMem = 0x18; private const uint RDmaData = 0x19; + private const uint RIndexBase = 0x1B; + private const uint RIndexCount = 0x1C; private const uint SpiShaderPgmLoPs = 0x8; private const uint SpiShaderPgmHiPs = 0x9; private const uint SpiShaderPgmLoEs = 0xC8; @@ -69,6 +83,9 @@ public static class AgcExports private const uint ComputePgmLo = 0x20C; private const uint ComputePgmHi = 0x20D; private const uint ComputePgmRsrc2 = 0x213; + private const uint ComputeStartX = 0x204; + private const uint ComputeStartY = 0x205; + private const uint ComputeStartZ = 0x206; private const uint ComputeNumThreadX = 0x207; private const uint ComputeNumThreadY = 0x208; private const uint ComputeNumThreadZ = 0x209; @@ -90,11 +107,11 @@ public static class AgcExports private const uint PaClVportYOffset = 0x112; private const uint PaScVportZMin0 = 0xB4; private const uint PaScVportZMax0 = 0xB5; + private const uint CbColorControl = 0x202; private const uint CbBlendRed = 0x105; private const uint CbBlendGreen = 0x106; private const uint CbBlendBlue = 0x107; private const uint CbBlendAlpha = 0x108; - private const uint CbColorControl = 0x202; private const uint CbColor0Base = 0x318; private const uint CbColorRegisterStride = 15; private const uint CbColor0Info = 0x31C; @@ -103,6 +120,15 @@ public static class AgcExports private const uint CbColor0Attrib3 = 0x3B8; private const uint CbBlend0Control = 0x1E0; private const uint PaScModeCntl0 = 0x292; + // GFX10 DB context registers (register byte address minus 0x28000, / 4). + private const uint DbDepthView = 0x002; + private const uint DbDepthSizeXy = 0x007; + private const uint DbDepthClear = 0x00B; + private const uint DbZInfo = 0x010; + private const uint DbZReadBase = 0x012; + private const uint DbZWriteBase = 0x014; + private const uint DbZReadBaseHi = 0x01A; + private const uint DbZWriteBaseHi = 0x01C; private const int ColorTargetCount = 8; private const uint PsTextureUserDataRegister = 0xC; private const uint VsUserDataRegister = 0x4C; @@ -110,23 +136,27 @@ public static class AgcExports private const uint EsUserDataRegister = 0xCC; private const uint ComputeUserDataRegister = 0x240; private const uint NggUserDataScalarRegisterBase = 8; - private const uint Gen5TextureFormatR8G8B8A8Unorm = 56; - private const uint Gen5TextureFormatR16G16B16A16Float = 71; + private const uint Gen5TextureFormatR8G8B8A8Unorm = 10; + private const uint Gen5TextureFormatR16G16B16A16Float = 12; + private const uint Gen5TextureType1D = 8; private const uint Gen5TextureType2D = 9; private const ulong MaxPresentedTextureBytes = 128UL * 1024UL * 1024UL; private const ulong VideoOutPixelFormatA8R8G8B8Srgb = 0x80000000; private const ulong VideoOutPixelFormatA8B8G8R8Srgb = 0x80002200; - private const ulong VideoOutPixelFormatB8G8R8A8Unorm = 0x8100000000000000; - private const ulong VideoOutPixelFormatR8G8B8A8Unorm = 0x8100000022000000; + private const ulong VideoOutPixelFormat2R8G8B8A8Srgb = 0x8000000022000000; + private const ulong VideoOutPixelFormat2B8G8R8A8Srgb = 0x8000000000000000; + private const ulong VideoOutPixelFormat2R10G10B10A2 = 0x8100000622000000; + private const ulong VideoOutPixelFormat2B10G10R10A2 = 0x8100000600000000; + private const ulong VideoOutPixelFormat2R10G10B10A2Srgb = 0x8100000022000000; + private const ulong VideoOutPixelFormat2B10G10R10A2Srgb = 0x8100000000000000; + private const ulong VideoOutPixelFormat2R10G10B10A2Bt2100Pq = 0x8100070422000000; + private const ulong VideoOutPixelFormat2B10G10R10A2Bt2100Pq = 0x8100070400000000; private const uint RegisterDefaultsVersion7 = 7; private const uint RegisterDefaultsVersion8 = 8; private const uint RegisterDefaultsVersion10 = 10; private const uint RegisterDefaultsVersion13 = 13; private const int RegisterDefaultsSize = 0x40; private const int RegisterDefaultBlockSize = 16 * 8; - private const ulong ResourceRegistrationBytesPerResource = 0x118; - private const ulong ResourceRegistrationBytesPerOwner = 0x1E0; - private const int ResourceRegistrationMaxNameLength = 256; private const ulong ShaderUserDataOffset = 0x08; private const ulong ShaderCodeOffset = 0x10; @@ -135,6 +165,9 @@ public static class AgcExports private const ulong ShaderSpecialsOffset = 0x28; private const ulong ShaderInputSemanticsOffset = 0x30; private const ulong ShaderOutputSemanticsOffset = 0x38; + private const ulong ResourceRegistrationBytesPerResource = 0x118; + private const ulong ResourceRegistrationBytesPerOwner = 0x1E0; + private const int ResourceRegistrationMaxNameLength = 256; private const ulong ShaderNumInputSemanticsOffset = 0x50; private const ulong ShaderNumOutputSemanticsOffset = 0x56; private const ulong ShaderTypeOffset = 0x5A; @@ -142,6 +175,7 @@ public static class AgcExports private const ulong CommandBufferCursorUpOffset = 0x10; private const ulong CommandBufferCursorDownOffset = 0x18; private const ulong CommandBufferCallbackOffset = 0x20; + private const ulong CommandBufferUserDataOffset = 0x28; private const ulong CommandBufferReservedDwOffset = 0x30; private const ulong ShaderSpecialGeCntlOffset = 0x00; private const ulong ShaderSpecialVgtShaderStagesEnOffset = 0x08; @@ -149,7 +183,6 @@ public static class AgcExports private const ulong ShaderSpecialGeUserVgprEnOffset = 0x28; private const uint CbSetShRegisterRangeMarker = 0x6875000D; private static readonly object _submitTraceGate = new(); - private static readonly object _textureHashTraceGate = new(); private static readonly HashSet _tracedDcbSizes = new(); private static readonly HashSet<(ulong Es, ulong Ps, GuestDrawKind Kind)> _tracedShaderTranslations = new(); private static readonly HashSet<(ulong Es, ulong Ps)> _tracedShaderDecodePairs = new(); @@ -157,39 +190,88 @@ public static class AgcExports private static readonly HashSet<(ulong Ps, string Error)> _tracedShaderFailures = new(); private static readonly HashSet<(int Handle, int Index, ulong Address, string Path)> _tracedDisplayBuffers = new(); private static readonly HashSet _tracedComputeShaders = new(); - private static readonly Dictionary<(ulong Address, uint Width, uint Height), ulong> _tracedTextureHashes = []; + private static readonly HashSet<(ulong Address, uint X, uint Y, uint Z)> + _tracedDispatchArguments = new(); + private static readonly HashSet<(ulong Address, uint Initiator, string Reason)> + _rejectedDispatchArguments = new(); private static readonly HashSet _tracedSubmittedDrawOpcodes = new(); // Concurrent so the per-draw/per-dispatch hit path is lock-free (and no longer // shares _submitTraceGate with tracing). private static readonly ConcurrentDictionary< - (ulong Es, ulong EsState, ulong Ps, ulong PsState, ulong OutputLayout, uint OutputCount, uint Attributes), + (ulong Es, ulong EsState, ulong Ps, ulong PsState, ulong OutputLayout, + uint OutputCount, uint Attributes, uint PsInputEna, uint PsInputAddr, + ulong AliasAlignment), (IGuestCompiledShader Vertex, IGuestCompiledShader Pixel)> _graphicsShaderCache = new(); private static readonly ConcurrentDictionary< - (ulong Cs, ulong State, uint LocalX, uint LocalY, uint LocalZ), + (ulong Cs, ulong State, uint LocalX, uint LocalY, uint LocalZ, + uint WaveLanes, ulong AliasAlignment), IGuestCompiledShader> _computeShaderCache = new(); + private static readonly ConcurrentDictionary< + (ulong Es, ulong State, ulong AliasAlignment), + IGuestCompiledShader> _depthOnlyVertexShaderCache = new(); private static readonly Dictionary _shaderHeadersByCode = new(); private static readonly bool _traceAgc = string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal); + // Drop a draw on an undecodable texture descriptor instead of substituting + // a 1x1 fallback binding. Off by default so a garbage descriptor degrades + // the pass rather than dropping it (Demon's Souls composite feeders). + private static readonly bool _strictShaderDescriptors = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_STRICT_SHADER_DESCRIPTORS"), + "1", + StringComparison.Ordinal); private static readonly bool _traceAgcShader = _traceAgc || string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC_SHADER"), "1", StringComparison.Ordinal); - private static readonly bool _traceTextureHashes = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TEXTURE_HASHES"), + private static readonly ulong? _traceComputeShaderAddress = ParseOptionalHexAddress( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_COMPUTE_SHADER_ADDRESS")); + private static readonly ulong? _tracePixelShaderAddress = ParseOptionalHexAddress( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_PIXEL_SHADER_ADDRESS")); + private static readonly ulong? _traceRenderTargetAddress = ParseOptionalHexAddress( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_RENDER_TARGET_ADDRESS")); + private static readonly bool _traceDraws = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_DRAWS"), + "1", + StringComparison.Ordinal); + private static readonly bool _traceFramePackets = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_FRAME_PACKETS"), + "1", + StringComparison.Ordinal); + private static readonly bool _traceVertexRanges = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_VERTEX_RANGES"), + "1", + StringComparison.Ordinal); + private static readonly bool _compatibilitySubmitCompletionEvent = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_AGC_SUBMIT_COMPLETION_EVENT"), + "1", + StringComparison.Ordinal); + // Escape hatch for the cached-texture copy skip (per-draw texel copies + // are re-enabled unconditionally when set), for A/B-ing rendering issues. + private static readonly bool _textureCopySkipDisabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_NO_TEXTURE_SKIP"), "1", StringComparison.Ordinal); private static long _dcbWriteDataTraceCount; + private static int _tracedVertexRangeCount; private static long _dcbWaitRegMemTraceCount; private static long _createShaderTraceCount; private static long _packetPayloadTraceCount; private static bool _tracedMissingPixelShaderBindings; + private static long _unsatisfiedWaitTraceCount; + private static long _labelProducerSequence; + private static readonly object _labelProducerGate = new(); + private static readonly List _labelProducers = []; + private static readonly HashSet<(object Memory, ulong Address, ulong SubmissionId)> + _tracedProducerlessWaits = new(); private static long _shaderTranslationMissTraceCount; private static long _translatedDrawTraceCount; private static long _standardDmaTraceCount; + private static long _packetParseFailureTraceCount; + private static int _textureFallbackTraceCount; private static readonly object _softwarePresenterGate = new(); private static readonly Dictionary<(ulong Source, ulong Destination), ulong> _softwarePresenterFingerprints = new(); private static readonly Dictionary<(ulong Shader, ulong Source, ulong Destination), ulong> _softwareComputeBlitFingerprints = new(); @@ -198,72 +280,7 @@ public static class AgcExports private static readonly ConditionalWeakTable _submittedGpuStates = new(); private static readonly RegisterDefaultGroup[] PrimaryRegisterDefaults = - [ - new(0, 0, 0xE24F806D, [new(CbColorControl, 0x00CC0010)]), - new(0, 3, 0x0BC65DA4, [new(0x08F, 0)]), - new(0, 4, 0x9E5AD592, [new(0x08E, 0)]), - new(0, 12, 0x6DE4C312, [new(0x203, 0)]), - new(0, 28, 0x1EB8D73A, [new(PaScModeCntl0, 0x00000002)]), - new(0, 31, 0xA20EFC70, [new(PaScWindowOffset, 0)]), - new(0, 58, 0x43FBD769, - [ - new(CbBlendRed, 0), - new(CbBlendBlue, 0), - new(CbBlendGreen, 0), - new(CbBlendAlpha, 0), - ]), - new(0, 59, 0xEF550356, [new(CbBlend0Control, 0x20010001)]), - new(0, 67, 0x918106BB, - [ - new(PaScGenericScissorTl, 0x80000000), - new(PaScGenericScissorBr, 0x40004000), - ]), - new(0, 72, 0x38E92C91, - [ - new(0x318, 0), - new(0x31B, 0), - new(0x31C, 0), - new(0x31D, 0), - new(0x31E, 0x48), - new(0x31F, 0), - new(0x321, 0), - new(0x323, 0), - new(0x324, 0), - new(0x325, 0), - new(0x390, 0), - new(0x398, 0), - new(0x3A0, 0), - new(0x3A8, 0), - new(0x3B0, 0), - new(0x3B8, 0x0006C000), - ]), - new(0, 73, 0x0B177B43, [new(0x00C, 0), new(0x00D, 0x40004000)]), - new(0, 74, 0x48531062, [new(0x191, 0)]), - new(0, 76, 0x7690AF6F, - [ - new(0x10F, 0x4E7E0000), - new(0x111, 0x4E7E0000), - new(0x113, 0x4E7E0000), - new(0x110, 0), - new(0x112, 0), - new(0x114, 0), - new(0x094, 0x80000000), - new(0x095, 0x40004000), - new(0x0B4, 0), - new(0x0B5, 0), - ]), - new(0, 77, 0x078D7060, - [ - new(PaScWindowScissorTl, 0x80000000), - new(PaScWindowScissorBr, 0x40004000), - ]), - new(1, 13, 0xC918DF3E, [new(0x20C, 0), new(0x20D, 0)]), - new(1, 14, 0xC9751C9C, [new(0x0C8, 0), new(0x0C9, 0)]), - new(1, 18, 0xC9E01B31, [new(0x008, 0), new(0x009, 0)]), - new(2, 3, 0x105971C2, [new(0x25B, 0)]), - new(2, 7, 0x40D49AD1, [new(0x262, 0)]), - new(2, 12, 0x9EBFAB10, [new(0x242, 0)]), - ]; + CreatePrimaryRegisterDefaults(); private static readonly RegisterDefaultGroup[] InternalRegisterDefaults = [ @@ -302,26 +319,79 @@ public static class AgcExports uint BaseLevel, uint LastLevel, uint Pitch, - uint DstSelect) + uint DstSelect, + uint Depth = 1, + uint BaseArray = 0, + uint ArrayPitch = 0, + uint MaxMip = 0, + uint MinLod = 0, + uint MinLodWarn = 0, + uint BcSwizzle = 0, + ulong MetadataAddress = 0, + uint DescriptorFlags = 0, + bool HasExtendedDescriptor = false) { + public uint ResourceMipLevels + { + get + { + // RDNA2 table 45 explicitly distinguishes MAX_MIP (the + // resource allocation) from BASE_LEVEL/LAST_LEVEL (the + // resource view). Do not size a Vulkan image from a view: + // another descriptor for the same allocation may expose a + // different subset of its mip chain. + var maximumMipLevels = GetMaximumMipLevels(); + var resourceMipLevels = HasExtendedDescriptor + ? MaxMip + 1 + : maximumMipLevels; + return Math.Min(Math.Max(resourceMipLevels, 1u), maximumMipLevels); + } + } + public uint MipLevels { get { - var largestDimension = Math.Max(Width, Height); - uint maximumMipLevels = 1; - while (largestDimension > 1) - { - largestDimension >>= 1; - maximumMipLevels++; - } - - var descriptorMipLevels = LastLevel >= BaseLevel - ? LastLevel - BaseLevel + 1 + var descriptorMipLevels = LastLevel >= ViewBaseLevel + ? LastLevel - ViewBaseLevel + 1 : 1; - return Math.Min(descriptorMipLevels, maximumMipLevels); + return Math.Min( + descriptorMipLevels, + ResourceMipLevels - ViewBaseLevel); } } + + public uint ViewBaseLevel + { + get + { + // Some single-mip Gen5 descriptors use the reserved/inverted + // 15-0 range as a mip-disabled sentinel. The resource still + // has exactly one addressable level (MAX_MIP=0). Treating 15 + // literally makes Vulkan reject an otherwise compatible GPU + // image and falls back to stale guest-memory pixels. For any + // malformed range, keep BASE_LEVEL's meaning and clamp it to + // the allocation's last addressable mip. In particular, the + // common 15-0/MAX_MIP=0 sentinel resolves to mip 0 without + // making LAST_LEVEL the base of unrelated inverted views. + return Math.Min(BaseLevel, ResourceMipLevels - 1); + } + } + + private uint GetMaximumMipLevels() + { + var largestDimension = Type == 10 + ? Math.Max(Math.Max(Width, Height), Depth) + : Math.Max(Width, Height); + uint maximumMipLevels = 1; + while (largestDimension > 1) + { + largestDimension >>= 1; + maximumMipLevels++; + } + + return maximumMipLevels; + } } private readonly record struct RenderTargetDescriptor( @@ -347,10 +417,15 @@ public static class AgcExports IReadOnlyList GlobalMemoryBindings, IReadOnlyList VertexInputs, IReadOnlyList RenderTargets, - // The seam-shaped view of RenderTargets, built once here so the per-frame - // submit path does not rebuild it for every draw of a cached translation. + GuestDepthTarget? DepthTarget, + // Seam-shaped color targets are built once with the cached translation. IReadOnlyList GuestTargets, - GuestRenderState RenderState); + GuestRenderState RenderState, + IReadOnlyList PixelUserData, + uint RawBlendControl, + uint RawColorInfo, + IReadOnlyList PixelInitialScalars, + IReadOnlyList VertexInitialScalars); private sealed record TranslatedImageBinding( TextureDescriptor Descriptor, @@ -373,16 +448,80 @@ public static class AgcExports private readonly record struct ComputeDispatch( uint GroupCountX, uint GroupCountY, - uint GroupCountZ); + uint GroupCountZ, + uint BaseGroupX, + uint BaseGroupY, + uint BaseGroupZ, + uint WaveLaneCount, + bool IsIndirect, + uint ThreadCountX, + uint ThreadCountY, + uint ThreadCountZ); + + private readonly record struct SubmittedAcquireMem( + uint Engine, + uint CbDbControl, + ulong BaseAddress, + ulong SizeBytes, + uint PollInterval, + uint GcrControl) + { + // GFX10 GCR_CNTL invalidation controls. The host has no separate GLI, + // GLM, GLK, GLV, GL1 and GL2 caches; they all converge on the guest + // memory snapshots used to build Vulkan resources. + private const uint GliInvalidateMask = 0x3u; + private const int Gl1RangeShift = 2; + private const uint Gl1RangeMask = 0x3u; + private const uint GlmInvalidate = 1u << 5; + private const uint GlkInvalidate = 1u << 7; + private const uint GlvInvalidate = 1u << 8; + private const uint Gl1Invalidate = 1u << 9; + private const uint Gl2Discard = 1u << 13; + private const uint Gl2Invalidate = 1u << 14; + private const int Gl2RangeShift = 11; + private const uint Gl2RangeMask = 0x3u; + + public bool InvalidatesGuestResources => + (GcrControl & (GliInvalidateMask | + GlmInvalidate | + GlkInvalidate | + GlvInvalidate | + Gl1Invalidate | + Gl2Discard | + Gl2Invalidate)) != 0; + + // sceAgc encodes its all-memory sentinel with a zero COHER_SIZE. GFX10 + // can also request ALL independently in GLI_INV, GL1_RANGE or + // GL2_RANGE; in the host's unified resource cache, any invalidated + // domain with ALL scope expands the operation to all tracked images. + public bool CoversAllGuestMemory => + SizeBytes == 0 || + (GcrControl & GliInvalidateMask) == 1u || + ((GcrControl & (GlmInvalidate | + GlkInvalidate | + GlvInvalidate | + Gl1Invalidate)) != 0 && + ((GcrControl >> Gl1RangeShift) & Gl1RangeMask) == 0) || + ((GcrControl & (Gl2Discard | Gl2Invalidate)) != 0 && + ((GcrControl >> Gl2RangeShift) & Gl2RangeMask) == 0); + } private sealed class SubmittedDcbState { + public readonly record struct PendingSubmission( + ulong CommandAddress, + uint DwordCount, + ulong SubmissionId, + bool TracePackets); + public Dictionary CxRegisters { get; } = new(); public Dictionary ShRegisters { get; } = new(); public Dictionary UcRegisters { get; } = new(); public TextureDescriptor? PresenterTexture { get; set; } public GuestDrawKind GuestDrawKind { get; set; } public TranslatedGuestDraw? TranslatedDraw { get; set; } + public TranslatedGuestDraw? PendingTargetlessDraw { get; set; } + public Dictionary KnownRenderTargets { get; } = new(); public Dictionary RenderTargetWriters { get; } = new(); public ulong IndirectArgsAddress { get; set; } public bool SawIndexedDraw { get; set; } @@ -391,6 +530,17 @@ public static class AgcExports public uint IndexSize { get; set; } public uint InstanceCount { get; set; } = 1; public uint DrawIndexOffset { get; set; } + public string QueueName { get; set; } = "graphics"; + public ulong ActiveSubmissionId { get; set; } + public Queue PendingSubmissions { get; } = new(); + public bool HasActiveSubmission { get; set; } + public bool IsSuspended { get; set; } + public ulong CompletionEventNotifiedSubmissionId { get; set; } + public Dictionary<(uint Op, uint Register), uint> FramePacketCounts { get; } = new(); + public uint FramePacketCount { get; set; } + public uint FrameDrawCount { get; set; } + public uint FrameDispatchCount { get; set; } + public ulong FlipCount { get; set; } } private sealed class SubmittedGpuState @@ -409,6 +559,8 @@ public static class AgcExports public uint NextOwner { get; set; } = 1; public uint NextResource { get; set; } = 1; public ulong WorkSequence { get; set; } + public ulong SubmissionSequence { get; set; } + public bool WaitMonitorRunning { get; set; } } private readonly record struct RegisteredAgcResource( @@ -419,6 +571,19 @@ public static class AgcExports uint Type, uint Flags); + private sealed class LabelProducerTrace + { + public long Sequence; + public required object Memory; + public ulong Address; + public ulong Length; + public ulong PacketAddress; + public ulong SubmissionId; + public required string QueueName; + public required string DebugName; + public bool Completed; + } + private readonly record struct RegisterDefaultValue(uint Offset, uint Value); private readonly record struct RegisterDefaultGroup( @@ -442,11 +607,11 @@ public static class AgcExports var version = (uint)ctx[CpuRegister.Rsi]; if (stateAddress == 0 || !IsSupportedRegisterDefaultsVersion(version)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } TraceAgc($"agc.init state=0x{stateAddress:X16} version={version}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } #pragma warning restore SHEM004 @@ -478,19 +643,19 @@ public static class AgcExports var codeAddress = ctx[CpuRegister.Rdx]; if (headerAddress == 0 || codeAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadUInt32(headerAddress, out var fileHeader) || - !ctx.TryReadUInt32(headerAddress + sizeof(uint), out var version)) + if (!TryReadUInt32(ctx, headerAddress, out var fileHeader) || + !TryReadUInt32(ctx, headerAddress + sizeof(uint), out var version)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (fileHeader != ShaderFileHeader || version != ShaderVersion) { TraceCreateShader(destinationAddress, headerAddress, codeAddress, $"invalid-header file=0x{fileHeader:X8} version=0x{version:X8}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } if (!RelocatePointerField(ctx, headerAddress + ShaderCxRegistersOffset) || @@ -501,12 +666,12 @@ public static class AgcExports !RelocatePointerField(ctx, headerAddress + ShaderOutputSemanticsOffset) || !ctx.TryWriteUInt64(headerAddress + ShaderCodeOffset, codeAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - if (!ctx.TryReadUInt64(headerAddress + ShaderUserDataOffset, out var userDataAddress)) + if (!TryReadUInt64(ctx, headerAddress + ShaderUserDataOffset, out var userDataAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (userDataAddress != 0 && @@ -516,18 +681,18 @@ public static class AgcExports !RelocatePointerField(ctx, userDataAddress + 0x18) || !RelocatePointerField(ctx, userDataAddress + 0x20))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (!PatchShaderProgramRegisters(ctx, headerAddress, codeAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } if (destinationAddress != 0 && !ctx.TryWriteUInt64(destinationAddress, headerAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } lock (_submitTraceGate) @@ -603,24 +768,24 @@ public static class AgcExports if (cxRegistersAddress == 0 || ucRegistersAddress == 0 || hullShaderAddress != 0 || geometryShaderAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadByte(geometryShaderAddress + ShaderTypeOffset, out var shaderType) || !IsEsGeometryShaderType(shaderType) || - !ctx.TryReadUInt64(geometryShaderAddress + ShaderSpecialsOffset, out var specialsAddress) || + if (!TryReadByte(ctx, geometryShaderAddress + ShaderTypeOffset, out var shaderType) || !IsEsGeometryShaderType(shaderType) || + !TryReadUInt64(ctx, geometryShaderAddress + ShaderSpecialsOffset, out var specialsAddress) || specialsAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } if (!CopyShaderRegister(ctx, specialsAddress + ShaderSpecialVgtShaderStagesEnOffset, cxRegistersAddress) || !CopyShaderRegister(ctx, specialsAddress + ShaderSpecialVgtGsOutPrimTypeOffset, cxRegistersAddress + 8) || !CopyShaderRegister(ctx, specialsAddress + ShaderSpecialGeCntlOffset, ucRegistersAddress) || !CopyShaderRegister(ctx, specialsAddress + ShaderSpecialGeUserVgprEnOffset, ucRegistersAddress + 8) || - !ctx.TryWriteUInt32(ucRegistersAddress + 16, VgtPrimitiveType) || - !ctx.TryWriteUInt32(ucRegistersAddress + 20, primitiveType)) + !TryWriteUInt32(ctx, ucRegistersAddress + 16, VgtPrimitiveType) || + !TryWriteUInt32(ctx, ucRegistersAddress + 20, primitiveType)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceAgc($"agc.create_prim_state cx=0x{cxRegistersAddress:X16} uc=0x{ucRegistersAddress:X16} gs=0x{geometryShaderAddress:X16} type={shaderType} prim=0x{primitiveType:X8}"); @@ -643,21 +808,21 @@ public static class AgcExports if (registersAddress == 0 || geometryShaderAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadUInt64(geometryShaderAddress + ShaderOutputSemanticsOffset, out var outputSemanticsAddress) || - !ctx.TryReadUInt32(geometryShaderAddress + ShaderNumOutputSemanticsOffset, out var outputSemanticsCount)) + if (!TryReadUInt64(ctx, geometryShaderAddress + ShaderOutputSemanticsOffset, out var outputSemanticsAddress) || + !TryReadUInt32(ctx, geometryShaderAddress + ShaderNumOutputSemanticsOffset, out var outputSemanticsCount)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } ulong inputSemanticsAddress = 0; if (pixelShaderAddress != 0 && - (!ctx.TryReadUInt64(pixelShaderAddress + ShaderInputSemanticsOffset, out inputSemanticsAddress) || - !ctx.TryReadUInt32(pixelShaderAddress + ShaderNumInputSemanticsOffset, out _))) + (!TryReadUInt64(ctx, pixelShaderAddress + ShaderInputSemanticsOffset, out inputSemanticsAddress) || + !TryReadUInt32(ctx, pixelShaderAddress + ShaderNumInputSemanticsOffset, out _))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } for (uint i = 0; i < 32; i++) @@ -667,7 +832,7 @@ public static class AgcExports { var flat = false; if (pixelShaderAddress != 0 && inputSemanticsAddress != 0 && - ctx.TryReadUInt32(inputSemanticsAddress + (i * sizeof(uint)), out var inputSemantic)) + TryReadUInt32(ctx, inputSemanticsAddress + (i * sizeof(uint)), out var inputSemantic)) { flat = ((inputSemantic >> 22) & 0x1) != 0; } @@ -676,10 +841,10 @@ public static class AgcExports } var destination = registersAddress + (i * 8); - if (!ctx.TryWriteUInt32(destination, SpiPsInputCntl0 + i) || - !ctx.TryWriteUInt32(destination + sizeof(uint), value)) + if (!TryWriteUInt32(ctx, destination, SpiPsInputCntl0 + i) || + !TryWriteUInt32(ctx, destination + sizeof(uint), value)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } @@ -703,15 +868,15 @@ public static class AgcExports var type = (int)ctx[CpuRegister.Rdx]; if (outputAddress == 0 || commandAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var payloadAddress = commandAddress + 8; if (type == 0) { - if (!ctx.TryReadUInt32(commandAddress, out var header)) + if (!TryReadUInt32(ctx, commandAddress, out var header)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } payloadAddress = (header & 0x3FFF_0000u) == 0x3FFF_0000u @@ -721,7 +886,7 @@ public static class AgcExports if (!ctx.TryWriteUInt64(outputAddress, payloadAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (ShouldTraceHotPath(ref _packetPayloadTraceCount)) @@ -747,20 +912,20 @@ public static class AgcExports var dwordCount = (uint)ctx[CpuRegister.Rsi]; if (commandBufferAddress == 0 || dwordCount < 2 || dwordCount > 0x4001) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return ReturnPointer(ctx, 0); } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, dwordCount, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(dwordCount, ItNop, RZero))) + !TryWriteUInt32(ctx, commandAddress, Pm4(dwordCount, ItNop, RZero))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return ReturnPointer(ctx, 0); } for (uint index = 1; index < dwordCount; index++) { - if (!ctx.TryWriteUInt32(commandAddress + ((ulong)index * sizeof(uint)), 0)) + if (!TryWriteUInt32(ctx, commandAddress + ((ulong)index * sizeof(uint)), 0)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return ReturnPointer(ctx, 0); } } @@ -781,11 +946,11 @@ public static class AgcExports var modifier = (uint)ctx[CpuRegister.R8]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(5, ItDispatchDirect, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, groupCountX) || - !ctx.TryWriteUInt32(commandAddress + 8, groupCountY) || - !ctx.TryWriteUInt32(commandAddress + 12, groupCountZ) || - !ctx.TryWriteUInt32(commandAddress + 16, (modifier & 0xA038u) | 0x41u)) + !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDispatchDirect, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, groupCountX) || + !TryWriteUInt32(ctx, commandAddress + 8, groupCountY) || + !TryWriteUInt32(ctx, commandAddress + 12, groupCountZ) || + !TryWriteUInt32(ctx, commandAddress + 16, DirectDispatchInitiator(modifier))) { return ReturnPointer(ctx, 0); } @@ -793,6 +958,14 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + private static uint DirectDispatchInitiator(uint modifier) => + // AGC's direct API takes workgroup counts by default. Preserve the + // caller's USE_THREAD_DIMENSIONS bit when explicitly requested; do not + // force it. Demon's Souls' 0xF00100 dispatch is paired with a + // 0x3C004000 element bound (exactly 64 lanes per group), proving the + // default packet is group-dimensional. + (modifier & 0xA038u) | 0x41u; + [SysAbiExport( Nid = "UZbQjYAwwXM", ExportName = "sceAgcCbSetShRegistersDirect", @@ -817,8 +990,8 @@ public static class AgcExports for (uint index = 0; index < registerCount; index++) { var entryAddress = registersAddress + ((ulong)index * 8); - if (!ctx.TryReadUInt32(entryAddress, out var offset) || - !ctx.TryReadUInt32(entryAddress + sizeof(uint), out var value)) + if (!TryReadUInt32(ctx, entryAddress, out var offset) || + !TryReadUInt32(ctx, entryAddress + sizeof(uint), out var value)) { return ReturnPointer(ctx, 0); } @@ -841,8 +1014,8 @@ public static class AgcExports var valueCount = (uint)(endIndex - startIndex); var packetDwords = valueCount + 2; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItSetShReg, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, registers[startIndex].Offset & 0xFFFFu)) + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItSetShReg, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, registers[startIndex].Offset & 0xFFFFu)) { return ReturnPointer(ctx, 0); } @@ -850,7 +1023,8 @@ public static class AgcExports firstCommandAddress = firstCommandAddress == 0 ? commandAddress : firstCommandAddress; for (var index = startIndex; index < endIndex; index++) { - if (!ctx.TryWriteUInt32( + if (!TryWriteUInt32( + ctx, commandAddress + 8 + ((ulong)(index - startIndex) * sizeof(uint)), registers[index].Value)) { @@ -874,8 +1048,8 @@ public static class AgcExports var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItNop, RAcbReset)) || - !ctx.TryWriteUInt32(commandAddress + 4, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RAcbReset)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0)) { return ReturnPointer(ctx, 0); } @@ -901,15 +1075,15 @@ public static class AgcExports var hasAddress = (eventType & ~1u) == 0x38; var packetDwords = hasAddress ? 4u : 2u; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItEventWrite, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, hasAddress ? eventType | 0x100u : eventType & 0x3Fu)) + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItEventWrite, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, hasAddress ? eventType | 0x100u : eventType & 0x3Fu)) { return ReturnPointer(ctx, 0); } if (hasAddress && - (!ctx.TryWriteUInt32(commandAddress + 8, (uint)eventAddress & ~7u) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(eventAddress >> 32)))) + (!TryWriteUInt32(ctx, commandAddress + 8, (uint)eventAddress & ~7u) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(eventAddress >> 32)))) { return ReturnPointer(ctx, 0); } @@ -940,14 +1114,14 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(8, ItNop, RAcquireMem)) || - !ctx.TryWriteUInt32(commandAddress + 4, 0x8000_0000u) || - !ctx.TryWriteUInt32(commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || - !ctx.TryWriteUInt32(commandAddress + 12, 0) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)(baseAddress >> 8)) || - !ctx.TryWriteUInt32(commandAddress + 20, 0) || - !ctx.TryWriteUInt32(commandAddress + 24, pollCycles / 40) || - !ctx.TryWriteUInt32(commandAddress + 28, gcrControl)) + !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RAcquireMem)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0x8000_0000u) || + !TryWriteUInt32(ctx, commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)(baseAddress >> 8)) || + !TryWriteUInt32(ctx, commandAddress + 20, 0) || + !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40) || + !TryWriteUInt32(ctx, commandAddress + 28, gcrControl)) { return ReturnPointer(ctx, 0); } @@ -969,8 +1143,8 @@ public static class AgcExports var address = ctx[CpuRegister.R8]; var reference = ctx[CpuRegister.R9]; var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + sizeof(ulong), out var mask) || - !ctx.TryReadUInt32(stackAddress + (2 * sizeof(ulong)), out var pollCycles) || + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var mask) || + !TryReadUInt32(ctx, stackAddress + (2 * sizeof(ulong)), out var pollCycles) || commandBufferAddress == 0 || size > 1 || compareFunction > 7 || @@ -982,27 +1156,27 @@ public static class AgcExports var packetDwords = size == 0 ? 6u : 9u; var packetRegister = size == 0 ? RWaitMem32 : RWaitMem64; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItNop, packetRegister)) || - !ctx.TryWriteUInt32(commandAddress + 4, (uint)address) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(address >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)mask)) + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItNop, packetRegister)) || + !TryWriteUInt32(ctx, commandAddress + 4, (uint)address) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(address >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)mask)) { return ReturnPointer(ctx, 0); } if (size == 0) { - if (!ctx.TryWriteUInt32(commandAddress + 16, compareFunction) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)reference)) + if (!TryWriteUInt32(ctx, commandAddress + 16, compareFunction) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)reference)) { return ReturnPointer(ctx, 0); } } - else if (!ctx.TryWriteUInt32(commandAddress + 16, (uint)(mask >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)reference) || - !ctx.TryWriteUInt32(commandAddress + 24, (uint)(reference >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 28, compareFunction) || - !ctx.TryWriteUInt32(commandAddress + 32, pollCycles / 40)) + else if (!TryWriteUInt32(ctx, commandAddress + 16, (uint)(mask >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)reference) || + !TryWriteUInt32(ctx, commandAddress + 24, (uint)(reference >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 28, compareFunction) || + !TryWriteUInt32(ctx, commandAddress + 32, pollCycles / 40)) { return ReturnPointer(ctx, 0); } @@ -1030,10 +1204,10 @@ public static class AgcExports var modifier = (uint)ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(4, ItDispatchIndirect, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, (uint)argumentsAddress) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(argumentsAddress >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 12, (modifier & 0xA038u) | 0x41u)) + !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItDispatchIndirect, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, (uint)argumentsAddress) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(argumentsAddress >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 12, (modifier & 0xA038u) | 0x41u)) { return ReturnPointer(ctx, 0); } @@ -1058,11 +1232,11 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var markerAddress) || - !ctx.TryWriteUInt32(markerAddress, Pm4(2, ItNop, RZero)) || - !ctx.TryWriteUInt32(markerAddress + 4, CbSetShRegisterRangeMarker) || + !TryWriteUInt32(ctx, markerAddress, Pm4(2, ItNop, RZero)) || + !TryWriteUInt32(ctx, markerAddress + 4, CbSetShRegisterRangeMarker) || !TryAllocateCommandDwords(ctx, commandBufferAddress, valueCount + 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(valueCount + 2, ItSetShReg, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, offset)) + !TryWriteUInt32(ctx, commandAddress, Pm4(valueCount + 2, ItSetShReg, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, offset)) { return ReturnPointer(ctx, 0); } @@ -1071,12 +1245,12 @@ public static class AgcExports { var value = 0u; if (valuesAddress != 0 && - !ctx.TryReadUInt32(valuesAddress + (i * sizeof(uint)), out value)) + !TryReadUInt32(ctx, valuesAddress + (i * sizeof(uint)), out value)) { return ReturnPointer(ctx, 0); } - if (!ctx.TryWriteUInt32(commandAddress + 8 + (i * sizeof(uint)), value)) + if (!TryWriteUInt32(ctx, commandAddress + 8 + (i * sizeof(uint)), value)) { return ReturnPointer(ctx, 0); } @@ -1100,12 +1274,12 @@ public static class AgcExports var cachePolicy = (uint)(ctx[CpuRegister.R8] & 0xFF); var destinationAddress = ctx[CpuRegister.R9]; var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + 8, out var dataSelectionRaw) || - !ctx.TryReadUInt64(stackAddress + 16, out var data) || - !ctx.TryReadUInt64(stackAddress + 24, out var gdsOffsetRaw) || - !ctx.TryReadUInt64(stackAddress + 32, out var gdsSizeRaw) || - !ctx.TryReadUInt64(stackAddress + 40, out var interruptRaw) || - !ctx.TryReadUInt64(stackAddress + 48, out var interruptContextIdRaw)) + if (!TryReadUInt64(ctx, stackAddress + 8, out var dataSelectionRaw) || + !TryReadUInt64(ctx, stackAddress + 16, out var data) || + !TryReadUInt64(ctx, stackAddress + 24, out var gdsOffsetRaw) || + !TryReadUInt64(ctx, stackAddress + 32, out var gdsSizeRaw) || + !TryReadUInt64(ctx, stackAddress + 40, out var interruptRaw) || + !TryReadUInt64(ctx, stackAddress + 48, out var interruptContextIdRaw)) { return ReturnPointer(ctx, 0); } @@ -1126,16 +1300,17 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(8, ItNop, RReleaseMem)) || - !ctx.TryWriteUInt32(commandAddress + 4, action | (cachePolicy << 8)) || - !ctx.TryWriteUInt32( + !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RReleaseMem)) || + !TryWriteUInt32(ctx, commandAddress + 4, action | (cachePolicy << 8)) || + !TryWriteUInt32( + ctx, commandAddress + 8, gcrControl | (dataSelection << 16) | (interrupt << 24)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)destinationAddress) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)(destinationAddress >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)data) || - !ctx.TryWriteUInt32(commandAddress + 24, (uint)(data >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 28, interruptContextId)) + !TryWriteUInt32(ctx, commandAddress + 12, (uint)destinationAddress) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)(destinationAddress >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)data) || + !TryWriteUInt32(ctx, commandAddress + 24, (uint)(data >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 28, interruptContextId)) { return ReturnPointer(ctx, 0); } @@ -1162,8 +1337,8 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItNop, RDrawReset)) || - !ctx.TryWriteUInt32(commandAddress + 4, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RDrawReset)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0)) { return ReturnPointer(ctx, 0); } @@ -1212,8 +1387,8 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItIndexType, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, indexSize)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItIndexType, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, indexSize)) { return ReturnPointer(ctx, 0); } @@ -1222,6 +1397,37 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "8N2tmT3jmC8", + ExportName = "sceAgcDcbSetIndexCount", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbSetIndexCount(CpuContext ctx) + { + var commandBufferAddress = ctx[CpuRegister.Rdi]; + var indexCount = (uint)ctx[CpuRegister.Rsi]; + if (commandBufferAddress == 0 || + !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RIndexCount)) || + !TryWriteUInt32(ctx, commandAddress + 4, indexCount)) + { + return ReturnPointer(ctx, 0); + } + + return ReturnPointer(ctx, commandAddress); + } + + [SysAbiExport( + Nid = "mljzuGDZRQ4", + ExportName = "sceAgcDcbSetIndexCountGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbSetIndexCountGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 7u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + [SysAbiExport( Nid = "tSBxhAPyytQ", ExportName = "sceAgcDcbSetNumInstances", @@ -1237,8 +1443,8 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItNumInstances, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, instanceCount)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNumInstances, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, instanceCount)) { return ReturnPointer(ctx, 0); } @@ -1265,21 +1471,27 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var baseCommand) || - !ctx.TryWriteUInt32(baseCommand, Pm4(3, ItIndexBase, 0)) || - !ctx.TryWriteUInt32(baseCommand + 4, (uint)indexAddress) || - !ctx.TryWriteUInt32(baseCommand + 8, (uint)(indexAddress >> 32)) || - !ctx.TryWriteUInt32(baseCommand + 12, Pm4(2, ItIndexBufferSize, 0)) || - !ctx.TryWriteUInt32(baseCommand + 16, indexCount)) + !TryWriteUInt32(ctx, baseCommand, Pm4(3, ItIndexBase, 0)) || + !TryWriteUInt32(ctx, baseCommand + 4, (uint)indexAddress) || + !TryWriteUInt32(ctx, baseCommand + 8, (uint)(indexAddress >> 32)) || + !TryWriteUInt32(ctx, baseCommand + 12, Pm4(2, ItIndexBufferSize, 0)) || + !TryWriteUInt32(ctx, baseCommand + 16, indexCount)) { return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var drawCommand) || - !ctx.TryWriteUInt32(drawCommand, Pm4(5, ItDrawIndex2, 0)) || - !ctx.TryWriteUInt32(drawCommand + 4, indexCount) || - !ctx.TryWriteUInt32(drawCommand + 8, 0) || - !ctx.TryWriteUInt32(drawCommand + 12, 0) || - !ctx.TryWriteUInt32(drawCommand + 16, 0)) + // DRAW_INDEX_2 is six dwords: header, maximum index count, the + // 64-bit index-buffer base, the draw count and the initiator. The + // former five-dword packet omitted both the real base and the count + // field, so every call made by Unity looked like a zero-count draw to + // the submitted-command parser and the complete scene was discarded. + if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 6, out var drawCommand) || + !TryWriteUInt32(ctx, drawCommand, Pm4(6, ItDrawIndex2, 0)) || + !TryWriteUInt32(ctx, drawCommand + 4, indexCount) || + !TryWriteUInt32(ctx, drawCommand + 8, (uint)indexAddress) || + !TryWriteUInt32(ctx, drawCommand + 12, (uint)(indexAddress >> 32)) || + !TryWriteUInt32(ctx, drawCommand + 16, indexCount) || + !TryWriteUInt32(ctx, drawCommand + 20, 0)) { return ReturnPointer(ctx, 0); } @@ -1308,13 +1520,13 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(7, ItNop, RDrawIndexAuto)) || - !ctx.TryWriteUInt32(commandAddress + 4, indexCount) || - !ctx.TryWriteUInt32(commandAddress + 8, 0) || - !ctx.TryWriteUInt32(commandAddress + 12, 0) || - !ctx.TryWriteUInt32(commandAddress + 16, 0) || - !ctx.TryWriteUInt32(commandAddress + 20, 0) || - !ctx.TryWriteUInt32(commandAddress + 24, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RDrawIndexAuto)) || + !TryWriteUInt32(ctx, commandAddress + 4, indexCount) || + !TryWriteUInt32(ctx, commandAddress + 8, 0) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, 0) || + !TryWriteUInt32(ctx, commandAddress + 20, 0) || + !TryWriteUInt32(ctx, commandAddress + 24, 0)) { return ReturnPointer(ctx, 0); } @@ -1323,6 +1535,44 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "t1vNu082-jM", + ExportName = "sceAgcDcbDrawIndexIndirect", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbDrawIndexIndirect(CpuContext ctx) + { + var commandBufferAddress = ctx[CpuRegister.Rdi]; + var dataOffset = (uint)ctx[CpuRegister.Rsi]; + var modifier = (uint)ctx[CpuRegister.Rdx]; + if (commandBufferAddress == 0 || + !TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || + !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDrawIndexIndirect, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, dataOffset) || + !TryWriteUInt32(ctx, commandAddress + 8, 0) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, modifier)) + { + return ReturnPointer(ctx, 0); + } + + TraceAgc( + $"agc.dcb_draw_index_indirect buf=0x{commandBufferAddress:X16} " + + $"cmd=0x{commandAddress:X16} offset=0x{dataOffset:X8} modifier=0x{modifier:X8}"); + return ReturnPointer(ctx, commandAddress); + } + + [SysAbiExport( + Nid = "mStuvI0zOtc", + ExportName = "sceAgcDcbDrawIndexIndirectGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbDrawIndexIndirectGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 5u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + [SysAbiExport( Nid = "rUuVjyR+Rd4", ExportName = "sceAgcDcbGetLodStatsGetSize", @@ -1348,8 +1598,8 @@ public static class AgcExports var control = (uint)ctx[CpuRegister.Rcx]; var counterMask = (uint)ctx[CpuRegister.R8] & 0xFFu; var resetCounters = (uint)ctx[CpuRegister.R9] & 0x1u; - if (!ctx.TryReadUInt64(ctx[CpuRegister.Rsp] + sizeof(ulong), out var enableRaw) || - !ctx.TryReadUInt64(ctx[CpuRegister.Rsp] + (2 * sizeof(ulong)), out var counterSelectRaw) || + if (!TryReadUInt64(ctx, ctx[CpuRegister.Rsp] + sizeof(ulong), out var enableRaw) || + !TryReadUInt64(ctx, ctx[CpuRegister.Rsp] + (2 * sizeof(ulong)), out var counterSelectRaw) || commandBufferAddress == 0) { return ReturnPointer(ctx, 0); @@ -1364,11 +1614,11 @@ public static class AgcExports (counterMask << 10) | (counterSelect << 2); if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(5, ItGetLodStats, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, control) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)destinationAddress & ~0x3Fu) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(destinationAddress >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 16, packetControl)) + !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItGetLodStats, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, control) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)destinationAddress & ~0x3Fu) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(destinationAddress >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 16, packetControl)) { return ReturnPointer(ctx, 0); } @@ -1395,8 +1645,8 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItEventWrite, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, eventType)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItEventWrite, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, eventType)) { return ReturnPointer(ctx, 0); } @@ -1418,7 +1668,7 @@ public static class AgcExports var gcrControl = (uint)ctx[CpuRegister.Rcx]; var baseAddress = ctx[CpuRegister.R8]; var sizeBytes = ctx[CpuRegister.R9]; - if (!ctx.TryReadUInt32(ctx[CpuRegister.Rsp] + sizeof(ulong), out var pollCycles)) + if (!TryReadUInt32(ctx, ctx[CpuRegister.Rsp] + sizeof(ulong), out var pollCycles)) { return ReturnPointer(ctx, 0); } @@ -1435,14 +1685,14 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(8, ItNop, RAcquireMem)) || - !ctx.TryWriteUInt32(commandAddress + 4, (engine << 31) | cbDbOp) || - !ctx.TryWriteUInt32(commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || - !ctx.TryWriteUInt32(commandAddress + 12, 0) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)(baseAddress >> 8)) || - !ctx.TryWriteUInt32(commandAddress + 20, 0) || - !ctx.TryWriteUInt32(commandAddress + 24, pollCycles / 40) || - !ctx.TryWriteUInt32(commandAddress + 28, gcrControl)) + !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RAcquireMem)) || + !TryWriteUInt32(ctx, commandAddress + 4, (engine << 31) | cbDbOp) || + !TryWriteUInt32(ctx, commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)(baseAddress >> 8)) || + !TryWriteUInt32(ctx, commandAddress + 20, 0) || + !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40) || + !TryWriteUInt32(ctx, commandAddress + 28, gcrControl)) { return ReturnPointer(ctx, 0); } @@ -1467,8 +1717,8 @@ public static class AgcExports var dataAddress = ctx[CpuRegister.R8]; var dwordCount = (uint)ctx[CpuRegister.R9]; var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + sizeof(ulong), out var incrementRaw) || - !ctx.TryReadUInt64(stackAddress + (2 * sizeof(ulong)), out var writeConfirmRaw)) + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var incrementRaw) || + !TryReadUInt64(ctx, stackAddress + (2 * sizeof(ulong)), out var writeConfirmRaw)) { return ReturnPointer(ctx, 0); } @@ -1485,20 +1735,21 @@ public static class AgcExports var packetDwords = dwordCount + 4; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItNop, RWriteData)) || - !ctx.TryWriteUInt32( + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItNop, RWriteData)) || + !TryWriteUInt32( + ctx, commandAddress + 4, destination | (cachePolicy << 8) | (increment << 16) | (writeConfirm << 24)) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)destinationAddress) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(destinationAddress >> 32))) + !TryWriteUInt32(ctx, commandAddress + 8, (uint)destinationAddress) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(destinationAddress >> 32))) { return ReturnPointer(ctx, 0); } for (uint index = 0; index < dwordCount; index++) { - if (!ctx.TryReadUInt32(dataAddress + ((ulong)index * sizeof(uint)), out var value) || - !ctx.TryWriteUInt32(commandAddress + 16 + ((ulong)index * sizeof(uint)), value)) + if (!TryReadUInt32(ctx, dataAddress + ((ulong)index * sizeof(uint)), out var value) || + !TryWriteUInt32(ctx, commandAddress + 16 + ((ulong)index * sizeof(uint)), value)) { return ReturnPointer(ctx, 0); } @@ -1529,9 +1780,9 @@ public static class AgcExports var cachePolicy = (uint)(ctx[CpuRegister.R8] & 0xFF); var address = ctx[CpuRegister.R9]; var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + sizeof(ulong), out var reference) || - !ctx.TryReadUInt64(stackAddress + (2 * sizeof(ulong)), out var mask) || - !ctx.TryReadUInt32(stackAddress + (3 * sizeof(ulong)), out var pollCycles)) + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var reference) || + !TryReadUInt64(ctx, stackAddress + (2 * sizeof(ulong)), out var mask) || + !TryReadUInt32(ctx, stackAddress + (3 * sizeof(ulong)), out var pollCycles)) { return ReturnPointer(ctx, 0); } @@ -1555,37 +1806,37 @@ public static class AgcExports if (standardWait) { - if (!ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItWaitRegMem, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, compareFunction | ((operation & 1) << 8)) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)address) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(address >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)reference) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)mask) || - !ctx.TryWriteUInt32(commandAddress + 24, pollCycles / 40)) + if (!TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItWaitRegMem, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, compareFunction | ((operation & 1) << 8)) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)address) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(address >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)reference) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)mask) || + !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40)) { return ReturnPointer(ctx, 0); } } - else if (!ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItNop, packetRegister)) || - !ctx.TryWriteUInt32(commandAddress + 4, (uint)address) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(address >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)mask)) + else if (!TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItNop, packetRegister)) || + !TryWriteUInt32(ctx, commandAddress + 4, (uint)address) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(address >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)mask)) { return ReturnPointer(ctx, 0); } else if (size == 0) { - if (!ctx.TryWriteUInt32(commandAddress + 16, compareFunction | (operation << 8)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)reference)) + if (!TryWriteUInt32(ctx, commandAddress + 16, compareFunction | (operation << 8)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)reference)) { return ReturnPointer(ctx, 0); } } - else if (!ctx.TryWriteUInt32(commandAddress + 16, (uint)(mask >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)reference) || - !ctx.TryWriteUInt32(commandAddress + 24, (uint)(reference >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 28, compareFunction | (operation << 8)) || - !ctx.TryWriteUInt32(commandAddress + 32, pollCycles / 40)) + else if (!TryWriteUInt32(ctx, commandAddress + 16, (uint)(mask >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)reference) || + !TryWriteUInt32(ctx, commandAddress + 24, (uint)(reference >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 28, compareFunction | (operation << 8)) || + !TryWriteUInt32(ctx, commandAddress + 32, pollCycles / 40)) { return ReturnPointer(ctx, 0); } @@ -1601,6 +1852,45 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "u2T2DiA5hRI", + ExportName = "sceAgcDcbStallCommandBufferParser", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbStallCommandBufferParser(CpuContext ctx) + { + var commandBufferAddress = ctx[CpuRegister.Rdi]; + var size = (uint)(ctx[CpuRegister.Rsi] & 0xFF); + var address = ctx[CpuRegister.Rdx]; + var reference = ctx[CpuRegister.Rcx]; + if (commandBufferAddress == 0 || size > 1 || + !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RZero)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0)) + { + return ReturnPointer(ctx, 0); + } + + // Direct execution submits work synchronously, so there is no independent + // hardware command processor to stall. Keep a well-formed no-op in the DCB + // so packet addresses and the command-buffer cursor remain coherent. + TraceAgc( + $"agc.dcb_stall_parser buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + + $"size={size} addr=0x{address:X16} reference=0x{reference:X16}"); + return ReturnPointer(ctx, commandAddress); + } + + [SysAbiExport( + Nid = "+u6dKSLWM2o", + ExportName = "sceAgcDcbStallCommandBufferParserGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbStallCommandBufferParserGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 2u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + [SysAbiExport( Nid = "WmAc2MEj6Io", ExportName = "sceAgcDcbDmaData", @@ -1615,12 +1905,12 @@ public static class AgcExports var destinationAddress = ctx[CpuRegister.R8]; var sourceCachePolicy = (uint)(ctx[CpuRegister.R9] & 0xFF); var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + sizeof(ulong), out var control4Raw) || - !ctx.TryReadUInt64(stackAddress + (2 * sizeof(ulong)), out var sourceAddress) || - !ctx.TryReadUInt32(stackAddress + (3 * sizeof(ulong)), out var byteCount) || - !ctx.TryReadUInt64(stackAddress + (4 * sizeof(ulong)), out var control7Raw) || - !ctx.TryReadUInt64(stackAddress + (5 * sizeof(ulong)), out var control8Raw) || - !ctx.TryReadUInt64(stackAddress + (6 * sizeof(ulong)), out var control9Raw)) + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var control4Raw) || + !TryReadUInt64(ctx, stackAddress + (2 * sizeof(ulong)), out var sourceAddress) || + !TryReadUInt32(ctx, stackAddress + (3 * sizeof(ulong)), out var byteCount) || + !TryReadUInt64(ctx, stackAddress + (4 * sizeof(ulong)), out var control7Raw) || + !TryReadUInt64(ctx, stackAddress + (5 * sizeof(ulong)), out var control8Raw) || + !TryReadUInt64(ctx, stackAddress + (6 * sizeof(ulong)), out var control9Raw)) { return ReturnPointer(ctx, 0); } @@ -1635,17 +1925,19 @@ public static class AgcExports var control8 = (uint)(control8Raw & 0xFF); var control9 = (uint)(control9Raw & 0xFF); if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(8, ItNop, RDmaData)) || - !ctx.TryWriteUInt32( + !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RDmaData)) || + !TryWriteUInt32( + ctx, commandAddress + 4, destination | (destinationCachePolicy << 8) | (source << 16) | (sourceCachePolicy << 24)) || - !ctx.TryWriteUInt32( + !TryWriteUInt32( + ctx, commandAddress + 8, control4 | (control7 << 8) | (control8 << 16) | (control9 << 24)) || - !ctx.TryWriteUInt32(commandAddress + 12, byteCount) || + !TryWriteUInt32(ctx, commandAddress + 12, byteCount) || !ctx.TryWriteUInt64(commandAddress + 16, destinationAddress) || !ctx.TryWriteUInt64(commandAddress + 24, sourceAddress)) { @@ -1659,6 +1951,61 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "2ccJz9LQI+w", + ExportName = "sceAgcDcbDmaDataGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbDmaDataGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 8u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + + [SysAbiExport( + Nid = "-RnpfpxIhec", + ExportName = "sceAgcAcbDmaData", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int AcbDmaData(CpuContext ctx) + { + var commandBufferAddress = ctx[CpuRegister.Rdi]; + var sourceSelector = (uint)(ctx[CpuRegister.Rsi] & 0xFF); + var destinationSelector = (uint)(ctx[CpuRegister.Rdx] & 0xFF); + var destinationAddress = ctx[CpuRegister.Rcx]; + var stackAddress = ctx[CpuRegister.Rsp]; + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var sourceOrImmediate) || + !TryReadUInt32(ctx, stackAddress + (2 * sizeof(ulong)), out var byteCount) || + commandBufferAddress == 0 || + byteCount == 0 || + byteCount > 256u * 1024u * 1024u || + !TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || + !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RDmaData)) || + !ctx.TryWriteUInt64(commandAddress + 4, destinationAddress) || + !ctx.TryWriteUInt64(commandAddress + 12, sourceOrImmediate) || + !TryWriteUInt32(ctx, commandAddress + 20, byteCount) || + !TryWriteUInt32( + ctx, + commandAddress + 24, + sourceSelector | (destinationSelector << 8))) + { + return ReturnPointer(ctx, 0); + } + + return ReturnPointer(ctx, commandAddress); + } + + [SysAbiExport( + Nid = "M0ttm8h7SKA", + ExportName = "sceAgcAcbDmaDataGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int AcbDmaDataGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 8u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + [SysAbiExport( Nid = "RmaJwLtc8rY", ExportName = "sceAgcDcbSetBaseIndirectArgs", @@ -1671,10 +2018,10 @@ public static class AgcExports var address = ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(4, ItSetBase, 0) | (baseIndex << 1)) || - !ctx.TryWriteUInt32(commandAddress + 4, 1) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)address & ~7u) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(address >> 32))) + !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItSetBase, 0) | (baseIndex << 1)) || + !TryWriteUInt32(ctx, commandAddress + 4, 1) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)address & ~7u) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(address >> 32))) { return ReturnPointer(ctx, 0); } @@ -1694,9 +2041,9 @@ public static class AgcExports var modifier = (uint)ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 3, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(3, ItDispatchIndirect, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, dataOffset) || - !ctx.TryWriteUInt32(commandAddress + 8, (modifier & 0xA038u) | 0x41u)) + !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItDispatchIndirect, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, dataOffset) || + !TryWriteUInt32(ctx, commandAddress + 8, (modifier & 0xA038u) | 0x41u)) { return ReturnPointer(ctx, 0); } @@ -1722,7 +2069,7 @@ public static class AgcExports var payloadDwords = Math.Max(((uint)marker.Length + 4) / 4, 1); var packetDwords = payloadDwords + 1; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItNop, RPushMarker))) + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItNop, RPushMarker))) { return ReturnPointer(ctx, 0); } @@ -1739,7 +2086,7 @@ public static class AgcExports } } - if (!ctx.TryWriteUInt32(commandAddress + 4 + ((ulong)index * sizeof(uint)), value)) + if (!TryWriteUInt32(ctx, commandAddress + 4 + ((ulong)index * sizeof(uint)), value)) { return ReturnPointer(ctx, 0); } @@ -1748,6 +2095,13 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "cpCILPya5Zk", + ExportName = "sceAgcAcbPushMarker", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int AcbPushMarker(CpuContext ctx) => DcbPushMarker(ctx); + [SysAbiExport( Nid = "H7uZqCoNuWk", ExportName = "sceAgcDcbPopMarker", @@ -1758,8 +2112,8 @@ public static class AgcExports var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItNop, RPopMarker)) || - !ctx.TryWriteUInt32(commandAddress + 4, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RPopMarker)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0)) { return ReturnPointer(ctx, 0); } @@ -1767,6 +2121,13 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "6mFxkVqdmbQ", + ExportName = "sceAgcAcbPopMarker", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int AcbPopMarker(CpuContext ctx) => DcbPopMarker(ctx); + [SysAbiExport( Nid = "IxYiarKlXxM", ExportName = "sceAgcDmaDataPatchSetDstAddressOrOffset", @@ -1778,14 +2139,116 @@ public static class AgcExports var destinationAddress = ctx[CpuRegister.Rsi]; if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register) || op != ItNop || - register != RDmaData) + register != RDmaData || + !TryReadUInt32(ctx, commandAddress, out var header)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - return ctx.TryWriteUInt64(commandAddress + 16, destinationAddress) - ? ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK) - : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + var packetLength = ((header >> 16) & 0x3FFFu) + 2; + var destinationOffset = packetLength == 7 ? 4UL : 16UL; + return ctx.TryWriteUInt64(commandAddress + destinationOffset, destinationAddress) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + // The SRC counterpart of sceAgcDmaDataPatchSetDstAddressOrOffset. Patches + // the source field (offset +24, matching the layout written by + // sceAgcDcbDmaData) of a NOP/RDmaData packet. Games patch this to point a + // GPU DMA at the data it should copy — commonly a completion/label write. + // When it is missing the source stays 0, ApplySubmittedDmaData skips the + // copy (copied=False), and whatever the guest waits on that label for never + // fires (observed: Void Terrarium's first draw batch presents a black frame + // then the render pipeline stalls with no further flips). + [SysAbiExport( + Nid = "cdDRpqcFGbU", + ExportName = "sceAgcDmaDataPatchSetSrcAddressOrOffsetOrImmediate", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DmaDataPatchSetSrcAddressOrOffsetOrImmediate(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var sourceValue = ctx[CpuRegister.Rsi]; + if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register) || + op != ItNop || + register != RDmaData || + !TryReadUInt32(ctx, commandAddress, out var header)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var packetLength = ((header >> 16) & 0x3FFFu) + 2; + var sourceOffset = packetLength == 7 ? 12UL : 24UL; + return ctx.TryWriteUInt64(commandAddress + sourceOffset, sourceValue) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "eAy8eGNsCuU", + ExportName = "sceAgcWriteDataPatchSetCachePolicy", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WriteDataPatchSetCachePolicy(CpuContext ctx) => + PatchWriteDataControlByte(ctx, byteIndex: 1); + + [SysAbiExport( + Nid = "tmy-+rBpspY", + ExportName = "sceAgcWriteDataPatchSetDst", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WriteDataPatchSetDst(CpuContext ctx) => + PatchWriteDataControlByte(ctx, byteIndex: 0); + + [SysAbiExport( + Nid = "fPSCdQxgpSw", + ExportName = "sceAgcWriteDataPatchSetAddressOrOffset", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WriteDataPatchSetAddressOrOffset(CpuContext ctx) + { + // SDK revisions disagree on whether the packet or destination is the + // first argument. Astro passes (destination, packet), while older + // captures use (packet, destination), so identify the packet by its + // header instead of hard-coding one ordering. + var first = ctx[CpuRegister.Rdi]; + var second = ctx[CpuRegister.Rsi]; + ulong commandAddress; + ulong destinationAddress; + if (TryGetPacketIdentity(ctx, first, out var firstOp, out var firstRegister) && + firstOp == ItNop && firstRegister == RWriteData) + { + commandAddress = first; + destinationAddress = second; + } + else if (TryGetPacketIdentity(ctx, second, out var secondOp, out var secondRegister) && + secondOp == ItNop && secondRegister == RWriteData) + { + commandAddress = second; + destinationAddress = first; + } + else + { + // Astro's SDK 9 ABI passes (address-or-offset, pointer-to-field) + // rather than the whole packet. The field is already the packet's + // 64-bit address payload, so patch it directly. + if (second == 0 || !ctx.TryWriteUInt64(second, first)) + { + return SetReturn(ctx, second == 0 + ? OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT + : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + TraceAgc( + $"agc.patch_write_data_field field=0x{second:X16} value=0x{first:X16}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + + TraceAgc( + $"agc.patch_write_data_addr cmd=0x{commandAddress:X16} dst=0x{destinationAddress:X16}"); + return ctx.TryWriteUInt64(commandAddress + 8, destinationAddress) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -1799,7 +2262,7 @@ public static class AgcExports var address = ctx[CpuRegister.Rsi]; if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var fieldOffset = op == ItWaitRegMem @@ -1809,12 +2272,98 @@ public static class AgcExports : 0; if (fieldOffset == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } return ctx.TryWriteUInt64(commandAddress + fieldOffset, address) - ? ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK) - : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "n485EBnIWmk", + ExportName = "sceAgcWaitRegMemPatchCompareFunction", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WaitRegMemPatchCompareFunction(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var compareFunction = (uint)ctx[CpuRegister.Rsi]; + if (compareFunction > 7 || + !TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var fieldOffset = op == ItWaitRegMem + ? 4UL + : op == ItNop && register == RWaitMem32 + ? 16UL + : op == ItNop && register == RWaitMem64 + ? 28UL + : 0; + return fieldOffset != 0 && + TryPatchUInt32Bits(ctx, commandAddress + fieldOffset, 0x7u, compareFunction) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, fieldOffset == 0 + ? OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT + : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "7nOoijNPvEU", + ExportName = "sceAgcWaitRegMemPatchReference", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WaitRegMemPatchReference(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var reference = ctx[CpuRegister.Rsi]; + if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var wrote = op == ItWaitRegMem + ? TryWriteUInt32(ctx, commandAddress + 16, (uint)reference) + : op == ItNop && register == RWaitMem32 + ? TryWriteUInt32(ctx, commandAddress + 20, (uint)reference) + : op == ItNop && register == RWaitMem64 && + ctx.TryWriteUInt64(commandAddress + 20, reference); + return wrote + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, op == ItWaitRegMem || + (op == ItNop && register is RWaitMem32 or RWaitMem64) + ? OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT + : OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + [SysAbiExport( + Nid = "hXAnLgDHCoI", + ExportName = "sceAgcWaitRegMemPatchMask", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WaitRegMemPatchMask(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var mask = ctx[CpuRegister.Rsi]; + if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var wrote = op == ItWaitRegMem + ? TryWriteUInt32(ctx, commandAddress + 20, (uint)mask) + : op == ItNop && register == RWaitMem32 + ? TryWriteUInt32(ctx, commandAddress + 12, (uint)mask) + : op == ItNop && register == RWaitMem64 && + ctx.TryWriteUInt64(commandAddress + 12, mask); + return wrote + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, op == ItWaitRegMem || + (op == ItNop && register is RWaitMem32 or RWaitMem64) + ? OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT + : OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } [SysAbiExport( @@ -1830,14 +2379,97 @@ public static class AgcExports op != ItNop || register != RReleaseMem) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } return ctx.TryWriteUInt64(commandAddress + 12, address) - ? ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK) - : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "J8YCgfKAMQs", + ExportName = "sceAgcQueueEndOfPipeActionPatchGcrCntl", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int QueueEndOfPipeActionPatchGcrCntl(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + if (!IsAgcReleaseMemPacket(ctx, commandAddress)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + return TryPatchUInt32Bits( + ctx, + commandAddress + 8, + 0x0000_FFFFu, + (uint)ctx[CpuRegister.Rsi] & 0xFFFFu) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "MlEw1feXcjg", + ExportName = "sceAgcQueueEndOfPipeActionPatchData", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int QueueEndOfPipeActionPatchData(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + if (!IsAgcReleaseMemPacket(ctx, commandAddress)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + return ctx.TryWriteUInt64(commandAddress + 20, ctx[CpuRegister.Rsi]) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "T9fjQIINoeE", + ExportName = "sceAgcQueueEndOfPipeActionPatchType", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int QueueEndOfPipeActionPatchType(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var dataSelection = (uint)ctx[CpuRegister.Rsi]; + TraceAgc( + $"agc.eop_patch_type cmd=0x{commandAddress:X16} value=0x{dataSelection:X8}"); + if (dataSelection > 3 || !IsAgcReleaseMemPacket(ctx, commandAddress)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + return TryPatchUInt32Bits( + ctx, + commandAddress + 8, + 0x00FF_0000u, + dataSelection << 16) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + private static bool IsAgcReleaseMemPacket(CpuContext ctx, ulong commandAddress) => + TryGetPacketIdentity(ctx, commandAddress, out var op, out var register) && + op == ItNop && + register == RReleaseMem; + + private static bool TryPatchUInt32Bits( + CpuContext ctx, + ulong address, + uint mask, + uint value) + { + return TryReadUInt32(ctx, address, out var current) && + TryWriteUInt32(ctx, address, PatchUInt32Bits(current, mask, value)); + } + + private static uint PatchUInt32Bits(uint current, uint mask, uint value) => + (current & ~mask) | (value & mask); + [SysAbiExport( Nid = "l4fM9K-Lyks", ExportName = "sceAgcDcbSetIndexBuffer", @@ -1854,11 +2486,11 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(3, ItIndexBase, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, (uint)(indexBufferAddress & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(indexBufferAddress >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 12, Pm4(2, ItIndexBufferSize, 0)) || - !ctx.TryWriteUInt32(commandAddress + 16, indexCount)) + !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItIndexBase, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, (uint)(indexBufferAddress & 0xFFFF_FFFFUL)) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(indexBufferAddress >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 12, Pm4(2, ItIndexBufferSize, 0)) || + !TryWriteUInt32(ctx, commandAddress + 16, indexCount)) { return ReturnPointer(ctx, 0); } @@ -1884,11 +2516,11 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(5, ItDrawIndexOffset2, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, indexCount) || - !ctx.TryWriteUInt32(commandAddress + 8, indexOffset) || - !ctx.TryWriteUInt32(commandAddress + 12, indexCount) || - !ctx.TryWriteUInt32(commandAddress + 16, flags & 0xE000_0001u)) + !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDrawIndexOffset2, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, indexCount) || + !TryWriteUInt32(ctx, commandAddress + 8, indexOffset) || + !TryWriteUInt32(ctx, commandAddress + 12, indexCount) || + !TryWriteUInt32(ctx, commandAddress + 16, flags & 0xE000_0001u)) { return ReturnPointer(ctx, 0); } @@ -1913,13 +2545,13 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(7, ItNop, RWaitFlipDone)) || - !ctx.TryWriteUInt32(commandAddress + 4, videoOutHandle) || - !ctx.TryWriteUInt32(commandAddress + 8, displayBufferIndex) || - !ctx.TryWriteUInt32(commandAddress + 12, 0) || - !ctx.TryWriteUInt32(commandAddress + 16, 0) || - !ctx.TryWriteUInt32(commandAddress + 20, 0) || - !ctx.TryWriteUInt32(commandAddress + 24, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RWaitFlipDone)) || + !TryWriteUInt32(ctx, commandAddress + 4, videoOutHandle) || + !TryWriteUInt32(ctx, commandAddress + 8, displayBufferIndex) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, 0) || + !TryWriteUInt32(ctx, commandAddress + 20, 0) || + !TryWriteUInt32(ctx, commandAddress + 24, 0)) { return ReturnPointer(ctx, 0); } @@ -1946,12 +2578,12 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 6, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(6, ItNop, RFlip)) || - !ctx.TryWriteUInt32(commandAddress + 4, videoOutHandle) || - !ctx.TryWriteUInt32(commandAddress + 8, unchecked((uint)displayBufferIndex)) || - !ctx.TryWriteUInt32(commandAddress + 12, flipMode) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)(flipArg & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)(flipArg >> 32))) + !TryWriteUInt32(ctx, commandAddress, Pm4(6, ItNop, RFlip)) || + !TryWriteUInt32(ctx, commandAddress + 4, videoOutHandle) || + !TryWriteUInt32(ctx, commandAddress + 8, unchecked((uint)displayBufferIndex)) || + !TryWriteUInt32(ctx, commandAddress + 12, flipMode) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)(flipArg & 0xFFFF_FFFFUL)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)(flipArg >> 32))) { return ReturnPointer(ctx, 0); } @@ -1960,96 +2592,6 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } - // Guest draw-command builders: emit valid (skippable) packets and return a live - // command pointer so the guest's command-buffer build succeeds; full draw processing is TODO. - [SysAbiExport( - Nid = "8N2tmT3jmC8", - ExportName = "sceAgcDcbSetIndexCount", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DcbSetIndexCount(CpuContext ctx) - { - var dcb = ctx[CpuRegister.Rdi]; - var indexCount = (uint)ctx[CpuRegister.Rsi]; - if (dcb == 0) - { - return ReturnPointer(ctx, 0); - } - - if (!TryAllocateCommandDwords(ctx, dcb, 2, out var cmd) || - !ctx.TryWriteUInt32(cmd, Pm4(2, ItNop, RZero)) || - !ctx.TryWriteUInt32(cmd + 4, indexCount)) - { - return ReturnPointer(ctx, 0); - } - - return ReturnPointer(ctx, cmd); - } - - [SysAbiExport( - Nid = "xSAR0LTcRKM", - ExportName = "sceAgcDcbJump", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DcbJump(CpuContext ctx) - { - var dcb = ctx[CpuRegister.Rdi]; - var target = ctx[CpuRegister.Rsi]; - var sizeDwords = (uint)ctx[CpuRegister.Rdx]; - if (dcb == 0) - { - return ReturnPointer(ctx, 0); - } - - if (!TryAllocateCommandDwords(ctx, dcb, 4, out var cmd) || - !ctx.TryWriteUInt32(cmd, Pm4(4, ItIndirectBuffer, RZero)) || - !ctx.TryWriteUInt32(cmd + 4, (uint)(target & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(cmd + 8, (uint)((target >> 32) & 0xFFFFUL)) || - !ctx.TryWriteUInt32(cmd + 12, sizeDwords & 0xFFFFF)) - { - return ReturnPointer(ctx, 0); - } - - return ReturnPointer(ctx, cmd); - } - - [SysAbiExport( - Nid = "bbFueFP+J4k", - ExportName = "sceAgcDcbSetPredication", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DcbSetPredication(CpuContext ctx) - { - var dcb = ctx[CpuRegister.Rdi]; - var address = ctx[CpuRegister.Rsi]; - if (dcb == 0) - { - return ReturnPointer(ctx, 0); - } - - if (!TryAllocateCommandDwords(ctx, dcb, 3, out var cmd) || - !ctx.TryWriteUInt32(cmd, Pm4(3, ItNop, RZero)) || - !ctx.TryWriteUInt32(cmd + 4, (uint)(address & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(cmd + 8, (uint)(address >> 32))) - { - return ReturnPointer(ctx, 0); - } - - return ReturnPointer(ctx, cmd); - } - - [SysAbiExport( - Nid = "w6Dj1VJt5qY", - ExportName = "sceAgcSetPacketPredication", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int SetPacketPredication(CpuContext ctx) - { - // Global predication toggle on a packet; a no-op is safe for rendering. - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "w2rJhmD+dsE", ExportName = "sceAgcDriverAddEqEvent", @@ -2066,11 +2608,11 @@ public static class AgcExports KernelEventQueueCompatExports.KernelEventFilterGraphics, userData)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } TraceAgc($"agc.driver_add_eq_event eq=0x{equeue:X16} id=0x{eventId:X16} udata=0x{userData:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -2087,11 +2629,11 @@ public static class AgcExports eventId, KernelEventQueueCompatExports.KernelEventFilterGraphics)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } TraceAgc($"agc.driver_delete_eq_event eq=0x{equeue:X16} id=0x{eventId:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -2103,14 +2645,14 @@ public static class AgcExports { var packetAddress = ctx[CpuRegister.Rdi]; if (packetAddress == 0 || - !ctx.TryReadUInt64(packetAddress, out var commandAddress) || - !ctx.TryReadUInt32(packetAddress + 8, out var dwordCount)) + !TryReadUInt64(ctx, packetAddress, out var commandAddress) || + !TryReadUInt32(ctx, packetAddress + 8, out var dwordCount)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var tracePackets = false; - if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal)) + if (_traceAgc) { lock (_submitTraceGate) { @@ -2123,77 +2665,20 @@ public static class AgcExports TraceAgc($"agc.driver_submit_dcb packet=0x{packetAddress:X16} addr=0x{commandAddress:X16} dwords={dwordCount}"); } + VulkanVideoPresenter.AttachGuestMemory(ctx.Memory); var gpuState = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); lock (gpuState.Gate) { - Gen5ShaderScalarEvaluator.BeginGlobalMemoryReadScope(); - try - { - ParseSubmittedDcb(ctx, gpuState, gpuState.Graphics, commandAddress, dwordCount, tracePackets); - DrainResumableDcbs(ctx, gpuState, tracePackets); - } - finally - { - Gen5ShaderScalarEvaluator.EndGlobalMemoryReadScope(); - } - } - - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - // ABI (reversed from Quake): rdi = array of DCB base addresses (u64 each), - // rsi = array of DCB sizes in dwords (u32 each), rdx = buffer count. - [SysAbiExport( - Nid = "6UzEidRZwkg", - ExportName = "sceAgcDriverSubmitMultiDcbs", - Target = Generation.Gen5, - LibraryName = "libSceAgcDriver")] - public static int DriverSubmitMultiDcbs(CpuContext ctx) - { - var addressArray = ctx[CpuRegister.Rdi]; - var sizeArray = ctx[CpuRegister.Rsi]; - var bufferCount = (uint)ctx[CpuRegister.Rdx]; - if (addressArray == 0 || sizeArray == 0 || bufferCount == 0 || bufferCount > 4096) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var tracePackets = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal); - - var gpuState = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - lock (gpuState.Gate) - { - Gen5ShaderScalarEvaluator.BeginGlobalMemoryReadScope(); - try - { - for (uint i = 0; i < bufferCount; i++) - { - if (!ctx.TryReadUInt64(addressArray + i * 8, out var commandAddress) || - commandAddress == 0 || - !ctx.TryReadUInt32(sizeArray + i * 4, out var dwordCount) || - dwordCount == 0) - { - continue; - } - - if (tracePackets) - { - TraceAgc( - $"agc.driver_submit_multi_dcbs index={i}/{bufferCount} " + - $"addr=0x{commandAddress:X16} dwords={dwordCount}"); - } - - ParseSubmittedDcb(ctx, gpuState, gpuState.Graphics, commandAddress, dwordCount, tracePackets); - } - - DrainResumableDcbs(ctx, gpuState, tracePackets); - } - finally - { - Gen5ShaderScalarEvaluator.EndGlobalMemoryReadScope(); - } + gpuState.Graphics.QueueName = "dcb.graphics"; + EnqueueSubmittedDcb( + ctx, + gpuState, + gpuState.Graphics, + commandAddress, + dwordCount, + ++gpuState.SubmissionSequence, + tracePackets); + DrainResumableDcbs(ctx, gpuState, tracePackets); } ctx[CpuRegister.Rax] = 0; @@ -2210,14 +2695,14 @@ public static class AgcExports var ownerHandle = (uint)ctx[CpuRegister.Rdi]; var packetAddress = ctx[CpuRegister.Rsi]; if (packetAddress == 0 || - !ctx.TryReadUInt64(packetAddress, out var commandAddress) || - !ctx.TryReadUInt32(packetAddress + 8, out var dwordCount)) + !TryReadUInt64(ctx, packetAddress, out var commandAddress) || + !TryReadUInt32(ctx, packetAddress + 8, out var dwordCount)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var tracePackets = false; - if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal)) + if (_traceAgc) { lock (_submitTraceGate) { @@ -2232,6 +2717,7 @@ public static class AgcExports $"addr=0x{commandAddress:X16} dwords={dwordCount}"); } + VulkanVideoPresenter.AttachGuestMemory(ctx.Memory); var gpuState = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); lock (gpuState.Gate) { @@ -2241,16 +2727,16 @@ public static class AgcExports gpuState.ComputeQueues.Add(ownerHandle, queueState); } - Gen5ShaderScalarEvaluator.BeginGlobalMemoryReadScope(); - try - { - ParseSubmittedDcb(ctx, gpuState, queueState, commandAddress, dwordCount, tracePackets); - DrainResumableDcbs(ctx, gpuState, tracePackets); - } - finally - { - Gen5ShaderScalarEvaluator.EndGlobalMemoryReadScope(); - } + queueState.QueueName = $"acb.compute[{ownerHandle}]"; + EnqueueSubmittedDcb( + ctx, + gpuState, + queueState, + commandAddress, + dwordCount, + ++gpuState.SubmissionSequence, + tracePackets); + DrainResumableDcbs(ctx, gpuState, tracePackets); } ctx[CpuRegister.Rax] = 0; @@ -2268,91 +2754,16 @@ public static class AgcExports if (outAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryWriteUInt32(outAddress, ResourceRegistrationMaxNameLength)) + if (!TryWriteUInt32(ctx, outAddress, 256)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - TraceAgc( - $"agc.driver_get_resource_registration_max_name_length " + - $"out=0x{outAddress:X16} value={ResourceRegistrationMaxNameLength}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "AOLcoIkQDgM", - ExportName = "sceAgcDriverQueryResourceRegistrationUserMemoryRequirements", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverQueryResourceRegistrationUserMemoryRequirements(CpuContext ctx) - { - var sizeAddress = ctx[CpuRegister.Rdi]; - var resourceCount = ctx[CpuRegister.Rsi]; - var ownerCount = ctx[CpuRegister.Rdx]; - if (sizeAddress == 0 || resourceCount == 0 || ownerCount == 0) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - ulong requiredSize; - try - { - requiredSize = checked( - resourceCount * ResourceRegistrationBytesPerResource + - ownerCount * ResourceRegistrationBytesPerOwner); - } - catch (OverflowException) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - if (!ctx.TryWriteUInt64(sizeAddress, requiredSize)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - TraceAgc( - $"agc.driver_query_resource_registration_memory resources={resourceCount} " + - $"owners={ownerCount} bytes=0x{requiredSize:X}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "F0Y42t-3e18", - ExportName = "sceAgcDriverInitResourceRegistration", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverInitResourceRegistration(CpuContext ctx) - { - var memoryAddress = ctx[CpuRegister.Rdi]; - var memorySize = ctx[CpuRegister.Rsi]; - var ownerCount = ctx[CpuRegister.Rdx]; - if (memoryAddress == 0 || memorySize == 0 || ownerCount == 0 || ownerCount > uint.MaxValue) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - lock (state.Gate) - { - state.ResourceRegistrationInitialized = true; - state.ResourceRegistrationMemory = memoryAddress; - state.ResourceRegistrationMemorySize = memorySize; - state.ResourceRegistrationMaxOwners = (uint)ownerCount; - state.ResourceOwners.Clear(); - state.RegisteredResources.Clear(); - state.DefaultOwner = DefaultAgcOwner; - state.NextOwner = 1; - state.NextResource = 1; - } - - TraceAgc( - $"agc.driver_init_resource_registration memory=0x{memoryAddress:X16} " + - $"bytes=0x{memorySize:X} owners={ownerCount}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceAgc($"agc.driver_get_resource_registration_max_name_length out=0x{outAddress:X16} value=256"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } private const uint DefaultAgcOwner = 1; @@ -2367,101 +2778,16 @@ public static class AgcExports if (ownerAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - uint owner; - lock (state.Gate) + if (!TryWriteUInt32(ctx, ownerAddress, DefaultAgcOwner)) { - owner = state.DefaultOwner; + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - if (!ctx.TryWriteUInt32(ownerAddress, owner)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - TraceAgc($"agc.driver_get_default_owner out=0x{ownerAddress:X16} owner={owner}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "U9ueyEhSkF4", - ExportName = "sceAgcDriverRegisterDefaultOwner", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverRegisterDefaultOwner(CpuContext ctx) - { - var owner = (uint)ctx[CpuRegister.Rdi]; - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - lock (state.Gate) - { - state.DefaultOwner = owner; - } - - TraceAgc($"agc.driver_register_default_owner owner={owner}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "X-Nm5KLREeg", - ExportName = "sceAgcDriverRegisterOwner", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverRegisterOwner(CpuContext ctx) - { - var ownerAddress = ctx[CpuRegister.Rdi]; - var nameAddress = ctx[CpuRegister.Rsi]; - if (ownerAddress == 0 || nameAddress == 0 || - !TryReadGuestCString( - ctx, - nameAddress, - ResourceRegistrationMaxNameLength, - out var nameBytes)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - uint owner; - lock (state.Gate) - { - if (!state.ResourceRegistrationInitialized || - state.ResourceRegistrationMaxOwners != 0 && - state.ResourceOwners.Count >= state.ResourceRegistrationMaxOwners) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - owner = state.NextOwner; - while (owner == state.DefaultOwner || state.ResourceOwners.ContainsKey(owner)) - { - owner++; - if (owner == 0) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - } - - state.NextOwner = owner + 1; - state.ResourceOwners.Add(owner, System.Text.Encoding.UTF8.GetString(nameBytes)); - } - - if (!ctx.TryWriteUInt32(ownerAddress, owner)) - { - lock (state.Gate) - { - state.ResourceOwners.Remove(owner); - } - - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - TraceAgc( - $"agc.driver_register_owner out=0x{ownerAddress:X16} owner={owner} " + - $"name={System.Text.Encoding.UTF8.GetString(nameBytes)}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceAgc($"agc.driver_get_default_owner out=0x{ownerAddress:X16} owner={DefaultAgcOwner}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -2471,88 +2797,17 @@ public static class AgcExports LibraryName = "libSceAgc")] public static int DriverRegisterResource(CpuContext ctx) { - var resourceHandleAddress = ctx[CpuRegister.Rdi]; + var resourceAddress = ctx[CpuRegister.Rdi]; var owner = (uint)ctx[CpuRegister.Rsi]; - var resourceAddress = ctx[CpuRegister.Rdx]; - var resourceSize = ctx[CpuRegister.Rcx]; - var nameAddress = ctx[CpuRegister.R8]; - var type = (uint)ctx[CpuRegister.R9]; - if (resourceHandleAddress == 0 || resourceAddress == 0 || resourceSize == 0 || - !ctx.TryReadUInt32(ctx[CpuRegister.Rsp] + sizeof(ulong), out var flags) || - !TryReadGuestCString( - ctx, - nameAddress, - ResourceRegistrationMaxNameLength, - out var nameBytes)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - uint resourceHandle; - lock (state.Gate) - { - if (!state.ResourceRegistrationInitialized || - owner != state.DefaultOwner && - !state.ResourceOwners.ContainsKey(owner)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - resourceHandle = state.NextResource++; - if (resourceHandle == 0) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - state.RegisteredResources.Add( - resourceHandle, - new RegisteredAgcResource( - owner, - resourceAddress, - resourceSize, - System.Text.Encoding.UTF8.GetString(nameBytes), - type, - flags)); - } - - if (!ctx.TryWriteUInt32(resourceHandleAddress, resourceHandle)) - { - lock (state.Gate) - { - state.RegisteredResources.Remove(resourceHandle); - } - - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } + var nameAddress = ctx[CpuRegister.Rdx]; + var type = (uint)ctx[CpuRegister.R8]; + var flags = (uint)ctx[CpuRegister.R9]; TraceAgc( - $"agc.driver_register_resource handle={resourceHandle} owner={owner} " + - $"resource=0x{resourceAddress:X16} bytes=0x{resourceSize:X} " + - $"name={System.Text.Encoding.UTF8.GetString(nameBytes)} type={type} flags={flags}"); + $"agc.driver_register_resource resource=0x{resourceAddress:X16} owner={owner} " + + $"name=0x{nameAddress:X16} type={type} flags={flags}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "pWLG7WOpVcw", - ExportName = "sceAgcDriverUnregisterResource", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverUnregisterResource(CpuContext ctx) - { - var resourceHandle = (uint)ctx[CpuRegister.Rdi]; - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - lock (state.Gate) - { - if (!state.RegisteredResources.Remove(resourceHandle)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - } - - TraceAgc($"agc.driver_unregister_resource handle={resourceHandle}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } // Synthetic label for an uncatalogued NID (the Unknown* convention); the NID is authoritative. @@ -2569,7 +2824,7 @@ public static class AgcExports $"rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} " + $"rcx=0x{ctx[CpuRegister.Rcx]:X16} r8=0x{ctx[CpuRegister.R8]:X16} r9=0x{ctx[CpuRegister.R9]:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } #pragma warning restore SHEM006 @@ -2597,7 +2852,7 @@ public static class AgcExports var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 1, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, 0x8000_0000)) + !TryWriteUInt32(ctx, commandAddress, 0x8000_0000)) { return ReturnPointer(ctx, 0); } @@ -2609,64 +2864,99 @@ public static class AgcExports } #pragma warning restore SHEM006 - // WAIT_REG_MEM packets whose condition is not met suspend their DCB into - // GpuWaitRegistry. Each submit re-checks every suspended DCB against current guest - // memory (labels are advanced by ReleaseMem/WriteData/DmaData packets or by direct - // CPU writes) and resumes the ones whose condition is now satisfied. A resumed DCB - // can itself write labels that unblock others, so loop until a fixed point. - private static void DrainResumableDcbs( + private static void EnqueueSubmittedDcb( CpuContext ctx, SubmittedGpuState gpuState, + SubmittedDcbState state, + ulong commandAddress, + uint dwordCount, + ulong submissionId, bool tracePackets) { - for (var pass = 0; pass < 256; pass++) - { - var woken = GpuWaitRegistry.CollectSatisfied((address, is64Bit) => - { - if (is64Bit) - { - return ctx.TryReadUInt64(address, out var value64) - ? value64 - : (ulong?)null; - } + state.PendingSubmissions.Enqueue(new SubmittedDcbState.PendingSubmission( + commandAddress, + dwordCount, + submissionId, + tracePackets)); + PumpSubmittedQueue(ctx, gpuState, state); + } - return ctx.TryReadUInt32(address, out var value32) - ? value32 - : (ulong?)null; - }); - if (woken is null) + private static void PumpSubmittedQueue( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state) + { + if (state.IsSuspended) + { + return; + } + + while (!state.HasActiveSubmission && + state.PendingSubmissions.TryDequeue(out var submission)) + { + state.HasActiveSubmission = true; + state.ActiveSubmissionId = submission.SubmissionId; + state.IsSuspended = ParseSubmittedDcb( + ctx, + gpuState, + state, + submission.CommandAddress, + submission.DwordCount, + submission.TracePackets); + if (state.IsSuspended) { return; } - foreach (var waiter in woken) - { - var remainingDwords = waiter.TotalDwords - waiter.ResumeOffset; - if (remainingDwords == 0) - { - continue; - } - - if (tracePackets) - { - TraceAgc( - $"agc.dcb.resumed addr=0x{waiter.WaitAddress:X16} " + - $"resume=0x{waiter.ResumeAddress:X16} dwords={remainingDwords}"); - } - - var state = waiter.State as SubmittedDcbState ?? gpuState.Graphics; - ParseSubmittedDcb( - ctx, - gpuState, - state, - waiter.ResumeAddress, - remainingDwords, - tracePackets); - } + state.HasActiveSubmission = false; + NotifySubmittedDcbCompleted(gpuState, state, submission.SubmissionId); } } - private static void ParseSubmittedDcb( + private static void NotifySubmittedDcbCompleted( + SubmittedGpuState gpuState, + SubmittedDcbState state, + ulong submissionId) + { + if (!ReferenceEquals(state, gpuState.Graphics) || + state.CompletionEventNotifiedSubmissionId == submissionId) + { + return; + } + + state.CompletionEventNotifiedSubmissionId = submissionId; + void TriggerCompletionEvents() + { + var triggered = KernelEventQueueCompatExports.TriggerRegisteredEvents( + ident: 0, + KernelEventQueueCompatExports.KernelEventFilterGraphics, + data: 0); + if (_compatibilitySubmitCompletionEvent) + { + triggered += KernelEventQueueCompatExports.TriggerRegisteredEventsDistinct( + KernelEventQueueCompatExports.KernelEventFilterGraphics); + } + TraceAgc( + $"agc.driver_submit_dcb completion submission={submissionId} " + + $"queues={triggered}"); + } + + // A DCB is complete only after its translated Vulkan work and ordered + // guest-memory writes have finished. Put the notification on that same + // logical graphics queue instead of approximating completion with a + // timer, which can wake Unity while its upload data is still stale. + if (VulkanVideoPresenter.SubmitOrderedGuestAction( + TriggerCompletionEvents, + $"agc submit completion {submissionId}") == 0) + { + TriggerCompletionEvents(); + } + } + + // Returns true only when parsing stopped on an unsatisfied WAIT_REG_MEM. + // Malformed packets are dropped as completed so one bad submission cannot + // permanently wedge all later work on the same hardware queue. + private static bool ParseSubmittedDcb( CpuContext ctx, SubmittedGpuState gpuState, SubmittedDcbState state, @@ -2676,16 +2966,55 @@ public static class AgcExports { if (commandAddress == 0 || dwordCount == 0 || dwordCount > 1_000_000) { - return; + return false; } + using var guestQueueScope = VulkanVideoPresenter.EnterGuestQueue( + state.QueueName, + state.ActiveSubmissionId); + var windowByteCount = checked((int)(dwordCount * sizeof(uint))); + var rented = VulkanVideoPresenter.GuestDataPool.Rent(windowByteCount); + try + { + if (ctx.Memory.TryRead(commandAddress, rented.AsSpan(0, windowByteCount))) + { + _dcbWindowBuffer = rented; + _dcbWindowStart = commandAddress; + _dcbWindowByteLength = windowByteCount; + } + + return ParseSubmittedDcbCore( + ctx, + gpuState, + state, + commandAddress, + dwordCount, + tracePackets); + } + finally + { + _dcbWindowBuffer = null; + _dcbWindowByteLength = 0; + VulkanVideoPresenter.GuestDataPool.Return(rented); + } + } + + private static bool ParseSubmittedDcbCore( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, + ulong commandAddress, + uint dwordCount, + bool tracePackets) + { var offset = 0u; while (offset < dwordCount) { var currentAddress = commandAddress + ((ulong)offset * sizeof(uint)); - if (!ctx.TryReadUInt32(currentAddress, out var header)) + if (!TryReadUInt32(ctx, currentAddress, out var header)) { - return; + TracePacketParseFailure(state, currentAddress, offset, 0, "header-read"); + return false; } var packetType = header >> 30; @@ -2697,28 +3026,76 @@ public static class AgcExports $"agc.dcb.packet dw={offset} addr=0x{currentAddress:X16} " + $"header=0x{header:X8} len=1 type=2"); } + offset++; continue; } if (packetType != 3) { - return; + TracePacketParseFailure( + state, + currentAddress, + offset, + header, + $"packet-type-{packetType}"); + return false; } var length = Pm4Length(header); if (length == 0 || offset + length > dwordCount) { - return; + TracePacketParseFailure( + state, + currentAddress, + offset, + header, + $"length-{length}-remaining-{dwordCount - offset}"); + return false; } var op = (header >> 8) & 0xFFu; var register = (header >> 2) & 0x3Fu; + if (_traceFramePackets && ReferenceEquals(state, gpuState.Graphics)) + { + var packetKey = (op, op == ItNop ? register : uint.MaxValue); + state.FramePacketCounts[packetKey] = + state.FramePacketCounts.TryGetValue(packetKey, out var packetCount) + ? packetCount + 1 + : 1; + state.FramePacketCount++; + } if (tracePackets) { TraceSubmittedPacket(ctx, currentAddress, offset, header, length, op, register); } + if (_traceDraws) + { + CountSubmittedOpcode(op, register); + } + + if (op == ItNop && + register is RDrawReset or RAcbReset && + length >= 2) + { + ResetSubmittedParserState(state); + TraceAgc( + $"agc.queue_reset queue={state.QueueName} " + + $"submission={state.ActiveSubmissionId} " + + $"kind={(register == RDrawReset ? "draw" : "acb")} " + + $"packet=0x{currentAddress:X16}"); + } + + if (op == ItNop && register == RAcquireMem && length >= 8) + { + ApplySubmittedAcquireMem( + ctx, + state, + currentAddress, + tracePackets); + } + if (op == ItSetShReg && TryReadTextureDescriptor(ctx, currentAddress, length, out var texture)) { @@ -2729,61 +3106,95 @@ public static class AgcExports if (op == ItSetBase && length >= 4 && - ctx.TryReadUInt32(currentAddress + 4, out var baseSelector) && + TryReadUInt32(ctx, currentAddress + 4, out var baseSelector) && baseSelector == 1 && - ctx.TryReadUInt64(currentAddress + 8, out var indirectArgsAddress)) + TryReadUInt64(ctx, currentAddress + 8, out var indirectArgsAddress)) { state.IndirectArgsAddress = indirectArgsAddress; } if (op == ItEventWrite && length >= 2 && - ctx.TryReadUInt32(currentAddress + sizeof(uint), out var eventTypeRaw)) + TryReadUInt32(ctx, currentAddress + sizeof(uint), out var eventTypeRaw)) { var eventType = eventTypeRaw & 0x3Fu; - - // The guest registers AGC events with a full eventId, but the command buffer - // only carries a 6-bit EVENT_TYPE. Those two values are not the same numbering - // scheme, so exact ident matching never wakes anything. Trigger every graphics - // event registration instead (workaround for issue #173). - var triggered = KernelEventQueueCompatExports.TriggerRegisteredEventsByFilter( - KernelEventQueueCompatExports.KernelEventFilterGraphics, - eventType); - if (tracePackets) - { - TraceAgc($"agc.dcb.event type=0x{eventType:X2} queues={triggered}"); - } + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + var triggered = KernelEventQueueCompatExports.TriggerRegisteredEventsByFilter( + KernelEventQueueCompatExports.KernelEventFilterGraphics, + eventType); + if (tracePackets) + { + TraceAgc($"agc.dcb.event type=0x{eventType:X2} queues={triggered}"); + } + }, + $"event_write type=0x{eventType:X2}", + currentAddress); } if (op == ItNop && register == RReleaseMem && length >= 7) { - ApplySubmittedReleaseMem(ctx, currentAddress, tracePackets); + ApplySubmittedReleaseMem(ctx, gpuState, state, currentAddress, tracePackets); + } + + if (op == ItReleaseMem && length >= 8) + { + ApplySubmittedStandardReleaseMem( + ctx, + gpuState, + state, + currentAddress, + tracePackets); } if (op == ItNop && register == RWriteData && length >= 4) { - ApplySubmittedWriteData(ctx, currentAddress, length, tracePackets); + ApplySubmittedWriteData( + ctx, + gpuState, + state, + currentAddress, + length, + standardPacket: false, + tracePacket: tracePackets); } if (op == ItWriteData && length >= 4) { - ApplySubmittedWriteData(ctx, currentAddress, length, tracePackets); + ApplySubmittedWriteData( + ctx, + gpuState, + state, + currentAddress, + length, + standardPacket: true, + tracePacket: tracePackets); } - if (op == ItNop && register == RDmaData && length >= 8) + if (op == ItNop && register == RDmaData && length >= 7) { - ApplySubmittedDmaData(ctx, currentAddress, tracePackets); + ApplySubmittedDmaData( + ctx, + gpuState, + state, + currentAddress, + compactLayout: length == 7, + tracePacket: tracePackets); } if (op == ItDmaData && length >= 7) { - ApplySubmittedStandardDmaData(ctx, currentAddress); + ApplySubmittedStandardDmaData(ctx, gpuState, state, currentAddress); } if (op == ItIndexBase && length >= 3 && - ctx.TryReadUInt32(currentAddress + 4, out var indexBaseLo) && - ctx.TryReadUInt32(currentAddress + 8, out var indexBaseHi)) + TryReadUInt32(ctx, currentAddress + 4, out var indexBaseLo) && + TryReadUInt32(ctx, currentAddress + 8, out var indexBaseHi)) { state.IndexBufferAddress = indexBaseLo | ((ulong)indexBaseHi << 32); @@ -2791,109 +3202,53 @@ public static class AgcExports if (op == ItIndexBufferSize && length >= 2 && - ctx.TryReadUInt32(currentAddress + 4, out var indexBufferCount)) + TryReadUInt32(ctx, currentAddress + 4, out var indexBufferCount)) { state.IndexBufferCount = indexBufferCount; } + if (op == ItNop && + register == RIndexCount && + length >= 2 && + TryReadUInt32(ctx, currentAddress + 4, out var customIndexCount)) + { + state.IndexBufferCount = customIndexCount; + } + if (op == ItIndexType && length >= 2 && - ctx.TryReadUInt32(currentAddress + 4, out var indexSize)) + TryReadUInt32(ctx, currentAddress + 4, out var indexSize)) { state.IndexSize = indexSize & 0x3; } if (op == ItNumInstances && length >= 2 && - ctx.TryReadUInt32(currentAddress + 4, out var instanceCount)) + TryReadUInt32(ctx, currentAddress + 4, out var instanceCount)) { state.InstanceCount = Math.Max(instanceCount, 1); } - // WAIT_REG_MEM (AGC NOP-encapsulated 32/64-bit variants): when the condition is - // not yet satisfied, suspend this DCB; DrainResumableDcbs resumes it once the - // watched memory advances. - if (op == ItNop && register is RWaitMem32 or RWaitMem64 && + if (op == ItNop && + register is RWaitMem32 or RWaitMem64 && length >= (register == RWaitMem32 ? 6u : 9u)) { - if (TryParseWaitRegMem(ctx, currentAddress, register == RWaitMem64, - out var waitAddr, out var refVal, out var waitMask, out var cmpFunc)) + if (HandleSubmittedWaitRegMem( + ctx, state, commandAddress, currentAddress, offset, length, + dwordCount, is64Bit: register == RWaitMem64, isStandard: false, + tracePackets)) { - ulong curVal = 0; - bool hasCurVal; - if (register == RWaitMem64) - { - hasCurVal = ctx.TryReadUInt64(waitAddr, out curVal); - } - else if (ctx.TryReadUInt32(waitAddr, out var curVal32)) - { - curVal = curVal32; - hasCurVal = true; - } - else - { - hasCurVal = false; - } - - var waiter = new GpuWaitRegistry.WaitingDcb - { - CommandBufferAddress = commandAddress, - ResumeAddress = currentAddress + ((ulong)length * sizeof(uint)), - TotalDwords = dwordCount, - ResumeOffset = offset + length, - ReferenceValue = refVal, - Mask = waitMask, - CompareFunction = cmpFunc, - Is64Bit = register == RWaitMem64, - State = state, - }; - if (hasCurVal && !GpuWaitRegistry.Compare(waiter, curVal)) - { - GpuWaitRegistry.Register(waitAddr, waiter); - - if (tracePackets) - { - TraceAgc( - $"agc.dcb.suspended addr=0x{waitAddr:X16} ref=0x{refVal:X16} " + - $"mask=0x{waitMask:X16} cur=0x{curVal:X16} cmp={cmpFunc}"); - } - - return; // suspend parsing of this DCB - } + return true; // DCB suspended until the awaited label is written } } if (op == ItWaitRegMem && length >= 7) { - if (TryParseStandardWaitRegMem(ctx, currentAddress, - out var waitAddr, out var refVal, out var waitMask, out var cmpFunc) && - ctx.TryReadUInt32(waitAddr, out var curVal)) + if (HandleSubmittedWaitRegMem( + ctx, state, commandAddress, currentAddress, offset, length, + dwordCount, is64Bit: false, isStandard: true, tracePackets)) { - var waiter = new GpuWaitRegistry.WaitingDcb - { - CommandBufferAddress = commandAddress, - ResumeAddress = currentAddress + ((ulong)length * sizeof(uint)), - TotalDwords = dwordCount, - ResumeOffset = offset + length, - ReferenceValue = refVal, - Mask = waitMask, - CompareFunction = cmpFunc, - Is64Bit = false, - State = state, - }; - if (!GpuWaitRegistry.Compare(waiter, curVal)) - { - GpuWaitRegistry.Register(waitAddr, waiter); - - if (tracePackets) - { - TraceAgc( - $"agc.dcb.suspended_std addr=0x{waitAddr:X16} ref=0x{refVal:X16} " + - $"mask=0x{waitMask:X16} cur=0x{curVal:X16} cmp={cmpFunc}"); - } - - return; - } + return true; // DCB suspended until the awaited label is written } } @@ -2906,12 +3261,16 @@ public static class AgcExports out var indexCount) && indexCount != 0) { - lock (_submitTraceGate) + state.FrameDrawCount++; + if (_traceAgcShader) { - if (_tracedSubmittedDrawOpcodes.Add(op)) + lock (_submitTraceGate) { - TraceAgcShader( - $"agc.draw_packet op=0x{op:X2} count={indexCount}"); + if (_tracedSubmittedDrawOpcodes.Add(op)) + { + TraceAgcShader( + $"agc.draw_packet op=0x{op:X2} count={indexCount}"); + } } } @@ -2926,9 +3285,10 @@ public static class AgcExports if (op == ItNop && register == RDrawIndexAuto && length >= 2 && - ctx.TryReadUInt32(currentAddress + 4, out var autoIndexCount) && + TryReadUInt32(ctx, currentAddress + 4, out var autoIndexCount) && autoIndexCount != 0) { + state.FrameDrawCount++; TryTranslateGuestDraw( ctx, gpuState, @@ -2944,30 +3304,98 @@ public static class AgcExports currentAddress, length, op, - out var dispatch)) + out var dispatch)) { + state.FrameDispatchCount++; ObserveComputeDispatch(ctx, gpuState, state, dispatch); } + if (op == ItNop && + register == RWaitFlipDone && + length >= 3 && + TryReadUInt32(ctx, currentAddress + 4, out var waitVideoOutHandle) && + TryReadUInt32(ctx, currentAddress + 8, out var waitDisplayBufferIndex)) + { + var waitSequence = VulkanVideoPresenter.SubmitOrderedGuestFlipWait( + unchecked((int)waitVideoOutHandle), + unchecked((int)waitDisplayBufferIndex)); + TraceAgcShader( + $"agc.flip_wait_safe queue={state.QueueName} " + + $"submission={state.ActiveSubmissionId} " + + $"handle={waitVideoOutHandle} index={waitDisplayBufferIndex} " + + $"work_sequence={waitSequence}"); + } + if (op == ItNop && register == RFlip && length >= 6) { - if (!ctx.TryReadUInt32(currentAddress + 4, out var videoOutHandle) || - !ctx.TryReadUInt32(currentAddress + 8, out var displayBufferIndexRaw) || - !ctx.TryReadUInt32(currentAddress + 12, out var flipMode) || - !ctx.TryReadUInt32(currentAddress + 16, out var flipArgLo) || - !ctx.TryReadUInt32(currentAddress + 20, out var flipArgHi)) + TraceFramePacketSummary(state); + SyncCpuWrittenGuestImages(ctx); + if (!TryReadUInt32(ctx, currentAddress + 4, out var videoOutHandle) || + !TryReadUInt32(ctx, currentAddress + 8, out var displayBufferIndexRaw) || + !TryReadUInt32(ctx, currentAddress + 12, out var flipMode) || + !TryReadUInt32(ctx, currentAddress + 16, out var flipArgLo) || + !TryReadUInt32(ctx, currentAddress + 20, out var flipArgHi)) { - return; + return false; } var flipArg = unchecked((long)(((ulong)flipArgHi << 32) | flipArgLo)); var displayBufferIndex = unchecked((int)displayBufferIndexRaw); var handle = unchecked((int)videoOutHandle); + if (state.PendingTargetlessDraw is { } pendingComposite && + VideoOutExports.TryGetDisplayBufferInfo( + handle, + displayBufferIndex, + out var pendingDisplayBuffer) && + state.KnownRenderTargets.TryGetValue( + pendingDisplayBuffer.Address, + out var pendingDisplayTarget)) + { + var textures = CreateGuestDrawTextures( + ctx, + pendingComposite.Textures, + out _); + var globalMemoryBuffers = + CreateTranslatedDrawGlobalBuffers(pendingComposite); + var vertexBuffers = + CreateGuestVertexBuffers(pendingComposite.VertexInputs); + ProvideRenderTargetInitialData(ctx, pendingDisplayTarget); + GuestGpu.Current.SubmitOffscreenTranslatedDraw( + pendingComposite.PixelShader, + textures, + globalMemoryBuffers, + pendingComposite.AttributeCount, + [new GuestRenderTarget( + pendingDisplayTarget.Address, + pendingDisplayTarget.Width, + pendingDisplayTarget.Height, + pendingDisplayTarget.Format, + pendingDisplayTarget.NumberType)], + pendingComposite.VertexShader, + pendingComposite.VertexCount, + pendingComposite.InstanceCount, + pendingComposite.PrimitiveType, + pendingComposite.IndexBuffer, + vertexBuffers, + pendingComposite.RenderState, + pendingComposite.DepthTarget, + pendingComposite.PixelShaderAddress); + TraceAgcShader( + $"agc.deferred_composite ps=0x{pendingComposite.PixelShaderAddress:X16} " + + $"src=0x{pendingComposite.Textures.FirstOrDefault()?.Descriptor.Address ?? 0:X16} " + + $"dst=0x{pendingDisplayTarget.Address:X16} " + + $"size={pendingDisplayTarget.Width}x{pendingDisplayTarget.Height}"); + state.PendingTargetlessDraw = null; + state.TranslatedDraw = null; + } + if (VideoOutExports.TryGetDisplayBufferInfo( handle, displayBufferIndex, out var cachedDisplayBuffer) && - GuestGpu.Current.TrySubmitGuestImage( + GuestGpu.Current.TrySubmitOrderedGuestImageFlip( + handle, + displayBufferIndex, cachedDisplayBuffer.Address, cachedDisplayBuffer.Width, cachedDisplayBuffer.Height, @@ -2993,7 +3421,7 @@ public static class AgcExports "draw-fallback"); var textures = CreateGuestDrawTextures(ctx, translatedDraw.Textures, out var fallbackTextureCount); var globalMemoryBuffers = - CreateGuestMemoryBuffers(translatedDraw.GlobalMemoryBindings); + CreateTranslatedDrawGlobalBuffersForPresent(ctx, translatedDraw); GuestGpu.Current.SubmitTranslatedDraw( translatedDraw.PixelShader, textures, @@ -3047,11 +3475,72 @@ public static class AgcExports _ = VideoOutExports.SubmitFlipFromAgc(ctx, handle, displayBufferIndex, unchecked((int)flipMode), flipArg); state.SawIndexedDraw = false; state.GuestDrawKind = GuestDrawKind.None; + if (state.PendingTargetlessDraw is { } unusedPendingDraw) + { + ReturnPooledDrawArrays( + unusedPendingDraw, + globals: true, + vertex: true, + index: true); + state.PendingTargetlessDraw = null; + } state.TranslatedDraw = null; } offset += length; } + + return false; + } + + private static void TraceFramePacketSummary(SubmittedDcbState state) + { + if (!_traceFramePackets) + { + return; + } + + var flip = ++state.FlipCount; + if (flip <= 8 || flip % 60 == 0 || state.FrameDrawCount == 0) + { + var opcodes = string.Join( + ',', + state.FramePacketCounts + .OrderByDescending(entry => entry.Value) + .ThenBy(entry => entry.Key.Op) + .Take(32) + .Select(entry => entry.Key.Register == uint.MaxValue + ? $"0x{entry.Key.Op:X2}:{entry.Value}" + : $"0x{entry.Key.Op:X2}/r{entry.Key.Register}:{entry.Value}")); + Console.Error.WriteLine( + $"[FRAMEPKT] flip={flip} submission={state.ActiveSubmissionId} " + + $"packets={state.FramePacketCount} draws={state.FrameDrawCount} " + + $"dispatches={state.FrameDispatchCount} opcodes=[{opcodes}]"); + } + + state.FramePacketCounts.Clear(); + state.FramePacketCount = 0; + state.FrameDrawCount = 0; + state.FrameDispatchCount = 0; + } + + private static void TracePacketParseFailure( + SubmittedDcbState state, + ulong address, + uint offset, + uint header, + string reason) + { + if (!_traceFramePackets || + Interlocked.Increment(ref _packetParseFailureTraceCount) > 128) + { + return; + } + + Console.Error.WriteLine( + $"[FRAMEPKT] parse-failure queue={state.QueueName} " + + $"submission={state.ActiveSubmissionId} offset={offset} " + + $"address=0x{address:X16} header=0x{header:X8} reason={reason}"); } private static void TraceDisplayBuffer( @@ -3077,44 +3566,623 @@ public static class AgcExports private static void ApplySubmittedDmaData( CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, ulong packetAddress, + bool compactLayout, bool tracePacket) { - if (!ctx.TryReadUInt32(packetAddress + 12, out var byteCount) || - !ctx.TryReadUInt64(packetAddress + 16, out var destinationAddress) || - !ctx.TryReadUInt64(packetAddress + 24, out var sourceAddress)) + var byteCountOffset = compactLayout ? 20UL : 12UL; + var destinationOffset = compactLayout ? 4UL : 16UL; + var sourceOffset = compactLayout ? 12UL : 24UL; + if (!TryReadUInt32(ctx, packetAddress + byteCountOffset, out var byteCount) || + !TryReadUInt64(ctx, packetAddress + destinationOffset, out var destinationAddress) || + !TryReadUInt64(ctx, packetAddress + sourceOffset, out var sourceAddress)) { return; } - var copied = - byteCount != 0 && - byteCount <= 256u * 1024u * 1024u && - destinationAddress != 0 && - sourceAddress != 0 && - TryCopyGuestMemory(ctx, sourceAddress, destinationAddress, byteCount); + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + InvalidateDcbWindowIfOverlaps(destinationAddress, byteCount); + var immediateFill = + compactLayout && + destinationAddress >= 0x10000 && + sourceAddress <= uint.MaxValue; + var copied = + byteCount != 0 && + byteCount <= 256u * 1024u * 1024u && + destinationAddress != 0 && + (immediateFill + ? TryFillGuestMemory(ctx, (uint)sourceAddress, destinationAddress, byteCount) + : sourceAddress != 0 && + TryCopyGuestMemory(ctx, sourceAddress, destinationAddress, byteCount)); + if (copied) + { + MirrorDmaWriteToGuestImage( + ctx, + destinationAddress, + byteCount, + immediateFill ? (uint)sourceAddress : null); + } + + if (tracePacket) + { + TraceAgc( + $"agc.dcb.dma_data dst=0x{destinationAddress:X16} " + + $"src=0x{sourceAddress:X16} bytes={byteCount} " + + $"fill={immediateFill} copied={copied}"); + } + }, + $"agc_dma_data dst=0x{destinationAddress:X16} bytes={byteCount}", + packetAddress, + destinationAddress, + byteCount); + } + + private static void SubmitOrderedGpuSideEffect( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, + Action action, + string debugName, + ulong packetAddress, + ulong producerAddress = 0, + ulong producerLength = 0) + { + var producer = RegisterLabelProducer( + ctx.Memory, + state, + packetAddress, + producerAddress, + producerLength, + debugName); + + void CompleteAndWake() + { + CompleteLabelProducer(producer); + if (GpuWaitRegistry.Count == 0) + { + return; + } + + // Resuming a DCB can enqueue another compute dispatch and wait for + // it. Never do that reentrantly on the Vulkan render thread. + ThreadPool.UnsafeQueueUserWorkItem( + static state => + { + var (resumeContext, resumeGpuState) = state; + lock (resumeGpuState.Gate) + { + DrainResumableDcbs( + resumeContext, + resumeGpuState, + tracePackets: _traceAgc); + } + }, + (ctx, gpuState), + preferLocal: false); + } + + void ApplyAndQueueCompletion() + { + action(); + // DMA side effects can enqueue a Vulkan image mirror while this + // ordered action is executing. Completing the label here would + // wake another queue before that mirror is visible. Queue a + // second same-queue ordered action after all immediate follow-up + // writes; it fences those writes before publishing the producer. + if (VulkanVideoPresenter.SubmitOrderedGuestAction( + CompleteAndWake, + $"{debugName} completion") == 0) + { + CompleteAndWake(); + } + } + + if (VulkanVideoPresenter.SubmitOrderedGuestAction( + ApplyAndQueueCompletion, + debugName) == 0) + { + // Headless/startup submissions have no Vulkan queue to order + // against, so retaining the previous immediate behavior is exact. + ApplyAndQueueCompletion(); + } + } + + private static LabelProducerTrace? RegisterLabelProducer( + object memory, + SubmittedDcbState state, + ulong packetAddress, + ulong address, + ulong length, + string debugName) + { + if (address == 0 || length == 0) + { + return null; + } + + var producer = new LabelProducerTrace + { + Sequence = Interlocked.Increment(ref _labelProducerSequence), + Memory = memory, + Address = address, + Length = length, + PacketAddress = packetAddress, + SubmissionId = state.ActiveSubmissionId, + QueueName = state.QueueName, + DebugName = debugName, + }; + lock (_labelProducerGate) + { + if (_labelProducers.Count >= 4096) + { + _labelProducers.RemoveRange(0, 1024); + } + + _labelProducers.Add(producer); + } + + foreach (var waiting in GpuWaitRegistry.SnapshotInRange(memory, address, length)) + { + TraceAgc( + $"agc.wait_producer_scheduled label=0x{waiting.Address:X16} " + + $"waiters={waiting.Count} producer_seq={producer.Sequence} " + + $"queue={producer.QueueName} submission={producer.SubmissionId} " + + $"packet=0x{packetAddress:X16} action='{debugName}'"); + } + + return producer; + } + + private static void CompleteLabelProducer(LabelProducerTrace? producer) + { + if (producer is null) + { + return; + } + + lock (_labelProducerGate) + { + producer.Completed = true; + } + + foreach (var waiting in GpuWaitRegistry.SnapshotInRange( + producer.Memory, + producer.Address, + producer.Length)) + { + TraceAgc( + $"agc.wait_producer_completed label=0x{waiting.Address:X16} " + + $"waiters={waiting.Count} producer_seq={producer.Sequence} " + + $"queue={producer.QueueName} submission={producer.SubmissionId} " + + $"action='{producer.DebugName}'"); + } + } + + private static void TraceWaitProducerState( + object memory, + in GpuWaitRegistry.WaitingDcb waiter, + ulong commandAddress, + ulong packetAddress, + bool stale, + ulong? currentValue = null) + { + LabelProducerTrace? producer = null; + lock (_labelProducerGate) + { + for (var index = _labelProducers.Count - 1; index >= 0; index--) + { + var candidate = _labelProducers[index]; + if (!ReferenceEquals(candidate.Memory, memory) || + !RangesOverlap( + candidate.Address, + candidate.Length, + waiter.WaitAddress, + waiter.Is64Bit ? (ulong)sizeof(ulong) : sizeof(uint))) + { + continue; + } + + producer = candidate; + break; + } + + if (_tracedProducerlessWaits.Count >= 4096) + { + _tracedProducerlessWaits.Clear(); + } + + if (!stale && producer is null && + !_tracedProducerlessWaits.Add( + (memory, waiter.WaitAddress, waiter.SubmissionId))) + { + return; + } + } + + var prefix = stale ? "agc.wait_stale" : "agc.wait_suspended"; + var current = currentValue.HasValue + ? $"0x{currentValue.Value:X16}" + : "unreadable"; + var condition = + $"value={current} mask=0x{waiter.Mask:X16} " + + $"ref=0x{waiter.ReferenceValue:X16} cmp={waiter.CompareFunction} " + + $"control=0x{waiter.ControlValue:X8} bits={(waiter.Is64Bit ? 64 : 32)} " + + $"form={(waiter.IsStandard ? "standard" : "agc-nop")}"; + if (producer is null) + { + Console.Error.WriteLine( + $"[LOADER][WARN] {prefix} label=0x{waiter.WaitAddress:X16} " + + $"queue={waiter.QueueName} submission={waiter.SubmissionId} " + + $"command=0x{commandAddress:X16} packet=0x{packetAddress:X16} " + + condition + " " + + "producer=none-observed; remaining-suspended"); + return; + } + + TraceAgc( + $"{prefix} label=0x{waiter.WaitAddress:X16} " + + $"queue={waiter.QueueName} submission={waiter.SubmissionId} " + + condition + " " + + $"producer_seq={producer.Sequence} producer_state=" + + $"{(producer.Completed ? "completed" : "queued")} " + + $"producer_queue={producer.QueueName} " + + $"producer_submission={producer.SubmissionId} " + + $"producer_packet=0x{producer.PacketAddress:X16} " + + $"action='{producer.DebugName}'"); + } + + private static void ApplySubmittedAcquireMem( + CpuContext ctx, + SubmittedDcbState state, + ulong packetAddress, + bool tracePacket) + { + if (!TryDecodeSubmittedAcquireMem(ctx, packetAddress, out var acquire)) + { + TraceAgc( + $"agc.acquire_mem_decode_failed queue={state.QueueName} " + + $"submission={state.ActiveSubmissionId} packet=0x{packetAddress:X16}"); + return; + } + + var queueName = state.QueueName; + var submissionId = state.ActiveSubmissionId; + var debugName = + $"acquire_mem base=0x{acquire.BaseAddress:X16} size=0x{acquire.SizeBytes:X16} " + + $"gcr=0x{acquire.GcrControl:X8}"; + void ApplyAcquire() + { + // ExecuteOrderedGuestAction first flushes and waits for this guest + // queue, then writes back dirty guest buffers. At that exact PM4 + // point, refresh only tracked guest images covered by the acquire + // range. Cached sampled textures use the same dirty tracker and + // are evicted by the presenter without throwing away clean cache + // entries (hardware invalidation does not imply changed bytes). + if (acquire.InvalidatesGuestResources) + { + SyncCpuWrittenGuestImages( + ctx, + acquire.BaseAddress, + acquire.CoversAllGuestMemory + ? ulong.MaxValue + : acquire.SizeBytes); + } + + if (tracePacket) + { + TraceAgc( + $"agc.acquire_mem_applied queue={queueName} " + + $"submission={submissionId} packet=0x{packetAddress:X16} " + + $"work_sequence={VulkanVideoPresenter.CurrentGuestWorkSequenceForDiagnostics}"); + } + } + + var sequence = VulkanVideoPresenter.SubmitOrderedGuestAction( + ApplyAcquire, + debugName); + if (sequence == 0) + { + // Headless startup has no host GPU queue, but the guest-memory + // cache model still needs the same invalidation semantics. + ApplyAcquire(); + } + + // The bulk PM4 read is itself a parser-side cache. Do not retain it + // across a guest cache-invalidation point; subsequent packets return + // to live guest memory while the host barrier remains ordered in the + // logical GPU queue. Submission stays asynchronous, matching hardware + // and avoiding a CPU stall for every ACQUIRE_MEM packet. + _dcbWindowBuffer = null; + _dcbWindowByteLength = 0; + if (tracePacket) { TraceAgc( - $"agc.dcb.dma_data dst=0x{destinationAddress:X16} src=0x{sourceAddress:X16} " + - $"bytes={byteCount} copied={copied}"); + $"agc.acquire_mem queue={queueName} " + + $"submission={submissionId} packet=0x{packetAddress:X16} " + + $"engine={acquire.Engine} cbdb=0x{acquire.CbDbControl:X8} " + + $"base=0x{acquire.BaseAddress:X16} size=0x{acquire.SizeBytes:X16} " + + $"scope={(acquire.CoversAllGuestMemory ? "all" : "range")} " + + $"poll={acquire.PollInterval} gcr=0x{acquire.GcrControl:X8} " + + $"resource_inv={acquire.InvalidatesGuestResources} " + + $"sequence={sequence} scheduled={(sequence != 0)}"); + } + } + + private static bool TryDecodeSubmittedAcquireMem( + CpuContext ctx, + ulong packetAddress, + out SubmittedAcquireMem acquire) + { + if (!TryReadUInt32(ctx, packetAddress + 4, out var coherControl) || + !TryReadUInt32(ctx, packetAddress + 8, out var sizeLow) || + !TryReadUInt32(ctx, packetAddress + 12, out var sizeHigh) || + !TryReadUInt32(ctx, packetAddress + 16, out var baseLow) || + !TryReadUInt32(ctx, packetAddress + 20, out var baseHigh) || + !TryReadUInt32(ctx, packetAddress + 24, out var pollInterval) || + !TryReadUInt32(ctx, packetAddress + 28, out var gcrControl)) + { + acquire = default; + return false; + } + + acquire = DecodeSubmittedAcquireMem( + coherControl, + sizeLow, + sizeHigh, + baseLow, + baseHigh, + pollInterval, + gcrControl); + return true; + } + + private static SubmittedAcquireMem DecodeSubmittedAcquireMem( + uint coherControl, + uint sizeLow, + uint sizeHigh, + uint baseLow, + uint baseHigh, + uint pollInterval, + uint gcrControl) + { + // GFX10 ACQUIRE_MEM expresses COHER_SIZE and COHER_BASE in 256-byte + // units. SIZE_HI is 8 bits and BASE_HI is 24 bits in the packet. + var sizeUnits = sizeLow | ((ulong)(sizeHigh & 0xFFu) << 32); + var baseUnits = baseLow | ((ulong)(baseHigh & 0x00FF_FFFFu) << 32); + return new SubmittedAcquireMem( + Engine: coherControl >> 31, + CbDbControl: coherControl & 0x7FFF_FFFFu, + BaseAddress: baseUnits << 8, + SizeBytes: sizeUnits << 8, + PollInterval: pollInterval & 0xFFFFu, + GcrControl: gcrControl & 0x7FFFFu); + } + + private static void ResetSubmittedParserState(SubmittedDcbState state) + { + // Queue ownership, pending submissions and suspension bookkeeping are + // deliberately retained. Work emitted before this packet already owns + // immutable snapshots; clearing these fields affects only commands + // translated after RESET at this precise packet position. + state.CxRegisters.Clear(); + state.ShRegisters.Clear(); + state.UcRegisters.Clear(); + state.PresenterTexture = null; + state.GuestDrawKind = GuestDrawKind.None; + state.TranslatedDraw = null; + state.RenderTargetWriters.Clear(); + state.IndirectArgsAddress = 0; + state.SawIndexedDraw = false; + state.IndexBufferAddress = 0; + state.IndexBufferCount = 0; + state.IndexSize = 0; + state.InstanceCount = 1; + state.DrawIndexOffset = 0; + } + + private static bool RangesOverlap( + ulong leftAddress, + ulong leftLength, + ulong rightAddress, + ulong rightLength) + { + var leftEnd = leftAddress > ulong.MaxValue - leftLength + ? ulong.MaxValue + : leftAddress + leftLength; + var rightEnd = rightAddress > ulong.MaxValue - rightLength + ? ulong.MaxValue + : rightAddress + rightLength; + return leftAddress < rightEnd && rightAddress < leftEnd; + } + + /// + /// PS5 render targets alias guest memory, so a CP DMA fill or copy that + /// lands on an RT is visible to later GPU reads. Our render targets live + /// in Vulkan images, so mirror DMA writes into them: fills become + /// vkCmdClearColorImage, copies re-upload the guest bytes. Without this, + /// per-frame DMA clears never reach the image (the fog layer in Dreaming + /// Sarah accumulates until it saturates, washing the scene out). + /// + /// + /// PS5 render targets alias unified memory, so the game's CPU can rewrite a + /// surface (Chowdren memsets its fog-noise layer every frame) and the GPU + /// observes it. Our Vulkan guest images are separate storage, so re-upload + /// CPU-authored surfaces once per flip. Surfaces only the GPU writes keep + /// all-zero guest memory and are skipped, preserving their GPU content. + /// + private static long _guestImageSyncTraceCount; + + private static void SyncCpuWrittenGuestImages( + CpuContext ctx, + ulong scopeAddress = 0, + ulong scopeByteCount = ulong.MaxValue) + { + if (!SharpEmu.HLE.GuestImageWriteTracker.Enabled || scopeByteCount == 0) + { + return; + } + + foreach (var (address, width, height, byteCount) in VulkanVideoPresenter.GetGuestImageExtents()) + { + if (scopeByteCount != ulong.MaxValue && + !RangesOverlap(address, byteCount, scopeAddress, scopeByteCount)) + { + continue; + } + + if (!SharpEmu.HLE.GuestImageWriteTracker.ConsumeDirty(address)) + { + continue; + } + + if (byteCount == 0 || byteCount > MaxPresentedTextureBytes) + { + continue; + } + + var pixels = new byte[byteCount]; + if (ctx.Memory.TryRead(address, pixels)) + { + VulkanVideoPresenter.SubmitGuestImageWrite(address, pixels); + if (Interlocked.Increment(ref _guestImageSyncTraceCount) <= 64) + { + Console.Error.WriteLine( + $"[SYNC] cpu-write addr=0x{address:X} {width}x{height}"); + } + } + + SharpEmu.HLE.GuestImageWriteTracker.Rearm(address); + } + } + + private static long _dmaMirrorTraceCount; + private static readonly Dictionary<(uint Op, uint Register), long> _submittedOpcodeCounts = new(); + private static long _submittedOpcodeTotal; + + private static void CountSubmittedOpcode(uint op, uint register) + { + var key = (op, op == ItNop ? register : uint.MaxValue); + lock (_submittedOpcodeCounts) + { + _submittedOpcodeCounts[key] = + _submittedOpcodeCounts.TryGetValue(key, out var count) ? count + 1 : 1; + if (++_submittedOpcodeTotal % 500_000 == 0) + { + var summary = string.Join( + ' ', + _submittedOpcodeCounts + .OrderByDescending(entry => entry.Value) + .Select(entry => entry.Key.Register == uint.MaxValue + ? $"0x{entry.Key.Op:X2}:{entry.Value}" + : $"0x{entry.Key.Op:X2}/r{entry.Key.Register}:{entry.Value}")); + Console.Error.WriteLine($"[PKT] total={_submittedOpcodeTotal} {summary}"); + } + } + } + + private static void MirrorDmaWriteToGuestImage( + CpuContext ctx, + ulong destinationAddress, + ulong byteCount, + uint? fillValue) + { + var hasImage = VulkanVideoPresenter.TryGetGuestImageExtent( + destinationAddress, + out var width, + out var height, + out var imageBytes); + if (_traceDraws && Interlocked.Increment(ref _dmaMirrorTraceCount) <= 400) + { + Console.Error.WriteLine( + $"[DMA] dst=0x{destinationAddress:X} bytes={byteCount} " + + $"fill={(fillValue is { } f ? $"0x{f:X8}" : "copy")} image={hasImage}"); + } + + if (!hasImage) + { + return; + } + + if (imageBytes == 0 || byteCount < imageBytes) + { + return; + } + + if (fillValue is { } fill) + { + VulkanVideoPresenter.SubmitGuestImageFill(destinationAddress, fill); + return; + } + + var pixels = new byte[imageBytes]; + if (ctx.Memory.TryRead(destinationAddress, pixels)) + { + VulkanVideoPresenter.SubmitGuestImageWrite(destinationAddress, pixels); } } private static void ApplySubmittedStandardDmaData( CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, ulong packetAddress) { - if (!ctx.TryReadUInt32(packetAddress + 4, out var control) || - !ctx.TryReadUInt32(packetAddress + 8, out var sourceLow) || - !ctx.TryReadUInt32(packetAddress + 12, out var sourceHigh) || - !ctx.TryReadUInt32(packetAddress + 16, out var destinationLow) || - !ctx.TryReadUInt32(packetAddress + 20, out var destinationHigh) || - !ctx.TryReadUInt32(packetAddress + 24, out var command)) + if (!TryReadUInt32(ctx, packetAddress + 4, out var control) || + !TryReadUInt32(ctx, packetAddress + 8, out var sourceLow) || + !TryReadUInt32(ctx, packetAddress + 12, out var sourceHigh) || + !TryReadUInt32(ctx, packetAddress + 16, out var destinationLow) || + !TryReadUInt32(ctx, packetAddress + 20, out var destinationHigh) || + !TryReadUInt32(ctx, packetAddress + 24, out var command)) { return; } + var byteCount = command & 0x1F_FFFFu; + var destinationSelect = (control >> 20) & 0x3u; + var destinationSwap = (command >> 24) & 0x3u; + var destinationAddressSpace = (command >> 27) & 0x1u; + var destinationAddress = destinationLow | ((ulong)destinationHigh << 32); + var writesGuestMemory = + byteCount != 0 && + destinationSwap == 0 && + destinationSelect is 0 or 3 && + (destinationSelect == 3 || destinationAddressSpace == 0); + + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => ApplySubmittedStandardDmaDataSnapshot( + ctx, + control, + sourceLow, + sourceHigh, + destinationLow, + destinationHigh, + command), + $"dma_data dst=0x{destinationHigh:X8}{destinationLow:X8} bytes={byteCount}", + packetAddress, + writesGuestMemory ? destinationAddress : 0, + writesGuestMemory ? byteCount : 0); + } + + private static void ApplySubmittedStandardDmaDataSnapshot( + CpuContext ctx, + uint control, + uint sourceLow, + uint sourceHigh, + uint destinationLow, + uint destinationHigh, + uint command) + { var byteCount = command & 0x1F_FFFFu; var sourceSelect = (control >> 29) & 0x3u; var destinationSelect = (control >> 20) & 0x3u; @@ -3132,6 +4200,7 @@ public static class AgcExports var destinationAddress = destinationLow | ((ulong)destinationHigh << 32); + InvalidateDcbWindowIfOverlaps(destinationAddress, byteCount); bool copied; ulong sourceAddress; if (sourceSelect is 0 or 3 && @@ -3141,12 +4210,16 @@ public static class AgcExports if (sourceAddressIncrement != 0) { copied = - ctx.TryReadUInt32(sourceAddress, out var fillValue) && + TryReadUInt32(ctx, sourceAddress, out var fillValue) && TryFillGuestMemory( ctx, fillValue, destinationAddress, byteCount); + if (copied) + { + MirrorDmaWriteToGuestImage(ctx, destinationAddress, byteCount, fillValue); + } } else { @@ -3155,6 +4228,10 @@ public static class AgcExports sourceAddress, destinationAddress, byteCount); + if (copied) + { + MirrorDmaWriteToGuestImage(ctx, destinationAddress, byteCount, fillValue: null); + } } } else if (sourceSelect == 2) @@ -3165,6 +4242,10 @@ public static class AgcExports sourceLow, destinationAddress, byteCount); + if (copied) + { + MirrorDmaWriteToGuestImage(ctx, destinationAddress, byteCount, sourceLow); + } } else { @@ -3183,48 +4264,792 @@ public static class AgcExports private static void ApplySubmittedWriteData( CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, ulong packetAddress, uint packetLength, + bool standardPacket, bool tracePacket) { - if (!ctx.TryReadUInt32(packetAddress + 4, out var control) || - !ctx.TryReadUInt64(packetAddress + 8, out var destinationAddress)) + if (!TryReadUInt32(ctx, packetAddress + 4, out var control) || + !TryReadUInt64(ctx, packetAddress + 8, out var destinationAddress)) { return; } - var destination = control & 0xFFu; - var increment = (control >> 16) & 0xFFu; + var (destination, incrementAddress, writeConfirm, cachePolicy) = standardPacket + ? DecodeStandardWriteDataControl(control) + : DecodeAgcWriteDataControl(control); var dwordCount = packetLength - 4; - var wroteData = destination is 1 or 2 or 4 or 5; - for (uint index = 0; wroteData && index < dwordCount; index++) + var values = new uint[dwordCount]; + for (uint index = 0; index < dwordCount; index++) { var sourceAddress = packetAddress + 16 + ((ulong)index * sizeof(uint)); - var targetAddress = destinationAddress + - (increment == 0 ? (ulong)index * sizeof(uint) : 0); - wroteData = - ctx.TryReadUInt32(sourceAddress, out var value) && - ctx.TryWriteUInt32(targetAddress, value); + if (!TryReadUInt32(ctx, sourceAddress, out values[index])) + { + return; + } } + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + InvalidateDcbWindowIfOverlaps( + destinationAddress, + incrementAddress ? (ulong)dwordCount * sizeof(uint) : sizeof(uint)); + var wroteData = destination is 1 or 2 or 4 or 5; + for (uint index = 0; wroteData && index < dwordCount; index++) + { + var targetAddress = destinationAddress + + (incrementAddress ? (ulong)index * sizeof(uint) : 0); + wroteData = TryWriteUInt32(ctx, targetAddress, values[index]); + } + + if (tracePacket) + { + TraceAgc( + $"agc.dcb.write_data dst={destination} " + + $"addr=0x{destinationAddress:X16} count={dwordCount} " + + $"increment={incrementAddress} confirm={writeConfirm} " + + $"cache={cachePolicy} standard={standardPacket} wrote={wroteData}"); + } + }, + $"write_data dst=0x{destinationAddress:X16} count={dwordCount}", + packetAddress, + destination is 1 or 2 or 4 or 5 ? destinationAddress : 0, + destination is 1 or 2 or 4 or 5 + ? incrementAddress ? (ulong)dwordCount * sizeof(uint) : sizeof(uint) + : 0); + } + + private static (uint Destination, bool IncrementAddress, bool WriteConfirm, uint CachePolicy) + DecodeStandardWriteDataControl(uint control) + { + // GFX10 PKT3_WRITE_DATA is not byte-packed like sceAgcDcbWriteData's + // NOP wrapper: DST_SEL is 11:8, ADDR_INCR is bit 16 (0 increments), + // WR_CONFIRM is bit 20, and CACHE_POLICY is 26:25. In particular, the + // low byte is reserved and must never be interpreted as DST_SEL. + return ( + Destination: (control >> 8) & 0xFu, + IncrementAddress: (control & (1u << 16)) == 0, + WriteConfirm: (control & (1u << 20)) != 0, + CachePolicy: (control >> 25) & 0x3u); + } + + private static (uint Destination, bool IncrementAddress, bool WriteConfirm, uint CachePolicy) + DecodeAgcWriteDataControl(uint control) => + ( + Destination: control & 0xFFu, + IncrementAddress: ((control >> 16) & 0xFFu) == 0, + WriteConfirm: ((control >> 24) & 0xFFu) != 0, + CachePolicy: (control >> 8) & 0xFFu); + +#if DEBUG + private static void ValidateWriteDataControlDecoders() + { + // Regression vector: reserved low-byte noise previously decoded 0xA5 + // as DST_SEL, causing a valid standard memory write to be discarded. + const uint standardControl = 0xA5u | (5u << 8) | (1u << 16) | (1u << 20) | (2u << 25); + var standard = DecodeStandardWriteDataControl(standardControl); + System.Diagnostics.Debug.Assert(standard.Destination == 5u); + System.Diagnostics.Debug.Assert(!standard.IncrementAddress); + System.Diagnostics.Debug.Assert(standard.WriteConfirm); + System.Diagnostics.Debug.Assert(standard.CachePolicy == 2u); + + const uint agcControl = 4u | (3u << 8) | (1u << 24); + var agc = DecodeAgcWriteDataControl(agcControl); + System.Diagnostics.Debug.Assert(agc.Destination == 4u); + System.Diagnostics.Debug.Assert(agc.IncrementAddress); + System.Diagnostics.Debug.Assert(agc.WriteConfirm); + System.Diagnostics.Debug.Assert(agc.CachePolicy == 3u); + } + + private static void ValidateDispatchInitiators() + { + const uint threadCount = 0x00F0_0100u; + const uint localSize = 64u; + var initiator = DirectDispatchInitiator(0); + System.Diagnostics.Debug.Assert((initiator & (1u << 5)) == 0); + System.Diagnostics.Debug.Assert((initiator & (1u << 6)) != 0); + System.Diagnostics.Debug.Assert(threadCount * localSize == 0x3C00_4000u); + System.Diagnostics.Debug.Assert(CeilDivide(20, 8) == 3); + System.Diagnostics.Debug.Assert(CeilDivide(12, 8) == 2); + } + + private static void ValidateSubmittedQueueAndReleaseMemDecoders() + { + var nggRegisters = new Dictionary + { + [GsUserDataRegister - 1] = 3u << 1, + }; + System.Diagnostics.Debug.Assert( + SelectExportUserDataRegister(nggRegisters) == GsUserDataRegister); + + var queue = new SubmittedDcbState(); + queue.PendingSubmissions.Enqueue(new(0x1000, 8, 11, false)); + queue.PendingSubmissions.Enqueue(new(0x2000, 16, 12, true)); + System.Diagnostics.Debug.Assert( + queue.PendingSubmissions.Dequeue().SubmissionId == 11); + System.Diagnostics.Debug.Assert( + queue.PendingSubmissions.Dequeue().SubmissionId == 12); + + var control = (1u << 16) | (2u << 29); + var decoded = DecodeStandardReleaseMemControl(control); + System.Diagnostics.Debug.Assert(decoded.Destination == 1u); + System.Diagnostics.Debug.Assert(decoded.DataSelection == 2u); + System.Diagnostics.Debug.Assert( + PatchUInt32Bits(0xABCD_1234u, 0x00FF_0000u, 3u << 16) == + 0xAB03_1234u); + } + + private static void ValidateAcquireMemAndQueueResetDecoders() + { + var range = DecodeSubmittedAcquireMem( + 0x8000_7FC0u, + 0x0000_0123u, + 0x45u, + 0x89AB_CDEFu, + 0x0012_3456u, + 0x1_000Au, + 0x0001_0388u); + System.Diagnostics.Debug.Assert(range.Engine == 1u); + System.Diagnostics.Debug.Assert(range.CbDbControl == 0x7FC0u); + System.Diagnostics.Debug.Assert(range.SizeBytes == 0x0000_4500_0001_2300UL); + System.Diagnostics.Debug.Assert(range.BaseAddress == 0x1234_5689_ABCD_EF00UL); + System.Diagnostics.Debug.Assert(range.PollInterval == 0xAu); + System.Diagnostics.Debug.Assert(range.InvalidatesGuestResources); + System.Diagnostics.Debug.Assert(!range.CoversAllGuestMemory); + + var all = DecodeSubmittedAcquireMem(0, 0, 0, 0, 0, 0, 0x280u); + System.Diagnostics.Debug.Assert(all.CoversAllGuestMemory); + System.Diagnostics.Debug.Assert(all.InvalidatesGuestResources); + var explicitAll = DecodeSubmittedAcquireMem(0, 1, 0, 0, 0, 0, 0x103C0u); + System.Diagnostics.Debug.Assert(explicitAll.CoversAllGuestMemory); + + var queue = new SubmittedDcbState + { + QueueName = "validator", + ActiveSubmissionId = 7, + HasActiveSubmission = true, + IsSuspended = true, + IndexBufferAddress = 0x1000, + IndexBufferCount = 12, + IndexSize = 1, + InstanceCount = 4, + DrawIndexOffset = 2, + IndirectArgsAddress = 0x2000, + SawIndexedDraw = true, + GuestDrawKind = GuestDrawKind.FullscreenBarycentric, + }; + queue.CxRegisters.Add(1, 2); + queue.ShRegisters.Add(3, 4); + queue.UcRegisters.Add(5, 6); + queue.PendingSubmissions.Enqueue(new(0x3000, 2, 8, false)); + ResetSubmittedParserState(queue); + System.Diagnostics.Debug.Assert(queue.CxRegisters.Count == 0); + System.Diagnostics.Debug.Assert(queue.ShRegisters.Count == 0); + System.Diagnostics.Debug.Assert(queue.UcRegisters.Count == 0); + System.Diagnostics.Debug.Assert(queue.IndexBufferAddress == 0); + System.Diagnostics.Debug.Assert(queue.IndexBufferCount == 0); + System.Diagnostics.Debug.Assert(queue.IndexSize == 0); + System.Diagnostics.Debug.Assert(queue.InstanceCount == 1); + System.Diagnostics.Debug.Assert(queue.DrawIndexOffset == 0); + System.Diagnostics.Debug.Assert(queue.IndirectArgsAddress == 0); + System.Diagnostics.Debug.Assert(!queue.SawIndexedDraw); + System.Diagnostics.Debug.Assert(queue.GuestDrawKind == GuestDrawKind.None); + System.Diagnostics.Debug.Assert(queue.QueueName == "validator"); + System.Diagnostics.Debug.Assert(queue.ActiveSubmissionId == 7); + System.Diagnostics.Debug.Assert(queue.HasActiveSubmission); + System.Diagnostics.Debug.Assert(queue.IsSuspended); + System.Diagnostics.Debug.Assert(queue.PendingSubmissions.Count == 1); + } + + private static void ValidateDepthTargetDecoder() + { + var registers = new Dictionary + { + [DbDepthControl] = 0x2u | 0x4u | (1u << 4), + [DbDepthSizeXy] = 1919u | (1079u << 16), + [DbDepthClear] = BitConverter.SingleToUInt32Bits(1f), + [DbZInfo] = 3u | (24u << 4), + [DbZReadBase] = 0x0123_4567u, + [DbZWriteBase] = 0x0123_4567u, + [DbZReadBaseHi] = 2u, + [DbZWriteBaseHi] = 2u, + }; + var depth = DecodeDepthTarget(registers); + System.Diagnostics.Debug.Assert(depth is not null); + System.Diagnostics.Debug.Assert(depth.Width == 1920 && depth.Height == 1080); + System.Diagnostics.Debug.Assert(depth.GuestFormat == 3u); + System.Diagnostics.Debug.Assert(depth.SwizzleMode == 24u); + System.Diagnostics.Debug.Assert(depth.Address == 0x0000_0201_2345_6700UL); + System.Diagnostics.Debug.Assert(depth.ClearDepth == 1f); + } +#endif + + // SHARPEMU_GPU_WAIT_MODE=force reverts to the legacy behaviour of faking a + // satisfying value at parse time. Default (suspend) properly suspends the + // DCB on an unmet WAIT_REG_MEM and resumes it once the awaited completion + // label is genuinely written by a later submit — preserving cross-submit + // ordering so the work after a wait (e.g. the final composite) does not run + // ahead of the compute it samples. + private static readonly bool _gpuWaitSuspendEnabled = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_GPU_WAIT_MODE"), + "force", + StringComparison.OrdinalIgnoreCase); + + // Optional age for one-shot missing-producer diagnostics. Stale waits are + // never removed or force-satisfied in the default suspend mode: doing so + // advances a queue without its real cross-queue producer and can publish + // incomplete CPU/GPU state. Only SHARPEMU_GPU_WAIT_MODE=force retains the + // explicit legacy mutation path above. Default 0 disables age diagnostics. + private static readonly long _gpuWaitStaleTicks = + (long.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_GPU_WAIT_FALLBACK_MS"), + out var fallbackMs) && fallbackMs >= 0 + ? fallbackMs + : 0L) * System.Diagnostics.Stopwatch.Frequency / 1000L; + + // Reads the WAIT_REG_MEM watched address, reference, mask, and 3-bit compare + // function for both the AGC NOP-encapsulated (RWaitMem32/64) and the standard + // ItWaitRegMem packet layouts. + private static bool TryParseSubmittedWait( + CpuContext ctx, + ulong packetAddress, + bool is64Bit, + bool isStandard, + out ulong waitAddress, + out ulong reference, + out ulong mask, + out uint compareFunction, + out uint controlValue) + { + waitAddress = 0; + reference = 0; + mask = 0; + compareFunction = 0; + controlValue = 0; + if (isStandard) + { + if (!TryReadUInt32(ctx, packetAddress + 4, out var stdControl) || + !TryReadUInt64(ctx, packetAddress + 8, out waitAddress) || + !TryReadUInt32(ctx, packetAddress + 16, out var stdRef) || + !TryReadUInt32(ctx, packetAddress + 20, out var stdMask)) + { + return false; + } + + compareFunction = stdControl & 0x7u; + controlValue = stdControl; + reference = stdRef; + mask = stdMask; + return true; + } + + if (!TryReadUInt64(ctx, packetAddress + 4, out waitAddress) || + !TryReadUInt32(ctx, packetAddress + (is64Bit ? 28u : 16u), out var control)) + { + return false; + } + + compareFunction = control & 0x7u; + controlValue = control; + if (is64Bit) + { + return TryReadUInt64(ctx, packetAddress + 12, out mask) && + TryReadUInt64(ctx, packetAddress + 20, out reference); + } + + if (!TryReadUInt32(ctx, packetAddress + 12, out var mask32) || + !TryReadUInt32(ctx, packetAddress + 20, out var reference32)) + { + return false; + } + + mask = mask32; + reference = reference32; + return true; + } + + // Returns true when the DCB should suspend parsing at this wait (its + // continuation was registered into GpuWaitRegistry); false to keep parsing + // (already satisfied, unreadable, or legacy force-satisfy mode). + private static bool HandleSubmittedWaitRegMem( + CpuContext ctx, + SubmittedDcbState state, + ulong commandAddress, + ulong packetAddress, + uint offset, + uint length, + uint dwordCount, + bool is64Bit, + bool isStandard, + bool tracePacket) + { + if (!TryParseSubmittedWait( + ctx, packetAddress, is64Bit, isStandard, + out var waitAddress, out var reference, out var mask, out var compareFunction, + out var controlValue)) + { + return false; + } + + // COMPARE_FUNC=0 is the hardware "always" condition. Reserved 7 is + // also fail-open; neither condition may register a waiter. Validate + // the watched memory before any read so null/malformed packets cannot + // become permanent entries keyed by address zero. + if (compareFunction is 0 or 7) + { + TraceSubmittedWait( + waitAddress, + 0, + mask, + reference, + compareFunction, + is64Bit ? 64 : 32, + tracePacket); + return false; + } + + var requiredAlignment = is64Bit ? sizeof(ulong) : sizeof(uint); + if (waitAddress == 0 || + mask == 0 || + waitAddress % (ulong)requiredAlignment != 0) + { + TraceAgc( + $"agc.dcb.wait_reject addr=0x{waitAddress:X16} " + + $"mask=0x{mask:X16} compare={compareFunction} bits=" + + $"{(is64Bit ? 64 : 32)} standard={isStandard} " + + $"packet=0x{packetAddress:X16} reason=invalid-address-or-mask"); + return false; + } + + ulong currentValue = 0; + bool hasCurrent; + if (is64Bit) + { + hasCurrent = TryReadUInt64(ctx, waitAddress, out currentValue); + } + else if (TryReadUInt32(ctx, waitAddress, out var current32)) + { + currentValue = current32; + hasCurrent = true; + } + else + { + hasCurrent = false; + } + + TraceSubmittedWait( + waitAddress, currentValue, mask, reference, compareFunction, + is64Bit ? 64 : 32, tracePacket); + + var waiter = new GpuWaitRegistry.WaitingDcb + { + CommandBufferAddress = commandAddress, + ResumeAddress = packetAddress + ((ulong)length * sizeof(uint)), + TotalDwords = dwordCount, + ResumeOffset = offset + length, + ReferenceValue = reference, + Mask = mask, + CompareFunction = compareFunction, + ControlValue = controlValue, + Is64Bit = is64Bit, + IsStandard = isStandard, + WaitAddress = waitAddress, + Memory = ctx.Memory, + QueueName = state.QueueName, + SubmissionId = state.ActiveSubmissionId, + RegisteredTicks = System.Diagnostics.Stopwatch.GetTimestamp(), + State = state, + }; + + if (hasCurrent && GpuWaitRegistry.Compare(waiter, currentValue)) + { + return false; // already satisfied — keep parsing + } + + if (!_gpuWaitSuspendEnabled) + { + if (hasCurrent) + { + ForceSatisfyGpuWait(ctx, waiter, currentValue); + } + + return false; + } + + if (!hasCurrent) + { + return false; // cannot evaluate the label — do not stall the DCB + } + + GpuWaitRegistry.Register(waitAddress, waiter); + var gpuState = _submittedGpuStates.GetValue( + ctx.Memory, + static _ => new SubmittedGpuState()); + EnsureGpuWaitMonitor(ctx, gpuState); + TraceWaitProducerState( + ctx.Memory, + waiter, + commandAddress, + packetAddress, + stale: false, + currentValue); if (tracePacket) { TraceAgc( - $"agc.dcb.write_data dst={destination} addr=0x{destinationAddress:X16} " + - $"count={dwordCount} increment={increment} wrote={wroteData}"); + $"agc.dcb.suspended addr=0x{waitAddress:X16} ref=0x{reference:X16} " + + $"mask=0x{mask:X16} cur=0x{currentValue:X16} cmp={compareFunction}"); + } + + return true; + } + + /// + /// Direct guest CPU stores can satisfy a GPU wait without crossing another + /// AGC import. Keep one low-frequency monitor per guest memory while waits + /// exist so those real stores wake their queues. The monitor never changes + /// a label: it uses the same masked comparison as submission-time parsing + /// and resumes only after the guest value genuinely satisfies the packet. + /// + private static void EnsureGpuWaitMonitor( + CpuContext submitContext, + SubmittedGpuState gpuState) + { + if (gpuState.WaitMonitorRunning) + { + return; + } + + gpuState.WaitMonitorRunning = true; + var monitorContext = new CpuContext( + submitContext.Memory, + submitContext.TargetGeneration); + ThreadPool.UnsafeQueueUserWorkItem( + static state => MonitorGpuWaits(state.Context, state.GpuState), + (Context: monitorContext, GpuState: gpuState), + preferLocal: false); + } + + private static void MonitorGpuWaits( + CpuContext ctx, + SubmittedGpuState gpuState) + { + var delayMilliseconds = 1; + while (true) + { + var madeProgress = false; + lock (gpuState.Gate) + { + var before = GpuWaitRegistry.CountForMemory(ctx.Memory); + if (before == 0) + { + gpuState.WaitMonitorRunning = false; + return; + } + + DrainResumableDcbs(ctx, gpuState, tracePackets: _traceAgc); + var after = GpuWaitRegistry.CountForMemory(ctx.Memory); + madeProgress = after < before; + if (madeProgress) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] agc.wait_monitor_resumed count={before - after} " + + $"remaining={after}"); + } + if (after == 0) + { + gpuState.WaitMonitorRunning = false; + return; + } + } + + delayMilliseconds = madeProgress + ? 1 + : Math.Min(delayMilliseconds * 2, 16); + Thread.Sleep(delayMilliseconds); } } - private static void ApplySubmittedReleaseMem( + /// + /// Writes a value that satisfies the waiter's comparison. This deliberately + /// exists only behind SHARPEMU_GPU_WAIT_MODE=force for legacy A/B testing; + /// normal and stale waits must never mutate their watched label. + /// + private static void ForceSatisfyGpuWait( CpuContext ctx, + in GpuWaitRegistry.WaitingDcb waiter, + ulong value) + { + var address = waiter.WaitAddress; + var mask = waiter.Mask; + if (address == 0 || mask == 0) + { + return; + } + + var maskedRef = waiter.ReferenceValue & mask; + ulong? satisfyMasked = waiter.CompareFunction switch + { + 1 => maskedRef == 0 ? null : (maskedRef - 1) & mask, // < + 2 => maskedRef, // <= + 3 => maskedRef, // == + 4 => (~maskedRef) & mask, // != + 5 => maskedRef, // >= + 6 => maskedRef == mask ? null : (maskedRef + 1) & mask, // > + _ => null, + }; + + if (satisfyMasked is not { } satisfy) + { + return; + } + + var newValue = (value & ~mask) | (satisfy & mask); + if (waiter.Is64Bit) + { + ctx.TryWriteUInt64(address, newValue); + } + else + { + TryWriteUInt32(ctx, address, unchecked((uint)newValue)); + } + } + + // WAIT_REG_MEM packets whose condition is not met suspend their DCB into + // GpuWaitRegistry. Each submit re-checks every suspended DCB against current + // guest memory (labels are advanced by ReleaseMem/WriteData/DmaData packets + // or direct CPU writes) and resumes the ones now satisfied. A resumed DCB + // can itself write labels that unblock others, so loop to a fixed point. + private static void DrainResumableDcbs( + CpuContext ctx, + SubmittedGpuState gpuState, + bool tracePackets) + { + if (!_gpuWaitSuspendEnabled) + { + return; + } + + for (var pass = 0; pass < 256; pass++) + { + var woken = GpuWaitRegistry.CollectSatisfied(ctx.Memory, (address, is64Bit) => + is64Bit + ? TryReadUInt64(ctx, address, out var value64) ? value64 : (ulong?)null + : TryReadUInt32(ctx, address, out var value32) ? value32 : (ulong?)null); + + if (woken is null) + { + if (_gpuWaitStaleTicks > 0 && + GpuWaitRegistry.CollectUnreportedStale( + ctx.Memory, + System.Diagnostics.Stopwatch.GetTimestamp(), + _gpuWaitStaleTicks) is { } stale) + { + foreach (var waiter in stale) + { + ulong? currentValue = waiter.Is64Bit + ? TryReadUInt64(ctx, waiter.WaitAddress, out var value64) + ? value64 + : null + : TryReadUInt32(ctx, waiter.WaitAddress, out var value32) + ? value32 + : null; + TraceWaitProducerState( + ctx.Memory, + waiter, + waiter.CommandBufferAddress, + waiter.ResumeAddress, + stale: true, + currentValue); + } + } + + return; + } + + foreach (var waiter in woken) + { + ResumeSuspendedDcb(ctx, gpuState, waiter, tracePackets); + } + } + } + + private static void ResumeSuspendedDcb( + CpuContext ctx, + SubmittedGpuState gpuState, + in GpuWaitRegistry.WaitingDcb waiter, + bool tracePackets) + { + var state = waiter.State as SubmittedDcbState ?? gpuState.Graphics; + var remainingDwords = waiter.TotalDwords - waiter.ResumeOffset; + var waitedMilliseconds = waiter.RegisteredTicks == 0 + ? 0.0 + : (System.Diagnostics.Stopwatch.GetTimestamp() - waiter.RegisteredTicks) * + 1000.0 / System.Diagnostics.Stopwatch.Frequency; + TraceAgcShader( + $"agc.queue_resumed queue={waiter.QueueName} " + + $"submission={waiter.SubmissionId} label=0x{waiter.WaitAddress:X16} " + + $"resume=0x{waiter.ResumeAddress:X16} remaining_dwords={remainingDwords} " + + $"waited_ms={waitedMilliseconds:F3}"); + if (remainingDwords == 0) + { + state.IsSuspended = false; + state.HasActiveSubmission = false; + NotifySubmittedDcbCompleted(gpuState, state, waiter.SubmissionId); + PumpSubmittedQueue(ctx, gpuState, state); + return; + } + + if (tracePackets) + { + TraceAgc( + $"agc.dcb.resumed addr=0x{waiter.WaitAddress:X16} " + + $"resume=0x{waiter.ResumeAddress:X16} dwords={remainingDwords} forced=False"); + } + + System.Diagnostics.Debug.Assert(state.HasActiveSubmission); + System.Diagnostics.Debug.Assert(state.IsSuspended); + state.QueueName = waiter.QueueName ?? state.QueueName; + state.ActiveSubmissionId = waiter.SubmissionId; + state.IsSuspended = false; + if (ParseSubmittedDcb( + ctx, + gpuState, + state, + waiter.ResumeAddress, + remainingDwords, + tracePackets)) + { + state.IsSuspended = true; + return; + } + + state.HasActiveSubmission = false; + NotifySubmittedDcbCompleted(gpuState, state, waiter.SubmissionId); + PumpSubmittedQueue(ctx, gpuState, state); + } + + private static void TraceSubmittedWait( + ulong address, + ulong value, + ulong mask, + ulong reference, + uint compareFunction, + int bits, + bool tracePacket) + { + var maskedValue = value & mask; + var satisfied = compareFunction switch + { + 0 => true, + 1 => maskedValue < reference, + 2 => maskedValue <= reference, + 3 => maskedValue == reference, + 4 => maskedValue != reference, + 5 => maskedValue >= reference, + 6 => maskedValue > reference, + _ => true, + }; + if (!tracePacket && (satisfied || !ShouldTraceHotPath(ref _unsatisfiedWaitTraceCount))) + { + return; + } + + TraceAgc( + $"agc.dcb.wait_reg_mem bits={bits} addr=0x{address:X16} " + + $"value=0x{value:X16} mask=0x{mask:X16} ref=0x{reference:X16} " + + $"compare={compareFunction} satisfied={satisfied}"); + } + + private static void ApplySubmittedStandardReleaseMem( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, ulong packetAddress, bool tracePacket) { - if (!ctx.TryReadUInt32(packetAddress + 8, out var control) || - !ctx.TryReadUInt32(packetAddress + 12, out var destinationLo) || - !ctx.TryReadUInt32(packetAddress + 16, out var destinationHi) || - !ctx.TryReadUInt32(packetAddress + 20, out var dataLo) || - !ctx.TryReadUInt32(packetAddress + 24, out var dataHi)) + if (!TryReadUInt32(ctx, packetAddress + 8, out var control) || + !TryReadUInt32(ctx, packetAddress + 12, out var destinationLo) || + !TryReadUInt32(ctx, packetAddress + 16, out var destinationHi) || + !TryReadUInt32(ctx, packetAddress + 20, out var dataLo) || + !TryReadUInt32(ctx, packetAddress + 24, out var dataHi)) + { + return; + } + + var (destination, dataSelection) = DecodeStandardReleaseMemControl(control); + var destinationAddress = ((ulong)destinationHi << 32) | destinationLo; + var data = ((ulong)dataHi << 32) | dataLo; + var writeLength = dataSelection switch + { + 1 => (ulong)sizeof(uint), + 2 or 3 or 4 => (ulong)sizeof(ulong), + _ => 0UL, + }; + var writesGuestMemory = destination is 0 or 1 && + destinationAddress != 0 && + writeLength != 0; + + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + if (writesGuestMemory) + { + InvalidateDcbWindowIfOverlaps(destinationAddress, writeLength); + } + + var wroteData = writesGuestMemory && (dataSelection switch + { + 1 => TryWriteUInt32(ctx, destinationAddress, dataLo), + 2 => ctx.TryWriteUInt64(destinationAddress, data), + // Hardware counter writes are timing values sampled at the + // release point, not the immediate payload in ordinal 6/7. + 3 or 4 => ctx.TryWriteUInt64( + destinationAddress, + unchecked((ulong)System.Diagnostics.Stopwatch.GetTimestamp())), + _ => false, + }); + + if (tracePacket) + { + TraceAgc( + $"agc.dcb.release_mem_standard dst_sel={destination} " + + $"dst=0x{destinationAddress:X16} data_sel={dataSelection} " + + $"data=0x{data:X16} wrote={wroteData}"); + } + }, + $"release_mem_standard dst=0x{destinationAddress:X16} data=0x{data:X16}", + packetAddress, + writesGuestMemory ? destinationAddress : 0, + writesGuestMemory ? writeLength : 0); + } + + private static (uint Destination, uint DataSelection) + DecodeStandardReleaseMemControl(uint control) => + ( + Destination: (control >> 16) & 0x3u, + DataSelection: (control >> 29) & 0x7u); + + private static void ApplySubmittedReleaseMem( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, + ulong packetAddress, + bool tracePacket) + { + if (!TryReadUInt32(ctx, packetAddress + 8, out var control) || + !TryReadUInt32(ctx, packetAddress + 12, out var destinationLo) || + !TryReadUInt32(ctx, packetAddress + 16, out var destinationHi) || + !TryReadUInt32(ctx, packetAddress + 20, out var dataLo) || + !TryReadUInt32(ctx, packetAddress + 24, out var dataHi)) { return; } @@ -3232,20 +5057,43 @@ public static class AgcExports var dataSelection = (control >> 16) & 0xFFu; var destinationAddress = ((ulong)destinationHi << 32) | destinationLo; var data = ((ulong)dataHi << 32) | dataLo; - - var wroteData = dataSelection switch + var writeLength = dataSelection switch { - 1 => ctx.TryWriteUInt32(destinationAddress, dataLo), - 2 or 3 => ctx.TryWriteUInt64(destinationAddress, data), - _ => false, + 1 => (ulong)sizeof(uint), + 2 or 3 => (ulong)sizeof(ulong), + _ => 0UL, }; + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + InvalidateDcbWindowIfOverlaps(destinationAddress, writeLength); + var wroteData = dataSelection switch + { + 1 => TryWriteUInt32(ctx, destinationAddress, dataLo), + 2 => ctx.TryWriteUInt64(destinationAddress, data), + // Data selection 3 samples the GPU clock at the release + // point. The packet payload is ignored by hardware; Unity + // uses the nonzero timestamp as submit-completion state. + 3 => ctx.TryWriteUInt64( + destinationAddress, + unchecked((ulong)System.Diagnostics.Stopwatch.GetTimestamp())), + _ => false, + }; - if (tracePacket) - { - TraceAgc( - $"agc.dcb.release_mem dst=0x{destinationAddress:X16} data_sel={dataSelection} " + - $"data=0x{data:X16} wrote={wroteData}"); - } + if (tracePacket) + { + TraceAgc( + $"agc.dcb.release_mem dst=0x{destinationAddress:X16} " + + $"data_sel={dataSelection} data=0x{data:X16} wrote={wroteData}"); + } + }, + $"release_mem dst=0x{destinationAddress:X16} data=0x{data:X16}", + packetAddress, + dataSelection is 1 or 2 or 3 ? destinationAddress : 0, + writeLength); } private static void ApplySubmittedRegisters( @@ -3259,7 +5107,7 @@ public static class AgcExports if (op is ItSetShReg or ItSetContextReg or ItSetUconfigReg) { if (packetLength < 3 || - !ctx.TryReadUInt32(packetAddress + sizeof(uint), out var startRegister)) + !TryReadUInt32(ctx, packetAddress + sizeof(uint), out var startRegister)) { return; } @@ -3272,7 +5120,8 @@ public static class AgcExports }; for (uint index = 0; index < packetLength - 2; index++) { - if (!ctx.TryReadUInt32( + if (!TryReadUInt32( + ctx, packetAddress + 8 + ((ulong)index * sizeof(uint)), out var value)) { @@ -3288,8 +5137,8 @@ public static class AgcExports if (op != ItNop || register is not (RCxRegsIndirect or RShRegsIndirect or RUcRegsIndirect) || packetLength < 4 || - !ctx.TryReadUInt32(packetAddress + sizeof(uint), out var registerCount) || - !ctx.TryReadUInt64(packetAddress + 8, out var registersAddress)) + !TryReadUInt32(ctx, packetAddress + sizeof(uint), out var registerCount) || + !TryReadUInt64(ctx, packetAddress + 8, out var registersAddress)) { return; } @@ -3303,16 +5152,17 @@ public static class AgcExports for (uint index = 0; index < registerCount; index++) { var entryAddress = registersAddress + ((ulong)index * 8); - if (!ctx.TryReadUInt32(entryAddress, out var registerOffset) || - !ctx.TryReadUInt32(entryAddress + sizeof(uint), out var value)) + if (!TryReadUInt32(ctx, entryAddress, out var registerOffset) || + !TryReadUInt32(ctx, entryAddress + sizeof(uint), out var value)) { return; } - if (registerOffset != 0) - { - destination[registerOffset] = value; - } + // The indirect table has an explicit count; offset zero is a real + // context-register index (DB_RENDER_CONTROL), not a terminator. + // Dropping it leaves stale depth/render-control state active in + // later passes. + destination[registerOffset] = value; } } @@ -3328,25 +5178,20 @@ public static class AgcExports switch (op) { case ItDrawIndexAuto when packetLength >= 3: - return ctx.TryReadUInt32(packetAddress + 4, out drawCount); + return TryReadUInt32(ctx, packetAddress + 4, out drawCount); case ItDrawIndex2 when packetLength >= 6: state.DrawIndexOffset = 0; - return ctx.TryReadUInt32(packetAddress + 16, out drawCount); - // 5-dword form emitted by DcbDrawIndex (count at +4, ItIndexBase/ - // ItIndexBufferSize carried by separate preceding packets). - case ItDrawIndex2 when packetLength >= 5: - state.DrawIndexOffset = 0; - return ctx.TryReadUInt32(packetAddress + 4, out drawCount); + return TryReadUInt32(ctx, packetAddress + 16, out drawCount); case ItDrawIndexOffset2 when packetLength >= 5: - if (!ctx.TryReadUInt32(packetAddress + 8, out var indexOffset)) + if (!TryReadUInt32(ctx, packetAddress + 8, out var indexOffset)) { return false; } state.DrawIndexOffset = indexOffset; - return ctx.TryReadUInt32(packetAddress + 12, out drawCount); + return TryReadUInt32(ctx, packetAddress + 12, out drawCount); case ItDrawIndexMultiAuto when packetLength >= 4: - if (!ctx.TryReadUInt32(packetAddress + 12, out var control)) + if (!TryReadUInt32(ctx, packetAddress + 12, out var control)) { return false; } @@ -3355,12 +5200,13 @@ public static class AgcExports return true; case ItDrawIndirect or ItDrawIndexIndirect when packetLength >= 5 && state.IndirectArgsAddress != 0: - if (!ctx.TryReadUInt32(packetAddress + 4, out var dataOffset)) + if (!TryReadUInt32(ctx, packetAddress + 4, out var dataOffset)) { return false; } - return ctx.TryReadUInt32( + return TryReadUInt32( + ctx, state.IndirectArgsAddress + dataOffset, out drawCount); default: @@ -3390,24 +5236,48 @@ public static class AgcExports state.UcRegisters.TryGetValue(VgtPrimitiveType, out var primitiveType); var renderTargets = GetRenderTargets(state.CxRegisters); var drawSequence = ++gpuState.WorkSequence; + if (state.PendingTargetlessDraw is { } stalePendingDraw) + { + ReturnPooledDrawArrays( + stalePendingDraw, + globals: true, + vertex: true, + index: true); + state.PendingTargetlessDraw = null; + } state.TranslatedDraw = null; state.GuestDrawKind = GuestDrawKind.None; foreach (var target in renderTargets) { - state.RenderTargetWriters[target.Address] = new RenderTargetWriter( - drawSequence, - hasExportShader ? exportShaderAddress : 0, - hasPixelShader ? pixelShaderAddress : 0, - vertexCount, - primitiveType); + state.KnownRenderTargets[target.Address] = target; + // Colour exports originate in the pixel stage. A depth-only draw + // can leave old CB registers bound, but it must not become the + // advertised writer of those surfaces merely because they remain + // in state. + if (hasPixelShader) + { + state.RenderTargetWriters[target.Address] = new RenderTargetWriter( + drawSequence, + hasExportShader ? exportShaderAddress : 0, + pixelShaderAddress, + vertexCount, + primitiveType); + } - TraceAgcShader( - $"agc.rt_writer seq={drawSequence} target=0x{target.Address:X16} " + - $"fmt={target.Format} tile={target.TileMode} " + - $"size={target.Width}x{target.Height} vertices={vertexCount} " + - $"prim=0x{primitiveType:X} indexed={indexed} " + - $"es=0x{(hasExportShader ? exportShaderAddress : 0):X16} " + - $"ps=0x{(hasPixelShader ? pixelShaderAddress : 0):X16}"); + if (_traceAgcShader || + _tracePixelShaderAddress == pixelShaderAddress || + _traceRenderTargetAddress == target.Address) + { + Console.Error.WriteLine( + "[LOADER][TRACE] " + + $"agc.rt_writer seq={drawSequence} target=0x{target.Address:X16} " + + $"fmt={target.Format} tile={target.TileMode} " + + $"size={target.Width}x{target.Height} vertices={vertexCount} " + + $"prim=0x{primitiveType:X} indexed={indexed} " + + $"es=0x{(hasExportShader ? exportShaderAddress : 0):X16} " + + $"ps=0x{(hasPixelShader ? pixelShaderAddress : 0):X16} " + + $"color_write={(hasPixelShader ? 1 : 0)}"); + } } if (vertexCount == 0 || vertexCount > 1_048_576) @@ -3416,6 +5286,78 @@ public static class AgcExports } var translationError = string.Empty; + var depthState = DecodeDepthState(state.CxRegisters); + var depthTarget = DecodeDepthTarget(state.CxRegisters); + var hasDepthOnlyCandidate = hasExportShader && + !hasPixelShader && + depthTarget is not null && + (depthState.TestEnable || depthState.WriteEnable); + if (hasDepthOnlyCandidate && + TryCreateTranslatedDepthOnlyGuestDraw( + ctx, + state, + exportShaderAddress, + vertexCount, + indexed, + depthTarget!, + out var depthOnlyDraw, + out translationError)) + { + state.TranslatedDraw = depthOnlyDraw; + var activeDepthTarget = depthOnlyDraw.DepthTarget!; + var textures = CreateGuestDrawTextures( + ctx, + depthOnlyDraw.Textures, + out _); + var globalMemoryBuffers = + CreateTranslatedDrawGlobalBuffers(depthOnlyDraw); + var vertexBuffers = + CreateGuestVertexBuffers(depthOnlyDraw.VertexInputs); + var renderState = depthOnlyDraw.RenderState; + if (activeDepthTarget.ReadOnly && renderState.Depth.WriteEnable) + { + renderState = renderState with + { + Depth = renderState.Depth with { WriteEnable = false }, + }; + } + + TraceDrawCompact( + drawSequence, + depthOnlyDraw, + textures, + vertexBuffers); + GuestGpu.Current.SubmitDepthOnlyTranslatedDraw( + depthOnlyDraw.PixelShader, + textures, + globalMemoryBuffers, + depthOnlyDraw.AttributeCount, + activeDepthTarget, + depthOnlyDraw.VertexShader, + depthOnlyDraw.VertexCount, + depthOnlyDraw.InstanceCount, + depthOnlyDraw.PrimitiveType, + depthOnlyDraw.IndexBuffer, + vertexBuffers, + renderState, + depthOnlyDraw.PixelShaderAddress); + + if (_traceAgcShader) + { + TraceAgcShader( + $"agc.depth_only_draw seq={drawSequence} " + + $"es=0x{exportShaderAddress:X16} " + + $"depth=0x{activeDepthTarget.Address:X16}:" + + $"{activeDepthTarget.Width}x{activeDepthTarget.Height}:" + + $"fmt{activeDepthTarget.GuestFormat}/sw{activeDepthTarget.SwizzleMode} " + + $"test={(renderState.Depth.TestEnable ? 1 : 0)} " + + $"write={(renderState.Depth.WriteEnable ? 1 : 0)} " + + $"func={renderState.Depth.CompareOp} ro={(activeDepthTarget.ReadOnly ? 1 : 0)}"); + } + + return; + } + if (hasExportShader && hasPixelShader && hasPsInputEna && @@ -3425,58 +5367,219 @@ public static class AgcExports state, exportShaderAddress, pixelShaderAddress, + psInputEna, + psInputAddr, vertexCount, indexed, out var translatedDraw, out translationError)) { state.TranslatedDraw = translatedDraw; + if (TryGetHardwareColorResolveTargets( + state.CxRegisters, + out var resolveSource, + out var resolveDestination)) + { + state.KnownRenderTargets[resolveSource.Address] = resolveSource; + state.KnownRenderTargets[resolveDestination.Address] = resolveDestination; + ProvideRenderTargetInitialData(ctx, resolveSource); + if (VulkanVideoPresenter.TrySubmitGuestImageBlit( + resolveSource.Address, + resolveSource.Width, + resolveSource.Height, + resolveSource.Format, + resolveSource.NumberType, + resolveDestination.Address, + resolveDestination.Width, + resolveDestination.Height, + resolveDestination.Format, + resolveDestination.NumberType)) + { + state.RenderTargetWriters[resolveDestination.Address] = + new RenderTargetWriter( + drawSequence, + exportShaderAddress, + pixelShaderAddress, + vertexCount, + primitiveType); + TraceAgcShader( + $"agc.hardware_color_resolve seq={drawSequence} " + + $"src=0x{resolveSource.Address:X16}:" + + $"{resolveSource.Width}x{resolveSource.Height}:" + + $"fmt{resolveSource.Format}/num{resolveSource.NumberType} " + + $"dst=0x{resolveDestination.Address:X16}:" + + $"{resolveDestination.Width}x{resolveDestination.Height}:" + + $"fmt{resolveDestination.Format}/num{resolveDestination.NumberType}"); + ReturnPooledDrawArrays( + translatedDraw, + globals: true, + vertex: true, + index: true); + state.TranslatedDraw = null; + return; + } + + TraceAgcShader( + $"agc.hardware_color_resolve_unavailable seq={drawSequence} " + + $"src=0x{resolveSource.Address:X16} " + + $"dst=0x{resolveDestination.Address:X16}"); + } + var firstTarget = translatedDraw.RenderTargets.FirstOrDefault(); if (firstTarget.Address != 0) { - var textures = CreateGuestDrawTextures( + // Render every bound color target. A deferred G-buffer draw + // writes several targets in one guest pass; we render one bound + // target per Vulkan pass, each with the pixel variant that + // routes that target's MRT export slot to the fragment output. + // Every pass is enqueued in order on the same guest render + // queue. Share the immutable snapshots between those passes + // and let only the final pass return pooled arrays after its + // host upload. Copying the full vertex/global payload for each + // secondary target made deferred G-buffer draws allocate + // hundreds of MiB per second on the managed large-object heap. + var drawRenderTargets = translatedDraw.RenderTargets; + var lastTargetIndex = 0; + for (var targetIndex = 1; targetIndex < drawRenderTargets.Count; targetIndex++) + { + if (drawRenderTargets[targetIndex].Address != 0) + { + lastTargetIndex = targetIndex; + } + } + + var sharedTextures = CreateGuestDrawTextures( ctx, translatedDraw.Textures, out _); - var globalMemoryBuffers = - CreateGuestMemoryBuffers(translatedDraw.GlobalMemoryBindings); - var vertexBuffers = + var sharedGlobalMemoryBuffers = + CreateTranslatedDrawGlobalBuffers(translatedDraw); + var sharedVertexBuffers = CreateGuestVertexBuffers(translatedDraw.VertexInputs); - TraceRectListVertices(translatedDraw, vertexBuffers); - TraceGrassDrawVertices(translatedDraw, textures, vertexBuffers); + TraceRectListVertices(translatedDraw, sharedVertexBuffers); + TraceGrassDrawVertices(translatedDraw, sharedTextures, sharedVertexBuffers); + TraceDrawCompact( + drawSequence, + translatedDraw, + sharedTextures, + sharedVertexBuffers); + foreach (var renderTarget in drawRenderTargets) + { + if (renderTarget.Address != 0) + { + ProvideRenderTargetInitialData(ctx, renderTarget); + } + } + GuestGpu.Current.SubmitOffscreenTranslatedDraw( translatedDraw.PixelShader, - textures, - globalMemoryBuffers, + sharedTextures, + sharedGlobalMemoryBuffers, translatedDraw.AttributeCount, translatedDraw.GuestTargets, - translatedDraw.VertexShader, - translatedDraw.VertexCount, - translatedDraw.InstanceCount, - translatedDraw.PrimitiveType, - translatedDraw.IndexBuffer, - vertexBuffers, - translatedDraw.RenderState); + translatedDraw.VertexShader, + translatedDraw.VertexCount, + translatedDraw.InstanceCount, + translatedDraw.PrimitiveType, + translatedDraw.IndexBuffer, + sharedVertexBuffers, + translatedDraw.RenderState, + translatedDraw.DepthTarget, + translatedDraw.PixelShaderAddress); } else { - var storageTarget = translatedDraw.Textures - .FirstOrDefault(binding => binding.IsStorage); - if (storageTarget is not null) + if (translatedDraw.DepthTarget is { } translatedDepthTarget) { var textures = CreateGuestDrawTextures( ctx, translatedDraw.Textures, out _); var globalMemoryBuffers = - CreateGuestMemoryBuffers(translatedDraw.GlobalMemoryBindings); - GuestGpu.Current.SubmitStorageTranslatedDraw( + CreateTranslatedDrawGlobalBuffers(translatedDraw); + var vertexBuffers = + CreateGuestVertexBuffers(translatedDraw.VertexInputs); + var renderState = translatedDraw.RenderState; + if (translatedDepthTarget.ReadOnly && renderState.Depth.WriteEnable) + { + renderState = renderState with + { + Depth = renderState.Depth with { WriteEnable = false }, + }; + } + + TraceDrawCompact( + drawSequence, + translatedDraw, + textures, + vertexBuffers); + GuestGpu.Current.SubmitDepthOnlyTranslatedDraw( translatedDraw.PixelShader, textures, globalMemoryBuffers, translatedDraw.AttributeCount, - storageTarget.Descriptor.Width, - storageTarget.Descriptor.Height); + translatedDepthTarget, + translatedDraw.VertexShader, + translatedDraw.VertexCount, + translatedDraw.InstanceCount, + translatedDraw.PrimitiveType, + translatedDraw.IndexBuffer, + vertexBuffers, + renderState, + translatedDraw.PixelShaderAddress); + } + else + { + var storageTarget = translatedDraw.Textures + .FirstOrDefault(binding => binding.IsStorage); + if (storageTarget is not null) + { + var textures = CreateGuestDrawTextures( + ctx, + translatedDraw.Textures, + out _); + var globalMemoryBuffers = + CreateTranslatedDrawGlobalBuffers(translatedDraw); + TraceDrawCompact(drawSequence, translatedDraw, textures, []); + GuestGpu.Current.SubmitStorageTranslatedDraw( + translatedDraw.PixelShader, + textures, + globalMemoryBuffers, + translatedDraw.AttributeCount, + storageTarget.Descriptor.Width, + storageTarget.Descriptor.Height, + translatedDraw.PixelShaderAddress); + // The storage submit consumes the global buffers (the + // presenter returns them) but never the vertex/index + // arrays; return those here so they don't leak the pool. + ReturnPooledDrawArrays( + translatedDraw, + globals: false, + vertex: true, + index: true); + } + else + { + if (translatedDraw.Textures.Count != 0) + { + // Unity's PS5 final blit can omit CB registers and + // rely on the following AGC flip to name the scanout + // target. Retain that sampled draw until RFlip, then + // enqueue it against the known display surface before + // the ordered capture. + state.PendingTargetlessDraw = translatedDraw; + } + else + { + // No render target, storage sink or sampled source: + // nothing can consume this draw. + ReturnPooledDrawArrays( + translatedDraw, + globals: true, + vertex: true, + index: true); + } + } } } @@ -3489,25 +5592,40 @@ public static class AgcExports $"textures={translatedDraw.Textures.Count}"); } - lock (_submitTraceGate) + // Trace-only: gated on the flag so the dedup set and the dump — + // which reads pooled buffer data the presenter may already have + // recycled (harmless for diagnostics, garbage bytes at worst) — + // cost nothing in normal runs. + if (_traceAgcShader) { - var firstTextureAddress = translatedDraw.Textures.FirstOrDefault()?.Descriptor.Address ?? 0; - if (_tracedShaderDraws.Add( - (exportShaderAddress, pixelShaderAddress, firstTarget.Address, firstTextureAddress, vertexCount))) + lock (_submitTraceGate) { - TraceTranslatedGuestDraw( - ctx, - gpuState, - state, - translatedDraw, - psInputEna, - psInputAddr); + var firstTextureAddress = translatedDraw.Textures.FirstOrDefault()?.Descriptor.Address ?? 0; + if (_tracedShaderDraws.Add( + (exportShaderAddress, pixelShaderAddress, firstTarget.Address, firstTextureAddress, vertexCount))) + { + TraceTranslatedGuestDraw( + ctx, + gpuState, + state, + translatedDraw, + psInputEna, + psInputAddr); + } } } return; } + TraceDrawCompactMiss( + drawSequence, + vertexCount, + hasExportShader && hasPixelShader + ? translationError + : hasDepthOnlyCandidate && !string.IsNullOrEmpty(translationError) + ? $"depth-only: {translationError}" + : $"missing-shaders es={hasExportShader} ps={hasPixelShader} ena={hasPsInputEna} addr={hasPsInputAddr}"); TraceShaderTranslationMiss( ctx, state, @@ -3520,27 +5638,27 @@ public static class AgcExports psInputEna, hasPsInputAddr, psInputAddr, - hasExportShader && hasPixelShader ? translationError : null); + hasExportShader && hasPixelShader || hasDepthOnlyCandidate + ? translationError + : null); } - private static bool TryCreateTranslatedGuestDraw( + private static bool TryCreateTranslatedDepthOnlyGuestDraw( CpuContext ctx, SubmittedDcbState state, ulong exportShaderAddress, - ulong pixelShaderAddress, uint vertexCount, bool indexed, + GuestDepthTarget depthTarget, out TranslatedGuestDraw draw, out string error) { draw = default!; error = string.Empty; ulong exportShaderHeader; - ulong pixelShaderHeader; lock (_submitTraceGate) { _shaderHeadersByCode.TryGetValue(exportShaderAddress, out exportShaderHeader); - _shaderHeadersByCode.TryGetValue(pixelShaderAddress, out pixelShaderHeader); } if (!Gen5ShaderTranslator.TryCreateState( @@ -3558,28 +5676,285 @@ public static class AgcExports out var exportEvaluation, out error, resolveVertexInputs: true, - vertexRecordLimit: indexed ? null : vertexCount) || - !Gen5ShaderTranslator.TryCreateState( + requiredVertexRecordCount: TryGetRequiredVertexRecordCount( + ctx, + state, + vertexCount, + indexed, + out var depthVertexRecords) + ? depthVertexRecords + : null)) + { + return false; + } + + var exportFingerprint = _bakeScalars + ? ComputeShaderStateFingerprint(exportEvaluation) + : ComputeShaderStructuralFingerprint(exportEvaluation); + var cacheKey = ( + exportShaderAddress, + exportFingerprint, + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment); + _depthOnlyVertexShaderCache.TryGetValue(cacheKey, out var vertexShader); + + if (vertexShader is null) + { + var guestGlobalBufferCount = exportEvaluation.GlobalMemoryBindings.Count; + // CreateTranslatedDrawGlobalBuffers appends both stage scalar + // blocks. The pixel block is unused by the fixed fragment stage; + // the vertex block remains at guestCount+1, matching this layout. + var totalGlobalBufferCount = _bakeScalars + ? guestGlobalBufferCount + : guestGlobalBufferCount + 2; + if (!GuestGpu.Current.TryCompileVertexShader( + exportState, + exportEvaluation, + out vertexShader, + out error, + globalBufferBase: 0, + totalGlobalBufferCount: totalGlobalBufferCount, + imageBindingBase: 0, + scalarRegisterBufferIndex: _bakeScalars + ? -1 + : guestGlobalBufferCount + 1, + requiredVertexOutputCount: 0, + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment)) + { + ReturnPooledEvaluationArrays(exportEvaluation); + return false; + } + + DumpCompiledShader( + "depth-vs", + exportShaderAddress, + exportFingerprint, + vertexShader!, + exportState.Program); + VulkanVideoPresenter.CountSpirvCompilation(); + _depthOnlyVertexShaderCache.TryAdd(cacheKey, vertexShader!); + } + + var textures = new List( + exportEvaluation.ImageBindings.Count); + foreach (var binding in exportEvaluation.ImageBindings) + { + if (!TryDecodeTextureDescriptor(binding.ResourceDescriptor, out var texture)) + { + if (_strictShaderDescriptors) + { + error = $"invalid export texture descriptor at pc=0x{binding.Pc:X}"; + ReturnPooledEvaluationArrays(exportEvaluation); + return false; + } + + texture = new TextureDescriptor( + 0, + 1, + 1, + Gen5TextureFormatR8G8B8A8Unorm, + 0, + 0, + 0, + 0, + 0, + 1, + 0xFAC); + } + + textures.Add(new TranslatedImageBinding( + texture, + Gen5ShaderTranslator.IsStorageImageOperation(binding.Opcode), + binding.MipLevel ?? 0, + binding.SamplerDescriptor)); + } + + IReadOnlyList vertexInputs = + exportEvaluation.VertexInputs ?? []; + state.UcRegisters.TryGetValue(VgtPrimitiveType, out var primitiveType); + var syntheticTarget = new RenderTargetDescriptor( + Slot: 0, + Address: 0, + depthTarget.Width, + depthTarget.Height, + Format: 0, + NumberType: 0, + TileMode: 0); + var renderState = CreateRenderState(state.CxRegisters, syntheticTarget) with + { + // A guest pass without a pixel shader has no colour exports. The + // presenter uses a private compatibility attachment, so disable + // all writes to it and expose only the persistent DB result. + Blends = [GuestBlendState.Default with { WriteMask = 0 }], + }; + if (depthTarget.Width == 1 && + depthTarget.Height == 1 && + renderState.Viewport is { } depthViewport) + { + var inferredWidth = (uint)Math.Clamp( + MathF.Ceiling(MathF.Abs(depthViewport.Width)), + 1f, + 16384f); + var inferredHeight = (uint)Math.Clamp( + MathF.Ceiling(MathF.Abs(depthViewport.Height)), + 1f, + 16384f); + if (inferredWidth > 1 || inferredHeight > 1) + { + depthTarget = depthTarget with + { + Width = inferredWidth, + Height = inferredHeight, + }; + syntheticTarget = syntheticTarget with + { + Width = inferredWidth, + Height = inferredHeight, + }; + renderState = CreateRenderState(state.CxRegisters, syntheticTarget) with + { + Blends = [GuestBlendState.Default with { WriteMask = 0 }], + }; + } + } + draw = new TranslatedGuestDraw( + exportShaderAddress, + PixelShaderAddress: 0, + primitiveType, + vertexShader!, + GuestGpu.Current.GetDepthOnlyFragmentShader(), + AttributeCount: 0, + vertexCount, + state.InstanceCount, + indexed ? CreateGuestIndexBuffer(ctx, state, vertexCount) : null, + textures, + exportEvaluation.GlobalMemoryBindings, + vertexInputs, + RenderTargets: [], + depthTarget, + GuestTargets: [], + renderState, + PixelUserData: [], + RawBlendControl: 0, + RawColorInfo: 0, + PixelInitialScalars: [], + exportEvaluation.InitialScalarRegisters); + return true; + } + + private static bool TryCreateTranslatedGuestDraw( + CpuContext ctx, + SubmittedDcbState state, + ulong exportShaderAddress, + ulong pixelShaderAddress, + uint psInputEna, + uint psInputAddr, + uint vertexCount, + bool indexed, + out TranslatedGuestDraw draw, + out string error) + { + draw = default!; + error = string.Empty; + ulong exportShaderHeader; + ulong pixelShaderHeader; + lock (_submitTraceGate) + { + _shaderHeadersByCode.TryGetValue(exportShaderAddress, out exportShaderHeader); + _shaderHeadersByCode.TryGetValue(pixelShaderAddress, out pixelShaderHeader); + } + + // Sequential (not short-circuited into one condition) so a failure + // after an evaluation succeeded can return that evaluation's pooled + // buffer arrays to the pool instead of leaking them. + if (!Gen5ShaderTranslator.TryCreateState( + ctx, + exportShaderAddress, + exportShaderHeader, + state.ShRegisters, + SelectExportUserDataRegister(state.ShRegisters), + out var exportState, + out error, + userDataScalarRegisterBase: NggUserDataScalarRegisterBase)) + { + return false; + } + + if (!Gen5ShaderScalarEvaluator.TryEvaluate( + ctx, + exportState, + out var exportEvaluation, + out error, + resolveVertexInputs: true, + requiredVertexRecordCount: TryGetRequiredVertexRecordCount( + ctx, + state, + vertexCount, + indexed, + out var vertexRecords) + ? vertexRecords + : null)) + { + return false; + } + + if (!Gen5ShaderTranslator.TryCreateState( ctx, pixelShaderAddress, pixelShaderHeader, state.ShRegisters, PsTextureUserDataRegister, out var pixelState, - out error) || - !Gen5ShaderScalarEvaluator.TryEvaluate( + out error)) + { + ReturnPooledEvaluationArrays(exportEvaluation); + return false; + } + + if (!Gen5ShaderScalarEvaluator.TryEvaluate( ctx, pixelState, out var pixelEvaluation, out error)) { + ReturnPooledEvaluationArrays(exportEvaluation); return false; } - var renderTargets = GetRenderTargets(state.CxRegisters) + if (pixelShaderAddress == 0x0000000500781200 && + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TITLE_GLOBALS") == "1") + { + TraceAstroTitlePixelGlobals(pixelEvaluation); + } + + if (pixelShaderAddress == 0x0000000500781200 && + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TITLE_GLOBALS_LIVE") == "1") + { + TraceAstroTitlePixelGlobalProbe(pixelEvaluation); + } + + // Every bound color target the shader exports to. Deferred renderers + // draw a multi-render-target G-buffer (up to eight slots) in one pass. + // Fall back to slot 0 if we cannot match any export to a bound target. + var allBoundTargets = GetRenderTargets(state.CxRegisters); + var renderTargets = allBoundTargets .Where(target => HasPixelColorExport(pixelState, target.Slot)) .OrderBy(target => target.Slot) .ToArray(); + if (renderTargets.Length == 0) + { + renderTargets = allBoundTargets + .Where(target => target.Slot == 0) + .ToArray(); + } + if (_traceAgcShader && allBoundTargets.Count > 1) + { + TraceAgcShader( + $"agc.mrt_filter ps=0x{pixelShaderAddress:X16} " + + $"bound=[{string.Join(",", allBoundTargets.Select(t => $"s{t.Slot}:0x{t.Address:X}:exp{(HasPixelColorExport(pixelState, t.Slot) ? 1 : 0)}"))}] " + + $"kept={renderTargets.Length}"); + } + var renderTargetOutputKinds = new Gen5PixelOutputKind[renderTargets.Length]; for (var index = 0; index < renderTargets.Length; index++) { @@ -3591,6 +5966,8 @@ public static class AgcExports { error = $"unsupported color target format={target.Format} number_type={target.NumberType}"; + ReturnPooledEvaluationArrays(exportEvaluation); + ReturnPooledEvaluationArrays(pixelEvaluation); return false; } } @@ -3607,8 +5984,12 @@ public static class AgcExports } var attributeCount = GetInterpolatedAttributeCount(pixelState); - var exportStateFingerprint = ComputeShaderStructureFingerprint(exportEvaluation); - var pixelStateFingerprint = ComputeShaderStructureFingerprint(pixelEvaluation); + var exportStateFingerprint = _bakeScalars + ? ComputeShaderStateFingerprint(exportEvaluation) + : ComputeShaderStructuralFingerprint(exportEvaluation); + var pixelStateFingerprint = _bakeScalars + ? ComputeShaderStateFingerprint(pixelEvaluation) + : ComputeShaderStructuralFingerprint(pixelEvaluation); var shaderKey = ( exportShaderAddress, exportStateFingerprint, @@ -3616,10 +5997,19 @@ public static class AgcExports pixelStateFingerprint, outputLayout, (uint)renderTargets.Length, - attributeCount); - var totalGlobalBuffers = + attributeCount, + psInputEna, + psInputAddr, + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment); + + var guestGlobalBuffers = pixelEvaluation.GlobalMemoryBindings.Count + exportEvaluation.GlobalMemoryBindings.Count; + // Two per-draw initial-scalar buffers ride after the guest buffers: + // [pixel guest][vertex guest][pixel sgprs][vertex sgprs]. + var totalGlobalBuffers = _bakeScalars + ? guestGlobalBuffers + : guestGlobalBuffers + 2; _graphicsShaderCache.TryGetValue(shaderKey, out var compiled); if (compiled.Vertex is null || compiled.Pixel is null) @@ -3640,19 +6030,28 @@ public static class AgcExports out var pixelShader, out error, globalBufferBase: 0, - totalGlobalBufferCount: totalGlobalBuffers + 2, + totalGlobalBufferCount: totalGlobalBuffers, imageBindingBase: 0, - scalarRegisterBufferIndex: totalGlobalBuffers) || + scalarRegisterBufferIndex: _bakeScalars ? -1 : guestGlobalBuffers, + pixelInputEnable: psInputEna, + pixelInputAddress: psInputAddr, + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment) || !GuestGpu.Current.TryCompileVertexShader( exportState, exportEvaluation, out var vertexShader, out error, globalBufferBase: pixelEvaluation.GlobalMemoryBindings.Count, - totalGlobalBufferCount: totalGlobalBuffers + 2, + totalGlobalBufferCount: totalGlobalBuffers, imageBindingBase: pixelEvaluation.ImageBindings.Count, - scalarRegisterBufferIndex: totalGlobalBuffers + 1)) + scalarRegisterBufferIndex: _bakeScalars ? -1 : guestGlobalBuffers + 1, + requiredVertexOutputCount: (int)GetInterpolatedAttributeCount(pixelState), + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment)) { + ReturnPooledEvaluationArrays(exportEvaluation); + ReturnPooledEvaluationArrays(pixelEvaluation); return false; } @@ -3669,6 +6068,7 @@ public static class AgcExports pixelStateFingerprint, compiled.Pixel, pixelState.Program); + VulkanVideoPresenter.CountSpirvCompilation(); _graphicsShaderCache.TryAdd(shaderKey, compiled); } @@ -3681,15 +6081,34 @@ public static class AgcExports { if (!TryDecodeTextureDescriptor(binding.ResourceDescriptor, out var texture)) { - error = $"invalid texture descriptor at pc=0x{binding.Pc:X}"; - return false; + // A garbage/zeroed texture descriptor (from a per-draw descriptor + // setup race — the same root as scalar-load-failed) would drop + // the whole draw, so Demon's Souls' deferred-lighting/composite + // passes that produce the composite's feeder targets never run + // and the frame stays black. Substitute a 1x1 fallback binding so + // the pass still renders (that one sampled texture reads black) + // rather than dropping it entirely. STRICT reverts. + if (_strictShaderDescriptors) + { + error = $"invalid texture descriptor at pc=0x{binding.Pc:X}"; + ReturnPooledEvaluationArrays(exportEvaluation); + ReturnPooledEvaluationArrays(pixelEvaluation); + return false; + } + + texture = new TextureDescriptor( + 0, 1, 1, Gen5TextureFormatR8G8B8A8Unorm, 0, 0, 0, 0, 0, 1, 0xFAC); } - TraceAgcShader( - $"agc.texture_binding ps=0x{pixelShaderAddress:X16} es=0x{exportShaderAddress:X16} " + - $"pc=0x{binding.Pc:X} op={binding.Opcode} storage={(Gen5ShaderTranslator.IsStorageImageOperation(binding.Opcode) ? 1 : 0)} " + - $"decoded={FormatTextureDescriptor(texture)} " + - $"raw={FormatShaderDwords(binding.ResourceDescriptor)} sampler={FormatShaderDwords(binding.SamplerDescriptor)}"); + if (_traceAgcShader || _tracePixelShaderAddress == pixelShaderAddress) + { + Console.Error.WriteLine( + "[LOADER][TRACE] " + + $"agc.texture_binding ps=0x{pixelShaderAddress:X16} es=0x{exportShaderAddress:X16} " + + $"pc=0x{binding.Pc:X} op={binding.Opcode} storage={(Gen5ShaderTranslator.IsStorageImageOperation(binding.Opcode) ? 1 : 0)} " + + $"decoded={FormatTextureDescriptor(texture)} " + + $"raw={FormatShaderDwords(binding.ResourceDescriptor)} sampler={FormatShaderDwords(binding.SamplerDescriptor)}"); + } textures.Add( new TranslatedImageBinding( texture, @@ -3700,8 +6119,6 @@ public static class AgcExports var globalMemoryBindings = pixelEvaluation.GlobalMemoryBindings .Concat(exportEvaluation.GlobalMemoryBindings) - .Append(CreateScalarRegisterBinding(pixelEvaluation)) - .Append(CreateScalarRegisterBinding(exportEvaluation)) .ToArray(); IReadOnlyList vertexInputs = exportEvaluation.VertexInputs ?? []; @@ -3723,7 +6140,7 @@ public static class AgcExports primitiveType, compiled.Vertex, compiled.Pixel, - attributeCount, + GetInterpolatedAttributeCount(pixelState), vertexCount, state.InstanceCount, indexed ? CreateGuestIndexBuffer(ctx, state, vertexCount) : null, @@ -3731,26 +6148,100 @@ public static class AgcExports globalMemoryBindings, vertexInputs, renderTargets, + DecodeDepthTarget(state.CxRegisters), guestTargets, ApplyTransparentPremultipliedFillClear( CreateRenderState(state.CxRegisters, renderTargets, pixelState), textures, vertexInputs, - pixelEvaluation.InitialScalarRegisters)); + pixelEvaluation.InitialScalarRegisters), + pixelEvaluation.InitialScalarRegisters.Take(8).ToArray(), + state.CxRegisters.TryGetValue(CbBlend0Control, out var rawBlend) ? rawBlend : 0, + state.CxRegisters.TryGetValue( + CbColor0Info + renderTargets.FirstOrDefault().Slot * CbColorRegisterStride, + out var rawInfo) + ? rawInfo + : 0, + pixelEvaluation.InitialScalarRegisters, + exportEvaluation.InitialScalarRegisters); return true; } - private static readonly bool _transparentFillClearEnabled = !string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_TRANSPARENT_FILL_CLEAR"), + private static int _tracedAstroTitlePixelGlobals; + private static int _tracedAstroTitlePixelGlobalProbe; + + private static void TraceAstroTitlePixelGlobalProbe(Gen5ShaderEvaluation evaluation) + { + const int probeOffset = 17216; + var draw = Interlocked.Increment(ref _tracedAstroTitlePixelGlobalProbe); + foreach (var (binding, index) in evaluation.GlobalMemoryBindings.Select((value, index) => (value, index))) + { + if (probeOffset + 16 > binding.DataLength) + { + continue; + } + + Console.Error.WriteLine( + $"[TITLE-GLOBALS-LIVE] draw={draw} binding={index} " + + $"base=0x{binding.BaseAddress:X16} offset=0x{probeOffset:X} " + + $"bytes={Convert.ToHexString(binding.Data.AsSpan(probeOffset, 16))}"); + } + } + + private static void TraceAstroTitlePixelGlobals(Gen5ShaderEvaluation evaluation) + { + if (Interlocked.Exchange(ref _tracedAstroTitlePixelGlobals, 1) != 0) + { + return; + } + + Console.Error.WriteLine( + $"[TITLE-GLOBALS] initial_s0_31=" + + string.Join(',', evaluation.InitialScalarRegisters + .Take(32) + .Select((value, index) => $"s{index}={value:X8}"))); + + var probeOffsets = new[] + { + 0, 16, 24, 32, 48, + 192, 256, 400, 432, + 17100, 17104, 17136, 17168, 17184, 17200, 17216, + }; + foreach (var binding in evaluation.GlobalMemoryBindings) + { + Console.Error.WriteLine( + $"[TITLE-GLOBALS] binding s{binding.ScalarAddress} " + + $"base=0x{binding.BaseAddress:X16} bytes={binding.DataLength} " + + $"pcs={string.Join(',', binding.InstructionPcs.Select(pc => $"0x{pc:X}"))}"); + foreach (var offset in probeOffsets) + { + if (offset < 0 || offset + 16 > binding.DataLength) + { + continue; + } + + Console.Error.WriteLine( + $"[TITLE-GLOBALS] s{binding.ScalarAddress}+0x{offset:X}=" + + Convert.ToHexString(binding.Data.AsSpan(offset, 16))); + } + } + } + + private static readonly bool _fillClearHack = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_FILL_CLEAR"), "1", StringComparison.Ordinal); /// - /// Chowdren resets its effect layers with an untextured transparent-black - /// fill using premultiplied blending. With One/OneMinusSrcAlpha that draw - /// is otherwise a no-op, causing fog and vignette layers to accumulate. - /// Treat precisely that draw shape as an overwrite only when every MRT - /// attachment uses the same premultiplied blend pattern. + /// Treat an untextured fill that outputs pure transparent black through + /// premultiplied blending as an overwrite. Chowdren issues exactly this + /// draw once per frame to reset its effect layers (fog smoke, vignette + /// masks); under the blend factors it sets (One, OneMinusSrcAlpha) a + /// (0,0,0,0) source is a mathematical no-op, so without this the layers + /// accumulate until they saturate and the fog composites as a flat veil + /// over the whole scene. The workaround applies only when every MRT + /// attachment uses the same blend pattern. Disable with + /// SHARPEMU_DISABLE_FILL_CLEAR=1. /// private static GuestRenderState ApplyTransparentPremultipliedFillClear( GuestRenderState renderState, @@ -3758,7 +6249,7 @@ public static class AgcExports IReadOnlyList vertexInputs, IReadOnlyList pixelUserData) { - if (!_transparentFillClearEnabled || + if (!_fillClearHack || textures.Count != 0 || vertexInputs.Count != 0 || pixelUserData.Count < 4 || @@ -3769,6 +6260,7 @@ public static class AgcExports for (var index = 0; index < 4; index++) { + // Positive or negative zero. if ((pixelUserData[index] & 0x7FFF_FFFFu) != 0) { return renderState; @@ -3806,12 +6298,101 @@ public static class AgcExports var bytesPerIndex = is32Bit ? sizeof(uint) : sizeof(ushort); var byteOffset = checked((ulong)state.DrawIndexOffset * (uint)bytesPerIndex); var byteCount = checked((int)(indexCount * (uint)bytesPerIndex)); - var data = new byte[byteCount]; + var data = VulkanVideoPresenter.GuestDataPool.Rent(byteCount); + var span = data.AsSpan(0, byteCount); var address = state.IndexBufferAddress + byteOffset; - return (ctx.Memory.TryRead(address, data) || - KernelMemoryCompatExports.TryReadTrackedLibcHeap(address, data)) - ? new GuestIndexBuffer(data, is32Bit) - : null; + if (ctx.Memory.TryRead(address, span) || + KernelMemoryCompatExports.TryReadTrackedLibcHeap(address, span)) + { + return new GuestIndexBuffer(data, byteCount, is32Bit, Pooled: true); + } + + VulkanVideoPresenter.GuestDataPool.Return(data); + return null; + } + + private static bool TryGetRequiredVertexRecordCount( + CpuContext ctx, + SubmittedDcbState state, + uint drawCount, + bool indexed, + out uint recordCount) + { + recordCount = Math.Max(drawCount, Math.Max(state.InstanceCount, 1u)); + if (!indexed) + { + return true; + } + + if (state.IndexBufferAddress == 0 || drawCount == 0) + { + return false; + } + + var is32Bit = state.IndexSize != 0; + var bytesPerIndex = is32Bit ? sizeof(uint) : sizeof(ushort); + var byteOffset = checked((ulong)state.DrawIndexOffset * (uint)bytesPerIndex); + var address = state.IndexBufferAddress + byteOffset; + const int chunkBytes = 64 * 1024; + var scratch = VulkanVideoPresenter.GuestDataPool.Rent(chunkBytes); + var remaining = drawCount; + var maxIndex = 0u; + var sawIndex = false; + try + { + while (remaining != 0) + { + var chunkIndices = (int)Math.Min( + remaining, + (uint)(chunkBytes / bytesPerIndex)); + var bytes = chunkIndices * bytesPerIndex; + var span = scratch.AsSpan(0, bytes); + if (!ctx.Memory.TryRead(address, span) && + !KernelMemoryCompatExports.TryReadTrackedLibcHeap(address, span)) + { + return false; + } + + for (var index = 0; index < chunkIndices; index++) + { + var value = is32Bit + ? BinaryPrimitives.ReadUInt32LittleEndian( + span.Slice(index * sizeof(uint), sizeof(uint))) + : BinaryPrimitives.ReadUInt16LittleEndian( + span.Slice(index * sizeof(ushort), sizeof(ushort))); + if (value == (is32Bit ? uint.MaxValue : ushort.MaxValue)) + { + // Primitive-restart markers do not address vertex data. + continue; + } + + maxIndex = Math.Max(maxIndex, value); + sawIndex = true; + } + + address += (uint)bytes; + remaining -= (uint)chunkIndices; + } + } + finally + { + VulkanVideoPresenter.GuestDataPool.Return(scratch); + } + + var indexedRecords = sawIndex && maxIndex != uint.MaxValue + ? maxIndex + 1 + : 1u; + recordCount = Math.Max(indexedRecords, Math.Max(state.InstanceCount, 1u)); + if (_traceVertexRanges && + Interlocked.Increment(ref _tracedVertexRangeCount) <= 512) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] agc.vertex_range indexed=1 draw_count={drawCount} " + + $"max_index={(sawIndex ? maxIndex : 0)} records={recordCount} " + + $"instances={state.InstanceCount} index_size={(is32Bit ? 32 : 16)} " + + $"index_addr=0x{state.IndexBufferAddress:X16} offset={state.DrawIndexOffset}"); + } + return true; } private static bool HasPixelColorExport(Gen5ShaderState state, uint target) => @@ -3838,35 +6419,40 @@ public static class AgcExports return (uint)(maxAttribute + 1); } - private const int ShaderScalarRegisterCount = 256; + private static readonly bool _bakeScalars = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_BAKE_SGPRS"), + "1", + StringComparison.Ordinal); - private static Gen5GlobalMemoryBinding CreateScalarRegisterBinding( - Gen5ShaderEvaluation evaluation) + /// + /// Fingerprint of everything that shapes the translated SPIR-V besides + /// scalar register values (those arrive in a per-draw buffer): the + /// resolved binding set with its format-shaping descriptor words, vertex + /// input layouts, and compute system registers. Value churn in user data + /// no longer forces a new translation and pipeline. + /// + private static ulong ComputeShaderStructuralFingerprint(Gen5ShaderEvaluation evaluation) { - var data = new byte[ShaderScalarRegisterCount * sizeof(uint)]; - var registers = evaluation.InitialScalarRegisters; - var count = Math.Min(registers.Count, ShaderScalarRegisterCount); - for (var index = 0; index < count; index++) - { - var value = registers[index]; - var offset = index * sizeof(uint); - data[offset] = (byte)value; - data[offset + 1] = (byte)(value >> 8); - data[offset + 2] = (byte)(value >> 16); - data[offset + 3] = (byte)(value >> 24); - } - - return new Gen5GlobalMemoryBinding(0, 0, [], data); - } - - private static ulong ComputeShaderStructureFingerprint(Gen5ShaderEvaluation evaluation) - { - const ulong offsetBasis = 14695981039346656037UL; const ulong prime = 1099511628211UL; - var hash = offsetBasis; + var hash = 14695981039346656037UL; void Mix(ulong value) => hash = (hash ^ value) * prime; - Mix((ulong)evaluation.GlobalMemoryBindings.Count); + foreach (var binding in evaluation.ImageBindings) + { + Mix(binding.Pc); + Mix((ulong)(uint)binding.Opcode.GetHashCode()); + if (binding.ResourceDescriptor.Count > 1) + { + // The generated image type depends only on unified format. + // Bounds are queried from the bound view in SPIR-V; guest image + // addresses, dimensions, swizzles and sampler state are all + // runtime descriptor data and must not create pipeline variants. + Mix(binding.ResourceDescriptor[1] & 0x1FF0_0000u); + } + + Mix(binding.MipLevel ?? 0xFFFF_FFFFUL); + } + foreach (var binding in evaluation.GlobalMemoryBindings) { Mix(binding.ScalarAddress); @@ -3877,27 +6463,8 @@ public static class AgcExports } } - Mix((ulong)evaluation.ImageBindings.Count); - foreach (var image in evaluation.ImageBindings) - { - Mix(image.Pc); - Mix((ulong)(uint)image.Opcode.GetHashCode()); - foreach (var word in image.ResourceDescriptor) - { - Mix(word); - } - - foreach (var word in image.SamplerDescriptor) - { - Mix(word); - } - - Mix(image.MipLevel ?? uint.MaxValue); - } - if (evaluation.VertexInputs is { } vertexInputs) { - Mix((ulong)vertexInputs.Count); foreach (var input in vertexInputs) { Mix(input.Pc); @@ -3906,6 +6473,7 @@ public static class AgcExports Mix(input.DataFormat); Mix(input.NumberFormat); Mix(input.Stride); + Mix(input.OffsetBytes); } } @@ -3930,6 +6498,16 @@ public static class AgcExports hash = (hash ^ value) * prime; } + // Baked-scalar mode has no runtime state block from which the shader + // can load descriptor-alignment biases, so the low guest address bits + // remain part of the generated module and must participate in its key. + foreach (var binding in evaluation.GlobalMemoryBindings) + { + hash = (hash ^ ( + binding.BaseAddress & + (VulkanVideoPresenter.GuestStorageBufferOffsetAlignment - 1))) * prime; + } + if (evaluation.ComputeSystemRegisters is { } computeSystemRegisters) { hash = (hash ^ (computeSystemRegisters.WorkGroupXRegister ?? uint.MaxValue)) * prime; @@ -3941,8 +6519,36 @@ public static class AgcExports return hash; } + private static bool TryGetHardwareColorResolveTargets( + IReadOnlyDictionary registers, + out RenderTargetDescriptor source, + out RenderTargetDescriptor destination) + { + source = default; + destination = default; + if (!registers.TryGetValue(CbColorControl, out var colorControl) || + ((colorControl >> 4) & 0x7u) != 3u) + { + return false; + } + + // CB_COLOR_CONTROL.MODE=RESOLVE uses color slot 0 as the multisampled + // source and slot 1 as the single-sample destination. CB_TARGET_MASK + // still enables only slot 0, so treating this like a normal MRT draw + // rewrites the source and leaves the following composite's input blank. + var boundTargets = GetRenderTargets(registers, includeMaskedTargets: true); + source = boundTargets.FirstOrDefault(target => target.Slot == 0); + destination = boundTargets.FirstOrDefault(target => target.Slot == 1); + return source.Address != 0 && + destination.Address != 0 && + source.Width == destination.Width && + source.Height == destination.Height && + source.Format == destination.Format; + } + private static IReadOnlyList GetRenderTargets( - IReadOnlyDictionary registers) + IReadOnlyDictionary registers, + bool includeMaskedTargets = false) { var hasTargetMask = registers.TryGetValue(CbTargetMask, out var targetMask); var targets = new List(ColorTargetCount); @@ -3960,7 +6566,8 @@ public static class AgcExports var address = ((ulong)(baseHigh & 0xFFu) << 40) | ((ulong)baseLow << 8); var writeMask = (targetMask >> ((int)slot * 4)) & 0xFu; - if (address == 0 || (hasTargetMask && writeMask == 0)) + if (address == 0 || + (!includeMaskedTargets && hasTargetMask && writeMask == 0)) { continue; } @@ -3978,6 +6585,19 @@ public static class AgcExports return targets; } + private static GuestRenderState CreateRenderState( + IReadOnlyDictionary registers, + RenderTargetDescriptor target) + { + var scissor = DecodeScissor(registers, target.Width, target.Height); + return new GuestRenderState( + [DecodeBlendState(registers, target.Slot)], + scissor, + DecodeViewport(registers, target.Width, target.Height, scissor), + DecodeRasterState(registers), + DecodeDepthState(registers)); + } + private static GuestRenderState CreateRenderState( IReadOnlyDictionary registers, IReadOnlyList targets, @@ -4000,7 +6620,108 @@ public static class AgcExports }; }).ToArray(), scissor, - DecodeViewport(registers, target.Width, target.Height, scissor)); + DecodeViewport(registers, target.Width, target.Height, scissor), + DecodeRasterState(registers), + DecodeDepthState(registers)); + } + + // DB_DEPTH_CONTROL (context register 0x200): Z_ENABLE bit1, Z_WRITE_ENABLE + // bit2, ZFUNC bits[6:4] (GCN compare, matches Vulkan CompareOp ordering). + private const uint DbDepthControl = 0x200; + + private static GuestDepthState DecodeDepthState( + IReadOnlyDictionary registers) + { + if (!registers.TryGetValue(DbDepthControl, out var control)) + { + return GuestDepthState.Default; + } + + var testEnable = (control & 0x2u) != 0; + var writeEnable = (control & 0x4u) != 0; + var compareOp = (control >> 4) & 0x7u; + return new GuestDepthState(testEnable, writeEnable, compareOp); + } + + private static GuestDepthTarget? DecodeDepthTarget( + IReadOnlyDictionary registers) + { + var depthState = DecodeDepthState(registers); + if (!depthState.TestEnable && !depthState.WriteEnable) + { + return null; + } + + if (!registers.TryGetValue(DbZInfo, out var zInfo) || + !registers.TryGetValue(DbDepthSizeXy, out var sizeXy)) + { + return null; + } + + var guestFormat = zInfo & 0x3u; + if (guestFormat == 0) + { + return null; + } + + registers.TryGetValue(DbZReadBase, out var readBase); + registers.TryGetValue(DbZWriteBase, out var writeBase); + registers.TryGetValue(DbZReadBaseHi, out var readBaseHi); + registers.TryGetValue(DbZWriteBaseHi, out var writeBaseHi); + var readAddress = ((ulong)(readBaseHi & 0xFFu) << 40) | ((ulong)readBase << 8); + var writeAddress = ((ulong)(writeBaseHi & 0xFFu) << 40) | ((ulong)writeBase << 8); + if (readAddress == 0 && writeAddress == 0) + { + return null; + } + + var width = (sizeXy & 0x3FFFu) + 1; + var height = ((sizeXy >> 16) & 0x3FFFu) + 1; + if (width == 0 || height == 0 || width > 16384 || height > 16384) + { + return null; + } + + registers.TryGetValue(DbDepthView, out var depthView); + var clearDepth = registers.TryGetValue(DbDepthClear, out var clearBits) + ? BitConverter.UInt32BitsToSingle(clearBits) + : 1f; + if (!float.IsFinite(clearDepth) || clearDepth < 0f || clearDepth > 1f) + { + clearDepth = 1f; + } + + return new GuestDepthTarget( + readAddress, + writeAddress, + width, + height, + guestFormat, + (zInfo >> 4) & 0x1Fu, + clearDepth, + ReadOnly: (depthView & (1u << 24)) != 0 || writeAddress == 0); + } + + // PA_SU_SC_MODE_CNTL (context register 0x205) carries face culling, the + // front-face winding and polygon (wireframe) mode. + private const uint PaSuScModeCntl = 0x205; + + private static GuestRasterState DecodeRasterState( + IReadOnlyDictionary registers) + { + if (!registers.TryGetValue(PaSuScModeCntl, out var mode)) + { + return GuestRasterState.Default; + } + + var cullFront = (mode & 0x1u) != 0; + var cullBack = (mode & 0x2u) != 0; + var frontFaceClockwise = (mode & 0x4u) != 0; + var polyMode = (mode >> 3) & 0x3u; + var frontPtype = (mode >> 5) & 0x7u; + // POLY_MODE != 0 with a line front primitive type renders wireframe. + var wireframe = polyMode != 0 && frontPtype == 1; + return new GuestRasterState(cullFront, cullBack, frontFaceClockwise, wireframe); } private static GuestBlendState DecodeBlendState( @@ -4056,7 +6777,19 @@ public static class AgcExports windowOffsetY = (short)(windowOffset >> 16); } - IntersectScissorPair(registers, PaScScreenScissorTl, PaScScreenScissorBr, ref left, ref top, ref right, ref bottom); + // AGC reset-state blocks can carry an all-zero screen-scissor pair as + // an unpatched placeholder while the generic/viewport scissors hold + // the active bounds. Treat only that exact reset value as absent. A + // nonzero empty rectangle remains meaningful and still clips the draw. + IntersectScissorPair( + registers, + PaScScreenScissorTl, + PaScScreenScissorBr, + ref left, + ref top, + ref right, + ref bottom, + ignoreAllZeroPair: true); IntersectScissorPair( registers, PaScWindowScissorTl, @@ -4188,9 +6921,19 @@ public static class AgcExports ref int right, ref int bottom, int offsetX = 0, - int offsetY = 0) + int offsetY = 0, + bool ignoreAllZeroPair = false) { - if (!TryDecodeScissorPair(registers, tlRegister, brRegister, out var pairLeft, out var pairTop, out var pairRight, out var pairBottom)) + if (!TryDecodeScissorPair( + registers, + tlRegister, + brRegister, + out var pairLeft, + out var pairTop, + out var pairRight, + out var pairBottom, + out var allZero) || + (ignoreAllZeroPair && allZero)) { return; } @@ -4213,18 +6956,21 @@ public static class AgcExports out int left, out int top, out int right, - out int bottom) + out int bottom, + out bool allZero) { left = 0; top = 0; right = 0; bottom = 0; + allZero = false; if (!registers.TryGetValue(tlRegister, out var tl) || !registers.TryGetValue(brRegister, out var br)) { return false; } + allZero = tl == 0 && br == 0; left = (int)(tl & 0x7FFFu); top = (int)((tl >> 16) & 0x7FFFu); right = (int)(br & 0x7FFFu); @@ -4247,6 +6993,12 @@ public static class AgcExports draw.RenderTargets.Select(target => $"{target.Slot}:0x{target.Address:X16}:{target.Width}x{target.Height}:" + $"fmt{target.Format}/num{target.NumberType}/tile{target.TileMode}")); + var depthTarget = draw.DepthTarget is { } depth + ? $"0x{depth.Address:X16}:{depth.Width}x{depth.Height}:" + + $"fmt{depth.GuestFormat}/sw{depth.SwizzleMode}:" + + $"read=0x{depth.ReadAddress:X16}/write=0x{depth.WriteAddress:X16}:" + + $"clear={depth.ClearDepth:0.######}/ro={(depth.ReadOnly ? 1 : 0)}" + : "none"; var probes = new Dictionary(); var textures = string.Join( ',', @@ -4284,11 +7036,11 @@ public static class AgcExports var buffers = string.Join( ',', draw.GlobalMemoryBindings.Select((binding, index) => - $"{index}:0x{binding.BaseAddress:X16}:{binding.Data.Length}:" + - Convert.ToHexString(binding.Data.AsSpan(0, Math.Min(binding.Data.Length, 256))))); + $"{index}:0x{binding.BaseAddress:X16}:{binding.DataLength}:" + + Convert.ToHexString(binding.Data.AsSpan(0, Math.Min(binding.DataLength, 256))))); var indices = draw.IndexBuffer is { } indexBuffer ? $"{(indexBuffer.Is32Bit ? 32 : 16)}:" + - Convert.ToHexString(indexBuffer.Data.AsSpan(0, Math.Min(indexBuffer.Data.Length, 32))) + Convert.ToHexString(indexBuffer.Data.AsSpan(0, Math.Min(indexBuffer.Length, 32))) : "none"; var vertexInputs = draw.VertexInputs.Count == 0 ? "none" @@ -4338,7 +7090,7 @@ public static class AgcExports $"write_mask=0x{blend.WriteMask:X} scissor={scissor} viewport={viewport} " + $"raster=[{raster}] " + $"ps_ena=0x{psInputEna:X8} ps_addr=0x{psInputAddr:X8} " + - $"targets=[{targets}] textures=[{textures}] " + + $"targets=[{targets}] depth=[{depthTarget}] textures=[{textures}] " + $"buffers=[{buffers}] vertex=[{vertexInputs}] indices=[{indices}]"); } @@ -4370,6 +7122,243 @@ public static class AgcExports return textures; } + /// + /// Guest storage buffers for a translated draw, followed by the per-draw + /// initial scalar registers of each stage (pixel then vertex), matching + /// the binding layout the shaders were compiled against. + /// + private static IReadOnlyList CreateTranslatedDrawGlobalBuffers( + TranslatedGuestDraw translatedDraw) + { + var buffers = CreateGuestMemoryBuffers(translatedDraw.GlobalMemoryBindings); + if (_bakeScalars) + { + return buffers; + } + + var combined = new List(buffers.Count + 2); + combined.AddRange(buffers); + var runtimeStateLength = GetRuntimeScalarBufferLength( + translatedDraw.GlobalMemoryBindings.Count); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarState( + translatedDraw.PixelInitialScalars, + translatedDraw.GlobalMemoryBindings), + runtimeStateLength, + Pooled: true)); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarState( + translatedDraw.VertexInitialScalars, + translatedDraw.GlobalMemoryBindings), + runtimeStateLength, + Pooled: true)); + return combined; + } + + private static IReadOnlyList + CreateGlobalBufferOwnershipView( + IReadOnlyList buffers, + bool ownsPooledData) + { + var view = new GuestMemoryBuffer[buffers.Count]; + for (var index = 0; index < buffers.Count; index++) + { + var buffer = buffers[index]; + view[index] = buffer with + { + Pooled = ownsPooledData && buffer.Pooled, + }; + } + + return view; + } + + /// + /// Present-time variant: the flip path can reuse the same translated + /// draw across several flips and swapchain retries, so it must not wrap + /// the (pooled, single-consumption) binding arrays. Buffer contents are + /// re-read from guest memory instead, which also presents current data. + /// + private static IReadOnlyList CreateTranslatedDrawGlobalBuffersForPresent( + CpuContext ctx, + TranslatedGuestDraw translatedDraw) + { + var bindings = translatedDraw.GlobalMemoryBindings; + var combined = new List(bindings.Count + 2); + foreach (var binding in bindings) + { + var data = new byte[Math.Max(binding.DataLength, sizeof(uint))]; + var guestMemoryBacked = binding.BaseAddress != 0 && + (ctx.Memory.TryRead(binding.BaseAddress, data) || + KernelMemoryCompatExports.TryReadTrackedLibcHeap(binding.BaseAddress, data)); + if (!guestMemoryBacked) + { + // Keep the zero-filled buffer; layout must match the shader. + } + + combined.Add(new GuestMemoryBuffer( + binding.BaseAddress, + data, + data.Length, + Pooled: false, + Writable: binding.Writable, + WriteBackToGuest: binding.WriteBackToGuest && guestMemoryBacked)); + } + + if (!_bakeScalars) + { + var runtimeStateLength = GetRuntimeScalarBufferLength(bindings.Count); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarStateUnpooled( + translatedDraw.PixelInitialScalars, + bindings), + runtimeStateLength, + Pooled: false)); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarStateUnpooled( + translatedDraw.VertexInitialScalars, + bindings), + runtimeStateLength, + Pooled: false)); + } + + return combined; + } + + private static int GetRuntimeScalarBufferLength(int bindingCount) => + checked((256 + bindingCount) * sizeof(uint)); + + private static byte[] PackRuntimeScalarState( + IReadOnlyList registers, + IReadOnlyList bindings) + { + var bytes = VulkanVideoPresenter.GuestDataPool.Rent( + GetRuntimeScalarBufferLength(bindings.Count)); + PackRuntimeScalarStateInto(bytes, registers, bindings); + return bytes; + } + + private static byte[] PackRuntimeScalarStateUnpooled( + IReadOnlyList registers, + IReadOnlyList bindings) + { + var bytes = new byte[GetRuntimeScalarBufferLength(bindings.Count)]; + PackRuntimeScalarStateInto(bytes, registers, bindings); + return bytes; + } + + private static void PackRuntimeScalarStateInto( + byte[] bytes, + IReadOnlyList registers, + IReadOnlyList bindings) + { + PackScalarRegistersInto(bytes, registers); + var biasOffset = 256 * sizeof(uint); + for (var index = 0; index < bindings.Count; index++) + { + var byteBias = checked((uint)( + bindings[index].BaseAddress & + (VulkanVideoPresenter.GuestStorageBufferOffsetAlignment - 1))); + BinaryPrimitives.WriteUInt32LittleEndian( + bytes.AsSpan(biasOffset + index * sizeof(uint), sizeof(uint)), + byteBias); + } + } + + private static void PackScalarRegistersInto(byte[] bytes, IReadOnlyList registers) + { + if (registers is uint[] { Length: >= 256 } array) + { + // Guest scalar registers are little-endian dwords and the host + // is x86-64, so a bulk copy replaces 256 per-element writes. + System.Runtime.InteropServices.MemoryMarshal + .AsBytes(array.AsSpan(0, 256)) + .CopyTo(bytes); + return; + } + + // Rented arrays carry stale bytes; clear the packed window first. + Array.Clear(bytes, 0, 256 * sizeof(uint)); + var count = Math.Min(registers.Count, 256); + for (var index = 0; index < count; index++) + { + BinaryPrimitives.WriteUInt32LittleEndian( + bytes.AsSpan(index * sizeof(uint)), + registers[index]); + } + } + + /// + /// Returns the pooled buffer arrays an evaluation produced. Called only + /// on translation-failure paths, where no + /// is built to take ownership; on success the draw's consumers return them. + /// + private static void ReturnPooledEvaluationArrays(Gen5ShaderEvaluation evaluation) + { + foreach (var binding in evaluation.GlobalMemoryBindings) + { + if (binding.DataPooled) + { + VulkanVideoPresenter.GuestDataPool.Return(binding.Data); + } + } + + if (evaluation.VertexInputs is { } vertexInputs) + { + foreach (var binding in vertexInputs) + { + if (binding.DataPooled) + { + VulkanVideoPresenter.GuestDataPool.Return(binding.Data); + } + } + } + } + + /// + /// Returns pooled data arrays a translated draw owns but did not hand to + /// a presenter consumer. The offscreen path hands globals, vertex and + /// index buffers to the presenter (which returns them), so it passes all + /// three false; other draw sinks pass true for whatever they dropped. + /// + private static void ReturnPooledDrawArrays( + TranslatedGuestDraw draw, + bool globals, + bool vertex, + bool index) + { + if (globals) + { + foreach (var binding in draw.GlobalMemoryBindings) + { + if (binding.DataPooled) + { + VulkanVideoPresenter.GuestDataPool.Return(binding.Data); + } + } + } + + if (vertex) + { + foreach (var binding in draw.VertexInputs) + { + if (binding.DataPooled) + { + VulkanVideoPresenter.GuestDataPool.Return(binding.Data); + } + } + } + + if (index && draw.IndexBuffer is { Pooled: true } indexBuffer) + { + VulkanVideoPresenter.GuestDataPool.Return(indexBuffer.Data); + } + } + private static IReadOnlyList CreateGuestMemoryBuffers( IReadOnlyList bindings) { @@ -4378,12 +7367,42 @@ public static class AgcExports { buffers[index] = new GuestMemoryBuffer( bindings[index].BaseAddress, - bindings[index].Data); + bindings[index].Data, + bindings[index].DataLength, + bindings[index].DataPooled, + bindings[index].Writable, + bindings[index].WriteBackToGuest); } return buffers; } + /// + /// Guest storage buffers for a compute dispatch followed by its initial + /// scalar registers. Dispatch-specific SGPR values remain runtime data so + /// one translated pipeline serves every matching shader/resource shape. + /// + private static IReadOnlyList CreateTranslatedComputeGlobalBuffers( + Gen5ShaderEvaluation evaluation) + { + var buffers = CreateGuestMemoryBuffers(evaluation.GlobalMemoryBindings); + if (_bakeScalars) + { + return buffers; + } + + var combined = new List(buffers.Count + 1); + combined.AddRange(buffers); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarState( + evaluation.InitialScalarRegisters, + evaluation.GlobalMemoryBindings), + GetRuntimeScalarBufferLength(evaluation.GlobalMemoryBindings.Count), + Pooled: true)); + return combined; + } + private static IReadOnlyList CreateGuestVertexBuffers( IReadOnlyList bindings) { @@ -4399,171 +7418,113 @@ public static class AgcExports binding.BaseAddress, binding.Stride, binding.OffsetBytes, - binding.Data); + binding.Data, + binding.DataLength, + binding.DataPooled); } return buffers; } - private static bool TryCreateGuestDrawTexture( - CpuContext ctx, - TextureDescriptor descriptor, - bool isStorage, - uint mipLevel, - IReadOnlyList samplerDescriptor, - out GuestDrawTexture texture) + private static IReadOnlyList + CreateVertexBufferOwnershipView( + IReadOnlyList buffers, + bool ownsPooledData) { - texture = default!; - if (descriptor.Type != Gen5TextureType2D || - descriptor.Width == 0 || - descriptor.Height == 0 || - descriptor.Width > 8192 || - descriptor.Height > 8192) + var view = new GuestVertexBuffer[buffers.Count]; + for (var index = 0; index < buffers.Count; index++) { - TraceTextureFallback(descriptor, "invalid-descriptor"); - texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); - return true; - } - - var sourceWidth = descriptor.TileMode == 0 - ? GetLinearTexturePitch( - Math.Max(descriptor.Width, descriptor.Pitch), - descriptor.Format) - : descriptor.Width; - var sourceByteCount = GetTextureByteCount( - descriptor.Format, - sourceWidth, - descriptor.Height); - if (sourceByteCount == 0 || - sourceByteCount > MaxPresentedTextureBytes || - sourceByteCount > int.MaxValue) - { - TraceTextureFallback( - descriptor, - $"invalid-byte-count:{sourceByteCount}"); - texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); - return true; - } - - if (!isStorage && - descriptor.Address != 0 && - GuestGpu.Current.IsGpuGuestImageAvailable( - descriptor.Address, - descriptor.Format, - descriptor.NumberType)) - { - texture = new GuestDrawTexture( - descriptor.Address, - descriptor.Width, - descriptor.Height, - descriptor.Format, - descriptor.NumberType, - [], - IsFallback: false, - IsStorage: false, - MipLevels: descriptor.MipLevels, - MipLevel: mipLevel, - Pitch: sourceWidth, - TileMode: descriptor.TileMode, - DstSelect: descriptor.DstSelect, - Sampler: ToGuestSampler(samplerDescriptor)); - return true; - } - - if (isStorage) - { - var initialPixels = Array.Empty(); - if (descriptor.Address != 0) + var buffer = buffers[index]; + view[index] = buffer with { - var storageSource = new byte[(int)sourceByteCount]; - if ((ctx.Memory.TryRead(descriptor.Address, storageSource) || - KernelMemoryCompatExports.TryReadTrackedLibcHeapGpuAlias( - descriptor.Address, - storageSource)) && - storageSource.AsSpan().IndexOfAnyExcept((byte)0) >= 0) - { - initialPixels = storageSource; - } + Pooled = ownsPooledData && buffer.Pooled, + }; + } + + return view; + } + + private static GuestIndexBuffer? CreateIndexBufferOwnershipView( + GuestIndexBuffer? buffer, + bool ownsPooledData) => + buffer is null + ? null + : buffer with { Pooled = ownsPooledData && buffer.Pooled }; + + // BCn block-compressed guest formats and the bytes per 4x4 block. + private static int GetBlockCompressedBlockBytes(uint format) => format switch + { + 169 or 170 or 175 or 176 => 8, + 171 or 172 or 173 or 174 or 177 or 178 or 179 or 180 or 181 or 182 => 16, + _ => 0, + }; + + /// + /// Deswizzles a tiled texture source into linear layout when tiling is + /// enabled and the format is understood; returns null to keep the raw + /// bytes (linear surfaces, unknown modes, or non-power-of-two elements). + /// + private static bool TryGetTextureElementLayout( + TextureDescriptor descriptor, + uint sourceWidth, + out int elementsWide, + out int elementsHigh, + out int bytesPerElement) + { + var blockBytes = GetBlockCompressedBlockBytes(descriptor.Format); + if (blockBytes != 0) + { + bytesPerElement = blockBytes; + elementsWide = (int)((sourceWidth + 3) / 4); + elementsHigh = (int)((descriptor.Height + 3) / 4); + } + else + { + bytesPerElement = (int)GetTextureBytesPerTexel(descriptor.Format); + if (bytesPerElement == 0) + { + elementsWide = 0; + elementsHigh = 0; + return false; } - texture = new GuestDrawTexture( - descriptor.Address, - descriptor.Width, - descriptor.Height, - descriptor.Format, - descriptor.NumberType, - initialPixels, - IsFallback: descriptor.Address == 0, - IsStorage: true, - MipLevels: descriptor.MipLevels, - MipLevel: mipLevel, - Pitch: sourceWidth, - TileMode: descriptor.TileMode, - DstSelect: descriptor.DstSelect, - Sampler: ToGuestSampler(samplerDescriptor)); - return true; + elementsWide = (int)sourceWidth; + elementsHigh = (int)descriptor.Height; } - var source = new byte[(int)sourceByteCount]; - if (!ctx.Memory.TryRead(descriptor.Address, source) && - !KernelMemoryCompatExports.TryReadTrackedLibcHeapGpuAlias( - descriptor.Address, - source)) - { - TraceTextureFallback( - descriptor, - $"guest-read-failed:{sourceByteCount}"); - texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); - return true; - } - - TraceTextureHash(descriptor, source); - - var nonZero = 0; - for (var i = 0; i < source.Length; i++) - { - if (source[i] != 0) - { - nonZero++; - if (nonZero >= 64) - { - break; - } - } - } - - TraceAgcShader( - $"agc.texture_source addr=0x{descriptor.Address:X16} " + - $"fmt={descriptor.Format} num={descriptor.NumberType} tile={descriptor.TileMode} " + - $"size={descriptor.Width}x{descriptor.Height} pitch={descriptor.Pitch} " + - $"dst=0x{descriptor.DstSelect:X3} " + - $"bytes={source.Length} nonzero64={nonZero}"); - DumpTextureSourceIfRequested(descriptor, sourceWidth, source); - - var rgba = source; - texture = new GuestDrawTexture( - descriptor.Address, - descriptor.Width, - descriptor.Height, - descriptor.Format, - descriptor.NumberType, - rgba, - IsFallback: false, - IsStorage: isStorage, - MipLevels: descriptor.MipLevels, - MipLevel: mipLevel, - Pitch: sourceWidth, - TileMode: descriptor.TileMode, - DstSelect: descriptor.DstSelect, - Sampler: ToGuestSampler(samplerDescriptor)); return true; } - private static int _textureFallbackTraceCount; - - private static void TraceTextureFallback( + private static byte[]? TryDetileTextureSource( TextureDescriptor descriptor, - string reason) + uint sourceWidth, + int logicalByteCount, + byte[] source) + { + if (!GnmTiling.NeedsDetile(descriptor.TileMode) || + !TryGetTextureElementLayout( + descriptor, + sourceWidth, + out var elementsWide, + out var elementsHigh, + out var bytesPerElement)) + { + return null; + } + + var linear = new byte[logicalByteCount]; + return GnmTiling.TryDetile( + source, + linear, + descriptor.TileMode, + elementsWide, + elementsHigh, + bytesPerElement) + ? linear + : null; + } + + private static void TraceTextureFallback(TextureDescriptor descriptor, string reason) { var mode = Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"); if ((!string.Equals(mode, "1", StringComparison.Ordinal) && @@ -4582,7 +7543,383 @@ public static class AgcExports $"dst=0x{descriptor.DstSelect:X3}"); } + private static bool TryCreateGuestDrawTexture( + CpuContext ctx, + TextureDescriptor descriptor, + bool isStorage, + uint mipLevel, + IReadOnlyList samplerDescriptor, + out GuestDrawTexture texture) + { + texture = default!; + if ((descriptor.Type != Gen5TextureType1D && + descriptor.Type != Gen5TextureType2D) || + descriptor.Width == 0 || + descriptor.Height == 0 || + descriptor.Width > 8192 || + descriptor.Height > 8192) + { + TraceTextureFallback(descriptor, "invalid-descriptor"); + texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); + return true; + } + var sourceWidth = descriptor.TileMode == 0 + ? GetLinearTexturePitch( + Math.Max(descriptor.Width, descriptor.Pitch), + descriptor.Height, + descriptor.Format) + : descriptor.Width; + var sourceByteCount = GetTextureByteCount( + descriptor.Format, + sourceWidth, + descriptor.Height); + if (sourceByteCount == 0 || + sourceByteCount > MaxPresentedTextureBytes || + sourceByteCount > int.MaxValue) + { + TraceTextureFallback( + descriptor, + $"invalid-byte-count:{sourceByteCount}"); + texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); + return true; + } + + var physicalSourceByteCount = sourceByteCount; + if (GnmTiling.NeedsDetile(descriptor.TileMode) && + TryGetTextureElementLayout( + descriptor, + sourceWidth, + out var elementsWide, + out var elementsHigh, + out var bytesPerElement) && + GnmTiling.TryGetTiledByteCount( + descriptor.TileMode, + elementsWide, + elementsHigh, + bytesPerElement, + out var tiledByteCount)) + { + physicalSourceByteCount = tiledByteCount; + } + + if (physicalSourceByteCount > MaxPresentedTextureBytes || + physicalSourceByteCount > int.MaxValue) + { + texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); + return true; + } + + if (!isStorage && + descriptor.Address != 0 && + VulkanVideoPresenter.IsGuestImageAvailable( + descriptor.Address, + descriptor.Format, + descriptor.NumberType)) + { + texture = new GuestDrawTexture( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + [], + IsFallback: false, + IsStorage: false, + MipLevels: descriptor.MipLevels, + MipLevel: mipLevel, + BaseMipLevel: descriptor.ViewBaseLevel, + ResourceMipLevels: descriptor.ResourceMipLevels, + Pitch: sourceWidth, + TileMode: descriptor.TileMode, + DstSelect: descriptor.DstSelect, + Sampler: ToGuestSampler(samplerDescriptor)); + return true; + } + + if (isStorage) + { + var initialPixels = Array.Empty(); + var uploadKnown = descriptor.Address != 0 && + VulkanVideoPresenter.IsGuestImageUploadKnown( + descriptor.Address, + descriptor.Format, + descriptor.NumberType); + var readSucceeded = false; + var linearNonzero = false; + if (descriptor.Address != 0 && !uploadKnown) + { + // Storage images can be pre-populated in tiled guest memory + // just like sampled images. Reading only the logical linear + // byte count both truncates 64 KiB swizzle blocks and uploads + // tiled bytes as scanlines. Read the full physical footprint + // and run the same AddrLib-derived detile path used below for + // sampled textures before seeding the Vulkan image. + var storageSource = new byte[(int)physicalSourceByteCount]; + if (ctx.Memory.TryRead(descriptor.Address, storageSource)) + { + readSucceeded = true; + var linearStorage = TryDetileTextureSource( + descriptor, + sourceWidth, + checked((int)sourceByteCount), + storageSource) ?? storageSource + .AsSpan(0, checked((int)sourceByteCount)) + .ToArray(); + if (linearStorage.AsSpan().IndexOfAnyExcept((byte)0) >= 0) + { + linearNonzero = true; + initialPixels = linearStorage; + } + } + } + + if (ParseOptionalHexAddress( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_STORAGE_IMAGE_INIT_ADDRESS")) == + descriptor.Address) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] agc.storage_initial_data " + + $"addr=0x{descriptor.Address:X16} op_storage={isStorage} " + + $"upload_known={uploadKnown} read={readSucceeded} " + + $"nonzero={linearNonzero} initial_bytes={initialPixels.Length} " + + $"logical_bytes={sourceByteCount} physical_bytes={physicalSourceByteCount} " + + $"size={descriptor.Width}x{descriptor.Height} pitch={sourceWidth} " + + $"fmt={descriptor.Format} num={descriptor.NumberType} " + + $"tile={descriptor.TileMode} mip={mipLevel}"); + } + + texture = new GuestDrawTexture( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + initialPixels, + IsFallback: descriptor.Address == 0, + IsStorage: true, + MipLevels: descriptor.MipLevels, + MipLevel: mipLevel, + BaseMipLevel: descriptor.ViewBaseLevel, + ResourceMipLevels: descriptor.ResourceMipLevels, + Pitch: sourceWidth, + TileMode: descriptor.TileMode, + DstSelect: descriptor.DstSelect, + Sampler: ToGuestSampler(samplerDescriptor)); + return true; + } + + // When the presenter already holds this exact texture identity in + // its cache, the texel copy below would be discarded on arrival; for + // scenes that sample large textures every draw this copy dominated + // CPU time. The dirty peek closes the race with eviction: a texture + // the guest rewrote must ship fresh texels with this draw, because + // the render thread evicts the stale cache entry before executing it + // (skipping would leave the draw with no pixels and a fallback + // texture for the frame — visible flicker on animated textures). + var sampler = ToGuestSampler(samplerDescriptor); + if (!_textureCopySkipDisabled && + descriptor.Address != 0 && + !SharpEmu.HLE.GuestImageWriteTracker.PeekDirty(descriptor.Address) && + VulkanVideoPresenter.IsTextureContentCached( + new VulkanVideoPresenter.TextureContentIdentity( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + descriptor.DstSelect, + descriptor.TileMode, + sourceWidth, + sampler))) + { + texture = new GuestDrawTexture( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + [], + IsFallback: false, + IsStorage: false, + MipLevels: descriptor.MipLevels, + MipLevel: mipLevel, + BaseMipLevel: descriptor.ViewBaseLevel, + ResourceMipLevels: descriptor.ResourceMipLevels, + Pitch: sourceWidth, + TileMode: descriptor.TileMode, + DstSelect: descriptor.DstSelect, + Sampler: sampler); + return true; + } + + var source = new byte[(int)physicalSourceByteCount]; + if (!ctx.Memory.TryRead(descriptor.Address, source)) + { + TraceTextureFallback( + descriptor, + $"guest-read-failed:{sourceByteCount}"); + texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); + return true; + } + + if (_traceAgcShader) + { + var nonZero = 0; + for (var i = 0; i < source.Length; i++) + { + if (source[i] != 0) + { + nonZero++; + if (nonZero >= 64) + { + break; + } + } + } + + TraceAgcShader( + $"agc.texture_source addr=0x{descriptor.Address:X16} " + + $"fmt={descriptor.Format} num={descriptor.NumberType} tile={descriptor.TileMode} " + + $"size={descriptor.Width}x{descriptor.Height} pitch={descriptor.Pitch} " + + $"dst=0x{descriptor.DstSelect:X3} " + + $"bytes={source.Length} logical_bytes={sourceByteCount} nonzero64={nonZero}"); + } + DumpTextureSourceIfRequested(descriptor, sourceWidth, source); + + var rgba = TryDetileTextureSource( + descriptor, + sourceWidth, + checked((int)sourceByteCount), + source) ?? source.AsSpan(0, checked((int)sourceByteCount)).ToArray(); + DumpLinearTextureIfRequested(descriptor, sourceWidth, rgba); + texture = new GuestDrawTexture( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + rgba, + IsFallback: false, + IsStorage: isStorage, + MipLevels: descriptor.MipLevels, + MipLevel: mipLevel, + BaseMipLevel: descriptor.ViewBaseLevel, + ResourceMipLevels: descriptor.ResourceMipLevels, + Pitch: sourceWidth, + TileMode: descriptor.TileMode, + DstSelect: descriptor.DstSelect, + Sampler: ToGuestSampler(samplerDescriptor)); + return true; + } + + + + /// + /// On PS5 render targets alias guest memory, so pixels the game wrote with + /// the CPU are visible before the first GPU draw (Chowdren pre-fills its + /// fog/overlay layers that way). Seed newly created Vulkan guest images + /// with the current guest memory contents to preserve that base layer. + /// + private static void ProvideRenderTargetInitialData( + CpuContext ctx, + RenderTargetDescriptor target) + { + if (!VulkanVideoPresenter.GuestImageWantsInitialData(target.Address)) + { + return; + } + + var byteCount = (ulong)target.Width * target.Height * 4; + if (byteCount == 0 || byteCount > MaxPresentedTextureBytes) + { + return; + } + + var initialData = new byte[byteCount]; + var readOk = ctx.Memory.TryRead(target.Address, initialData); + var nonZero = readOk && initialData.AsSpan().IndexOfAnyExcept((byte)0) >= 0; + if (_traceDraws && _rtSeedTraced.Add(target.Address)) + { + Console.Error.WriteLine( + $"[RTSEED] addr=0x{target.Address:X} {target.Width}x{target.Height} " + + $"read={readOk} nonZero={nonZero}"); + } + + if (nonZero) + { + VulkanVideoPresenter.ProvideGuestImageInitialData(target.Address, initialData); + } + } + + private static readonly HashSet _rtSeedTraced = new(); + + private static void TraceDrawCompact( + ulong sequence, + TranslatedGuestDraw draw, + IReadOnlyList textures, + IReadOnlyList vertexBuffers) + { + if (!_traceDraws) + { + return; + } + + var target = draw.RenderTargets.FirstOrDefault(); + var blend = draw.RenderState.Blend; + var viewport = draw.RenderState.Viewport is { } vp + ? $"{vp.X:0.#},{vp.Y:0.#},{vp.Width:0.#}x{vp.Height:0.#}" + : "none"; + var textureList = string.Join( + '|', + textures.Select(texture => + $"0x{texture.Address:X}:{texture.Width}x{texture.Height}" + + $":f{texture.Format}/n{texture.NumberType}/d{texture.DstSelect:X3}" + + (texture.IsFallback ? ":FALLBACK" : string.Empty))); + var positions = string.Empty; + var positionBuffer = vertexBuffers.FirstOrDefault(buffer => buffer.Location == 0); + if (positionBuffer is { Length: >= 8 }) + { + var stride = Math.Max(positionBuffer.Stride, 4u); + var vertexTotal = (int)((positionBuffer.Length - positionBuffer.OffsetBytes) / stride); + var sampled = new List(); + foreach (var vertex in new[] { 0, 1, vertexTotal - 1 }) + { + var baseOffset = (int)(positionBuffer.OffsetBytes + vertex * stride); + if (vertex < 0 || baseOffset + 8 > positionBuffer.Length) + { + continue; + } + + sampled.Add( + $"{BitConverter.ToSingle(positionBuffer.Data, baseOffset):0.##}," + + $"{BitConverter.ToSingle(positionBuffer.Data, baseOffset + 4):0.##}"); + } + + positions = string.Join(';', sampled); + } + + Console.Error.WriteLine( + $"[DRAW] seq={sequence} es=0x{draw.ExportShaderAddress:X} ps=0x{draw.PixelShaderAddress:X} " + + $"target=0x{target.Address:X}:{target.Width}x{target.Height}:f{target.Format}/n{target.NumberType} " + + $"prim=0x{draw.PrimitiveType:X} verts={draw.VertexCount} indexed={draw.IndexBuffer is not null} " + + $"blend={(blend.Enable ? 1 : 0)}:{blend.ColorSrcFactor}/{blend.ColorDstFactor}/{blend.ColorFunc}" + + $":a{blend.AlphaSrcFactor}/{blend.AlphaDstFactor}/{blend.AlphaFunc}/s{(blend.SeparateAlphaBlend ? 1 : 0)} " + + $"mask=0x{blend.WriteMask:X} viewport={viewport} textures={textureList} pos={positions} " + + $"ps_s0..3={string.Join(',', draw.PixelUserData.Take(4).Select(value => BitConverter.UInt32BitsToSingle(value).ToString("0.###")))} " + + $"rawblend=0x{draw.RawBlendControl:X8} info=0x{draw.RawColorInfo:X8}"); + } + + private static void TraceDrawCompactMiss(ulong sequence, uint vertexCount, string error) + { + if (!_traceDraws) + { + return; + } + + Console.Error.WriteLine($"[DRAW] seq={sequence} MISS verts={vertexCount} error={error}"); + } private static int _grassTraceCount; @@ -4606,13 +7943,13 @@ public static class AgcExports { text.Append( $"\n loc={buffer.Location} fmt={buffer.DataFormat}/{buffer.NumberFormat}x{buffer.ComponentCount} " + - $"stride={buffer.Stride} offset={buffer.OffsetBytes} bytes={buffer.Data.Length}"); + $"stride={buffer.Stride} offset={buffer.OffsetBytes} bytes={buffer.Length}"); var stride = Math.Max(buffer.Stride, 4u); - var maxVerts = Math.Min(6, (int)((buffer.Data.Length - buffer.OffsetBytes) / stride)); + var maxVerts = Math.Min(6, (int)((buffer.Length - buffer.OffsetBytes) / stride)); for (var vertex = 0; vertex < maxVerts; vertex++) { var baseOffset = (int)(buffer.OffsetBytes + vertex * stride); - var components = Math.Min(4, (int)((buffer.Data.Length - baseOffset) / 4)); + var components = Math.Min(4, (int)((buffer.Length - baseOffset) / 4)); text.Append($"\n v{vertex}:"); for (var c = 0; c < components; c++) { @@ -4645,7 +7982,7 @@ public static class AgcExports for (var vertex = 0; vertex < 3; vertex++) { var baseOffset = (int)(buffer.OffsetBytes + vertex * stride); - if (baseOffset + 16 > buffer.Data.Length) + if (baseOffset + 16 > buffer.Length) { break; } @@ -4702,12 +8039,47 @@ public static class AgcExports $"-p{sourcePitch}-f{descriptor.Format}-t{descriptor.TileMode}.bin"); File.WriteAllBytes(path, source); } - catch (Exception exception) + catch (IOException) + { + } + } + + /// + /// Writes the bytes after detiling when SHARPEMU_TEXTURE_LINEAR_DUMP_DIR is + /// set. Keeping this separate from the raw-source dump makes AddrLib + /// equation changes directly inspectable with ordinary image tools. + /// + private static void DumpLinearTextureIfRequested( + in TextureDescriptor descriptor, + uint sourcePitch, + byte[] source) + { + var directory = Environment.GetEnvironmentVariable("SHARPEMU_TEXTURE_LINEAR_DUMP_DIR"); + if (string.IsNullOrWhiteSpace(directory)) + { + return; + } + + var key = $"linear-0x{descriptor.Address:X}-{descriptor.Width}x{descriptor.Height}"; + var occurrence = _textureDumpKeys.AddOrUpdate(key, 1, static (_, count) => count + 1); + if ((occurrence > 3 && occurrence % 500 >= 3) || + Interlocked.Increment(ref _textureDumpCount) > 200) + { + return; + } + + var index = _textureDumpCount; + try + { + Directory.CreateDirectory(directory); + var path = Path.Combine( + directory, + $"{index:D3}-0x{descriptor.Address:X}-{descriptor.Width}x{descriptor.Height}" + + $"-p{sourcePitch}-f{descriptor.Format}-t{descriptor.TileMode}.linear.bin"); + File.WriteAllBytes(path, source); + } + catch (IOException) { - // A bad SHARPEMU_TEXTURE_DUMP_DIR (permissions, invalid path) - // must not take the emulator down; the dump is a debug aid. - Console.Error.WriteLine( - $"[LOADER][WARN] Texture dump failed: {exception.Message}"); } } @@ -4731,34 +8103,6 @@ public static class AgcExports MipLevel: 0); } - private static void TraceTextureHash(TextureDescriptor descriptor, ReadOnlySpan source) - { - if (!_traceTextureHashes || - descriptor.Address == 0 || - descriptor.Width > 256 || - descriptor.Height > 256) - { - return; - } - - var hash = ComputeFingerprint(source); - var key = (descriptor.Address, descriptor.Width, descriptor.Height); - lock (_textureHashTraceGate) - { - if (_tracedTextureHashes.TryGetValue(key, out var previousHash) && - previousHash == hash) - { - return; - } - - _tracedTextureHashes[key] = hash; - } - - Console.Error.WriteLine( - $"[LOADER][TRACE] agc.texture_hash addr=0x{descriptor.Address:X16} " + - $"size={descriptor.Width}x{descriptor.Height} bytes={source.Length} hash=0x{hash:X16}"); - } - private static GuestSampler ToGuestSampler(IReadOnlyList descriptor) => descriptor.Count >= 4 ? new GuestSampler( @@ -4807,52 +8151,246 @@ public static class AgcExports dispatch = default; ulong dimensionsAddress; uint initiator; + string dispatchSource; if (opcode == ItDispatchDirect) { if (packetLength < 5 || - !ctx.TryReadUInt32(packetAddress + 16, out initiator)) + !TryReadUInt32(ctx, packetAddress + 16, out initiator)) { return false; } dimensionsAddress = packetAddress + 4; + dispatchSource = "direct"; } else if (packetLength >= 4) { - if (!ctx.TryReadUInt64(packetAddress + 4, out dimensionsAddress) || - !ctx.TryReadUInt32(packetAddress + 12, out initiator)) + if (!TryReadUInt64(ctx, packetAddress + 4, out dimensionsAddress) || + !TryReadUInt32(ctx, packetAddress + 12, out initiator)) { return false; } + + dispatchSource = "absolute-indirect"; } else { if (packetLength < 3 || state.IndirectArgsAddress == 0 || - !ctx.TryReadUInt32(packetAddress + 4, out var dataOffset) || - !ctx.TryReadUInt32(packetAddress + 8, out initiator)) + !TryReadUInt32(ctx, packetAddress + 4, out var dataOffset) || + !TryReadUInt32(ctx, packetAddress + 8, out initiator)) { return false; } dimensionsAddress = state.IndirectArgsAddress + dataOffset; + dispatchSource = "base-indirect"; } if ((initiator & 1) == 0 || - !ctx.TryReadUInt32(dimensionsAddress, out var groupCountX) || - !ctx.TryReadUInt32(dimensionsAddress + 4, out var groupCountY) || - !ctx.TryReadUInt32(dimensionsAddress + 8, out var groupCountZ) || - groupCountX == 0 || - groupCountY == 0 || - groupCountZ == 0) + !TryReadUInt32(ctx, dimensionsAddress, out var dispatchEndX) || + !TryReadUInt32(ctx, dimensionsAddress + 4, out var dispatchEndY) || + !TryReadUInt32(ctx, dimensionsAddress + 8, out var dispatchEndZ)) { return false; } - dispatch = new ComputeDispatch(groupCountX, groupCountY, groupCountZ); + if (dispatchEndX == 0 || dispatchEndY == 0 || dispatchEndZ == 0) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + "zero-dimension"); + } + + // When FORCE_START_AT_000 is clear, RDNA2 interprets the three packet + // values as end coordinates, not group counts. Vulkan expresses the + // same operation as vkCmdDispatchBase(base, end - base). Ignoring the + // COMPUTE_START registers turned small high-base clears into apparent + // multi-million/billion-group dispatches and forced an unsafe cap. + const uint forceStartAtZero = 1u << 2; + const uint partialThreadGroupEnabled = 1u << 1; + const uint useThreadDimensions = 1u << 5; + uint baseGroupX = 0; + uint baseGroupY = 0; + uint baseGroupZ = 0; + if ((initiator & forceStartAtZero) == 0) + { + state.ShRegisters.TryGetValue(ComputeStartX, out baseGroupX); + state.ShRegisters.TryGetValue(ComputeStartY, out baseGroupY); + state.ShRegisters.TryGetValue(ComputeStartZ, out baseGroupZ); + } + + var localSizeX = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadX); + var localSizeY = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadY); + var localSizeZ = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadZ); + uint groupCountX; + uint groupCountY; + uint groupCountZ; + var threadCountX = uint.MaxValue; + var threadCountY = uint.MaxValue; + var threadCountZ = uint.MaxValue; + if ((initiator & useThreadDimensions) != 0) + { + // In thread-dimension mode the packet contains thread counts, not + // group end coordinates. Vulkan still dispatches whole workgroups, + // so round up and pass the exact exclusive thread bounds to the + // translated shader. Its entry guard disables invocations in the + // partially populated final group before any guest instruction. + var startThreadX = (ulong)baseGroupX * localSizeX; + var startThreadY = (ulong)baseGroupY * localSizeY; + var startThreadZ = (ulong)baseGroupZ * localSizeZ; + if ((ulong)dispatchEndX <= startThreadX || + (ulong)dispatchEndY <= startThreadY || + (ulong)dispatchEndZ <= startThreadZ) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + $"thread-end-not-after-base(" + + $"{startThreadX}x{startThreadY}x{startThreadZ})"); + } + + groupCountX = CeilDivide((ulong)dispatchEndX - startThreadX, localSizeX); + groupCountY = CeilDivide((ulong)dispatchEndY - startThreadY, localSizeY); + groupCountZ = CeilDivide((ulong)dispatchEndZ - startThreadZ, localSizeZ); + threadCountX = dispatchEndX; + threadCountY = dispatchEndY; + threadCountZ = dispatchEndZ; + } + else + { + if (dispatchEndX <= baseGroupX || + dispatchEndY <= baseGroupY || + dispatchEndZ <= baseGroupZ) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + $"end-not-after-base({baseGroupX}x{baseGroupY}x{baseGroupZ})"); + } + + groupCountX = dispatchEndX - baseGroupX; + groupCountY = dispatchEndY - baseGroupY; + groupCountZ = dispatchEndZ - baseGroupZ; + } + + if ((initiator & partialThreadGroupEnabled) != 0) + { + var partialSizeX = GetComputePartialSize(state.ShRegisters, ComputeNumThreadX); + var partialSizeY = GetComputePartialSize(state.ShRegisters, ComputeNumThreadY); + var partialSizeZ = GetComputePartialSize(state.ShRegisters, ComputeNumThreadZ); + if (partialSizeX == 0 || partialSizeX > localSizeX || + partialSizeY == 0 || partialSizeY > localSizeY || + partialSizeZ == 0 || partialSizeZ > localSizeZ) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + $"invalid-partial-size({partialSizeX}x{partialSizeY}x{partialSizeZ}/" + + $"{localSizeX}x{localSizeY}x{localSizeZ})"); + } + + if (partialSizeX != localSizeX || + partialSizeY != localSizeY || + partialSizeZ != localSizeZ) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + $"unrepresentable-partial-group({partialSizeX}x{partialSizeY}x{partialSizeZ}/" + + $"{localSizeX}x{localSizeY}x{localSizeZ})"); + } + } + + var waveLaneCount = (initiator & (1u << 15)) != 0 ? 32u : 64u; + + if (_traceAgcShader && + ((ulong)groupCountX * groupCountY * groupCountZ >= 1_000_000UL || + groupCountX >= 1_000_000u)) + { + lock (_submitTraceGate) + { + if (_tracedDispatchArguments.Add( + (dimensionsAddress, groupCountX, groupCountY, groupCountZ))) + { + TraceAgcShader( + $"agc.dispatch_args source={dispatchSource} op=0x{opcode:X2} " + + $"queue={state.QueueName} submission={state.ActiveSubmissionId} " + + $"packet=0x{packetAddress:X16} len={packetLength} " + + $"dims=0x{dimensionsAddress:X16} " + + $"raw={dispatchEndX:X8}/{dispatchEndY:X8}/{dispatchEndZ:X8} " + + $"base={baseGroupX:X8}/{baseGroupY:X8}/{baseGroupZ:X8} " + + $"count={groupCountX:X8}/{groupCountY:X8}/{groupCountZ:X8} " + + $"wave={waveLaneCount} " + + $"initiator=0x{initiator:X8} " + + $"indirect_base=0x{state.IndirectArgsAddress:X16}"); + } + } + } + + dispatch = new ComputeDispatch( + groupCountX, + groupCountY, + groupCountZ, + baseGroupX, + baseGroupY, + baseGroupZ, + waveLaneCount, + IsIndirect: opcode == ItDispatchIndirect, + threadCountX, + threadCountY, + threadCountZ); return true; } + private static uint CeilDivide(ulong value, uint divisor) => + checked((uint)((value + divisor - 1) / divisor)); + + private static bool RejectComputeDispatch( + ulong dimensionsAddress, + uint initiator, + string source, + uint rawX, + uint rawY, + uint rawZ, + string reason) + { + lock (_submitTraceGate) + { + if (_rejectedDispatchArguments.Count < 256 && + _rejectedDispatchArguments.Add((dimensionsAddress, initiator, reason))) + { + Console.Error.WriteLine( + $"[LOADER][WARN] agc.dispatch_reject source={source} " + + $"dims=0x{dimensionsAddress:X16} raw={rawX:X8}/{rawY:X8}/{rawZ:X8} " + + $"initiator=0x{initiator:X8} reason={reason}"); + } + } + + return false; + } + private static void ObserveComputeDispatch( CpuContext ctx, SubmittedGpuState gpuState, @@ -4948,17 +8486,73 @@ public static class AgcExports var localSizeX = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadX); var localSizeY = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadY); var localSizeZ = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadZ); + if (_traceComputeShaderAddress == shaderAddress) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] agc.compute_dispatch_trace seq={sequence} " + + $"cs=0x{shaderAddress:X16} " + + $"groups={dispatch.GroupCountX}x{dispatch.GroupCountY}x{dispatch.GroupCountZ} " + + $"base={dispatch.BaseGroupX}x{dispatch.BaseGroupY}x{dispatch.BaseGroupZ} " + + $"local={localSizeX}x{localSizeY}x{localSizeZ} " + + $"bindings=[{string.Join(',', descriptions)}]"); + } + + var writesGlobalMemory = evaluation.GlobalMemoryBindings.Any(static binding => + binding.Writable); var gpuDispatch = false; + var evaluationHandledByCpu = false; var computeError = string.Empty; - if ((hasStorageBinding || evaluation.GlobalMemoryBindings.Count != 0) && + if (!hasStorageBinding && + writesGlobalMemory && + TrySubmitMaskedDwordCopyKernel( + ctx, + shaderState.Program, + evaluation, + dispatch, + localSizeX, + localSizeY, + localSizeZ, + out var semanticCopySequence, + out var copyDescription)) + { + gpuDispatch = true; + evaluationHandledByCpu = true; + TraceAgcShader( + $"agc.compute_semantic_fast_path cs=0x{shaderAddress:X16} " + + $"queue={state.QueueName} submission={state.ActiveSubmissionId} " + + copyDescription); + // The scalar evaluator snapshots guest buffers while parsing the + // command stream. Do not let another submission (or the CPU) + // observe that snapshot until the semantic replacement has + // reached the same CPU-visible completion point as a translated + // writable-buffer dispatch below. Returning early here allowed + // the guest to reuse a transient heap while its delayed clear was + // still queued, so the clear could erase newly constructed CPU + // objects. Waiting on the work sequence also retires preceding + // Vulkan writes before the next evaluator snapshot is captured. + if (!VulkanVideoPresenter.WaitForGuestWork(semanticCopySequence)) + { + computeError = + $"semantic-global-write-sync-timeout sequence={semanticCopySequence}"; + } + } + else if ((hasStorageBinding || writesGlobalMemory) && (ulong)localSizeX * localSizeY * localSizeZ <= 1024) { var shaderKey = ( shaderAddress, - ComputeShaderStateFingerprint(evaluation), + _bakeScalars + ? ComputeShaderStateFingerprint(evaluation) + : ComputeShaderStructuralFingerprint(evaluation), localSizeX, localSizeY, - localSizeZ); + localSizeZ, + dispatch.WaveLaneCount, + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment); + var guestGlobalBufferCount = evaluation.GlobalMemoryBindings.Count; + var totalGlobalBufferCount = _bakeScalars + ? guestGlobalBufferCount + : guestGlobalBufferCount + 1; _computeShaderCache.TryGetValue(shaderKey, out var computeShader); if (computeShader is null && @@ -4969,7 +8563,14 @@ public static class AgcExports localSizeY, localSizeZ, out computeShader, - out computeError)) + out computeError, + totalGlobalBufferCount, + initialScalarBufferIndex: _bakeScalars + ? -1 + : guestGlobalBufferCount, + waveLaneCount: dispatch.WaveLaneCount, + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment)) { DumpCompiledShader( "cs", @@ -4988,16 +8589,32 @@ public static class AgcExports translatedBindings, out _); var globalMemoryBuffers = - CreateGuestMemoryBuffers(evaluation.GlobalMemoryBindings); - GuestGpu.Current.SubmitComputeDispatch( + CreateTranslatedComputeGlobalBuffers(evaluation); + var workSequence = GuestGpu.Current.SubmitComputeDispatch( shaderAddress, computeShader, textures, globalMemoryBuffers, dispatch.GroupCountX, dispatch.GroupCountY, - dispatch.GroupCountZ); + dispatch.GroupCountZ, + dispatch.BaseGroupX, + dispatch.BaseGroupY, + dispatch.BaseGroupZ, + localSizeX, + localSizeY, + localSizeZ, + dispatch.IsIndirect, + writesGlobalMemory, + dispatch.ThreadCountX, + dispatch.ThreadCountY, + dispatch.ThreadCountZ); gpuDispatch = true; + if (writesGlobalMemory && + !VulkanVideoPresenter.WaitForGuestWork(workSequence)) + { + computeError = $"global-write-sync-timeout sequence={workSequence}"; + } } } @@ -5007,17 +8624,324 @@ public static class AgcExports { if (_tracedComputeShaders.Add(shaderAddress)) { + var globalBuffers = evaluation.GlobalMemoryBindings.Count == 0 + ? string.Empty + : $" global_buffers=[{string.Join(',', evaluation.GlobalMemoryBindings.Select( + binding => $"0x{binding.BaseAddress:X16}:{binding.DataLength}"))}]"; + var scalarProbe = string.Join( + ',', + evaluation.InitialScalarRegisters + .Take(16) + .Select((value, index) => $"s{index}={value:X8}")); + var globalProbes = evaluation.GlobalMemoryBindings.Count == 0 + ? string.Empty + : $" global_heads=[{string.Join(',', evaluation.GlobalMemoryBindings.Select( + binding => + $"0x{binding.BaseAddress:X16}:" + + Convert.ToHexString(binding.Data.AsSpan( + 0, + Math.Min(binding.DataLength, 16)))))}]"; + var globalDescriptors = evaluation.GlobalMemoryBindings.Count == 0 + ? string.Empty + : $" global_descriptors=[{string.Join(',', evaluation.GlobalMemoryBindings.Select( + binding => + $"s{binding.ScalarAddress}=" + + string.Join(':', evaluation.ScalarRegisters + .Skip(checked((int)binding.ScalarAddress)) + .Take(4) + .Select(value => $"{value:X8}"))))}]"; + var opcodes = string.Join( + ',', + shaderState.Program.Instructions + .Select(instruction => instruction.Opcode) + .Distinct() + .Take(48)); TraceAgcShader( $"agc.compute_shader cs=0x{shaderAddress:X16} " + $"groups={dispatch.GroupCountX}x{dispatch.GroupCountY}x{dispatch.GroupCountZ} " + + $"base={dispatch.BaseGroupX}x{dispatch.BaseGroupY}x{dispatch.BaseGroupZ} " + + $"wave={dispatch.WaveLaneCount} " + $"local={localSizeX}x{localSizeY}x{localSizeZ} " + $"sys={DescribeComputeSystemRegisters(computeSystemRegisters)} " + - $"gpu={gpuDispatch} blits={blitCount} " + - $"globals={evaluation.GlobalMemoryBindings.Count}" + + $"gpu={gpuDispatch} blits={blitCount} globals={evaluation.GlobalMemoryBindings.Count} " + + $"global_writes={writesGlobalMemory}" + (computeError.Length == 0 ? string.Empty : $" error={computeError}") + + $" sgprs=[{scalarProbe}]" + + globalBuffers + + globalProbes + + globalDescriptors + + $" opcodes=[{opcodes}]" + $" bindings=[{string.Join(',', descriptions)}]"); } } + + if (evaluationHandledByCpu) + { + ReturnPooledEvaluationArrays(evaluation); + } + } + + /// + /// Recognizes the SDK's masked-dword resource initialization kernel and + /// executes its exact semantics over the guest-memory window that the + /// emulator can map. The guest dispatches this kernel over multi-gigabyte + /// virtual heaps (up to ~67 million 64-lane workgroups); translating every + /// out-of-window invocation to Vulkan dominated startup despite those + /// stores being bounds-discarded. This is a semantic kernel replacement, + /// not a generic dispatch cap: the complete instruction shape and SGPR + /// bindings must match before the ordered CPU action is used. + /// + private static bool TrySubmitMaskedDwordCopyKernel( + CpuContext ctx, + Gen5ShaderProgram program, + Gen5ShaderEvaluation evaluation, + ComputeDispatch dispatch, + uint localSizeX, + uint localSizeY, + uint localSizeZ, + out long workSequence, + out string description) + { + workSequence = 0; + description = string.Empty; + var instructions = program.Instructions; + string[] expectedOpcodes = + [ + "SMovB32", + "STtraceData", + "SInstPrefetch", + "VLshlAddU32", + "SBufferLoadDword", + "SWaitcnt", + "VCmpxGtU32", + "SCbranchExecz", + "SBufferLoadDword", + "SWaitcnt", + "VAndB32", + "BufferLoadFormatX", + "SWaitcnt", + "BufferStoreFormatX", + "SEndpgm", + ]; + if (instructions.Count != expectedOpcodes.Length || + !instructions.Select(static instruction => instruction.Opcode) + .SequenceEqual(expectedOpcodes) || + !IsExactMaskedDwordCopyInstructionShape(instructions) || + dispatch.BaseGroupX != 0 || + dispatch.BaseGroupY != 0 || + dispatch.BaseGroupZ != 0 || + dispatch.GroupCountY != 1 || + dispatch.GroupCountZ != 1 || + localSizeX != 64 || + localSizeY != 1 || + localSizeZ != 1 || + evaluation.ComputeSystemRegisters?.WorkGroupXRegister != 12) + { + return false; + } + + var control = evaluation.GlobalMemoryBindings.SingleOrDefault( + static binding => binding.ScalarAddress == 8 && !binding.Writable); + var source = evaluation.GlobalMemoryBindings.SingleOrDefault( + static binding => binding.ScalarAddress == 0 && !binding.Writable); + var destination = evaluation.GlobalMemoryBindings.SingleOrDefault( + static binding => binding.ScalarAddress == 4 && + binding.Writable && + binding.WriteBackToGuest); + if (control is null || source is null || destination is null || + control.DataLength < 2 * sizeof(uint) || + source.DataLength < sizeof(uint) || + destination.BaseAddress == 0 || + destination.DataLength < sizeof(uint) || + !IsExactMaskedDwordCopyDescriptor( + evaluation.InitialScalarRegisters, + source.ScalarAddress, + source.BaseAddress) || + !IsExactMaskedDwordCopyDescriptor( + evaluation.InitialScalarRegisters, + destination.ScalarAddress, + destination.BaseAddress)) + { + return false; + } + + var elementCount = BinaryPrimitives.ReadUInt32LittleEndian( + control.Data.AsSpan(0, sizeof(uint))); + var sourceMask = BinaryPrimitives.ReadUInt32LittleEndian( + control.Data.AsSpan(sizeof(uint), sizeof(uint))); + var dispatchedThreads = dispatch.ThreadCountX != uint.MaxValue + ? dispatch.ThreadCountX + : Math.Min( + (ulong)uint.MaxValue, + (ulong)dispatch.GroupCountX * localSizeX); + var writableDwords = (uint)(destination.DataLength / sizeof(uint)); + var outputDwords = (uint)Math.Min( + Math.Min((ulong)elementCount, dispatchedThreads), + writableDwords); + if (outputDwords == 0) + { + return false; + } + + var output = new byte[checked((int)outputDwords * sizeof(uint))]; + var outputWords = System.Runtime.InteropServices.MemoryMarshal.Cast( + output.AsSpan()); + if (sourceMask == 0) + { + outputWords.Fill(BinaryPrimitives.ReadUInt32LittleEndian( + source.Data.AsSpan(0, sizeof(uint)))); + } + else + { + var sourceWords = System.Runtime.InteropServices.MemoryMarshal.Cast( + source.Data.AsSpan(0, source.DataLength - (source.DataLength % sizeof(uint)))); + for (uint index = 0; index < outputDwords; index++) + { + var sourceIndex = index & sourceMask; + outputWords[(int)index] = sourceIndex < (uint)sourceWords.Length + ? sourceWords[(int)sourceIndex] + : 0; + } + } + + var destinationAddress = destination.BaseAddress; + workSequence = VulkanVideoPresenter.SubmitOrderedGuestAction( + () => + { + if (!ctx.Memory.TryWrite(destinationAddress, output)) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] AGC masked-copy fast path failed " + + $"dst=0x{destinationAddress:X16} bytes={output.Length}"); + return; + } + + GuestImageWriteTracker.Track( + destinationAddress, + (ulong)output.Length, + VulkanVideoPresenter.CurrentGuestWorkSequenceForDiagnostics, + "agc.masked-dword-copy"); + }, + $"masked_dword_copy dst=0x{destinationAddress:X16} bytes={output.Length}"); + description = + $"dst=0x{destinationAddress:X16} bytes={output.Length} " + + $"elements={elementCount} mask=0x{sourceMask:X8} " + + $"dispatch={dispatch.GroupCountX}x{localSizeX}"; + return workSequence > 0; + } + + private static bool IsExactMaskedDwordCopyInstructionShape( + IReadOnlyList instructions) + { + static bool IsOperand( + Gen5Operand operand, + Gen5OperandKind kind, + uint value) => + operand.Kind == kind && operand.Value == value; + + static bool IsBufferControl( + Gen5ShaderInstruction instruction, + uint vectorAddress, + uint vectorData, + uint scalarResource) => + instruction.Control is Gen5BufferMemoryControl + { + DwordCount: 1, + OffsetBytes: 0, + IndexEnabled: true, + OffsetEnabled: false, + } control && + control.VectorAddress == vectorAddress && + control.VectorData == vectorData && + control.ScalarResource == scalarResource; + + static bool IsScalarLoad( + Gen5ShaderInstruction instruction, + int offsetBytes) => + instruction.Control is Gen5ScalarMemoryControl + { + DestinationCount: 1, + DynamicOffsetRegister: null, + } control && + control.ImmediateOffsetBytes == offsetBytes && + instruction.Destinations.Count == 1 && + IsOperand( + instruction.Destinations[0], + Gen5OperandKind.ScalarRegister, + 106) && + instruction.Sources.Count >= 1 && + IsOperand( + instruction.Sources[0], + Gen5OperandKind.ScalarRegister, + 8); + + // This replacement depends on the operands as much as the opcode + // sequence. Reversing V_CMPX_GT or enabling offen on either MUBUF + // operation changes the set or address of written lanes. + var globalId = instructions[3]; + var compare = instructions[6]; + var sourceIndex = instructions[10]; + var load = instructions[11]; + var store = instructions[13]; + return + globalId.Destinations.Count == 1 && + IsOperand(globalId.Destinations[0], Gen5OperandKind.VectorRegister, 0) && + globalId.Sources.Count == 3 && + IsOperand(globalId.Sources[0], Gen5OperandKind.ScalarRegister, 12) && + IsOperand(globalId.Sources[1], Gen5OperandKind.EncodedConstant, 134) && + IsOperand(globalId.Sources[2], Gen5OperandKind.VectorRegister, 0) && + IsScalarLoad(instructions[4], offsetBytes: 0) && + compare.Sources.Count == 2 && + IsOperand(compare.Sources[0], Gen5OperandKind.ScalarRegister, 106) && + IsOperand(compare.Sources[1], Gen5OperandKind.VectorRegister, 0) && + instructions[7].Words.Count == 1 && + (instructions[7].Words[0] & 0xFFFFu) == 9 && + IsScalarLoad(instructions[8], offsetBytes: sizeof(uint)) && + sourceIndex.Destinations.Count == 1 && + IsOperand(sourceIndex.Destinations[0], Gen5OperandKind.VectorRegister, 1) && + sourceIndex.Sources.Count == 2 && + IsOperand(sourceIndex.Sources[0], Gen5OperandKind.ScalarRegister, 106) && + IsOperand(sourceIndex.Sources[1], Gen5OperandKind.VectorRegister, 0) && + IsBufferControl(load, vectorAddress: 1, vectorData: 1, scalarResource: 0) && + IsBufferControl(store, vectorAddress: 0, vectorData: 1, scalarResource: 4); + } + + private static bool IsExactMaskedDwordCopyDescriptor( + IReadOnlyList scalarRegisters, + uint scalarBase, + ulong expectedBaseAddress) + { + if (scalarBase + 3 >= scalarRegisters.Count) + { + return false; + } + + var word0 = scalarRegisters[(int)scalarBase]; + var word1 = scalarRegisters[(int)scalarBase + 1]; + var word3 = scalarRegisters[(int)scalarBase + 3]; + var baseAddress = word0 | ((ulong)(word1 & 0xFFFFu) << 32); + var stride = (word1 >> 16) & 0x3FFFu; + var cacheSwizzle = (word1 & (1u << 30)) != 0; + var swizzleEnabled = (word1 & (1u << 31)) != 0; + var unifiedFormat = (word3 >> 12) & 0x7Fu; + var addTidEnabled = (word3 & (1u << 23)) != 0; + var outOfBoundsSelect = (word3 >> 28) & 0x3u; + var type = word3 >> 30; + var dstSelectX = word3 & 0x7u; + + // RDNA2 tables 35 and 37: OOB_SELECT=0 is structured indexing, so + // NUM_RECORDS counts stride-sized records. FORMAT=20 is 32_UINT and + // dst_sel_x=4 selects its R component. ADD_TID and either swizzle bit + // alter addressing and are therefore outside this replacement. + return baseAddress == expectedBaseAddress && + stride == sizeof(uint) && + !cacheSwizzle && + !swizzleEnabled && + unifiedFormat == 20 && + !addTidEnabled && + outOfBoundsSelect == 0 && + type == 0 && + dstSelectX == 4; } private static Gen5ComputeSystemRegisters DecodeComputeSystemRegisters( @@ -5069,6 +8993,29 @@ public static class AgcExports private static uint SelectExportUserDataRegister( IReadOnlyDictionary registers) { + // RSRC2 is the authoritative stage selector: its USER_SGPR field + // describes the hardware SGPR window even when the shader has zero + // user-data dwords and therefore no USER_DATA register was written. + // GFX10 NGG export shaders use the GS user-data bank (RSRC2 at 0x8B), + // while their program address is carried in the ES/NGG registers. + // Looking only for a populated USER_DATA range made those shaders + // fall through to ES (0xCC) and reject every graphics draw because + // the unrelated ES RSRC2 register at 0xCB was legitimately absent. + if (HasShaderResource2(registers, GsUserDataRegister)) + { + return GsUserDataRegister; + } + + if (HasShaderResource2(registers, EsUserDataRegister)) + { + return EsUserDataRegister; + } + + if (HasShaderResource2(registers, VsUserDataRegister)) + { + return VsUserDataRegister; + } + if (HasUserDataRange(registers, GsUserDataRegister)) { return GsUserDataRegister; @@ -5091,6 +9038,11 @@ public static class AgcExports : EsUserDataRegister; } + private static bool HasShaderResource2( + IReadOnlyDictionary registers, + uint userDataBaseRegister) => + registers.ContainsKey(userDataBaseRegister - 1); + private static bool HasUserDataRange( IReadOnlyDictionary registers, uint startRegister) @@ -5127,10 +9079,17 @@ public static class AgcExports uint register) { return registers.TryGetValue(register, out var value) - ? Math.Max(value & 0x3FFu, 1u) + ? Math.Max(value & 0xFFFFu, 1u) : 1u; } + private static uint GetComputePartialSize( + IReadOnlyDictionary registers, + uint register) => + registers.TryGetValue(register, out var value) + ? value >> 16 + : 0u; + private static int TryApplySoftwareComputeBlits( CpuContext ctx, ulong shaderAddress, @@ -5167,10 +9126,12 @@ public static class AgcExports cachedSourceTexture.Width, cachedSourceTexture.Height, cachedSourceTexture.Format, + cachedSourceTexture.NumberType, texture.Address, texture.Width, texture.Height, - texture.Format)) + texture.Format, + texture.NumberType)) { blits++; TraceAgcShader( @@ -5362,16 +9323,6 @@ public static class AgcExports 12 => 8UL, 13 => 12UL, 14 => 16UL, - 20 => 4UL, - 22 => 8UL, - 29 => 4UL, - 36 => 1UL, - 49 => 1UL, - Gen5TextureFormatR8G8B8A8Unorm => 4UL, - 62 => 4UL, - 64 => 4UL, - Gen5TextureFormatR16G16B16A16Float => 8UL, - 75 => 8UL, _ => 0UL, }; @@ -5383,29 +9334,27 @@ public static class AgcExports return checked((ulong)width * height * bytesPerTexel); } - var blockBytes = format switch - { - 169 or 170 => 8UL, - 171 or 172 or 173 or 174 or 175 or 176 or - 177 or 178 or 179 or 180 or 181 or 182 => 16UL, - _ => 0UL, - }; + var blockBytes = (ulong)GetBlockCompressedBlockBytes(format); return blockBytes == 0 ? 0 : checked(((ulong)width + 3) / 4 * (((ulong)height + 3) / 4) * blockBytes); } - private static uint GetLinearTexturePitch(uint pitch, uint format) + private static uint GetLinearTexturePitch(uint pitch, uint height, uint format) { var bytesPerTexel = GetTextureBytesPerTexel(format); - if (bytesPerTexel == 0) + if (bytesPerTexel == 0 || height == 0) { return pitch; } - // GFX10 ADDR_SW_LINEAR aligns each row to 256 bytes. - var pitchAlignment = Math.Max(1UL, 256UL / bytesPerTexel); - return checked((uint)AlignUp(pitch, pitchAlignment)); + // GNM linear surfaces align the row pitch to 256 bytes, so a 32px + // RGBA8 texture is stored with a 64px (256-byte) pitch and a 288px + // one with 320px. Reading at the unpadded width made every padded + // tail land on the next row, which showed as transparent gaps every + // other row on small tiles and diagonal dashes on wider surfaces. + var pitchBytes = AlignUp((ulong)pitch * bytesPerTexel, 256UL); + return checked((uint)(pitchBytes / bytesPerTexel)); } private static ulong AlignUp(ulong value, ulong alignment) => @@ -5441,6 +9390,17 @@ public static class AgcExports return; } + // Translation failures are compatibility issues, not merely verbose + // shader diagnostics. Report each distinct failure once even when AGC + // tracing is disabled so normal runs preserve the missing opcode or + // unsupported translation reason needed to fix the game. + if (firstFailure) + { + Console.Error.WriteLine( + $"[COMPAT][SHADER] ps=0x{pixelShaderAddress:X16} " + + $"es=0x{exportShaderAddress:X16} error={translationError}"); + } + if ((!hasPixelShader || !hasPsInputEna || !hasPsInputAddr) && TryMarkMissingPixelShaderBindingsTrace()) { @@ -5449,6 +9409,29 @@ public static class AgcExports DescribeShaderRegisterCandidates(ctx, state.ShRegisters)); } + if (!hasPixelShader) + { + state.CxRegisters.TryGetValue(DbDepthControl, out var rawDepthControl); + state.CxRegisters.TryGetValue(DbZInfo, out var rawZInfo); + state.CxRegisters.TryGetValue(DbDepthSizeXy, out var rawDepthSize); + state.CxRegisters.TryGetValue(DbDepthView, out var rawDepthView); + var depthState = DecodeDepthState(state.CxRegisters); + var depthTarget = DecodeDepthTarget(state.CxRegisters); + TraceAgcShader( + $"agc.shader_depth_state control=0x{rawDepthControl:X8} " + + $"zinfo=0x{rawZInfo:X8} size=0x{rawDepthSize:X8} " + + $"view=0x{rawDepthView:X8} " + + $"test={(depthState.TestEnable ? 1 : 0)} " + + $"write={(depthState.WriteEnable ? 1 : 0)} " + + $"func={depthState.CompareOp} " + + (depthTarget is null + ? "target=none" + : $"target=0x{depthTarget.Address:X16}:" + + $"{depthTarget.Width}x{depthTarget.Height}:" + + $"fmt{depthTarget.GuestFormat}/sw{depthTarget.SwizzleMode}:" + + $"ro={(depthTarget.ReadOnly ? 1 : 0)}")); + } + var shaderDecode = string.Empty; if (hasExportShader && hasPixelShader) { @@ -5512,7 +9495,7 @@ public static class AgcExports TraceAgcShader( $"agc.shader_global_binding ps=0x{pixelShaderAddress:X16} " + $"saddr=s{binding.ScalarAddress} " + - $"base=0x{binding.BaseAddress:X16} bytes={binding.Data.Length} " + + $"base=0x{binding.BaseAddress:X16} bytes={binding.DataLength} " + $"pcs={string.Join(',', binding.InstructionPcs.Select(pc => $"0x{pc:X}"))}"); } @@ -5521,7 +9504,11 @@ public static class AgcExports evaluation, [new(0, 0, Gen5PixelOutputKind.Float)], out var compiledPixel, - out var compileError)) + out var compileError, + pixelInputEnable: psInputEna, + pixelInputAddress: psInputAddr, + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment)) { TraceAgcShader( $"agc.shader_spirv ps=0x{pixelShaderAddress:X16} " + @@ -5604,7 +9591,8 @@ public static class AgcExports .Take(16) .Select(candidate => { - var type = ctx.TryReadByte( + var type = TryReadByte( + ctx, candidate.Header + ShaderTypeOffset, out var shaderType) ? shaderType.ToString() @@ -5640,7 +9628,7 @@ public static class AgcExports { descriptor = default; if (packetLength < 10 || - !ctx.TryReadUInt32(packetAddress + 4, out var startRegister)) + !TryReadUInt32(ctx, packetAddress + 4, out var startRegister)) { return false; } @@ -5656,10 +9644,10 @@ public static class AgcExports packetAddress + 8 + ((ulong)(PsTextureUserDataRegister - startRegister) * sizeof(uint)); - Span fields = stackalloc uint[4]; + Span fields = stackalloc uint[8]; for (var i = 0; i < fields.Length; i++) { - if (!ctx.TryReadUInt32(descriptorAddress + ((ulong)i * sizeof(uint)), out fields[i])) + if (!TryReadUInt32(ctx, descriptorAddress + ((ulong)i * sizeof(uint)), out fields[i])) { return false; } @@ -5678,32 +9666,51 @@ public static class AgcExports return false; } - // GFX10/RDNA2 T# layout: WIDTH is split across word1[31:30] (lo 2 bits) - // and word2[11:0] (hi 12 bits); FORMAT is the combined 9-bit field at - // word1[28:20]. Verified against Kyty's decode of the same game - // descriptors (fmt=56=8_8_8_8_UNORM, extent 1280x720, sw_mode 27). - // GNM T# exposes a 38-bit baseaddr256 field, but RPCSX and the - // Demon's Souls descriptors both show that only the low 32 bits are - // part of the guest GPU VA. The upper baseaddr bits carry resource - // metadata and produce bogus addresses such as 0x00003804... if used. - var address = ((ulong)(uint)((((ulong)fields[1] << 32) | fields[0]) & 0x3F_FFFF_FFFFUL)) << 8; - var width = (((fields[1] >> 30) & 0x3u) | ((fields[2] & 0xFFFu) << 2)) + 1; - var height = ((fields[2] >> 14) & 0x3FFFu) + 1; - var format = (fields[1] >> 20) & 0x1FFu; - var numberType = (fields[1] >> 26) & 0xFu; + // RDNA2 ISA table 45: BASE_ADDRESS is addr[47:8], WIDTH is the full + // 16-bit field split across word1/word2, and HEIGHT is word2[29:14]. + // Keeping the high base byte is required for legal guest VAs above + // 1 TiB; it is not descriptor metadata. + var address = (((ulong)(fields[1] & 0xFFu) << 32) | fields[0]) << 8; + var width = (((fields[1] >> 30) & 0x3u) | ((fields[2] & 0x3FFFu) << 2)) + 1; + var height = ((fields[2] >> 14) & 0xFFFFu) + 1; + var unifiedFormat = (fields[1] >> 20) & 0x1FFu; + if (unifiedFormat == 0 || + !Gfx10UnifiedFormat.TryDecode( + unifiedFormat, + out var format, + out var numberType)) + { + return false; + } var tileMode = (fields[3] >> 20) & 0x1Fu; var type = (fields[3] >> 28) & 0xFu; var baseLevel = (fields[3] >> 12) & 0xFu; var lastLevel = (fields[3] >> 16) & 0xFu; - // The 128-bit RDNA2 2D resource derives pitch[12:0] from width; - // the optional extension word only supplies pitch[13]. - var pitch = width; - if (fields.Count >= 5) - { - pitch = ((((width - 1) & 0x1FFFu) | (((fields[4] >> 13) & 1u) << 13)) + 1); - } + var bcSwizzle = (fields[3] >> 25) & 0x7u; + var hasExtendedDescriptor = fields.Count >= 8; + var word4 = fields.Count >= 5 ? fields[4] : 0u; + var depthOrLastSlice = (word4 & 0x1FFFu) + 1; + var baseArray = (word4 >> 16) & 0x1FFFu; + // In a 256-bit 1D/2D/2D-MSAA descriptor word4[13:0] is + // (pitch-1). A zeroed upper half denotes the common 128-bit resource, + // where pitch is implicit; use width rather than inventing pitch=1. + var pitch = type is 8u or 9u or 14u && word4 != 0 + ? (word4 & 0x3FFFu) + 1 + : width; + var depth = type is 10u or 11u or 12u or 13u or 15u + ? depthOrLastSlice + : 1u; + var word5 = fields.Count >= 6 ? fields[5] : 0u; + var arrayPitch = word5 & 0xFu; + var maxMip = (word5 >> 4) & 0xFu; + var minLod = (fields[1] >> 8) & 0xFFFu; + var minLodWarn = (word5 >> 8) & 0xFFFu; + var word6 = fields.Count >= 7 ? fields[6] : 0u; + var word7 = fields.Count >= 8 ? fields[7] : 0u; + var metadataAddress = ((((ulong)word7 << 8) | (word6 >> 24)) << 8); + var descriptorFlags = word6 & 0x00FF_FFFFu; var dstSelect = fields[3] & 0xFFFu; - if (address == 0 || width == 0 || height == 0) + if (address == 0 || width == 0 || height == 0 || type is >= 1 and <= 7) { return false; } @@ -5719,7 +9726,17 @@ public static class AgcExports baseLevel, lastLevel, pitch, - dstSelect); + dstSelect, + depth, + baseArray, + arrayPitch, + maxMip, + minLod, + minLodWarn, + bcSwizzle, + metadataAddress, + descriptorFlags, + hasExtendedDescriptor); return true; } @@ -5730,8 +9747,15 @@ public static class AgcExports var tileMode = 0u; if (fields.Count >= 4) { - format = (fields[1] >> 20) & 0x1FFu; - numberType = (fields[1] >> 26) & 0xFu; + var unifiedFormat = (fields[1] >> 20) & 0x1FFu; + if (!Gfx10UnifiedFormat.TryDecode( + unifiedFormat, + out format, + out numberType)) + { + format = Gen5TextureFormatR8G8B8A8Unorm; + numberType = 0; + } tileMode = (fields[3] >> 20) & 0x1Fu; if (format == 0) { @@ -5774,8 +9798,14 @@ public static class AgcExports destination.PixelFormat is not ( VideoOutPixelFormatA8R8G8B8Srgb or VideoOutPixelFormatA8B8G8R8Srgb or - VideoOutPixelFormatB8G8R8A8Unorm or - VideoOutPixelFormatR8G8B8A8Unorm)) + VideoOutPixelFormat2R8G8B8A8Srgb or + VideoOutPixelFormat2B8G8R8A8Srgb or + VideoOutPixelFormat2R10G10B10A2 or + VideoOutPixelFormat2B10G10R10A2 or + VideoOutPixelFormat2R10G10B10A2Srgb or + VideoOutPixelFormat2B10G10R10A2Srgb or + VideoOutPixelFormat2R10G10B10A2Bt2100Pq or + VideoOutPixelFormat2B10G10R10A2Bt2100Pq)) { return false; } @@ -5814,7 +9844,9 @@ public static class AgcExports var destinationRow = new byte[checked((int)destinationPitch * 4)]; var rgbaDestination = destination.PixelFormat is VideoOutPixelFormatA8B8G8R8Srgb or - VideoOutPixelFormatR8G8B8A8Unorm; + VideoOutPixelFormat2R8G8B8A8Srgb; + var packed10Destination = + VideoOutExports.IsPacked10BitPixelFormat(destination.PixelFormat); for (uint y = 0; y < destination.Height; y++) { var sourceY = (uint)(((ulong)y * source.Height) / destination.Height); @@ -5823,7 +9855,24 @@ public static class AgcExports var sourceX = (uint)(((ulong)x * source.Width) / destination.Width); var sourceOffset = checked((int)(((ulong)sourceY * source.Width + sourceX) * 4)); var destinationOffset = checked((int)x * 4); - if (rgbaDestination) + if (packed10Destination) + { + if (!VideoOutExports.TryPackRgba8Pixel( + destination.PixelFormat, + sourceBytes[sourceOffset + 0], + sourceBytes[sourceOffset + 1], + sourceBytes[sourceOffset + 2], + sourceBytes[sourceOffset + 3], + out var packed)) + { + return false; + } + + BinaryPrimitives.WriteUInt32LittleEndian( + destinationRow.AsSpan(destinationOffset, sizeof(uint)), + packed); + } + else if (rgbaDestination) { destinationRow[destinationOffset + 0] = sourceBytes[sourceOffset + 0]; destinationRow[destinationOffset + 1] = sourceBytes[sourceOffset + 1]; @@ -5836,7 +9885,10 @@ public static class AgcExports destinationRow[destinationOffset + 2] = sourceBytes[sourceOffset + 0]; } - destinationRow[destinationOffset + 3] = sourceBytes[sourceOffset + 3]; + if (!packed10Destination) + { + destinationRow[destinationOffset + 3] = sourceBytes[sourceOffset + 3]; + } } var destinationAddress = destination.Address + ((ulong)y * destinationPitch * 4); @@ -5886,7 +9938,7 @@ public static class AgcExports var payloadCount = Math.Min(length - 1, 32u); for (uint i = 0; i < payloadCount; i++) { - if (!ctx.TryReadUInt32(packetAddress + ((ulong)(i + 1) * sizeof(uint)), out var value)) + if (!TryReadUInt32(ctx, packetAddress + ((ulong)(i + 1) * sizeof(uint)), out var value)) { return; } @@ -5897,8 +9949,8 @@ public static class AgcExports if (op != ItNop || register is not (RCxRegsIndirect or RShRegsIndirect or RUcRegsIndirect) || length < 4 || - !ctx.TryReadUInt32(packetAddress + 4, out var registerCount) || - !ctx.TryReadUInt64(packetAddress + 8, out var registersAddress)) + !TryReadUInt32(ctx, packetAddress + 4, out var registerCount) || + !TryReadUInt64(ctx, packetAddress + 8, out var registersAddress)) { return; } @@ -5909,8 +9961,8 @@ public static class AgcExports for (uint i = 0; i < tracedCount; i++) { var entryAddress = registersAddress + ((ulong)i * 8); - if (!ctx.TryReadUInt32(entryAddress, out var registerOffset) || - !ctx.TryReadUInt32(entryAddress + 4, out var value)) + if (!TryReadUInt32(ctx, entryAddress, out var registerOffset) || + !TryReadUInt32(ctx, entryAddress + 4, out var value)) { TraceAgc($"agc.dcb.indirect_read_failed space={registerSpace} index={i} addr=0x{entryAddress:X16}"); return; @@ -5927,9 +9979,9 @@ public static class AgcExports private static bool PatchShaderProgramRegisters(CpuContext ctx, ulong headerAddress, ulong codeAddress) { - if (!ctx.TryReadUInt64(headerAddress + ShaderShRegistersOffset, out var shRegistersAddress) || - !ctx.TryReadByte(headerAddress + ShaderTypeOffset, out var shaderType) || - !ctx.TryReadByte(headerAddress + ShaderNumShRegistersOffset, out var registerCount)) + if (!TryReadUInt64(ctx, headerAddress + ShaderShRegistersOffset, out var shRegistersAddress) || + !TryReadByte(ctx, headerAddress + ShaderTypeOffset, out var shaderType) || + !TryReadByte(ctx, headerAddress + ShaderNumShRegistersOffset, out var registerCount)) { return false; } @@ -5939,8 +9991,8 @@ public static class AgcExports return false; } - if (!ctx.TryReadUInt32(shRegistersAddress, out var loRegister) || - !ctx.TryReadUInt32(shRegistersAddress + 8, out var hiRegister)) + if (!TryReadUInt32(ctx, shRegistersAddress, out var loRegister) || + !TryReadUInt32(ctx, shRegistersAddress + 8, out var hiRegister)) { return false; } @@ -5971,12 +10023,12 @@ public static class AgcExports var loValue = (uint)((codeAddress >> 8) & 0xFFFF_FFFFUL); var hiValue = (uint)((codeAddress >> 40) & 0xFFUL); - return ctx.TryWriteUInt32(shRegistersAddress + sizeof(uint), loValue) && - ctx.TryWriteUInt32(shRegistersAddress + 8 + sizeof(uint), hiValue); + return TryWriteUInt32(ctx, shRegistersAddress + sizeof(uint), loValue) && + TryWriteUInt32(ctx, shRegistersAddress + 8 + sizeof(uint), hiValue); } private static bool IsEsGeometryShaderType(byte shaderType) => - shaderType is 2 or 4 or 6; + shaderType is 2 or 6; private static int SetIndirectPatchAddress(CpuContext ctx, string registerSpace) { @@ -5984,13 +10036,13 @@ public static class AgcExports var registersAddress = ctx[CpuRegister.Rsi]; if (commandAddress == 0 || registersAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryWriteUInt32(commandAddress + 8, (uint)(registersAddress & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(registersAddress >> 32))) + if (!TryWriteUInt32(ctx, commandAddress + 8, (uint)(registersAddress & 0xFFFF_FFFFUL)) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(registersAddress >> 32))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceAgc($"agc.patch_{registerSpace}_addr cmd=0x{commandAddress:X16} regs=0x{registersAddress:X16}"); @@ -5998,19 +10050,75 @@ public static class AgcExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + private static int PatchWriteDataControlByte(CpuContext ctx, int byteIndex) + { + if (!TryResolveWriteDataPatchArguments( + ctx, + ctx[CpuRegister.Rdi], + ctx[CpuRegister.Rsi], + out var commandAddress, + out var value) || + !TryReadUInt32(ctx, commandAddress + 4, out var control)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var shift = byteIndex * 8; + var patchedControl = (control & ~(0xFFu << shift)) | (((uint)value & 0xFFu) << shift); + return TryWriteUInt32(ctx, commandAddress + 4, patchedControl) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + private static bool TryResolveWriteDataPatchArguments( + CpuContext ctx, + ulong first, + ulong second, + out ulong commandAddress, + out ulong value) + { + if (IsWriteDataPacket(ctx, first)) + { + commandAddress = first; + value = second; + return true; + } + + if (IsWriteDataPacket(ctx, second)) + { + commandAddress = second; + value = first; + return true; + } + + commandAddress = 0; + value = 0; + return false; + } + + private static bool IsWriteDataPacket(CpuContext ctx, ulong commandAddress) + { + if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) + { + return false; + } + + return op == ItWriteData || (op == ItNop && register == RWriteData); + } + private static int AddIndirectPatchRegisters(CpuContext ctx, string registerSpace) { var commandAddress = ctx[CpuRegister.Rdi]; var registerCount = (uint)ctx[CpuRegister.Rsi]; if (commandAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadUInt32(commandAddress + 4, out var currentCount) || - !ctx.TryWriteUInt32(commandAddress + 4, currentCount + registerCount)) + if (!TryReadUInt32(ctx, commandAddress + 4, out var currentCount) || + !TryWriteUInt32(ctx, commandAddress + 4, currentCount + registerCount)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceAgc($"agc.patch_{registerSpace}_add cmd=0x{commandAddress:X16} add={registerCount} total={currentCount + registerCount}"); @@ -6029,10 +10137,10 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(4, ItNop, packetRegister)) || - !ctx.TryWriteUInt32(commandAddress + 4, registerCount) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(registersAddress & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(registersAddress >> 32))) + !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItNop, packetRegister)) || + !TryWriteUInt32(ctx, commandAddress + 4, registerCount) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(registersAddress & 0xFFFF_FFFFUL)) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(registersAddress >> 32))) { return ReturnPointer(ctx, 0); } @@ -6045,22 +10153,55 @@ public static class AgcExports { commandAddress = 0; if (sizeDwords == 0 || - !ctx.TryReadUInt64(commandBufferAddress + CommandBufferCursorUpOffset, out var cursorUp) || - !ctx.TryReadUInt64(commandBufferAddress + CommandBufferCursorDownOffset, out var cursorDown) || - !ctx.TryReadUInt64(commandBufferAddress + CommandBufferCallbackOffset, out var callback) || - !ctx.TryReadUInt32(commandBufferAddress + CommandBufferReservedDwOffset, out var reservedDwords)) + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorUpOffset, out var cursorUp) || + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorDownOffset, out var cursorDown) || + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCallbackOffset, out var callback) || + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferUserDataOffset, out var userData) || + !TryReadUInt32(ctx, commandBufferAddress + CommandBufferReservedDwOffset, out var reservedDwords)) { return false; } - var availableDwords = cursorDown >= cursorUp - ? Math.Min((cursorDown - cursorUp) / sizeof(uint), uint.MaxValue) - : 0; - var remainingDwords = (uint)Math.Max(availableDwords, reservedDwords) - reservedDwords; + var remainingDwords = GetRemainingCommandDwords(cursorUp, cursorDown, reservedDwords); if (sizeDwords > remainingDwords) { TraceAgc($"agc.cmd_alloc_full buf=0x{commandBufferAddress:X16} need={sizeDwords} remaining={remainingDwords} callback=0x{callback:X16}"); - return false; + var scheduler = GuestThreadExecution.Scheduler; + ulong callbackResult = 0; + string? callbackError = null; + if (callback == 0 || + scheduler is null || + !scheduler.TryCallGuestFunction( + ctx, + callback, + commandBufferAddress, + (ulong)sizeDwords + reservedDwords, + userData, + 0, + 0, + "agc_command_buffer_full", + out callbackResult, + out callbackError)) + { + TraceAgc( + $"agc.cmd_alloc_callback_failed buf=0x{commandBufferAddress:X16} " + + $"callback=0x{callback:X16} result=0x{callbackResult:X16} " + + $"error={callbackError ?? "none"}"); + return false; + } + + TraceAgc( + $"agc.cmd_alloc_callback_complete buf=0x{commandBufferAddress:X16} " + + $"callback=0x{callback:X16} result=0x{callbackResult:X16}"); + + if (!TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorUpOffset, out cursorUp) || + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorDownOffset, out cursorDown) || + !TryReadUInt32(ctx, commandBufferAddress + CommandBufferReservedDwOffset, out reservedDwords) || + sizeDwords > GetRemainingCommandDwords(cursorUp, cursorDown, reservedDwords)) + { + TraceAgc($"agc.cmd_alloc_callback_no_space buf=0x{commandBufferAddress:X16} need={sizeDwords}"); + return false; + } } var nextCursor = cursorUp + ((ulong)sizeDwords * sizeof(uint)); @@ -6073,21 +10214,34 @@ public static class AgcExports return true; } + private static uint GetRemainingCommandDwords( + ulong cursorUp, + ulong cursorDown, + uint reservedDwords) + { + var availableDwords = cursorDown >= cursorUp + ? Math.Min((cursorDown - cursorUp) / sizeof(uint), uint.MaxValue) + : 0; + return availableDwords > reservedDwords + ? (uint)availableDwords - reservedDwords + : 0; + } + private static bool CopyShaderRegister(CpuContext ctx, ulong sourceAddress, ulong destinationAddress) { - if (!ctx.TryReadUInt32(sourceAddress, out var offset) || - !ctx.TryReadUInt32(sourceAddress + sizeof(uint), out var value)) + if (!TryReadUInt32(ctx, sourceAddress, out var offset) || + !TryReadUInt32(ctx, sourceAddress + sizeof(uint), out var value)) { return false; } - return ctx.TryWriteUInt32(destinationAddress, offset) && - ctx.TryWriteUInt32(destinationAddress + sizeof(uint), value); + return TryWriteUInt32(ctx, destinationAddress, offset) && + TryWriteUInt32(ctx, destinationAddress + sizeof(uint), value); } private static bool RelocatePointerField(CpuContext ctx, ulong fieldAddress) { - if (!ctx.TryReadUInt64(fieldAddress, out var relativeAddress)) + if (!TryReadUInt64(ctx, fieldAddress, out var relativeAddress)) { return false; } @@ -6254,6 +10408,12 @@ public static class AgcExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)(int)result); + return (int)result; + } + private static uint Pm4(uint lengthDwords, uint op, uint register) => 0xC0000000u | ((((ushort)lengthDwords - 2u) & 0x3FFFu) << 16) | @@ -6263,6 +10423,102 @@ public static class AgcExports private static uint Pm4Length(uint header) => ((header >> 16) & 0x3FFFu) + 2u; + private static bool TryReadByte(CpuContext ctx, ulong address, out byte value) + { + Span buffer = stackalloc byte[1]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = buffer[0]; + return true; + } + + // A submitted command buffer is bulk-copied once per submit and served + // from this thread-local window: the previous per-dword reads each took + // the guest-memory reader lock and ran a region binary search, which + // dominated submit parsing (thousands of locked 4-byte reads per DCB). + [ThreadStatic] + private static byte[]? _dcbWindowBuffer; + [ThreadStatic] + private static ulong _dcbWindowStart; + [ThreadStatic] + private static int _dcbWindowByteLength; + + /// + /// Drops the bulk-read window when a self-patching command buffer writes + /// into its own bytes during parse, so subsequent reads see live guest + /// memory instead of the pre-write snapshot. Self-patching is rare, so + /// paying live-read cost for the rest of that one submit is acceptable. + /// + private static void InvalidateDcbWindowIfOverlaps(ulong address, ulong length) + { + if (_dcbWindowBuffer is null || length == 0) + { + return; + } + + var windowEnd = _dcbWindowStart + (ulong)_dcbWindowByteLength; + if (address < windowEnd && address + length > _dcbWindowStart) + { + _dcbWindowBuffer = null; + _dcbWindowByteLength = 0; + } + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + if (_dcbWindowBuffer is { } window && + address >= _dcbWindowStart && + address - _dcbWindowStart + sizeof(uint) <= (ulong)_dcbWindowByteLength) + { + value = BinaryPrimitives.ReadUInt32LittleEndian( + window.AsSpan((int)(address - _dcbWindowStart))); + return true; + } + + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) + { + if (_dcbWindowBuffer is { } window && + address >= _dcbWindowStart && + address - _dcbWindowStart + sizeof(ulong) <= (ulong)_dcbWindowByteLength) + { + value = BinaryPrimitives.ReadUInt64LittleEndian( + window.AsSpan((int)(address - _dcbWindowStart))); + return true; + } + + Span buffer = stackalloc byte[sizeof(ulong)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(buffer); + return true; + } + private static bool TryReadGuestCString( CpuContext ctx, ulong address, @@ -6278,7 +10534,7 @@ public static class AgcExports var values = new List(Math.Min(maximumLength, 128)); for (var index = 0; index < maximumLength; index++) { - if (!ctx.TryReadByte(address + (ulong)index, out var value)) + if (!TryReadByte(ctx, address + (ulong)index, out var value)) { bytes = []; return false; @@ -6305,7 +10561,7 @@ public static class AgcExports { op = 0; register = 0; - if (commandAddress == 0 || !ctx.TryReadUInt32(commandAddress, out var header)) + if (commandAddress == 0 || !TryReadUInt32(ctx, commandAddress, out var header)) { return false; } @@ -6384,6 +10640,62 @@ public static class AgcExports return count <= 8 || count % 100_000 == 0; } + // Interpolated-string handlers gated on the trace flags: when tracing is + // off (the normal case) the compiler skips every AppendFormatted call, so + // the interpolation never runs. These functions are on the hottest guest + // paths — e.g. AddIndirectPatchRegisters fires tens of thousands of times + // per second — and previously formatted a discarded string every call. + [System.Runtime.CompilerServices.InterpolatedStringHandler] + private ref struct AgcTraceHandler + { + private System.Runtime.CompilerServices.DefaultInterpolatedStringHandler _inner; + private readonly bool _enabled; + + public AgcTraceHandler(int literalLength, int formattedCount, out bool shouldAppend) + { + _enabled = _traceAgc; + shouldAppend = _enabled; + _inner = _enabled + ? new System.Runtime.CompilerServices.DefaultInterpolatedStringHandler(literalLength, formattedCount) + : default; + } + + public void AppendLiteral(string value) => _inner.AppendLiteral(value); + public void AppendFormatted(T value) => _inner.AppendFormatted(value); + public void AppendFormatted(T value, string? format) => _inner.AppendFormatted(value, format); + public string ToStringAndClear() => _enabled ? _inner.ToStringAndClear() : string.Empty; + } + + [System.Runtime.CompilerServices.InterpolatedStringHandler] + private ref struct AgcShaderTraceHandler + { + private System.Runtime.CompilerServices.DefaultInterpolatedStringHandler _inner; + private readonly bool _enabled; + + public AgcShaderTraceHandler(int literalLength, int formattedCount, out bool shouldAppend) + { + _enabled = _traceAgcShader; + shouldAppend = _enabled; + _inner = _enabled + ? new System.Runtime.CompilerServices.DefaultInterpolatedStringHandler(literalLength, formattedCount) + : default; + } + + public void AppendLiteral(string value) => _inner.AppendLiteral(value); + public void AppendFormatted(T value) => _inner.AppendFormatted(value); + public void AppendFormatted(T value, string? format) => _inner.AppendFormatted(value, format); + public string ToStringAndClear() => _enabled ? _inner.ToStringAndClear() : string.Empty; + } + + private static void TraceAgc( + [System.Runtime.CompilerServices.InterpolatedStringHandlerArgument] ref AgcTraceHandler message) + { + if (_traceAgc) + { + Console.Error.WriteLine($"[LOADER][TRACE] {message.ToStringAndClear()}"); + } + } + private static void TraceAgc(string message) { if (!_traceAgc) @@ -6394,6 +10706,15 @@ public static class AgcExports Console.Error.WriteLine($"[LOADER][TRACE] {message}"); } + private static void TraceAgcShader( + [System.Runtime.CompilerServices.InterpolatedStringHandlerArgument] ref AgcShaderTraceHandler message) + { + if (_traceAgcShader) + { + Console.Error.WriteLine($"[LOADER][TRACE] {message.ToStringAndClear()}"); + } + } + private static void TraceAgcShader(string message) { if (!_traceAgcShader) @@ -6412,8 +10733,34 @@ public static class AgcExports private static string FormatTextureDescriptor(TextureDescriptor descriptor) => $"addr=0x{descriptor.Address:X16} {descriptor.Width}x{descriptor.Height} " + $"fmt={descriptor.Format} num={descriptor.NumberType} tile={descriptor.TileMode} " + - $"type={descriptor.Type} levels={descriptor.BaseLevel}-{descriptor.LastLevel} " + - $"pitch={descriptor.Pitch} dst=0x{descriptor.DstSelect:X3}"; + $"type={descriptor.Type} depth={descriptor.Depth} base_array={descriptor.BaseArray} " + + $"levels={descriptor.BaseLevel}-{descriptor.LastLevel}/max{descriptor.MaxMip} " + + $"pitch={descriptor.Pitch} array_pitch={descriptor.ArrayPitch} " + + $"lod={descriptor.MinLod:X3}/{descriptor.MinLodWarn:X3} " + + $"bc={descriptor.BcSwizzle} meta=0x{descriptor.MetadataAddress:X16} " + + $"flags=0x{descriptor.DescriptorFlags:X6} dst=0x{descriptor.DstSelect:X3}"; + + private static ulong? ParseOptionalHexAddress(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var span = value.AsSpan().Trim(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) + ? address + : null; + } private static void DumpCompiledShader( string stage, @@ -6431,6 +10778,27 @@ public static class AgcExports return; } + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_DUMP_SPIRV_ADDRESS"); + if (!string.IsNullOrWhiteSpace(addressFilter)) + { + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + if (!ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var filteredAddress) || + shaderAddress != filteredAddress) + { + return; + } + } + var directory = Path.Combine(AppContext.BaseDirectory, "shader-dumps"); Directory.CreateDirectory(directory); var name = $"{shaderAddress:X16}-{stateFingerprint:X16}.{stage}"; @@ -6471,85 +10839,297 @@ public static class AgcExports $"[LOADER][TRACE] agc.create_shader dst=0x{destinationAddress:X16} header=0x{headerAddress:X16} code=0x{codeAddress:X16} {detail}"); } - // Packet layouts mirror what DcbWaitRegMem emits. - // 32-bit (ItNop/RWaitMem32): +4 addrLo, +8 addrHi, +12 mask, +16 cmp|op<<8, +20 ref. - // 64-bit (ItNop/RWaitMem64): +4 addrLo, +8 addrHi, +12 maskLo, +16 maskHi, - // +20 refLo, +24 refHi, +28 cmp|op<<8, +32 poll. - private static bool TryParseWaitRegMem( - CpuContext ctx, - ulong addr, - bool is64, - out ulong waitAddr, - out ulong refVal, - out ulong mask, - out uint cmpFunc) + [SysAbiExport( + Nid = "xSAR0LTcRKM", + ExportName = "sceAgcDcbJump", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbJump(CpuContext ctx) { - waitAddr = refVal = mask = 0; - cmpFunc = 0; - - if (!ctx.TryReadUInt32(addr + 4, out var lo) || - !ctx.TryReadUInt32(addr + 8, out var hi)) + var dcb = ctx[CpuRegister.Rdi]; + var target = ctx[CpuRegister.Rsi]; + var sizeDwords = (uint)ctx[CpuRegister.Rdx]; + if (dcb == 0) { - return false; + return ReturnPointer(ctx, 0); } - waitAddr = ((ulong)hi << 32) | lo; - if (!is64) + if (!TryAllocateCommandDwords(ctx, dcb, 4, out var cmd) || + !ctx.TryWriteUInt32(cmd, Pm4(4, ItIndirectBuffer, RZero)) || + !ctx.TryWriteUInt32(cmd + 4, (uint)(target & 0xFFFF_FFFFUL)) || + !ctx.TryWriteUInt32(cmd + 8, (uint)((target >> 32) & 0xFFFFUL)) || + !ctx.TryWriteUInt32(cmd + 12, sizeDwords & 0xFFFFF)) { - if (!ctx.TryReadUInt32(addr + 12, out var mask32) || - !ctx.TryReadUInt32(addr + 16, out var cmpRaw) || - !ctx.TryReadUInt32(addr + 20, out var refVal32)) + return ReturnPointer(ctx, 0); + } + + return ReturnPointer(ctx, cmd); + } + + [SysAbiExport( + Nid = "bbFueFP+J4k", + ExportName = "sceAgcDcbSetPredication", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbSetPredication(CpuContext ctx) + { + var dcb = ctx[CpuRegister.Rdi]; + var address = ctx[CpuRegister.Rsi]; + if (dcb == 0) + { + return ReturnPointer(ctx, 0); + } + + if (!TryAllocateCommandDwords(ctx, dcb, 3, out var cmd) || + !ctx.TryWriteUInt32(cmd, Pm4(3, ItNop, RZero)) || + !ctx.TryWriteUInt32(cmd + 4, (uint)(address & 0xFFFF_FFFFUL)) || + !ctx.TryWriteUInt32(cmd + 8, (uint)(address >> 32))) + { + return ReturnPointer(ctx, 0); + } + + return ReturnPointer(ctx, cmd); + } + + [SysAbiExport( + Nid = "w6Dj1VJt5qY", + ExportName = "sceAgcSetPacketPredication", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int SetPacketPredication(CpuContext ctx) + { + // Global predication toggle on a packet; a no-op is safe for rendering. + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ABI (reversed from Quake): rdi = array of DCB base addresses (u64 each), + // rsi = array of DCB sizes in dwords (u32 each), rdx = buffer count. + [SysAbiExport( + Nid = "6UzEidRZwkg", + ExportName = "sceAgcDriverSubmitMultiDcbs", + Target = Generation.Gen5, + LibraryName = "libSceAgcDriver")] + public static int DriverSubmitMultiDcbs(CpuContext ctx) + { + var addressArray = ctx[CpuRegister.Rdi]; + var sizeArray = ctx[CpuRegister.Rsi]; + var bufferCount = (uint)ctx[CpuRegister.Rdx]; + if (addressArray == 0 || sizeArray == 0 || bufferCount == 0 || bufferCount > 4096) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var tracePackets = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal); + + var gpuState = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + lock (gpuState.Gate) + { + Gen5ShaderScalarEvaluator.BeginGlobalMemoryReadScope(); + try { - return false; + for (uint i = 0; i < bufferCount; i++) + { + if (!ctx.TryReadUInt64(addressArray + i * 8, out var commandAddress) || + commandAddress == 0 || + !ctx.TryReadUInt32(sizeArray + i * 4, out var dwordCount) || + dwordCount == 0) + { + continue; + } + + if (tracePackets) + { + TraceAgc( + $"agc.driver_submit_multi_dcbs index={i}/{bufferCount} " + + $"addr=0x{commandAddress:X16} dwords={dwordCount}"); + } + + ParseSubmittedDcb(ctx, gpuState, gpuState.Graphics, commandAddress, dwordCount, tracePackets); + } + + DrainResumableDcbs(ctx, gpuState, tracePackets); + } + finally + { + Gen5ShaderScalarEvaluator.EndGlobalMemoryReadScope(); + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "AOLcoIkQDgM", + ExportName = "sceAgcDriverQueryResourceRegistrationUserMemoryRequirements", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverQueryResourceRegistrationUserMemoryRequirements(CpuContext ctx) + { + var sizeAddress = ctx[CpuRegister.Rdi]; + var resourceCount = ctx[CpuRegister.Rsi]; + var ownerCount = ctx[CpuRegister.Rdx]; + if (sizeAddress == 0 || resourceCount == 0 || ownerCount == 0) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + ulong requiredSize; + try + { + requiredSize = checked( + resourceCount * ResourceRegistrationBytesPerResource + + ownerCount * ResourceRegistrationBytesPerOwner); + } + catch (OverflowException) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + if (!ctx.TryWriteUInt64(sizeAddress, requiredSize)) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + TraceAgc( + $"agc.driver_query_resource_registration_memory resources={resourceCount} " + + $"owners={ownerCount} bytes=0x{requiredSize:X}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + } + + [SysAbiExport( + Nid = "F0Y42t-3e18", + ExportName = "sceAgcDriverInitResourceRegistration", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverInitResourceRegistration(CpuContext ctx) + { + var memoryAddress = ctx[CpuRegister.Rdi]; + var memorySize = ctx[CpuRegister.Rsi]; + var ownerCount = ctx[CpuRegister.Rdx]; + if (memoryAddress == 0 || memorySize == 0 || ownerCount == 0 || ownerCount > uint.MaxValue) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + lock (state.Gate) + { + state.ResourceRegistrationInitialized = true; + state.ResourceRegistrationMemory = memoryAddress; + state.ResourceRegistrationMemorySize = memorySize; + state.ResourceRegistrationMaxOwners = (uint)ownerCount; + state.ResourceOwners.Clear(); + state.RegisteredResources.Clear(); + state.DefaultOwner = DefaultAgcOwner; + state.NextOwner = 1; + state.NextResource = 1; + } + + TraceAgc( + $"agc.driver_init_resource_registration memory=0x{memoryAddress:X16} " + + $"bytes=0x{memorySize:X} owners={ownerCount}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + } + + [SysAbiExport( + Nid = "U9ueyEhSkF4", + ExportName = "sceAgcDriverRegisterDefaultOwner", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverRegisterDefaultOwner(CpuContext ctx) + { + var owner = (uint)ctx[CpuRegister.Rdi]; + var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + lock (state.Gate) + { + state.DefaultOwner = owner; + } + + TraceAgc($"agc.driver_register_default_owner owner={owner}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + } + + [SysAbiExport( + Nid = "X-Nm5KLREeg", + ExportName = "sceAgcDriverRegisterOwner", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverRegisterOwner(CpuContext ctx) + { + var ownerAddress = ctx[CpuRegister.Rdi]; + var nameAddress = ctx[CpuRegister.Rsi]; + if (ownerAddress == 0 || nameAddress == 0 || + !TryReadGuestCString( + ctx, + nameAddress, + ResourceRegistrationMaxNameLength, + out var nameBytes)) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + uint owner; + lock (state.Gate) + { + if (!state.ResourceRegistrationInitialized || + state.ResourceRegistrationMaxOwners != 0 && + state.ResourceOwners.Count >= state.ResourceRegistrationMaxOwners) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - mask = mask32; - refVal = refVal32; - cmpFunc = cmpRaw & 0x7; - return true; + owner = state.NextOwner; + while (owner == state.DefaultOwner || state.ResourceOwners.ContainsKey(owner)) + { + owner++; + if (owner == 0) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + } + + state.NextOwner = owner + 1; + state.ResourceOwners.Add(owner, System.Text.Encoding.UTF8.GetString(nameBytes)); } - if (!ctx.TryReadUInt32(addr + 12, out var maskLo) || - !ctx.TryReadUInt32(addr + 16, out var maskHi) || - !ctx.TryReadUInt32(addr + 20, out var refLo) || - !ctx.TryReadUInt32(addr + 24, out var refHi) || - !ctx.TryReadUInt32(addr + 28, out var cmpRaw64)) + if (!ctx.TryWriteUInt32(ownerAddress, owner)) { - return false; + lock (state.Gate) + { + state.ResourceOwners.Remove(owner); + } + + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - mask = ((ulong)maskHi << 32) | maskLo; - refVal = ((ulong)refHi << 32) | refLo; - cmpFunc = cmpRaw64 & 0x7; - return true; + TraceAgc( + $"agc.driver_register_owner out=0x{ownerAddress:X16} owner={owner} " + + $"name={System.Text.Encoding.UTF8.GetString(nameBytes)}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } - // Standard ItWaitRegMem (7 dwords, 32-bit compare): - // +4 cmp|(op&1)<<8, +8 addrLo, +12 addrHi, +16 ref, +20 mask, +24 poll. - private static bool TryParseStandardWaitRegMem( - CpuContext ctx, - ulong addr, - out ulong waitAddr, - out ulong refVal, - out ulong mask, - out uint cmpFunc) + [SysAbiExport( + Nid = "pWLG7WOpVcw", + ExportName = "sceAgcDriverUnregisterResource", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverUnregisterResource(CpuContext ctx) { - waitAddr = refVal = mask = 0; - cmpFunc = 0; - - if (!ctx.TryReadUInt32(addr + 4, out var cmpRaw) || - !ctx.TryReadUInt32(addr + 8, out var lo) || - !ctx.TryReadUInt32(addr + 12, out var hi) || - !ctx.TryReadUInt32(addr + 16, out var reference) || - !ctx.TryReadUInt32(addr + 20, out var mask32)) + var resourceHandle = (uint)ctx[CpuRegister.Rdi]; + var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + lock (state.Gate) { - return false; + if (!state.RegisteredResources.Remove(resourceHandle)) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } } - cmpFunc = cmpRaw & 0x7; - waitAddr = ((ulong)hi << 32) | lo; - refVal = reference; - mask = mask32; - return true; + TraceAgc($"agc.driver_unregister_resource handle={resourceHandle}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } } diff --git a/src/SharpEmu.Libs/Agc/AgcPrimaryRegisterDefaults.cs b/src/SharpEmu.Libs/Agc/AgcPrimaryRegisterDefaults.cs new file mode 100644 index 0000000..1b089fc --- /dev/null +++ b/src/SharpEmu.Libs/Agc/AgcPrimaryRegisterDefaults.cs @@ -0,0 +1,143 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Agc; + +public static partial class AgcExports +{ + // Exact Gen5 primary register defaults originally reverse-engineered by Kyty (MIT). + // Hashes, offsets, and values match the tables consumed by the PS5 AGC SDK. + private static RegisterDefaultGroup[] CreatePrimaryRegisterDefaults() => + [ + // context register groups + new(0u, 0u, 0xE24F806Du, [new(0x202u, 0x00CC0010u)]), // CB_COLOR_CONTROL + new(0u, 1u, 0xF6C28182u, [new(0x109u, 0x00000000u)]), // CB_DCC_CONTROL + new(0u, 2u, 0x6F6E55A5u, [new(0x104u, 0x00000000u)]), // CB_RMI_GL2_CACHE_CONTROL + new(0u, 3u, 0x0BC65DA4u, [new(0x08Fu, 0x00000000u)]), // CB_SHADER_MASK + new(0u, 4u, 0x9E5AD592u, [new(0x08Eu, 0x0000000Fu)]), // CB_TARGET_MASK + new(0u, 5u, 0xBB513B98u, [new(0x2DCu, 0x0000AA00u)]), // DB_ALPHA_TO_MASK + new(0u, 6u, 0xAB64B23Bu, [new(0x001u, 0x00000000u)]), // DB_COUNT_CONTROL + new(0u, 7u, 0x53C39964u, [new(0x200u, 0x00000000u)]), // DB_DEPTH_CONTROL + new(0u, 8u, 0x01396B11u, [new(0x201u, 0x00000000u)]), // DB_EQAA + new(0u, 9u, 0x7D42019Au, [new(0x000u, 0x00000000u)]), // DB_RENDER_CONTROL + new(0u, 10u, 0x3548F523u, [new(0x006u, 0x00000000u)]), // PS_SHADER_SAMPLE_EXCLUSION_MASK + new(0u, 11u, 0xF43AD28Au, [new(0x01Fu, 0x00000000u)]), // DB_RMI_L2_CACHE_CONTROL + new(0u, 12u, 0x6DE4C312u, [new(0x203u, 0x00000000u)]), // DB_SHADER_CONTROL + new(0u, 13u, 0x00A77AE0u, [new(0x2B0u, 0x00000000u)]), // DB_SRESULTS_COMPARE_STATE0 + new(0u, 14u, 0x00A779B7u, [new(0x2B1u, 0x00000000u)]), // DB_SRESULTS_COMPARE_STATE1 + new(0u, 15u, 0x5100100Cu, [new(0x10Cu, 0x00000000u)]), // DB_STENCILREFMASK + new(0u, 16u, 0x59958BBAu, [new(0x10Du, 0x00000000u)]), // DB_STENCILREFMASK_BF + new(0u, 17u, 0x0C06F17Cu, [new(0x10Bu, 0x00000000u)]), // DB_STENCIL_CONTROL + new(0u, 18u, 0x6F104B72u, [new(0x1FFu, 0x00000000u)]), // GE_MAX_OUTPUT_PER_SUBGROUP + new(0u, 19u, 0x25C70D9Cu, [new(0x204u, 0x00000000u)]), // PA_CL_CLIP_CNTL + new(0u, 20u, 0x3881201Eu, [new(0x20Du, 0x00000000u)]), // PA_CL_OBJPRIM_ID_CNTL + new(0u, 21u, 0x09AFDDAFu, [new(0x206u, 0x0000043Fu)]), // PA_CL_VTE_CNTL + new(0u, 22u, 0x367D63CFu, [new(0x2F8u, 0x00000000u)]), // PA_SC_AA_CONFIG + new(0u, 23u, 0x43707DB8u, [new(0x083u, 0x0000FFFFu)]), // PA_SC_CLIPRECT_RULE + new(0u, 24u, 0xF6AE26BAu, [new(0x313u, 0x00000000u)]), // PA_SC_CONSERVATIVE_RASTERIZATION_CNTL + new(0u, 25u, 0x1B917652u, [new(0x800003FEu, 0x00000000u)]), // PA_SC_FSR_ENABLE + new(0u, 26u, 0x94B1E4F7u, [new(0x0EAu, 0x00000000u)]), // PA_SC_HORIZ_GRID + new(0u, 27u, 0xE3661B6Cu, [new(0x0E9u, 0x00000000u)]), // PA_SC_LEFT_VERT_GRID + new(0u, 28u, 0x1EB8D73Au, [new(0x292u, 0x00000002u)]), // PA_SC_MODE_CNTL_0 + new(0u, 29u, 0x15051FA3u, [new(0x293u, 0x00000000u)]), // PA_SC_MODE_CNTL_1 + new(0u, 30u, 0x9C51A7F1u, [new(0x0E8u, 0x00000000u)]), // PA_SC_RIGHT_VERT_GRID + new(0u, 31u, 0xA20EFC70u, [new(0x080u, 0x00000000u)]), // PA_SC_WINDOW_OFFSET + new(0u, 32u, 0x0EC09F6Eu, [new(0x211u, 0x00000000u)]), // PA_STATE_STEREO_X + new(0u, 33u, 0x34A7D6D3u, [new(0x210u, 0x00000000u)]), // PA_STEREO_CNTL + new(0u, 34u, 0xCE831B94u, [new(0x08Du, 0x00000000u)]), // PA_SU_HARDWARE_SCREEN_OFFSET + new(0u, 35u, 0x5CC72A74u, [new(0x282u, 0x00000008u)]), // PA_SU_LINE_CNTL + new(0u, 36u, 0x3B77713Cu, [new(0x281u, 0xFFFF0000u)]), // PA_SU_POINT_MINMAX + new(0u, 37u, 0x40F64410u, [new(0x280u, 0x00080008u)]), // PA_SU_POINT_SIZE + new(0u, 38u, 0x69441268u, [new(0x2DFu, 0x00000000u)]), // PA_SU_POLY_OFFSET_CLAMP + new(0u, 39u, 0x2E418B83u, [new(0x2DEu, 0x000001E9u)]), // PA_SU_POLY_OFFSET_DB_FMT_CNTL + new(0u, 40u, 0xA00D0C8Du, [new(0x205u, 0x00000240u)]), // PA_SU_SC_MODE_CNTL + new(0u, 41u, 0xB1289FB3u, [new(0x20Cu, 0x00000001u)]), // PA_SU_SMALL_PRIM_FILTER_CNTL + new(0u, 42u, 0x144832FBu, [new(0x2F9u, 0x0000002Du)]), // PA_SU_VTX_CNTL + new(0u, 43u, 0x9890D9FAu, [new(0x1BAu, 0x00000000u)]), // SPI_TMPRING_SIZE + new(0u, 44u, 0x9016FAF1u, [new(0x2A6u, 0x00000000u)]), // VGT_DRAW_PAYLOAD_CNTL + new(0u, 45u, 0x4B73CE27u, [new(0x2CEu, 0x00000400u)]), // VGT_GS_MAX_VERT_OUT + new(0u, 46u, 0x5F5A3E7Bu, [new(0x29Bu, 0x00000002u)]), // VGT_GS_OUT_PRIM_TYPE + new(0u, 47u, 0xD4AF3A51u, [new(0x2D6u, 0x00000000u)]), // VGT_LS_HS_CONFIG + new(0u, 48u, 0x6CF4F543u, [new(0x2A3u, 0xFFFFFFFFu)]), // VGT_PRIMITIVEID_RESET + new(0u, 49u, 0x5FB86CCBu, [new(0x2A1u, 0x00000000u)]), // VGT_PRIMITIVEID_EN + new(0u, 50u, 0xEDEFA188u, [new(0x2ADu, 0x00000000u)]), // VGT_REUSE_OFF + new(0u, 51u, 0xD0DE9EE6u, [new(0x2D5u, 0x00000000u)]), // VGT_SHADER_STAGES_EN + new(0u, 52u, 0xC5831803u, [new(0x2D4u, 0x88101000u)]), // VGT_TESS_DISTRIBUTION + new(0u, 53u, 0x8E6DE84Bu, [new(0x2DBu, 0x00000000u)]), // VGT_TF_PARAM + new(0u, 54u, 0xD0771662u, [new(0x2F5u, 0x00000000u), new(0x2F6u, 0x00000000u)]), // PA_SC_CENTROID_PRIORITY_0, PA_SC_CENTROID_PRIORITY_1 + new(0u, 55u, 0x569F7444u, [new(0x2FEu, 0x00000000u)]), // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + new(0u, 56u, 0x5C6637CDu, [new(0x30Eu, 0xFFFFFFFFu), new(0x30Fu, 0xFFFFFFFFu)]), // PA_SC_AA_MASK_X0Y0_X1Y0, PA_SC_AA_MASK_X0Y1_X1Y1 + new(0u, 57u, 0xCAE3E690u, [new(0x311u, 0x00000002u), new(0x312u, 0x03FF0080u)]), // PA_SC_BINNER_CNTL_0, PA_SC_BINNER_CNTL_1 + new(0u, 58u, 0x43FBD769u, [new(0x105u, 0x00000000u), new(0x107u, 0x00000000u), new(0x106u, 0x00000000u), new(0x108u, 0x00000000u)]), // CB_BLEND_RED, CB_BLEND_BLUE, CB_BLEND_GREEN, CB_BLEND_ALPHA + new(0u, 59u, 0xEF550356u, [new(0x1E0u, 0x20010001u)]), // CB_BLEND0_CONTROL + new(0u, 60u, 0x8F52E279u, [new(0x020u, 0x00000000u), new(0x021u, 0x00000000u)]), // TA_BC_BASE_ADDR, TA_BC_BASE_ADDR_HI + new(0u, 61u, 0x1F2D8149u, [new(0x084u, 0x00000000u), new(0x085u, 0x20002000u)]), // PA_SC_CLIPRECT_0_TL, PA_SC_CLIPRECT_0_BR + new(0u, 62u, 0x853D0614u, [new(0x800003FFu, 0x00000000u)]), // CX_NOP + new(0u, 63u, 0x4413C6F9u, [new(0x008u, 0x00000000u), new(0x009u, 0x00000000u)]), // DB_DEPTH_BOUNDS_MIN, DB_DEPTH_BOUNDS_MAX + new(0u, 64u, 0x67096014u, [new(0x010u, 0x80000000u), new(0x011u, 0x20000000u), new(0x012u, 0x00000000u), new(0x013u, 0x00000000u), new(0x014u, 0x00000000u), new(0x015u, 0x00000000u), new(0x01Au, 0x00000000u), new(0x01Bu, 0x00000000u), new(0x01Cu, 0x00000000u), new(0x01Du, 0x00000000u), new(0x01Eu, 0x00000000u), new(0x002u, 0x00000000u), new(0x005u, 0x00000000u), new(0x007u, 0x00000000u), new(0x00Bu, 0x00000000u), new(0x00Au, 0x00000000u)]), // DB_Z_INFO, DB_STENCIL_INFO, DB_Z_READ_BASE, DB_STENCIL_READ_BASE, DB_Z_WRITE_BASE, DB_STENCIL_WRITE_BASE, DB_Z_READ_BASE_HI, DB_STENCIL_READ_BASE_HI, DB_Z_WRITE_BASE_HI, DB_STENCIL_WRITE_BASE_HI, DB_HTILE_DATA_BASE_HI, DB_DEPTH_VIEW, DB_HTILE_DATA_BASE, DB_DEPTH_SIZE_XY, DB_DEPTH_CLEAR, DB_STENCIL_CLEAR + new(0u, 65u, 0x88F5E915u, [new(0x0EBu, 0xFF00FF00u), new(0x0ECu, 0x00000000u)]), // PA_SC_FOV_WINDOW_LR, PA_SC_FOV_WINDOW_TB + new(0u, 66u, 0x033F1EFFu, [new(0x800003FCu, 0x00000000u), new(0x800003FDu, 0x00000000u)]), // FSR_RECURSIONS0, FSR_RECURSIONS1 + new(0u, 67u, 0x918106BBu, [new(0x090u, 0x80000000u), new(0x091u, 0x40004000u)]), // PA_SC_GENERIC_SCISSOR_TL, PA_SC_GENERIC_SCISSOR_BR + new(0u, 68u, 0x95F0E7ACu, [new(0x2FAu, 0x4E7E0000u), new(0x2FBu, 0x4E7E0000u), new(0x2FCu, 0x4E7E0000u), new(0x2FDu, 0x4E7E0000u)]), // PA_CL_GB_VERT_CLIP_ADJ, PA_CL_GB_VERT_DISC_ADJ, PA_CL_GB_HORZ_CLIP_ADJ, PA_CL_GB_HORZ_DISC_ADJ + new(0u, 69u, 0xB48CBAB2u, [new(0x2E2u, 0x00000000u), new(0x2E3u, 0x00000000u)]), // PA_SU_POLY_OFFSET_BACK_SCALE, PA_SU_POLY_OFFSET_BACK_OFFSET + new(0u, 70u, 0x05BB3BC6u, [new(0x2E0u, 0x00000000u), new(0x2E1u, 0x00000000u)]), // PA_SU_POLY_OFFSET_FRONT_SCALE, PA_SU_POLY_OFFSET_FRONT_OFFSET + new(0u, 71u, 0x94FABA07u, [new(0x003u, 0x00000000u), new(0x004u, 0x00000000u)]), // DB_RENDER_OVERRIDE, DB_RENDER_OVERRIDE2 + new(0u, 72u, 0x38E92C91u, [new(0x318u, 0x00000000u), new(0x31Bu, 0x00000000u), new(0x31Cu, 0x00000000u), new(0x31Du, 0x00000000u), new(0x31Eu, 0x00000048u), new(0x31Fu, 0x00000000u), new(0x321u, 0x00000000u), new(0x323u, 0x00000000u), new(0x324u, 0x00000000u), new(0x325u, 0x00000000u), new(0x390u, 0x00000000u), new(0x398u, 0x00000000u), new(0x3A0u, 0x00000000u), new(0x3A8u, 0x00000000u), new(0x3B0u, 0x00000000u), new(0x3B8u, 0x0006C000u)]), // CB_COLOR0_BASE, CB_COLOR0_VIEW, CB_COLOR0_INFO, CB_COLOR0_ATTRIB, CB_COLOR0_DCC_CONTROL, CB_COLOR0_CMASK, CB_COLOR0_FMASK, CB_COLOR0_CLEAR_WORD0, CB_COLOR0_CLEAR_WORD1, CB_COLOR0_DCC_BASE, CB_COLOR0_BASE_EXT, CB_COLOR0_CMASK_BASE_EXT, CB_COLOR0_FMASK_BASE_EXT, CB_COLOR0_DCC_BASE_EXT, CB_COLOR0_ATTRIB2, CB_COLOR0_ATTRIB3 + new(0u, 73u, 0x0B177B43u, [new(0x00Cu, 0x00000000u), new(0x00Du, 0x40004000u)]), // PA_SC_SCREEN_SCISSOR_TL, PA_SC_SCREEN_SCISSOR_BR + new(0u, 74u, 0x48531062u, [new(0x191u, 0x00000000u)]), // SPI_PS_INPUT_CNTL_0 + new(0u, 75u, 0xAAA964B9u, [new(0x16Fu, 0x00000000u), new(0x170u, 0x00000000u), new(0x171u, 0x00000000u), new(0x172u, 0x00000000u)]), // PA_CL_UCP_0_X, PA_CL_UCP_0_Y, PA_CL_UCP_0_Z, PA_CL_UCP_0_W + new(0u, 76u, 0x7690AF6Fu, [new(0x10Fu, 0x4E7E0000u), new(0x111u, 0x4E7E0000u), new(0x113u, 0x4E7E0000u), new(0x110u, 0x00000000u), new(0x112u, 0x00000000u), new(0x114u, 0x00000000u), new(0x094u, 0x80000000u), new(0x095u, 0x40004000u), new(0x0B4u, 0x00000000u), new(0x0B5u, 0x00000000u)]), // PA_CL_VPORT_XSCALE, PA_CL_VPORT_YSCALE, PA_CL_VPORT_ZSCALE, PA_CL_VPORT_XOFFSET, PA_CL_VPORT_YOFFSET, PA_CL_VPORT_ZOFFSET, PA_SC_VPORT_SCISSOR_0_TL, PA_SC_VPORT_SCISSOR_0_BR, PA_SC_VPORT_ZMIN_0, PA_SC_VPORT_ZMAX_0 + new(0u, 77u, 0x078D7060u, [new(0x081u, 0x80000000u), new(0x082u, 0x40004000u)]), // PA_SC_WINDOW_SCISSOR_TL, PA_SC_WINDOW_SCISSOR_BR + // shader register groups + new(1u, 0u, 0x5D6E3EC7u, [new(0x212u, 0x00000000u)]), // COMPUTE_PGM_RSRC1 + new(1u, 1u, 0x57E7079Au, [new(0x213u, 0x00000000u)]), // COMPUTE_PGM_RSRC2 + new(1u, 2u, 0x7467FAFDu, [new(0x228u, 0x00000000u)]), // COMPUTE_PGM_RSRC3 + new(1u, 3u, 0x9E826B50u, [new(0x215u, 0x00000000u)]), // COMPUTE_RESOURCE_LIMITS + new(1u, 4u, 0xDC484F18u, [new(0x218u, 0x00000000u)]), // COMPUTE_TMPRING_SIZE + new(1u, 5u, 0x5DA8BCA3u, [new(0x08Au, 0x00000000u)]), // SPI_SHADER_PGM_RSRC1_GS + new(1u, 6u, 0x5CA726D8u, [new(0x10Au, 0x00000000u)]), // SPI_SHADER_PGM_RSRC1_HS + new(1u, 7u, 0x5DD28360u, [new(0x00Au, 0x00000000u)]), // SPI_SHADER_PGM_RSRC1_PS + new(1u, 8u, 0x57EFA0BEu, [new(0x08Bu, 0x00000000u)]), // SPI_SHADER_PGM_RSRC2_GS + new(1u, 9u, 0x502363D5u, [new(0x10Bu, 0x00000000u)]), // SPI_SHADER_PGM_RSRC2_HS + new(1u, 10u, 0x506D14BDu, [new(0x00Bu, 0x00000000u)]), // SPI_SHADER_PGM_RSRC2_PS + new(1u, 11u, 0xB2609506u, [new(0x224u, 0x00000000u)]), // COMPUTE_USER_ACCUM_0 + new(1u, 12u, 0x9E5CFB8Au, [new(0x107u, 0x00000000u), new(0x087u, 0x00000000u), new(0x007u, 0x00000000u)]), // SPI_SHADER_PGM_RSRC3_HS, SPI_SHADER_PGM_RSRC3_GS, SPI_SHADER_PGM_RSRC3_PS + new(1u, 13u, 0xC918DF3Eu, [new(0x20Cu, 0x00000000u), new(0x20Du, 0x00000000u)]), // COMPUTE_PGM_LO, COMPUTE_PGM_HI + new(1u, 14u, 0xC9751C9Cu, [new(0x0C8u, 0x00000000u), new(0x0C9u, 0x00000000u)]), // SPI_SHADER_PGM_LO_ES, SPI_SHADER_PGM_HI_ES + new(1u, 15u, 0xC97EF77Au, [new(0x088u, 0x00000000u), new(0x089u, 0x00000000u)]), // SPI_SHADER_PGM_LO_GS, SPI_SHADER_PGM_HI_GS + new(1u, 16u, 0xC927C6B9u, [new(0x108u, 0x00000000u), new(0x109u, 0x00000000u)]), // SPI_SHADER_PGM_LO_HS, SPI_SHADER_PGM_HI_HS + new(1u, 17u, 0xC92A1EC5u, [new(0x148u, 0x00000000u), new(0x149u, 0x00000000u)]), // SPI_SHADER_PGM_LO_LS, SPI_SHADER_PGM_HI_LS + new(1u, 18u, 0xC9E01B31u, [new(0x008u, 0x00000000u), new(0x009u, 0x00000000u)]), // SPI_SHADER_PGM_LO_PS, SPI_SHADER_PGM_HI_PS + new(1u, 19u, 0x50685F29u, [new(0x800002FFu, 0x00000000u)]), // SH_NOP + new(1u, 20u, 0xB26219CAu, [new(0x0B2u, 0x00000000u)]), // SPI_SHADER_USER_ACCUM_ESGS_0 + new(1u, 21u, 0xB25B6CF9u, [new(0x132u, 0x00000000u)]), // SPI_SHADER_USER_ACCUM_LSHS_0 + new(1u, 22u, 0xB2F86101u, [new(0x032u, 0x00000000u)]), // SPI_SHADER_USER_ACCUM_PS_0 + new(1u, 23u, 0x07E3B155u, [new(0x082u, 0x00000000u), new(0x083u, 0x00000000u)]), // SPI_SHADER_USER_DATA_ADDR_LO_GS, SPI_SHADER_USER_DATA_ADDR_HI_GS + new(1u, 24u, 0x07E383C6u, [new(0x102u, 0x00000000u), new(0x103u, 0x00000000u)]), // SPI_SHADER_USER_DATA_ADDR_LO_HS, SPI_SHADER_USER_DATA_ADDR_HI_HS + new(1u, 25u, 0xBDA98653u, [new(0x240u, 0x00000000u)]), // COMPUTE_USER_DATA_0 + new(1u, 26u, 0xBDBD1D0Fu, [new(0x08Cu, 0x00000000u)]), // SPI_SHADER_USER_DATA_GS_0 + new(1u, 27u, 0xBD946FD4u, [new(0x10Cu, 0x00000000u)]), // SPI_SHADER_USER_DATA_HS_0 + new(1u, 28u, 0xBDF02A4Cu, [new(0x00Cu, 0x00000000u)]), // SPI_SHADER_USER_DATA_PS_0 + // uconfig register groups + new(2u, 0u, 0x19E93E85u, [new(0x41Fu, 0x00000000u)]), // GDS_OA_ADDRESS + new(2u, 1u, 0x3B5C2AF3u, [new(0x41Du, 0x00000000u)]), // GDS_OA_CNTL + new(2u, 2u, 0x47974A35u, [new(0x41Eu, 0x00000000u)]), // GDS_OA_COUNTER + new(2u, 3u, 0x105971C2u, [new(0x25Bu, 0x00000000u)]), // GE_CNTL + new(2u, 4u, 0x7D137765u, [new(0x24Au, 0x00000000u)]), // GE_INDX_OFFSET + new(2u, 5u, 0xD187FEBCu, [new(0x24Bu, 0x00000000u)]), // GE_MULTI_PRIM_IB_RESET_EN + new(2u, 6u, 0x12F854ACu, [new(0x25Fu, 0x00000000u)]), // GE_STEREO_CNTL + new(2u, 7u, 0x40D49AD1u, [new(0x262u, 0x00000000u)]), // GE_USER_VGPR_EN + new(2u, 8u, 0x8C0923DAu, [new(0x80003FF4u, 0x00000000u)]), // FSR_EXTEND_SUBPIXEL_ROUNDING + new(2u, 9u, 0xBB8DF494u, [new(0x80003FFDu, 0x00000000u)]), // TEXTURE_GRADIENT_CONTROL + new(2u, 10u, 0xF6D8A76Eu, [new(0x382u, 0x40000040u)]), // TEXTURE_GRADIENT_FACTORS + new(2u, 11u, 0x7620F1E9u, [new(0x248u, 0x00000000u)]), // VGT_OBJECT_ID + new(2u, 12u, 0x9EBFAB10u, [new(0x242u, 0x00000000u)]), // VGT_PRIMITIVE_TYPE + new(2u, 13u, 0x98A09D0Eu, [new(0x380u, 0x00000000u), new(0x381u, 0x00000000u)]), // TA_CS_BC_BASE_ADDR, TA_CS_BC_BASE_ADDR_HI + new(2u, 14u, 0x195D37D2u, [new(0x80003FF5u, 0x00000000u), new(0x80003FF6u, 0x00000000u)]), // FSR_ALPHA_VALUE0, FSR_ALPHA_VALUE1 + new(2u, 15u, 0xF9EC4F85u, [new(0x80003FF7u, 0x00000000u), new(0x80003FF8u, 0x00000000u), new(0x80003FF9u, 0x00000000u), new(0x80003FFAu, 0x00000000u)]), // FSR_CONTROL_POINT0, FSR_CONTROL_POINT1, FSR_CONTROL_POINT2, FSR_CONTROL_POINT3 + new(2u, 16u, 0x4626B750u, [new(0x80003FFBu, 0x00000000u), new(0x80003FFCu, 0x00000000u)]), // FSR_WINDOW0, FSR_WINDOW1 + new(2u, 17u, 0x4CC673A0u, [new(0x80003FFEu, 0x00000000u)]), // MEMORY_MAPPING_MASK + new(2u, 18u, 0xDE5B3431u, [new(0x80003FFFu, 0x00000000u)]), // UC_NOP + new(2u, 19u, 0x036AC8A6u, [new(0x25Cu, 0x00000000u)]), // GE_USER_VGPR1 + ]; +} diff --git a/src/SharpEmu.Libs/Agc/AgcShaderCompilerHooks.cs b/src/SharpEmu.Libs/Agc/AgcShaderCompilerHooks.cs index aa824cc..d6dedbc 100644 --- a/src/SharpEmu.Libs/Agc/AgcShaderCompilerHooks.cs +++ b/src/SharpEmu.Libs/Agc/AgcShaderCompilerHooks.cs @@ -4,6 +4,7 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using SharpEmu.Libs.Kernel; +using SharpEmu.Libs.VideoOut; using SharpEmu.ShaderCompiler; namespace SharpEmu.Libs.Agc; @@ -26,5 +27,7 @@ internal static class AgcShaderCompilerHooks { Gen5ShaderScalarEvaluator.FallbackMemoryReader = KernelMemoryCompatExports.TryReadTrackedLibcHeap; + Gen5ShaderScalarEvaluator.GlobalMemoryPool = + VulkanVideoPresenter.GuestDataPool; } } diff --git a/src/SharpEmu.Libs/Agc/GnmTiling.cs b/src/SharpEmu.Libs/Agc/GnmTiling.cs new file mode 100644 index 0000000..2fe0c00 --- /dev/null +++ b/src/SharpEmu.Libs/Agc/GnmTiling.cs @@ -0,0 +1,486 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Agc; + +/// +/// Deswizzles RDNA2 (GFX10) tiled texture surfaces into linear layout so they +/// can be uploaded to Vulkan. PS5 stores most textures in a swizzled layout +/// selected by the 5-bit SWIZZLE_MODE in the image descriptor; uploading those +/// bytes verbatim samples as garbage. +/// +/// The GFX10 addressing uses power-of-two swizzle blocks (256 B / 4 KiB / +/// 64 KiB) whose internal element order follows the standard (S), display (D), +/// render (R), or z-order/depth (Z) equations. This implements the exact base +/// S and Z 2D single-sample modes plus the PS5's RB+ 64 KiB Z_X/R_X equations; +/// other D/R and pipe/bank-XOR modes stay opt-in while their complete AddrLib +/// equations are being ported. +/// +internal static class GnmTiling +{ + // Oberon uses the 16-pipe / 8-pixel-packer RB+ topology. These are the + // single-sample 64 KiB equations generated by AMD AddrLib for that exact + // topology. Each entry describes one address bit as an XOR of X/Y bits. + private readonly record struct AddressBit(uint XMask, uint YMask); + + private static readonly AddressBit[][] RbPlus64KRenderX = + [ + // 1 byte/element: nibble01=0, nibble2=307, nibble3=379. + [X(0), X(1), X(2), X(3), Y(0), Y(1), Y(2), Y(3), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(6), Y(6), XY(7, 8), XY(8, 7)], + // 2 bytes/element: nibble01=1, nibble2=307, nibble3=389. + [Zero, X(0), X(1), X(2), Y(0), Y(1), Y(2), X(3), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + Y(3), X(6), XY(7, 7), XY(8, 6)], + // 4 bytes/element: nibble01=39, nibble2=307, nibble3=381. + [Zero, Zero, X(0), X(1), Y(0), Y(1), X(2), Y(2), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(3), Y(3), XY(6, 7), XY(7, 6)], + // 8 bytes/element: nibble01=6, nibble2=307, nibble3=382. + [Zero, Zero, Zero, X(0), Y(0), X(1), X(2), Y(1), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + Y(2), X(3), XY(7, 3), XY(6, 6)], + // 16 bytes/element: nibble01=7, nibble2=307, nibble3=390. + [Zero, Zero, Zero, Zero, X(0), Y(0), X(1), Y(1), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(2), Y(2), XY(6, 3), XY(3, 6)], + ]; + + private static readonly AddressBit[][] RbPlus64KDepthX = + [ + // 1 byte/element: nibble01=8, nibble2=306, nibble3=379. + [X(0), Y(0), X(1), Y(1), X(2), Y(2), X(3), Y(3), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(6), Y(6), XY(7, 8), XY(8, 7)], + // 2 bytes/element: nibble01=9, nibble2=306, nibble3=389. + [Zero, X(0), Y(0), X(1), Y(1), X(2), Y(2), X(3), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + Y(3), X(6), XY(7, 7), XY(8, 6)], + // 4 bytes/element: nibble01=10, nibble2=306, nibble3=381. + [Zero, Zero, X(0), Y(0), X(1), Y(1), X(2), Y(2), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(3), Y(3), XY(6, 7), XY(7, 6)], + // 8 bytes/element: nibble01=11, nibble2=307, nibble3=382. + [Zero, Zero, Zero, X(0), Y(0), X(1), Y(1), X(2), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + Y(2), X(3), XY(7, 3), XY(6, 6)], + // 16 bytes/element is identical to R_X for a 2D single-sample image. + [Zero, Zero, Zero, Zero, X(0), Y(0), X(1), Y(1), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(2), Y(2), XY(6, 3), XY(3, 6)], + ]; + + private static readonly AddressBit[][] RbPlus64KStandard = + [ + // GFX10_SW_64K_S_RBPLUS_PATINFO, 1 byte/element. + [X(0), X(1), X(2), X(3), Y(0), Y(1), Y(2), Y(3), + Y(4), X(4), Y(5), X(5), Y(6), X(6), Y(7), X(7)], + // 2 bytes/element. + [Zero, X(0), X(1), X(2), Y(0), Y(1), Y(2), X(3), + Y(3), X(4), Y(4), X(5), Y(5), X(6), Y(6), X(7)], + // 4 bytes/element. + [Zero, Zero, X(0), X(1), Y(0), Y(1), Y(2), X(2), + Y(3), X(3), Y(4), X(4), Y(5), X(5), Y(6), X(6)], + // 8 bytes/element (also BC1/BC4 compressed blocks). + [Zero, Zero, Zero, X(0), Y(0), Y(1), X(1), X(2), + Y(2), X(3), Y(3), X(4), Y(4), X(5), Y(5), X(6)], + // 16 bytes/element (also 16-byte BC compressed blocks). + [Zero, Zero, Zero, Zero, Y(0), Y(1), X(0), X(1), + Y(2), X(2), Y(3), X(3), Y(4), X(4), Y(5), X(5)], + ]; + + // GFX10 4K_S has a separate 12-bit micro-tile equation. It is not the + // generic x/y interleave used by the 64K standard block; using that larger + // equation leaves a regular grid in linearized atlases. + private static readonly AddressBit[][] Standard4K = + [ + [X(0), X(1), X(2), X(3), Y(0), Y(1), Y(2), Y(3), + Y(4), X(4), Y(5), X(5)], + [Zero, X(0), X(1), X(2), Y(0), Y(1), Y(2), X(3), + Y(3), X(4), Y(4), X(5)], + [Zero, Zero, X(0), X(1), Y(0), Y(1), Y(2), X(2), + Y(3), X(3), Y(4), X(4)], + [Zero, Zero, Zero, X(0), Y(0), Y(1), X(1), X(2), + Y(2), X(3), Y(3), X(4)], + [Zero, Zero, Zero, Zero, Y(0), Y(1), X(0), X(1), + Y(2), X(2), Y(3), X(3)], + ]; + + private static readonly bool _enabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DETILE"), + "1", + StringComparison.Ordinal); + + private static readonly bool _disabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DETILE"), + "0", + StringComparison.Ordinal); + + private static readonly HashSet _reportedModes = new(); + + public static bool Enabled => _enabled || !_disabled; + + /// + /// Base S/Z modes and the Oberon RB+ 64 KiB Z_X/R_X modes for which this + /// implementation carries the exact AddrLib address equations. Other + /// bank/pipe-XOR variants remain opt-in. + /// + private static bool IsTrustedByDefault(uint swizzleMode) => + // Exact base S/Z modes. D/R use different GFX10 swizzle equations and + // the T/X modes additionally apply pipe/bank XOR between blocks. + swizzleMode is 1 or 4 or 5 or 8 or 9 or 24 or 27; + + // Detile a surface when it is verified-correct by default (trusted base mode), + // or when the user opts the approximate modes in with SHARPEMU_DETILE=1. + // SHARPEMU_DETILE=0 forces the old raw-upload behavior for everything. + private static bool ShouldDetile(uint swizzleMode) => + swizzleMode != 0 && !_disabled && (_enabled || IsTrustedByDefault(swizzleMode)); + + /// + /// True when a surface with the given swizzle mode needs deswizzling. + /// Mode 0 is linear and never needs it. + /// + public static bool NeedsDetile(uint swizzleMode) => ShouldDetile(swizzleMode); + + /// + /// Gets the physical byte span occupied by a tiled mip. GNM allocates whole + /// swizzle blocks even when the logical image is much smaller than a block; + /// reading only the logical texel count truncates most source offsets during + /// detiling (a 64x64 BC1 mode-9 image is 2 KiB logically but occupies one + /// 64 KiB swizzle block). + /// + public static bool TryGetTiledByteCount( + uint swizzleMode, + int elementsWide, + int elementsHigh, + int bytesPerElement, + out ulong byteCount) + { + byteCount = 0; + if (!ShouldDetile(swizzleMode) || + elementsWide <= 0 || + elementsHigh <= 0 || + bytesPerElement <= 0 || + !TryGetSwizzleKind(swizzleMode, out _, out var blockBytes)) + { + return false; + } + + var bppLog2 = BitLog2((uint)bytesPerElement); + if (bppLog2 < 0) + { + return false; + } + + var blockElements = blockBytes >> bppLog2; + var (blockWidth, blockHeight) = SquareBlockDimensions(blockElements); + if (blockWidth == 0 || blockHeight == 0) + { + return false; + } + + var blocksWide = ((ulong)elementsWide + (ulong)blockWidth - 1) / (ulong)blockWidth; + var blocksHigh = ((ulong)elementsHigh + (ulong)blockHeight - 1) / (ulong)blockHeight; + try + { + byteCount = checked(blocksWide * blocksHigh * (ulong)blockBytes); + return true; + } + catch (OverflowException) + { + byteCount = 0; + return false; + } + } + + /// + /// Deswizzles into linear row-major order. + /// Elements are pixels for uncompressed formats and 4x4 blocks for + /// block-compressed formats, so callers pass the element grid dimensions + /// and the bytes per element. Returns false (leaving output untouched) for + /// unsupported swizzle modes so the caller can fall back to the raw bytes. + /// + public static bool TryDetile( + ReadOnlySpan tiled, + Span linear, + uint swizzleMode, + int elementsWide, + int elementsHigh, + int bytesPerElement) + { + if (!ShouldDetile(swizzleMode) || elementsWide <= 0 || elementsHigh <= 0 || bytesPerElement <= 0) + { + return false; + } + + if (!TryGetSwizzleKind(swizzleMode, out var kind, out var blockBytes)) + { + ReportUnsupported(swizzleMode); + return false; + } + + var bppLog2 = BitLog2((uint)bytesPerElement); + if (bppLog2 < 0) + { + // Non-power-of-two element size (e.g. 24-bit) is not swizzled in a + // way this equation models. + ReportUnsupported(swizzleMode); + return false; + } + + // Block dimensions in elements: a swizzle block holds blockBytes bytes, + // laid out as a square-ish power-of-two element grid scaled by bpp. + var blockElements = blockBytes >> bppLog2; + var (blockWidth, blockHeight) = SquareBlockDimensions(blockElements); + if (blockWidth == 0 || blockHeight == 0) + { + ReportUnsupported(swizzleMode); + return false; + } + + var blocksPerRow = (elementsWide + blockWidth - 1) / blockWidth; + var requiredLinear = (long)elementsWide * elementsHigh * bytesPerElement; + if (linear.Length < requiredLinear) + { + return false; + } + + // Precompute the within-block element offset for each (x, y) inside a + // single block. The swizzle equation only depends on the in-block + // coordinates, so this table is reused for every block — turning the + // per-pixel bit-interleave (a loop + calls) into a single array lookup. + // Detiling a 2048x2048 texture is millions of elements; without this the + // per-pixel math makes DETILE unusably slow during asset streaming. + var hasExactXorPattern = TryGetExactXorPattern(swizzleMode, bppLog2, out var xorPattern); + var blockTable = hasExactXorPattern ? [] : new int[blockWidth * blockHeight]; + for (var by = 0; !hasExactXorPattern && by < blockHeight; by++) + { + for (var bx = 0; bx < blockWidth; bx++) + { + blockTable[by * blockWidth + bx] = (int)(kind == SwizzleKind.ZOrder + ? MortonInterleave((uint)bx, (uint)by, blockWidth, blockHeight) + : StandardSwizzleOffset((uint)bx, (uint)by, blockWidth, blockHeight)); + } + } + + for (var y = 0; y < elementsHigh; y++) + { + var blockY = y / blockHeight; + var inBlockY = y % blockHeight; + var rowBlockBase = (long)blockY * blocksPerRow; + var tableRowBase = inBlockY * blockWidth; + var destRowBase = (long)y * elementsWide * bytesPerElement; + for (var x = 0; x < elementsWide; x++) + { + var blockX = x / blockWidth; + var inBlockX = x % blockWidth; + + var blockIndex = rowBlockBase + blockX; + var sourceByte = hasExactXorPattern + ? blockIndex * blockBytes + ComputePatternOffset((uint)x, (uint)y, xorPattern) + : (blockIndex * blockElements + blockTable[tableRowBase + inBlockX]) * + (long)bytesPerElement; + var destByte = destRowBase + (long)x * bytesPerElement; + if (sourceByte + bytesPerElement > tiled.Length || + destByte + bytesPerElement > linear.Length) + { + continue; + } + + tiled.Slice((int)sourceByte, bytesPerElement) + .CopyTo(linear.Slice((int)destByte, bytesPerElement)); + } + } + + return true; + } + + private enum SwizzleKind + { + Standard, + ZOrder, + } + + private static readonly AddressBit Zero = new(0, 0); + + private static AddressBit X(int bit) => new(1u << bit, 0); + + private static AddressBit Y(int bit) => new(0, 1u << bit); + + private static AddressBit XY(int xBit, int yBit) => new(1u << xBit, 1u << yBit); + + private static AddressBit XY(int xBit, int yBit1, int yBit2) => + new(1u << xBit, (1u << yBit1) | (1u << yBit2)); + + private static bool TryGetExactXorPattern( + uint swizzleMode, + int bytesPerElementLog2, + out AddressBit[] pattern) + { + pattern = []; + if ((uint)bytesPerElementLog2 >= RbPlus64KRenderX.Length) + { + return false; + } + + pattern = swizzleMode switch + { + 5 => Standard4K[bytesPerElementLog2], + 9 => RbPlus64KStandard[bytesPerElementLog2], + 24 => RbPlus64KDepthX[bytesPerElementLog2], + 27 => RbPlus64KRenderX[bytesPerElementLog2], + _ => [], + }; + return pattern.Length != 0; + } + + private static long ComputePatternOffset(uint x, uint y, AddressBit[] pattern) + { + uint offset = 0; + for (var bit = 0; bit < pattern.Length; bit++) + { + var equation = pattern[bit]; + var parity = (System.Numerics.BitOperations.PopCount(x & equation.XMask) + + System.Numerics.BitOperations.PopCount(y & equation.YMask)) & 1; + offset |= (uint)parity << bit; + } + + return offset; + } + + private static bool TryGetSwizzleKind(uint swizzleMode, out SwizzleKind kind, out int blockBytes) + { + // GFX10 AddrLib SWIZZLE_MODE enumeration: + // 1-3 = 256 B S/D/R + // 4-7 = 4 KiB Z/S/D/R + // 8-11 = 64 KiB Z/S/D/R + // 16-19 = 64 KiB Z/S/D/R _T + // 20-23 = 4 KiB Z/S/D/R _X + // 24-27 = 64 KiB Z/S/D/R _X + // The pipe/bank XOR (_T/_X) affects which block a given tile lands in, + // but the *within-block* element order matches the base S/Z equation, + // which is what dominates visible correctness. We model the block + // interior and treat blocks as linear-ordered. + kind = SwizzleKind.Standard; + blockBytes = 0; + switch (swizzleMode) + { + case 4: case 8: case 16: case 20: case 24: + kind = SwizzleKind.ZOrder; + break; + case 1: case 2: case 3: + case 5: case 6: case 7: + case 9: case 10: case 11: + case 17: case 18: case 19: + case 21: case 22: case 23: + case 25: case 26: case 27: + kind = SwizzleKind.Standard; + break; + default: + return false; + } + + blockBytes = swizzleMode switch + { + >= 1 and <= 3 => 256, + >= 4 and <= 7 => 4096, + >= 20 and <= 23 => 4096, + _ => 65536, + }; + return true; + } + + // Standard (S) 2D swizzle: within a block, the element order interleaves x + // and y bits with x taking the low bit — the AMD "standard" microtile. + private static long StandardSwizzleOffset(uint x, uint y, int blockWidth, int blockHeight) + { + var widthBits = BitLog2((uint)blockWidth); + var heightBits = BitLog2((uint)blockHeight); + long offset = 0; + var outBit = 0; + var xi = 0; + var yi = 0; + while (xi < widthBits || yi < heightBits) + { + if (xi < widthBits) + { + offset |= (long)((x >> xi) & 1u) << outBit++; + xi++; + } + + if (yi < heightBits) + { + offset |= (long)((y >> yi) & 1u) << outBit++; + yi++; + } + } + + return offset; + } + + // Z-order (Morton) swizzle: pure bit interleave, y taking the low bit. + private static long MortonInterleave(uint x, uint y, int blockWidth, int blockHeight) + { + var widthBits = BitLog2((uint)blockWidth); + var heightBits = BitLog2((uint)blockHeight); + long offset = 0; + var outBit = 0; + var xi = 0; + var yi = 0; + while (xi < widthBits || yi < heightBits) + { + if (yi < heightBits) + { + offset |= (long)((y >> yi) & 1u) << outBit++; + yi++; + } + + if (xi < widthBits) + { + offset |= (long)((x >> xi) & 1u) << outBit++; + xi++; + } + } + + return offset; + } + + private static (int Width, int Height) SquareBlockDimensions(int blockElements) + { + if (blockElements <= 0 || (blockElements & (blockElements - 1)) != 0) + { + return (0, 0); + } + + var totalBits = BitLog2((uint)blockElements); + // Split as evenly as possible with width >= height (x gets the extra bit). + var widthBits = (totalBits + 1) / 2; + var heightBits = totalBits - widthBits; + return (1 << widthBits, 1 << heightBits); + } + + private static int BitLog2(uint value) + { + if (value == 0 || (value & (value - 1)) != 0) + { + return -1; + } + + return System.Numerics.BitOperations.TrailingZeroCount(value); + } + + private static void ReportUnsupported(uint swizzleMode) + { + lock (_reportedModes) + { + if (!_reportedModes.Add(swizzleMode)) + { + return; + } + } + + Console.Error.WriteLine( + $"[LOADER][WARN] GNM detile: unsupported swizzle mode {swizzleMode}; texture uploaded linear."); + } +} diff --git a/src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs b/src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs index 80c4822..dd8b3fe 100644 --- a/src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs +++ b/src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs @@ -3,10 +3,17 @@ namespace SharpEmu.Libs.Agc; -// Holds DCBs whose parsing was suspended on an unsatisfied WAIT_REG_MEM condition. -// AgcExports re-checks every waiter against guest memory on each submit and resumes -// the ones whose condition became true (labels are advanced by ReleaseMem/WriteData/ -// DmaData packets or by direct CPU writes). +/// +/// Holds DCBs whose parsing was suspended on an unsatisfied WAIT_REG_MEM +/// condition. AgcExports re-checks every waiter against guest memory on each +/// submit and resumes the ones whose condition became true (labels are advanced +/// by ReleaseMem / WriteData / DmaData packets, or by direct CPU writes). +/// +/// This preserves cross-submit ordering: the work that follows a wait inside a +/// DCB is only queued once the awaited completion label is genuinely written, +/// instead of being force-satisfied at parse time and running ahead of the +/// compute/graphics work it depends on (which produced a black composite). +/// internal static class GpuWaitRegistry { public struct WaitingDcb @@ -19,13 +26,56 @@ internal static class GpuWaitRegistry public ulong ReferenceValue; public ulong Mask; public uint CompareFunction; + public uint ControlValue; public bool Is64Bit; + public bool IsStandard; + public object? Memory; + public string? QueueName; + public ulong SubmissionId; + // Stopwatch timestamp captured at registration. Stale waiters remain + // registered; this only controls one-shot diagnostics. + public long RegisteredTicks; + public bool StaleReported; public object? State; } private static readonly object _gate = new(); private static readonly Dictionary> _waiters = new(); + public static int Count + { + get + { + lock (_gate) + { + var total = 0; + foreach (var (_, list) in _waiters) + { + total += list.Count; + } + + return total; + } + } + } + + public static int CountForMemory(object memory) + { + lock (_gate) + { + var total = 0; + foreach (var (_, list) in _waiters) + { + foreach (var waiter in list) + { + total += ReferenceEquals(waiter.Memory, memory) ? 1 : 0; + } + } + + return total; + } + } + public static void Register(ulong address, WaitingDcb waiter) { waiter.WaitAddress = address; @@ -41,9 +91,15 @@ internal static class GpuWaitRegistry } } - // Re-evaluates every registered waiter. readValue receives (address, is64Bit) and - // returns null when the memory is unreadable; such waiters are kept registered. - public static List? CollectSatisfied(Func readValue) + /// + /// Re-evaluates every registered waiter. + /// receives (address, is64Bit) and returns null when the memory is + /// unreadable; such waiters are kept registered. Returns the waiters whose + /// condition is now satisfied (removed from the registry), or null. + /// + public static List? CollectSatisfied( + object memory, + Func readValue) { List? woken = null; lock (_gate) @@ -53,6 +109,11 @@ internal static class GpuWaitRegistry { for (var i = list.Count - 1; i >= 0; i--) { + if (!ReferenceEquals(list[i].Memory, memory)) + { + continue; + } + var value = readValue(address, list[i].Is64Bit); if (value is null || !Compare(list[i], value.Value)) { @@ -71,7 +132,7 @@ internal static class GpuWaitRegistry } } - if (emptied != null) + if (emptied is not null) { foreach (var address in emptied) { @@ -83,20 +144,122 @@ internal static class GpuWaitRegistry return woken; } + /// + /// Returns waiters that have remained unsatisfied longer than + /// exactly once, without removing them or + /// changing their labels. Missing GPU work must fail closed: advancing a + /// command buffer without its real producer corrupts cross-queue ordering. + /// + public static List? CollectUnreportedStale( + object memory, + long nowTicks, + long maxAgeTicks) + { + List? stale = null; + lock (_gate) + { + foreach (var (_, list) in _waiters) + { + for (var i = list.Count - 1; i >= 0; i--) + { + var waiter = list[i]; + if (!ReferenceEquals(waiter.Memory, memory) || + waiter.StaleReported || + nowTicks - waiter.RegisteredTicks < maxAgeTicks) + { + continue; + } + + stale ??= new List(); + waiter.StaleReported = true; + list[i] = waiter; + stale.Add(waiter); + } + } + } + + return stale; + } + + /// + /// Returns watched labels overlapped by a newly discovered producer. Used + /// only for diagnostics; producer completion still wakes through the + /// normal CollectSatisfied path after the ordered memory write executes. + /// + public static List<(ulong Address, int Count)> SnapshotInRange( + object memory, + ulong start, + ulong length) + { + var matches = new List<(ulong Address, int Count)>(); + if (length == 0) + { + return matches; + } + + var end = start > ulong.MaxValue - length ? ulong.MaxValue : start + length; + lock (_gate) + { + foreach (var (address, list) in _waiters) + { + var matchingCount = 0; + var any64Bit = false; + foreach (var waiter in list) + { + if (!ReferenceEquals(waiter.Memory, memory)) + { + continue; + } + + matchingCount++; + any64Bit |= waiter.Is64Bit; + } + + if (matchingCount == 0) + { + continue; + } + + var width = any64Bit + ? sizeof(ulong) + : sizeof(uint); + var waitEnd = address > ulong.MaxValue - (ulong)width + ? ulong.MaxValue + : address + (ulong)width; + if (start < waitEnd && address < end) + { + matches.Add((address, matchingCount)); + } + } + } + + return matches; + } + public static bool Compare(in WaitingDcb waiter, ulong value) { var masked = value & waiter.Mask; + var reference = waiter.ReferenceValue & waiter.Mask; return waiter.CompareFunction switch { - 1 => masked < waiter.ReferenceValue, - 2 => masked <= waiter.ReferenceValue, - 3 => masked == waiter.ReferenceValue, - 4 => masked != waiter.ReferenceValue, - 5 => masked >= waiter.ReferenceValue, - 6 => masked > waiter.ReferenceValue, - // 0 is "always" in the PM4 encoding and 7 is reserved; treating both as - // satisfied keeps a malformed packet from suspending its DCB forever. + 0 => true, + 1 => masked < reference, + 2 => masked <= reference, + 3 => masked == reference, + 4 => masked != reference, + 5 => masked >= reference, + 6 => masked > reference, + // 7 is reserved; treating it as satisfied keeps a malformed packet + // from suspending forever. _ => true, }; } + + public static void Clear() + { + lock (_gate) + { + _waiters.Clear(); + } + } } diff --git a/src/SharpEmu.Libs/Ajm/AjmExports.cs b/src/SharpEmu.Libs/Ajm/AjmExports.cs index c9b8741..7c1dfd5 100644 --- a/src/SharpEmu.Libs/Ajm/AjmExports.cs +++ b/src/SharpEmu.Libs/Ajm/AjmExports.cs @@ -7,31 +7,11 @@ namespace SharpEmu.Libs.Ajm; public static class AjmExports { - private const int OrbisAjmErrorInvalidParameter = unchecked((int)0x80930005); - private static int _nextContextId; - [SysAbiExport( Nid = "dl+4eHSzUu4", ExportName = "sceAjmInitialize", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceAjm")] - public static int Initialize(CpuContext ctx) - { - var reserved = ctx[CpuRegister.Rdi]; - var outContextAddress = ctx[CpuRegister.Rsi]; - - // AJM requires a zero reserved argument. - if (reserved != 0 || outContextAddress == 0) - { - return ctx.SetReturn(OrbisAjmErrorInvalidParameter); - } - - var contextId = unchecked((uint)Interlocked.Increment(ref _nextContextId)); - if (!ctx.TryWriteUInt32(outContextAddress, contextId)) - { - return ctx.SetReturn(OrbisAjmErrorInvalidParameter); - } - - return ctx.SetReturn(0); - } + public static int Initialize(CpuContext ctx) => + SharpEmu.Libs.Audio.AjmExports.AjmInitialize(ctx); } diff --git a/src/SharpEmu.Libs/Ampr/AmprExports.cs b/src/SharpEmu.Libs/Ampr/AmprExports.cs index 0c41b05..0dc0cfe 100644 --- a/src/SharpEmu.Libs/Ampr/AmprExports.cs +++ b/src/SharpEmu.Libs/Ampr/AmprExports.cs @@ -36,6 +36,7 @@ public static class AmprExports public ulong Buffer; public ulong Size; public ulong WriteOffset; + public ulong CommandCount; } private sealed class CachedHostFile @@ -267,53 +268,19 @@ public static class AmprExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - ulong bytesRead = 0; - - // Unregistered/missing files are zero-filled instead of failing: games queue - // speculative reads and only consume the bytes on success paths. - if (!AmprFileRegistry.TryGetHostPath(fileId, out var hostPath) || !File.Exists(hostPath)) + if (!AmprFileRegistry.TryGetHostPath(fileId, out var hostPath)) { - if (destination != 0 && size > 0) - { - int chunkSize = (int)Math.Min(size, 4096); - Span zeros = stackalloc byte[chunkSize]; - zeros.Clear(); - while (bytesRead < size) - { - int currentChunk = (int)Math.Min((ulong)chunkSize, size - bytesRead); - if (!ctx.Memory.TryWrite(destination + bytesRead, zeros[..currentChunk])) - { - break; - } - - bytesRead += (ulong)currentChunk; - } - } - - TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead, "(missing)", (int)OrbisGen2Result.ORBIS_GEN2_OK); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead: 0, hostPath, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - // Offset -1 means "continue after the previous read of this file id". - if (fileOffset == unchecked((ulong)(long)-1)) - { - fileOffset = PakDirectoryTracker.ResolveSequentialOffset(fileId, size); - } - else if (fileOffset > long.MaxValue) - { - fileOffset = 0; - } - - var result = TryReadFileToGuestMemory(ctx, hostPath, fileOffset, destination, size, out bytesRead); + var result = TryReadFileToGuestMemory(ctx, hostPath, fileOffset, destination, size, out var bytesRead); if (result != (int)OrbisGen2Result.ORBIS_GEN2_OK) { TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead, hostPath, result); return result; } - PakDirectoryTracker.OnReadCompleted(ctx, fileId, destination, fileOffset, bytesRead); - if (!AppendReadFileRecord(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; @@ -418,6 +385,35 @@ public static class AmprExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + [SysAbiExport( + Nid = "gzndltBEzWc", + ExportName = "sceAmprCommandBufferGetNumCommands", + Target = Generation.Gen5, + LibraryName = "libSceAmpr")] + public static int CommandBufferGetNumCommands(CpuContext ctx) + { + var commandBuffer = ctx[CpuRegister.Rdi]; + if (commandBuffer == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!TryGetCommandBufferState(ctx, commandBuffer, out _, out _, out var state) || state is null) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ulong commandCount; + lock (state) + { + commandCount = state.CommandCount; + } + + TraceAmpr(ctx, "get_num_commands", commandBuffer, commandCount, 0); + ctx[CpuRegister.Rax] = commandCount; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + [SysAbiExport( Nid = "H896Pt-yB4I", ExportName = "sceAmprCommandBufferWriteKernelEventQueue_04_00", @@ -533,7 +529,7 @@ public static class AmprExports var offset = 0UL; while (offset < writeOffset) { - if (!ctx.TryReadUInt32(buffer + offset, out var recordType)) + if (!TryReadUInt32(ctx, buffer + offset, out var recordType)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -649,6 +645,7 @@ public static class AmprExports state.Buffer = buffer; state.Size = size; state.WriteOffset = writeOffset; + state.CommandCount = 0; } } @@ -681,6 +678,7 @@ public static class AmprExports state.Buffer = buffer; state.Size = size; state.WriteOffset = 0; + state.CommandCount = 0; } return true; @@ -911,6 +909,7 @@ public static class AmprExports } state.WriteOffset += recordSize; + state.CommandCount++; } return true; @@ -963,6 +962,19 @@ public static class AmprExports return true; } + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + private static void TraceAmpr(CpuContext ctx, string operation, ulong commandBuffer, ulong arg0, ulong arg1) { if (!_traceAmpr) diff --git a/src/SharpEmu.Libs/Audio/AjmExports.cs b/src/SharpEmu.Libs/Audio/AjmExports.cs new file mode 100644 index 0000000..7ac0a1a --- /dev/null +++ b/src/SharpEmu.Libs/Audio/AjmExports.cs @@ -0,0 +1,104 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Threading; + +namespace SharpEmu.Libs.Audio; + +public static class AjmExports +{ + private static readonly ConcurrentDictionary Contexts = new(); + private static int _nextContextId; + public static int AjmInitialize(CpuContext ctx) + { + var reserved = ctx[CpuRegister.Rdi]; + var outputAddress = ctx[CpuRegister.Rsi]; + if (reserved != 0 || outputAddress == 0) + { + return unchecked((int)0x806A0001); + } + + var contextId = unchecked((uint)Interlocked.Increment(ref _nextContextId)); + Span value = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(value, contextId); + if (!ctx.Memory.TryWrite(outputAddress, value)) + { + return unchecked((int)0x806A0001); + } + + Contexts[contextId] = 0; + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] ajm.initialize reserved={reserved} out=0x{outputAddress:X16} context={contextId}"); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "MHur6qCsUus", + ExportName = "sceAjmFinalize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAjm")] + public static int AjmFinalize(CpuContext ctx) + { + Contexts.TryRemove(unchecked((uint)ctx[CpuRegister.Rdi]), out _); + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "Q3dyFuwGn64", + ExportName = "sceAjmModuleRegister", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAjm")] + public static int AjmModuleRegister(CpuContext ctx) + { + var contextId = unchecked((uint)ctx[CpuRegister.Rdi]); + var codecType = unchecked((uint)ctx[CpuRegister.Rsi]); + var reserved = ctx[CpuRegister.Rdx]; + if (reserved != 0 || !Contexts.ContainsKey(contextId)) + { + return unchecked((int)0x806A0001); + } + + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] ajm.module_register context={contextId} codec={codecType} reserved={reserved}"); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "Wi7DtlLV+KI", + ExportName = "sceAjmModuleUnregister", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAjm")] + public static int AjmModuleUnregister(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "MmpF1XsQiHw", + ExportName = "sceAjmBatchInitialize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAjm")] + public static int AjmBatchInitialize(CpuContext ctx) + { + // The caller owns and initializes the batch storage. This API resets + // its submission cursor on hardware; FMOD does not consume a return + // value or an additional output object here. + ctx[CpuRegister.Rax] = 0; + return 0; + } +} diff --git a/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs b/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs index 730e3e9..18b3751 100644 --- a/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs +++ b/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs @@ -3,22 +3,70 @@ using SharpEmu.HLE; using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Threading; namespace SharpEmu.Libs.Audio; public static class AudioOut2Exports { - // Sized from guest evidence, not SDK headers: Quake keeps its - // SceAudioOut2ContextParam on the stack with the frame canary at param+0x60, - // and an earlier 0x80-byte ResetParam write zeroed that canary - // (__stack_chk_fail right after sceAudioOut2UserCreate, which silently killed - // the whole audio init). Stay well below 0x60 and only write the prefix we - // populate. - private const int AudioOut2ContextParamSize = 0x30; + // FMOD's PS5 backend allocates this ABI structure as four 16-byte lanes. + // Clearing 0x80 bytes here overwrote the caller's stack canary immediately + // following the 0x40-byte parameter block. + private const int AudioOut2ContextParamSize = 0x40; private const int AudioOut2ContextMemorySize = 0x10000; + private const int AudioOut2ContextMemoryAlignment = 0x10000; private static long _nextContextHandle = 1; private static long _nextUserHandle = 1; private static int _nextPortId; + private static long _pushTraceCount; + + // Per-context audio parameters captured at ContextCreate so ContextAdvance + // can pace to the real playback cadence (grain samples at the sample rate). + private static readonly ConcurrentDictionary Contexts = new(); + + private sealed class ContextState + { + private readonly object _paceGate = new(); + private long _nextAdvanceTimestamp; + + public ContextState(uint frequency, uint channels, uint grainSamples) + { + Frequency = frequency == 0 ? 48000 : frequency; + Channels = channels == 0 ? 2 : channels; + GrainSamples = grainSamples == 0 ? 256 : grainSamples; + } + + public uint Frequency { get; } + public uint Channels { get; } + public uint GrainSamples { get; } + + // Blocks the advancing thread until one grain worth of wall-clock time + // has elapsed since the previous advance, matching hardware timing so + // audio-gated titles neither spin nor drift ahead. + public void PaceAdvance() + { + long delay; + lock (_paceGate) + { + var now = Stopwatch.GetTimestamp(); + if (_nextAdvanceTimestamp < now) + { + _nextAdvanceTimestamp = now; + } + + delay = _nextAdvanceTimestamp - now; + _nextAdvanceTimestamp += checked( + (long)Math.Ceiling(Stopwatch.Frequency * (double)GrainSamples / Frequency)); + } + + if (delay > 0) + { + Thread.Sleep(TimeSpan.FromSeconds((double)delay / Stopwatch.Frequency)); + } + } + } [SysAbiExport( Nid = "g2tViFIohHE", @@ -41,7 +89,7 @@ public static class AudioOut2Exports var paramAddress = ctx[CpuRegister.Rdi]; if (paramAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } Span param = stackalloc byte[AudioOut2ContextParamSize]; @@ -52,8 +100,8 @@ public static class AudioOut2Exports BinaryPrimitives.WriteUInt32LittleEndian(param[0x0C..], 0x400); return ctx.Memory.TryWrite(paramAddress, param) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -64,15 +112,22 @@ public static class AudioOut2Exports public static int AudioOut2ContextQueryMemory(CpuContext ctx) { var paramAddress = ctx[CpuRegister.Rdi]; - var outMemorySizeAddress = ctx[CpuRegister.Rsi]; - if (paramAddress == 0 || outMemorySizeAddress == 0) + var memoryInfoAddress = ctx[CpuRegister.Rsi]; + if (paramAddress == 0 || memoryInfoAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - - return ctx.TryWriteUInt64(outMemorySizeAddress, AudioOut2ContextMemorySize) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + + Span memoryInfo = stackalloc byte[0x20]; + memoryInfo.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(memoryInfo[0x00..], AudioOut2ContextMemorySize); + BinaryPrimitives.WriteUInt64LittleEndian(memoryInfo[0x08..], AudioOut2ContextMemoryAlignment); + BinaryPrimitives.WriteUInt64LittleEndian(memoryInfo[0x10..], AudioOut2ContextMemorySize); + BinaryPrimitives.WriteUInt64LittleEndian(memoryInfo[0x18..], AudioOut2ContextMemoryAlignment); + + return ctx.Memory.TryWrite(memoryInfoAddress, memoryInfo) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -88,13 +143,34 @@ public static class AudioOut2Exports var outContextAddress = ctx[CpuRegister.Rcx]; if (paramAddress == 0 || memoryAddress == 0 || memorySize == 0 || outContextAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + // Read channels/frequency/grain from the reset-param blob so the + // context can pace advances to the real audio cadence. + uint channels = 2; + uint frequency = 48000; + uint grain = 256; + Span param = stackalloc byte[AudioOut2ContextParamSize]; + if (ctx.Memory.TryRead(paramAddress, param)) + { + var pc = BinaryPrimitives.ReadUInt32LittleEndian(param[0x04..]); + var pf = BinaryPrimitives.ReadUInt32LittleEndian(param[0x08..]); + var pg = BinaryPrimitives.ReadUInt32LittleEndian(param[0x0C..]); + if (pc is > 0 and <= 8) channels = pc; + if (pf is >= 8000 and <= 192000) frequency = pf; + // Values below one cache line are flags/counts in observed PS5 + // callers, not audio grains. Keep the hardware-sized default. + if (pg is >= 64 and <= 0x4000) grain = pg; + TraceAudioOut2($"context-param address=0x{paramAddress:X} bytes={Convert.ToHexString(param)}"); } var handle = (ulong)Interlocked.Increment(ref _nextContextHandle); - return ctx.TryWriteUInt64(outContextAddress, handle) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + Contexts[handle] = new ContextState(frequency, channels, grain); + TraceAudioOut2($"context-create handle=0x{handle:X} frequency={frequency} channels={channels} grain={grain} memory=0x{memoryAddress:X} size=0x{memorySize:X}"); + return TryWriteUInt64(ctx, outContextAddress, handle) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -102,7 +178,77 @@ public static class AudioOut2Exports ExportName = "sceAudioOut2ContextDestroy", Target = Generation.Gen5, LibraryName = "libSceAudioOut2")] - public static int AudioOut2ContextDestroy(CpuContext ctx) => ctx.SetReturn(0); + public static int AudioOut2ContextDestroy(CpuContext ctx) + { + Contexts.TryRemove(ctx[CpuRegister.Rdi], out _); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "DxGyV8dtOR8", + ExportName = "sceAudioOut2ContextBedWrite", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2ContextBedWrite(CpuContext ctx) => SetReturn(ctx, 0); + + [SysAbiExport( + Nid = "aII9h5nli9U", + ExportName = "sceAudioOut2ContextPush", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2ContextPush(CpuContext ctx) + { + var handle = ctx[CpuRegister.Rdi]; + var traceCount = Interlocked.Increment(ref _pushTraceCount); + if (traceCount <= 16) + { + TraceAudioOut2($"context-push count={traceCount} rdi=0x{handle:X} rsi=0x{ctx[CpuRegister.Rsi]:X} rdx=0x{ctx[CpuRegister.Rdx]:X} rcx=0x{ctx[CpuRegister.Rcx]:X}"); + } + + if (Contexts.TryGetValue(handle, out var context)) + { + // FMOD's PS5 output path uses ContextPush as the submission clock + // and does not call ContextAdvance. Pace pushes to one hardware + // grain so the feeder cannot outrun playback and starve the game. + context.PaceAdvance(); + } + + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "PE2zHMqLSHs", + ExportName = "sceAudioOut2ContextAdvance", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2ContextAdvance(CpuContext ctx) + { + // Advancing renders one grain of audio on hardware; pace it to the same + // wall-clock cadence so the guest audio thread runs at the right speed. + if (Contexts.TryGetValue(ctx[CpuRegister.Rdi], out var context)) + { + context.PaceAdvance(); + } + + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "R7d0F1g2qsU", + ExportName = "sceAudioOut2ContextGetQueueLevel", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2ContextGetQueueLevel(CpuContext ctx) + { + // The advance path paces synchronously, so the queue is always drained. + var levelAddress = ctx[CpuRegister.Rsi]; + if (levelAddress != 0) + { + _ = TryWriteUInt64(ctx, levelAddress, 0); + } + + return SetReturn(ctx, 0); + } [SysAbiExport( Nid = "JK2wamZPzwM", @@ -117,16 +263,23 @@ public static class AudioOut2Exports var contextAddress = ctx[CpuRegister.Rcx]; if (type < 0 || type > 255 || paramAddress == 0 || outPortAddress == 0 || contextAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var portId = unchecked((uint)Interlocked.Increment(ref _nextPortId)) & 0xFF; var handle = 0x2000_0000UL | ((ulong)(uint)type << 16) | portId; - return ctx.TryWriteUInt64(outPortAddress, handle) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return TryWriteUInt64(ctx, outPortAddress, handle) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "8XTArSPyWHk", + ExportName = "sceAudioOut2PortSetAttributes", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2PortSetAttributes(CpuContext ctx) => SetReturn(ctx, 0); + [SysAbiExport( Nid = "gatEUKG+Ea4", ExportName = "sceAudioOut2PortGetState", @@ -138,7 +291,7 @@ public static class AudioOut2Exports var stateAddress = ctx[CpuRegister.Rsi]; if (handle == 0 || stateAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var type = (int)((handle >> 16) & 0xFF); @@ -151,8 +304,8 @@ public static class AudioOut2Exports BinaryPrimitives.WriteInt16LittleEndian(state[0x04..], -1); return ctx.Memory.TryWrite(stateAddress, state) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -165,7 +318,7 @@ public static class AudioOut2Exports var infoAddress = ctx[CpuRegister.Rdi]; if (infoAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } Span info = stackalloc byte[0x40]; @@ -175,8 +328,8 @@ public static class AudioOut2Exports BinaryPrimitives.WriteUInt32LittleEndian(info[0x08..], 48000); return ctx.Memory.TryWrite(infoAddress, info) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -184,14 +337,14 @@ public static class AudioOut2Exports ExportName = "sceAudioOut2PortDestroy", Target = Generation.Gen5, LibraryName = "libSceAudioOut2")] - public static int AudioOut2PortDestroy(CpuContext ctx) => ctx.SetReturn(0); + public static int AudioOut2PortDestroy(CpuContext ctx) => SetReturn(ctx, 0); [SysAbiExport( Nid = "IaZXJ9M79uo", ExportName = "sceAudioOut2UserDestroy", Target = Generation.Gen5, LibraryName = "libSceAudioOut2")] - public static int AudioOut2UserDestroy(CpuContext ctx) => ctx.SetReturn(0); + public static int AudioOut2UserDestroy(CpuContext ctx) => SetReturn(ctx, 0); [SysAbiExport( Nid = "xywYcRB7nbQ", @@ -202,14 +355,36 @@ public static class AudioOut2Exports { var userId = unchecked((int)ctx[CpuRegister.Rdi]); var outUserAddress = ctx[CpuRegister.Rsi]; - if ((userId != 0 && userId != 1 && userId != 1000 && userId != 255) || outUserAddress == 0) + if ((userId != 0 && userId != 1 && userId != 1000 && userId != 0x10000000 && userId != 255) || + outUserAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var handle = (ulong)Interlocked.Increment(ref _nextUserHandle); - return ctx.TryWriteUInt64(outUserAddress, handle) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return TryWriteUInt64(ctx, outUserAddress, handle) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + private static bool TryWriteUInt64(CpuContext ctx, ulong address, ulong value) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + + private static void TraceAudioOut2(string message) + { + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AUDIO_OUT2"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine($"[LOADER][TRACE] audio_out2.{message}"); + } } } diff --git a/src/SharpEmu.Libs/Audio/AudioOutExports.cs b/src/SharpEmu.Libs/Audio/AudioOutExports.cs index a044e58..a736b8c 100644 --- a/src/SharpEmu.Libs/Audio/AudioOutExports.cs +++ b/src/SharpEmu.Libs/Audio/AudioOutExports.cs @@ -50,6 +50,7 @@ public static class AudioOutExports public int BytesPerSample { get; } public bool IsFloat { get; } public IHostAudioStream? Backend { get; } + public volatile float Volume = 1.0f; public int BufferByteLength => checked((int)BufferLength * Channels * BytesPerSample); @@ -198,7 +199,8 @@ public static class AudioOutExports checked((int)port.BufferLength), port.Channels, port.BytesPerSample, - port.IsFloat); + port.IsFloat, + port.Volume); if (!port.Backend.Submit(output.AsSpan(0, outputLength))) { port.PaceSilence(); @@ -225,10 +227,43 @@ public static class AudioOutExports public static int AudioOutSetVolume(CpuContext ctx) { var handle = unchecked((int)ctx[CpuRegister.Rdi]); - return ctx.SetReturn( - Ports.ContainsKey(handle) - ? 0 - : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + var channelFlags = unchecked((uint)ctx[CpuRegister.Rsi]); + var volumeArrayAddress = ctx[CpuRegister.Rdx]; + if (!Ports.TryGetValue(handle, out var port)) + { + return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + const int unityVolume = 32768; + var maxVolume = 0; + var found = false; + if (volumeArrayAddress != 0) + { + Span raw = stackalloc byte[sizeof(int)]; + for (var channel = 0; channel < 8; channel++) + { + if ((channelFlags & (1u << channel)) == 0) + { + continue; + } + + if (!ctx.Memory.TryRead(volumeArrayAddress + (ulong)(channel * sizeof(int)), raw)) + { + return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + var value = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(raw); + maxVolume = Math.Max(maxVolume, value); + found = true; + } + } + + if (found) + { + port.Volume = Math.Clamp(maxVolume / (float)unityVolume, 0f, 1f); + } + + return ctx.SetReturn(0); } public static void ShutdownAllPorts() diff --git a/src/SharpEmu.Libs/Audio/AudioPcmConversion.cs b/src/SharpEmu.Libs/Audio/AudioPcmConversion.cs index a15ce19..6e65b4c 100644 --- a/src/SharpEmu.Libs/Audio/AudioPcmConversion.cs +++ b/src/SharpEmu.Libs/Audio/AudioPcmConversion.cs @@ -21,7 +21,8 @@ internal static class AudioPcmConversion int frames, int channels, int bytesPerSample, - bool isFloat) + bool isFloat, + float volume) { var sourceFrameSize = checked(channels * bytesPerSample); for (var frame = 0; frame < frames; frame++) @@ -31,6 +32,8 @@ internal static class AudioPcmConversion var right = channels == 1 ? left : ReadSample(sourceFrame, 1, bytesPerSample, isFloat); + left = ApplyVolume(left, volume); + right = ApplyVolume(right, volume); BinaryPrimitives.WriteInt16LittleEndian(destination[(frame * OutputFrameSize)..], left); BinaryPrimitives.WriteInt16LittleEndian(destination[((frame * OutputFrameSize) + 2)..], right); } @@ -52,4 +55,10 @@ internal static class AudioPcmConversion var value = Math.Clamp(BitConverter.Int32BitsToSingle(bits), -1.0f, 1.0f); return checked((short)MathF.Round(value * short.MaxValue)); } + + private static short ApplyVolume(short sample, float volume) + { + var scaled = MathF.Round(sample * Math.Clamp(volume, 0.0f, 1.0f)); + return (short)Math.Clamp(scaled, short.MinValue, short.MaxValue); + } } diff --git a/src/SharpEmu.Libs/Audio/FmodCompatExports.cs b/src/SharpEmu.Libs/Audio/FmodCompatExports.cs new file mode 100644 index 0000000..35e65fc --- /dev/null +++ b/src/SharpEmu.Libs/Audio/FmodCompatExports.cs @@ -0,0 +1,47 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using System.Buffers.Binary; +using System.Collections.Concurrent; + +namespace SharpEmu.Libs.Audio; + +public static class FmodCompatExports +{ + private static readonly ConcurrentDictionary SetOutputCalls = new(); + + [SysAbiExport( + Nid = "uPLTdl3psGk", + Target = Generation.Gen5, + LibraryName = "libfmod")] + public static int FmodSystemSetOutput(CpuContext ctx) + { + var system = ctx[CpuRegister.Rdi]; + var output = unchecked((int)ctx[CpuRegister.Rsi]); + var callCount = SetOutputCalls.AddOrUpdate(system, 1, static (_, count) => count + 1); + var resetPrematureInit = callCount <= 2; + if (resetPrematureInit && system != 0) + { + Span zeroByte = stackalloc byte[1]; + Span outputBytes = stackalloc byte[sizeof(int)]; + Span zeroInt = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(outputBytes, output); + _ = ctx.Memory.TryWrite(system + 0x08, zeroByte); + _ = ctx.Memory.TryWrite(system + 0x116D0, outputBytes); + _ = ctx.Memory.TryWrite(system + 0x116D4, zeroInt); + } + + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_FMOD"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] fmod.system_set_output system=0x{system:X16} output={output} call={callCount} reset_premature_init={resetPrematureInit} -> 0"); + } + + // The PS5 FMOD object arrives with its initialized byte set before Unity has + // applied the startup output configuration. Preserve those settings while + // clearing the premature marker so Studio can execute the real core init. + ctx[CpuRegister.Rax] = 0; + return 0; + } +} diff --git a/src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs b/src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs index 2153bc7..f34c0f3 100644 --- a/src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs +++ b/src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs @@ -3,14 +3,114 @@ using SharpEmu.HLE; using SharpEmu.Libs.Kernel; +using System.Buffers.Binary; +using System.Diagnostics; +using System.Globalization; +using System.Text; namespace SharpEmu.Libs.AvPlayer; public static class AvPlayerExports { private const int InvalidParameters = unchecked((int)0x806A0001); + private const int OperationFailed = unchecked((int)0x806A0002); + private const int FrameBufferCount = 3; + private const int FrameInfoSize = 40; + private const int FrameInfoExSize = 104; + private const int StreamInfoSize = 40; + private const int MaxGuestPathLength = 4096; private static readonly object StateGate = new(); - private static readonly HashSet Players = new(); + private static readonly Dictionary Players = new(); + private static int _traceCount; + + private sealed class PlayerState : IDisposable + { + public required ulong Handle { get; init; } + public bool AutoStart { get; init; } + public ulong AllocatorObject { get; init; } + public ulong AllocateTextureCallback { get; init; } + public ulong EventObject { get; init; } + public ulong EventCallback { get; init; } + public string? SourcePath { get; set; } + public int Width { get; set; } + public int Height { get; set; } + public double FramesPerSecond { get; set; } = 30.0; + public ulong DurationMilliseconds { get; set; } + public bool Started { get; set; } + public bool Paused { get; set; } + public bool Looping { get; set; } + public bool EndOfStream { get; set; } + public Process? Decoder { get; set; } + public Stream? DecoderOutput { get; set; } + public Process? AudioDecoder { get; set; } + public Stream? AudioDecoderOutput { get; set; } + public Stopwatch PlaybackClock { get; } = new(); + public byte[]? RawFrame { get; set; } + public byte[]? RawAudioFrame { get; set; } + public byte[]? PaddedFrame { get; set; } + public ulong[] GuestBuffers { get; } = new ulong[FrameBufferCount]; + public bool TextureAllocatorFailed { get; set; } + public int GuestBufferStride { get; set; } + public int NextGuestBuffer { get; set; } + public ulong LastGuestBuffer { get; set; } + public long NextFrameIndex { get; set; } + public ulong AudioBufferBase { get; set; } + public int NextAudioBuffer { get; set; } + public long NextAudioFrameIndex { get; set; } + + public void Dispose() + { + DecoderOutput?.Dispose(); + DecoderOutput = null; + AudioDecoderOutput?.Dispose(); + AudioDecoderOutput = null; + if (Decoder is not null) + { + try + { + if (!Decoder.HasExited) + { + Decoder.Kill(entireProcessTree: true); + } + } + catch (InvalidOperationException) + { + } + finally + { + Decoder.Dispose(); + Decoder = null; + } + } + if (AudioDecoder is not null) + { + try + { + if (!AudioDecoder.HasExited) + { + AudioDecoder.Kill(entireProcessTree: true); + } + } + catch (InvalidOperationException) + { + } + finally + { + AudioDecoder.Dispose(); + AudioDecoder = null; + } + } + } + + public void ResetPlayback() + { + Dispose(); + PlaybackClock.Reset(); + NextFrameIndex = 0; + NextAudioFrameIndex = 0; + EndOfStream = false; + } + } [SysAbiExport( Nid = "aS66RI0gGgo", @@ -19,7 +119,8 @@ public static class AvPlayerExports LibraryName = "libSceAvPlayer")] public static int AvPlayerInit(CpuContext ctx) { - if (ctx[CpuRegister.Rdi] == 0 || + var initDataAddress = ctx[CpuRegister.Rdi]; + if (initDataAddress == 0 || !KernelMemoryCompatExports.TryAllocateHleData(ctx, 0x40, 16, out var handle)) { ctx[CpuRegister.Rax] = 0; @@ -28,60 +129,22 @@ public static class AvPlayerExports lock (StateGate) { - Players.Add(handle); + Players.Add(handle, new PlayerState + { + Handle = handle, + AutoStart = TryReadByte(ctx, initDataAddress + 108, out var autoStart) && autoStart != 0, + AllocatorObject = TryReadUInt64(ctx, initDataAddress, out var allocatorObject) ? allocatorObject : 0, + AllocateTextureCallback = TryReadUInt64(ctx, initDataAddress + 24, out var allocateTexture) ? allocateTexture : 0, + EventObject = TryReadUInt64(ctx, initDataAddress + 80, out var eventObject) ? eventObject : 0, + EventCallback = TryReadUInt64(ctx, initDataAddress + 88, out var eventCallback) ? eventCallback : 0, + }); } + Trace($"init handle=0x{handle:X16} alloc_texture=0x{Players[handle].AllocateTextureCallback:X16}"); ctx[CpuRegister.Rax] = handle; return unchecked((int)handle); } - [SysAbiExport( - Nid = "KMcEa+rHsIo", - ExportName = "sceAvPlayerAddSource", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceAvPlayer")] - public static int AvPlayerAddSource(CpuContext ctx) - { - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "JdksQu8pNdQ", - ExportName = "sceAvPlayerGetVideoDataEx", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceAvPlayer")] - public static int AvPlayerGetVideoDataEx(CpuContext ctx) - { - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - // "UbQoYawOsfY" is sceAvPlayerIsActive, not sceAvPlayerGetVideoDataEx (the previous NID here - // was wrong - verified by hashing both names against scripts/ps5_names.txt). No player is - // ever actually active in this HLE implementation, so report false/inactive. - [SysAbiExport( - Nid = "UbQoYawOsfY", - ExportName = "sceAvPlayerIsActive", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceAvPlayer")] - public static int AvPlayerIsActive(CpuContext ctx) - { - ctx[CpuRegister.Rax] = 0; - return 0; - } - - [SysAbiExport( - Nid = "Wnp1OVcrZgk", - ExportName = "sceAvPlayerGetAudioData", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceAvPlayer")] - public static int AvPlayerGetAudioData(CpuContext ctx) - { - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "HD1YKVU26-M", ExportName = "sceAvPlayerPostInit", @@ -93,13 +156,55 @@ public static class AvPlayerExports var dataAddress = ctx[CpuRegister.Rsi]; lock (StateGate) { - return ctx.SetReturn( - handle != 0 && dataAddress != 0 && Players.Contains(handle) + return SetReturn( + ctx, + handle != 0 && dataAddress != 0 && Players.ContainsKey(handle) ? 0 : InvalidParameters); } } + [SysAbiExport( + Nid = "o9eWRkSL+M4", + ExportName = "sceAvPlayerInitEx", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerInitEx(CpuContext ctx) + { + var initDataAddress = ctx[CpuRegister.Rdi]; + var playerOutAddress = ctx[CpuRegister.Rsi]; + if (initDataAddress == 0 || + playerOutAddress == 0 || + !KernelMemoryCompatExports.TryAllocateHleData(ctx, 0x40, 16, out var handle) || + !ctx.TryWriteUInt64(playerOutAddress, handle)) + { + return SetReturn(ctx, InvalidParameters); + } + + lock (StateGate) + { + Players.Add(handle, new PlayerState + { + Handle = handle, + AutoStart = TryReadByte(ctx, initDataAddress + 164, out var autoStart) && autoStart != 0, + AllocatorObject = TryReadUInt64(ctx, initDataAddress + 8, out var allocatorObject) ? allocatorObject : 0, + AllocateTextureCallback = TryReadUInt64(ctx, initDataAddress + 32, out var allocateTexture) ? allocateTexture : 0, + EventObject = TryReadUInt64(ctx, initDataAddress + 88, out var eventObject) ? eventObject : 0, + EventCallback = TryReadUInt64(ctx, initDataAddress + 96, out var eventCallback) ? eventCallback : 0, + }); + } + + Trace($"init_ex handle=0x{handle:X16} alloc_texture=0x{Players[handle].AllocateTextureCallback:X16}"); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "eBTreZ84JFY", + ExportName = "sceAvPlayerSetLogCallback", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerSetLogCallback(CpuContext ctx) => SetReturn(ctx, 0); + [SysAbiExport( Nid = "NkJwDzKmIlw", ExportName = "sceAvPlayerClose", @@ -107,9 +212,1063 @@ public static class AvPlayerExports LibraryName = "libSceAvPlayer")] public static int AvPlayerClose(CpuContext ctx) { + PlayerState? player; lock (StateGate) { - return ctx.SetReturn(Players.Remove(ctx[CpuRegister.Rdi]) ? 0 : InvalidParameters); + if (!Players.Remove(ctx[CpuRegister.Rdi], out player)) + { + return SetReturn(ctx, InvalidParameters); + } + } + + player.Dispose(); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "KMcEa+rHsIo", + ExportName = "sceAvPlayerAddSource", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerAddSource(CpuContext ctx) + { + if (!TryReadNullTerminatedUtf8(ctx, ctx[CpuRegister.Rsi], MaxGuestPathLength, out var path)) + { + return SetReturn(ctx, InvalidParameters); + } + + return AddSource(ctx, path); + } + + [SysAbiExport( + Nid = "x8uvuFOPZhU", + ExportName = "sceAvPlayerAddSourceEx", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerAddSourceEx(CpuContext ctx) + { + var uriType = unchecked((uint)ctx[CpuRegister.Rsi]); + var detailsAddress = ctx[CpuRegister.Rdx]; + if (uriType != 0 || detailsAddress == 0 || + !ctx.TryReadUInt64(detailsAddress, out var pathAddress) || + !TryReadUInt32(ctx, detailsAddress + sizeof(ulong), out var pathLength) || + pathLength == 0 || pathLength > MaxGuestPathLength || + !TryReadUtf8(ctx, pathAddress, checked((int)pathLength), out var path)) + { + return SetReturn(ctx, InvalidParameters); + } + + return AddSource(ctx, path.TrimEnd('\0')); + } + + [SysAbiExport( + Nid = "ET4Gr-Uu07s", + ExportName = "sceAvPlayerStart", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerStart(CpuContext ctx) + { + PlayerState player; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var foundPlayer) || foundPlayer.SourcePath is null) + { + return SetReturn(ctx, InvalidParameters); + } + player = foundPlayer; + + player.Started = true; + player.Paused = false; + player.EndOfStream = false; + Trace($"start handle=0x{player.Handle:X16}"); + } + + // Event callbacks are guest code and can immediately query the player. + // Never hold StateGate while waiting for one or the callback deadlocks + // when it re-enters an AvPlayer export on another guest worker. + NotifyEvent(ctx, player, 3); // StatePlay + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "ZC17w3vB5Lo", + ExportName = "sceAvPlayerStop", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerStop(CpuContext ctx) + { + PlayerState player; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var foundPlayer)) + { + return SetReturn(ctx, InvalidParameters); + } + player = foundPlayer; + + player.ResetPlayback(); + player.Started = false; + } + + NotifyEvent(ctx, player, 1); // StateStop + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "9y5v+fGN4Wk", + ExportName = "sceAvPlayerPause", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerPause(CpuContext ctx) + { + PlayerState player; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var foundPlayer)) + { + return SetReturn(ctx, InvalidParameters); + } + player = foundPlayer; + + player.Paused = true; + player.PlaybackClock.Stop(); + } + + + NotifyEvent(ctx, player, 4); // StatePause + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "w5moABNwnRY", + ExportName = "sceAvPlayerResume", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerResume(CpuContext ctx) + { + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player)) + { + return SetReturn(ctx, InvalidParameters); + } + + player.Paused = false; + if (player.Decoder is not null) + { + player.PlaybackClock.Start(); + } + return SetReturn(ctx, 0); + } + } + + [SysAbiExport( + Nid = "OVths0xGfho", + ExportName = "sceAvPlayerSetLooping", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerSetLooping(CpuContext ctx) + { + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player)) + { + return SetReturn(ctx, InvalidParameters); + } + + player.Looping = ctx[CpuRegister.Rsi] != 0; + return SetReturn(ctx, 0); + } + } + + [SysAbiExport( + Nid = "ODJK2sn9w4A", + ExportName = "sceAvPlayerEnableStream", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerEnableStream(CpuContext ctx) => ValidatePlayer(ctx); + + [SysAbiExport( + Nid = "k-q+xOxdc3E", + ExportName = "sceAvPlayerSetAvSyncMode", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerSetAvSyncMode(CpuContext ctx) + { + Trace($"set_av_sync_mode handle=0x{ctx[CpuRegister.Rdi]:X16} mode={ctx[CpuRegister.Rsi]}"); + return ValidatePlayer(ctx); + } + + [SysAbiExport( + Nid = "ctTAcF5DiKQ", + ExportName = "sceAvPlayerGetStreamInfoEx", + Target = Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerSetDecoderMode(CpuContext ctx) => ValidatePlayer(ctx); + + [SysAbiExport( + Nid = "XC9wM+xULz8", + ExportName = "sceAvPlayerJumpToTime", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerJumpToTime(CpuContext ctx) + { + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player)) + { + return SetReturn(ctx, InvalidParameters); + } + + player.ResetPlayback(); + player.Started = true; + return SetReturn(ctx, 0); + } + } + + [SysAbiExport( + Nid = "yN7Jhuv8g24", + ExportName = "sceAvPlayerVprintf", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerVprintf(CpuContext ctx) => SetReturn(ctx, 0); + + [SysAbiExport( + Nid = "UbQoYawOsfY", + ExportName = "sceAvPlayerIsActive", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerIsActive(CpuContext ctx) + { + lock (StateGate) + { + return SetReturn( + ctx, + Players.TryGetValue(ctx[CpuRegister.Rdi], out var player) && + player.Started && !player.EndOfStream ? 1 : 0); + } + } + + [SysAbiExport( + Nid = "o3+RWnHViSg", + ExportName = "sceAvPlayerGetVideoData", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerGetVideoData(CpuContext ctx) => GetVideoData(ctx, extended: false); + + [SysAbiExport( + Nid = "JdksQu8pNdQ", + ExportName = "sceAvPlayerGetVideoDataEx", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerGetVideoDataEx(CpuContext ctx) => GetVideoData(ctx, extended: true); + + [SysAbiExport( + Nid = "Wnp1OVcrZgk", + ExportName = "sceAvPlayerGetAudioData", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerGetAudioData(CpuContext ctx) + { + var infoAddress = ctx[CpuRegister.Rsi]; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player) || + infoAddress == 0 || !player.Started || player.Paused || player.EndOfStream || + player.SourcePath is null || !EnsureAudioDecoder(player)) + { + return SetReturn(ctx, 0); + } + + const int samplesPerFrame = 1024; + const int channelCount = 2; + const int sampleRate = 48_000; + const int audioFrameSize = samplesPerFrame * channelCount * sizeof(short); + if (player.RawAudioFrame is null || + !ReadExactly(player.AudioDecoderOutput, player.RawAudioFrame)) + { + return SetReturn(ctx, 0); + } + if (player.AudioBufferBase == 0) + { + if (!KernelMemoryCompatExports.TryAllocateHleData( + ctx, + audioFrameSize * 8UL, + 0x100, + out var audioBufferBase)) + { + return SetReturn(ctx, 0); + } + player.AudioBufferBase = audioBufferBase; + } + + var bufferAddress = player.AudioBufferBase + + checked((ulong)(player.NextAudioBuffer * audioFrameSize)); + player.NextAudioBuffer = (player.NextAudioBuffer + 1) % 8; + if (!ctx.Memory.TryWrite(bufferAddress, player.RawAudioFrame)) + { + return SetReturn(ctx, 0); + } + + var timestamp = checked((ulong)(player.NextAudioFrameIndex * samplesPerFrame * 1000L / sampleRate)); + player.NextAudioFrameIndex++; + Span info = stackalloc byte[FrameInfoSize]; + info.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(info[0..], bufferAddress); + BinaryPrimitives.WriteUInt64LittleEndian(info[16..], timestamp); + BinaryPrimitives.WriteUInt16LittleEndian(info[24..], channelCount); + BinaryPrimitives.WriteUInt32LittleEndian(info[28..], sampleRate); + BinaryPrimitives.WriteUInt32LittleEndian(info[32..], audioFrameSize); + if (!ctx.Memory.TryWrite(infoAddress, info)) + { + return SetReturn(ctx, 0); + } + Trace($"audio_frame handle=0x{player.Handle:X16} ts={timestamp} data=0x{bufferAddress:X16}"); + return SetReturn(ctx, 1); + } + } + + [SysAbiExport( + Nid = "wwM99gjFf1Y", + ExportName = "sceAvPlayerCurrentTime", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerCurrentTime(CpuContext ctx) + { + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player)) + { + return SetReturn(ctx, InvalidParameters); + } + + var milliseconds = (ulong)player.PlaybackClock.ElapsedMilliseconds; + ctx[CpuRegister.Rax] = milliseconds; + return unchecked((int)milliseconds); + } + } + + [SysAbiExport( + Nid = "hdTyRzCXQeQ", + ExportName = "sceAvPlayerStreamCount", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerStreamCount(CpuContext ctx) + { + lock (StateGate) + { + return SetReturn(ctx, Players.ContainsKey(ctx[CpuRegister.Rdi]) ? 2 : InvalidParameters); + } + } + + [SysAbiExport( + Nid = "d8FcbzfAdQw", + ExportName = "sceAvPlayerGetStreamInfo", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerGetStreamInfo(CpuContext ctx) + { + var streamIndex = unchecked((uint)ctx[CpuRegister.Rsi]); + var infoAddress = ctx[CpuRegister.Rdx]; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player) || + streamIndex > 1 || infoAddress == 0 || player.Width <= 0 || player.Height <= 0) + { + return SetReturn(ctx, InvalidParameters); + } + + Span info = stackalloc byte[StreamInfoSize]; + info.Clear(); + BinaryPrimitives.WriteUInt32LittleEndian(info[0..], streamIndex); // 0=video, 1=audio + if (streamIndex == 0) + { + BinaryPrimitives.WriteUInt32LittleEndian(info[8..], checked((uint)player.Width)); + BinaryPrimitives.WriteUInt32LittleEndian(info[12..], checked((uint)player.Height)); + BinaryPrimitives.WriteSingleLittleEndian(info[16..], (float)player.Width / player.Height); + } + else + { + BinaryPrimitives.WriteUInt16LittleEndian(info[8..], 2); + BinaryPrimitives.WriteUInt32LittleEndian(info[12..], 48_000); + } + BinaryPrimitives.WriteUInt64LittleEndian(info[24..], player.DurationMilliseconds); + if (!ctx.Memory.TryWrite(infoAddress, info)) + { + return SetReturn(ctx, InvalidParameters); + } + + return SetReturn(ctx, 0); + } + } + + private static int AddSource(CpuContext ctx, string guestPath) + { + PlayerState player; + bool autoStart; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var foundPlayer)) + { + return SetReturn(ctx, InvalidParameters); + } + player = foundPlayer; + + var hostPath = ResolveGuestPath(guestPath); + if (hostPath is null || !ProbeVideo(hostPath, out var width, out var height, out var fps, out var duration)) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] Could not open guest video '{guestPath}' (resolved '{hostPath ?? ""}')."); + return SetReturn(ctx, OperationFailed); + } + + player.ResetPlayback(); + player.SourcePath = hostPath; + player.Width = width; + player.Height = height; + player.FramesPerSecond = fps; + player.DurationMilliseconds = duration; + player.Started = player.AutoStart; + autoStart = player.AutoStart; + Trace($"source guest='{guestPath}' host='{hostPath}' {width}x{height} fps={fps:F3} duration_ms={duration} auto_start={player.AutoStart}"); + } + + + NotifyEvent(ctx, player, 2); // StateReady + if (autoStart) + { + NotifyEvent(ctx, player, 3); // StatePlay + } + return SetReturn(ctx, 0); + } + + private static int GetVideoData(CpuContext ctx, bool extended) + { + var infoAddress = ctx[CpuRegister.Rsi]; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player) || + infoAddress == 0 || !player.Started || player.Paused || player.EndOfStream || + player.SourcePath is null) + { + return SetReturn(ctx, 0); + } + + if (!EnsureDecoder(player)) + { + player.EndOfStream = true; + return SetReturn(ctx, 0); + } + + var fps = Math.Max(1.0, player.FramesPerSecond); + var expectedFrame = (long)Math.Floor(player.PlaybackClock.Elapsed.TotalSeconds * fps); + while (player.NextFrameIndex < expectedFrame) + { + if (!ReadFrame(player)) + { + return FinishStream(ctx, player); + } + player.NextFrameIndex++; + } + + if (!ReadFrame(player)) + { + return FinishStream(ctx, player); + } + + var timestamp = checked((ulong)Math.Round(player.NextFrameIndex * 1000.0 / fps)); + player.NextFrameIndex++; + if (!WriteVideoFrame(ctx, player, infoAddress, timestamp, extended)) + { + return SetReturn(ctx, 0); + } + + Trace($"video_frame handle=0x{player.Handle:X16} ex={extended} ts={timestamp} data=0x{player.LastGuestBuffer:X16}"); + return SetReturn(ctx, 1); + } + } + + private static int FinishStream(CpuContext ctx, PlayerState player) + { + if (player.Looping) + { + player.ResetPlayback(); + player.Started = true; + } + else + { + player.EndOfStream = true; + player.PlaybackClock.Stop(); + } + return SetReturn(ctx, 0); + } + + private static bool EnsureDecoder(PlayerState player) + { + if (player.DecoderOutput is not null) + { + return true; + } + + var ffmpeg = FindFfmpeg(); + if (ffmpeg is null || player.SourcePath is null) + { + Console.Error.WriteLine("[AVPLAYER][ERROR] FFmpeg was not found. Set SHARPEMU_FFMPEG_PATH."); + return false; + } + + var startInfo = new ProcessStartInfo(ffmpeg) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("-nostdin"); + startInfo.ArgumentList.Add("-hide_banner"); + startInfo.ArgumentList.Add("-loglevel"); + startInfo.ArgumentList.Add("error"); + startInfo.ArgumentList.Add("-i"); + startInfo.ArgumentList.Add(player.SourcePath); + startInfo.ArgumentList.Add("-map"); + startInfo.ArgumentList.Add("0:v:0"); + startInfo.ArgumentList.Add("-an"); + startInfo.ArgumentList.Add("-pix_fmt"); + startInfo.ArgumentList.Add("nv12"); + startInfo.ArgumentList.Add("-f"); + startInfo.ArgumentList.Add("rawvideo"); + startInfo.ArgumentList.Add("pipe:1"); + + try + { + player.Decoder = Process.Start(startInfo); + if (player.Decoder is null) + { + return false; + } + player.Decoder.ErrorDataReceived += (_, eventArgs) => + { + if (!string.IsNullOrWhiteSpace(eventArgs.Data)) + { + Console.Error.WriteLine($"[AVPLAYER][FFMPEG] {eventArgs.Data}"); + } + }; + player.Decoder.BeginErrorReadLine(); + player.DecoderOutput = player.Decoder.StandardOutput.BaseStream; + player.RawFrame = new byte[checked(player.Width * player.Height * 3 / 2)]; + player.PlaybackClock.Start(); + Trace($"decoder_started pid={player.Decoder.Id} source='{player.SourcePath}'"); + return true; + } + catch (Exception exception) when (exception is IOException or InvalidOperationException or System.ComponentModel.Win32Exception) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] Failed to launch FFmpeg: {exception.Message}"); + player.Dispose(); + return false; + } + } + + private static bool EnsureAudioDecoder(PlayerState player) + { + if (player.AudioDecoderOutput is not null) + { + return true; + } + + var ffmpeg = FindFfmpeg(); + if (ffmpeg is null || player.SourcePath is null) + { + return false; + } + + var startInfo = new ProcessStartInfo(ffmpeg) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("-nostdin"); + startInfo.ArgumentList.Add("-hide_banner"); + startInfo.ArgumentList.Add("-loglevel"); + startInfo.ArgumentList.Add("error"); + startInfo.ArgumentList.Add("-i"); + startInfo.ArgumentList.Add(player.SourcePath); + startInfo.ArgumentList.Add("-map"); + startInfo.ArgumentList.Add("0:a:0"); + startInfo.ArgumentList.Add("-vn"); + startInfo.ArgumentList.Add("-ac"); + startInfo.ArgumentList.Add("2"); + startInfo.ArgumentList.Add("-ar"); + startInfo.ArgumentList.Add("48000"); + startInfo.ArgumentList.Add("-f"); + startInfo.ArgumentList.Add("s16le"); + startInfo.ArgumentList.Add("pipe:1"); + + try + { + player.AudioDecoder = Process.Start(startInfo); + if (player.AudioDecoder is null) + { + return false; + } + player.AudioDecoder.ErrorDataReceived += (_, eventArgs) => + { + if (!string.IsNullOrWhiteSpace(eventArgs.Data)) + { + Console.Error.WriteLine($"[AVPLAYER][FFMPEG-AUDIO] {eventArgs.Data}"); + } + }; + player.AudioDecoder.BeginErrorReadLine(); + player.AudioDecoderOutput = player.AudioDecoder.StandardOutput.BaseStream; + player.RawAudioFrame = new byte[1024 * 2 * sizeof(short)]; + Trace($"audio_decoder_started pid={player.AudioDecoder.Id} source='{player.SourcePath}'"); + return true; + } + catch (Exception exception) when (exception is IOException or InvalidOperationException or System.ComponentModel.Win32Exception) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] Failed to launch FFmpeg audio decoder: {exception.Message}"); + player.AudioDecoderOutput?.Dispose(); + player.AudioDecoderOutput = null; + player.AudioDecoder?.Dispose(); + player.AudioDecoder = null; + return false; + } + } + + private static bool ReadFrame(PlayerState player) + { + if (player.DecoderOutput is null || player.RawFrame is null) + { + return false; + } + + try + { + return ReadExactly(player.DecoderOutput, player.RawFrame); + } + catch (IOException exception) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] FFmpeg stream read failed: {exception.Message}"); + return false; + } + } + + private static bool ReadExactly(Stream? stream, byte[] buffer) + { + if (stream is null) + { + return false; + } + var offset = 0; + while (offset < buffer.Length) + { + var read = stream.Read(buffer, offset, buffer.Length - offset); + if (read == 0) + { + return false; + } + offset += read; + } + return true; + } + + private static bool WriteVideoFrame( + CpuContext ctx, + PlayerState player, + ulong infoAddress, + ulong timestamp, + bool extended) + { + if (player.RawFrame is null) + { + return false; + } + + var alignedWidth = AlignUp(player.Width, 16); + var alignedHeight = AlignUp(player.Height, 16); + var bufferStride = checked(alignedWidth * alignedHeight * 3 / 2); + if (player.GuestBuffers[0] == 0) + { + if (!AllocateGuestVideoBuffers(ctx, player, bufferStride)) + { + return false; + } + player.GuestBufferStride = bufferStride; + } + + var frameData = player.RawFrame; + if (!extended && (alignedWidth != player.Width || alignedHeight != player.Height)) + { + player.PaddedFrame ??= new byte[bufferStride]; + player.PaddedFrame.AsSpan().Clear(); + for (var row = 0; row < player.Height; row++) + { + player.RawFrame.AsSpan(row * player.Width, player.Width) + .CopyTo(player.PaddedFrame.AsSpan(row * alignedWidth, player.Width)); + } + var rawChromaOffset = player.Width * player.Height; + var paddedChromaOffset = alignedWidth * alignedHeight; + for (var row = 0; row < player.Height / 2; row++) + { + player.RawFrame.AsSpan(rawChromaOffset + (row * player.Width), player.Width) + .CopyTo(player.PaddedFrame.AsSpan(paddedChromaOffset + (row * alignedWidth), player.Width)); + } + frameData = player.PaddedFrame; + } + + var bufferAddress = player.GuestBuffers[player.NextGuestBuffer]; + player.NextGuestBuffer = (player.NextGuestBuffer + 1) % FrameBufferCount; + player.LastGuestBuffer = bufferAddress; + if (!ctx.Memory.TryWrite(bufferAddress, frameData)) + { + return false; + } + + Span info = extended + ? stackalloc byte[FrameInfoExSize] + : stackalloc byte[FrameInfoSize]; + info.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(info[0..], bufferAddress); + BinaryPrimitives.WriteUInt64LittleEndian(info[16..], timestamp); + BinaryPrimitives.WriteUInt32LittleEndian(info[24..], checked((uint)(extended ? player.Width : alignedWidth))); + BinaryPrimitives.WriteUInt32LittleEndian(info[28..], checked((uint)(extended ? player.Height : alignedHeight))); + BinaryPrimitives.WriteSingleLittleEndian(info[32..], 1.0f); + if (extended) + { + BinaryPrimitives.WriteUInt32LittleEndian(info[60..], checked((uint)player.Width)); + info[64] = 8; + info[65] = 8; + } + return ctx.Memory.TryWrite(infoAddress, info); + } + + private static bool AllocateGuestVideoBuffers(CpuContext ctx, PlayerState player, int bufferSize) + { + var scheduler = GuestThreadExecution.Scheduler; + if (!player.TextureAllocatorFailed && player.AllocateTextureCallback != 0 && scheduler is not null) + { + for (var index = 0; index < player.GuestBuffers.Length; index++) + { + if (!scheduler.TryCallGuestFunction( + ctx, + player.AllocateTextureCallback, + player.AllocatorObject, + 0x100, + checked((ulong)bufferSize), + 0, + 0, + "avplayer_allocate_texture", + out var buffer, + out var error) || buffer == 0) + { + Console.Error.WriteLine( + $"[AVPLAYER][ERROR] Guest texture allocation failed index={index} " + + $"callback=0x{player.AllocateTextureCallback:X16}: {error ?? "returned null"}"); + player.TextureAllocatorFailed = true; + Array.Clear(player.GuestBuffers); + break; + } + player.GuestBuffers[index] = buffer; + Trace($"texture_buffer index={index} data=0x{buffer:X16} size={bufferSize}"); + } + if (!player.TextureAllocatorFailed) + { + return true; + } + } + + if (!KernelMemoryCompatExports.TryAllocateHleData( + ctx, + checked((ulong)bufferSize * FrameBufferCount), + 0x1000, + out var bufferBase)) + { + return false; + } + for (var index = 0; index < player.GuestBuffers.Length; index++) + { + player.GuestBuffers[index] = bufferBase + checked((ulong)(index * bufferSize)); + } + Console.Error.WriteLine("[AVPLAYER][WARN] Guest texture allocator unavailable; using generic HLE memory."); + return true; + } + + private static bool ProbeVideo( + string path, + out int width, + out int height, + out double framesPerSecond, + out ulong durationMilliseconds) + { + width = 0; + height = 0; + framesPerSecond = 30.0; + durationMilliseconds = 0; + var ffmpeg = FindFfmpeg(); + if (ffmpeg is null) + { + return false; + } + var ffprobe = Path.Combine(Path.GetDirectoryName(ffmpeg) ?? string.Empty, "ffprobe"); + if (!File.Exists(ffprobe)) + { + return false; + } + + var startInfo = new ProcessStartInfo(ffprobe) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("-v"); + startInfo.ArgumentList.Add("error"); + startInfo.ArgumentList.Add("-select_streams"); + startInfo.ArgumentList.Add("v:0"); + startInfo.ArgumentList.Add("-show_entries"); + startInfo.ArgumentList.Add("stream=width,height,avg_frame_rate,duration"); + startInfo.ArgumentList.Add("-of"); + startInfo.ArgumentList.Add("default=noprint_wrappers=1"); + startInfo.ArgumentList.Add(path); + + try + { + using var process = Process.Start(startInfo); + if (process is null) + { + return false; + } + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + if (process.ExitCode != 0) + { + Console.Error.WriteLine($"[AVPLAYER][FFPROBE] {error.Trim()}"); + return false; + } + + foreach (var line in output.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var separator = line.IndexOf('='); + if (separator < 1) + { + continue; + } + var key = line[..separator]; + var value = line[(separator + 1)..]; + switch (key) + { + case "width": + _ = int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out width); + break; + case "height": + _ = int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out height); + break; + case "avg_frame_rate": + var parts = value.Split('/'); + if (parts.Length == 2 && + double.TryParse(parts[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var numerator) && + double.TryParse(parts[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var denominator) && + denominator != 0) + { + framesPerSecond = numerator / denominator; + } + break; + case "duration": + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var duration)) + { + durationMilliseconds = checked((ulong)Math.Max(0, Math.Round(duration * 1000.0))); + } + break; + } + } + return width > 0 && height > 0 && framesPerSecond > 0; + } + catch (Exception exception) when (exception is IOException or InvalidOperationException or System.ComponentModel.Win32Exception) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] Failed to probe video: {exception.Message}"); + return false; + } + } + + private static string? FindFfmpeg() + { + var configured = Environment.GetEnvironmentVariable("SHARPEMU_FFMPEG_PATH"); + if (!string.IsNullOrWhiteSpace(configured) && File.Exists(configured)) + { + return configured; + } + foreach (var candidate in new[] { "/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg" }) + { + if (File.Exists(candidate)) + { + return candidate; + } + } + return null; + } + + private static string? ResolveGuestPath(string guestPath) + { + if (string.IsNullOrWhiteSpace(guestPath)) + { + return null; + } + + var normalized = guestPath.Replace('\\', '/'); + if (Uri.TryCreate(normalized, UriKind.Absolute, out var uri) && uri.IsFile) + { + normalized = uri.LocalPath; + } + if (File.Exists(normalized)) + { + return Path.GetFullPath(normalized); + } + + var app0 = Environment.GetEnvironmentVariable("SHARPEMU_APP0_DIR"); + if (string.IsNullOrWhiteSpace(app0)) + { + return null; + } + foreach (var prefix in new[] { "app0:/", "/app0/", "app0:", "/app0" }) + { + if (normalized.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + normalized = normalized[prefix.Length..]; + break; + } + } + var candidate = Path.GetFullPath(Path.Combine(app0, normalized.TrimStart('/'))); + var root = Path.GetFullPath(app0).TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar; + return candidate.StartsWith(root, StringComparison.OrdinalIgnoreCase) && File.Exists(candidate) + ? candidate + : null; + } + + private static bool TryReadNullTerminatedUtf8(CpuContext ctx, ulong address, int maxLength, out string value) + { + value = string.Empty; + if (address == 0 || maxLength <= 0) + { + return false; + } + var bytes = new List(Math.Min(maxLength, 256)); + Span single = stackalloc byte[1]; + for (var index = 0; index < maxLength; index++) + { + if (!ctx.Memory.TryRead(address + (ulong)index, single)) + { + return false; + } + if (single[0] == 0) + { + value = Encoding.UTF8.GetString(bytes.ToArray()); + return true; + } + bytes.Add(single[0]); + } + return false; + } + + private static bool TryReadUtf8(CpuContext ctx, ulong address, int length, out string value) + { + value = string.Empty; + if (address == 0 || length <= 0) + { + return false; + } + var bytes = new byte[length]; + if (!ctx.Memory.TryRead(address, bytes)) + { + return false; + } + value = Encoding.UTF8.GetString(bytes); + return true; + } + + private static bool TryReadByte(CpuContext ctx, ulong address, out byte value) + { + Span buffer = stackalloc byte[1]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + value = buffer[0]; + return true; + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) => + ctx.TryReadUInt64(address, out value); + + private static void NotifyEvent(CpuContext ctx, PlayerState player, ulong eventId) + { + if (player.EventCallback == 0) + { + Trace($"event skipped handle=0x{player.Handle:X16} id={eventId} callback=0"); + return; + } + + var scheduler = GuestThreadExecution.Scheduler; + string? error = null; + if (scheduler is null || + !scheduler.TryCallGuestFunction( + ctx, + player.EventCallback, + player.EventObject, + eventId, + 0, + 0, + 0, + $"avplayer_event_{eventId}", + out _, + out error)) + { + Console.Error.WriteLine( + $"[AVPLAYER][WARN] Event callback failed handle=0x{player.Handle:X16} " + + $"event={eventId} callback=0x{player.EventCallback:X16}: {error ?? "scheduler unavailable"}"); + return; + } + + Trace($"event handle=0x{player.Handle:X16} id={eventId} callback=0x{player.EventCallback:X16}"); + } + + private static int AlignUp(int value, int alignment) => + checked((value + alignment - 1) & -alignment); + + private static int ValidatePlayer(CpuContext ctx) + { + lock (StateGate) + { + return SetReturn(ctx, Players.ContainsKey(ctx[CpuRegister.Rdi]) ? 0 : InvalidParameters); + } + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + + private static void Trace(string message) + { + var count = Interlocked.Increment(ref _traceCount); + if (count <= 32 || count % 300 == 0) + { + Console.Error.WriteLine($"[AVPLAYER][INFO] {message}"); } } } diff --git a/src/SharpEmu.Libs/Bink/Bink2MovieBridge.cs b/src/SharpEmu.Libs/Bink/Bink2MovieBridge.cs new file mode 100644 index 0000000..d3d5ead --- /dev/null +++ b/src/SharpEmu.Libs/Bink/Bink2MovieBridge.cs @@ -0,0 +1,403 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using System.Buffers.Binary; + +namespace SharpEmu.Libs.Bink; + +/// +/// Optional host-side Bink 2 bridge for games that ship a static Bink player. +/// +/// The game in that case never imports libSceVideodec, so an HLE video-decoder +/// export cannot see its movie frames. Kernel file opens identify the active +/// .bk2 file and the presenter requests BGRA frames from a tiny native adapter. +/// The adapter is deliberately a separate, user-supplied library: Bink 2 is a +/// proprietary SDK and SharpEmu must neither bundle it nor depend on its ABI. +/// +internal static class Bink2MovieBridge +{ + private const uint MaxDimension = 16384; + private static readonly object Gate = new(); + private static NativeAdapter? _adapter; + private static string? _activePath; + private static IntPtr _activeMovie; + private static Bink2MovieInfo _activeInfo; + private static byte[]? _frameBuffer; + private static bool _usingDummyMovie; + private static bool _loadAttempted; + private static bool _availabilityReported; + + /// + /// Returns true when the guest should receive a normal "file not found" + /// result for a Bink movie. This is the safe default without a decoder: + /// games that treat movies as optional fall through to their next state + /// rather than submitting an empty Bink GPU texture forever. + /// + internal static bool ShouldSkipGuestMovie(string hostPath) => + hostPath.EndsWith(".bk2", StringComparison.OrdinalIgnoreCase) && + ResolveMode() == MovieMode.Skip; + + internal static void ObserveGuestMovie(string hostPath) + { + if (!hostPath.EndsWith(".bk2", StringComparison.OrdinalIgnoreCase) || + !File.Exists(hostPath)) + { + return; + } + + lock (Gate) + { + if (string.Equals(_activePath, hostPath, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (ResolveMode() == MovieMode.Dummy) + { + AttachDummyMovieLocked(hostPath); + return; + } + + var adapter = GetAdapterLocked(); + if (adapter is null) + { + return; + } + + CloseActiveLocked(); + if (!adapter.TryOpen(hostPath, out var movie, out var info)) + { + Console.Error.WriteLine( + "[LOADER][WARN] Bink2 bridge could not open movie '" + + Path.GetFileName(hostPath) + "'."); + return; + } + + if (!IsValid(info)) + { + adapter.Close(movie); + Console.Error.WriteLine( + "[LOADER][WARN] Bink2 bridge rejected invalid movie dimensions for '" + + Path.GetFileName(hostPath) + "'."); + return; + } + + _activePath = hostPath; + _activeMovie = movie; + _activeInfo = info; + _frameBuffer = GC.AllocateUninitializedArray(GetFrameBufferLength(info)); + Console.Error.WriteLine( + "[LOADER][INFO] Bink2 bridge attached: " + Path.GetFileName(hostPath) + " " + + info.Width + "x" + info.Height + " @ " + + info.FramesPerSecondNumerator + "/" + info.FramesPerSecondDenominator + " fps."); + } + } + + internal static bool TryDecodeNextFrame( + out byte[] pixels, + out uint width, + out uint height) + { + lock (Gate) + { + pixels = []; + width = 0; + height = 0; + if (_adapter is null || _activeMovie == IntPtr.Zero || _frameBuffer is null) + { + if (_usingDummyMovie && _frameBuffer is not null) + { + pixels = _frameBuffer; + width = _activeInfo.Width; + height = _activeInfo.Height; + return true; + } + + return false; + } + + unsafe + { + fixed (byte* destination = _frameBuffer) + { + if (!_adapter.DecodeNextBgra( + _activeMovie, + (IntPtr)destination, + _activeInfo.Width * 4, + (uint)_frameBuffer.Length)) + { + return false; + } + } + } + + pixels = _frameBuffer; + width = _activeInfo.Width; + height = _activeInfo.Height; + return true; + } + } + + private static bool IsValid(Bink2MovieInfo info) => + info.Width > 0 && info.Height > 0 && + info.Width <= MaxDimension && info.Height <= MaxDimension && + (ulong)info.Width * info.Height * 4 <= int.MaxValue; + + private static int GetFrameBufferLength(Bink2MovieInfo info) => + checked((int)((ulong)info.Width * info.Height * 4)); + + private static MovieMode ResolveMode() + { + var configured = Environment.GetEnvironmentVariable("SHARPEMU_BINK_MODE"); + if (string.Equals(configured, "dummy", StringComparison.OrdinalIgnoreCase)) + { + return MovieMode.Dummy; + } + + if (string.Equals(configured, "native", StringComparison.OrdinalIgnoreCase)) + { + return MovieMode.Native; + } + + if (string.Equals(configured, "skip", StringComparison.OrdinalIgnoreCase)) + { + return MovieMode.Skip; + } + + // With no SDK adapter present, returning "not found" makes optional + // cinematics advance. Supplying either an explicit path or the normal + // side-by-side adapter enables native playback automatically. + if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("SHARPEMU_BINK2_BRIDGE")) || + EnumerateAdapterCandidates().Any(File.Exists)) + { + return MovieMode.Native; + } + + return MovieMode.Skip; + } + + private static void AttachDummyMovieLocked(string hostPath) + { + if (!TryReadBinkInfo(hostPath, out var info) || !IsValid(info)) + { + Console.Error.WriteLine( + "[LOADER][WARN] Bink dummy could not read movie header '" + + Path.GetFileName(hostPath) + "'."); + return; + } + + CloseActiveLocked(); + _activePath = hostPath; + _activeInfo = info; + _frameBuffer = GC.AllocateUninitializedArray(GetFrameBufferLength(info)); + FillDummyFrame(_frameBuffer, info.Width, info.Height); + _usingDummyMovie = true; + Console.Error.WriteLine( + "[LOADER][INFO] Bink dummy attached: " + Path.GetFileName(hostPath) + " " + + info.Width + "x" + info.Height + "."); + } + + private static bool TryReadBinkInfo(string path, out Bink2MovieInfo info) + { + info = default; + Span header = stackalloc byte[32]; + try + { + using var stream = File.OpenRead(path); + if (stream.Read(header) != header.Length || + !header[..4].SequenceEqual("KB2j"u8)) + { + return false; + } + + info = new Bink2MovieInfo( + BinaryPrimitives.ReadUInt32LittleEndian(header.Slice(0x14, 4)), + BinaryPrimitives.ReadUInt32LittleEndian(header.Slice(0x18, 4)), + BinaryPrimitives.ReadUInt32LittleEndian(header.Slice(0x1C, 4)), + 1); + return true; + } + catch (IOException) + { + return false; + } + } + + private static void FillDummyFrame(byte[] pixels, uint width, uint height) + { + for (var y = 0u; y < height; y++) + { + for (var x = 0u; x < width; x++) + { + var offset = checked((int)(((ulong)y * width + x) * 4)); + var band = ((x / 96) + (y / 96)) & 1; + pixels[offset] = band == 0 ? (byte)0x28 : (byte)0x18; + pixels[offset + 1] = band == 0 ? (byte)0x18 : (byte)0x28; + pixels[offset + 2] = 0x10; + pixels[offset + 3] = 0xFF; + } + } + } + + private static NativeAdapter? GetAdapterLocked() + { + if (_loadAttempted) + { + return _adapter; + } + + _loadAttempted = true; + foreach (var candidate in EnumerateAdapterCandidates()) + { + if (!NativeLibrary.TryLoad(candidate, out var library)) + { + continue; + } + + if (NativeAdapter.TryCreate(library, out var adapter)) + { + _adapter = adapter; + Console.Error.WriteLine("[LOADER][INFO] Bink2 bridge loaded: " + candidate); + return adapter; + } + + NativeLibrary.Free(library); + } + + if (!_availabilityReported) + { + _availabilityReported = true; + Console.Error.WriteLine( + "[LOADER][INFO] Bink2 bridge unavailable; install the licensed adapter and set SHARPEMU_BINK2_BRIDGE."); + } + + return null; + } + + private static IEnumerable EnumerateAdapterCandidates() + { + var configured = Environment.GetEnvironmentVariable("SHARPEMU_BINK2_BRIDGE"); + if (!string.IsNullOrWhiteSpace(configured)) + { + yield return configured; + } + + var baseDirectory = AppContext.BaseDirectory; + if (OperatingSystem.IsMacOS()) + { + yield return Path.Combine(baseDirectory, "libsharpemu_bink2_bridge.dylib"); + } + else if (OperatingSystem.IsWindows()) + { + yield return Path.Combine(baseDirectory, "sharpemu_bink2_bridge.dll"); + } + else + { + yield return Path.Combine(baseDirectory, "libsharpemu_bink2_bridge.so"); + } + } + + private static void CloseActiveLocked() + { + if (_activeMovie != IntPtr.Zero) + { + _adapter?.Close(_activeMovie); + } + + _activePath = null; + _activeMovie = IntPtr.Zero; + _activeInfo = default; + _frameBuffer = null; + _usingDummyMovie = false; + } + + [StructLayout(LayoutKind.Sequential)] + private readonly struct Bink2MovieInfo + { + public readonly uint Width; + public readonly uint Height; + public readonly uint FramesPerSecondNumerator; + public readonly uint FramesPerSecondDenominator; + + internal Bink2MovieInfo( + uint width, + uint height, + uint framesPerSecondNumerator, + uint framesPerSecondDenominator) + { + Width = width; + Height = height; + FramesPerSecondNumerator = framesPerSecondNumerator; + FramesPerSecondDenominator = framesPerSecondDenominator; + } + } + + private enum MovieMode + { + Skip, + Dummy, + Native, + } + + private sealed class NativeAdapter + { + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int OpenUtf8Delegate(IntPtr pathUtf8, out IntPtr movie, out Bink2MovieInfo info); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int DecodeNextBgraDelegate(IntPtr movie, IntPtr destination, uint stride, uint destinationBytes); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate void CloseDelegate(IntPtr movie); + + private readonly OpenUtf8Delegate _openUtf8; + private readonly DecodeNextBgraDelegate _decodeNextBgra; + private readonly CloseDelegate _close; + + private NativeAdapter( + OpenUtf8Delegate openUtf8, + DecodeNextBgraDelegate decodeNextBgra, + CloseDelegate close) + { + _openUtf8 = openUtf8; + _decodeNextBgra = decodeNextBgra; + _close = close; + } + + internal static bool TryCreate(IntPtr library, out NativeAdapter? adapter) + { + adapter = null; + if (!NativeLibrary.TryGetExport(library, "sharpemu_bink2_open_utf8", out var open) || + !NativeLibrary.TryGetExport(library, "sharpemu_bink2_decode_next_bgra", out var decode) || + !NativeLibrary.TryGetExport(library, "sharpemu_bink2_close", out var close)) + { + return false; + } + + adapter = new NativeAdapter( + Marshal.GetDelegateForFunctionPointer(open), + Marshal.GetDelegateForFunctionPointer(decode), + Marshal.GetDelegateForFunctionPointer(close)); + return true; + } + + internal bool TryOpen(string path, out IntPtr movie, out Bink2MovieInfo info) + { + var utf8 = Marshal.StringToCoTaskMemUTF8(path); + try + { + return _openUtf8(utf8, out movie, out info) != 0 && movie != IntPtr.Zero; + } + finally + { + Marshal.FreeCoTaskMem(utf8); + } + } + + internal bool DecodeNextBgra(IntPtr movie, IntPtr destination, uint stride, uint destinationBytes) => + _decodeNextBgra(movie, destination, stride, destinationBytes) != 0; + + internal void Close(IntPtr movie) => _close(movie); + } +} diff --git a/src/SharpEmu.Libs/Codec/CodecExports.cs b/src/SharpEmu.Libs/Codec/CodecExports.cs new file mode 100644 index 0000000..fbf85ac --- /dev/null +++ b/src/SharpEmu.Libs/Codec/CodecExports.cs @@ -0,0 +1,104 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using SharpEmu.Libs.Kernel; +using System.Collections.Concurrent; +using System.Threading; + +namespace SharpEmu.Libs.Codec; + +/// +/// libSceVideodec / libSceAudiodec handle management. Actual H.264/HEVC and +/// AAC/AT9 decoding requires an external codec, which is out of scope; these +/// exports keep the decoder lifecycle resolvable (create/decode/flush/delete) +/// and report "no output produced" so guests advance instead of failing on +/// unresolved imports. +/// +public static class CodecExports +{ + private const int Ok = 0; + private const int VideodecErrorInvalidArg = unchecked((int)0x80620801); + private const int AudiodecErrorInvalidArg = unchecked((int)0x807F0002); + + private static readonly ConcurrentDictionary VideoDecoders = new(); + private static readonly ConcurrentDictionary AudioDecoders = new(); + private static long _nextHandle = 1; + + // ---- Video decoder ---- + + [SysAbiExport(Nid = "qkgRiwHyheU", ExportName = "sceVideodecCreateDecoder", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideodec")] + public static int VideodecCreateDecoder(CpuContext ctx) + { + var outHandleAddress = ctx[CpuRegister.Rdx]; + if (outHandleAddress == 0) + { + return SetReturn(ctx, VideodecErrorInvalidArg); + } + + var handle = (ulong)Interlocked.Increment(ref _nextHandle); + VideoDecoders[handle] = 1; + return TryWriteHandle(ctx, outHandleAddress, handle) ? SetReturn(ctx, Ok) : SetReturn(ctx, VideodecErrorInvalidArg); + } + + [SysAbiExport(Nid = "q0W5GJMovMs", ExportName = "sceVideodecDecode", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideodec")] + public static int VideodecDecode(CpuContext ctx) + { + // No decoder is present: report success with no picture produced. + return SetReturn(ctx, VideoDecoders.ContainsKey(ctx[CpuRegister.Rdi]) ? Ok : VideodecErrorInvalidArg); + } + + [SysAbiExport(Nid = "jeigLlKdp5I", ExportName = "sceVideodecFlush", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideodec")] + public static int VideodecFlush(CpuContext ctx) => + SetReturn(ctx, VideoDecoders.ContainsKey(ctx[CpuRegister.Rdi]) ? Ok : VideodecErrorInvalidArg); + + [SysAbiExport(Nid = "U0kpGF1cl90", ExportName = "sceVideodecDeleteDecoder", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideodec")] + public static int VideodecDeleteDecoder(CpuContext ctx) + { + VideoDecoders.TryRemove(ctx[CpuRegister.Rdi], out _); + return SetReturn(ctx, Ok); + } + + // ---- Audio decoder ---- + + [SysAbiExport(Nid = "O3f1sLMWRvs", ExportName = "sceAudiodecCreateDecoder", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceAudiodec")] + public static int AudiodecCreateDecoder(CpuContext ctx) + { + var handle = (ulong)Interlocked.Increment(ref _nextHandle); + AudioDecoders[handle] = 1; + // sceAudiodec returns the handle directly (>= 0) or a negative error. + ctx[CpuRegister.Rax] = handle; + return unchecked((int)handle); + } + + [SysAbiExport(Nid = "KHXHMDLkILw", ExportName = "sceAudiodecDecode", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceAudiodec")] + public static int AudiodecDecode(CpuContext ctx) + { + // No decoder present: report success with zero output samples so the + // caller treats the frame as silent rather than erroring. + return SetReturn(ctx, AudioDecoders.ContainsKey(ctx[CpuRegister.Rdi]) ? Ok : AudiodecErrorInvalidArg); + } + + [SysAbiExport(Nid = "Tp+ZEy69mLk", ExportName = "sceAudiodecDeleteDecoder", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceAudiodec")] + public static int AudiodecDeleteDecoder(CpuContext ctx) + { + AudioDecoders.TryRemove(ctx[CpuRegister.Rdi], out _); + return SetReturn(ctx, Ok); + } + + private static bool TryWriteHandle(CpuContext ctx, ulong address, ulong handle) => + ctx.TryWriteUInt64(address, handle); + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } +} diff --git a/src/SharpEmu.Libs/CommonDialog/ErrorDialogExports.cs b/src/SharpEmu.Libs/CommonDialog/ErrorDialogExports.cs new file mode 100644 index 0000000..71a4624 --- /dev/null +++ b/src/SharpEmu.Libs/CommonDialog/ErrorDialogExports.cs @@ -0,0 +1,106 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using System.Threading; + +namespace SharpEmu.Libs.CommonDialog; + +/// +/// libSceErrorDialog. There is no host error dialog, so an opened dialog +/// immediately reports finished — this keeps guests that block on the dialog +/// status (a common fatal-error path) from spinning forever. +/// +public static class ErrorDialogExports +{ + private const int AlreadyInitialized = unchecked((int)0x80ED0001); + private const int NotInitialized = unchecked((int)0x80ED0002); + private const int ArgNull = unchecked((int)0x80ED0005); + + private const int StatusNone = 0; + private const int StatusInitialized = 1; + private const int StatusFinished = 3; + + private static int _initialized; + private static int _status; + + [SysAbiExport( + Nid = "I88KChlynSs", + ExportName = "sceErrorDialogInitialize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogInitialize(CpuContext ctx) + { + var result = Interlocked.Exchange(ref _initialized, 1) == 0 ? 0 : AlreadyInitialized; + if (result == 0) + { + Volatile.Write(ref _status, StatusInitialized); + } + + return SetReturn(ctx, result); + } + + [SysAbiExport( + Nid = "M2ZF-ClLhgY", + ExportName = "sceErrorDialogOpen", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogOpen(CpuContext ctx) + { + if (ctx[CpuRegister.Rdi] == 0) + { + return SetReturn(ctx, ArgNull); + } + + if (Volatile.Read(ref _initialized) == 0) + { + return SetReturn(ctx, NotInitialized); + } + + Volatile.Write(ref _status, StatusFinished); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "t2FvHRXzgqk", + ExportName = "sceErrorDialogGetStatus", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogGetStatus(CpuContext ctx) => SetReturn(ctx, Volatile.Read(ref _status)); + + [SysAbiExport( + Nid = "WWiGuh9XfgQ", + ExportName = "sceErrorDialogUpdateStatus", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogUpdateStatus(CpuContext ctx) => SetReturn(ctx, Volatile.Read(ref _status)); + + [SysAbiExport( + Nid = "ekXHb1kDBl0", + ExportName = "sceErrorDialogClose", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogClose(CpuContext ctx) + { + Volatile.Write(ref _status, StatusFinished); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "9XAxK2PMwk8", + ExportName = "sceErrorDialogTerminate", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogTerminate(CpuContext ctx) + { + Volatile.Write(ref _status, StatusNone); + Interlocked.Exchange(ref _initialized, 0); + return SetReturn(ctx, 0); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } +} diff --git a/src/SharpEmu.Libs/Fiber/FiberExports.cs b/src/SharpEmu.Libs/Fiber/FiberExports.cs index 30a873d..ea2f99e 100644 --- a/src/SharpEmu.Libs/Fiber/FiberExports.cs +++ b/src/SharpEmu.Libs/Fiber/FiberExports.cs @@ -3,6 +3,7 @@ using System.Buffers.Binary; using System.Collections.Concurrent; +using System.Diagnostics; using System.Text; using SharpEmu.HLE; @@ -22,6 +23,10 @@ public static class FiberExports private const uint FiberStateIdle = 2; private const uint FiberStateTerminated = 3; private const uint FiberFlagContextSizeCheck = 0x10; + private const uint FiberFlagSetFpuRegs = 0x100; + private const uint Firmware350BuildVersion = 0x03500000; + private const ushort InitialFpuControlWord = 0x037F; + private const uint InitialMxcsr = 0x9FC0; private const int FiberErrorNull = unchecked((int)0x80590001); private const int FiberErrorAlignment = unchecked((int)0x80590002); @@ -45,13 +50,26 @@ public static class FiberExports private static int _contextSizeCheck; - [ThreadStatic] - private static ulong _currentFiberAddress; - private static readonly object _fiberGate = new(); private static readonly ConcurrentDictionary _continuations = new(); - private static readonly ConcurrentDictionary _returnTargets = new(); private static readonly ConcurrentDictionary _stackRanges = new(); + private static readonly ConcurrentDictionary _threadStates = new(); + + static FiberExports() + { + RunFiberSelfChecks(); + } + + public static void ResetRuntimeState() + { + lock (_fiberGate) + { + _continuations.Clear(); + _stackRanges.Clear(); + _threadStates.Clear(); + Volatile.Write(ref _contextSizeCheck, 0); + } + } [SysAbiExport( Nid = "hVYD7Ou2pCQ", @@ -61,6 +79,7 @@ public static class FiberExports public static int FiberInitialize(CpuContext ctx) { var optParam = ReadStackArg64(ctx, 0); + var buildVersion = unchecked((uint)ReadStackArg64(ctx, 1)); return FiberInitializeCore( ctx, ctx[CpuRegister.Rdi], @@ -70,7 +89,8 @@ public static class FiberExports ctx[CpuRegister.R8], ctx[CpuRegister.R9], optParam, - flags: 0); + flags: 0, + buildVersion); } [SysAbiExport( @@ -82,6 +102,7 @@ public static class FiberExports { var optParam = ReadStackArg64(ctx, 0); var flags = unchecked((uint)ReadStackArg64(ctx, 1)); + var buildVersion = unchecked((uint)ReadStackArg64(ctx, 2)); return FiberInitializeCore( ctx, ctx[CpuRegister.Rdi], @@ -91,7 +112,8 @@ public static class FiberExports ctx[CpuRegister.R8], ctx[CpuRegister.R9], optParam, - flags); + flags, + buildVersion); } [SysAbiExport( @@ -104,17 +126,17 @@ public static class FiberExports var optParam = ctx[CpuRegister.Rdi]; if (optParam == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } if ((optParam & 7) != 0) { - return ctx.SetReturn(FiberErrorAlignment); + return SetReturn(ctx, FiberErrorAlignment); } - return ctx.TryWriteUInt32(optParam, FiberOptSignature) - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorInvalid); + return TryWriteUInt32(ctx, optParam, FiberOptSignature) + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorInvalid); } [SysAbiExport( @@ -127,24 +149,23 @@ public static class FiberExports var fiber = ctx[CpuRegister.Rdi]; if (!TryValidateFiber(ctx, fiber, out var error)) { - return ctx.SetReturn(error); + return SetReturn(ctx, error); } - if (!ctx.TryReadUInt32(fiber + FiberStateOffset, out var state)) + if (!TryReadUInt32(ctx, fiber + FiberStateOffset, out var state)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (state != FiberStateIdle) { - return ctx.SetReturn(FiberErrorState); + return SetReturn(ctx, FiberErrorState); } _continuations.TryRemove(fiber, out _); - _returnTargets.TryRemove(fiber, out _); _stackRanges.TryRemove(fiber, out _); - _ = ctx.TryWriteUInt32(fiber + FiberStateOffset, FiberStateTerminated); - return ctx.SetReturn(0); + _ = TryWriteUInt32(ctx, fiber + FiberStateOffset, FiberStateTerminated); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -229,74 +250,47 @@ public static class FiberExports var fiberAddress = ResolveCurrentFiberAddress(ctx); if (fiberAddress == 0) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } if (GuestThreadExecution.Scheduler is not { SupportsGuestContextTransfer: true } || !GuestThreadExecution.TryGetCurrentImportCallFrame(out var frame)) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } var returnArgument = ctx[CpuRegister.Rdi]; var argOnRunAddress = ctx[CpuRegister.Rsi]; - if (argOnRunAddress != 0 && !ctx.TryWriteUInt64(argOnRunAddress, 0)) - { - return ctx.SetReturn(FiberErrorInvalid); - } - GuestCpuContinuation transferTarget; - ulong previousFiber; lock (_fiberGate) { _continuations[fiberAddress] = new FiberContinuation( CaptureContinuation(ctx, frame.ReturnRip, frame.ResumeRsp, frame.ReturnSlotAddress), argOnRunAddress); - if (!_returnTargets.TryRemove(fiberAddress, out var returnTarget)) + var threadKey = GetThreadKey(ctx); + if (!_threadStates.TryGetValue(threadKey, out var threadState) || + !TryWriteResumeArgument(ctx, threadState.RootContinuation, returnArgument)) { _continuations.TryRemove(fiberAddress, out _); - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } - previousFiber = returnTarget.PreviousFiber; - if (previousFiber != 0) + if (!TryWriteUInt32(ctx, fiberAddress + FiberStateOffset, FiberStateIdle)) { - if (!_continuations.TryRemove(previousFiber, out var previousContinuation) || - !TryWriteResumeArgument(ctx, previousContinuation, returnArgument) || - !ctx.TryWriteUInt32(previousFiber + FiberStateOffset, FiberStateRun)) - { - _continuations.TryRemove(fiberAddress, out _); - return ctx.SetReturn(FiberErrorState); - } - - transferTarget = previousContinuation.Context with { Rax = 0 }; - } - else - { - if (!returnTarget.ThreadContinuation.HasValue || - !TryWriteResumeArgument(ctx, returnTarget.ThreadContinuation.Value, returnArgument)) - { - _continuations.TryRemove(fiberAddress, out _); - return ctx.SetReturn(FiberErrorState); - } - - transferTarget = returnTarget.ThreadContinuation.Value.Context with { Rax = 0 }; + return SetReturn(ctx, FiberErrorInvalid); } - if (!ctx.TryWriteUInt32(fiberAddress + FiberStateOffset, FiberStateIdle)) - { - return ctx.SetReturn(FiberErrorInvalid); - } + transferTarget = threadState.RootContinuation.Context with { Rax = 0 }; + _threadStates.TryRemove(threadKey, out _); } - _currentFiberAddress = previousFiber; - _ = GuestThreadExecution.EnterFiber(previousFiber); + _ = GuestThreadExecution.EnterFiber(0); GuestThreadExecution.RequestCurrentContextTransfer(transferTarget); TraceFiber( - $"return fiber=0x{fiberAddress:X16} to=0x{previousFiber:X16} " + + $"return-to-thread fiber=0x{fiberAddress:X16} " + $"resume=0x{transferTarget.Rip:X16} rsp=0x{transferTarget.Rsp:X16} arg=0x{returnArgument:X16}"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -309,18 +303,18 @@ public static class FiberExports var outAddress = ctx[CpuRegister.Rdi]; if (outAddress == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } var fiberAddress = ResolveCurrentFiberAddress(ctx); if (fiberAddress == 0) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } - return ctx.TryWriteUInt64(outAddress, fiberAddress) - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorInvalid); + return TryWriteUInt64(ctx, outAddress, fiberAddress) + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorInvalid); } [SysAbiExport( @@ -334,35 +328,35 @@ public static class FiberExports var info = ctx[CpuRegister.Rsi]; if (info == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } if (!TryValidateFiber(ctx, fiber, out var error)) { - return ctx.SetReturn(error); + return SetReturn(ctx, error); } - if (!ctx.TryReadUInt64(info, out var size) || size != FiberInfoSize) + if (!TryReadUInt64(ctx, info, out var size) || size != FiberInfoSize) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (!TryReadFiberFields(ctx, fiber, out var fields)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } - if (!ctx.TryWriteUInt64(info + 8, fields.Entry) || - !ctx.TryWriteUInt64(info + 16, fields.ArgOnInitialize) || - !ctx.TryWriteUInt64(info + 24, fields.ContextAddress) || - !ctx.TryWriteUInt64(info + 32, fields.ContextSize) || + if (!TryWriteUInt64(ctx, info + 8, fields.Entry) || + !TryWriteUInt64(ctx, info + 16, fields.ArgOnInitialize) || + !TryWriteUInt64(ctx, info + 24, fields.ContextAddress) || + !TryWriteUInt64(ctx, info + 32, fields.ContextSize) || !TryWriteName(ctx, info + 40, fields.Name) || - !ctx.TryWriteUInt64(info + 72, ulong.MaxValue)) + !TryWriteUInt64(ctx, info + 72, ulong.MaxValue)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -376,22 +370,22 @@ public static class FiberExports var nameAddress = ctx[CpuRegister.Rsi]; if (!TryValidateFiber(ctx, fiber, out var error)) { - return ctx.SetReturn(error); + return SetReturn(ctx, error); } if (nameAddress == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } - if (!ctx.TryReadNullTerminatedUtf8(nameAddress, MaxNameLength + 1, out var name)) + if (!TryReadNullTerminatedUtf8(ctx, nameAddress, MaxNameLength + 1, out var name)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } return TryWriteName(ctx, fiber + FiberNameOffset, name) - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorInvalid); + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorInvalid); } [SysAbiExport( @@ -403,12 +397,12 @@ public static class FiberExports { if (ctx[CpuRegister.Rdi] != 0) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } return Interlocked.Exchange(ref _contextSizeCheck, 1) == 0 - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorState); + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorState); } [SysAbiExport( @@ -419,8 +413,8 @@ public static class FiberExports public static int FiberStopContextSizeCheck(CpuContext ctx) { return Interlocked.Exchange(ref _contextSizeCheck, 0) == 1 - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorState); + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorState); } [SysAbiExport( @@ -433,17 +427,18 @@ public static class FiberExports var outAddress = ctx[CpuRegister.Rdi]; if (outAddress == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } - if (ResolveCurrentFiberAddress(ctx) == 0) + if (ResolveCurrentFiberAddress(ctx) == 0 || + !_threadStates.TryGetValue(GetThreadKey(ctx), out var threadState)) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } - return ctx.TryWriteUInt64(outAddress, ctx[CpuRegister.Rbp]) - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorInvalid); + return TryWriteUInt64(ctx, outAddress, threadState.RootContinuation.Context.Rbp) + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorInvalid); } private static int FiberInitializeCore( @@ -455,69 +450,70 @@ public static class FiberExports ulong contextAddress, ulong contextSize, ulong optParam, - uint flags) + uint flags, + uint buildVersion) { if (fiber == 0 || nameAddress == 0 || entry == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } if ((fiber & 7) != 0 || (contextAddress & 15) != 0 || (optParam & 7) != 0) { - return ctx.SetReturn(FiberErrorAlignment); + return SetReturn(ctx, FiberErrorAlignment); } if (contextSize != 0 && contextSize < FiberContextMinimumSize) { - return ctx.SetReturn(FiberErrorRange); + return SetReturn(ctx, FiberErrorRange); } if ((contextSize & 15) != 0 || (contextAddress == 0 && contextSize != 0) || (contextAddress != 0 && contextSize == 0)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (optParam != 0 && - (!ctx.TryReadUInt32(optParam, out var optMagic) || optMagic != FiberOptSignature)) + (!TryReadUInt32(ctx, optParam, out var optMagic) || optMagic != FiberOptSignature)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } - if (!ctx.TryReadNullTerminatedUtf8(nameAddress, MaxNameLength + 1, out var name)) + if (!TryReadNullTerminatedUtf8(ctx, nameAddress, MaxNameLength + 1, out var name)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } - if (Volatile.Read(ref _contextSizeCheck) != 0) - { - flags |= FiberFlagContextSizeCheck; - } + flags = ApplyInitializationFlags( + flags, + buildVersion, + Volatile.Read(ref _contextSizeCheck) != 0); - if (!ctx.TryWriteUInt32(fiber + FiberMagicStartOffset, FiberSignature0) || - !ctx.TryWriteUInt32(fiber + FiberStateOffset, FiberStateIdle) || - !ctx.TryWriteUInt64(fiber + FiberEntryOffset, entry) || - !ctx.TryWriteUInt64(fiber + FiberArgOnInitializeOffset, argOnInitialize) || - !ctx.TryWriteUInt64(fiber + FiberContextAddressOffset, contextAddress) || - !ctx.TryWriteUInt64(fiber + FiberContextSizeOffset, contextSize) || + if (!TryWriteUInt32(ctx, fiber + FiberMagicStartOffset, FiberSignature0) || + !TryWriteUInt32(ctx, fiber + FiberStateOffset, FiberStateIdle) || + !TryWriteUInt64(ctx, fiber + FiberEntryOffset, entry) || + !TryWriteUInt64(ctx, fiber + FiberArgOnInitializeOffset, argOnInitialize) || + !TryWriteUInt64(ctx, fiber + FiberContextAddressOffset, contextAddress) || + !TryWriteUInt64(ctx, fiber + FiberContextSizeOffset, contextSize) || !TryWriteName(ctx, fiber + FiberNameOffset, name) || - !ctx.TryWriteUInt64(fiber + FiberContextPointerOffset, 0) || - !ctx.TryWriteUInt32(fiber + FiberFlagsOffset, flags) || - !ctx.TryWriteUInt64(fiber + FiberContextStartOffset, contextAddress) || - !ctx.TryWriteUInt64(fiber + FiberContextEndOffset, contextAddress == 0 ? 0 : contextAddress + contextSize) || - !ctx.TryWriteUInt32(fiber + FiberMagicEndOffset, FiberSignature1)) + !TryWriteUInt64(ctx, fiber + FiberContextPointerOffset, 0) || + !TryWriteUInt32(ctx, fiber + FiberFlagsOffset, flags) || + !TryWriteUInt64(ctx, fiber + FiberContextStartOffset, contextAddress) || + !TryWriteUInt64(ctx, fiber + FiberContextEndOffset, contextAddress == 0 ? 0 : contextAddress + contextSize) || + !TryWriteUInt32(ctx, fiber + FiberMagicEndOffset, FiberSignature1)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (contextAddress != 0) { - if (!ctx.TryWriteUInt64(contextAddress, FiberStackSignature)) + if (!TryWriteUInt64(ctx, contextAddress, FiberStackSignature)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if ((flags & FiberFlagContextSizeCheck) != 0) @@ -531,8 +527,10 @@ public static class FiberExports _stackRanges[fiber] = new FiberStackRange(contextAddress, contextSize); } - TraceFiber($"init fiber=0x{fiber:X16} entry=0x{entry:X16} ctx=0x{contextAddress:X16} size=0x{contextSize:X} name='{name}'"); - return ctx.SetReturn(0); + TraceFiber( + $"init fiber=0x{fiber:X16} entry=0x{entry:X16} ctx=0x{contextAddress:X16} " + + $"size=0x{contextSize:X} flags=0x{flags:X} build=0x{buildVersion:X8} name='{name}'"); + return SetReturn(ctx, 0); } private static int FiberRunCore( @@ -547,12 +545,12 @@ public static class FiberExports { if (!TryValidateFiber(ctx, fiber, out var error)) { - return ctx.SetReturn(error); + return SetReturn(ctx, error); } if (!TryReadFiberFields(ctx, fiber, out var fields)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (attachContextAddress != 0 || attachContextSize != 0) @@ -560,7 +558,7 @@ public static class FiberExports var attachResult = AttachContext(ctx, fiber, attachContextAddress, attachContextSize, ref fields); if (attachResult != 0) { - return ctx.SetReturn(attachResult); + return SetReturn(ctx, attachResult); } } @@ -568,78 +566,96 @@ public static class FiberExports if ((isSwitch && previousFiber == 0) || (!isSwitch && previousFiber != 0)) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } if (previousFiber == fiber) { - return ctx.SetReturn(FiberErrorState); + return SetReturn(ctx, FiberErrorState); } if (GuestThreadExecution.Scheduler is not { SupportsGuestContextTransfer: true } || !GuestThreadExecution.TryGetCurrentImportCallFrame(out var frame)) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } GuestCpuContinuation transferTarget; var resumed = false; lock (_fiberGate) { + var threadKey = GetThreadKey(ctx); if (!TryReadFiberFields(ctx, fiber, out fields)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (fields.State != FiberStateIdle) { TraceFiber($"run-state-error reason={reason} fiber=0x{fiber:X16} state=0x{fields.State:X8}"); - return ctx.SetReturn(FiberErrorState); + return SetReturn(ctx, FiberErrorState); } - FiberContinuation targetContinuation; + FiberContinuation targetContinuation = default; if (_continuations.TryGetValue(fiber, out var savedContinuation)) { targetContinuation = savedContinuation; resumed = true; } - else if (!TryCreateInitialContinuation(ctx, fields, argOnRun, out targetContinuation)) - { - return ctx.SetReturn(FiberErrorInvalid); - } - - if (resumed && !TryWriteResumeArgument(ctx, targetContinuation, argOnRun)) - { - return ctx.SetReturn(FiberErrorInvalid); - } - var callerContinuation = new FiberContinuation( CaptureContinuation(ctx, frame.ReturnRip, frame.ResumeRsp, frame.ReturnSlotAddress), outArgumentAddress); + if (!resumed) + { + var rootStackTop = _threadStates.TryGetValue(threadKey, out var existingThreadState) + ? existingThreadState.RootContinuation.Context.Rsp + : callerContinuation.Context.Rsp; + if (!TryCreateInitialContinuation( + ctx, + fields, + argOnRun, + rootStackTop, + out targetContinuation)) + { + return SetReturn(ctx, FiberErrorInvalid); + } + } + else if (!TryWriteResumeArgument(ctx, targetContinuation, argOnRun)) + { + return SetReturn(ctx, FiberErrorInvalid); + } + if (previousFiber != 0) { - if (!ctx.TryReadUInt32(previousFiber + FiberStateOffset, out var previousState) || + if (!TryReadUInt32(ctx, previousFiber + FiberStateOffset, out var previousState) || previousState != FiberStateRun || - !ctx.TryWriteUInt32(previousFiber + FiberStateOffset, FiberStateIdle)) + !TryWriteUInt32(ctx, previousFiber + FiberStateOffset, FiberStateIdle)) { - return ctx.SetReturn(FiberErrorState); + return SetReturn(ctx, FiberErrorState); } _continuations[previousFiber] = callerContinuation; - _returnTargets[fiber] = new FiberReturnTarget(previousFiber, null); } else { - _returnTargets[fiber] = new FiberReturnTarget(0, callerContinuation); + if (_threadStates.ContainsKey(threadKey)) + { + return SetReturn(ctx, FiberErrorPermission); + } + + _threadStates[threadKey] = new FiberThreadState(callerContinuation, fiber, previousFiber); } - if (!ctx.TryWriteUInt32(fiber + FiberStateOffset, FiberStateRun)) + if (!TryWriteUInt32(ctx, fiber + FiberStateOffset, FiberStateRun)) { if (previousFiber != 0) { _continuations.TryRemove(previousFiber, out _); - _ = ctx.TryWriteUInt32(previousFiber + FiberStateOffset, FiberStateRun); + _ = TryWriteUInt32(ctx, previousFiber + FiberStateOffset, FiberStateRun); } - _returnTargets.TryRemove(fiber, out _); - return ctx.SetReturn(FiberErrorInvalid); + else + { + _threadStates.TryRemove(threadKey, out _); + } + return SetReturn(ctx, FiberErrorInvalid); } if (resumed) @@ -648,35 +664,66 @@ public static class FiberExports } transferTarget = targetContinuation.Context with { Rax = 0 }; + if (_threadStates.TryGetValue(threadKey, out var activeState)) + { + _threadStates[threadKey] = activeState with + { + CurrentFiber = fiber, + PreviousFiber = previousFiber, + }; + } } - _currentFiberAddress = fiber; _ = GuestThreadExecution.EnterFiber(fiber); GuestThreadExecution.RequestCurrentContextTransfer(transferTarget); TraceFiber( $"transfer reason={reason} from=0x{previousFiber:X16} to=0x{fiber:X16} resume={resumed} " + $"rip=0x{transferTarget.Rip:X16} rsp=0x{transferTarget.Rsp:X16} arg=0x{argOnRun:X16}"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } private static bool TryCreateInitialContinuation( CpuContext ctx, FiberFields fields, ulong argOnRun, + ulong rootStackTop, out FiberContinuation continuation) { continuation = default; - if (fields.ContextAddress == 0 || fields.ContextSize < FiberContextMinimumSize) + ulong stackEnd; + if (fields.ContextAddress == 0) + { + if (rootStackTop < 32) + { + return false; + } + + // Contextless fibers are specified to execute on the root + // sceFiberRun stack. Keep the suspended import return record + // above the entry stack and use a synthetic transfer slot below it. + stackEnd = rootStackTop & ~15UL; + } + else + { + if (fields.ContextSize < FiberContextMinimumSize) + { + return false; + } + + stackEnd = fields.ContextAddress + fields.ContextSize; + } + + if (!TryCalculateInitialStackLayout(stackEnd, out var entryRsp, out var transferSlot)) + { + return false; + } + if (!TryWriteUInt64(ctx, transferSlot, fields.Entry) || + !TryWriteUInt64(ctx, entryRsp, 0)) { return false; } - var stackEnd = fields.ContextAddress + fields.ContextSize; - var entryRsp = (stackEnd & ~15UL) - sizeof(ulong); - if (!ctx.TryWriteUInt64(entryRsp, 0)) - { - return false; - } + var setFpuRegisters = (fields.Flags & FiberFlagSetFpuRegs) != 0; continuation = new FiberContinuation( new GuestCpuContinuation( @@ -698,7 +745,12 @@ public static class FiberExports 0, 0, 0, - 0), + 0, + 0, + 0, + setFpuRegisters ? InitialFpuControlWord : ctx.FpuControlWord, + setFpuRegisters ? InitialMxcsr : ctx.Mxcsr, + RestoreFullFpuState: setFpuRegisters), 0); return true; } @@ -708,7 +760,7 @@ public static class FiberExports FiberContinuation continuation, ulong argument) => continuation.ArgOnRunAddress == 0 || - ctx.TryWriteUInt64(continuation.ArgOnRunAddress, argument); + TryWriteUInt64(ctx, continuation.ArgOnRunAddress, argument); private static GuestCpuContinuation CaptureContinuation( CpuContext ctx, @@ -731,26 +783,99 @@ public static class FiberExports ctx[CpuRegister.Rdi], ctx[CpuRegister.R8], ctx[CpuRegister.R9], + ctx[CpuRegister.R10], + ctx[CpuRegister.R11], ctx[CpuRegister.R12], ctx[CpuRegister.R13], ctx[CpuRegister.R14], - ctx[CpuRegister.R15]); + ctx[CpuRegister.R15], + ctx.FpuControlWord, + ctx.Mxcsr, + RestoreFullFpuState: false); private static ulong ResolveCurrentFiberAddress(CpuContext ctx) { - if (_currentFiberAddress != 0) - { - return _currentFiberAddress; - } - if (GuestThreadExecution.CurrentFiberAddress != 0) { return GuestThreadExecution.CurrentFiberAddress; } + if (_threadStates.TryGetValue(GetThreadKey(ctx), out var threadState) && + threadState.CurrentFiber != 0) + { + return threadState.CurrentFiber; + } + return TryFindFiberByStack(ctx, out var fiberAddress) ? fiberAddress : 0; } + private static ulong GetThreadKey(CpuContext ctx) + { + var handle = GuestThreadExecution.CurrentGuestThreadHandle; + if (handle != 0) + { + return handle; + } + + // The main guest thread does not always have a scheduler handle. Its + // ABI thread pointer is stable across native/managed transitions. + return ctx.FsBase != 0 ? ctx.FsBase : 1; + } + + private static uint ApplyInitializationFlags( + uint flags, + uint buildVersion, + bool contextSizeCheck) + { + if (buildVersion >= Firmware350BuildVersion) + { + flags |= FiberFlagSetFpuRegs; + } + if (contextSizeCheck) + { + flags |= FiberFlagContextSizeCheck; + } + + return flags; + } + + private static bool TryCalculateInitialStackLayout( + ulong stackEnd, + out ulong entryRsp, + out ulong transferSlot) + { + var alignedEnd = stackEnd & ~15UL; + if (alignedEnd < 2 * sizeof(ulong)) + { + entryRsp = 0; + transferSlot = 0; + return false; + } + + entryRsp = alignedEnd - sizeof(ulong); + transferSlot = entryRsp - sizeof(ulong); + return true; + } + + [Conditional("DEBUG")] + private static void RunFiberSelfChecks() + { + Debug.Assert( + ApplyInitializationFlags(0, Firmware350BuildVersion - 1, false) == 0, + "Pre-3.50 fibers unexpectedly enable SetFpuRegs."); + Debug.Assert( + ApplyInitializationFlags(0, Firmware350BuildVersion, false) == FiberFlagSetFpuRegs, + "3.50+ fibers must initialize x87/MXCSR control state."); + Debug.Assert( + ApplyInitializationFlags(1, Firmware350BuildVersion, true) == + (1u | FiberFlagSetFpuRegs | FiberFlagContextSizeCheck), + "Fiber initialization discarded caller/internal flags."); + Debug.Assert( + TryCalculateInitialStackLayout(0x1017, out var rsp, out var slot) && + rsp == 0x1008 && slot == 0x1000 && (rsp & 15) == 8, + "Initial fiber transfer slot does not produce SysV entry alignment."); + } + internal static ulong GetCurrentFiberAddressForDiagnostics(CpuContext ctx) => ResolveCurrentFiberAddress(ctx); @@ -779,11 +904,11 @@ public static class FiberExports return FiberErrorInvalid; } - if (!ctx.TryWriteUInt64(fiber + FiberContextAddressOffset, contextAddress) || - !ctx.TryWriteUInt64(fiber + FiberContextSizeOffset, contextSize) || - !ctx.TryWriteUInt64(fiber + FiberContextStartOffset, contextAddress) || - !ctx.TryWriteUInt64(fiber + FiberContextEndOffset, contextAddress + contextSize) || - !ctx.TryWriteUInt64(contextAddress, FiberStackSignature)) + if (!TryWriteUInt64(ctx, fiber + FiberContextAddressOffset, contextAddress) || + !TryWriteUInt64(ctx, fiber + FiberContextSizeOffset, contextSize) || + !TryWriteUInt64(ctx, fiber + FiberContextStartOffset, contextAddress) || + !TryWriteUInt64(ctx, fiber + FiberContextEndOffset, contextAddress + contextSize) || + !TryWriteUInt64(ctx, contextAddress, FiberStackSignature)) { return FiberErrorInvalid; } @@ -839,8 +964,8 @@ public static class FiberExports return false; } - if (!ctx.TryReadUInt32(fiber + FiberMagicStartOffset, out var magicStart) || - !ctx.TryReadUInt32(fiber + FiberMagicEndOffset, out var magicEnd) || + if (!TryReadUInt32(ctx, fiber + FiberMagicStartOffset, out var magicStart) || + !TryReadUInt32(ctx, fiber + FiberMagicEndOffset, out var magicEnd) || magicStart != FiberSignature0 || magicEnd != FiberSignature1) { @@ -855,12 +980,12 @@ public static class FiberExports private static bool TryReadFiberFields(CpuContext ctx, ulong fiber, out FiberFields fields) { fields = default; - if (!ctx.TryReadUInt32(fiber + FiberStateOffset, out var state) || - !ctx.TryReadUInt64(fiber + FiberEntryOffset, out var entry) || - !ctx.TryReadUInt64(fiber + FiberArgOnInitializeOffset, out var argOnInitialize) || - !ctx.TryReadUInt64(fiber + FiberContextAddressOffset, out var contextAddress) || - !ctx.TryReadUInt64(fiber + FiberContextSizeOffset, out var contextSize) || - !ctx.TryReadUInt32(fiber + FiberFlagsOffset, out var flags) || + if (!TryReadUInt32(ctx, fiber + FiberStateOffset, out var state) || + !TryReadUInt64(ctx, fiber + FiberEntryOffset, out var entry) || + !TryReadUInt64(ctx, fiber + FiberArgOnInitializeOffset, out var argOnInitialize) || + !TryReadUInt64(ctx, fiber + FiberContextAddressOffset, out var contextAddress) || + !TryReadUInt64(ctx, fiber + FiberContextSizeOffset, out var contextSize) || + !TryReadUInt32(ctx, fiber + FiberFlagsOffset, out var flags) || !TryReadInlineName(ctx, fiber + FiberNameOffset, out var name)) { return false; @@ -898,6 +1023,46 @@ public static class FiberExports return 0; } + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(buffer); + return true; + } + + private static bool TryWriteUInt64(CpuContext ctx, ulong address, ulong value) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + private static bool TryWriteName(CpuContext ctx, ulong address, string name) { Span buffer = stackalloc byte[MaxNameLength + 1]; @@ -925,6 +1090,37 @@ public static class FiberExports return true; } + private static bool TryReadNullTerminatedUtf8(CpuContext ctx, ulong address, int capacity, out string value) + { + var bytes = new byte[capacity]; + Span current = stackalloc byte[1]; + for (var index = 0; index < bytes.Length; index++) + { + if (!ctx.Memory.TryRead(address + (ulong)index, current)) + { + value = string.Empty; + return false; + } + + if (current[0] == 0) + { + value = Encoding.UTF8.GetString(bytes, 0, index); + return true; + } + + bytes[index] = current[0]; + } + + value = Encoding.UTF8.GetString(bytes); + return true; + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + private static void TraceFiber(string message) { if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_FIBER"), "1", StringComparison.Ordinal)) @@ -946,9 +1142,10 @@ public static class FiberExports GuestCpuContinuation Context, ulong ArgOnRunAddress); - private readonly record struct FiberReturnTarget( - ulong PreviousFiber, - FiberContinuation? ThreadContinuation); + private sealed record FiberThreadState( + FiberContinuation RootContinuation, + ulong CurrentFiber, + ulong PreviousFiber); private readonly record struct FiberStackRange(ulong Start, ulong Size) { diff --git a/src/SharpEmu.Libs/Font/FontExports.cs b/src/SharpEmu.Libs/Font/FontExports.cs new file mode 100644 index 0000000..b02d14d --- /dev/null +++ b/src/SharpEmu.Libs/Font/FontExports.cs @@ -0,0 +1,293 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.HLE; + +namespace SharpEmu.Libs.Font; + +public static class FontExports +{ + private static readonly object AllocationGate = new(); + private static ulong _librarySelectionAddress; + private static ulong _rendererSelectionAddress; + + [SysAbiExport( + Nid = "whrS4oksXc4", + ExportName = "sceFontMemoryInit", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int MemoryInit(CpuContext ctx) + { + var descriptorAddress = ctx[CpuRegister.Rdi]; + var regionAddress = ctx[CpuRegister.Rsi]; + var regionSize = (uint)ctx[CpuRegister.Rdx]; + var interfaceAddress = ctx[CpuRegister.Rcx]; + var mspaceAddress = ctx[CpuRegister.R8]; + var destroyCallback = ctx[CpuRegister.R9]; + if (descriptorAddress == 0 || + !TryWriteUInt32(ctx, descriptorAddress, 0x00000F00) || + !TryWriteUInt32(ctx, descriptorAddress + 0x04, regionSize) || + !ctx.TryWriteUInt64(descriptorAddress + 0x08, regionAddress) || + !ctx.TryWriteUInt64(descriptorAddress + 0x10, mspaceAddress) || + !ctx.TryWriteUInt64(descriptorAddress + 0x18, interfaceAddress) || + !ctx.TryWriteUInt64(descriptorAddress + 0x20, destroyCallback) || + !ctx.TryWriteUInt64(descriptorAddress + 0x28, 0) || + !ctx.TryWriteUInt64(descriptorAddress + 0x30, 0) || + !ctx.TryWriteUInt64(descriptorAddress + 0x38, mspaceAddress)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetSuccess(ctx); + } + + [SysAbiExport( + Nid = "oM+XCzVG3oM", + ExportName = "sceFontSelectLibraryFt", + Target = Generation.Gen5, + LibraryName = "libSceFontFt")] + public static int SelectLibraryFt(CpuContext ctx) => + ReturnSelection(ctx, ref _librarySelectionAddress, 0x38); + + [SysAbiExport( + Nid = "Xx974EW-QFY", + ExportName = "sceFontSelectRendererFt", + Target = Generation.Gen5, + LibraryName = "libSceFontFt")] + public static int SelectRendererFt(CpuContext ctx) => + ReturnSelection(ctx, ref _rendererSelectionAddress, 0x100); + + [SysAbiExport( + Nid = "n590hj5Oe-k", + ExportName = "sceFontCreateLibraryWithEdition", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int CreateLibraryWithEdition(CpuContext ctx) => + CreateOpaqueHandle(ctx, ctx[CpuRegister.Rcx], 0x100, magic: 0x0F01); + + [SysAbiExport( + Nid = "WaSFJoRWXaI", + ExportName = "sceFontCreateRendererWithEdition", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int CreateRendererWithEdition(CpuContext ctx) => + CreateOpaqueHandle(ctx, ctx[CpuRegister.Rcx], 0x100, magic: 0x0F07); + + [SysAbiExport( + Nid = "cKYtVmeSTcw", + ExportName = "sceFontOpenFontSet", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int OpenFontSet(CpuContext ctx) => + CreateOpaqueHandle(ctx, ctx[CpuRegister.R8], 0x100, magic: 0x0F02); + + [SysAbiExport( + Nid = "KXUpebrFk1U", + ExportName = "sceFontOpenFontMemory", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int OpenFontMemory(CpuContext ctx) => + CreateOpaqueHandle(ctx, ctx[CpuRegister.R8], 0x100, magic: 0x0F02); + + [SysAbiExport( + Nid = "JzCH3SCFnAU", + ExportName = "sceFontOpenFontInstance", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int OpenFontInstance(CpuContext ctx) + { + var sourceHandle = ctx[CpuRegister.Rdi]; + var setupHandle = ctx[CpuRegister.Rsi]; + var outputAddress = ctx[CpuRegister.Rdx]; + if (outputAddress == 0) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + if (setupHandle != 0) + { + return ctx.TryWriteUInt64(outputAddress, setupHandle) + ? SetSuccess(ctx) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + if (!TryAllocateOpaque(ctx, 0x100, out var handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + if (sourceHandle != 0) + { + Span source = stackalloc byte[0x100]; + if (ctx.Memory.TryRead(sourceHandle, source)) + { + _ = ctx.Memory.TryWrite(handle, source); + } + } + + _ = TryWriteUInt16(ctx, handle, 0x0F02); + return ctx.TryWriteUInt64(outputAddress, handle) + ? SetSuccess(ctx) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "SsRbbCiWoGw", + ExportName = "sceFontSupportSystemFonts", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int SupportSystemFonts(CpuContext ctx) => SetSuccess(ctx); + + [SysAbiExport( + Nid = "mz2iTY0MK4A", + ExportName = "sceFontSupportExternalFonts", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int SupportExternalFonts(CpuContext ctx) => SetSuccess(ctx); + + [SysAbiExport( + Nid = "CUKn5pX-NVY", + ExportName = "sceFontAttachDeviceCacheBuffer", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int AttachDeviceCacheBuffer(CpuContext ctx) => SetSuccess(ctx); + + [SysAbiExport( + Nid = "IQtleGLL5pQ", + ExportName = "sceFontGetRenderCharGlyphMetrics", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int GetRenderCharGlyphMetrics(CpuContext ctx) + { + var metricsAddress = ctx[CpuRegister.Rdx]; + if (metricsAddress == 0) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var values = new[] { 8.0f, 16.0f, 0.0f, 12.0f, 8.0f, 0.0f, 0.0f, 16.0f }; + for (var index = 0; index < values.Length; index++) + { + if (!TryWriteUInt32( + ctx, + metricsAddress + (ulong)(index * sizeof(float)), + BitConverter.SingleToUInt32Bits(values[index]))) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + } + + return SetSuccess(ctx); + } + + [SysAbiExport( + Nid = "gdUCnU0gHdI", + ExportName = "sceFontRenderSurfaceInit", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int RenderSurfaceInit(CpuContext ctx) + { + var surfaceAddress = ctx[CpuRegister.Rdi]; + var bufferAddress = ctx[CpuRegister.Rsi]; + var widthBytes = (uint)ctx[CpuRegister.Rdx]; + var pixelBytes = (uint)ctx[CpuRegister.Rcx] & 0xFF; + var width = (uint)ctx[CpuRegister.R8]; + var height = (uint)ctx[CpuRegister.R9]; + if (surfaceAddress == 0 || + !ctx.TryWriteUInt64(surfaceAddress, bufferAddress) || + !TryWriteUInt32(ctx, surfaceAddress + 0x08, widthBytes) || + !TryWriteUInt32(ctx, surfaceAddress + 0x0C, pixelBytes) || + !TryWriteUInt32(ctx, surfaceAddress + 0x10, width) || + !TryWriteUInt32(ctx, surfaceAddress + 0x14, height) || + !TryWriteUInt32(ctx, surfaceAddress + 0x18, 0) || + !TryWriteUInt32(ctx, surfaceAddress + 0x1C, 0) || + !TryWriteUInt32(ctx, surfaceAddress + 0x20, width) || + !TryWriteUInt32(ctx, surfaceAddress + 0x24, height)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetSuccess(ctx); + } + + private static int ReturnSelection(CpuContext ctx, ref ulong selectionAddress, uint objectSize) + { + if (ctx[CpuRegister.Rdi] != 0) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + lock (AllocationGate) + { + if (selectionAddress == 0) + { + if (!TryAllocateOpaque(ctx, 0x20, out selectionAddress) || + !TryWriteUInt32(ctx, selectionAddress, 0) || + !TryWriteUInt32(ctx, selectionAddress + 4, objectSize)) + { + selectionAddress = 0; + } + } + } + + ctx[CpuRegister.Rax] = selectionAddress; + return 0; + } + + private static int CreateOpaqueHandle(CpuContext ctx, ulong outputAddress, int size, ushort magic) + { + if (outputAddress == 0 || !TryAllocateOpaque(ctx, size, out var handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + if (!TryWriteUInt16(ctx, handle, magic) || !ctx.TryWriteUInt64(outputAddress, handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetSuccess(ctx); + } + + private static bool TryAllocateOpaque(CpuContext ctx, int size, out ulong address) + { + address = 0; + if (ctx.Memory is not IGuestMemoryAllocator allocator || + !allocator.TryAllocateGuestMemory((ulong)size, 0x10, out address)) + { + return false; + } + + Span bytes = stackalloc byte[size]; + bytes.Clear(); + return ctx.Memory.TryWrite(address, bytes); + } + + private static bool TryWriteUInt16(CpuContext ctx, ulong address, ushort value) + { + Span bytes = stackalloc byte[sizeof(ushort)]; + BinaryPrimitives.WriteUInt16LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + + private static int SetSuccess(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)(int)result); + return (int)result; + } +} diff --git a/src/SharpEmu.Libs/Gpu/GuestGpuTypes.cs b/src/SharpEmu.Libs/Gpu/GuestGpuTypes.cs index 759b40d..99f90dc 100644 --- a/src/SharpEmu.Libs/Gpu/GuestGpuTypes.cs +++ b/src/SharpEmu.Libs/Gpu/GuestGpuTypes.cs @@ -22,6 +22,8 @@ internal sealed record GuestDrawTexture( bool IsStorage, uint MipLevels = 1, uint MipLevel = 0, + uint BaseMipLevel = 0, + uint ResourceMipLevels = 1, uint Pitch = 0, uint TileMode = 0, uint DstSelect = 0xFAC, @@ -36,7 +38,11 @@ internal readonly record struct GuestSampler( internal sealed record GuestMemoryBuffer( ulong BaseAddress, - byte[] Data); + byte[] Data, + int Length, + bool Pooled, + bool Writable = false, + bool WriteBackToGuest = true); /// DataFormat/NumberFormat are raw guest vertex-attribute codes. internal sealed record GuestVertexBuffer( @@ -47,11 +53,15 @@ internal sealed record GuestVertexBuffer( ulong BaseAddress, uint Stride, uint OffsetBytes, - byte[] Data); + byte[] Data, + int Length, + bool Pooled); internal sealed record GuestIndexBuffer( byte[] Data, - bool Is32Bit); + int Length, + bool Is32Bit, + bool Pooled); internal readonly record struct GuestRect( int X, @@ -67,6 +77,25 @@ internal readonly record struct GuestViewport( float MinDepth, float MaxDepth); +internal readonly record struct GuestRasterState( + bool CullFront, + bool CullBack, + bool FrontFaceClockwise, + bool Wireframe) +{ + public static GuestRasterState Default { get; } = new(false, false, false, false); +} + +// CompareOp uses the GCN DB_DEPTH_CONTROL ZFUNC encoding, which matches the +// Vulkan CompareOp ordering (0=Never through 7=Always). +internal readonly record struct GuestDepthState( + bool TestEnable, + bool WriteEnable, + uint CompareOp) +{ + public static GuestDepthState Default { get; } = new(false, false, 7); +} + /// Factors/funcs are raw guest CB_BLEND*_CONTROL register bitfields; the /// defaults (1/0) are the guest ONE/ZERO codes. internal readonly record struct GuestBlendState( @@ -95,12 +124,16 @@ internal readonly record struct GuestBlendState( internal sealed record GuestRenderState( IReadOnlyList Blends, GuestRect? Scissor, - GuestViewport? Viewport) + GuestViewport? Viewport, + GuestRasterState Raster, + GuestDepthState Depth) { public static GuestRenderState Default { get; } = new( [GuestBlendState.Default], Scissor: null, - Viewport: null); + Viewport: null, + GuestRasterState.Default, + GuestDepthState.Default); public GuestBlendState Blend => Blends.Count == 0 ? GuestBlendState.Default : Blends[0]; @@ -114,3 +147,17 @@ internal sealed record GuestRenderTarget( uint Format, uint NumberType, uint MipLevels = 1); + +/// Guest DB surface bound alongside a color render target. +internal sealed record GuestDepthTarget( + ulong ReadAddress, + ulong WriteAddress, + uint Width, + uint Height, + uint GuestFormat, + uint SwizzleMode, + float ClearDepth, + bool ReadOnly) +{ + public ulong Address => WriteAddress != 0 ? WriteAddress : ReadAddress; +} diff --git a/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs b/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs index 0963fc2..371ad82 100644 --- a/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs +++ b/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs @@ -33,7 +33,9 @@ internal interface IGuestGpuBackend int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1); + int scalarRegisterBufferIndex = -1, + int requiredVertexOutputCount = 0, + ulong storageBufferOffsetAlignment = 1); bool TryCompilePixelShader( Gen5ShaderState state, @@ -44,7 +46,10 @@ internal interface IGuestGpuBackend int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1); + int scalarRegisterBufferIndex = -1, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + ulong storageBufferOffsetAlignment = 1); bool TryCompileComputeShader( Gen5ShaderState state, @@ -53,7 +58,14 @@ internal interface IGuestGpuBackend uint localSizeY, uint localSizeZ, out IGuestCompiledShader? shader, - out string error); + out string error, + int totalGlobalBufferCount = -1, + int initialScalarBufferIndex = -1, + uint waveLaneCount = 32, + ulong storageBufferOffsetAlignment = 1); + + /// Returns the backend's no-color-output fragment shader. + IGuestCompiledShader GetDepthOnlyFragmentShader(); void HideSplashScreen(); @@ -78,6 +90,21 @@ internal interface IGuestGpuBackend IReadOnlyList? vertexBuffers = null, GuestRenderState? renderState = null); + void SubmitDepthOnlyTranslatedDraw( + IGuestCompiledShader pixelShader, + IReadOnlyList textures, + IReadOnlyList globalMemoryBuffers, + uint attributeCount, + GuestDepthTarget depthTarget, + IGuestCompiledShader? vertexShader = null, + uint vertexCount = 3, + uint instanceCount = 1, + uint primitiveType = 4, + GuestIndexBuffer? indexBuffer = null, + IReadOnlyList? vertexBuffers = null, + GuestRenderState? renderState = null, + ulong shaderAddress = 0); + void SubmitOffscreenTranslatedDraw( IGuestCompiledShader pixelShader, IReadOnlyList textures, @@ -90,7 +117,9 @@ internal interface IGuestGpuBackend uint primitiveType = 4, GuestIndexBuffer? indexBuffer = null, IReadOnlyList? vertexBuffers = null, - GuestRenderState? renderState = null); + GuestRenderState? renderState = null, + GuestDepthTarget? depthTarget = null, + ulong shaderAddress = 0); void SubmitStorageTranslatedDraw( IGuestCompiledShader pixelShader, @@ -98,16 +127,28 @@ internal interface IGuestGpuBackend IReadOnlyList globalMemoryBuffers, uint attributeCount, uint width, - uint height); + uint height, + ulong shaderAddress = 0); - void SubmitComputeDispatch( + long SubmitComputeDispatch( ulong shaderAddress, IGuestCompiledShader computeShader, IReadOnlyList textures, IReadOnlyList globalMemoryBuffers, uint groupCountX, uint groupCountY, - uint groupCountZ); + uint groupCountZ, + uint baseGroupX, + uint baseGroupY, + uint baseGroupZ, + uint localSizeX, + uint localSizeY, + uint localSizeZ, + bool isIndirect, + bool writesGlobalMemory, + uint threadCountX = uint.MaxValue, + uint threadCountY = uint.MaxValue, + uint threadCountZ = uint.MaxValue); bool TrySubmitGuestImage( ulong address, @@ -115,6 +156,14 @@ internal interface IGuestGpuBackend uint height, uint pitchInPixel); + bool TrySubmitOrderedGuestImageFlip( + int videoOutHandle, + int displayBufferIndex, + ulong address, + uint width, + uint height, + uint pitchInPixel); + /// Registers a display buffer with its guest texture format tag. void RegisterKnownDisplayBuffer(ulong address, uint guestFormat); @@ -126,10 +175,12 @@ internal interface IGuestGpuBackend uint sourceWidth, uint sourceHeight, uint sourceFormat, + uint sourceNumberType, ulong destinationAddress, uint destinationWidth, uint destinationHeight, - uint destinationFormat); + uint destinationFormat, + uint destinationNumberType); /// /// Whether the backend supports the guest render-target format, and how its pixel diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanGuestGpuBackend.cs b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanGuestGpuBackend.cs index 702d6de..c221fe4 100644 --- a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanGuestGpuBackend.cs +++ b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanGuestGpuBackend.cs @@ -15,6 +15,9 @@ namespace SharpEmu.Libs.Gpu.Vulkan; /// internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend { + private static readonly IGuestCompiledShader DepthOnlyFragmentShader = + new VulkanCompiledGuestShader(SpirvFixedShaders.CreateDepthOnlyFragment()); + public bool TryCompileVertexShader( Gen5ShaderState state, Gen5ShaderEvaluation evaluation, @@ -23,7 +26,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) + int scalarRegisterBufferIndex = -1, + int requiredVertexOutputCount = 0, + ulong storageBufferOffsetAlignment = 1) { shader = null; if (!Gen5SpirvTranslator.TryCompileVertexShader( @@ -34,7 +39,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex)) + scalarRegisterBufferIndex, + requiredVertexOutputCount, + storageBufferOffsetAlignment)) { return false; } @@ -52,7 +59,10 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) + int scalarRegisterBufferIndex = -1, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + ulong storageBufferOffsetAlignment = 1) { shader = null; if (!Gen5SpirvTranslator.TryCompilePixelShader( @@ -64,7 +74,10 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex)) + scalarRegisterBufferIndex, + pixelInputEnable, + pixelInputAddress, + storageBufferOffsetAlignment)) { return false; } @@ -80,7 +93,11 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend uint localSizeY, uint localSizeZ, out IGuestCompiledShader? shader, - out string error) + out string error, + int totalGlobalBufferCount = -1, + int initialScalarBufferIndex = -1, + uint waveLaneCount = 32, + ulong storageBufferOffsetAlignment = 1) { shader = null; if (!Gen5SpirvTranslator.TryCompileComputeShader( @@ -90,7 +107,11 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend localSizeY, localSizeZ, out var compiled, - out error)) + out error, + totalGlobalBufferCount, + initialScalarBufferIndex, + waveLaneCount, + storageBufferOffsetAlignment)) { return false; } @@ -99,6 +120,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend return true; } + public IGuestCompiledShader GetDepthOnlyFragmentShader() => + DepthOnlyFragmentShader; + public void EnsureStarted(uint width, uint height) => VulkanVideoPresenter.EnsureStarted(width, height); @@ -140,6 +164,35 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend vertexBuffers, renderState); + public void SubmitDepthOnlyTranslatedDraw( + IGuestCompiledShader pixelShader, + IReadOnlyList textures, + IReadOnlyList globalMemoryBuffers, + uint attributeCount, + GuestDepthTarget depthTarget, + IGuestCompiledShader? vertexShader = null, + uint vertexCount = 3, + uint instanceCount = 1, + uint primitiveType = 4, + GuestIndexBuffer? indexBuffer = null, + IReadOnlyList? vertexBuffers = null, + GuestRenderState? renderState = null, + ulong shaderAddress = 0) => + VulkanVideoPresenter.SubmitDepthOnlyTranslatedDraw( + Spirv(pixelShader), + textures, + globalMemoryBuffers, + attributeCount, + depthTarget, + vertexShader is null ? null : Spirv(vertexShader), + vertexCount, + instanceCount, + primitiveType, + indexBuffer, + vertexBuffers, + renderState, + shaderAddress); + public void SubmitOffscreenTranslatedDraw( IGuestCompiledShader pixelShader, IReadOnlyList textures, @@ -152,7 +205,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend uint primitiveType = 4, GuestIndexBuffer? indexBuffer = null, IReadOnlyList? vertexBuffers = null, - GuestRenderState? renderState = null) => + GuestRenderState? renderState = null, + GuestDepthTarget? depthTarget = null, + ulong shaderAddress = 0) => VulkanVideoPresenter.SubmitOffscreenTranslatedDraw( Spirv(pixelShader), textures, @@ -165,7 +220,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend primitiveType, indexBuffer, vertexBuffers, - renderState); + renderState, + depthTarget, + shaderAddress); public void SubmitStorageTranslatedDraw( IGuestCompiledShader pixelShader, @@ -173,23 +230,36 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend IReadOnlyList globalMemoryBuffers, uint attributeCount, uint width, - uint height) => + uint height, + ulong shaderAddress = 0) => VulkanVideoPresenter.SubmitStorageTranslatedDraw( Spirv(pixelShader), textures, globalMemoryBuffers, attributeCount, width, - height); + height, + shaderAddress); - public void SubmitComputeDispatch( + public long SubmitComputeDispatch( ulong shaderAddress, IGuestCompiledShader computeShader, IReadOnlyList textures, IReadOnlyList globalMemoryBuffers, uint groupCountX, uint groupCountY, - uint groupCountZ) => + uint groupCountZ, + uint baseGroupX, + uint baseGroupY, + uint baseGroupZ, + uint localSizeX, + uint localSizeY, + uint localSizeZ, + bool isIndirect, + bool writesGlobalMemory, + uint threadCountX = uint.MaxValue, + uint threadCountY = uint.MaxValue, + uint threadCountZ = uint.MaxValue) => VulkanVideoPresenter.SubmitComputeDispatch( shaderAddress, Spirv(computeShader), @@ -197,7 +267,18 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend globalMemoryBuffers, groupCountX, groupCountY, - groupCountZ); + groupCountZ, + baseGroupX, + baseGroupY, + baseGroupZ, + localSizeX, + localSizeY, + localSizeZ, + isIndirect, + writesGlobalMemory, + threadCountX, + threadCountY, + threadCountZ); public bool TrySubmitGuestImage( ulong address, @@ -206,6 +287,21 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend uint pitchInPixel) => VulkanVideoPresenter.TrySubmitGuestImage(address, width, height, pitchInPixel); + public bool TrySubmitOrderedGuestImageFlip( + int videoOutHandle, + int displayBufferIndex, + ulong address, + uint width, + uint height, + uint pitchInPixel) => + VulkanVideoPresenter.TrySubmitOrderedGuestImageFlip( + videoOutHandle, + displayBufferIndex, + address, + width, + height, + pitchInPixel); + public void RegisterKnownDisplayBuffer(ulong address, uint guestFormat) => VulkanVideoPresenter.RegisterKnownDisplayBuffer(address, guestFormat); @@ -217,19 +313,23 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend uint sourceWidth, uint sourceHeight, uint sourceFormat, + uint sourceNumberType, ulong destinationAddress, uint destinationWidth, uint destinationHeight, - uint destinationFormat) => + uint destinationFormat, + uint destinationNumberType) => VulkanVideoPresenter.TrySubmitGuestImageBlit( sourceAddress, sourceWidth, sourceHeight, sourceFormat, + sourceNumberType, destinationAddress, destinationWidth, destinationHeight, - destinationFormat); + destinationFormat, + destinationNumberType); public bool TryGetRenderTargetOutputKind(uint dataFormat, uint numberType, out Gen5PixelOutputKind outputKind) { diff --git a/src/SharpEmu.Libs/HostTiming.cs b/src/SharpEmu.Libs/HostTiming.cs new file mode 100644 index 0000000..181a8f1 --- /dev/null +++ b/src/SharpEmu.Libs/HostTiming.cs @@ -0,0 +1,82 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; + +namespace SharpEmu.Libs; + +/// +/// High-resolution host sleeps for guest pacing. +/// routinely overshoots by a scheduler quantum, which turns per-frame waits +/// (flip pacing, usleep-based game loops) into a hard frame-rate cap; these +/// helpers sleep coarsely for the bulk of the interval and yield-spin the +/// remainder so wakeups land within tens of microseconds of the target. +/// +internal static class HostTiming +{ + /// + /// Blocks until reaches + /// . + /// + public static void SleepUntil(long targetTimestamp) + { + while (true) + { + var remainingTicks = targetTimestamp - Stopwatch.GetTimestamp(); + if (remainingTicks <= 0) + { + return; + } + + if (remainingTicks > Stopwatch.Frequency * 60) + { + // Far-future target: coarse sleep avoids overflowing the + // microsecond conversion below and needs no precision. + Thread.Sleep(30_000); + continue; + } + + var remainingMicroseconds = remainingTicks * 1_000_000 / Stopwatch.Frequency; + if (remainingMicroseconds > 2200) + { + // Coarse sleep for the bulk; macOS overshoots ~0.5-1.5 ms. + Thread.Sleep((int)((remainingMicroseconds - 1200) / 1000)); + } + else if (remainingMicroseconds > 1200) + { + // A 1 ms nap typically wakes within the margin and costs no + // CPU, unlike yield-spinning through the whole tail. + Thread.Sleep(1); + } + else if (remainingMicroseconds > 100) + { + Thread.Sleep(0); + } + else + { + Thread.SpinWait(64); + } + } + } + + /// Blocks for the given number of microseconds. + public static void SleepMicroseconds(long microseconds) + { + if (microseconds <= 0) + { + return; + } + + if (microseconds >= 10_000_000) + { + // Long/sentinel sleeps (usleep(-1) parks): sub-millisecond + // precision is irrelevant and the tick conversion below would + // overflow, which used to turn the park into a hot spin. + Thread.Sleep((int)Math.Min(microseconds / 1000, int.MaxValue)); + return; + } + + var ticks = microseconds * Stopwatch.Frequency / 1_000_000; + SleepUntil(Stopwatch.GetTimestamp() + ticks); + } +} diff --git a/src/SharpEmu.Libs/Json/JsonExports.cs b/src/SharpEmu.Libs/Json/JsonExports.cs index c62262b..322351e 100644 --- a/src/SharpEmu.Libs/Json/JsonExports.cs +++ b/src/SharpEmu.Libs/Json/JsonExports.cs @@ -2,12 +2,34 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System; +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.Text.Json; using SharpEmu.HLE; namespace SharpEmu.Libs.Json; public static class JsonExports { + private const int ValueObjectSize = 0x20; + private const int StringObjectSize = 0x08; + private const ulong MaximumJsonBufferSize = 16 * 1024 * 1024; + private const int SceJsonParserErrorInvalidToken = unchecked((int)0x80920101); + private const int SceJsonParserErrorEmptyBuffer = unchecked((int)0x80920105); + + private sealed record JsonValueState(JsonElement Element); + + private sealed record JsonStringState( + string Value, + ulong GuestBufferAddress = 0, + int GuestBufferCapacity = 0); + + private static readonly ConcurrentDictionary _values = new(); + private static readonly ConcurrentDictionary _strings = new(); + private static readonly JsonElement _nullElement = CreateNullElement(); + [SysAbiExport( Nid = "-hJRce8wn1U", ExportName = "_ZN3sce4Json12MemAllocatorC2Ev", @@ -78,9 +100,6 @@ public static class JsonExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - // sce::Json::Initializer::setGlobalNullAccessCallback(const Value& (*)(ValueType, const Value*, void*), void*) - // Registers the guest hook invoked when a Value is accessed as the wrong type. Quake calls it - // during kexPSNWebAPI::Initialize and treats a non-zero return as a fatal init failure. [SysAbiExport( Nid = "+drDFyAS6u4", ExportName = "_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_", @@ -91,26 +110,38 @@ public static class JsonExports var thisAddress = ctx[CpuRegister.Rdi]; if (thisAddress == 0) { - ctx[CpuRegister.Rax] = unchecked((ulong)(int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } JsonObjectHeap.GlobalNullAccessCallback = ctx[CpuRegister.Rsi]; JsonObjectHeap.GlobalNullAccessCallbackContext = ctx[CpuRegister.Rdx]; TraceJson("Initializer.setGlobalNullAccessCallback", thisAddress, ctx[CpuRegister.Rsi]); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + return SetReturn(ctx, 0); } [SysAbiExport( Nid = "WSOuge5IsCg", ExportName = "_ZN3sce4Json14InitParameter2C1Ev", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceJson2")] + Target = Generation.Gen5, + LibraryName = "libSceJson")] public static int InitParameter2Constructor(CpuContext ctx) { var thisAddress = ctx[CpuRegister.Rdi]; - TraceJson("InitParameter2.ctor", thisAddress, ctx[CpuRegister.Rsi]); + if (thisAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + // The PS5 ABI object occupies 0x28 bytes in the caller's frame. Its + // setters below replace the allocator and file-buffer fields. + Span parameter = stackalloc byte[0x28]; + parameter.Clear(); + if (!ctx.Memory.TryWrite(thisAddress, parameter)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + TraceJson("InitParameter2.ctor", thisAddress, 0); ctx[CpuRegister.Rax] = thisAddress; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -118,12 +149,24 @@ public static class JsonExports [SysAbiExport( Nid = "I2QC8PYhJWY", ExportName = "_ZN3sce4Json14InitParameter212setAllocatorEPNS0_12MemAllocatorEPv", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceJson2")] + Target = Generation.Gen5, + LibraryName = "libSceJson")] public static int InitParameter2SetAllocator(CpuContext ctx) { var thisAddress = ctx[CpuRegister.Rdi]; - TraceJson("InitParameter2.setAllocator", thisAddress, ctx[CpuRegister.Rsi]); + if (thisAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + Span fields = stackalloc byte[sizeof(ulong) * 2]; + BinaryPrimitives.WriteUInt64LittleEndian(fields, ctx[CpuRegister.Rsi]); + BinaryPrimitives.WriteUInt64LittleEndian(fields[sizeof(ulong)..], ctx[CpuRegister.Rdx]); + if (!ctx.Memory.TryWrite(thisAddress, fields)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + ctx[CpuRegister.Rax] = thisAddress; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -131,12 +174,16 @@ public static class JsonExports [SysAbiExport( Nid = "Eu95jmqn5Rw", ExportName = "_ZN3sce4Json14InitParameter217setFileBufferSizeEm", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceJson2")] + Target = Generation.Gen5, + LibraryName = "libSceJson")] public static int InitParameter2SetFileBufferSize(CpuContext ctx) { var thisAddress = ctx[CpuRegister.Rdi]; - TraceJson("InitParameter2.setFileBufferSize", thisAddress, ctx[CpuRegister.Rsi]); + if (thisAddress == 0 || !ctx.TryWriteUInt64(thisAddress + 0x10, ctx[CpuRegister.Rsi])) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + ctx[CpuRegister.Rax] = thisAddress; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -144,14 +191,474 @@ public static class JsonExports [SysAbiExport( Nid = "IXW-z8pggfg", ExportName = "_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceJson2")] - public static int Initializer2Constructor(CpuContext ctx) + Target = Generation.Gen5, + LibraryName = "libSceJson")] + public static int InitializerInitialize2(CpuContext ctx) { var thisAddress = ctx[CpuRegister.Rdi]; - TraceJson("Initializer2.ctor", thisAddress, ctx[CpuRegister.Rsi]); + var initParameterAddress = ctx[CpuRegister.Rsi]; + if (thisAddress == 0 || initParameterAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + TraceJson("Initializer.initialize2", thisAddress, initParameterAddress); + return SetReturn(ctx, 0); + } + public static int ValueConstructor(CpuContext ctx) + { + _ = ConstructValue(ctx); + return JsonValueExports.ValueDefaultConstructor(ctx); + } + + [SysAbiExport( + Nid = "-wa17B7TGnw", + ExportName = "_ZN3sce4Json5ValueC2Ev", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueBaseConstructor(CpuContext ctx) => ConstructValue(ctx); + public static int ValueDestructor(CpuContext ctx) + { + _ = DestroyValue(ctx); + return JsonValueExports.ValueDestructor(ctx); + } + + [SysAbiExport( + Nid = "0eUrW9JAxM0", + ExportName = "_ZN3sce4Json5ValueD2Ev", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueBaseDestructor(CpuContext ctx) => DestroyValue(ctx); + + [SysAbiExport( + Nid = "S5JxQnoGF3E", + ExportName = "_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ParserParseBuffer(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var bufferAddress = ctx[CpuRegister.Rsi]; + var bufferSize = ctx[CpuRegister.Rdx]; + if (valueAddress == 0 || bufferAddress == 0 || bufferSize == 0) + { + return SetReturn(ctx, SceJsonParserErrorEmptyBuffer); + } + + if (bufferSize > MaximumJsonBufferSize || bufferSize > int.MaxValue) + { + return SetReturn(ctx, SceJsonParserErrorInvalidToken); + } + + var buffer = new byte[(int)bufferSize]; + if (!ctx.Memory.TryRead(bufferAddress, buffer)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + try + { + using var document = JsonDocument.Parse(buffer); + var element = document.RootElement.Clone(); + StoreValue(ctx, valueAddress, element); + TraceJsonText("Parser.parse", valueAddress, Encoding.UTF8.GetString(buffer)); + return SetReturn(ctx, 0); + } + catch (JsonException) + { + return SetReturn(ctx, SceJsonParserErrorInvalidToken); + } + } + + [SysAbiExport( + Nid = "SHtAad20YYM", + ExportName = "_ZNK3sce4Json5Value7getTypeEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetType(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var element = GetValue(valueAddress); + ctx[CpuRegister.Rax] = (ulong)GetValueType(element); + return 0; + } + + [SysAbiExport( + Nid = "RBw+4NukeGQ", + ExportName = "_ZNK3sce4Json5Value5countEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueCount(CpuContext ctx) + { + var element = GetValue(ctx[CpuRegister.Rdi]); + ctx[CpuRegister.Rax] = element.ValueKind switch + { + System.Text.Json.JsonValueKind.Array => (ulong)element.GetArrayLength(), + System.Text.Json.JsonValueKind.Object => (ulong)element.EnumerateObject().Count(), + _ => 0, + }; + return 0; + } + + [SysAbiExport( + Nid = "zTwZdI8AZ5Y", + ExportName = "_ZNK3sce4Json5Value10getBooleanEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetBoolean(CpuContext ctx) => ReturnValueStorage(ctx); + + [SysAbiExport( + Nid = "DIxvoy7Ngvk", + ExportName = "_ZNK3sce4Json5Value10getIntegerEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetInteger(CpuContext ctx) => ReturnValueStorage(ctx); + + [SysAbiExport( + Nid = "sn4HNCtNRzY", + ExportName = "_ZNK3sce4Json5Value11getUIntegerEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetUnsignedInteger(CpuContext ctx) => ReturnValueStorage(ctx); + + [SysAbiExport( + Nid = "3qrge7L-AU4", + ExportName = "_ZNK3sce4Json5Value7getRealEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetReal(CpuContext ctx) => ReturnValueStorage(ctx); + + [SysAbiExport( + Nid = "HwDt5lD9Bfo", + ExportName = "_ZNK3sce4Json5ValueixEPKc", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueIndexCString(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var keyAddress = ctx[CpuRegister.Rsi]; + if (!TryReadUtf8CString(ctx, keyAddress, 4096, out var key) || + !TryAllocateGuestObject(ctx, ValueObjectSize, out var childAddress)) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + var parent = GetValue(valueAddress); + var child = parent.ValueKind == System.Text.Json.JsonValueKind.Object && + parent.TryGetProperty(key, out var property) + ? property.Clone() + : _nullElement; + StoreValue(ctx, childAddress, child); + ctx[CpuRegister.Rax] = childAddress; + TraceJsonText("Value.index", valueAddress, key); + return 0; + } + + [SysAbiExport( + Nid = "XlWbvieLj2M", + ExportName = "_ZNK3sce4Json5ValueixEm", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueIndexPosition(CpuContext ctx) => ReturnIndexedValue(ctx); + + [SysAbiExport( + Nid = "0YqYAoO-+Uo", + ExportName = "_ZNK3sce4Json5Value8getValueEm", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetPosition(CpuContext ctx) => ReturnIndexedValue(ctx); + + [SysAbiExport( + Nid = "4zrm6VrgIAw", + ExportName = "_ZN3sce4Json5ValueaSERKS1_", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueAssignment(CpuContext ctx) + { + var destinationAddress = ctx[CpuRegister.Rdi]; + var sourceAddress = ctx[CpuRegister.Rsi]; + if (destinationAddress != 0) + { + StoreValue(ctx, destinationAddress, GetValue(sourceAddress)); + } + + ctx[CpuRegister.Rax] = destinationAddress; + return 0; + } + public static int StringConstructor(CpuContext ctx) + { + _ = ConstructString(ctx); + return JsonValueExports.StringDefaultConstructor(ctx); + } + + [SysAbiExport( + Nid = "eG9E9M6XvTM", + ExportName = "_ZN3sce4Json6StringC2Ev", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int StringBaseConstructor(CpuContext ctx) => ConstructString(ctx); + public static int StringDestructor(CpuContext ctx) + { + _ = DestroyString(ctx); + return JsonValueExports.StringDestructor(ctx); + } + + [SysAbiExport( + Nid = "Ui7YFnSTCBw", + ExportName = "_ZN3sce4Json6StringD2Ev", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int StringBaseDestructor(CpuContext ctx) => DestroyString(ctx); + + [SysAbiExport( + Nid = "Ncel8t2Rrpc", + ExportName = "_ZNK3sce4Json5Value8toStringERNS0_6StringE", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueToString(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var stringAddress = ctx[CpuRegister.Rsi]; + if (stringAddress != 0) + { + var element = GetValue(valueAddress); + var value = element.ValueKind == System.Text.Json.JsonValueKind.String + ? element.GetString() ?? string.Empty + : element.GetRawText(); + _strings[stringAddress] = new JsonStringState(value); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "L1KAkYWml-M", + ExportName = "_ZNK3sce4Json6String5c_strEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int StringCStr(CpuContext ctx) + { + var stringAddress = ctx[CpuRegister.Rdi]; + if (!_strings.TryGetValue(stringAddress, out var state)) + { + state = new JsonStringState(string.Empty); + } + + var bytes = Encoding.UTF8.GetBytes(state.Value + '\0'); + var guestBufferAddress = state.GuestBufferAddress; + if (guestBufferAddress == 0 || state.GuestBufferCapacity < bytes.Length) + { + if (!TryAllocateGuestObject(ctx, bytes.Length, out guestBufferAddress)) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + } + + if (!ctx.Memory.TryWrite(guestBufferAddress, bytes)) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + _strings[stringAddress] = state with + { + GuestBufferAddress = guestBufferAddress, + GuestBufferCapacity = bytes.Length, + }; + ctx.TryWriteUInt64(stringAddress, guestBufferAddress); + ctx[CpuRegister.Rax] = guestBufferAddress; + TraceJsonText("String.c_str", stringAddress, state.Value); + return 0; + } + + private static int ConstructValue(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + if (thisAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + StoreValue(ctx, thisAddress, _nullElement); ctx[CpuRegister.Rax] = thisAddress; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + return 0; + } + + private static int ReturnIndexedValue(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var position = ctx[CpuRegister.Rsi]; + if (!TryAllocateGuestObject(ctx, ValueObjectSize, out var childAddress)) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + var parent = GetValue(valueAddress); + var child = parent.ValueKind == System.Text.Json.JsonValueKind.Array && + position < (ulong)parent.GetArrayLength() + ? parent[(int)position].Clone() + : _nullElement; + StoreValue(ctx, childAddress, child); + ctx[CpuRegister.Rax] = childAddress; + return 0; + } + + private static int ReturnValueStorage(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + ctx[CpuRegister.Rax] = thisAddress == 0 ? 0 : thisAddress + 0x10; + return 0; + } + + private static int DestroyValue(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + _values.TryRemove(thisAddress, out _); + if (thisAddress != 0) + { + Span empty = stackalloc byte[ValueObjectSize]; + empty.Clear(); + ctx.Memory.TryWrite(thisAddress, empty); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + private static int ConstructString(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + if (thisAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + _strings[thisAddress] = new JsonStringState(string.Empty); + ctx.TryWriteUInt64(thisAddress, 0); + ctx[CpuRegister.Rax] = thisAddress; + return 0; + } + + private static int DestroyString(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + _strings.TryRemove(thisAddress, out _); + if (thisAddress != 0) + { + ctx.TryWriteUInt64(thisAddress, 0); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + private static JsonElement CreateNullElement() + { + using var document = JsonDocument.Parse("null"); + return document.RootElement.Clone(); + } + + private static JsonElement GetValue(ulong address) => + address != 0 && _values.TryGetValue(address, out var state) + ? state.Element + : _nullElement; + + private static void StoreValue(CpuContext ctx, ulong address, JsonElement element) + { + if (address == 0) + { + return; + } + + var clone = element.Clone(); + _values[address] = new JsonValueState(clone); + + Span mirror = stackalloc byte[ValueObjectSize]; + mirror.Clear(); + var type = GetValueType(clone); + BinaryPrimitives.WriteInt32LittleEndian(mirror[0x1C..], type); + switch (clone.ValueKind) + { + case System.Text.Json.JsonValueKind.True: + mirror[0x10] = 1; + break; + case System.Text.Json.JsonValueKind.Number when clone.TryGetInt64(out var integer): + BinaryPrimitives.WriteInt64LittleEndian(mirror[0x10..], integer); + break; + case System.Text.Json.JsonValueKind.Number when clone.TryGetUInt64(out var unsignedInteger): + BinaryPrimitives.WriteUInt64LittleEndian(mirror[0x10..], unsignedInteger); + break; + case System.Text.Json.JsonValueKind.Number: + BinaryPrimitives.WriteInt64LittleEndian( + mirror[0x10..], + BitConverter.DoubleToInt64Bits(clone.GetDouble())); + break; + } + + ctx.Memory.TryWrite(address, mirror); + } + + private static int GetValueType(JsonElement element) => element.ValueKind switch + { + System.Text.Json.JsonValueKind.True or System.Text.Json.JsonValueKind.False => 1, + System.Text.Json.JsonValueKind.Number when element.TryGetInt64(out _) => 2, + System.Text.Json.JsonValueKind.Number when element.TryGetUInt64(out _) => 3, + System.Text.Json.JsonValueKind.Number => 4, + System.Text.Json.JsonValueKind.String => 5, + System.Text.Json.JsonValueKind.Array => 6, + System.Text.Json.JsonValueKind.Object => 7, + _ => 0, + }; + + private static bool TryAllocateGuestObject(CpuContext ctx, int size, out ulong address) + { + address = 0; + return size > 0 && + ctx.Memory is IGuestMemoryAllocator allocator && + allocator.TryAllocateGuestMemory((ulong)size, 0x10, out address); + } + + private static bool TryReadUtf8CString( + CpuContext ctx, + ulong address, + int maximumLength, + out string value) + { + value = string.Empty; + if (address == 0 || maximumLength <= 0) + { + return false; + } + + var bytes = new byte[maximumLength]; + Span current = stackalloc byte[1]; + for (var index = 0; index < bytes.Length; index++) + { + if (!ctx.Memory.TryRead(address + (ulong)index, current)) + { + return false; + } + + if (current[0] == 0) + { + value = Encoding.UTF8.GetString(bytes, 0, index); + return true; + } + + bytes[index] = current[0]; + } + + return false; + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; } private static void TraceJson(string operation, ulong thisAddress, ulong argument) @@ -164,4 +671,16 @@ public static class JsonExports Console.Error.WriteLine( $"[LOADER][TRACE] json.{operation} this=0x{thisAddress:X16} arg=0x{argument:X16}"); } + + private static void TraceJsonText(string operation, ulong thisAddress, string value) + { + if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_JSON"), "1", StringComparison.Ordinal)) + { + return; + } + + var preview = value.Length <= 128 ? value : value[..128]; + Console.Error.WriteLine( + $"[LOADER][TRACE] json.{operation} this=0x{thisAddress:X16} value={preview}"); + } } diff --git a/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs index c4febeb..552df52 100644 --- a/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs @@ -1,8 +1,8 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Buffers.Binary; using System.Collections.Concurrent; -using System.Diagnostics; using System.Text; using SharpEmu.HLE; using SharpEmu.Libs.Fiber; @@ -34,45 +34,6 @@ public static class KernelEventFlagCompatExports public object Gate { get; } = new(); } - private sealed class EventFlagWaiter : IGuestThreadBlockWaiter - { - public required CpuContext Ctx { get; init; } - public required EventFlagState State { get; init; } - public required ulong Pattern { get; init; } - public required uint WaitMode { get; init; } - public required ulong ResultAddress { get; init; } - public bool Timed { get; init; } - - // Timed-wait completion state; unused when Timed is false. - public ulong TimeoutAddress { get; init; } - public long DeadlineTimestamp { get; init; } - - public OrbisGen2Result? Result { get; set; } - - // Untimed waits stash the prepared result here at wake and return it at resume. - private OrbisGen2Result _blockedResult = OrbisGen2Result.ORBIS_GEN2_OK; - - public int Resume() => Timed - ? CompleteBlockedTimedWait(Ctx, State, this, Pattern, WaitMode, ResultAddress, TimeoutAddress, DeadlineTimestamp) - : (int)_blockedResult; - - public bool TryWake() - { - if (Timed) - { - return TryCompleteBlockedTimedWait(Ctx, State, this, Pattern, WaitMode, ResultAddress); - } - - if (!TryPrepareBlockedWait(Ctx, State, Pattern, WaitMode, ResultAddress, out var preparedResult)) - { - return false; - } - - _blockedResult = preparedResult; - return true; - } - } - [SysAbiExport( Nid = "BpFoboUJoZU", ExportName = "sceKernelCreateEventFlag", @@ -91,17 +52,17 @@ public static class KernelEventFlagCompatExports optionAddress != 0 || !IsValidAttributes(attributes)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadNullTerminatedUtf8(nameAddress, MaxEventFlagNameLength + 1, out var name)) + if (!TryReadNullTerminatedUtf8(ctx, nameAddress, MaxEventFlagNameLength + 1, out var name)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (Encoding.UTF8.GetByteCount(name) > MaxEventFlagNameLength) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var handle = unchecked((ulong)Interlocked.Increment(ref _nextEventFlagHandle)); @@ -115,11 +76,11 @@ public static class KernelEventFlagCompatExports if (!ctx.TryWriteUInt64(outAddress, handle)) { _eventFlags.TryRemove(handle, out _); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceEventFlag($"create handle=0x{handle:X16} name='{name}' attr=0x{attributes:X2} bits=0x{initialPattern:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -132,7 +93,7 @@ public static class KernelEventFlagCompatExports var handle = ctx[CpuRegister.Rdi]; if (!_eventFlags.TryRemove(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } lock (state.Gate) @@ -141,7 +102,7 @@ public static class KernelEventFlagCompatExports } TraceEventFlag($"delete handle=0x{handle:X16} name='{state.Name}'"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -156,7 +117,7 @@ public static class KernelEventFlagCompatExports var returnRip = GetCurrentReturnRip(); if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } lock (state.Gate) @@ -167,7 +128,7 @@ public static class KernelEventFlagCompatExports } _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(GetEventFlagWakeKey(handle)); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -181,7 +142,7 @@ public static class KernelEventFlagCompatExports var pattern = ctx[CpuRegister.Rsi]; if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } lock (state.Gate) @@ -190,7 +151,7 @@ public static class KernelEventFlagCompatExports TraceEventFlag($"clear handle=0x{handle:X16} mask=0x{pattern:X16} bits=0x{state.Bits:X16}"); } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -207,29 +168,29 @@ public static class KernelEventFlagCompatExports if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (pattern == 0 || !IsValidWaitMode(waitMode)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } lock (state.Gate) { if (!TryWriteResultPattern(ctx, resultAddress, state.Bits)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (!IsSatisfied(state.Bits, pattern, waitMode)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); } ApplyClearMode(state, pattern, waitMode); TraceEventFlag($"poll handle=0x{handle:X16} pattern=0x{pattern:X16} mode=0x{waitMode:X2} bits=0x{state.Bits:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } } @@ -249,18 +210,18 @@ public static class KernelEventFlagCompatExports if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (pattern == 0 || !IsValidWaitMode(waitMode)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } uint timeoutUsec = 0; - if (timeoutAddress != 0 && !ctx.TryReadUInt32(timeoutAddress, out timeoutUsec)) + if (timeoutAddress != 0 && !TryReadUInt32(ctx, timeoutAddress, out timeoutUsec)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } Monitor.Enter(state.Gate); @@ -268,65 +229,64 @@ public static class KernelEventFlagCompatExports { if (TryCompleteSatisfiedWait(ctx, state, pattern, waitMode, resultAddress, out var immediateWaitResult)) { - return ctx.SetReturn(immediateWaitResult); + return SetReturn(ctx, immediateWaitResult); } - if (timeoutAddress != 0) - { - if (timeoutUsec == 0) - { - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - _ = TryWriteResultPattern(ctx, resultAddress, state.Bits); - TraceEventFlag($"wait-timeout handle=0x{handle:X16} pattern=0x{pattern:X16} timeout=0 ret=0x{returnRip:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); - } - - var deadline = GuestThreadExecution.ComputeDeadlineTimestamp( - TimeSpan.FromTicks((long)timeoutUsec * 10L)); - var timedWaiter = new EventFlagWaiter - { - Ctx = ctx, - State = state, - Pattern = pattern, - WaitMode = waitMode, - ResultAddress = resultAddress, - Timed = true, - TimeoutAddress = timeoutAddress, - DeadlineTimestamp = deadline, - }; - if (GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - "sceKernelWaitEventFlag", - GetEventFlagWakeKey(handle), - timedWaiter, - blockDeadlineTimestamp: deadline)) - { - state.WaitingThreads++; - TraceEventFlag($"wait-block-timed handle=0x{handle:X16} pattern=0x{pattern:X16} timeout={timeoutUsec} waiters={state.WaitingThreads} ret=0x{returnRip:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - _ = TryWriteResultPattern(ctx, resultAddress, state.Bits); - TraceEventFlag($"wait-timeout-host handle=0x{handle:X16} pattern=0x{pattern:X16} timeout={timeoutUsec} ret=0x{returnRip:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); - } + // Timed waits block on a deadline instead of returning TIMED_OUT + // immediately; a zero-microsecond timeout still degrades to an + // instant poll because the deadline is already in the past. + var deadline = timeoutAddress != 0 + ? GuestThreadExecution.ComputeDeadlineTimestamp(TimeSpan.FromMicroseconds(timeoutUsec)) + : 0; + var hostDeadlineMs = timeoutAddress != 0 + ? Environment.TickCount64 + (timeoutUsec == 0 + ? 0L + : Math.Max(1L, (timeoutUsec + 999L) / 1000L)) + : long.MaxValue; var currentGuestThread = GuestThreadExecution.CurrentGuestThreadHandle; var currentFiber = FiberExports.GetCurrentFiberAddressForDiagnostics(ctx); var managedThread = Environment.CurrentManagedThreadId; + var blockedWaitResult = OrbisGen2Result.ORBIS_GEN2_OK; + var satisfied = false; var requestedBlock = GuestThreadExecution.RequestCurrentThreadBlock( ctx, "sceKernelWaitEventFlag", GetEventFlagWakeKey(handle), - new EventFlagWaiter + () => { - Ctx = ctx, - State = state, - Pattern = pattern, - WaitMode = waitMode, - ResultAddress = resultAddress, - }); + if (satisfied) + { + return (int)blockedWaitResult; + } + + // Deadline expiry: report timeout with the current bits. + if (timeoutAddress != 0) + { + _ = TryWriteUInt32(ctx, timeoutAddress, 0); + } + + _ = TryWriteResultPattern(ctx, resultAddress, state.Bits); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; + }, + () => + { + if (!TryPrepareBlockedWait( + ctx, + state, + pattern, + waitMode, + resultAddress, + out var preparedResult)) + { + return false; + } + + blockedWaitResult = preparedResult; + satisfied = true; + return true; + }, + deadline); TraceEventFlag($"wait-unsatisfied handle=0x{handle:X16} pattern=0x{pattern:X16} bits=0x{state.Bits:X16} guest_thread=0x{currentGuestThread:X16} fiber=0x{currentFiber:X16} managed={managedThread} block={requestedBlock} ret=0x{returnRip:X16} frames={FormatFrameChain(ctx)}"); TraceEventFlag($"wait-object handle=0x{handle:X16} name='{state.Name}' {FormatGuestWaitObject(ctx)}"); if (!requestedBlock) @@ -334,7 +294,7 @@ public static class KernelEventFlagCompatExports var scheduler = GuestThreadExecution.Scheduler; if (scheduler is null) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); } state.WaitingThreads++; @@ -359,10 +319,21 @@ public static class KernelEventFlagCompatExports state.WaitingThreads = Math.Max(0, state.WaitingThreads - 1); releaseWaiter = false; TraceEventFlag($"wait-wake handle=0x{handle:X16} pattern=0x{pattern:X16} bits=0x{state.Bits:X16} waiters={state.WaitingThreads} ret=0x{returnRip:X16}"); - return ctx.SetReturn(pumpedWaitResult); + return SetReturn(ctx, pumpedWaitResult); } - Monitor.Wait(state.Gate, HostWaitPumpMilliseconds); + var remaining = hostDeadlineMs - Environment.TickCount64; + if (timeoutAddress != 0 && remaining <= 0) + { + state.WaitingThreads = Math.Max(0, state.WaitingThreads - 1); + releaseWaiter = false; + _ = TryWriteUInt32(ctx, timeoutAddress, 0); + _ = TryWriteResultPattern(ctx, resultAddress, state.Bits); + TraceEventFlag($"wait-timeout handle=0x{handle:X16} pattern=0x{pattern:X16} bits=0x{state.Bits:X16} ret=0x{returnRip:X16}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); + } + + Monitor.Wait(state.Gate, (int)Math.Min(remaining, HostWaitPumpMilliseconds)); } } finally @@ -376,7 +347,7 @@ public static class KernelEventFlagCompatExports state.WaitingThreads++; TraceEventFlag($"wait-block handle=0x{handle:X16} pattern=0x{pattern:X16} waiters={state.WaitingThreads} guest_thread=0x{currentGuestThread:X16} fiber=0x{currentFiber:X16} managed={managedThread} ret=0x{returnRip:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } finally { @@ -396,24 +367,26 @@ public static class KernelEventFlagCompatExports var waiterCountAddress = ctx[CpuRegister.Rdx]; if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } lock (state.Gate) { if (waiterCountAddress != 0 && - !ctx.TryWriteUInt32(waiterCountAddress, unchecked((uint)state.WaitingThreads))) + !TryWriteUInt32(ctx, waiterCountAddress, unchecked((uint)state.WaitingThreads))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } state.Bits = setPattern; state.WaitingThreads = 0; Monitor.PulseAll(state.Gate); - TraceEventFlag($"cancel handle=0x{handle:X16} bits=0x{setPattern:X16}"); + TraceEventFlag( + $"cancel handle=0x{handle:X16} bits=0x{setPattern:X16} " + + $"guest_thread=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} ret=0x{GetCurrentReturnRip():X16}"); } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } private static bool IsValidAttributes(uint attributes) @@ -509,98 +482,90 @@ public static class KernelEventFlagCompatExports } } - private static bool TryCompleteBlockedTimedWait( - CpuContext ctx, - EventFlagState state, - EventFlagWaiter waiter, - ulong pattern, - uint waitMode, - ulong resultAddress) - { - lock (state.Gate) - { - if (waiter.Result is not null) - { - return true; - } - - if (!IsSatisfied(state.Bits, pattern, waitMode)) - { - return false; - } - - waiter.Result = TryWriteResultPattern(ctx, resultAddress, state.Bits) - ? OrbisGen2Result.ORBIS_GEN2_OK - : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - if (waiter.Result == OrbisGen2Result.ORBIS_GEN2_OK) - { - ApplyClearMode(state, pattern, waitMode); - } - - state.WaitingThreads = Math.Max(0, state.WaitingThreads - 1); - return true; - } - } - - private static int CompleteBlockedTimedWait( - CpuContext ctx, - EventFlagState state, - EventFlagWaiter waiter, - ulong pattern, - uint waitMode, - ulong resultAddress, - ulong timeoutAddress, - long deadlineTimestamp) - { - lock (state.Gate) - { - if (waiter.Result is null) - { - if (IsSatisfied(state.Bits, pattern, waitMode)) - { - waiter.Result = TryWriteResultPattern(ctx, resultAddress, state.Bits) - ? OrbisGen2Result.ORBIS_GEN2_OK - : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - if (waiter.Result == OrbisGen2Result.ORBIS_GEN2_OK) - { - ApplyClearMode(state, pattern, waitMode); - } - } - else - { - waiter.Result = TryWriteResultPattern(ctx, resultAddress, state.Bits) - ? OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT - : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - state.WaitingThreads = Math.Max(0, state.WaitingThreads - 1); - } - } - - if (waiter.Result == OrbisGen2Result.ORBIS_GEN2_OK) - { - var remainingTicks = deadlineTimestamp - Stopwatch.GetTimestamp(); - var remainingMicros = remainingTicks <= 0 - ? 0u - : (uint)Math.Min( - uint.MaxValue, - remainingTicks / (double)Stopwatch.Frequency * 1_000_000d); - _ = ctx.TryWriteUInt32(timeoutAddress, remainingMicros); - } - else - { - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - } - - return (int)waiter.Result.Value; - } - private static string GetEventFlagWakeKey(ulong handle) => $"event_flag:0x{handle:X16}"; private static bool TryWriteResultPattern(CpuContext ctx, ulong address, ulong bits) => address == 0 || ctx.TryWriteUInt64(address, bits); + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(buffer); + return true; + } + + private static bool TryReadByte(CpuContext ctx, ulong address, out byte value) + { + Span buffer = stackalloc byte[1]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = buffer[0]; + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadNullTerminatedUtf8(CpuContext ctx, ulong address, int capacity, out string value) + { + var bytes = new byte[capacity]; + Span current = stackalloc byte[1]; + for (var index = 0; index < bytes.Length; index++) + { + if (!ctx.Memory.TryRead(address + (ulong)index, current)) + { + value = string.Empty; + return false; + } + + if (current[0] == 0) + { + value = Encoding.UTF8.GetString(bytes, 0, index); + return true; + } + + bytes[index] = current[0]; + } + + value = Encoding.UTF8.GetString(bytes); + return true; + } + + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + var value = (int)result; + ctx[CpuRegister.Rax] = unchecked((ulong)value); + return value; + } + private static void TraceEventFlag(string message) { if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_EVENT_FLAG"), "1", StringComparison.Ordinal)) @@ -684,7 +649,7 @@ public static class KernelEventFlagCompatExports private static void AppendByte(StringBuilder builder, CpuContext ctx, ulong address, string name) { - if (ctx.TryReadByte(address, out var value)) + if (TryReadByte(ctx, address, out var value)) { builder.Append($" {name}=0x{value:X2}"); } @@ -692,7 +657,7 @@ public static class KernelEventFlagCompatExports private static void AppendUInt32(StringBuilder builder, CpuContext ctx, ulong address, string name) { - if (ctx.TryReadUInt32(address, out var value)) + if (TryReadUInt32(ctx, address, out var value)) { builder.Append($" {name}=0x{value:X8}"); } @@ -700,7 +665,7 @@ public static class KernelEventFlagCompatExports private static void AppendUInt64(StringBuilder builder, CpuContext ctx, ulong address, string name) { - if (ctx.TryReadUInt64(address, out var value)) + if (TryReadUInt64(ctx, address, out var value)) { builder.Append($" {name}=0x{value:X16}"); } diff --git a/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs index dafdb9c..b7223d7 100644 --- a/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs @@ -395,13 +395,13 @@ public static class KernelEventQueueCompatExports } uint timeoutUsec = 0; - if (timeoutAddress != 0 && !ctx.TryReadUInt32(timeoutAddress, out timeoutUsec)) + if (timeoutAddress != 0 && !TryReadUInt32(ctx, timeoutAddress, out timeoutUsec)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } var deliveredCount = DequeueEvents(ctx, handle, eventsAddress, eventCapacity); - if (outCountAddress != 0 && !ctx.TryWriteUInt32(outCountAddress, (uint)deliveredCount)) + if (outCountAddress != 0 && !TryWriteUInt32(ctx, outCountAddress, (uint)deliveredCount)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -450,7 +450,7 @@ public static class KernelEventQueueCompatExports } deliveredCount = DequeueEvents(ctx, handle, eventsAddress, eventCapacity); - if (outCountAddress != 0 && !ctx.TryWriteUInt32(outCountAddress, (uint)deliveredCount)) + if (outCountAddress != 0 && !TryWriteUInt32(ctx, outCountAddress, (uint)deliveredCount)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -649,6 +649,60 @@ public static class KernelEventQueueCompatExports return triggeredCount; } + /// + /// Queues one event for every registration using . + /// Unlike , this preserves distinct + /// event identifiers registered on the same queue. AGC driver completion + /// queues use this form because the driver, rather than a packet-provided + /// identifier, announces that the whole submission reached end-of-pipe. + /// + public static int TriggerRegisteredEventsDistinct(short filter) + { + HashSet? wakeHandles = null; + var triggeredCount = 0; + lock (_eventQueueGate) + { + foreach (var (handle, registrations) in _registeredEvents) + { + foreach (var registration in registrations.Values) + { + if (registration.Filter != filter) + { + continue; + } + + if (!_pendingEvents.TryGetValue(handle, out var queue)) + { + queue = new KernelEventDeque(); + _pendingEvents[handle] = queue; + } + + QueueOrUpdateEvent( + queue, + new KernelQueuedEvent( + registration.Ident, + registration.Filter, + 0, + 1, + registration.Ident, + registration.UserData)); + (wakeHandles ??= []).Add(handle); + triggeredCount++; + } + } + } + + if (wakeHandles is not null) + { + foreach (var handle in wakeHandles) + { + WakeEventQueue(handle); + } + } + + return triggeredCount; + } + private static bool TriggerRegisteredEvent( ulong handle, ulong ident, @@ -752,7 +806,7 @@ public static class KernelEventQueueCompatExports ulong outCountAddress) { var deliveredCount = DequeueEvents(ctx, handle, eventsAddress, eventCapacity); - if (outCountAddress != 0 && !ctx.TryWriteUInt32(outCountAddress, (uint)deliveredCount)) + if (outCountAddress != 0 && !TryWriteUInt32(ctx, outCountAddress, (uint)deliveredCount)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -865,8 +919,29 @@ public static class KernelEventQueueCompatExports return; } - _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out ulong returnRip); + var returnRip = 0UL; + _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out returnRip); Console.Error.WriteLine( - $"[LOADER][TRACE] equeue.{operation}: thread=0x{KernelPthreadState.GetCurrentThreadHandle():X16} handle=0x{handle:X16} rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} ret=0x{returnRip:X16}"); + $"[LOADER][TRACE] equeue.{operation}: handle=0x{handle:X16} rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} ret=0x{returnRip:X16}"); + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; } } diff --git a/src/SharpEmu.Libs/Kernel/KernelExceptionCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelExceptionCompatExports.cs index 417cdd0..4008f7b 100644 --- a/src/SharpEmu.Libs/Kernel/KernelExceptionCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelExceptionCompatExports.cs @@ -62,4 +62,75 @@ public static class KernelExceptionCompatExports ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + + [SysAbiExport( + Nid = "il03nluKfMk", + ExportName = "sceKernelRaiseException", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int RaiseException(CpuContext ctx) + { + var targetThread = ctx[CpuRegister.Rdi]; + var exceptionType = unchecked((int)ctx[CpuRegister.Rsi]); + if (targetThread == 0 || exceptionType is < 0 or >= 128) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + ulong handler; + lock (_gate) + { + if (!_installedHandlers.TryGetValue(exceptionType, out handler)) + { + // The kernel accepts a raise for a type with no process + // handler; there is simply no user callback to deliver. + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + } + + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_IGNORE_GUEST_EXCEPTIONS"), + "1", + StringComparison.Ordinal)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + + var scheduler = GuestThreadExecution.Scheduler; + string? error = null; + if (scheduler is null || + !scheduler.TryRaiseGuestException( + ctx, + targetThread, + handler, + exceptionType, + out error)) + { + Console.Error.WriteLine( + $"[LOADER][WARN] sceKernelRaiseException delivery failed: " + + $"target=0x{targetThread:X16} type=0x{exceptionType:X2} " + + $"error={error ?? "scheduler unavailable"}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + } + + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_EXCEPTIONS"), + "1", + StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.raise " + + $"target=0x{targetThread:X16} type=0x{exceptionType:X2} " + + $"handler=0x{handler:X16}"); + } + + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + var value = (int)result; + ctx[CpuRegister.Rax] = unchecked((ulong)value); + return value; + } } diff --git a/src/SharpEmu.Libs/Kernel/KernelExports.cs b/src/SharpEmu.Libs/Kernel/KernelExports.cs index 0235c92..09fb5b2 100644 --- a/src/SharpEmu.Libs/Kernel/KernelExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelExports.cs @@ -13,8 +13,6 @@ public static class KernelExports private static readonly object _coredumpGate = new(); private static ulong _coredumpHandler; private static ulong _coredumpHandlerContext; - private const uint Gen4CompiledSdkVersion = 0x05000000; - private const uint Gen5CompiledSdkVersion = 0x09000000; private readonly record struct CxaDestructorEntry( ulong Function, @@ -28,24 +26,7 @@ public static class KernelExports LibraryName = "libKernel")] public static int KernelGetCompiledSdkVersion(CpuContext ctx) { - var versionAddress = ctx[CpuRegister.Rdi]; - if (versionAddress == 0) - { - ctx[CpuRegister.Rax] = unchecked((ulong)(int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; - } - - var sdkVersion = ctx.TargetGeneration == Generation.Gen5 - ? Gen5CompiledSdkVersion - : Gen4CompiledSdkVersion; - - if (!ctx.TryWriteUInt32(versionAddress, sdkVersion)) - { - ctx[CpuRegister.Rax] = unchecked((ulong)(int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ctx[CpuRegister.Rax] = 0; + _ = ctx; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -71,20 +52,12 @@ public static class KernelExports ExportName = "exit", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int Exit(CpuContext ctx) => RequestProcessExit(ctx, "exit"); - - [SysAbiExport( - Nid = "L1SBTkC+Cvw", - ExportName = "abort", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Abort(CpuContext ctx) + public static int Exit(CpuContext ctx) { - // Route through the same graceful guest-entry-exit path as exit(): letting the call - // fall through to the host's native abort() does not unwind the guest thread cleanly. - Console.Error.WriteLine("[LOADER][INFO] abort() called by guest - terminating"); - GuestThreadExecution.RequestCurrentEntryExit("abort", -1); - ctx[CpuRegister.Rax] = unchecked((ulong)(-1L)); + var status = unchecked((int)ctx[CpuRegister.Rdi]); + Console.Error.WriteLine($"[LOADER][INFO] exit(status={status})"); + GuestThreadExecution.RequestCurrentEntryExit("exit", status); + ctx[CpuRegister.Rax] = unchecked((ulong)status); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -199,7 +172,11 @@ public static class KernelExports ExportName = "_exit", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int UnderscoreExit(CpuContext ctx) => RequestProcessExit(ctx, "_exit"); + public static int UnderscoreExit(CpuContext ctx) + { + _ = ctx; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } [SysAbiExport( Nid = "Ac86z8q7T8A", @@ -218,12 +195,14 @@ public static class KernelExports Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] public static int PthreadCreate(CpuContext ctx) + => PthreadCreateCore(ctx, ctx[CpuRegister.R8]); + + private static int PthreadCreateCore(CpuContext ctx, ulong nameAddress) { var threadIdAddress = ctx[CpuRegister.Rdi]; var attrAddress = ctx[CpuRegister.Rsi]; var entryAddress = ctx[CpuRegister.Rdx]; var argument = ctx[CpuRegister.Rcx]; - var nameAddress = ctx[CpuRegister.R8]; var name = nameAddress == 0 ? string.Empty : ReadCString(ctx, nameAddress, 256); var threadHandle = KernelPthreadState.CreateThreadHandle(name); KernelPthreadExtendedCompatExports.GetThreadStartScheduling( @@ -278,14 +257,20 @@ public static class KernelExports ExportName = "pthread_create", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int PosixPthreadCreate(CpuContext ctx) => PthreadCreate(ctx); + public static int PosixPthreadCreate(CpuContext ctx) + { + return PthreadCreateCore(ctx, nameAddress: 0); + } [SysAbiExport( Nid = "Jmi+9w9u0E4", ExportName = "pthread_create_name_np", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int PosixPthreadCreateNameNp(CpuContext ctx) => PthreadCreate(ctx); + public static int PosixPthreadCreateNameNp(CpuContext ctx) + { + return PthreadCreateCore(ctx, ctx[CpuRegister.R8]); + } [SysAbiExport( Nid = "3kg7rT0NQIs", @@ -295,6 +280,9 @@ public static class KernelExports public static int PthreadExit(CpuContext ctx) { var value = ctx[CpuRegister.Rdi]; + // Run cleanup on the still-executable thread before unwinding it. + KernelPthreadExtendedCompatExports.RunThreadLocalDestructors(ctx); + KernelMemoryCompatExports.RunThreadDtors(ctx); GuestThreadExecution.RequestCurrentEntryExit("scePthreadExit", value); ctx[CpuRegister.Rax] = value; return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -308,6 +296,8 @@ public static class KernelExports public static int PosixPthreadExit(CpuContext ctx) { var value = ctx[CpuRegister.Rdi]; + KernelPthreadExtendedCompatExports.RunThreadLocalDestructors(ctx); + KernelMemoryCompatExports.RunThreadDtors(ctx); GuestThreadExecution.RequestCurrentEntryExit("pthread_exit", value); ctx[CpuRegister.Rax] = value; return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -362,7 +352,10 @@ public static class KernelExports ExportName = "pthread_join", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int PosixPthreadJoin(CpuContext ctx) => PthreadJoin(ctx); + public static int PosixPthreadJoin(CpuContext ctx) + { + return PthreadJoin(ctx); + } [SysAbiExport( Nid = "wuCroIGjt2g", @@ -437,14 +430,21 @@ public static class KernelExports private static string ReadCString(CpuContext ctx, ulong address, int maxLen) { Span buf = stackalloc byte[maxLen]; - if (!ctx.Memory.TryRead(address, buf)) - return $""; + Span one = stackalloc byte[1]; + var len = 0; + while (len < buf.Length) + { + if (!ctx.Memory.TryRead(address + (ulong)len, one)) + return len == 0 ? $"" : System.Text.Encoding.UTF8.GetString(buf[..len]); - int len = 0; - while (len < buf.Length && buf[len] != 0) len++; + if (one[0] == 0) + break; - try { return System.Text.Encoding.UTF8.GetString(buf.Slice(0, len)); } - catch { return System.Text.Encoding.ASCII.GetString(buf.Slice(0, len)); } + buf[len++] = one[0]; + } + + try { return System.Text.Encoding.UTF8.GetString(buf[..len]); } + catch { return System.Text.Encoding.ASCII.GetString(buf[..len]); } } private static bool ShouldTracePthread() @@ -452,19 +452,18 @@ public static class KernelExports return string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_PTHREADS"), "1", StringComparison.Ordinal); } - private static int RequestProcessExit(CpuContext ctx, string syscallName) + [SysAbiExport( + Nid = "L1SBTkC+Cvw", + ExportName = "abort", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Abort(CpuContext ctx) { - var status = unchecked((int)ctx[CpuRegister.Rdi]); - Console.Error.WriteLine($"[LOADER][INFO] {syscallName}(status={status})"); - GuestThreadExecution.RequestCurrentEntryExit(syscallName, status); - ctx[CpuRegister.Rax] = unchecked((ulong)status); + // Route through the same graceful guest-entry-exit path as exit(): letting the call + // fall through to the host's native abort() does not unwind the guest thread cleanly. + Console.Error.WriteLine("[LOADER][INFO] abort() called by guest - terminating"); + GuestThreadExecution.RequestCurrentEntryExit("abort", -1); + ctx[CpuRegister.Rax] = unchecked((ulong)(-1L)); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - - // NOTE: "SHtAad20YYM"/"DIxvoy7Ngvk" are sce::Json::Value::getType/getInteger, and - // "3GPpjQdAMTw"/"9rAeANT2tyE"/"tsvEmnenz48" are __cxa_guard_acquire/__cxa_guard_release/ - // __cxa_atexit (verified by hashing against scripts/ps5_names.txt). Do not register them - // here as kernel mutex functions: the cxa guards are implemented in CxxAbiExports.cs and - // shadowing them breaks every C++ static-init guard in the game. - } diff --git a/src/SharpEmu.Libs/Kernel/KernelFileExtendedExports.cs b/src/SharpEmu.Libs/Kernel/KernelFileExtendedExports.cs new file mode 100644 index 0000000..67c468c --- /dev/null +++ b/src/SharpEmu.Libs/Kernel/KernelFileExtendedExports.cs @@ -0,0 +1,662 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Threading; + +namespace SharpEmu.Libs.Kernel; + +/// +/// Positional and scatter/gather file I/O, synchronization, renaming, fcntl, +/// polling and asynchronous I/O (AIO) that complement the base file exports. +/// These share the same guest FD table and path-resolution helpers via the +/// partial class. +/// +public static partial class KernelMemoryCompatExports +{ + // fcntl commands (FreeBSD numbering used by the PS4/PS5 libc). + private const int F_DUPFD = 0; + private const int F_GETFD = 1; + private const int F_SETFD = 2; + private const int F_GETFL = 3; + private const int F_SETFL = 4; + + // poll event bits. + private const short POLLIN = 0x0001; + private const short POLLOUT = 0x0004; + + // AIO request state (SCE_KERNEL_AIO_STATE_*). + private const uint AioStateCompleted = 3; + private const int SizeofAioRwRequest = 40; + + private static long _nextAioSubmitId = 1; + private static readonly ConcurrentDictionary _aioResults = new(); + + private static FileStream? GetOpenFile(int fd) + { + lock (_fdGate) + { + return _openFiles.TryGetValue(fd, out var stream) ? stream : null; + } + } + + // ---- Positional read/write (do not move the file offset) ---- + + [SysAbiExport(Nid = "ezv-RSBNKqI", ExportName = "pread", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixPread(CpuContext ctx) => KernelPreadCore(ctx); + + [SysAbiExport(Nid = "+r3rMFwItV4", ExportName = "sceKernelPread", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelPread(CpuContext ctx) => KernelPreadCore(ctx); + + private static int KernelPreadCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var bufferAddress = ctx[CpuRegister.Rsi]; + var requested = (int)Math.Min(ctx[CpuRegister.Rdx], int.MaxValue); + var offset = unchecked((long)ctx[CpuRegister.Rcx]); + if (requested < 0 || (requested > 0 && bufferAddress == 0) || offset < 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + var stream = GetOpenFile(fd); + if (stream is null) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (requested == 0) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + var buffer = GC.AllocateUninitializedArray(requested); + int read; + try + { + read = RandomAccess.Read(stream.SafeFileHandle, buffer, offset); + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (read > 0 && !ctx.Memory.TryWrite(bufferAddress, buffer.AsSpan(0, read))) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = unchecked((ulong)read); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "C2kJ-byS5rM", ExportName = "pwrite", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixPwrite(CpuContext ctx) => KernelPwriteCore(ctx); + + [SysAbiExport(Nid = "nKWi-N2HBV4", ExportName = "sceKernelPwrite", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelPwrite(CpuContext ctx) => KernelPwriteCore(ctx); + + private static int KernelPwriteCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var bufferAddress = ctx[CpuRegister.Rsi]; + var requested = (int)Math.Min(ctx[CpuRegister.Rdx], int.MaxValue); + var offset = unchecked((long)ctx[CpuRegister.Rcx]); + if (requested < 0 || (requested > 0 && bufferAddress == 0) || offset < 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + var stream = GetOpenFile(fd); + if (stream is null) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (requested == 0) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + var payload = GC.AllocateUninitializedArray(requested); + if (!ctx.Memory.TryRead(bufferAddress, payload)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + try + { + RandomAccess.Write(stream.SafeFileHandle, payload, offset); + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + ctx[CpuRegister.Rax] = unchecked((ulong)requested); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Synchronization ---- + + [SysAbiExport(Nid = "juWbTNM+8hw", ExportName = "fsync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixFsync(CpuContext ctx) => KernelFsyncCore(ctx); + + [SysAbiExport(Nid = "fTx66l5iWIA", ExportName = "sceKernelFsync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelFsync(CpuContext ctx) => KernelFsyncCore(ctx); + + [SysAbiExport(Nid = "KIbJFQ0I1Cg", ExportName = "fdatasync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixFdatasync(CpuContext ctx) => KernelFsyncCore(ctx); + + [SysAbiExport(Nid = "30Rh4ixbKy4", ExportName = "sceKernelFdatasync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelFdatasync(CpuContext ctx) => KernelFsyncCore(ctx); + + private static int KernelFsyncCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var stream = GetOpenFile(fd); + if (stream is null) + { + if (fd is 0 or 1 or 2) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + try + { + stream.Flush(flushToDisk: true); + } + catch (IOException) + { + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "Y2OqwJQ3lr8", ExportName = "sync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixSync(CpuContext ctx) + { + lock (_fdGate) + { + foreach (var stream in _openFiles.Values) + { + try { stream.Flush(flushToDisk: true); } catch (IOException) { } + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Truncation ---- + + [SysAbiExport(Nid = "ih4CD9-gghM", ExportName = "ftruncate", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixFtruncate(CpuContext ctx) => KernelFtruncateCore(ctx); + + [SysAbiExport(Nid = "VW3TVZiM4-E", ExportName = "sceKernelFtruncate", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelFtruncate(CpuContext ctx) => KernelFtruncateCore(ctx); + + private static int KernelFtruncateCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var length = unchecked((long)ctx[CpuRegister.Rsi]); + if (length < 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + var stream = GetOpenFile(fd); + if (stream is null) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + try + { + stream.SetLength(length); + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "ayrtszI7GBg", ExportName = "truncate", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixTruncate(CpuContext ctx) => KernelTruncateCore(ctx); + + [SysAbiExport(Nid = "WlyEA-sLDf0", ExportName = "sceKernelTruncate", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelTruncate(CpuContext ctx) => KernelTruncateCore(ctx); + + private static int KernelTruncateCore(CpuContext ctx) + { + var pathAddress = ctx[CpuRegister.Rdi]; + var length = unchecked((long)ctx[CpuRegister.Rsi]); + if (length < 0 || !TryReadNullTerminatedUtf8(ctx, pathAddress, MaxGuestStringLength, out var guestPath)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (IsReadOnlyGuestMutationPath(guestPath)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; + } + + var hostPath = ResolveGuestPath(guestPath); + try + { + using var stream = new FileStream(hostPath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); + stream.SetLength(length); + } + catch (FileNotFoundException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Rename ---- + + [SysAbiExport(Nid = "NN01qLRhiqU", ExportName = "rename", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixRename(CpuContext ctx) => KernelRenameCore(ctx); + + [SysAbiExport(Nid = "52NcYU9+lEo", ExportName = "sceKernelRename", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelRename(CpuContext ctx) => KernelRenameCore(ctx); + + private static int KernelRenameCore(CpuContext ctx) + { + if (!TryReadNullTerminatedUtf8(ctx, ctx[CpuRegister.Rdi], MaxGuestStringLength, out var fromGuest) || + !TryReadNullTerminatedUtf8(ctx, ctx[CpuRegister.Rsi], MaxGuestStringLength, out var toGuest)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (IsReadOnlyGuestMutationPath(fromGuest) || IsReadOnlyGuestMutationPath(toGuest)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; + } + + var fromHost = ResolveGuestPath(fromGuest); + var toHost = ResolveGuestPath(toGuest); + try + { + if (Directory.Exists(fromHost)) + { + Directory.Move(fromHost, toHost); + } + else + { + File.Move(fromHost, toHost, overwrite: true); + } + } + catch (FileNotFoundException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + catch (DirectoryNotFoundException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + InvalidateNegativeStatCacheForPathAndAncestors(toGuest); + AddNegativeStatCacheForGuestPath(fromGuest); + InvalidateAprFileSizeCache(fromHost); + InvalidateAprFileSizeCache(toHost); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Descriptor duplication ---- + + [SysAbiExport(Nid = "iiQjzvfWDq0", ExportName = "dup", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixDup(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + lock (_fdGate) + { + if (!_openFiles.TryGetValue(fd, out var stream)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + // POSIX dup shares the open file description (and offset), which is + // exactly the shared FileStream reference. + var newFd = (int)Interlocked.Increment(ref _nextFileDescriptor); + _openFiles[newFd] = stream; + ctx[CpuRegister.Rax] = unchecked((ulong)newFd); + } + + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "wdUufa9g-D8", ExportName = "dup2", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixDup2(CpuContext ctx) + { + var oldFd = unchecked((int)ctx[CpuRegister.Rdi]); + var newFd = unchecked((int)ctx[CpuRegister.Rsi]); + lock (_fdGate) + { + if (!_openFiles.TryGetValue(oldFd, out var stream)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (oldFd == newFd) + { + ctx[CpuRegister.Rax] = unchecked((ulong)newFd); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // If newFd names an open file, dup2 closes it first. + if (_openFiles.TryGetValue(newFd, out var existing) && !ReferenceEquals(existing, stream)) + { + try { existing.Dispose(); } catch (IOException) { } + } + + _openFiles[newFd] = stream; + ctx[CpuRegister.Rax] = unchecked((ulong)newFd); + } + + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- fcntl ---- + + [SysAbiExport(Nid = "8nY19bKoiZk", ExportName = "fcntl", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixFcntl(CpuContext ctx) => KernelFcntlCore(ctx); + + [SysAbiExport(Nid = "SoZkxZkCHaw", ExportName = "sceKernelFcntl", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelFcntl(CpuContext ctx) => KernelFcntlCore(ctx); + + private static int KernelFcntlCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var command = unchecked((int)ctx[CpuRegister.Rsi]); + var argument = unchecked((int)ctx[CpuRegister.Rdx]); + + switch (command) + { + case F_DUPFD: + lock (_fdGate) + { + if (!_openFiles.TryGetValue(fd, out var stream)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + var newFd = Math.Max((int)Interlocked.Increment(ref _nextFileDescriptor), argument); + _openFiles[newFd] = stream; + ctx[CpuRegister.Rax] = unchecked((ulong)newFd); + } + + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + + case F_GETFD: + case F_GETFL: + // No close-on-exec / status flags are tracked; report cleared. + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + + case F_SETFD: + case F_SETFL: + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + + default: + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + } + + // ---- poll / select ---- + // Regular files are always ready for both read and write, so report every + // requested descriptor as immediately ready. + + [SysAbiExport(Nid = "ku7D4q1Y9PI", ExportName = "poll", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixPoll(CpuContext ctx) + { + var fdsAddress = ctx[CpuRegister.Rdi]; + var count = unchecked((uint)ctx[CpuRegister.Rsi]); + if (fdsAddress == 0 || count == 0) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // struct pollfd { int fd; short events; short revents; } == 8 bytes. + var ready = 0; + Span buffer = stackalloc byte[8]; + for (uint i = 0; i < count && i < 4096; i++) + { + var entry = fdsAddress + i * 8; + if (!ctx.Memory.TryRead(entry, buffer)) + { + break; + } + + var events = BinaryPrimitives.ReadInt16LittleEndian(buffer[4..]); + var revents = (short)(events & (POLLIN | POLLOUT)); + BinaryPrimitives.WriteInt16LittleEndian(buffer[6..], revents); + if (revents != 0) + { + ready++; + } + + _ = ctx.Memory.TryWrite(entry, buffer); + } + + ctx[CpuRegister.Rax] = unchecked((ulong)ready); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "T8fER+tIGgk", ExportName = "select", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixSelect(CpuContext ctx) + { + // nfds in Rdi; the fd_sets are left as-is (all reported ready) and the + // ready count returned is nfds so callers proceed without blocking. + var nfds = unchecked((int)ctx[CpuRegister.Rdi]); + ctx[CpuRegister.Rax] = unchecked((ulong)Math.Max(nfds, 0)); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Asynchronous I/O (executed synchronously) ---- + + [SysAbiExport(Nid = "HgX7+AORI58", ExportName = "sceKernelAioSubmitReadCommands", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioSubmitReadCommands(CpuContext ctx) => KernelAioSubmit(ctx, write: false); + + [SysAbiExport(Nid = "lXT0m3P-vs4", ExportName = "sceKernelAioSubmitReadCommandsMultiple", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioSubmitReadCommandsMultiple(CpuContext ctx) => KernelAioSubmit(ctx, write: false); + + [SysAbiExport(Nid = "XQ8C8y+de+E", ExportName = "sceKernelAioSubmitWriteCommands", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioSubmitWriteCommands(CpuContext ctx) => KernelAioSubmit(ctx, write: true); + + private static int KernelAioSubmit(CpuContext ctx, bool write) + { + var requestsAddress = ctx[CpuRegister.Rdi]; + var count = unchecked((int)ctx[CpuRegister.Rsi]); + var outIdAddress = ctx[CpuRegister.Rcx]; + if (requestsAddress == 0 || count <= 0 || count > 0x10000) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + // Each SceKernelAioRWRequest: offset(8) nbyte(8) buf(8) result(8) fd(4). + Span request = stackalloc byte[SizeofAioRwRequest]; + Span result = stackalloc byte[16]; + for (var i = 0; i < count; i++) + { + var entry = requestsAddress + (ulong)(i * SizeofAioRwRequest); + if (!ctx.Memory.TryRead(entry, request)) + { + break; + } + + var offset = BinaryPrimitives.ReadInt64LittleEndian(request[0..]); + var nbyte = (int)Math.Min(BinaryPrimitives.ReadUInt64LittleEndian(request[8..]), int.MaxValue); + var buf = BinaryPrimitives.ReadUInt64LittleEndian(request[16..]); + var resultPtr = BinaryPrimitives.ReadUInt64LittleEndian(request[24..]); + var fd = BinaryPrimitives.ReadInt32LittleEndian(request[32..]); + + long transferred = KernelAioTransfer(ctx, fd, offset, nbyte, buf, write); + if (resultPtr != 0) + { + BinaryPrimitives.WriteInt64LittleEndian(result[0..], transferred); + BinaryPrimitives.WriteUInt32LittleEndian(result[8..], AioStateCompleted); + _ = ctx.Memory.TryWrite(resultPtr, result); + } + } + + var submitId = unchecked((uint)Interlocked.Increment(ref _nextAioSubmitId)); + _aioResults[submitId] = 0; + if (outIdAddress != 0) + { + Span idBuffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(idBuffer, submitId); + _ = ctx.Memory.TryWrite(outIdAddress, idBuffer); + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + private static long KernelAioTransfer(CpuContext ctx, int fd, long offset, int nbyte, ulong buf, bool write) + { + if (nbyte <= 0 || buf == 0 || offset < 0) + { + return 0; + } + + var stream = GetOpenFile(fd); + if (stream is null) + { + return -1; + } + + var scratch = GC.AllocateUninitializedArray(nbyte); + try + { + if (write) + { + if (!ctx.Memory.TryRead(buf, scratch)) + { + return -1; + } + + RandomAccess.Write(stream.SafeFileHandle, scratch, offset); + return nbyte; + } + + var read = RandomAccess.Read(stream.SafeFileHandle, scratch, offset); + if (read > 0 && !ctx.Memory.TryWrite(buf, scratch.AsSpan(0, read))) + { + return -1; + } + + return read; + } + catch (IOException) + { + return -1; + } + } + + [SysAbiExport(Nid = "o7O4z3jwKzo", ExportName = "sceKernelAioPollRequests", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioPollRequests(CpuContext ctx) => KernelAioComplete(ctx); + + [SysAbiExport(Nid = "lgK+oIWkJyA", ExportName = "sceKernelAioWaitRequests", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioWaitRequests(CpuContext ctx) => KernelAioComplete(ctx); + + private static int KernelAioComplete(CpuContext ctx) + { + // Submission already performed the I/O synchronously, so every request + // reports completed. Rsi points at the state-out array, Rdx = count. + var statesAddress = ctx[CpuRegister.Rsi]; + var count = unchecked((int)ctx[CpuRegister.Rdx]); + if (statesAddress != 0 && count > 0 && count <= 0x10000) + { + Span state = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(state, AioStateCompleted); + for (var i = 0; i < count; i++) + { + _ = ctx.Memory.TryWrite(statesAddress + (ulong)(i * sizeof(uint)), state); + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "fR521KIGgb8", ExportName = "sceKernelAioCancelRequest", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioCancelRequest(CpuContext ctx) + { + // Already completed; report the completed state if an out pointer given. + var stateAddress = ctx[CpuRegister.Rsi]; + if (stateAddress != 0) + { + Span state = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(state, AioStateCompleted); + _ = ctx.Memory.TryWrite(stateAddress, state); + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "5TgME6AYty4", ExportName = "sceKernelAioDeleteRequest", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioDeleteRequest(CpuContext ctx) + { + var submitId = unchecked((uint)ctx[CpuRegister.Rdi]); + _aioResults.TryRemove(submitId, out _); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } +} diff --git a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs index 77ba324..81848a7 100644 --- a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs @@ -2,27 +2,25 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; -using SharpEmu.HLE.Host; using SharpEmu.Libs.Ampr; +using SharpEmu.Libs.Bink; using System.Buffers; using System.Buffers.Binary; using System.Collections.Concurrent; using System.Text; +using System.Threading; using System.Runtime.InteropServices; +using System.Linq; using System.Globalization; namespace SharpEmu.Libs.Kernel; -public static class KernelMemoryCompatExports +public static partial class KernelMemoryCompatExports { private const int MaxGuestStringLength = 4096; private const int WideCharSize = sizeof(ushort); private const int MemsetChunkSize = 16 * 1024; - private const int MemcpyChunkSize = 256 * 1024; - - // Shared all-zero scratch for chunked zero-fill loops; never written to. private static readonly byte[] _zeroChunk = new byte[MemsetChunkSize]; - private const int TlsModuleBlockSize = 0x10000; private const int O_WRONLY = 0x1; private const int O_RDWR = 0x2; private const int O_APPEND = 0x8; @@ -57,13 +55,12 @@ public static class KernelMemoryCompatExports private const ulong FlexibleMemorySizeBytes = 448UL * 1024 * 1024; private const int OrbisVirtualQueryInfoSize = 72; private const int OrbisKernelMaximumNameLength = 32; - // Raw Windows PAGE_* values used only against HostRegionInfo.RawProtection, - // which by contract carries the untranslated protection word of the host - // platform in use (see IHostMemory). + private const uint MemCommit = 0x1000; private const uint HostPageNoAccess = 0x01; private const uint HostPageReadOnly = 0x02; private const uint HostPageReadWrite = 0x04; private const uint HostPageWriteCopy = 0x08; + private const uint HostPageExecute = 0x10; private const uint HostPageExecuteRead = 0x20; private const uint HostPageExecuteReadWrite = 0x40; private const uint HostPageExecuteWriteCopy = 0x80; @@ -100,7 +97,6 @@ public static class KernelMemoryCompatExports private static readonly Dictionary _openDirectories = new(); private static readonly object _libcAllocGate = new(); private static readonly object _memoryGate = new(); - private static readonly object _tlsGate = new(); private static readonly object _ioTraceGate = new(); private static readonly object _statCacheGate = new(); private static readonly object _guestMountGate = new(); @@ -108,7 +104,6 @@ public static class KernelMemoryCompatExports private static readonly Dictionary _libcAllocations = new(); private static readonly Dictionary _mappedRegions = new(); private static readonly Dictionary _mappedRegionNames = new(); - private static readonly Dictionary _tlsModuleBlocks = new(); private static readonly Dictionary _guestMounts = new(StringComparer.OrdinalIgnoreCase); private static readonly HashSet _tracedStatResults = new(StringComparer.Ordinal); private static readonly HashSet _negativeStatCache = new(StringComparer.OrdinalIgnoreCase); @@ -147,9 +142,35 @@ public static class KernelMemoryCompatExports private static string? _cachedApp0Root; private static string? _cachedDownload0Root; - // Property (not a cached field) so merely touching this type never resolves - // the platform backend; non-Windows hosts only throw if a call is reached. - private static IHostMemory HostMemory => HostPlatform.Current.Memory; + [StructLayout(LayoutKind.Sequential)] + private struct MemoryBasicInformation + { + public nint BaseAddress; + public nint AllocationBase; + public uint AllocationProtect; + public nuint RegionSize; + public uint State; + public uint Protect; + public uint Type; + } + + private static unsafe nuint VirtualQuery(nint lpAddress, out MemoryBasicInformation lpBuffer, nuint dwLength) + { + _ = dwLength; + var result = HostMemory.Query((void*)lpAddress, out var info); + lpBuffer = default; + lpBuffer.BaseAddress = (nint)info.BaseAddress; + lpBuffer.AllocationBase = (nint)info.AllocationBase; + lpBuffer.AllocationProtect = info.AllocationProtect; + lpBuffer.RegionSize = (nuint)info.RegionSize; + lpBuffer.State = info.State; + lpBuffer.Protect = info.Protect; + lpBuffer.Type = info.Type; + return result; + } + + private static unsafe bool VirtualProtect(nint lpAddress, nuint dwSize, uint flNewProtect, out uint lpflOldProtect) => + HostMemory.Protect((void*)lpAddress, dwSize, flNewProtect, out lpflOldProtect); private sealed class OpenDirectory { @@ -159,7 +180,7 @@ public static class KernelMemoryCompatExports } private readonly record struct DirectAllocation(ulong Start, ulong Length, int MemoryType); - private readonly record struct LibcHeapAllocation(nint BaseAddress, nuint Size, nuint Alignment, bool IsGuarded); + private readonly record struct LibcHeapAllocation(nint BaseAddress, nuint Size, nuint Alignment); private readonly record struct MappedRegion(ulong Address, ulong Length, int Protection, bool IsFlexible, bool IsDirect, ulong DirectStart); private readonly record struct BatchMapEntry(ulong Start, ulong Offset, ulong Length, byte Protection, byte Type, int Operation); @@ -189,53 +210,11 @@ public static class KernelMemoryCompatExports } } - public static bool TryUnregisterGuestPathMount(string guestMountPoint) - { - if (string.IsNullOrWhiteSpace(guestMountPoint)) - { - return false; - } - - var normalizedMountPoint = NormalizeGuestStatCachePath(guestMountPoint); - if (normalizedMountPoint is null || normalizedMountPoint == "/") - { - return false; - } - - var removed = false; - lock (_guestMountGate) - { - removed = _guestMounts.Remove(normalizedMountPoint); - } - - if (!removed) - { - return false; - } - - lock (_statCacheGate) - { - _negativeStatCache.RemoveWhere(path => - string.Equals(path, normalizedMountPoint, StringComparison.OrdinalIgnoreCase) || - path.StartsWith(normalizedMountPoint + "/", StringComparison.OrdinalIgnoreCase)); - } - - return true; - } - internal static bool TryAllocateHleData( CpuContext ctx, ulong length, ulong alignment, out ulong address) - => TryAllocateHleData(ctx, length, alignment, OrbisProtCpuReadWrite, out address); - - internal static bool TryAllocateHleData( - CpuContext ctx, - ulong length, - ulong alignment, - int protection, - out ulong address) { address = 0; if (length == 0 || length > int.MaxValue) @@ -250,7 +229,7 @@ public static class KernelMemoryCompatExports var desiredAddress = AlignUp( _nextVirtualAddress == 0 ? DefaultMapSearchBase : _nextVirtualAddress, effectiveAlignment); - if (!TryReserveGuestVirtualRange(ctx, desiredAddress, mappedLength, protection, effectiveAlignment, out address) || + if (!TryReserveGuestVirtualRange(ctx, desiredAddress, mappedLength, OrbisProtCpuReadWrite, effectiveAlignment, out address) || address == 0) { return false; @@ -260,7 +239,7 @@ public static class KernelMemoryCompatExports _mappedRegions[address] = new MappedRegion( address, mappedLength, - protection, + OrbisProtCpuReadWrite, IsFlexible: false, IsDirect: false, DirectStart: 0); @@ -280,69 +259,6 @@ public static class KernelMemoryCompatExports return true; } - private static ulong _dummyVtableAddress; - private const int DummyVtableSlotCount = 64; - - // Some KexEngine objects (mutex wrappers, NGS2 system/rack/voice handles, etc.) are treated - // by guest code as C++ instances with a vtable pointer at offset 0. When an HLE constructor - // leaves that slot zeroed, the game's own virtual dispatch (call [ [obj]+N ]) reads a null - // vtable and jumps through address N, crashing (e.g. "CALL qword ptr [RAX+0x58]" with RAX=0). - // This allocates one shared, executable no-op stub plus a vtable of pointers to it, so any - // HLE object constructor can make virtual calls on its output land safely. - internal static bool TryWriteDummyVtable(CpuContext ctx, ulong objectAddress) - { - if (objectAddress == 0 || !TryEnsureDummyVtable(ctx, out var vtableAddress)) - { - return false; - } - - return ctx.TryWriteUInt64(objectAddress, vtableAddress); - } - - private static bool TryEnsureDummyVtable(CpuContext ctx, out ulong vtableAddress) - { - lock (_memoryGate) - { - if (_dummyVtableAddress != 0) - { - vtableAddress = _dummyVtableAddress; - return true; - } - - const int executableReadWrite = OrbisProtCpuRead | OrbisProtCpuWrite | OrbisProtCpuExec; - if (!TryAllocateHleData(ctx, 0x1000, 0x1000, executableReadWrite, out var block)) - { - vtableAddress = 0; - return false; - } - - // xor eax, eax; ret - every dummy virtual method just returns 0 and does nothing else. - if (!ctx.Memory.TryWrite(block, new byte[] { 0x31, 0xC0, 0xC3 })) - { - vtableAddress = 0; - return false; - } - - var table = new byte[DummyVtableSlotCount * sizeof(ulong)]; - for (var i = 0; i < DummyVtableSlotCount; i++) - { - BinaryPrimitives.WriteUInt64LittleEndian(table.AsSpan(i * sizeof(ulong), sizeof(ulong)), block); - } - - var tableAddress = block + 0x100; - if (!ctx.Memory.TryWrite(tableAddress, table)) - { - vtableAddress = 0; - return false; - } - - Console.Error.WriteLine($"[LOADER][INFO] Dummy vtable ready: stub=0x{block:X16} vtable=0x{tableAddress:X16} slots={DummyVtableSlotCount}"); - _dummyVtableAddress = tableAddress; - vtableAddress = tableAddress; - return true; - } - } - internal static void RegisterReservedVirtualRange(ulong address, ulong length) { if (address == 0 || length == 0) @@ -393,18 +309,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - // Longer null-dst memsets are unrecoverable, but RAX must still be set - leaving it - // stale here previously let callers that do `buf = memset(...)` carry on with a - // garbage "buffer" pointer instead of a clean NULL, causing a *different*, - // confusingly-located crash further downstream. - var largeRecoveryIndex = Interlocked.Increment(ref _nullMemsetRecoveryCount); - if (largeRecoveryIndex <= 8) - { - Console.Error.WriteLine( - $"[LOADER][WARNING] memset null-dst (len>0x20) recovery#{largeRecoveryIndex}: rip=0x{ctx.Rip:X16} len=0x{length:X} val=0x{value:X2}"); - } - - ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } @@ -422,13 +326,16 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - const ulong MaxSane = 512UL * 1024 * 1024; + // Only obviously-bogus destinations or absurd sizes are rejected. Large + // but plausible clears (engines zero multi-hundred-MB resource pools in + // one call) are allowed; the write loop below clamps to the mapped range + // if the request runs off the end of a region instead of hard-faulting. + const ulong MaxSane = 2UL * 1024 * 1024 * 1024; if (destination < 0x1000 || destination >= CanonicalUserUpper || length > MaxSane) { Console.WriteLine("!!! CRITICAL: Bad Memset Call !!!"); Console.WriteLine($"Called from RIP: 0x{ctx.Rip:X}"); Console.WriteLine($"dst=0x{destination:X} val=0x{value:X2} len=0x{length:X}"); - ctx[CpuRegister.Rax] = destination; return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -450,6 +357,9 @@ public static class KernelMemoryCompatExports var take = (int)Math.Min((ulong)chunkLength, remaining); if (!TryWriteCompat(ctx, cursor, chunk.AsSpan(0, take))) { + // Clamp oversized clears to the valid mapped prefix. Small + // inaccessible writes are tolerated for compatibility with + // titles that probe optional state during startup. if (length <= 0x40) { var recoveryIndex = Interlocked.Increment(ref _inaccessibleMemsetRecoveryCount); @@ -463,7 +373,16 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + var clampIndex = Interlocked.Increment(ref _inaccessibleMemsetRecoveryCount); + if (clampIndex <= 8) + { + Console.Error.WriteLine( + $"[LOADER][WARNING] memset clamped to mapped range#{clampIndex}: rip=0x{ctx.Rip:X16} " + + $"dst=0x{destination:X16} len=0x{length:X} written=0x{cursor - destination:X} val=0x{value:X2}"); + } + + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; } cursor += (ulong)take; @@ -678,80 +597,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "AV6ipCNa4Rw", - ExportName = "strcasecmp", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strcasecmp(CpuContext ctx) - { - var left = ctx[CpuRegister.Rdi]; - var right = ctx[CpuRegister.Rsi]; - if (left == 0 || right == 0) - { - var recoveryIndex = Interlocked.Increment(ref _nullStrcasecmpRecoveryCount); - if (recoveryIndex <= 16) - { - var otherAddress = left == 0 ? right : left; - var otherText = otherAddress != 0 && TryReadNullTerminatedUtf8(ctx, otherAddress, 256, out var text) - ? text - : ""; - _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnRip); - Console.Error.WriteLine( - $"[LOADER][WARNING] strcasecmp null-arg recovery#{recoveryIndex}: ret=0x{returnRip:X16} left=0x{left:X16} right=0x{right:X16} other=\"{otherText}\""); - } - - // Real strcasecmp(NULL, x) is undefined behaviour and previously crashed inside the - // LLE-routed implementation. Treat it as "not equal" instead so callers doing - // `if (strcasecmp(a, b) == 0)` degrade gracefully rather than taking down the guest. - ctx[CpuRegister.Rax] = left == right ? 0uL : 1uL; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - if (!TryCompareStringsCaseInsensitive(ctx, left, right, limit: ulong.MaxValue, out var compare)) - { - ctx[CpuRegister.Rax] = 1; - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ctx[CpuRegister.Rax] = unchecked((ulong)compare); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - private static bool TryCompareStringsCaseInsensitive(CpuContext ctx, ulong left, ulong right, ulong limit, out int compare) - { - compare = 0; - if (left == 0 || right == 0) - { - return false; - } - - var max = limit == ulong.MaxValue ? 1_048_576UL : Math.Min(limit, 1_048_576UL); - Span leftByte = stackalloc byte[1]; - Span rightByte = stackalloc byte[1]; - for (ulong i = 0; i < max; i++) - { - if (!TryReadCompat(ctx, left + i, leftByte) || - !TryReadCompat(ctx, right + i, rightByte)) - { - return false; - } - - var leftLower = ToAsciiLower(leftByte[0]); - var rightLower = ToAsciiLower(rightByte[0]); - compare = leftLower - rightLower; - if (compare != 0 || leftByte[0] == 0 || rightByte[0] == 0) - { - return true; - } - } - - compare = 0; - return true; - } - - private static byte ToAsciiLower(byte value) => value is >= (byte)'A' and <= (byte)'Z' ? (byte)(value + 32) : value; - [SysAbiExport( Nid = "0nV21JjYCH8", ExportName = "wcsncpy", @@ -803,54 +648,6 @@ public static class KernelMemoryCompatExports return VsnprintfCore(ctx); } - // sprintf/vsprintf are served from the same HLE formatting engine as snprintf/vsnprintf - // instead of falling through to the game's bundled libc. - [SysAbiExport( - Nid = "tcVi5SivF7Q", - ExportName = "sprintf", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Sprintf(CpuContext ctx) - { - var destination = ctx[CpuRegister.Rdi]; - var formatAddress = ctx[CpuRegister.Rsi]; - - if (!TryReadCString(ctx, formatAddress, 1_048_576, out var formatBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - var format = Encoding.UTF8.GetString(formatBytes); - var rendered = FormatStringFromVarArgs(ctx, format, firstGpArgIndex: 2); - return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, rendered); - } - - [SysAbiExport( - Nid = "jbz9I9vkqkk", - ExportName = "vsprintf", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Vsprintf(CpuContext ctx) - { - var destination = ctx[CpuRegister.Rdi]; - var formatAddress = ctx[CpuRegister.Rsi]; - var vaListAddress = ctx[CpuRegister.Rdx]; - - if (!TryReadCString(ctx, formatAddress, 1_048_576, out var formatBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - var format = Encoding.UTF8.GetString(formatBytes); - if (!TryCreateVaListCursor(ctx, vaListAddress, out var vaCursor)) - { - return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, formatBytes); - } - - var rendered = FormatString(ctx, format, ref vaCursor); - return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, rendered); - } - [SysAbiExport( Nid = "nJz16JE1txM", ExportName = "swprintf", @@ -1017,20 +814,59 @@ public static class KernelMemoryCompatExports } var payload = new byte[count * WideCharSize]; - for (var copied = 0; copied < count; copied++) + if (count == 0) { - if (!TryReadUInt16Compat(ctx, source + ((ulong)copied * WideCharSize), out var unit)) + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // Keep host-pointer reads page-bounded and copy several UTF-16 code + // units per validation. Large scratch strings otherwise pay the host + // address-validation and temporary-buffer cost once per character. + const int maxReadBytes = 4096; + var readBuffer = GC.AllocateUninitializedArray( + Math.Min(maxReadBytes, payload.Length)); + var copied = 0; + while (copied < count) + { + var sourceAddress = source + ((ulong)copied * WideCharSize); + if (sourceAddress < source) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - BinaryPrimitives.WriteUInt16LittleEndian( - payload.AsSpan(copied * WideCharSize, WideCharSize), - unit); - - if (unit == 0) + var pageBytesRemaining = maxReadBytes - + (int)(sourceAddress & (maxReadBytes - 1)); + var remainingBytes = (count - copied) * WideCharSize; + var readBytes = Math.Min( + readBuffer.Length, + Math.Min(pageBytesRemaining, remainingBytes)); + readBytes &= ~(WideCharSize - 1); + if (readBytes == 0 || + !TryReadCompat(ctx, sourceAddress, readBuffer.AsSpan(0, readBytes))) { - break; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + for (var offset = 0; offset < readBytes; offset += WideCharSize) + { + var unit = BinaryPrimitives.ReadUInt16LittleEndian( + readBuffer.AsSpan(offset, WideCharSize)); + if (unit == 0) + { + // payload is zero-initialized, supplying wcsncpy padding. + copied = count; + break; + } + + BinaryPrimitives.WriteUInt16LittleEndian( + payload.AsSpan((copied * WideCharSize) + offset, WideCharSize), + unit); + } + + if (copied != count) + { + copied += readBytes / WideCharSize; } } @@ -1055,23 +891,48 @@ public static class KernelMemoryCompatExports private static int WcschrCore(CpuContext ctx, ulong address, ushort needle) { - for (ulong index = 0; index < 1_048_576; index++) + const int maxReadBytes = 4096; + var readBuffer = GC.AllocateUninitializedArray(maxReadBytes); + const ulong maxUnits = 1_048_576; + for (ulong index = 0; index < maxUnits;) { - if (!TryReadUInt16Compat(ctx, address + (index * WideCharSize), out var unit)) + var unitAddress = address + (index * WideCharSize); + if (unitAddress < address) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (unit == needle) + var remainingBytes = (maxUnits - index) * WideCharSize; + var pageBytesRemaining = maxReadBytes - + (int)(unitAddress & (maxReadBytes - 1)); + var readBytes = (int)Math.Min( + (ulong)Math.Min(readBuffer.Length, pageBytesRemaining), + remainingBytes); + readBytes &= ~(WideCharSize - 1); + if (readBytes == 0 || + !TryReadCompat(ctx, unitAddress, readBuffer.AsSpan(0, readBytes))) { - ctx[CpuRegister.Rax] = address + (index * WideCharSize); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (unit == 0) + for (var offset = 0; offset < readBytes; offset += WideCharSize) { - break; + var unit = BinaryPrimitives.ReadUInt16LittleEndian( + readBuffer.AsSpan(offset, WideCharSize)); + if (unit == needle) + { + ctx[CpuRegister.Rax] = unitAddress + (ulong)offset; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + if (unit == 0) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } } + + index += (ulong)(readBytes / WideCharSize); } ctx[CpuRegister.Rax] = 0; @@ -1209,65 +1070,16 @@ public static class KernelMemoryCompatExports { var destination = ctx[CpuRegister.Rdi]; var source = ctx[CpuRegister.Rsi]; - var rawCount = ctx[CpuRegister.Rdx]; - - // A garbage/absurd count (observed as e.g. 0xA7560035 from the same still-unidentified - // upstream bug that also feeds bad lengths to memset) must not turn into a multi-GB - // copy attempt from a guest-thread call context, which corrupted the CLR outright - // ("Invalid Program: attempted to call a UnmanagedCallersOnly method from managed - // code") instead of throwing a normal exception. Reject anything above a sane bound. - const ulong maxSaneCount = 512UL * 1024 * 1024; - if (rawCount > maxSaneCount) + var count = (int)Math.Min(ctx[CpuRegister.Rdx], int.MaxValue); + if (count < 0) { - Console.Error.WriteLine($"[LOADER][WARNING] memcpy oversized count rejected: dst=0x{destination:X16} src=0x{source:X16} count=0x{rawCount:X}"); - ctx[CpuRegister.Rax] = destination; return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - ctx[CpuRegister.Rax] = destination; - if (rawCount == 0) + var payload = GC.AllocateUninitializedArray(count); + if (count > 0 && (!TryReadCompat(ctx, source, payload) || !TryWriteCompat(ctx, destination, payload))) { - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - // Cap iterations at the requested chunk size, not chunk.Length: Rent may hand - // back a larger array, and the copy granularity should not depend on pool - // bucketing internals. - var chunkLength = (int)Math.Min(rawCount, (ulong)MemcpyChunkSize); - var chunk = ArrayPool.Shared.Rent(chunkLength); - try - { - // memmove aliases this export, so overlapping ranges must survive the chunked - // copy: when the destination starts inside the source range, copy high-to-low - // so no source byte is overwritten before it has been read. - var copyBackward = destination > source && destination - source < rawCount; - var remaining = rawCount; - ulong offset = copyBackward ? rawCount : 0; - while (remaining > 0) - { - var take = (int)Math.Min((ulong)chunkLength, remaining); - if (copyBackward) - { - offset -= (ulong)take; - } - - var span = chunk.AsSpan(0, take); - if (!TryReadCompat(ctx, source + offset, span) || !TryWriteCompat(ctx, destination + offset, span)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (!copyBackward) - { - offset += (ulong)take; - } - - remaining -= (ulong)take; - } - } - finally - { - ArrayPool.Shared.Return(chunk); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -1508,24 +1320,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - // Was unresolved (returning the 0x80020002 sentinel, then crashing when the guest - // dereferenced it) - the game's own heap instrumentation calls this hook when it - // detects a corrupted/invalid block, not the emulator's allocator, so this is purely - // a diagnostic sink: log what was reported and return success so the caller continues. - [SysAbiExport( - Nid = "al3JzFI9MQ0", - ExportName = "sceLibcInternalHeapErrorReportForGame", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceLibcInternal")] - public static int LibcInternalHeapErrorReportForGame(CpuContext ctx) - { - Console.Error.WriteLine( - $"[LOADER][WARN] sceLibcInternalHeapErrorReportForGame: rdi=0x{ctx[CpuRegister.Rdi]:X16} " + - $"rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} rcx=0x{ctx[CpuRegister.Rcx]:X16}"); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "DfivPArhucg", ExportName = "memcmp", @@ -1563,214 +1357,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "ob5xAW4ln-0", - ExportName = "strchr", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strchr(CpuContext ctx) - { - var address = ctx[CpuRegister.Rdi]; - var needle = unchecked((byte)ctx[CpuRegister.Rsi]); - if (address == 0) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - // The terminator counts as part of the scanned range, so strchr(s, '\0') - // returns a pointer to the string's null byte just like a native libc. - Span current = stackalloc byte[1]; - for (ulong index = 0; index < 1_048_576; index++) - { - if (!TryReadCompat(ctx, address + index, current)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (current[0] == needle) - { - ctx[CpuRegister.Rax] = address + index; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - if (current[0] == 0) - { - break; - } - } - - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "9yDWMxEFdJU", - ExportName = "strrchr", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strrchr(CpuContext ctx) - { - var address = ctx[CpuRegister.Rdi]; - var needle = unchecked((byte)ctx[CpuRegister.Rsi]); - if (address == 0) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ulong match = 0; - var found = false; - Span current = stackalloc byte[1]; - for (ulong index = 0; index < 1_048_576; index++) - { - if (!TryReadCompat(ctx, address + index, current)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (current[0] == needle) - { - match = address + index; - found = true; - } - - if (current[0] == 0) - { - break; - } - } - - ctx[CpuRegister.Rax] = found ? match : 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "8u8lPzUEq+U", - ExportName = "memchr", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Memchr(CpuContext ctx) - { - var address = ctx[CpuRegister.Rdi]; - var needle = unchecked((byte)ctx[CpuRegister.Rsi]); - var count = ctx[CpuRegister.Rdx]; - - Span current = stackalloc byte[1]; - for (ulong index = 0; index < count; index++) - { - if (!TryReadCompat(ctx, address + index, current)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (current[0] == needle) - { - ctx[CpuRegister.Rax] = address + index; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - } - - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "Ls4tzzhimqQ", - ExportName = "strcat", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strcat(CpuContext ctx) - { - var destination = ctx[CpuRegister.Rdi]; - var source = ctx[CpuRegister.Rsi]; - if (!TryReadCString(ctx, source, 1_048_576, out var sourceBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (!TryReadCString(ctx, destination, 1_048_576, out var destinationBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - // Overwrite the destination terminator and re-terminate after the copied bytes. - var appendAddress = destination + (ulong)destinationBytes.Length; - var payload = new byte[sourceBytes.Length + 1]; - sourceBytes.CopyTo(payload.AsSpan()); - if (!TryWriteCompat(ctx, appendAddress, payload)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ctx[CpuRegister.Rax] = destination; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "kHg45qPC6f0", - ExportName = "strncat", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strncat(CpuContext ctx) - { - var destination = ctx[CpuRegister.Rdi]; - var source = ctx[CpuRegister.Rsi]; - var limit = ctx[CpuRegister.Rdx]; - - // Bounding the source read by the count yields strncat's "at most n bytes" - // semantics while still stopping early at the source terminator. - if (!TryReadCString(ctx, source, limit, out var sourceBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (!TryReadCString(ctx, destination, 1_048_576, out var destinationBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - var appendAddress = destination + (ulong)destinationBytes.Length; - var payload = new byte[sourceBytes.Length + 1]; - sourceBytes.CopyTo(payload.AsSpan()); - if (!TryWriteCompat(ctx, appendAddress, payload)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ctx[CpuRegister.Rax] = destination; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "viiwFMaNamA", - ExportName = "strstr", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strstr(CpuContext ctx) - { - var haystack = ctx[CpuRegister.Rdi]; - var needle = ctx[CpuRegister.Rsi]; - if (!TryReadCString(ctx, haystack, 1_048_576, out var haystackBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (!TryReadCString(ctx, needle, 1_048_576, out var needleBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - // An empty needle matches at the start of the haystack. - if (needleBytes.Length == 0) - { - ctx[CpuRegister.Rax] = haystack; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - var matchIndex = haystackBytes.AsSpan().IndexOf(needleBytes.AsSpan()); - ctx[CpuRegister.Rax] = matchIndex >= 0 ? haystack + (ulong)matchIndex : 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "QrZZdJ8XsX0", ExportName = "fputs", @@ -1828,6 +1414,17 @@ public static class KernelMemoryCompatExports var mode = ResolveOpenMode(flags, access); try { + if (Bink2MovieBridge.ShouldSkipGuestMovie(hostPath)) + { + LogOpenTrace( + "_open bink-skip path='" + guestPath + "' host='" + hostPath + + "' flags=0x" + flags.ToString("X8")); + Console.Error.WriteLine( + "[LOADER][INFO] Skipping Bink movie without a decoder: " + + Path.GetFileName(hostPath)); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + if (IsMutatingOpen(flags) && IsReadOnlyGuestMutationPath(guestPath)) { LogOpenTrace($"_open readonly path='{guestPath}' host='{hostPath}' flags=0x{flags:X8}"); @@ -1849,10 +1446,9 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - int directoryFd; + var directoryFd = (int)Interlocked.Increment(ref _nextFileDescriptor); lock (_fdGate) { - directoryFd = AllocateGuestFileDescriptor(); _openDirectories[directoryFd] = new OpenDirectory { Path = hostPath, @@ -1873,13 +1469,18 @@ public static class KernelMemoryCompatExports stream.Seek(0, SeekOrigin.End); } - int fd; + var fd = (int)Interlocked.Increment(ref _nextFileDescriptor); lock (_fdGate) { - fd = AllocateGuestFileDescriptor(); _openFiles[fd] = stream; } + // Bink is linked directly into some games, so there is no media + // import for the HLE codec layer to intercept. The successful + // guest file open is the stable boundary at which the optional + // host Bink bridge can attach to the same movie. + Bink2MovieBridge.ObserveGuestMovie(hostPath); + if (IsMutatingOpen(flags)) { InvalidateNegativeStatCacheForPathAndAncestors(guestPath); @@ -1899,45 +1500,6 @@ public static class KernelMemoryCompatExports } } - [SysAbiExport( - Nid = "fgIsQ10xYVA", - ExportName = "sceKernelChmod", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int KernelChmod(CpuContext ctx) - { - var pathAddress = ctx[CpuRegister.Rdi]; - var mode = unchecked((uint)ctx[CpuRegister.Rsi]); - if (pathAddress == 0) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; - } - - if (!TryReadNullTerminatedUtf8(ctx, pathAddress, MaxGuestStringLength, out var guestPath)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - var hostPath = ResolveGuestPath(guestPath); - if (IsReadOnlyGuestMutationPath(guestPath)) - { - LogOpenTrace($"chmod readonly path='{guestPath}' host='{hostPath}' mode=0x{mode:X}"); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; - } - - if (!File.Exists(hostPath) && !Directory.Exists(hostPath)) - { - AddNegativeStatCacheForGuestPath(guestPath); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; - } - - // POSIX permission bits have no host equivalent on Windows; accept the call - // so guests that chmod their freshly created files/directories can proceed. - LogOpenTrace($"chmod path='{guestPath}' host='{hostPath}' mode=0x{mode:X}"); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "NNtFaKJbPt0", ExportName = "_close", @@ -2011,25 +1573,8 @@ public static class KernelMemoryCompatExports Nid = "E6ao34wPw+U", ExportName = "stat", Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int PosixStat(CpuContext ctx) - { - var result = KernelStat(ctx); - if (result == (int)OrbisGen2Result.ORBIS_GEN2_OK) - { - return 0; - } - - var errno = result switch - { - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT => Einval, - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT => Efault, - _ => 2, - }; - KernelRuntimeCompatExports.TrySetErrno(ctx, errno); - ctx[CpuRegister.Rax] = ulong.MaxValue; - return -1; - } + LibraryName = "libc")] + public static int PosixStat(CpuContext ctx) => KernelStat(ctx); [SysAbiExport( Nid = "gEpBkcwxUjw", @@ -2048,15 +1593,15 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - var entryCount = (int)count; - Span localIds = count <= 256 ? stackalloc uint[entryCount] : new uint[entryCount]; - Span localSizes = count <= 128 ? stackalloc ulong[entryCount] : new ulong[entryCount]; - var resolvedGuestPaths = new string[entryCount]; - var resolvedHostPaths = new string[entryCount]; - for (ulong i = 0; i < count; i++) { - var index = (int)i; + if (idsAddress != 0 && + !TryWriteUInt32Compat(ctx, idsAddress + (i * sizeof(uint)), uint.MaxValue)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + if (!TryResolveAprFilepath(ctx, pathListAddress, i, out var guestPath)) { KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); @@ -2066,52 +1611,137 @@ public static class KernelMemoryCompatExports var hostPath = ResolveGuestPath(guestPath); if (!TryGetAprFileSize(hostPath, out var fileSize)) { + // Per-file resolve: a missing entry gets an invalid id + // (0xFFFFFFFF, already written above) and size 0, and the batch + // CONTINUES. Aborting the whole batch on the first miss left the + // remaining paths unresolved and could stall the guest's asset + // streaming when a batch happens to include an absent (e.g. + // patch/DLC) file; the caller checks per-file id/size. LogIoTrace("apr_resolve", guestPath, $"host='{hostPath}' index={i} count={count} result=not_found"); - KernelRuntimeCompatExports.TrySetErrno(ctx, 2); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + if (sizesAddress != 0 && + !TryWriteUInt64Compat(ctx, sizesAddress + (i * sizeof(ulong)), 0)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + continue; } - var fileId = AmprFileRegistry.ComputeFileId(guestPath); + var fileId = AmprFileRegistry.Register(guestPath, hostPath); LogIoTrace("apr_resolve", guestPath, $"host='{hostPath}' index={i} count={count} id=0x{fileId:X8} size={fileSize}"); - localIds[index] = fileId; - localSizes[index] = fileSize; - resolvedGuestPaths[index] = guestPath; - resolvedHostPaths[index] = hostPath; - } - - Span sizePayload = count <= 64 ? stackalloc byte[entryCount * sizeof(ulong)] : new byte[entryCount * sizeof(ulong)]; - for (ulong i = 0; i < count; i++) - { - BinaryPrimitives.WriteUInt64LittleEndian(sizePayload[(int)(i * sizeof(ulong))..], localSizes[(int)i]); - } - - if (!TryWriteCompat(ctx, sizesAddress, sizePayload)) - { - KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (idsAddress != 0) - { - Span idPayload = count <= 128 ? stackalloc byte[entryCount * sizeof(uint)] : new byte[entryCount * sizeof(uint)]; - for (ulong i = 0; i < count; i++) + if (idsAddress != 0 && + !TryWriteUInt32Compat(ctx, idsAddress + (i * sizeof(uint)), fileId)) { - BinaryPrimitives.WriteUInt32LittleEndian(idPayload[(int)(i * sizeof(uint))..], localIds[(int)i]); + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (!TryWriteCompat(ctx, idsAddress, idPayload)) + if (!TryWriteUInt64Compat(ctx, sizesAddress + (i * sizeof(ulong)), fileSize)) { KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } } - for (var i = 0; i < entryCount; i++) + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // The IDs-only sibling of sceKernelAprResolveFilepathsToIdsAndFileSizes. + // Games that stream via AMPR APR call this to turn asset paths into file + // IDs, then hand those IDs to sceAmprAprCommandBufferReadFile. Without it + // the paths never register in AmprFileRegistry, so every subsequent + // ReadFile fails with NOT_FOUND and the streaming pipeline stalls forever. + // Signature: (const char* const* paths, size_t count, uint32_t* ids). + [SysAbiExport( + Nid = "WT-5NKy42fw", + ExportName = "sceKernelAprResolveFilepathsToIds", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelAprResolveFilepathsToIds(CpuContext ctx) + { + var pathListAddress = ctx[CpuRegister.Rdi]; + var count = ctx[CpuRegister.Rsi]; + var idsAddress = ctx[CpuRegister.Rdx]; + if (pathListAddress == 0 || count == 0 || idsAddress == 0 || count > 1024) { - AmprFileRegistry.Register(resolvedGuestPaths[i], resolvedHostPaths[i]); + KernelRuntimeCompatExports.TrySetErrno(ctx, Einval); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + for (ulong i = 0; i < count; i++) + { + if (!TryWriteUInt32Compat(ctx, idsAddress + (i * sizeof(uint)), uint.MaxValue)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (!TryResolveAprFilepath(ctx, pathListAddress, i, out var guestPath)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var hostPath = ResolveGuestPath(guestPath); + if (!TryGetAprFileSize(hostPath, out _)) + { + LogIoTrace("apr_resolve_ids", guestPath, $"host='{hostPath}' index={i} count={count} result=not_found"); + KernelRuntimeCompatExports.TrySetErrno(ctx, 2); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + var fileId = AmprFileRegistry.Register(guestPath, hostPath); + LogIoTrace("apr_resolve_ids", guestPath, $"host='{hostPath}' index={i} count={count} id=0x{fileId:X8}"); + + if (!TryWriteUInt32Compat(ctx, idsAddress + (i * sizeof(uint)), fileId)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // Stat an AMPR APR file by the id returned from sceKernelAprResolveFilepathsToIds. + // Games call this after resolving ids to learn each asset's size before + // issuing the streaming read. When it is missing the guest gets no size back + // and dereferences a null result pointer (observed SIGSEGV write to 0x0 in + // Void Terrarium). Signature: (SceKernelAprFileId id, SceKernelStat* stat). + [SysAbiExport( + Nid = "ApkYaHb8Sek", + ExportName = "sceKernelAprGetFileStat", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelAprGetFileStat(CpuContext ctx) + { + var fileId = unchecked((uint)ctx[CpuRegister.Rdi]); + var statAddress = ctx[CpuRegister.Rsi]; + if (statAddress == 0) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Einval); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!AmprFileRegistry.TryGetHostPath(fileId, out var hostPath)) + { + LogIoTrace("apr_get_file_stat", $"id=0x{fileId:X8}", "result=id_not_registered"); + KernelRuntimeCompatExports.TrySetErrno(ctx, 2); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (!TryWriteHostPathStat(ctx, statAddress, hostPath)) + { + LogIoTrace("apr_get_file_stat", hostPath, $"id=0x{fileId:X8} result=not_found"); + KernelRuntimeCompatExports.TrySetErrno(ctx, 2); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + LogIoTrace("apr_get_file_stat", hostPath, $"id=0x{fileId:X8}"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -2312,12 +1942,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (KernelSocketCompatExports.TryCloseSocketFd(fd)) - { - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - FileStream? stream; lock (_fdGate) { @@ -2361,17 +1985,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (KernelSocketCompatExports.TryReadSocketFd(ctx, fd, bufferAddress, requested, out var socketBytesRead, out var socketError)) - { - if (socketError != OrbisGen2Result.ORBIS_GEN2_OK) - { - return (int)socketError; - } - - ctx[CpuRegister.Rax] = socketBytesRead; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - FileStream? stream; lock (_fdGate) { @@ -2601,17 +2214,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (KernelSocketCompatExports.TryWriteSocketFd(ctx, fd, payload, out var socketError)) - { - if (socketError != OrbisGen2Result.ORBIS_GEN2_OK) - { - return (int)socketError; - } - - ctx[CpuRegister.Rax] = unchecked((ulong)requested); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - FileStream? stream; lock (_fdGate) { @@ -2650,37 +2252,23 @@ public static class KernelMemoryCompatExports LibraryName = "libKernel")] public static int ClockGettime(CpuContext ctx) { - var clockId = unchecked((int)ctx[CpuRegister.Rdi]); var timespecAddress = ctx[CpuRegister.Rsi]; if (timespecAddress == 0) { - KernelRuntimeCompatExports.TrySetErrno(ctx, Einval); - ctx[CpuRegister.Rax] = unchecked((ulong)-1); - return -1; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - long seconds; - long nanoseconds; - if (!KernelRuntimeCompatExports.ResolveClockTime(clockId, out seconds, out nanoseconds)) + var now = DateTimeOffset.UtcNow; + var seconds = now.ToUnixTimeSeconds(); + var nanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100; + if (!ctx.TryWriteUInt64(timespecAddress, unchecked((ulong)seconds)) || + !ctx.TryWriteUInt64(timespecAddress + sizeof(long), unchecked((ulong)nanoseconds))) { - KernelRuntimeCompatExports.TrySetErrno(ctx, Einval); - ctx[CpuRegister.Rax] = unchecked((ulong)-1); - return -1; - } - - Span timespecBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(timespecBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(timespecBuffer[sizeof(long)..], nanoseconds); - - if (!ctx.Memory.TryWrite(timespecAddress, timespecBuffer)) - { - KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); - ctx[CpuRegister.Rax] = unchecked((ulong)-1); - return -1; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } ctx[CpuRegister.Rax] = 0; - return 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; } [SysAbiExport( @@ -2708,31 +2296,7 @@ public static class KernelMemoryCompatExports private static ulong ResolveTlsAddress(CpuContext ctx, ulong moduleId, ulong offset) { - if (ctx.FsBase == 0) - { - return 0; - } - - if (moduleId <= 1) - { - return unchecked(ctx.FsBase + offset); - } - - var key = (ctx.FsBase << 16) ^ (moduleId & 0xFFFFUL); - ulong moduleBase; - lock (_tlsGate) - { - if (!_tlsModuleBlocks.TryGetValue(key, out moduleBase)) - { - var block = Marshal.AllocHGlobal(TlsModuleBlockSize); - Marshal.Copy(new byte[TlsModuleBlockSize], 0, block, TlsModuleBlockSize); - - moduleBase = unchecked((ulong)block); - _tlsModuleBlocks[key] = moduleBase; - } - } - - return unchecked(moduleBase + offset); + return SharpEmu.HLE.GuestTlsTemplate.ResolveAddress(ctx, moduleId, offset); } [SysAbiExport( @@ -2768,6 +2332,31 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + /// + /// Invokes the registered runtime thread-destructor callback (set via + /// _sceKernelSetThreadDtors) on the exiting guest thread. C++ + /// runtimes register this to flush per-thread cleanup that would + /// otherwise leak. + /// + public static void RunThreadDtors(CpuContext ctx) + { + var callback = _threadDtorsCallback; + if (callback == 0) + { + return; + } + + _ = GuestThreadExecution.Scheduler?.TryCallGuestFunction( + ctx, + callback, + 0, + 0, + 0, + 0, + "kernel_thread_dtors", + out _); + } + [SysAbiExport( Nid = "Tz4RNUCBbGI", ExportName = "_sceKernelRtldThreadAtexitIncrement", @@ -2960,7 +2549,10 @@ public static class KernelMemoryCompatExports searchStart = 0; } - var align = alignment == 0 ? 0x1000UL : alignment; + // PS5 direct memory is allocated in 16 KiB pages; when the guest does + // not care about alignment, default to that granularity rather than the + // host 4 KiB page so physical offsets stay on true page boundaries. + var align = alignment == 0 ? OrbisPageSize : alignment; ulong selectedAddress; lock (_memoryGate) { @@ -3029,7 +2621,7 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - var effectiveAlignment = alignment == 0 ? 0x1000UL : alignment; + var effectiveAlignment = alignment == 0 ? OrbisPageSize : alignment; ulong aligned; lock (_memoryGate) { @@ -3108,20 +2700,21 @@ public static class KernelMemoryCompatExports { var start = ctx[CpuRegister.Rdi]; var length = ctx[CpuRegister.Rsi]; - if (length == 0) + if (!IsAligned(start, OrbisPageSize) || !IsAligned(length, OrbisPageSize)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + if (length == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + lock (_memoryGate) { - if (!_directAllocations.TryGetValue(start, out var allocation) || allocation.Length != length) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; - } - - _directAllocations.Remove(start); - _nextPhysicalAddress = GetDirectMemoryHighWaterMarkLocked(); + // The unchecked API ignores an unallocated range, matching the + // kernel contract used by guest pool allocators during teardown. + _ = TryReleaseDirectMemoryRangeLocked(start, length); } return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -3137,21 +2730,22 @@ public static class KernelMemoryCompatExports var start = ctx[CpuRegister.Rdi]; var length = ctx[CpuRegister.Rsi]; - if (length == 0) + if (!IsAligned(start, OrbisPageSize) || !IsAligned(length, OrbisPageSize)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + if (length == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + lock (_memoryGate) { - if (!_directAllocations.TryGetValue(start, out var allocation) || - allocation.Length != length) + if (!TryReleaseDirectMemoryRangeLocked(start, length)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - - _directAllocations.Remove(start); - _nextPhysicalAddress = GetDirectMemoryHighWaterMarkLocked(); } return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -3188,7 +2782,7 @@ public static class KernelMemoryCompatExports ulong mappedAddress; lock (_memoryGate) { - var effectiveAlignment = alignment == 0 ? 0x1000UL : alignment; + var effectiveAlignment = alignment == 0 ? OrbisPageSize : alignment; var fixedMapping = (flags & 0x10UL) != 0; var desiredAddress = requestedAddress != 0 ? requestedAddress @@ -3209,7 +2803,34 @@ public static class KernelMemoryCompatExports if (!reserved) { - mappedAddress = 0; + // Rosetta places the x86-64 process stack in the low + // address window used by some fixed PS5 mappings. Do not + // clobber that host memory: relocate the mapping and + // return the actual address through the in/out pointer. + if (OperatingSystem.IsMacOS()) + { + var fallbackAddress = AlignUp( + _nextVirtualAddress == 0 ? DefaultMapSearchBase : _nextVirtualAddress, + effectiveAlignment); + reserved = TryReserveGuestVirtualRange( + ctx, + fallbackAddress, + length, + protection, + effectiveAlignment, + out mappedAddress); + + if (reserved && ShouldTraceDirectMemory()) + { + Console.Error.WriteLine( + $"[LOADER][WARN] map_direct relocated fixed mapping: requested=0x{requestedAddress:X16} mapped=0x{mappedAddress:X16} len=0x{length:X16}"); + } + } + + if (!reserved) + { + mappedAddress = 0; + } } } else @@ -3389,6 +3010,31 @@ public static class KernelMemoryCompatExports return KernelBatchMapCore(ctx, unchecked((int)ctx[CpuRegister.Rcx])); } + [SysAbiExport( + Nid = "yDBwVAolDgg", + ExportName = "sceKernelIsStack", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelIsStack(CpuContext ctx) + { + _ = ctx[CpuRegister.Rdi]; + var startAddress = ctx[CpuRegister.Rsi]; + var endAddress = ctx[CpuRegister.Rdx]; + + // The queried ranges used by libc's VM allocator are ordinary heap + // mappings. The kernel still initializes both outputs when a mapping + // is not a pthread stack; leaving them untouched makes libc consume + // stale stack values and issue invalid fixed-range reservations. + if ((startAddress != 0 && !ctx.TryWriteUInt64(startAddress, 0)) || + (endAddress != 0 && !ctx.TryWriteUInt64(endAddress, 0))) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + [SysAbiExport( Nid = "cQke9UuBQOk", ExportName = "sceKernelMunmap", @@ -3398,27 +3044,45 @@ public static class KernelMemoryCompatExports { var address = ctx[CpuRegister.Rdi]; var length = ctx[CpuRegister.Rsi]; - if (address == 0 || length == 0) + if (address == 0 || length == 0 || ulong.MaxValue - address < length) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + var rangeEnd = address + length; + var physicallyBacked = IsGuestRangeBacked(ctx, address, length); + var removedAny = false; lock (_memoryGate) { - if (!_mappedRegions.TryGetValue(address, out var mappedRegion) || mappedRegion.Length != length) + var removedRegions = _mappedRegions.Values + .Where(region => + region.Address >= address && + region.Address < rangeEnd && + region.Length <= rangeEnd - region.Address) + .ToArray(); + + if (removedRegions.Length == 0 && !physicallyBacked) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - _mappedRegions.Remove(address); - if (mappedRegion.IsFlexible) + foreach (var mappedRegion in removedRegions) { - _allocatedFlexibleBytes = mappedRegion.Length >= _allocatedFlexibleBytes - ? 0 - : _allocatedFlexibleBytes - mappedRegion.Length; + removedAny |= _mappedRegions.Remove(mappedRegion.Address); + if (mappedRegion.IsFlexible) + { + _allocatedFlexibleBytes = mappedRegion.Length >= _allocatedFlexibleBytes + ? 0 + : _allocatedFlexibleBytes - mappedRegion.Length; + } } } + if (physicallyBacked || removedAny) + { + KernelRuntimeCompatExports.RegisterReleasedVirtualRange(address, length); + } + return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -3574,7 +3238,7 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (protectionOut != 0 && !ctx.TryWriteInt32(protectionOut, region.Protection)) + if (protectionOut != 0 && !TryWriteInt32(ctx, protectionOut, region.Protection)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -3613,7 +3277,7 @@ public static class KernelMemoryCompatExports if (!ctx.TryWriteUInt64(infoAddress, block.Start) || !ctx.TryWriteUInt64(infoAddress + sizeof(ulong), block.Start + block.Length) || - !ctx.TryWriteInt32(infoAddress + (sizeof(ulong) * 2), block.MemoryType)) + !TryWriteInt32(ctx, infoAddress + (sizeof(ulong) * 2), block.MemoryType)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -3645,7 +3309,7 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!TryProtectHostRange(ctx, alignedAddress, alignedLength, protection)) + if (!TryProtectHostRange(alignedAddress, alignedLength, protection)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } @@ -3679,7 +3343,7 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!TryProtectHostRange(ctx, alignedAddress, alignedLength, protection)) + if (!TryProtectHostRange(alignedAddress, alignedLength, protection)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } @@ -3755,7 +3419,6 @@ public static class KernelMemoryCompatExports } var rendered = FormatString(ctx, format, ref vaCursor); - return WriteSnprintfOutput(ctx, destination, bufferSize, rendered); } @@ -3828,7 +3491,11 @@ public static class KernelMemoryCompatExports return true; } - internal static bool TryFormatStringFromVaList(CpuContext ctx, string format, ulong vaListAddress, out string rendered) + internal static bool TryFormatStringFromVaList( + CpuContext ctx, + string format, + ulong vaListAddress, + out string rendered) { if (!TryCreateVaListCursor(ctx, vaListAddress, out var vaCursor)) { @@ -4181,19 +3848,48 @@ public static class KernelMemoryCompatExports return FormatStringFromVarArgs(ctx, format, firstGpArgIndex: 3); } + [ThreadStatic] + private static StringBuilder? _formatBuilder; + + // printf length modifier collapsed to the widths our conversions care about. + // Kept as an enum rather than a per-argument substring so the hot format + // loop avoids a string allocation and string comparisons on every spec. + private enum PrintfLength + { + None, + Char, // hh + Short, // h + Long, // l / ll / j / z / t / L + } + private static string FormatString( CpuContext ctx, string format, ref TArgumentSource argumentSource) where TArgumentSource : struct, IPrintfArgumentSource { - var sb = new StringBuilder(format.Length + 32); + // printf formatting is one of the hottest HLE paths during loading, so + // reuse a per-thread builder and append literal runs as spans instead of + // allocating a builder and appending char-by-char on every call. + var sb = _formatBuilder ??= new StringBuilder(256); + sb.Clear(); + if (sb.Capacity < format.Length + 32) + { + sb.EnsureCapacity(format.Length + 32); + } for (var i = 0; i < format.Length; i++) { if (format[i] != '%') { - sb.Append(format[i]); + var literalStart = i; + while (i < format.Length && format[i] != '%') + { + i++; + } + + sb.Append(format.AsSpan(literalStart, i - literalStart)); + i--; continue; } @@ -4267,19 +3963,27 @@ public static class KernelMemoryCompatExports } } - var lengthMod = ""; + var lengthMod = PrintfLength.None; if (i < format.Length) { - if (i + 1 < format.Length && - ((format[i] == 'h' && format[i + 1] == 'h') || - (format[i] == 'l' && format[i + 1] == 'l'))) + if (i + 1 < format.Length && format[i] == 'h' && format[i + 1] == 'h') { - lengthMod = format.Substring(i, 2); + lengthMod = PrintfLength.Char; i += 2; } - else if (format[i] is 'h' or 'l' or 'j' or 'z' or 't' or 'L') + else if (i + 1 < format.Length && format[i] == 'l' && format[i + 1] == 'l') { - lengthMod = format[i].ToString(); + lengthMod = PrintfLength.Long; + i += 2; + } + else if (format[i] == 'h') + { + lengthMod = PrintfLength.Short; + i++; + } + else if (format[i] is 'l' or 'j' or 'z' or 't' or 'L') + { + lengthMod = PrintfLength.Long; i++; } } @@ -4303,17 +4007,13 @@ public static class KernelMemoryCompatExports { long value = lengthMod switch { - "hh" => unchecked((sbyte)argumentSource.NextGpArg()), - "h" => unchecked((short)argumentSource.NextGpArg()), - "l" => unchecked((long)argumentSource.NextGpArg()), - "ll" => unchecked((long)argumentSource.NextGpArg()), - "j" => unchecked((long)argumentSource.NextGpArg()), - "z" => unchecked((long)argumentSource.NextGpArg()), - "t" => unchecked((long)argumentSource.NextGpArg()), + PrintfLength.Char => unchecked((sbyte)argumentSource.NextGpArg()), + PrintfLength.Short => unchecked((short)argumentSource.NextGpArg()), + PrintfLength.Long => unchecked((long)argumentSource.NextGpArg()), _ => unchecked((int)argumentSource.NextGpArg()) }; - var formatted = value.ToString(CultureInfo.InvariantCulture); + var formatted = value.ToString(); if (showSign && value >= 0) formatted = "+" + formatted; else if (spaceForSign && value >= 0) @@ -4327,17 +4027,13 @@ public static class KernelMemoryCompatExports { ulong value = lengthMod switch { - "hh" => (byte)argumentSource.NextGpArg(), - "h" => (ushort)argumentSource.NextGpArg(), - "l" => argumentSource.NextGpArg(), - "ll" => argumentSource.NextGpArg(), - "j" => argumentSource.NextGpArg(), - "z" => argumentSource.NextGpArg(), - "t" => argumentSource.NextGpArg(), + PrintfLength.Char => (byte)argumentSource.NextGpArg(), + PrintfLength.Short => (ushort)argumentSource.NextGpArg(), + PrintfLength.Long => argumentSource.NextGpArg(), _ => (uint)argumentSource.NextGpArg() }; - var formatted = value.ToString(CultureInfo.InvariantCulture); + var formatted = value.ToString(); sb.Append(PadString(formatted, width, leftAlign, padWithZero && !leftAlign)); } break; @@ -4347,19 +4043,15 @@ public static class KernelMemoryCompatExports { ulong value = lengthMod switch { - "hh" => (byte)argumentSource.NextGpArg(), - "h" => (ushort)argumentSource.NextGpArg(), - "l" => argumentSource.NextGpArg(), - "ll" => argumentSource.NextGpArg(), - "j" => argumentSource.NextGpArg(), - "z" => argumentSource.NextGpArg(), - "t" => argumentSource.NextGpArg(), + PrintfLength.Char => (byte)argumentSource.NextGpArg(), + PrintfLength.Short => (ushort)argumentSource.NextGpArg(), + PrintfLength.Long => argumentSource.NextGpArg(), _ => (uint)argumentSource.NextGpArg() }; var formatted = specifier == 'x' - ? value.ToString("x", CultureInfo.InvariantCulture) - : value.ToString("X", CultureInfo.InvariantCulture); + ? value.ToString("x") + : value.ToString("X"); if (alternateForm && value != 0) formatted = specifier == 'x' ? "0x" + formatted : "0X" + formatted; @@ -4372,13 +4064,9 @@ public static class KernelMemoryCompatExports { ulong value = lengthMod switch { - "hh" => (byte)argumentSource.NextGpArg(), - "h" => (ushort)argumentSource.NextGpArg(), - "l" => argumentSource.NextGpArg(), - "ll" => argumentSource.NextGpArg(), - "j" => argumentSource.NextGpArg(), - "z" => argumentSource.NextGpArg(), - "t" => argumentSource.NextGpArg(), + PrintfLength.Char => (byte)argumentSource.NextGpArg(), + PrintfLength.Short => (ushort)argumentSource.NextGpArg(), + PrintfLength.Long => argumentSource.NextGpArg(), _ => (uint)argumentSource.NextGpArg() }; @@ -4403,12 +4091,12 @@ public static class KernelMemoryCompatExports case 's': { var strAddr = argumentSource.NextGpArg(); - TracePrintfStringArgument(ctx, lengthMod, strAddr); + TracePrintfStringArgument(ctx, lengthMod == PrintfLength.Long ? "l" : "", strAddr); if (strAddr == 0) { sb.Append("(null)"); } - else if (lengthMod == "l") + else if (lengthMod == PrintfLength.Long) { if (TryReadWideCString(ctx, strAddr, 1_048_576, out var wideUnits)) { @@ -4439,7 +4127,7 @@ public static class KernelMemoryCompatExports case 'c': { string renderedChar; - if (lengthMod == "l") + if (lengthMod == PrintfLength.Long) { var scalar = unchecked((ushort)argumentSource.NextGpArg()); renderedChar = TryConvertWideScalarToString(scalar, out var wideCharText) @@ -4467,10 +4155,7 @@ public static class KernelMemoryCompatExports var formatStr = precision >= 0 ? $"{{0:{specifier}{precision}}}" : $"{{0:{specifier}}}"; - var formatted = string.Format( - CultureInfo.InvariantCulture, - formatStr, - value); + var formatted = string.Format(formatStr, value); if (showSign && value >= 0) formatted = "+" + formatted; @@ -4486,7 +4171,7 @@ public static class KernelMemoryCompatExports var addr = argumentSource.NextGpArg(); if (addr != 0) { - _ = ctx.TryWriteInt32(addr, sb.Length); + _ = TryWriteInt32(ctx, addr, sb.Length); } } break; @@ -4532,59 +4217,33 @@ public static class KernelMemoryCompatExports private struct RegisterPrintfArgumentSource : IPrintfArgumentSource { - // SysV AMD64: variadic integer/pointer args use the GP registers (rdi..r9, up to - // 6) and variadic float/double args use xmm0..xmm7 (8) — INDEPENDENT counters. - // Args beyond the registers spill to the stack in source order, so GP-overflow - // and FP-overflow share one stack cursor. private readonly CpuContext _ctx; private int _gpIndex; - private int _fpIndex; - private int _stackIndex; public RegisterPrintfArgumentSource(CpuContext ctx, int gpIndex) { _ctx = ctx; _gpIndex = gpIndex; - _fpIndex = 0; - _stackIndex = 0; } public ulong NextGpArg() { - var index = _gpIndex; - if (index < 6) + var index = _gpIndex++; + return index switch { - _gpIndex++; - return index switch - { - 0 => _ctx[CpuRegister.Rdi], - 1 => _ctx[CpuRegister.Rsi], - 2 => _ctx[CpuRegister.Rdx], - 3 => _ctx[CpuRegister.Rcx], - 4 => _ctx[CpuRegister.R8], - _ => _ctx[CpuRegister.R9], - }; - } - - return ReadStackArg(_ctx, (ulong)(_stackIndex++) * 8); + 0 => _ctx[CpuRegister.Rdi], + 1 => _ctx[CpuRegister.Rsi], + 2 => _ctx[CpuRegister.Rdx], + 3 => _ctx[CpuRegister.Rcx], + 4 => _ctx[CpuRegister.R8], + 5 => _ctx[CpuRegister.R9], + _ => ReadStackArg(_ctx, (ulong)(index - 6) * 8) + }; } public double NextFloatArg() { - // Float/double variadic args live in xmm0..xmm7 (low 64 bits), NOT the GP - // registers. Reading them from GP prints garbage and desynchronizes every - // subsequent argument. - ulong bits; - if (_fpIndex < 8) - { - _ctx.GetXmmRegister(_fpIndex++, out bits, out _); - } - else - { - bits = ReadStackArg(_ctx, (ulong)(_stackIndex++) * 8); - } - - return BitConverter.Int64BitsToDouble(unchecked((long)bits)); + return BitConverter.Int64BitsToDouble(unchecked((long)NextGpArg())); } } @@ -4668,7 +4327,7 @@ public static class KernelMemoryCompatExports return 0; } - var effectiveAlignment = alignment == 0 ? 0x1000UL : alignment; + var effectiveAlignment = alignment == 0 ? OrbisPageSize : alignment; if (_nextVirtualAddress == 0) { _nextVirtualAddress = 0x0100_0000UL; @@ -4741,7 +4400,7 @@ public static class KernelMemoryCompatExports out _); } - private static bool IsGuestRangeBacked(CpuContext ctx, ulong address, ulong length) + internal static bool IsGuestRangeBacked(CpuContext ctx, ulong address, ulong length) { if (address == 0 || length == 0 || ulong.MaxValue - address < length - 1) { @@ -5206,6 +4865,7 @@ public static class KernelMemoryCompatExports writer.Advance(firstReadLength); } + Span one = stackalloc byte[1]; while (offset < (ulong)limit) { var current = address + offset; @@ -5237,7 +4897,6 @@ public static class KernelMemoryCompatExports continue; } - var one = chunk.AsSpan(0, 1); if (!TryReadCompat(ctx, current, one)) { return false; @@ -5290,35 +4949,20 @@ public static class KernelMemoryCompatExports private static bool TryReadWideCString(CpuContext ctx, ulong address, ulong maxLength, out ushort[] units) { - units = Array.Empty(); - if (address == 0) - { - return false; - } - - var limit = (int)Math.Min(maxLength, 1_048_576UL); - var buffer = new List(Math.Min(limit, 256)); - for (var i = 0; i < limit; i++) - { - if (!TryReadUInt16Compat(ctx, address + ((ulong)i * WideCharSize), out var unit)) - { - return false; - } - - if (unit == 0) - { - units = buffer.ToArray(); - return true; - } - - buffer.Add(unit); - } - - units = buffer.ToArray(); - return true; + return TryReadWideCStringCore(ctx, address, maxLength, out units, out _); } private static bool TryReadWideCStringBounded(CpuContext ctx, ulong address, ulong maxLength, out ushort[] units, out bool terminated) + { + return TryReadWideCStringCore(ctx, address, maxLength, out units, out terminated); + } + + private static bool TryReadWideCStringCore( + CpuContext ctx, + ulong address, + ulong maxLength, + out ushort[] units, + out bool terminated) { units = Array.Empty(); terminated = false; @@ -5329,21 +4973,45 @@ public static class KernelMemoryCompatExports var limit = (int)Math.Min(maxLength, 1_048_576UL); var buffer = new List(Math.Min(limit, 256)); - for (var i = 0; i < limit; i++) + const int maxReadBytes = 4096; + var readBuffer = GC.AllocateUninitializedArray( + Math.Min(maxReadBytes, Math.Max(WideCharSize, limit * WideCharSize))); + var index = 0; + while (index < limit) { - if (!TryReadUInt16Compat(ctx, address + ((ulong)i * WideCharSize), out var unit)) + var unitAddress = address + ((ulong)index * WideCharSize); + if (unitAddress < address) { return false; } - if (unit == 0) + var remainingBytes = (limit - index) * WideCharSize; + var pageBytesRemaining = maxReadBytes - + (int)(unitAddress & (maxReadBytes - 1)); + var readBytes = Math.Min( + readBuffer.Length, + Math.Min(pageBytesRemaining, remainingBytes)); + readBytes &= ~(WideCharSize - 1); + if (readBytes == 0 || + !TryReadCompat(ctx, unitAddress, readBuffer.AsSpan(0, readBytes))) { - terminated = true; - units = buffer.ToArray(); - return true; + return false; } - buffer.Add(unit); + for (var offset = 0; offset < readBytes; offset += WideCharSize) + { + var unit = BinaryPrimitives.ReadUInt16LittleEndian( + readBuffer.AsSpan(offset, WideCharSize)); + if (unit == 0) + { + terminated = true; + units = buffer.ToArray(); + return true; + } + + buffer.Add(unit); + index++; + } } units = buffer.ToArray(); @@ -5676,7 +5344,7 @@ public static class KernelMemoryCompatExports processedCount++; } - if (processedOutAddress != 0 && !ctx.TryWriteInt32(processedOutAddress, processedCount)) + if (processedOutAddress != 0 && !TryWriteInt32(ctx, processedOutAddress, processedCount)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -5879,40 +5547,42 @@ public static class KernelMemoryCompatExports return alignedLength != 0; } - private static bool TryProtectHostRange(CpuContext ctx, ulong address, ulong length, int orbisProtection) + private static bool TryProtectHostRange(ulong address, ulong length, int orbisProtection) { if (length == 0 || length > nuint.MaxValue) { return false; } - if (!KernelVirtualRangeAllocator.TryResolveAddressSpace(ctx.Memory, out var addressSpace)) + var hostProtection = ResolveHostProtection(orbisProtection); + if (!VirtualProtect((nint)address, (nuint)length, hostProtection, out _)) { return false; } - return addressSpace.TryProtect(address, length, ResolveGuestProtection(orbisProtection)); + return true; } - private static GuestPageProtection ResolveGuestProtection(int orbisProtection) + private static uint ResolveHostProtection(int orbisProtection) { - var protection = GuestPageProtection.None; - if ((orbisProtection & (OrbisProtCpuRead | OrbisProtGpuRead)) != 0) + var read = (orbisProtection & (OrbisProtCpuRead | OrbisProtGpuRead)) != 0; + var write = (orbisProtection & (OrbisProtCpuWrite | OrbisProtGpuWrite)) != 0; + var execute = (orbisProtection & OrbisProtCpuExec) != 0; + + if (execute) { - protection |= GuestPageProtection.Read; + return write + ? HostPageExecuteReadWrite + : read + ? HostPageExecuteRead + : HostPageExecute; } - if ((orbisProtection & (OrbisProtCpuWrite | OrbisProtGpuWrite)) != 0) - { - protection |= GuestPageProtection.Write; - } - - if ((orbisProtection & OrbisProtCpuExec) != 0) - { - protection |= GuestPageProtection.Execute; - } - - return protection; + return write + ? HostPageReadWrite + : read + ? HostPageReadOnly + : HostPageNoAccess; } private static bool TryFindVirtualQueryRegionLocked(ulong queryAddress, bool findNext, out MappedRegion region) @@ -5975,6 +5645,56 @@ public static class KernelMemoryCompatExports return string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_DIRECT_MEMORY"), "1", StringComparison.Ordinal); } + private static bool TryReleaseDirectMemoryRangeLocked(ulong start, ulong length) + { + if (!TryAddU64(start, length, out var releaseEnd)) + { + return false; + } + + DirectAllocation? owner = null; + ulong ownerEnd = 0; + foreach (var allocation in _directAllocations.Values) + { + if (TryAddU64(allocation.Start, allocation.Length, out var allocationEnd) && + start >= allocation.Start && + releaseEnd <= allocationEnd) + { + owner = allocation; + ownerEnd = allocationEnd; + break; + } + } + + if (owner is not { } releasedFrom) + { + return false; + } + + _directAllocations.Remove(releasedFrom.Start); + if (start > releasedFrom.Start) + { + _directAllocations[releasedFrom.Start] = releasedFrom with + { + Length = start - releasedFrom.Start, + }; + } + + if (releaseEnd < ownerEnd) + { + _directAllocations[releaseEnd] = new DirectAllocation( + releaseEnd, + ownerEnd - releaseEnd, + releasedFrom.MemoryType); + } + + _nextPhysicalAddress = GetDirectMemoryHighWaterMarkLocked(); + return true; + } + + private static bool IsAligned(ulong value, ulong alignment) => + alignment != 0 && value % alignment == 0; + private static bool TryAllocateDirectMemoryLocked( ulong searchStart, ulong searchEnd, @@ -5990,7 +5710,7 @@ public static class KernelMemoryCompatExports return false; } - var effectiveAlignment = alignment == 0 ? 0x1000UL : alignment; + var effectiveAlignment = alignment == 0 ? OrbisPageSize : alignment; if (!TryFindAllocatableDirectMemoryRangeLocked(searchStart, searchEnd, length, effectiveAlignment, allocationLimit, out var freePosition) || !TryAddU64(freePosition, length, out var endAddress)) { @@ -6262,26 +5982,6 @@ public static class KernelMemoryCompatExports alignment = NormalizeLibcAlignment(alignment); var actualSize = requestedSize == 0 ? 1u : requestedSize; - // This intentionally uses host guard pages for allocations made through the libc HLE. - // It turns an overrun that reaches the next page into an immediate access violation at - // the offending guest instruction, rather than allowing it to poison a later import. - var guardHeap = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_GUARD_HEAP"), - "1", - StringComparison.Ordinal); - - if (guardHeap && TryAllocateGuardedLibcHeap(actualSize, alignment, zeroFill, out address)) - { - return true; - } - - // Do not silently fall back to an unguarded allocation when guard heap was explicitly - // requested: a failed setup must remain visible to the caller/reproducer. - if (guardHeap) - { - return false; - } - nuint totalSize; try { @@ -6317,7 +6017,7 @@ public static class KernelMemoryCompatExports var alignedAddress = AlignUp(unchecked((ulong)baseAddress) + (ulong)IntPtr.Size, (ulong)alignment); lock (_libcAllocGate) { - _libcAllocations[alignedAddress] = new LibcHeapAllocation(baseAddress, actualSize, alignment, IsGuarded: false); + _libcAllocations[alignedAddress] = new LibcHeapAllocation(baseAddress, actualSize, alignment); } try @@ -6337,51 +6037,6 @@ public static class KernelMemoryCompatExports return true; } - private static unsafe bool TryAllocateGuardedLibcHeap(nuint actualSize, nuint alignment, bool zeroFill, out ulong address) - { - address = 0; - const nuint pageSize = 0x1000; - - try - { - var effectiveAlignment = Math.Max(alignment, pageSize); - var usableSize = checked((nuint)AlignUp((ulong)actualSize, (ulong)pageSize)); - var reservationSize = checked(pageSize + effectiveAlignment - 1 + usableSize + pageSize); - var baseAddress = unchecked((nint)HostMemory.Allocate(0, reservationSize, HostPageProtection.ReadWrite)); - if (baseAddress == 0) - { - return false; - } - - var alignedAddress = AlignUp(unchecked((ulong)baseAddress) + (ulong)pageSize, (ulong)effectiveAlignment); - var guardAddress = alignedAddress + (ulong)usableSize; - if (!HostMemory.Protect(guardAddress, pageSize, HostPageProtection.NoAccess, out _)) - { - _ = HostMemory.Free(unchecked((ulong)baseAddress)); - return false; - } - - lock (_libcAllocGate) - { - _libcAllocations[alignedAddress] = new LibcHeapAllocation(baseAddress, actualSize, alignment, IsGuarded: true); - } - - if (zeroFill) - { - NativeMemory.Clear((void*)alignedAddress, actualSize); - } - - Console.Error.WriteLine( - $"[LOADER][TRACE] guard-heap alloc: ptr=0x{alignedAddress:X16} size=0x{actualSize:X} guard=0x{guardAddress:X16}"); - address = alignedAddress; - return true; - } - catch (OverflowException) - { - return false; - } - } - private static unsafe bool TryReallocateLibcHeap(ulong existingAddress, ulong requestedSize, out ulong resizedAddress) { resizedAddress = 0; @@ -6490,14 +6145,7 @@ public static class KernelMemoryCompatExports } } - if (allocation.IsGuarded) - { - _ = HostMemory.Free(unchecked((ulong)allocation.BaseAddress)); - } - else - { - Marshal.FreeHGlobal(allocation.BaseAddress); - } + Marshal.FreeHGlobal(allocation.BaseAddress); } private static bool TryMultiplyAllocationSize(ulong left, ulong right, out nuint size) @@ -6588,7 +6236,7 @@ public static class KernelMemoryCompatExports return false; } - if (!TryQueryHostPage(address, out var startInfo) || !HasRequiredProtection(startInfo.RawProtection, writeAccess)) + if (!TryQueryHostPage(address, out var startInfo) || !HasRequiredProtection(startInfo.Protect, writeAccess)) { return false; } @@ -6599,7 +6247,7 @@ public static class KernelMemoryCompatExports return true; } - if (!TryQueryHostPage(endAddress, out var endInfo) || !HasRequiredProtection(endInfo.RawProtection, writeAccess)) + if (!TryQueryHostPage(endAddress, out var endInfo) || !HasRequiredProtection(endInfo.Protect, writeAccess)) { return false; } @@ -6607,14 +6255,16 @@ public static class KernelMemoryCompatExports return true; } - private static bool TryQueryHostPage(ulong address, out HostRegionInfo info) + private static bool TryQueryHostPage(ulong address, out MemoryBasicInformation info) { - if (!HostMemory.Query(address, out info)) + info = default; + var size = (nuint)Marshal.SizeOf(); + if (VirtualQuery((nint)address, out info, size) == 0) { return false; } - return info.State == HostRegionState.Committed; + return info.State == MemCommit; } private static bool HasRequiredProtection(uint protect, bool writeAccess) @@ -6630,6 +6280,13 @@ public static class KernelMemoryCompatExports return (protect & expected) != 0; } + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + BitConverter.TryWriteBytes(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + private static bool TryWriteOpenDescriptorStat(CpuContext ctx, int fd, ulong statAddress) { if (fd is 0 or 1 or 2) @@ -6821,19 +6478,19 @@ public static class KernelMemoryCompatExports } var currentIndex = directory.NextIndex; + if (basePointerAddress != 0 && !TryWriteUInt64Compat(ctx, basePointerAddress, (ulong)currentIndex)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + if (currentIndex >= directory.Entries.Length) { - if (basePointerAddress != 0 && - !TryWriteUInt64Compat(ctx, basePointerAddress, (ulong)currentIndex)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } var entryName = directory.Entries[currentIndex]; + directory.NextIndex = currentIndex + 1; var entryBytes = Encoding.UTF8.GetBytes(entryName); var nameLength = Math.Min(entryBytes.Length, 255); @@ -6852,13 +6509,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (basePointerAddress != 0 && - !TryWriteUInt64Compat(ctx, basePointerAddress, (ulong)currentIndex)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - directory.NextIndex = currentIndex + 1; ctx[CpuRegister.Rax] = 512; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -7108,4 +6758,396 @@ public static class KernelMemoryCompatExports sum = left + right; return sum >= left; } + + [SysAbiExport( + Nid = "AV6ipCNa4Rw", + ExportName = "strcasecmp", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strcasecmp(CpuContext ctx) + { + var left = ctx[CpuRegister.Rdi]; + var right = ctx[CpuRegister.Rsi]; + if (left == 0 || right == 0) + { + var recoveryIndex = Interlocked.Increment(ref _nullStrcasecmpRecoveryCount); + if (recoveryIndex <= 16) + { + var otherAddress = left == 0 ? right : left; + var otherText = otherAddress != 0 && TryReadNullTerminatedUtf8(ctx, otherAddress, 256, out var text) + ? text + : ""; + _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnRip); + Console.Error.WriteLine( + $"[LOADER][WARNING] strcasecmp null-arg recovery#{recoveryIndex}: ret=0x{returnRip:X16} left=0x{left:X16} right=0x{right:X16} other=\"{otherText}\""); + } + + // Real strcasecmp(NULL, x) is undefined behaviour and previously crashed inside the + // LLE-routed implementation. Treat it as "not equal" instead so callers doing + // `if (strcasecmp(a, b) == 0)` degrade gracefully rather than taking down the guest. + ctx[CpuRegister.Rax] = left == right ? 0uL : 1uL; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + if (!TryCompareStringsCaseInsensitive(ctx, left, right, limit: ulong.MaxValue, out var compare)) + { + ctx[CpuRegister.Rax] = 1; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = unchecked((ulong)compare); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // sprintf/vsprintf are served from the same HLE formatting engine as snprintf/vsnprintf + // instead of falling through to the game's bundled libc. + [SysAbiExport( + Nid = "tcVi5SivF7Q", + ExportName = "sprintf", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Sprintf(CpuContext ctx) + { + var destination = ctx[CpuRegister.Rdi]; + var formatAddress = ctx[CpuRegister.Rsi]; + + if (!TryReadCString(ctx, formatAddress, 1_048_576, out var formatBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var format = Encoding.UTF8.GetString(formatBytes); + var rendered = FormatStringFromVarArgs(ctx, format, firstGpArgIndex: 2); + return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, rendered); + } + + [SysAbiExport( + Nid = "jbz9I9vkqkk", + ExportName = "vsprintf", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Vsprintf(CpuContext ctx) + { + var destination = ctx[CpuRegister.Rdi]; + var formatAddress = ctx[CpuRegister.Rsi]; + var vaListAddress = ctx[CpuRegister.Rdx]; + + if (!TryReadCString(ctx, formatAddress, 1_048_576, out var formatBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var format = Encoding.UTF8.GetString(formatBytes); + if (!TryCreateVaListCursor(ctx, vaListAddress, out var vaCursor)) + { + return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, formatBytes); + } + + var rendered = FormatString(ctx, format, ref vaCursor); + return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, rendered); + } + + // Was unresolved (returning the 0x80020002 sentinel, then crashing when the guest + // dereferenced it) - the game's own heap instrumentation calls this hook when it + // detects a corrupted/invalid block, not the emulator's allocator, so this is purely + // a diagnostic sink: log what was reported and return success so the caller continues. + [SysAbiExport( + Nid = "al3JzFI9MQ0", + ExportName = "sceLibcInternalHeapErrorReportForGame", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceLibcInternal")] + public static int LibcInternalHeapErrorReportForGame(CpuContext ctx) + { + Console.Error.WriteLine( + $"[LOADER][WARN] sceLibcInternalHeapErrorReportForGame: rdi=0x{ctx[CpuRegister.Rdi]:X16} " + + $"rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} rcx=0x{ctx[CpuRegister.Rcx]:X16}"); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "ob5xAW4ln-0", + ExportName = "strchr", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strchr(CpuContext ctx) + { + var address = ctx[CpuRegister.Rdi]; + var needle = unchecked((byte)ctx[CpuRegister.Rsi]); + if (address == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + // The terminator counts as part of the scanned range, so strchr(s, '\0') + // returns a pointer to the string's null byte just like a native libc. + Span current = stackalloc byte[1]; + for (ulong index = 0; index < 1_048_576; index++) + { + if (!TryReadCompat(ctx, address + index, current)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (current[0] == needle) + { + ctx[CpuRegister.Rax] = address + index; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + if (current[0] == 0) + { + break; + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "9yDWMxEFdJU", + ExportName = "strrchr", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strrchr(CpuContext ctx) + { + var address = ctx[CpuRegister.Rdi]; + var needle = unchecked((byte)ctx[CpuRegister.Rsi]); + if (address == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ulong match = 0; + var found = false; + Span current = stackalloc byte[1]; + for (ulong index = 0; index < 1_048_576; index++) + { + if (!TryReadCompat(ctx, address + index, current)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (current[0] == needle) + { + match = address + index; + found = true; + } + + if (current[0] == 0) + { + break; + } + } + + ctx[CpuRegister.Rax] = found ? match : 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "8u8lPzUEq+U", + ExportName = "memchr", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Memchr(CpuContext ctx) + { + var address = ctx[CpuRegister.Rdi]; + var needle = unchecked((byte)ctx[CpuRegister.Rsi]); + var count = ctx[CpuRegister.Rdx]; + + Span current = stackalloc byte[1]; + for (ulong index = 0; index < count; index++) + { + if (!TryReadCompat(ctx, address + index, current)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (current[0] == needle) + { + ctx[CpuRegister.Rax] = address + index; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "Ls4tzzhimqQ", + ExportName = "strcat", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strcat(CpuContext ctx) + { + var destination = ctx[CpuRegister.Rdi]; + var source = ctx[CpuRegister.Rsi]; + if (!TryReadCString(ctx, source, 1_048_576, out var sourceBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (!TryReadCString(ctx, destination, 1_048_576, out var destinationBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + // Overwrite the destination terminator and re-terminate after the copied bytes. + var appendAddress = destination + (ulong)destinationBytes.Length; + var payload = new byte[sourceBytes.Length + 1]; + sourceBytes.CopyTo(payload.AsSpan()); + if (!TryWriteCompat(ctx, appendAddress, payload)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "kHg45qPC6f0", + ExportName = "strncat", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strncat(CpuContext ctx) + { + var destination = ctx[CpuRegister.Rdi]; + var source = ctx[CpuRegister.Rsi]; + var limit = ctx[CpuRegister.Rdx]; + + // Bounding the source read by the count yields strncat's "at most n bytes" + // semantics while still stopping early at the source terminator. + if (!TryReadCString(ctx, source, limit, out var sourceBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (!TryReadCString(ctx, destination, 1_048_576, out var destinationBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var appendAddress = destination + (ulong)destinationBytes.Length; + var payload = new byte[sourceBytes.Length + 1]; + sourceBytes.CopyTo(payload.AsSpan()); + if (!TryWriteCompat(ctx, appendAddress, payload)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "viiwFMaNamA", + ExportName = "strstr", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strstr(CpuContext ctx) + { + var haystack = ctx[CpuRegister.Rdi]; + var needle = ctx[CpuRegister.Rsi]; + if (!TryReadCString(ctx, haystack, 1_048_576, out var haystackBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (!TryReadCString(ctx, needle, 1_048_576, out var needleBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + // An empty needle matches at the start of the haystack. + if (needleBytes.Length == 0) + { + ctx[CpuRegister.Rax] = haystack; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + var matchIndex = haystackBytes.AsSpan().IndexOf(needleBytes.AsSpan()); + ctx[CpuRegister.Rax] = matchIndex >= 0 ? haystack + (ulong)matchIndex : 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "fgIsQ10xYVA", + ExportName = "sceKernelChmod", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelChmod(CpuContext ctx) + { + var pathAddress = ctx[CpuRegister.Rdi]; + var mode = unchecked((uint)ctx[CpuRegister.Rsi]); + if (pathAddress == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!TryReadNullTerminatedUtf8(ctx, pathAddress, MaxGuestStringLength, out var guestPath)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var hostPath = ResolveGuestPath(guestPath); + if (IsReadOnlyGuestMutationPath(guestPath)) + { + LogOpenTrace($"chmod readonly path='{guestPath}' host='{hostPath}' mode=0x{mode:X}"); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; + } + + if (!File.Exists(hostPath) && !Directory.Exists(hostPath)) + { + AddNegativeStatCacheForGuestPath(guestPath); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + // POSIX permission bits have no host equivalent on Windows; accept the call + // so guests that chmod their freshly created files/directories can proceed. + LogOpenTrace($"chmod path='{guestPath}' host='{hostPath}' mode=0x{mode:X}"); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + private static bool TryCompareStringsCaseInsensitive( + CpuContext ctx, + ulong left, + ulong right, + ulong limit, + out int compare) + { + compare = 0; + Span leftBuffer = stackalloc byte[1]; + Span rightBuffer = stackalloc byte[1]; + for (ulong index = 0; index < limit; index++) + { + if (!TryReadCompat(ctx, left + index, leftBuffer) || + !TryReadCompat(ctx, right + index, rightBuffer)) + { + return false; + } + + var leftValue = leftBuffer[0]; + var rightValue = rightBuffer[0]; + var leftLower = ToAsciiLower(leftValue); + var rightLower = ToAsciiLower(rightValue); + if (leftLower != rightLower) + { + compare = leftLower - rightLower; + return true; + } + + if (leftValue == 0) + { + return true; + } + } + + return true; + } + + private static byte ToAsciiLower(byte value) => + value is >= (byte)'A' and <= (byte)'Z' ? (byte)(value + 32) : value; } diff --git a/src/SharpEmu.Libs/Kernel/KernelModuleRegistry.cs b/src/SharpEmu.Libs/Kernel/KernelModuleRegistry.cs index 1e89323..f4eac33 100644 --- a/src/SharpEmu.Libs/Kernel/KernelModuleRegistry.cs +++ b/src/SharpEmu.Libs/Kernel/KernelModuleRegistry.cs @@ -9,8 +9,16 @@ namespace SharpEmu.Libs.Kernel; public static class KernelModuleRegistry { + public enum ModuleStartState + { + NotStarted, + Starting, + Started, + } + private static readonly object _gate = new(); private static readonly Dictionary _modulesByHandle = new(); + private static readonly Dictionary> _symbolsByHandle = new(); private static readonly Dictionary _handleByPath = new(StringComparer.OrdinalIgnoreCase); private static readonly Dictionary _handleByName = new(StringComparer.OrdinalIgnoreCase); private static readonly Dictionary _sysmoduleHandleById = new(); @@ -32,6 +40,11 @@ public static class KernelModuleRegistry ulong BaseAddress, ulong EndAddress, ulong EntryPoint, + ulong InitEntryPoint, + ulong EhFrameHeaderAddress, + ulong EhFrameAddress, + ulong EhFrameSize, + ModuleStartState StartState, bool IsMain, bool IsSystemModule); @@ -40,6 +53,7 @@ public static class KernelModuleRegistry lock (_gate) { _modulesByHandle.Clear(); + _symbolsByHandle.Clear(); _handleByPath.Clear(); _handleByName.Clear(); _sysmoduleHandleById.Clear(); @@ -52,6 +66,10 @@ public static class KernelModuleRegistry ulong baseAddress, ulong size, ulong entryPoint, + ulong initEntryPoint, + ulong ehFrameHeaderAddress, + ulong ehFrameAddress, + ulong ehFrameSize, bool isMain, bool isSystemModule = false) { @@ -67,6 +85,10 @@ public static class KernelModuleRegistry BaseAddress = baseAddress, EndAddress = ComputeEnd(baseAddress, size), EntryPoint = entryPoint, + InitEntryPoint = initEntryPoint, + EhFrameHeaderAddress = ehFrameHeaderAddress, + EhFrameAddress = ehFrameAddress, + EhFrameSize = ehFrameSize, IsMain = existing.IsMain || isMain, IsSystemModule = existing.IsSystemModule || isSystemModule, }; @@ -84,6 +106,11 @@ public static class KernelModuleRegistry BaseAddress: baseAddress, EndAddress: ComputeEnd(baseAddress, size), EntryPoint: entryPoint, + InitEntryPoint: initEntryPoint, + EhFrameHeaderAddress: ehFrameHeaderAddress, + EhFrameAddress: ehFrameAddress, + EhFrameSize: ehFrameSize, + StartState: ModuleStartState.NotStarted, IsMain: isMain, IsSystemModule: isSystemModule); _modulesByHandle[handle] = entry; @@ -119,6 +146,11 @@ public static class KernelModuleRegistry BaseAddress: 0, EndAddress: 0, EntryPoint: 0, + InitEntryPoint: 0, + EhFrameHeaderAddress: 0, + EhFrameAddress: 0, + EhFrameSize: 0, + StartState: ModuleStartState.Started, IsMain: false, IsSystemModule: isSystemModule); _modulesByHandle[handle] = entry; @@ -203,6 +235,97 @@ public static class KernelModuleRegistry } } + /// + /// Atomically claims a module initializer. A module can be observed while + /// it is starting (for recursive loader calls), but its DT_INIT routine is + /// executed at most once after a successful start. + /// + public static bool TryBeginModuleStart(int handle, out ModuleEntry module) + { + lock (_gate) + { + if (!_modulesByHandle.TryGetValue(handle, out module)) + { + return false; + } + + if (module.StartState != ModuleStartState.NotStarted) + { + return false; + } + + if (module.InitEntryPoint < 0x10000) + { + module = module with { StartState = ModuleStartState.Started }; + _modulesByHandle[handle] = module; + return false; + } + + module = module with { StartState = ModuleStartState.Starting }; + _modulesByHandle[handle] = module; + return true; + } + } + + public static void CompleteModuleStart(int handle, bool succeeded) + { + lock (_gate) + { + if (!_modulesByHandle.TryGetValue(handle, out var module) || + module.StartState != ModuleStartState.Starting) + { + return; + } + + _modulesByHandle[handle] = module with + { + StartState = succeeded ? ModuleStartState.Started : ModuleStartState.NotStarted, + }; + } + } + + public static void RegisterModuleSymbols(int handle, IReadOnlyDictionary symbols) + { + ArgumentNullException.ThrowIfNull(symbols); + lock (_gate) + { + if (!_modulesByHandle.ContainsKey(handle)) + { + return; + } + + if (!_symbolsByHandle.TryGetValue(handle, out var destination)) + { + destination = new Dictionary(StringComparer.Ordinal); + _symbolsByHandle[handle] = destination; + } + + foreach (var (name, address) in symbols) + { + if (!string.IsNullOrWhiteSpace(name) && address >= 0x10000) + { + destination.TryAdd(name, address); + } + } + } + } + + public static bool TryResolveModuleSymbol(int handle, string symbolName, out ulong address) + { + address = 0; + if (string.IsNullOrWhiteSpace(symbolName)) + { + return false; + } + + lock (_gate) + { + return _symbolsByHandle.TryGetValue(handle, out var symbols) && + symbols.TryGetValue(symbolName, out address) && + address >= 0x10000; + } + } + public static bool TryFindByPathOrName(string? modulePathOrName, out ModuleEntry module) { module = default; diff --git a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs index c2aa476..1d8fc45 100644 --- a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs @@ -3,7 +3,9 @@ using System.Collections.Concurrent; using SharpEmu.HLE; -using SharpEmu.Libs.Fiber; +using System.Buffers.Binary; +using System.Diagnostics; +using System.Threading; using System.Diagnostics.CodeAnalysis; namespace SharpEmu.Libs.Kernel; @@ -18,15 +20,12 @@ public static class KernelPthreadCompatExports private const int MutexObjectSize = 0x100; private const int MutexAttrObjectSize = 0x40; private const int CondObjectSize = 0x100; - private const int DefaultSpuriousCondWakeMilliseconds = 1; private const int PthreadOnceUninitialized = 0; private const int PthreadOnceInProgress = 1; private const int PthreadOnceDone = 2; private static readonly object _stateGate = new(); private static readonly ConcurrentDictionary _mutexStates = new(); - private static readonly ConcurrentDictionary _mutexWakeKeys = new(); - private static readonly ConcurrentDictionary _condWakeKeys = new(); private static readonly Dictionary _mutexAttrStates = new(); private static readonly Dictionary _condStates = new(); private static readonly Dictionary _onceGates = new(); @@ -38,71 +37,55 @@ public static class KernelPthreadCompatExports string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_PTHREAD_CONDS"), "1", StringComparison.Ordinal); private static readonly HashSet? _tracePthreadMutexFilter = ParseTraceAddressFilter( Environment.GetEnvironmentVariable("SHARPEMU_LOG_PTHREAD_MUTEX_FILTER")); - private static readonly bool _enableMutexLockBlocking = - string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_MUTEX_LOCK_BLOCKING"), "1", StringComparison.Ordinal); - - // On the outer class deliberately: a static on the nested state classes gives them a - // type initializer that first runs on a guest thread and fail-fasts the CLR. - private static long _nextMutexWakeId; - private static long _nextCondWakeId; - private static readonly ConcurrentDictionary _posixTimedWaitCounts = new(); - private static readonly ConcurrentDictionary _reportedContendedMutexes = new(); + private static long _nextSynchronizationWaiterId; private sealed class PthreadMutexState { - public SemaphoreSlim Semaphore { get; } = new(1, 1); public ulong OwnerThreadId { get; set; } public int RecursionCount { get; set; } public int Type { get; set; } = MutexTypeErrorCheck; public int Protocol { get; set; } - - // Keyed on state identity, not the resolved address (which can differ between a - // lock and its unlock for the same state, stranding the waiter). - public string WakeKey { get; } = "pthread_mutex#" + Interlocked.Increment(ref _nextMutexWakeId).ToString("X"); + public LinkedList Waiters { get; } = new(); } - private sealed class PthreadMutexWaiter : IGuestThreadBlockWaiter + private sealed class PthreadMutexWaiter { public required ulong ThreadId { get; init; } - public required CpuContext Ctx { get; init; } - public required ulong MutexAddress { get; init; } - public required ulong ResolvedAddress { get; init; } - public required PthreadMutexState State { get; init; } - public int Reserved; - - public int Resume() => CompleteBlockedMutexLock(Ctx, MutexAddress, ResolvedAddress, State, this); - - public bool TryWake() => TryReserveBlockedMutexLock(Ctx, MutexAddress, ResolvedAddress, State, this); - } - - private sealed class PthreadCondWaiter : IGuestThreadBlockWaiter - { - public required CpuContext Ctx { get; init; } - public required ulong CondAddress { get; init; } - public required ulong MutexAddress { get; init; } - public required PthreadCondState State { get; init; } - public required ulong ObservedEpoch { get; init; } - public required bool Timed { get; init; } - public required int ReleasedRecursion { get; init; } - public required bool PosixResult { get; init; } - - public int Resume() => ResumePthreadCondWait(Ctx, CondAddress, MutexAddress, State, ObservedEpoch, Timed, ReleasedRecursion, PosixResult); - - public bool TryWake() => State.SignalEpoch != ObservedEpoch; + public required string WakeKey { get; init; } + public required bool Cooperative { get; init; } + public LinkedListNode? Node { get; set; } + public int Granted; } private sealed class PthreadCondState { public object SyncRoot { get; } = new(); + public LinkedList WaiterQueue { get; } = new(); public ulong SignalEpoch { get; set; } public int Waiters { get; set; } + } - // See PthreadMutexState.WakeKey. - public string WakeKey { get; } = "pthread_cond#" + Interlocked.Increment(ref _nextCondWakeId).ToString("X"); + private sealed class PthreadCondWaiter + { + public required ulong ThreadId { get; init; } + public required PthreadMutexState MutexState { get; init; } + public required string WakeKey { get; init; } + public required bool Cooperative { get; init; } + public bool PosixErrors { get; init; } + public LinkedListNode? Node { get; set; } + public PthreadMutexWaiter? MutexWaiter { get; set; } + public Timer? TimeoutTimer { get; set; } + // 0 = waiting, 1 = signaled, 2 = timed out. + public int CompletionState { get; set; } } private readonly record struct PthreadMutexAttrState(int Type, int Protocol); + static KernelPthreadCompatExports() + { + RunSynchronizationSelfChecks(); + } + [SysAbiExport( Nid = "aI+OeCz8xrQ", ExportName = "scePthreadSelf", @@ -111,6 +94,7 @@ public static class KernelPthreadCompatExports public static int PthreadSelf(CpuContext ctx) { var currentThreadHandle = KernelPthreadState.GetCurrentThreadHandle(); + GuestThreadExecution.Scheduler?.RegisterGuestThreadContext(currentThreadHandle, ctx); ctx[CpuRegister.Rax] = currentThreadHandle; TracePthreadSelf(ctx, currentThreadHandle); return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -150,7 +134,7 @@ public static class KernelPthreadCompatExports LibraryName = "libKernel")] public static int PthreadYield(CpuContext ctx) { - GuestThreadExecution.Scheduler?.Pump(ctx, "scePthreadYield"); + _ = ctx; Thread.Yield(); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -165,10 +149,13 @@ public static class KernelPthreadCompatExports if (_tracePthreads) { var nameAddress = ctx[CpuRegister.Rsi]; - var name = nameAddress != 0 && - KernelMemoryCompatExports.TryReadNullTerminatedUtf8(ctx, nameAddress, 64, out var value) - ? value - : ""; + Span nameBytes = stackalloc byte[64]; + var name = ""; + if (nameAddress != 0 && ctx.Memory.TryRead(nameAddress, nameBytes)) + { + var length = nameBytes.IndexOf((byte)0); + name = System.Text.Encoding.UTF8.GetString(length >= 0 ? nameBytes[..length] : nameBytes); + } Console.Error.WriteLine( $"[LOADER][TRACE] pthread.rename thread=0x{ctx[CpuRegister.Rdi]:X16} name=\"{name}\""); } @@ -343,6 +330,13 @@ public static class KernelPthreadCompatExports LibraryName = "libKernel")] public static int PthreadCondDestroy(CpuContext ctx) => PthreadCondDestroyCore(ctx, ctx[CpuRegister.Rdi]); + [SysAbiExport( + Nid = "RXXqi4CtF8w", + ExportName = "pthread_cond_destroy", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixPthreadCondDestroy(CpuContext ctx) => PthreadCondDestroyCore(ctx, ctx[CpuRegister.Rdi]); + [SysAbiExport( Nid = "WKAXJ4XBPQ4", ExportName = "scePthreadCondWait", @@ -357,104 +351,6 @@ public static class KernelPthreadCompatExports LibraryName = "libKernel")] public static int PthreadCondTimedwait(CpuContext ctx) => PthreadCondWaitCore(ctx, ctx[CpuRegister.Rdi], ctx[CpuRegister.Rsi], timed: true, timeoutUsec: unchecked((uint)ctx[CpuRegister.Rdx])); - [SysAbiExport( - Nid = "27bAgiJmOh0", - ExportName = "pthread_cond_timedwait", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int PosixPthreadCondTimedwait(CpuContext ctx) - { - var absoluteTimeAddress = ctx[CpuRegister.Rdx]; - if (absoluteTimeAddress == 0 || - !ctx.TryReadUInt64(absoluteTimeAddress, out var secondsRaw) || - !ctx.TryReadUInt64(absoluteTimeAddress + sizeof(ulong), out var nanosecondsRaw)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var deadlineSeconds = unchecked((long)secondsRaw); - var deadlineNanoseconds = unchecked((long)nanosecondsRaw); - if (deadlineSeconds < 0 || deadlineNanoseconds is < 0 or >= 1_000_000_000) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - long nowSeconds; - long nowNanoseconds; - if (deadlineSeconds >= 1_000_000_000) - { - var now = DateTimeOffset.UtcNow; - nowSeconds = now.ToUnixTimeSeconds(); - nowNanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100; - } - else - { - // Engine condition variables use monotonic deadlines. - KernelRuntimeCompatExports.GetProcessMonotonicTime(out nowSeconds, out nowNanoseconds); - } - - var remainingNanoseconds = - ((Int128)deadlineSeconds * 1_000_000_000 + deadlineNanoseconds) - - ((Int128)nowSeconds * 1_000_000_000 + nowNanoseconds); - var timeoutUsec = remainingNanoseconds <= 0 - ? 0u - : (uint)Int128.Min(uint.MaxValue, (remainingNanoseconds + 999) / 1_000); - - TracePosixTimedWait( - ctx, - deadlineSeconds, - deadlineNanoseconds, - nowSeconds, - nowNanoseconds, - remainingNanoseconds, - timeoutUsec); - - return PthreadCondWaitCore( - ctx, - ctx[CpuRegister.Rdi], - ctx[CpuRegister.Rsi], - timed: true, - timeoutUsec, - posixResult: true); - } - - private static void TracePosixTimedWait( - CpuContext ctx, - long deadlineSeconds, - long deadlineNanoseconds, - long nowSeconds, - long nowNanoseconds, - Int128 remainingNanoseconds, - uint timeoutUsec) - { - var condAddress = ctx[CpuRegister.Rdi]; - var count = _posixTimedWaitCounts.AddOrUpdate(condAddress, 1, static (_, current) => current + 1); - if (count != 1 && count % 1_000_000 != 0) - { - return; - } - - _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnRip); - Console.Error.WriteLine( - $"[LOADER][DIAG] pthread_cond_timedwait#{count}: thread=0x{KernelPthreadState.GetCurrentThreadHandle():X16} " + - $"cond=0x{condAddress:X16} mutex=0x{ctx[CpuRegister.Rsi]:X16} abstime=0x{ctx[CpuRegister.Rdx]:X16} " + - $"deadline={deadlineSeconds}.{deadlineNanoseconds:D9} now={nowSeconds}.{nowNanoseconds:D9} " + - $"remaining_ns={remainingNanoseconds} timeout_us={timeoutUsec} ret=0x{returnRip:X16}"); - - if (GuestThreadExecution.Scheduler is not { } scheduler) - { - return; - } - - foreach (var snapshot in scheduler.SnapshotThreads()) - { - Console.Error.WriteLine( - $"[LOADER][DIAG] guest_thread handle=0x{snapshot.ThreadHandle:X16} name='{snapshot.Name}' " + - $"state={snapshot.State} imports={snapshot.ImportCount} nid={snapshot.LastImportNid ?? "none"} " + - $"ret=0x{snapshot.LastReturnRip:X16} block={snapshot.BlockReason ?? "none"}"); - } - } - [SysAbiExport( Nid = "kDh-NfxgMtE", ExportName = "scePthreadCondSignal", @@ -476,6 +372,62 @@ public static class KernelPthreadCompatExports LibraryName = "libKernel")] public static int PosixPthreadCondWait(CpuContext ctx) => PthreadCondWaitCore(ctx, ctx[CpuRegister.Rdi], ctx[CpuRegister.Rsi], timed: false); + [SysAbiExport( + Nid = "27bAgiJmOh0", + ExportName = "pthread_cond_timedwait", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixPthreadCondTimedwait(CpuContext ctx) + { + var deadlineAddress = ctx[CpuRegister.Rdx]; + if (deadlineAddress == 0 || + !KernelMemoryCompatExports.TryReadUInt64Compat(ctx, deadlineAddress, out var rawSeconds) || + !KernelMemoryCompatExports.TryReadUInt64Compat( + ctx, + deadlineAddress + sizeof(long), + out var rawNanoseconds)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var seconds = unchecked((long)rawSeconds); + var nanoseconds = unchecked((long)rawNanoseconds); + if (seconds < 0 || nanoseconds is < 0 or >= 1_000_000_000) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + var now = DateTimeOffset.UtcNow; + var deltaSeconds = seconds - now.ToUnixTimeSeconds(); + var nowNanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100L; + uint timeoutUsec; + if (deltaSeconds < 0) + { + timeoutUsec = 0; + } + else if (deltaSeconds > uint.MaxValue / 1_000_000L + 1) + { + timeoutUsec = uint.MaxValue; + } + else + { + var remainingNanoseconds = + deltaSeconds * 1_000_000_000L + nanoseconds - nowNanoseconds; + var remainingUsec = remainingNanoseconds <= 0 + ? 0 + : (remainingNanoseconds + 999L) / 1_000L; + timeoutUsec = (uint)Math.Min(remainingUsec, uint.MaxValue); + } + + return PthreadCondWaitCore( + ctx, + ctx[CpuRegister.Rdi], + ctx[CpuRegister.Rsi], + timed: true, + timeoutUsec, + posixErrors: true); + } + [SysAbiExport( Nid = "mkx2fVhNMsg", ExportName = "pthread_cond_broadcast", @@ -543,42 +495,42 @@ public static class KernelPthreadCompatExports var initRoutine = ctx[CpuRegister.Rsi]; if (onceAddress == 0 || initRoutine == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadInt32(onceAddress, out var onceValue)) + if (!TryReadInt32(ctx, onceAddress, out var onceValue)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (onceValue == PthreadOnceDone) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } var gate = GetPthreadOnceGate(onceAddress); var shouldCall = false; lock (gate) { - if (!ctx.TryReadInt32(onceAddress, out onceValue)) + if (!TryReadInt32(ctx, onceAddress, out onceValue)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } while (onceValue == PthreadOnceInProgress) { Monitor.Wait(gate, TimeSpan.FromMilliseconds(1)); - if (!ctx.TryReadInt32(onceAddress, out onceValue)) + if (!TryReadInt32(ctx, onceAddress, out onceValue)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } if (onceValue != PthreadOnceDone) { - if (!ctx.TryWriteInt32(onceAddress, PthreadOnceInProgress)) + if (!TryWriteInt32(ctx, onceAddress, PthreadOnceInProgress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } shouldCall = true; @@ -594,21 +546,21 @@ public static class KernelPthreadCompatExports { lock (gate) { - _ = ctx.TryWriteInt32(onceAddress, PthreadOnceUninitialized); + _ = TryWriteInt32(ctx, onceAddress, PthreadOnceUninitialized); Monitor.PulseAll(gate); } TracePthreadOnce(onceAddress, initRoutine, "failed", error); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); } lock (gate) { - if (!ctx.TryWriteInt32(onceAddress, PthreadOnceDone)) + if (!TryWriteInt32(ctx, onceAddress, PthreadOnceDone)) { - _ = ctx.TryWriteInt32(onceAddress, PthreadOnceUninitialized); + _ = TryWriteInt32(ctx, onceAddress, PthreadOnceUninitialized); Monitor.PulseAll(gate); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } Monitor.PulseAll(gate); @@ -616,7 +568,7 @@ public static class KernelPthreadCompatExports } TracePthreadOnce(onceAddress, initRoutine, shouldCall ? "call" : "done", null); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } private static int PthreadMutexInitCore(CpuContext ctx, ulong mutexAddress, ulong attrAddress) @@ -639,8 +591,6 @@ public static class KernelPthreadCompatExports } if (!InitializeMutexObject(ctx, handle, state)) { - TryFreeOpaqueObject(ctx, handle); - state.Semaphore.Dispose(); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -652,8 +602,6 @@ public static class KernelPthreadCompatExports _mutexStates.TryRemove(mutexAddress, out _); _mutexStates.TryRemove(handle, out _); - TryFreeOpaqueObject(ctx, handle); - state.Semaphore.Dispose(); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -668,20 +616,26 @@ public static class KernelPthreadCompatExports } var resolvedAddress = ResolveMutexHandle(ctx, mutexAddress); - _mutexStates.TryRemove(resolvedAddress, out var state); - if (resolvedAddress != mutexAddress) - { - _mutexStates.TryRemove(mutexAddress, out _); - } - - if (state is null) + if (!_mutexStates.TryGetValue(resolvedAddress, out var state)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } + lock (state) + { + if (state.OwnerThreadId != 0 || state.RecursionCount != 0 || state.Waiters.Count != 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; + } + + _mutexStates.TryRemove(resolvedAddress, out _); + if (resolvedAddress != mutexAddress) + { + _mutexStates.TryRemove(mutexAddress, out _); + } + } + _ = KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, mutexAddress, 0); - TryFreeOpaqueObject(ctx, resolvedAddress); - state.Semaphore.Dispose(); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -699,6 +653,10 @@ public static class KernelPthreadCompatExports } var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); + var canCooperativelyBlock = !tryOnly && + GuestThreadExecution.IsGuestThread && + GuestThreadExecution.TryGetCurrentImportCallFrame(out _); + PthreadMutexWaiter? waiter = null; lock (state) { if (state.OwnerThreadId == currentThreadId) @@ -718,9 +676,10 @@ public static class KernelPthreadCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; } - state.RecursionCount++; - TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + // FreeBSD maps NORMAL to checked non-recursive behavior: + // self-lock is an error, never implicit recursion. + TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK; } else { @@ -731,105 +690,39 @@ public static class KernelPthreadCompatExports return ownedResult; } } - } - var acquired = state.Semaphore.Wait(0); - if (!acquired) - { - TraceContendedMutex(ctx, mutexAddress, resolvedAddress, state, currentThreadId); - // Fibers retain the synchronous fallback to preserve switch state. - var currentFiber = FiberExports.GetCurrentFiberAddressForDiagnostics(ctx); - var canCooperativelyBlock = _enableMutexLockBlocking || currentFiber == 0; - if (canCooperativelyBlock && - !tryOnly && - GuestThreadExecution.IsGuestThread && - GuestThreadExecution.TryGetCurrentImportCallFrame(out _) && - GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - "pthread_mutex_lock", - state.WakeKey, - new PthreadMutexWaiter - { - ThreadId = currentThreadId, - Ctx = ctx, - MutexAddress = mutexAddress, - ResolvedAddress = resolvedAddress, - State = state, - })) + if (state.OwnerThreadId == 0 && state.Waiters.Count == 0) { - TracePthreadMutex(ctx, "lock-block", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); + state.OwnerThreadId = currentThreadId; + state.RecursionCount = 1; + TracePthreadMutex(ctx, tryOnly ? "trylock" : "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (!tryOnly) + if (tryOnly) { - state.Semaphore.Wait(); - acquired = true; - } - } - - if (!acquired) - { - TracePthreadMutex(ctx, tryOnly ? "trylock" : "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; - } - - lock (state) - { - DetectMutexDoubleOwner(resolvedAddress, state, currentThreadId, "lock"); - state.OwnerThreadId = currentThreadId; - state.RecursionCount = 1; - } - - TracePthreadMutex(ctx, tryOnly ? "trylock" : "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - private static void TraceContendedMutex( - CpuContext ctx, - ulong mutexAddress, - ulong resolvedAddress, - PthreadMutexState state, - ulong currentThreadId) - { - if (!_reportedContendedMutexes.TryAdd(resolvedAddress, 0)) - { - return; - } - - ulong ownerThreadId; - int recursionCount; - int type; - lock (state) - { - ownerThreadId = state.OwnerThreadId; - recursionCount = state.RecursionCount; - type = state.Type; - } - - _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnRip); - Console.Error.WriteLine( - $"[LOADER][DIAG] pthread_mutex_contended: waiter=0x{currentThreadId:X16} owner=0x{ownerThreadId:X16} " + - $"mutex=0x{mutexAddress:X16} resolved=0x{resolvedAddress:X16} recursion={recursionCount} " + - $"type={type} semaphore_count={state.Semaphore.CurrentCount} ret=0x{returnRip:X16}"); - - if (GuestThreadExecution.Scheduler is not { } scheduler) - { - return; - } - - foreach (var snapshot in scheduler.SnapshotThreads()) - { - if (snapshot.ThreadHandle != currentThreadId && snapshot.ThreadHandle != ownerThreadId) - { - continue; + TracePthreadMutex(ctx, "trylock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; } - Console.Error.WriteLine( - $"[LOADER][DIAG] pthread_mutex_party handle=0x{snapshot.ThreadHandle:X16} name='{snapshot.Name}' " + - $"state={snapshot.State} imports={snapshot.ImportCount} nid={snapshot.LastImportNid ?? "none"} " + - $"ret=0x{snapshot.LastReturnRip:X16} block={snapshot.BlockReason ?? "none"}"); + waiter = EnqueueMutexWaiterLocked(state, currentThreadId, canCooperativelyBlock); } + + if (canCooperativelyBlock && waiter is not null && + GuestThreadExecution.RequestCurrentThreadBlock( + ctx, + "pthread_mutex_lock", + waiter.WakeKey, + () => CompleteBlockedMutexLock(ctx, mutexAddress, resolvedAddress, state, waiter), + () => TryGrantBlockedMutexLock(ctx, mutexAddress, resolvedAddress, state, waiter))) + { + TracePthreadMutex(ctx, "lock-block", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + var hostResult = WaitForHostMutexLock(state, waiter!); + TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, hostResult); + return hostResult; } private static int PthreadMutexUnlockCore(CpuContext ctx, ulong mutexAddress, bool requireOwner) @@ -846,24 +739,16 @@ public static class KernelPthreadCompatExports } var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); - var lenientOwnership = state.Type is MutexTypeNormal or MutexTypeAdaptiveNp; - - var shouldRelease = false; + string? nextWakeKey = null; lock (state) { if (state.RecursionCount <= 0) { - if (lenientOwnership) - { - TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (requireOwner && !lenientOwnership && state.OwnerThreadId != currentThreadId) + if (requireOwner && state.OwnerThreadId != currentThreadId) { TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; @@ -873,25 +758,16 @@ public static class KernelPthreadCompatExports if (state.RecursionCount == 0) { state.OwnerThreadId = 0; - shouldRelease = true; + nextWakeKey = state.Waiters.First?.Value.Cooperative == true + ? state.Waiters.First.Value.WakeKey + : null; + Monitor.PulseAll(state); } } - if (shouldRelease) + if (nextWakeKey is not null) { - try - { - state.Semaphore.Release(); - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(state.WakeKey, 1); - } - catch (SemaphoreFullException) - { - if (!lenientOwnership) - { - TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; - } - } + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(nextWakeKey, 1); } TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); @@ -913,7 +789,6 @@ public static class KernelPthreadCompatExports var initialState = new PthreadMutexAttrState(MutexTypeErrorCheck, 0); if (!WriteMutexAttrObject(ctx, handle, initialState)) { - TryFreeOpaqueObject(ctx, handle); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -931,7 +806,6 @@ public static class KernelPthreadCompatExports _mutexAttrStates.Remove(handle); } - TryFreeOpaqueObject(ctx, handle); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -955,7 +829,6 @@ public static class KernelPthreadCompatExports } } - TryFreeOpaqueObject(ctx, resolvedAddress); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1120,7 +993,7 @@ public static class KernelPthreadCompatExports { if (attrAddress == 0) { - return default; + return new PthreadMutexAttrState(MutexTypeErrorCheck, 0); } var resolvedAddress = ResolveMutexAttrHandle(ctx, attrAddress); @@ -1214,14 +1087,6 @@ public static class KernelPthreadCompatExports lock (_stateGate) { - if (_condStates.TryGetValue(condAddress, out var raced)) - { - TryFreeOpaqueObject(ctx, handle); - resolvedAddress = condAddress; - state = raced; - return true; - } - _condStates[condAddress] = createdState; _condStates[handle] = createdState; } @@ -1234,7 +1099,6 @@ public static class KernelPthreadCompatExports _condStates.Remove(handle); } - TryFreeOpaqueObject(ctx, handle); return false; } @@ -1254,31 +1118,23 @@ public static class KernelPthreadCompatExports Span initialData = stackalloc byte[size]; initialData.Clear(); - if (ctx.Memory.TryWrite(address, initialData)) - { - return true; - } - - allocator.TryFreeGuestMemory(address); - address = 0; - return false; - } - - private static void TryFreeOpaqueObject(CpuContext ctx, ulong address) - { - if (ctx.Memory is IGuestMemoryAllocator allocator) - { - allocator.TryFreeGuestMemory(address); - } + return ctx.Memory.TryWrite(address, initialData); } private static bool InitializeMutexObject(CpuContext ctx, ulong address, PthreadMutexState state) => - ctx.TryWriteUInt32(address + 0x20, unchecked((uint)state.Type)) && - ctx.TryWriteUInt32(address + 0x3C, unchecked((uint)state.Protocol)); + TryWriteUInt32(ctx, address + 0x20, unchecked((uint)state.Type)) && + TryWriteUInt32(ctx, address + 0x3C, unchecked((uint)state.Protocol)); private static bool WriteMutexAttrObject(CpuContext ctx, ulong address, PthreadMutexAttrState state) => - ctx.TryWriteUInt32(address, unchecked((uint)state.Type)) && - ctx.TryWriteUInt32(address + 4, unchecked((uint)state.Protocol)); + TryWriteUInt32(ctx, address, unchecked((uint)state.Type)) && + TryWriteUInt32(ctx, address + 4, unchecked((uint)state.Protocol)); + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + BitConverter.TryWriteBytes(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } private static int PthreadCondInitCore(CpuContext ctx, ulong condAddress) { @@ -1307,7 +1163,6 @@ public static class KernelPthreadCompatExports _condStates.Remove(handle); } - TryFreeOpaqueObject(ctx, handle); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1324,6 +1179,19 @@ public static class KernelPthreadCompatExports var resolvedAddress = ResolveCondHandle(ctx, condAddress); lock (_stateGate) { + if (!_condStates.TryGetValue(resolvedAddress, out var state)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + lock (state.SyncRoot) + { + if (state.WaiterQueue.Count != 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; + } + } + _condStates.Remove(resolvedAddress); if (resolvedAddress != condAddress) { @@ -1332,7 +1200,6 @@ public static class KernelPthreadCompatExports } _ = KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, condAddress, 0); - TryFreeOpaqueObject(ctx, resolvedAddress); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1342,141 +1209,123 @@ public static class KernelPthreadCompatExports ulong mutexAddress, bool timed, uint timeoutUsec = 0, - bool posixResult = false) + bool posixErrors = false) { if (condAddress == 0 || mutexAddress == 0) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!TryResolveCondState(ctx, condAddress, createIfZero: true, out var resolvedCondAddress, out var state)) + if (!TryResolveCondState(ctx, condAddress, createIfZero: true, out _, out var state)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - var waitResult = (int)OrbisGen2Result.ORBIS_GEN2_OK; - var spuriousWake = false; - var releasedRecursion = 0; + if (!TryResolveMutexState(ctx, mutexAddress, createIfZero: true, out _, out var mutexState)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); + lock (mutexState) + { + if (mutexState.OwnerThreadId != currentThreadId || mutexState.RecursionCount != 1) + { + return mutexState.OwnerThreadId == currentThreadId + ? (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT + : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; + } + } + + var cooperative = GuestThreadExecution.IsGuestThread && + GuestThreadExecution.TryGetCurrentImportCallFrame(out _); + var waiter = new PthreadCondWaiter + { + ThreadId = currentThreadId, + MutexState = mutexState, + Cooperative = cooperative, + PosixErrors = posixErrors, + WakeKey = cooperative + ? $"pthread_cond_waiter:{Interlocked.Increment(ref _nextSynchronizationWaiterId)}" + : string.Empty, + }; + lock (state.SyncRoot) { + waiter.Node = state.WaiterQueue.AddLast(waiter); state.Waiters++; - var observedEpoch = state.SignalEpoch; - TracePthreadCond( - "wait-enter", - condAddress, - mutexAddress, - state, - timed, - waitResult); + TracePthreadCond("wait-enter", condAddress, mutexAddress, state, timed, (int)OrbisGen2Result.ORBIS_GEN2_OK); - var unlockResult = PthreadMutexFullUnlockForCondWait(ctx, mutexAddress, out releasedRecursion); + var unlockResult = PthreadMutexUnlockCore(ctx, mutexAddress, requireOwner: true); if (unlockResult != (int)OrbisGen2Result.ORBIS_GEN2_OK) { - state.Waiters--; + RemoveCondWaiterLocked(state, waiter); TracePthreadCond("wait-unlock-fail", condAddress, mutexAddress, state, timed, unlockResult); return unlockResult; } - var scheduler = GuestThreadExecution.Scheduler; - if (GuestThreadExecution.IsGuestThread && - GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - timed ? "pthread_cond_timedwait" : "pthread_cond_wait", - state.WakeKey, - new PthreadCondWaiter + if (cooperative && timed) + { + waiter.TimeoutTimer = new Timer( + static callbackState => { - Ctx = ctx, - CondAddress = condAddress, - MutexAddress = mutexAddress, - State = state, - ObservedEpoch = observedEpoch, - Timed = timed, - ReleasedRecursion = releasedRecursion, - PosixResult = posixResult, + var (condState, condWaiter) = ((PthreadCondState, PthreadCondWaiter))callbackState!; + CompleteCondWaiter(condState, condWaiter, timedOut: true); }, - timed ? GuestThreadExecution.ComputeDeadlineTimestamp(GetCondWaitTimeout(timeoutUsec)) : 0)) - { - TracePthreadCond(timed ? "wait-block-timed" : "wait-block", condAddress, mutexAddress, state, timed, waitResult); - return waitResult; + (state, waiter), + GetCondWaitTimeout(timeoutUsec), + Timeout.InfiniteTimeSpan); } + } - if (scheduler is not null) - { - Monitor.Exit(state.SyncRoot); - try - { - scheduler.Pump(ctx, "pthread_cond_wait"); - } - finally - { - Monitor.Enter(state.SyncRoot); - } - } + if (cooperative && + GuestThreadExecution.RequestCurrentThreadBlock( + ctx, + timed ? "pthread_cond_timedwait" : "pthread_cond_wait", + waiter.WakeKey, + () => CompleteBlockedCondWait(ctx, condAddress, mutexAddress, state, waiter), + () => TryGrantCondWaiterMutex(waiter))) + { + TracePthreadCond("wait-block", condAddress, mutexAddress, state, timed, (int)OrbisGen2Result.ORBIS_GEN2_OK); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } - while (state.SignalEpoch == observedEpoch) + // Non-guest callers have no resumable CPU continuation. Park only + // those host-side compatibility callers, preserving the same FIFO + // mutex reacquisition rules as cooperative guest waiters. + lock (state.SyncRoot) + { + var deadline = timed + ? GuestThreadExecution.ComputeDeadlineTimestamp(GetCondWaitTimeout(timeoutUsec)) + : long.MaxValue; + while (waiter.CompletionState == 0) { if (!timed) { - if (!Monitor.Wait(state.SyncRoot, GetCondSpuriousWakeTimeout())) - { - if (scheduler is not null && !GuestThreadExecution.IsGuestThread) - { - Monitor.Exit(state.SyncRoot); - try - { - scheduler.Pump(ctx, "pthread_cond_wait"); - } - finally - { - Monitor.Enter(state.SyncRoot); - } - - continue; - } - - spuriousWake = true; - break; - } - + Monitor.Wait(state.SyncRoot); continue; } - if (!Monitor.Wait(state.SyncRoot, GetCondWaitTimeout(timeoutUsec))) + var remaining = GetRemainingTimeout(deadline); + if (remaining <= TimeSpan.Zero || !Monitor.Wait(state.SyncRoot, remaining)) { - waitResult = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; + CompleteCondWaiterLocked(state, waiter, timedOut: true); break; } } - - state.Waiters = Math.Max(0, state.Waiters - 1); - TracePthreadCond( - spuriousWake - ? "wait-spurious" - : (waitResult == (int)OrbisGen2Result.ORBIS_GEN2_OK ? "wait-wake" : "wait-timeout"), - condAddress, - mutexAddress, - state, - timed, - waitResult); } - var lockResult = PthreadMutexRelockForCondWait(ctx, mutexAddress, releasedRecursion); - if (lockResult != (int)OrbisGen2Result.ORBIS_GEN2_OK) + if (waiter.MutexWaiter is null) { - TracePthreadCond("wait-relock-fail", condAddress, mutexAddress, state, timed, lockResult); - return lockResult; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - TracePthreadCond( - spuriousWake - ? "wait-exit-spurious" - : (waitResult == (int)OrbisGen2Result.ORBIS_GEN2_OK ? "wait-exit" : "wait-exit-timeout"), - condAddress, - mutexAddress, - state, - timed, - waitResult); - return TranslatePthreadResult(waitResult, posixResult); + _ = WaitForHostMutexLock(mutexState, waiter.MutexWaiter); + var waitResult = waiter.CompletionState == 2 + ? CondTimedOutResult(waiter) + : (int)OrbisGen2Result.ORBIS_GEN2_OK; + TracePthreadCond(waiter.CompletionState == 2 ? "wait-exit-timeout" : "wait-exit", condAddress, mutexAddress, state, timed, waitResult); + return waitResult; } private static int PthreadCondSignalCore(CpuContext ctx, ulong condAddress, bool broadcast) @@ -1486,209 +1335,157 @@ public static class KernelPthreadCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!TryResolveCondState(ctx, condAddress, createIfZero: true, out var resolvedCondAddress, out var state)) + if (!TryResolveCondState(ctx, condAddress, createIfZero: true, out _, out var state)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - var shouldWakeScheduler = false; + List? completedWaiters = null; lock (state.SyncRoot) { - // Advance the epoch even with no waiter registered — it's the wait predicate. state.SignalEpoch++; - if (state.Waiters > 0) + for (var node = state.WaiterQueue.First; node is not null;) { - shouldWakeScheduler = true; - if (broadcast) + var next = node.Next; + var waiter = node.Value; + if (waiter.CompletionState == 0 && CompleteCondWaiterLocked(state, waiter, timedOut: false)) { - Monitor.PulseAll(state.SyncRoot); - } - else - { - Monitor.Pulse(state.SyncRoot); + (completedWaiters ??= new List()).Add(waiter); + if (!broadcast) + { + break; + } } + + node = next; } TracePthreadCond(broadcast ? "broadcast" : "signal", condAddress, mutexAddress: 0, state, timed: false, (int)OrbisGen2Result.ORBIS_GEN2_OK); } - if (shouldWakeScheduler) + if (completedWaiters is not null) { - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads( - state.WakeKey, - broadcast ? int.MaxValue : 1); + foreach (var waiter in completedWaiters) + { + WakeCooperativeWaiter(waiter); + } } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - private static int ResumePthreadCondWait( - CpuContext ctx, - ulong condAddress, - ulong mutexAddress, - PthreadCondState state, - ulong observedEpoch, - bool timed, - int releasedRecursion, - bool posixResult) + private static PthreadMutexWaiter EnqueueMutexWaiterLocked( + PthreadMutexState state, + ulong threadId, + bool cooperative, + string? wakeKey = null) { - var waitResult = (int)OrbisGen2Result.ORBIS_GEN2_OK; - lock (state.SyncRoot) + var waiter = new PthreadMutexWaiter { - state.Waiters = Math.Max(0, state.Waiters - 1); - if (timed && state.SignalEpoch == observedEpoch) - { - waitResult = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; - } - - TracePthreadCond( - waitResult == (int)OrbisGen2Result.ORBIS_GEN2_OK ? "wait-resume" : "wait-resume-timeout", - condAddress, - mutexAddress, - state, - timed, - waitResult); - } - - var lockResult = PthreadMutexRelockForCondWait(ctx, mutexAddress, releasedRecursion); - return TranslatePthreadResult( - lockResult == (int)OrbisGen2Result.ORBIS_GEN2_OK ? waitResult : lockResult, - posixResult); - } - - private static int TranslatePthreadResult(int result, bool posixResult) - { - if (!posixResult || result == (int)OrbisGen2Result.ORBIS_GEN2_OK) - { - return result; - } - - return result switch - { - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED => 1, // EPERM - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND => 2, // ENOENT - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT => 22, // EINVAL - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK => 11, // EDEADLK - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DELETED => 13, // EACCES - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY => 16, // EBUSY - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN => 35, // EAGAIN - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT => 60, // ETIMEDOUT - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_CANCELED => 85, // ECANCELED - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT => 14, // EFAULT - _ => result + ThreadId = threadId, + Cooperative = cooperative, + WakeKey = cooperative + ? wakeKey ?? $"pthread_mutex_waiter:{Interlocked.Increment(ref _nextSynchronizationWaiterId)}" + : string.Empty, }; + waiter.Node = state.Waiters.AddLast(waiter); + return waiter; } - private static string GetCondWakeKey(ulong resolvedCondAddress) => - _condWakeKeys.GetOrAdd(resolvedCondAddress, static address => string.Create( - 31, - address, - static (destination, value) => - { - "pthread_cond:0x".AsSpan().CopyTo(destination); - _ = value.TryFormat(destination[15..], out _, "X16"); - })); - - private static int PthreadMutexFullUnlockForCondWait(CpuContext ctx, ulong mutexAddress, out int releasedRecursion) + [Conditional("DEBUG")] + private static void RunSynchronizationSelfChecks() { - releasedRecursion = 0; - if (!TryResolveMutexState(ctx, mutexAddress, createIfZero: false, out _, out var state)) + var mutex = new PthreadMutexState(); + PthreadMutexWaiter first; + PthreadMutexWaiter second; + lock (mutex) { - return PthreadMutexUnlockCore(ctx, mutexAddress, requireOwner: true); + first = EnqueueMutexWaiterLocked(mutex, 0x101, cooperative: false); + second = EnqueueMutexWaiterLocked(mutex, 0x202, cooperative: false); + Debug.Assert(!TryGrantMutexWaiterLocked(mutex, second), "A mutex waiter bypassed FIFO order."); + Debug.Assert(TryGrantMutexWaiterLocked(mutex, first), "The FIFO mutex head was not granted."); + Debug.Assert(mutex.OwnerThreadId == first.ThreadId && mutex.RecursionCount == 1, "Mutex ownership was not transferred atomically."); + mutex.OwnerThreadId = 0; + mutex.RecursionCount = 0; + Debug.Assert(TryGrantMutexWaiterLocked(mutex, second), "The second mutex waiter was not granted after release."); } - var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); + var cond = new PthreadCondState(); + var condMutex = new PthreadMutexState(); + var condWaiter = new PthreadCondWaiter + { + ThreadId = 0x303, + MutexState = condMutex, + WakeKey = string.Empty, + Cooperative = false, + }; + lock (cond.SyncRoot) + { + condWaiter.Node = cond.WaiterQueue.AddLast(condWaiter); + cond.Waiters++; + Debug.Assert(CompleteCondWaiterLocked(cond, condWaiter, timedOut: false), "A condition waiter was not completed."); + Debug.Assert(cond.WaiterQueue.Count == 0 && cond.Waiters == 0 && condWaiter.MutexWaiter is not null, "Condition completion did not atomically queue mutex reacquisition."); + } + } + + private static bool TryGrantMutexWaiterLocked(PthreadMutexState state, PthreadMutexWaiter waiter) + { + if (Volatile.Read(ref waiter.Granted) != 0) + { + return true; + } + + if (state.OwnerThreadId != 0 || + waiter.Node is null || + !ReferenceEquals(state.Waiters.First, waiter.Node)) + { + return false; + } + + state.Waiters.Remove(waiter.Node); + waiter.Node = null; + state.OwnerThreadId = waiter.ThreadId; + state.RecursionCount = 1; + Volatile.Write(ref waiter.Granted, 1); + Monitor.PulseAll(state); + return true; + } + + private static int WaitForHostMutexLock(PthreadMutexState state, PthreadMutexWaiter waiter) + { lock (state) { - if (state.RecursionCount <= 0 || state.OwnerThreadId != currentThreadId) + while (!TryGrantMutexWaiterLocked(state, waiter)) { - return PthreadMutexUnlockCore(ctx, mutexAddress, requireOwner: true); - } - - releasedRecursion = state.RecursionCount; - state.RecursionCount = 1; - } - - return PthreadMutexUnlockCore(ctx, mutexAddress, requireOwner: true); - } - - private static int PthreadMutexRelockForCondWait(CpuContext ctx, ulong mutexAddress, int releasedRecursion) - { - var result = PthreadMutexLockCore(ctx, mutexAddress, tryOnly: false); - if (result != (int)OrbisGen2Result.ORBIS_GEN2_OK) - { - return result; - } - - if (releasedRecursion > 1 && - TryResolveMutexState(ctx, mutexAddress, createIfZero: false, out _, out var state)) - { - lock (state) - { - if (state.OwnerThreadId == KernelPthreadState.GetCurrentThreadHandle()) - { - state.RecursionCount = releasedRecursion; - } + Monitor.Wait(state); } } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - public static string? DumpMutexStateForStall(ulong mutexAddress) - { - if (!_mutexStates.TryGetValue(mutexAddress, out var state)) - { - return null; - } - - var waiterThreads = string.Join(",", _mutexStates - .Where(pair => ReferenceEquals(pair.Value, state)) - .Select(pair => $"0x{pair.Key:X}")); - - return $"mutex=0x{mutexAddress:X16} owner=0x{state.OwnerThreadId:X} recursion={state.RecursionCount} " + - $"type={state.Type} semaphore_count={state.Semaphore.CurrentCount} aliases=[{waiterThreads}]"; - } - - private static string GetMutexWakeKey(ulong resolvedMutexAddress) => - _mutexWakeKeys.GetOrAdd(resolvedMutexAddress, static address => string.Create( - 32, - address, - static (destination, value) => - { - "pthread_mutex:0x".AsSpan().CopyTo(destination); - _ = value.TryFormat(destination[16..], out _, "X16"); - })); - - private static bool TryReserveBlockedMutexLock( + private static bool TryGrantBlockedMutexLock( CpuContext ctx, ulong mutexAddress, ulong resolvedAddress, PthreadMutexState state, PthreadMutexWaiter waiter) { + var granted = false; lock (state) { - if (state.OwnerThreadId != 0 || state.RecursionCount != 0) - { - TracePthreadMutex(ctx, "lock-reserve-busy", mutexAddress, resolvedAddress, state, waiter.ThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); - return false; - } - - if (!state.Semaphore.Wait(0)) - { - TracePthreadMutex(ctx, "lock-reserve-busy", mutexAddress, resolvedAddress, state, waiter.ThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); - return false; - } - - DetectMutexDoubleOwner(resolvedAddress, state, waiter.ThreadId, "reserve"); - state.OwnerThreadId = waiter.ThreadId; - state.RecursionCount = 1; - Interlocked.Exchange(ref waiter.Reserved, 1); + granted = TryGrantMutexWaiterLocked(state, waiter); } - TracePthreadMutex(ctx, "lock-reserve", mutexAddress, resolvedAddress, state, waiter.ThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return true; + TracePthreadMutex( + ctx, + granted ? "lock-reserve" : "lock-reserve-busy", + mutexAddress, + resolvedAddress, + state, + waiter.ThreadId, + granted ? (int)OrbisGen2Result.ORBIS_GEN2_OK : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + return granted; } private static int CompleteBlockedMutexLock( @@ -1699,39 +1496,121 @@ public static class KernelPthreadCompatExports PthreadMutexWaiter waiter) { var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); - if (Interlocked.Exchange(ref waiter.Reserved, 0) == 1) + if (Volatile.Read(ref waiter.Granted) == 1) { TracePthreadMutex(ctx, "lock-resume", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (!state.Semaphore.Wait(0)) - { - TracePthreadMutex(ctx, "lock-resume-busy", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; - } - - lock (state) - { - DetectMutexDoubleOwner(resolvedAddress, state, currentThreadId, "resume"); - state.OwnerThreadId = currentThreadId; - state.RecursionCount = 1; - } - - TracePthreadMutex(ctx, "lock-resume", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + TracePthreadMutex(ctx, "lock-resume-ungranted", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; } - // A nonzero foreign prior owner means two guest threads are in the same critical - // section. Call while holding lock(state). - private static void DetectMutexDoubleOwner(ulong resolvedAddress, PthreadMutexState state, ulong currentThreadId, string site) + private static bool CompleteCondWaiterLocked( + PthreadCondState state, + PthreadCondWaiter waiter, + bool timedOut) { - var priorOwner = state.OwnerThreadId; - if (priorOwner != 0 && priorOwner != currentThreadId) + if (waiter.CompletionState != 0) { - Console.Error.WriteLine( - $"[LOADER][ERROR] MUTEX DOUBLE-OWNER at {site}: resolved=0x{resolvedAddress:X} acquirer=0x{currentThreadId:X} " + - $"prior_owner=0x{priorOwner:X} recursion={state.RecursionCount} type={state.Type} sem_count={state.Semaphore.CurrentCount}"); + return false; + } + + waiter.CompletionState = timedOut ? 2 : 1; + RemoveCondWaiterLocked(state, waiter); + waiter.TimeoutTimer?.Dispose(); + waiter.TimeoutTimer = null; + + lock (waiter.MutexState) + { + waiter.MutexWaiter = EnqueueMutexWaiterLocked( + waiter.MutexState, + waiter.ThreadId, + waiter.Cooperative, + waiter.WakeKey); + } + + Monitor.PulseAll(state.SyncRoot); + return true; + } + + private static void CompleteCondWaiter( + PthreadCondState state, + PthreadCondWaiter waiter, + bool timedOut) + { + var completed = false; + lock (state.SyncRoot) + { + completed = CompleteCondWaiterLocked(state, waiter, timedOut); + } + + if (completed) + { + WakeCooperativeWaiter(waiter); + } + } + + private static void RemoveCondWaiterLocked(PthreadCondState state, PthreadCondWaiter waiter) + { + if (waiter.Node is not null) + { + state.WaiterQueue.Remove(waiter.Node); + waiter.Node = null; + state.Waiters = Math.Max(0, state.Waiters - 1); + } + } + + private static bool TryGrantCondWaiterMutex(PthreadCondWaiter waiter) + { + var mutexWaiter = waiter.MutexWaiter; + if (waiter.CompletionState == 0 || mutexWaiter is null) + { + return false; + } + + lock (waiter.MutexState) + { + return TryGrantMutexWaiterLocked(waiter.MutexState, mutexWaiter); + } + } + + private static int CompleteBlockedCondWait( + CpuContext ctx, + ulong condAddress, + ulong mutexAddress, + PthreadCondState state, + PthreadCondWaiter waiter) + { + waiter.TimeoutTimer?.Dispose(); + waiter.TimeoutTimer = null; + var result = waiter.MutexWaiter is not null && + Volatile.Read(ref waiter.MutexWaiter.Granted) == 1 + ? (waiter.CompletionState == 2 + ? CondTimedOutResult(waiter) + : (int)OrbisGen2Result.ORBIS_GEN2_OK) + : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; + TracePthreadCond( + waiter.CompletionState == 2 ? "wait-resume-timeout" : "wait-resume", + condAddress, + mutexAddress, + state, + waiter.CompletionState == 2, + result); + _ = ctx; + return result; + } + + private static int CondTimedOutResult(PthreadCondWaiter waiter) => + waiter.PosixErrors + ? 60 // ETIMEDOUT on Orbis/FreeBSD; pthread APIs return errno directly. + : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; + + private static void WakeCooperativeWaiter(PthreadCondWaiter waiter) + { + if (waiter.Cooperative) + { + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(waiter.WakeKey, 1); } } @@ -1742,20 +1621,18 @@ public static class KernelPthreadCompatExports return TimeSpan.Zero; } - // Round positive sub-millisecond waits to the host sleep quantum. - return timeoutUsec < 1_000 - ? TimeSpan.FromMilliseconds(1) - : TimeSpan.FromTicks((long)timeoutUsec * 10L); + return TimeSpan.FromTicks((long)timeoutUsec * 10L); } - private static TimeSpan GetCondSpuriousWakeTimeout() + private static TimeSpan GetRemainingTimeout(long deadlineTimestamp) { - if (int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_PTHREAD_COND_SPURIOUS_WAKE_MS"), out var milliseconds)) + var remainingTicks = deadlineTimestamp - Stopwatch.GetTimestamp(); + if (remainingTicks <= 0) { - return TimeSpan.FromMilliseconds(Math.Max(1, milliseconds)); + return TimeSpan.Zero; } - return TimeSpan.FromMilliseconds(DefaultSpuriousCondWakeMilliseconds); + return TimeSpan.FromSeconds(remainingTicks / (double)Stopwatch.Frequency); } private static int NormalizeMutexType(int type) @@ -1785,6 +1662,32 @@ public static class KernelPthreadCompatExports } } + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)(int)result); + return (int)result; + } + + private static bool TryReadInt32(CpuContext ctx, ulong address, out int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt32LittleEndian(bytes); + return true; + } + + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + private static bool CreateImplicitMutexState(CpuContext ctx, ulong mutexAddress, int type, out ulong resolvedAddress, [NotNullWhen(true)] out PthreadMutexState? state) { var createdState = new PthreadMutexState @@ -1800,7 +1703,6 @@ public static class KernelPthreadCompatExports } if (!InitializeMutexObject(ctx, handle, createdState)) { - TryFreeOpaqueObject(ctx, handle); resolvedAddress = 0; state = null; return false; @@ -1810,14 +1712,12 @@ public static class KernelPthreadCompatExports { if (_mutexStates.TryGetValue(mutexAddress, out state)) { - TryFreeOpaqueObject(ctx, handle); resolvedAddress = mutexAddress; return true; } if (_mutexStates.TryGetValue(handle, out state)) { - TryFreeOpaqueObject(ctx, handle); resolvedAddress = handle; return true; } @@ -1831,7 +1731,6 @@ public static class KernelPthreadCompatExports _mutexStates.TryRemove(mutexAddress, out _); _mutexStates.TryRemove(handle, out _); - TryFreeOpaqueObject(ctx, handle); resolvedAddress = 0; state = null; return false; @@ -1879,8 +1778,7 @@ public static class KernelPthreadCompatExports $"[LOADER][TRACE] pthread_{operation}: mutex=0x{mutexAddress:X16} resolved=0x{resolvedAddress:X16} " + $"guest[0]=0x{guestWord0:X16} guest[8]=0x{guestWord1:X16} " + $"current=0x{currentThreadId:X16} owner=0x{(state?.OwnerThreadId ?? 0):X16} " + - $"recursion={(state?.RecursionCount ?? 0)} type={(state?.Type ?? 0)} result=0x{unchecked((uint)result):X8} " + - $"guest_thread=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} managed={Environment.CurrentManagedThreadId}"); + $"recursion={(state?.RecursionCount ?? 0)} type={(state?.Type ?? 0)} result=0x{unchecked((uint)result):X8}"); } private static void TracePthreadCond(string operation, ulong condAddress, ulong mutexAddress, PthreadCondState? state, bool timed, int result) @@ -1891,8 +1789,7 @@ public static class KernelPthreadCompatExports } Console.Error.WriteLine( - $"[LOADER][TRACE] pthread_cond_{operation}: thread=0x{KernelPthreadState.GetCurrentThreadHandle():X16} " + - $"cond=0x{condAddress:X16} key={state?.WakeKey ?? "none"} mutex=0x{mutexAddress:X16} " + + $"[LOADER][TRACE] pthread_cond_{operation}: cond=0x{condAddress:X16} mutex=0x{mutexAddress:X16} " + $"waiters={(state?.Waiters ?? 0)} epoch=0x{(state?.SignalEpoch ?? 0):X} timed={timed} result=0x{unchecked((uint)result):X8}"); } diff --git a/src/SharpEmu.Libs/Kernel/KernelPthreadExtendedCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelPthreadExtendedCompatExports.cs index b744201..4785bb4 100644 --- a/src/SharpEmu.Libs/Kernel/KernelPthreadExtendedCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelPthreadExtendedCompatExports.cs @@ -2,8 +2,10 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; +using System.Buffers.Binary; using System.Collections.Concurrent; using System.Text; +using System.Threading; using System.Diagnostics.CodeAnalysis; namespace SharpEmu.Libs.Kernel; @@ -15,6 +17,8 @@ public static class KernelPthreadExtendedCompatExports private const int DefaultDetachState = 0; private const ulong DefaultGuardSize = 0x1000UL; private const ulong DefaultStackSize = 0x1_00000UL; + private const ulong NativeGuestStackSize = 0x20_0000UL; + private const ulong NativeGuestStackStride = 0x100_0000UL; private const int DefaultInheritSched = 4; private const int DefaultSchedPolicy = 1; private const int DefaultSchedPriority = DefaultThreadPriority; @@ -223,6 +227,13 @@ public static class KernelPthreadExtendedCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + [SysAbiExport( + Nid = "+U1R4WtXvoc", + ExportName = "pthread_detach", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixPthreadDetach(CpuContext ctx) => PthreadDetach(ctx); + [SysAbiExport( Nid = "How7B8Oet6k", ExportName = "scePthreadGetname", @@ -273,6 +284,7 @@ public static class KernelPthreadExtendedCompatExports state.Attributes = state.Attributes with { AffinityMask = mask }; } + _ = GuestThreadExecution.Scheduler?.TrySetGuestThreadAffinity(thread, mask); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -326,7 +338,7 @@ public static class KernelPthreadExtendedCompatExports priority = GetOrCreateThreadStateLocked(thread).Priority; } - if (!ctx.TryWriteInt32(outPriorityAddress, priority)) + if (!TryWriteInt32(ctx, outPriorityAddress, priority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -354,6 +366,9 @@ public static class KernelPthreadExtendedCompatExports GetOrCreateThreadStateLocked(thread).Priority = priority; } + // Apply to the live scheduler thread so runtime priority changes take + // effect, not just the local bookkeeping snapshot. + _ = GuestThreadExecution.Scheduler?.TrySetGuestThreadPriority(thread, priority); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -373,7 +388,7 @@ public static class KernelPthreadExtendedCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!ctx.TryReadInt32(schedParamAddress, out var schedPriority)) + if (!TryReadInt32(ctx, schedParamAddress, out var schedPriority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -408,9 +423,9 @@ public static class KernelPthreadExtendedCompatExports public static int PthreadGetschedparam(CpuContext ctx) { var thread = ctx[CpuRegister.Rdi]; - var outPolicyAddress = ctx[CpuRegister.Rsi]; - var outSchedParamAddress = ctx[CpuRegister.Rdx]; - if (thread == 0 || outPolicyAddress == 0 || outSchedParamAddress == 0) + var policyAddress = ctx[CpuRegister.Rsi]; + var schedParamAddress = ctx[CpuRegister.Rdx]; + if (thread == 0 || policyAddress == 0 || schedParamAddress == 0) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } @@ -424,8 +439,8 @@ public static class KernelPthreadExtendedCompatExports priority = state.Priority; } - if (!ctx.TryWriteInt32(outPolicyAddress, policy) || - !ctx.TryWriteInt32(outSchedParamAddress, priority)) + if (!TryWriteInt32(ctx, policyAddress, policy) || + !TryWriteInt32(ctx, schedParamAddress, priority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -528,6 +543,22 @@ public static class KernelPthreadExtendedCompatExports lock (_stateGate) { var threadState = GetOrCreateThreadStateLocked(thread); + + // The native executor maps guest pthread stacks itself, after the + // kernel-facing thread object has been created. Report that live + // mapping when a thread asks for its own attributes. IL2CPP's + // conservative collector uses these two fields to register the stack; + // returning the default null address lets it recycle objects that are + // still reachable only from guest registers/stack frames. + if (thread == KernelPthreadState.GetCurrentThreadHandle() && + TryInferNativeGuestStack(ctx[CpuRegister.Rsp], out var stackAddress)) + { + threadState.Attributes = threadState.Attributes with + { + StackAddress = stackAddress, + StackSize = NativeGuestStackSize, + }; + } _attrStates[outAttrAddress] = threadState.Attributes; } @@ -535,6 +566,28 @@ public static class KernelPthreadExtendedCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + private static bool TryInferNativeGuestStack(ulong stackPointer, out ulong stackAddress) + { + stackAddress = 0; + var candidate = stackPointer & ~(NativeGuestStackStride - 1); + if (stackPointer - candidate >= NativeGuestStackSize) + { + return false; + } + + var highestStack = OperatingSystem.IsWindows() + ? 0x00007FFF_F000_0000UL + : 0x00006FFF_F000_0000UL; + var lowestStack = highestStack - (63 * NativeGuestStackStride); + if (candidate < lowestStack || candidate > highestStack) + { + return false; + } + + stackAddress = candidate; + return true; + } + [SysAbiExport( Nid = "8+s5BzZjxSg", ExportName = "scePthreadAttrGetaffinity", @@ -584,7 +637,7 @@ public static class KernelPthreadExtendedCompatExports state = GetOrCreateAttrStateLocked(attrAddress); } - if (!ctx.TryWriteInt32(outStateAddress, state.DetachState)) + if (!TryWriteInt32(ctx, outStateAddress, state.DetachState)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -827,7 +880,7 @@ public static class KernelPthreadExtendedCompatExports state = GetOrCreateAttrStateLocked(attrAddress); } - if (!ctx.TryWriteInt32(schedParamAddress, state.SchedPriority)) + if (!TryWriteInt32(ctx, schedParamAddress, state.SchedPriority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -850,7 +903,7 @@ public static class KernelPthreadExtendedCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!ctx.TryReadInt32(schedParamAddress, out var schedPriority)) + if (!TryReadInt32(ctx, schedParamAddress, out var schedPriority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1207,7 +1260,7 @@ public static class KernelPthreadExtendedCompatExports } } - if (!ctx.TryWriteInt32(outKeyAddress, key)) + if (!TryWriteInt32(ctx, outKeyAddress, key)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1315,6 +1368,72 @@ public static class KernelPthreadExtendedCompatExports LibraryName = "libKernel")] public static int OrbisPthreadGetspecific(CpuContext ctx) => PosixPthreadGetspecific(ctx); + private const int PthreadDestructorIterations = 4; + + /// + /// Runs the current thread's pthread TLS-key destructors, as POSIX + /// requires on thread exit. Each key holding a non-null value with a + /// registered destructor has its value cleared first and the destructor + /// then invoked with the previous value; this repeats up to + /// PTHREAD_DESTRUCTOR_ITERATIONS times so destructors that set new + /// thread-local values are themselves cleaned up. Called on the exiting + /// guest thread while it is still executable. + /// + public static void RunThreadLocalDestructors(CpuContext ctx) + { + var scheduler = GuestThreadExecution.Scheduler; + if (scheduler is null) + { + return; + } + + var threadHandle = KernelPthreadState.GetCurrentThreadHandle(); + if (!_threadLocalSpecific.TryGetValue(threadHandle, out var values)) + { + return; + } + + for (var iteration = 0; iteration < PthreadDestructorIterations; iteration++) + { + var ranAny = false; + foreach (var entry in values) + { + var value = entry.Value; + if (value == 0 || + !_tlsKeys.TryGetValue(entry.Key, out var keyState) || + keyState.Destructor == 0) + { + continue; + } + + // Clear before invoking, per POSIX, so a destructor that + // re-sets the key is handled on the next iteration. + if (!values.TryUpdate(entry.Key, 0, value)) + { + continue; + } + + ranAny = true; + _ = scheduler.TryCallGuestFunction( + ctx, + keyState.Destructor, + value, + 0, + 0, + 0, + "pthread_tls_destructor", + out _); + } + + if (!ranAny) + { + break; + } + } + + _threadLocalSpecific.TryRemove(threadHandle, out _); + } + private static int PthreadRwlockLockCore(CpuContext ctx, ulong rwlockAddress, bool write) { if (rwlockAddress == 0) @@ -1680,4 +1799,24 @@ public static class KernelPthreadExtendedCompatExports payload[^1] = 0; return ctx.Memory.TryWrite(address, payload); } + + private static bool TryReadInt32(CpuContext ctx, ulong address, out int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt32LittleEndian(bytes); + return true; + } + + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } } diff --git a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs index a59a639..b63881f 100644 --- a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs @@ -2,35 +2,21 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; -using SharpEmu.HLE.Host; using SharpEmu.Libs.Fiber; using System.Buffers.Binary; +using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; using System.Runtime.Intrinsics.X86; using System.Security.Cryptography; using System.Text; +using System.Threading; namespace SharpEmu.Libs.Kernel; public static class KernelRuntimeCompatExports { - internal const int ClockRealtime = 0; - internal const int ClockVirtual = 1; - internal const int ClockProf = 2; - internal const int ClockMonotonic = 4; - internal const int ClockUptime = 5; - internal const int ClockUptimePrecise = 7; - internal const int ClockUptimeFast = 8; - internal const int ClockRealtimePrecise = 9; - internal const int ClockRealtimeFast = 10; - internal const int ClockMonotonicPrecise = 11; - internal const int ClockMonotonicFast = 12; - internal const int ClockSecond = 13; - internal const int ClockThreadCputimeId = 14; - internal const int ClockProcTime = 15; - private const int Efault = 14; - private const int Einval = 22; private const ulong TlsErrnoOffset = 0x40; private const ulong TlsStackChkGuardBaseOffset = 0x800; private const ulong StackChkGuardFieldOffset = 0x10; @@ -40,20 +26,32 @@ public static class KernelRuntimeCompatExports private const int MallocReplaceSize = 0x70; private const int NewReplaceSize = 0x68; private const int OrbisTimesecSize = sizeof(long) + sizeof(uint) + sizeof(uint); - private const ulong ModuleInfoHandleOffset = 0x108; - private const ulong ModuleInfoNameOffset = 0x10; - private const int ModuleInfoNameMaxBytes = 64; + // PS4/PS5 libkernel module-info ABI. The extended form is consumed by + // sceKernelGetModuleInfoFromAddr and ends at byte 0x1A8; libc commonly + // places its stack canary immediately after that caller-owned buffer. + private const int ModuleInfoNameMaxBytes = 256; + private const int ModuleInfoSize = 0x160; + private const int ModuleInfoExSize = 0x1A8; + private const ulong ModuleInfoNameOffset = 0x08; + private const ulong ModuleInfoExHandleOffset = 0x108; + private const ulong ModuleInfoExInitProcOffset = 0x128; + private const ulong ModuleInfoExSegmentsOffset = 0x160; + private const ulong ModuleInfoExSegmentCountOffset = 0x1A0; + private const int ModuleInfoSegmentSize = 16; private const ulong DefaultKernelTscFrequency = 10_000_000UL; private const ulong PrtAreaStartAddress = 0x0000001000000000UL; private const ulong PrtAreaSize = 0x000000EC00000000UL; private const int MapFlagFixed = 0x10; private const ulong DefaultVirtualRangeAlignment = 0x4000UL; private const int AioInitParamSize = 0x3C; + private const uint MemCommit = 0x1000; + private const uint MemReserve = 0x2000; + private const uint PageExecuteReadWrite = 0x40; private static readonly object _stateGate = new(); private static readonly long _processStartCounter = Stopwatch.GetTimestamp(); private static readonly RdtscDelegate? _rdtscReader = CreateRdtscReader(); private static readonly ulong _kernelTscFrequency = ResolveKernelTscFrequency(); - private static readonly ulong _stackChkGuardValue = 0xC0DEC0DECAFEBABEUL; + private static readonly ulong _stackChkGuardValue = 0xC0DEC0DECAFEBA00UL; private static readonly nint _stackChkGuardObjectAddress = HleDataSymbols.TryGetAddress("f7uOxY9mM1U", out var stackChkGuardAddress) ? unchecked((nint)stackChkGuardAddress) @@ -61,6 +59,7 @@ public static class KernelRuntimeCompatExports private static ulong _applicationHeapApiAddress; private static ulong _processProcParamAddress; private static ulong _nextReservedVirtualBase = 0x6000_0000_0UL; + private static readonly List _releasedVirtualRanges = new(); private static uint _gpoStateBits; private static readonly HashSet _loadedSysmodules = new(); private static readonly object _prtApertureGate = new(); @@ -75,101 +74,14 @@ public static class KernelRuntimeCompatExports [ThreadStatic] private static int _shortUsleepCount; + private static readonly bool _stopwatchTicksAreNanoseconds = + Stopwatch.Frequency == 1_000_000_000L; + + private readonly record struct ReleasedVirtualRange(ulong Address, ulong Length); + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate ulong RdtscDelegate(); - [SysAbiExport( - Nid = "QvsZxomvUHs", - ExportName = "sceKernelNanosleep", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int KernelNanosleep(CpuContext ctx) => NanosleepCore(ctx, posix: false); - - [SysAbiExport( - Nid = "yS8U2TGCe1A", - ExportName = "nanosleep", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int PosixNanosleep(CpuContext ctx) => NanosleepCore(ctx, posix: true); - - private static int NanosleepCore(CpuContext ctx, bool posix) - { - var requestAddress = ctx[CpuRegister.Rdi]; - var remainAddress = ctx[CpuRegister.Rsi]; - - if (requestAddress == 0) - { - return NanosleepFailure(ctx, posix, Einval, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - Span timespecBuffer = stackalloc byte[16]; - if (!ctx.Memory.TryRead(requestAddress, timespecBuffer)) - { - return NanosleepFailure(ctx, posix, Efault, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - var tvSec = BinaryPrimitives.ReadInt64LittleEndian(timespecBuffer); - var tvNsec = BinaryPrimitives.ReadInt64LittleEndian(timespecBuffer[sizeof(long)..]); - - if (tvSec < 0 || tvNsec < 0 || tvNsec >= 1_000_000_000L) - { - return NanosleepFailure(ctx, posix, Einval, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - if (tvSec == 0 && tvNsec == 0) - { - WriteRemainingTime(ctx, remainAddress, 0, 0); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - GuestThreadExecution.Scheduler?.Pump(ctx, posix ? "nanosleep" : "sceKernelNanosleep"); - - // TimeSpan resolution is 100 ns ticks, so sub-100 ns requests round up to - // a single tick rather than collapsing to a zero-length (no-op) sleep. - var totalTicks = tvSec * TimeSpan.TicksPerSecond + Math.Max(tvNsec / 100L, 1L); - try - { - Thread.Sleep(TimeSpan.FromTicks(totalTicks)); - } - catch (ArgumentOutOfRangeException) - { - Thread.Sleep(TimeSpan.FromMilliseconds(int.MaxValue)); - } - - WriteRemainingTime(ctx, remainAddress, 0, 0); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - private static int NanosleepFailure(CpuContext ctx, bool posix, int errnoValue, OrbisGen2Result sceResult) - { - if (posix) - { - TrySetErrno(ctx, errnoValue); - ctx[CpuRegister.Rax] = unchecked((ulong)(-1L)); - } - else - { - ctx[CpuRegister.Rax] = unchecked((ulong)errnoValue); - } - - return (int)sceResult; - } - - private static void WriteRemainingTime(CpuContext ctx, ulong remainAddress, long seconds, long nanoseconds) - { - if (remainAddress == 0) - { - return; - } - - Span remainBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(remainBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(remainBuffer[sizeof(long)..], nanoseconds); - ctx.Memory.TryWrite(remainAddress, remainBuffer); - } - [SysAbiExport( Nid = "1jfXLRVzisc", ExportName = "sceKernelUsleep", @@ -203,21 +115,16 @@ public static class KernelRuntimeCompatExports else { _shortUsleepCount = 0; - var sleepMilliseconds = (int)Math.Min((micros + 999UL) / 1000UL, int.MaxValue); - Thread.Sleep(sleepMilliseconds); + // Precise sleep: rounding microseconds up to Thread.Sleep + // milliseconds (which itself overshoots by a scheduler quantum) + // hard-caps games that pace their frame loop with usleep. + HostTiming.SleepMicroseconds((long)Math.Min(micros, long.MaxValue)); } ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "QcteRwbsnV0", - ExportName = "usleep", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int PosixUsleep(CpuContext ctx) => KernelUsleep(ctx); - private static void TraceUsleepSpin(CpuContext ctx, ulong micros) { if (!_traceUsleep) @@ -299,16 +206,31 @@ public static class KernelRuntimeCompatExports long seconds; long nanoseconds; - if (!ResolveClockTime(clockId, out seconds, out nanoseconds)) + if (clockId == 0) { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + var now = DateTimeOffset.UtcNow; + seconds = now.ToUnixTimeSeconds(); + nanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100; + } + else + { + var elapsedTicks = Stopwatch.GetTimestamp() - _processStartCounter; + if (_stopwatchTicksAreNanoseconds) + { + // Constant divisors let the JIT strength-reduce the division; + // games call this thousands of times per second. + seconds = elapsedTicks / 1_000_000_000L; + nanoseconds = elapsedTicks - seconds * 1_000_000_000L; + } + else + { + seconds = elapsedTicks / Stopwatch.Frequency; + nanoseconds = (elapsedTicks % Stopwatch.Frequency) * 1_000_000_000L / Stopwatch.Frequency; + } } - Span timespecBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(timespecBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(timespecBuffer[sizeof(long)..], nanoseconds); - - if (!ctx.Memory.TryWrite(timeAddress, timespecBuffer)) + if (!ctx.TryWriteUInt64(timeAddress, unchecked((ulong)seconds)) || + !ctx.TryWriteUInt64(timeAddress + sizeof(long), unchecked((ulong)nanoseconds))) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -333,11 +255,8 @@ public static class KernelRuntimeCompatExports var now = DateTimeOffset.UtcNow; var seconds = now.ToUnixTimeSeconds(); var microseconds = (now.Ticks % TimeSpan.TicksPerSecond) / 10; - - Span timevalBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(timevalBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(timevalBuffer[sizeof(long)..], microseconds); - if (!ctx.Memory.TryWrite(timeAddress, timevalBuffer)) + if (!ctx.TryWriteUInt64(timeAddress, unchecked((ulong)seconds)) || + !ctx.TryWriteUInt64(timeAddress + sizeof(long), unchecked((ulong)microseconds))) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -359,28 +278,18 @@ public static class KernelRuntimeCompatExports var seconds = now.ToUnixTimeSeconds(); var microseconds = (now.Ticks % TimeSpan.TicksPerSecond) / 10; - if (timeAddress != 0) + if (timeAddress != 0 && + (!ctx.TryWriteUInt64(timeAddress, unchecked((ulong)seconds)) || + !ctx.TryWriteUInt64(timeAddress + sizeof(long), unchecked((ulong)microseconds)))) { - Span timevalBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(timevalBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(timevalBuffer[sizeof(long)..], microseconds); - if (!ctx.Memory.TryWrite(timeAddress, timevalBuffer)) - { - TrySetErrno(ctx, Efault); - return -1; - } + return -1; } - if (timezoneAddress != 0) + if (timezoneAddress != 0 && + (!TryWriteInt32(ctx, timezoneAddress, 0) || + !TryWriteInt32(ctx, timezoneAddress + sizeof(int), 0))) { - Span timezoneBuffer = stackalloc byte[8]; - BinaryPrimitives.WriteInt32LittleEndian(timezoneBuffer, 0); - BinaryPrimitives.WriteInt32LittleEndian(timezoneBuffer[sizeof(int)..], 0); - if (!ctx.Memory.TryWrite(timezoneAddress, timezoneBuffer)) - { - TrySetErrno(ctx, Efault); - return -1; - } + return -1; } ctx[CpuRegister.Rax] = 0; @@ -429,18 +338,6 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "HoLVWNanBBc", - ExportName = "getpid", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int GetProcessId(CpuContext ctx) - { - var processId = Environment.ProcessId; - ctx[CpuRegister.Rax] = unchecked((uint)processId); - return processId; - } - [SysAbiExport( Nid = "fgxnMeTNUtY", ExportName = "sceKernelGetProcessTimeCounter", @@ -747,58 +644,7 @@ public static class KernelRuntimeCompatExports internal static bool TrySetErrno(CpuContext ctx, int value) { var address = GetTlsScratchAddress(ctx, TlsErrnoOffset); - return address != 0 && ctx.TryWriteInt32(address, value); - } - - internal static bool ResolveClockTime(int clockId, out long seconds, out long nanoseconds) - { - switch (clockId) - { - case ClockRealtime: - case ClockRealtimePrecise: - case ClockRealtimeFast: - case ClockVirtual: - case ClockProf: - { - var now = DateTimeOffset.UtcNow; - seconds = now.ToUnixTimeSeconds(); - nanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100; - return true; - } - - // CLOCK_SECOND is FreeBSD's cached whole-second realtime clock (Quake's - // audio_output_thread polls it and treated the previous EINVAL as a fatal - // init failure, exiting and getting respawned in a loop). - case ClockSecond: - seconds = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - nanoseconds = 0; - return true; - - case ClockMonotonic: - case ClockMonotonicPrecise: - case ClockMonotonicFast: - case ClockUptime: - case ClockUptimePrecise: - case ClockUptimeFast: - // Per-thread/process CPU time approximated with the monotonic clock; games - // use these for profiling deltas where monotonicity matters, not absolutes. - case ClockThreadCputimeId: - case ClockProcTime: - GetProcessMonotonicTime(out seconds, out nanoseconds); - return true; - - default: - seconds = 0; - nanoseconds = 0; - return false; - } - } - - internal static void GetProcessMonotonicTime(out long seconds, out long nanoseconds) - { - var elapsedTicks = Stopwatch.GetTimestamp() - _processStartCounter; - seconds = elapsedTicks / Stopwatch.Frequency; - nanoseconds = (elapsedTicks % Stopwatch.Frequency) * 1_000_000_000L / Stopwatch.Frequency; + return address != 0 && TryWriteInt32(ctx, address, value); } [SysAbiExport( @@ -914,7 +760,27 @@ public static class KernelRuntimeCompatExports : AlignUp(_nextReservedVirtualBase, effectiveAlignment); } - if (!TryReserveVirtualRange(ctx, desiredAddress, length, effectiveAlignment, allowSearch: !fixedMapping, out var mappedAddress)) + ulong releasedAddress = 0; + var reusedReleasedRange = !fixedMapping && requestedAddress == 0 && + TryTakeReleasedVirtualRange(length, effectiveAlignment, out releasedAddress); + var alreadyBacked = fixedMapping && requestedAddress != 0 && + KernelMemoryCompatExports.IsGuestRangeBacked(ctx, requestedAddress, length); + ulong mappedAddress; + if (reusedReleasedRange) + { + mappedAddress = releasedAddress; + } + else if (alreadyBacked) + { + mappedAddress = requestedAddress; + } + else if (!TryReserveVirtualRange( + ctx, + desiredAddress, + length, + effectiveAlignment, + allowSearch: !fixedMapping, + out mappedAddress)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } @@ -922,7 +788,7 @@ public static class KernelRuntimeCompatExports if (ShouldTraceVirtualMemory()) { Console.Error.WriteLine( - $"[LOADER][TRACE] reserve_virtual_range: req=0x{requestedAddress:X16} desired=0x{desiredAddress:X16} mapped=0x{mappedAddress:X16} len=0x{length:X16} flags=0x{flags:X8} align=0x{effectiveAlignment:X16}"); + $"[LOADER][TRACE] reserve_virtual_range: req=0x{requestedAddress:X16} desired=0x{desiredAddress:X16} mapped=0x{mappedAddress:X16} len=0x{length:X16} flags=0x{flags:X8} align=0x{effectiveAlignment:X16} already_backed={alreadyBacked} reused={reusedReleasedRange}"); } if (!ctx.TryWriteUInt64(inOutAddressPointer, mappedAddress)) @@ -939,6 +805,75 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + internal static void RegisterReleasedVirtualRange(ulong address, ulong length) + { + if (address == 0 || length == 0 || ulong.MaxValue - address < length) + { + return; + } + + lock (_stateGate) + { + var start = address; + var end = address + length; + for (var i = _releasedVirtualRanges.Count - 1; i >= 0; i--) + { + var existing = _releasedVirtualRanges[i]; + var existingEnd = existing.Address + existing.Length; + if (end < existing.Address || existingEnd < start) + { + continue; + } + + start = Math.Min(start, existing.Address); + end = Math.Max(end, existingEnd); + _releasedVirtualRanges.RemoveAt(i); + } + + _releasedVirtualRanges.Add(new ReleasedVirtualRange(start, end - start)); + } + } + + private static bool TryTakeReleasedVirtualRange( + ulong length, + ulong alignment, + out ulong address) + { + address = 0; + lock (_stateGate) + { + for (var i = 0; i < _releasedVirtualRanges.Count; i++) + { + var range = _releasedVirtualRanges[i]; + var rangeEnd = range.Address + range.Length; + var aligned = AlignUp(range.Address, alignment); + if (aligned >= rangeEnd || length > rangeEnd - aligned) + { + continue; + } + + _releasedVirtualRanges.RemoveAt(i); + if (aligned > range.Address) + { + _releasedVirtualRanges.Add( + new ReleasedVirtualRange(range.Address, aligned - range.Address)); + } + + var allocationEnd = aligned + length; + if (allocationEnd < rangeEnd) + { + _releasedVirtualRanges.Add( + new ReleasedVirtualRange(allocationEnd, rangeEnd - allocationEnd)); + } + + address = aligned; + return true; + } + } + + return false; + } + [SysAbiExport( Nid = "BohYr-F7-is", ExportName = "sceKernelSetPrtAperture", @@ -1000,22 +935,36 @@ public static class KernelRuntimeCompatExports public static int KernelGetModuleInfoFromAddr(CpuContext ctx) { var queriedAddress = ctx[CpuRegister.Rdi]; - _ = ctx[CpuRegister.Rsi]; // mode + var flags = unchecked((int)ctx[CpuRegister.Rsi]); var outInfoAddress = ctx[CpuRegister.Rdx]; if (outInfoAddress == 0) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - var moduleHandle = ResolveModuleHandleByAddress(queriedAddress); - if (!ctx.TryWriteInt32(outInfoAddress + ModuleInfoHandleOffset, moduleHandle)) + if (flags is < 0 or >= 3) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!ctx.TryReadUInt64(outInfoAddress, out var callerSize)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (KernelModuleRegistry.TryGetModuleByHandle(moduleHandle, out var module)) + if (callerSize != ModuleInfoExSize) { - _ = TryWriteModuleName(ctx, outInfoAddress, module.Name); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!KernelModuleRegistry.TryGetModuleByAddress(queriedAddress, out var module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (!TryWriteModuleInfoEx(ctx, outInfoAddress, module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } ctx[CpuRegister.Rax] = 0; @@ -1030,22 +979,36 @@ public static class KernelRuntimeCompatExports public static int KernelGetModuleInfoForUnwind(CpuContext ctx) { var queriedAddress = ctx[CpuRegister.Rdi]; - _ = ctx[CpuRegister.Rsi]; // flags + var flags = unchecked((int)ctx[CpuRegister.Rsi]); var outInfoAddress = ctx[CpuRegister.Rdx]; if (outInfoAddress == 0) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - var moduleHandle = ResolveModuleHandleByAddress(queriedAddress); - if (!ctx.TryWriteInt32(outInfoAddress + ModuleInfoHandleOffset, moduleHandle)) + if (flags is < 0 or >= 3) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!ctx.TryReadUInt64(outInfoAddress, out var callerSize)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (KernelModuleRegistry.TryGetModuleByHandle(moduleHandle, out var module)) + if (callerSize < 0x130) { - _ = TryWriteModuleName(ctx, outInfoAddress, module.Name); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!KernelModuleRegistry.TryGetModuleByAddress(queriedAddress, out var module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (!TryWriteModuleInfoForUnwind(ctx, outInfoAddress, module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } ctx[CpuRegister.Rax] = 0; @@ -1085,7 +1048,7 @@ public static class KernelRuntimeCompatExports LibraryName = "libKernel")] public static int KernelGetModuleInfoInternal(CpuContext ctx) { - return KernelGetModuleInfoByHandleCore( + return KernelGetModuleInfoExByHandleCore( ctx, unchecked((int)ctx[CpuRegister.Rdi]), ctx[CpuRegister.Rsi]); @@ -1239,7 +1202,7 @@ public static class KernelRuntimeCompatExports public static int KernelStopUnloadModule(CpuContext ctx) { var resultAddress = ctx[CpuRegister.R9]; - if (resultAddress != 0 && !ctx.TryWriteInt32(resultAddress, 0)) + if (resultAddress != 0 && !TryWriteInt32(ctx, resultAddress, 0)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1256,8 +1219,10 @@ public static class KernelRuntimeCompatExports public static int KernelLoadStartModule(CpuContext ctx) { var modulePathAddress = ctx[CpuRegister.Rdi]; + var argumentSize = ctx[CpuRegister.Rsi]; + var argumentAddress = ctx[CpuRegister.Rdx]; var resultAddress = ctx[CpuRegister.R9]; - if (resultAddress != 0 && !ctx.TryWriteInt32(resultAddress, 0)) + if (resultAddress != 0 && !TryWriteInt32(ctx, resultAddress, 0)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1283,6 +1248,40 @@ public static class KernelRuntimeCompatExports handle = KernelModuleRegistry.RegisterSyntheticModule("module.sprx", isSystemModule: false); } + if (KernelModuleRegistry.TryBeginModuleStart(handle, out var moduleToStart)) + { + var scheduler = GuestThreadExecution.Scheduler; + string? startError = null; + var started = scheduler is not null && scheduler.TryCallGuestFunction( + ctx, + moduleToStart.InitEntryPoint, + argumentSize, + argumentAddress, + 0, + 0, + $"sceKernelLoadStartModule:{moduleToStart.Name}", + out startError); + KernelModuleRegistry.CompleteModuleStart(handle, started); + if (!started) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] sceKernelLoadStartModule failed to start '{moduleToStart.Name}' " + + $"at 0x{moduleToStart.InitEntryPoint:X16}: {startError ?? "guest scheduler unavailable"}"); + var error = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_CPU_TRAP; + if (resultAddress != 0) + { + _ = TryWriteInt32(ctx, resultAddress, error); + } + + ctx[CpuRegister.Rax] = unchecked((ulong)(long)error); + return error; + } + + Console.Error.WriteLine( + $"[LOADER][INFO] sceKernelLoadStartModule started '{moduleToStart.Name}' " + + $"at 0x{moduleToStart.InitEntryPoint:X16}"); + } + ctx[CpuRegister.Rax] = unchecked((uint)handle); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1414,8 +1413,8 @@ public static class KernelRuntimeCompatExports : 0; var minutesWest = unchecked((int)-offset.TotalMinutes); - if (!ctx.TryWriteInt32(timezoneAddress, minutesWest) || - !ctx.TryWriteInt32(timezoneAddress + sizeof(int), dstSeconds / 60)) + if (!TryWriteInt32(ctx, timezoneAddress, minutesWest) || + !TryWriteInt32(ctx, timezoneAddress + sizeof(int), dstSeconds / 60)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1425,7 +1424,7 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (dstSecondsAddress != 0 && !ctx.TryWriteInt32(dstSecondsAddress, dstSeconds)) + if (dstSecondsAddress != 0 && !TryWriteInt32(ctx, dstSecondsAddress, dstSeconds)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1485,7 +1484,7 @@ public static class KernelRuntimeCompatExports _loadedSysmodules.Add(moduleId); } - if (resultAddress != 0 && !ctx.TryWriteInt32(resultAddress, 0)) + if (resultAddress != 0 && !TryWriteInt32(ctx, resultAddress, 0)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1547,22 +1546,6 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - private static int ResolveModuleHandleByAddress(ulong queriedAddress) - { - if (queriedAddress != 0 && - KernelModuleRegistry.TryGetModuleByAddress(queriedAddress, out var moduleFromAddress)) - { - return moduleFromAddress.Handle; - } - - if (KernelModuleRegistry.TryGetFirstModule(out var firstModule)) - { - return firstModule.Handle; - } - - return 1; - } - private static int KernelGetModuleInfoByHandleCore(CpuContext ctx, int handle, ulong outInfoAddress) { if (outInfoAddress == 0) @@ -1575,12 +1558,52 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - if (!ctx.TryWriteInt32(outInfoAddress + ModuleInfoHandleOffset, module.Handle)) + if (!ctx.TryReadUInt64(outInfoAddress, out var callerSize)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (callerSize != ModuleInfoSize) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!TryWriteModuleInfo(ctx, outInfoAddress, module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + private static int KernelGetModuleInfoExByHandleCore(CpuContext ctx, int handle, ulong outInfoAddress) + { + if (outInfoAddress == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!KernelModuleRegistry.TryGetModuleByHandle(handle, out var module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (!ctx.TryReadUInt64(outInfoAddress, out var callerSize)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (callerSize != ModuleInfoExSize) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!TryWriteModuleInfoEx(ctx, outInfoAddress, module)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - _ = TryWriteModuleName(ctx, outInfoAddress, module.Name); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1612,7 +1635,7 @@ public static class KernelRuntimeCompatExports var writableCount = (int)Math.Min(Math.Min(capacity, (ulong)int.MaxValue), (ulong)handles.Length); for (var i = 0; i < writableCount; i++) { - if (!ctx.TryWriteInt32(handlesAddress + (ulong)(i * sizeof(int)), handles[i])) + if (!TryWriteInt32(ctx, handlesAddress + (ulong)(i * sizeof(int)), handles[i])) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1627,22 +1650,88 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - private static bool TryWriteModuleName(CpuContext ctx, ulong outInfoAddress, string moduleName) + private static bool TryWriteModuleInfo( + CpuContext ctx, + ulong outInfoAddress, + KernelModuleRegistry.ModuleEntry module) { - if (outInfoAddress == 0 || string.IsNullOrWhiteSpace(moduleName)) + var payload = new byte[ModuleInfoSize]; + BinaryPrimitives.WriteUInt64LittleEndian(payload, ModuleInfoSize); + WriteModuleName(payload, module.Name); + WriteModuleSegment(payload, 0x108, module); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(0x148), 1); + return ctx.Memory.TryWrite(outInfoAddress, payload); + } + + private static bool TryWriteModuleInfoEx( + CpuContext ctx, + ulong outInfoAddress, + KernelModuleRegistry.ModuleEntry module) + { + var payload = new byte[ModuleInfoExSize]; + BinaryPrimitives.WriteUInt64LittleEndian(payload, ModuleInfoExSize); + WriteModuleName(payload, module.Name); + BinaryPrimitives.WriteInt32LittleEndian( + payload.AsSpan((int)ModuleInfoExHandleOffset), + module.Handle); + BinaryPrimitives.WriteUInt64LittleEndian( + payload.AsSpan((int)ModuleInfoExInitProcOffset), + module.EntryPoint); + WriteModuleSegment(payload, (int)ModuleInfoExSegmentsOffset, module); + BinaryPrimitives.WriteUInt32LittleEndian( + payload.AsSpan((int)ModuleInfoExSegmentCountOffset), + 1); + return ctx.Memory.TryWrite(outInfoAddress, payload); + } + + private static bool TryWriteModuleInfoForUnwind( + CpuContext ctx, + ulong outInfoAddress, + KernelModuleRegistry.ModuleEntry module) + { + const int unwindInfoSize = 0x130; + var payload = new byte[unwindInfoSize]; + BinaryPrimitives.WriteUInt64LittleEndian(payload, unwindInfoSize); + WriteModuleName(payload, module.Name); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(0x108), module.EhFrameHeaderAddress); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(0x110), module.EhFrameAddress); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(0x118), module.EhFrameSize); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(0x120), module.BaseAddress); + BinaryPrimitives.WriteUInt64LittleEndian( + payload.AsSpan(0x128), + module.EndAddress - module.BaseAddress); + return ctx.Memory.TryWrite(outInfoAddress, payload); + } + + private static void WriteModuleName(Span payload, string moduleName) + { + if (string.IsNullOrWhiteSpace(moduleName)) { - return false; + return; } var encoded = Encoding.UTF8.GetBytes(moduleName); var payloadLength = Math.Min(encoded.Length, ModuleInfoNameMaxBytes - 1); - var buffer = new byte[ModuleInfoNameMaxBytes]; if (payloadLength > 0) { - Array.Copy(encoded, 0, buffer, 0, payloadLength); + encoded.AsSpan(0, payloadLength).CopyTo( + payload.Slice((int)ModuleInfoNameOffset, payloadLength)); } + } - return ctx.Memory.TryWrite(outInfoAddress + ModuleInfoNameOffset, buffer); + private static void WriteModuleSegment( + Span payload, + int offset, + KernelModuleRegistry.ModuleEntry module) + { + Debug.Assert(offset >= 0 && offset + ModuleInfoSegmentSize <= payload.Length); + BinaryPrimitives.WriteUInt64LittleEndian(payload.Slice(offset), module.BaseAddress); + BinaryPrimitives.WriteUInt32LittleEndian( + payload.Slice(offset + sizeof(ulong)), + (uint)Math.Min(module.EndAddress - module.BaseAddress, uint.MaxValue)); + // Loaded modules are represented as a single aggregate readable and + // executable range until per-program-header registry data is exposed. + BinaryPrimitives.WriteInt32LittleEndian(payload.Slice(offset + 12), 5); } private static bool TryReadUtf8Z(CpuContext ctx, ulong address, int maxLength, out string value) @@ -1653,42 +1742,26 @@ public static class KernelRuntimeCompatExports return false; } - const int pageSize = 4096; - const int inlineChunkSize = 64; - Span buffer = stackalloc byte[Math.Min(maxLength, 512)]; - var length = 0; - - for (var offset = 0; offset < maxLength && length < buffer.Length;) + var bytes = new List(Math.Min(maxLength, 64)); + Span one = stackalloc byte[1]; + for (var i = 0; i < maxLength; i++) { - var current = address + (ulong)offset; - var pageRemaining = pageSize - (int)(current & (pageSize - 1)); - var chunkSize = Math.Min( - buffer.Length - length, - Math.Min(maxLength - offset, Math.Min(inlineChunkSize, pageRemaining))); - - if (chunkSize <= 0) + if (!ctx.Memory.TryRead(address + (ulong)i, one)) { return false; } - var chunk = buffer.Slice(length, chunkSize); - if (!ctx.Memory.TryRead(current, chunk)) + if (one[0] == 0) { - return false; - } - - var nulIndex = chunk.IndexOf((byte)0); - if (nulIndex >= 0) - { - value = Encoding.UTF8.GetString(buffer[..(length + nulIndex)]); + value = Encoding.UTF8.GetString(bytes.ToArray()); return true; } - length += chunkSize; - offset += chunkSize; + bytes.Add(one[0]); } - return false; + value = Encoding.UTF8.GetString(bytes.ToArray()); + return true; } private static ulong GetTlsScratchAddress(CpuContext ctx, ulong offset) @@ -1701,6 +1774,13 @@ public static class KernelRuntimeCompatExports return unchecked(ctx.FsBase + offset); } + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + private static nint AllocateStackChkGuardObject() { try @@ -1902,20 +1982,21 @@ public static class KernelRuntimeCompatExports private static RdtscDelegate? CreateRdtscReader() { - if (!OperatingSystem.IsWindows() || !Environment.Is64BitProcess) + if (!Environment.Is64BitProcess || + RuntimeInformation.ProcessArchitecture != Architecture.X64) { return null; } try { - nint stubAddress = unchecked((nint)HostPlatform.Current.Memory.Allocate(0, 16, HostPageProtection.ReadWriteExecute)); + nint stubAddress = VirtualAlloc(nint.Zero, (nuint)16, MemCommit | MemReserve, PageExecuteReadWrite); if (stubAddress == 0) { return null; } - ReadOnlySpan stub = stackalloc byte[] + Span stub = stackalloc byte[] { 0x0F, 0x31, 0x48, 0xC1, 0xE2, 0x20, @@ -1923,14 +2004,7 @@ public static class KernelRuntimeCompatExports 0xC3, }; - unsafe - { - fixed (byte* src = stub) - { - Buffer.MemoryCopy(src, (void*)stubAddress, stub.Length, stub.Length); - } - } - + Marshal.Copy(stub.ToArray(), 0, stubAddress, stub.Length); return Marshal.GetDelegateForFunctionPointer(stubAddress); } catch @@ -1939,6 +2013,9 @@ public static class KernelRuntimeCompatExports } } + private static unsafe nint VirtualAlloc(nint lpAddress, nuint dwSize, uint flAllocationType, uint flProtect) => + (nint)HostMemory.Alloc((void*)lpAddress, dwSize, flAllocationType, flProtect); + private static bool TryReserveVirtualRange( CpuContext ctx, ulong desiredAddress, @@ -1974,4 +2051,116 @@ public static class KernelRuntimeCompatExports { return string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIRTUAL_MEMORY"), "1", StringComparison.Ordinal); } + [SysAbiExport( + Nid = "QvsZxomvUHs", + ExportName = "sceKernelNanosleep", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelNanosleep(CpuContext ctx) => NanosleepCore(ctx, posix: false); + + [SysAbiExport( + Nid = "yS8U2TGCe1A", + ExportName = "nanosleep", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixNanosleep(CpuContext ctx) => NanosleepCore(ctx, posix: true); + + private static int NanosleepCore(CpuContext ctx, bool posix) + { + const int eFault = 14; + const int eInvalid = 22; + var requestAddress = ctx[CpuRegister.Rdi]; + var remainAddress = ctx[CpuRegister.Rsi]; + + if (requestAddress == 0) + { + return NanosleepFailure(ctx, posix, eInvalid, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + Span timespecBuffer = stackalloc byte[16]; + if (!ctx.Memory.TryRead(requestAddress, timespecBuffer)) + { + return NanosleepFailure(ctx, posix, eFault, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + var tvSec = BinaryPrimitives.ReadInt64LittleEndian(timespecBuffer); + var tvNsec = BinaryPrimitives.ReadInt64LittleEndian(timespecBuffer[sizeof(long)..]); + if (tvSec < 0 || tvNsec < 0 || tvNsec >= 1_000_000_000L) + { + return NanosleepFailure(ctx, posix, eInvalid, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + if (tvSec == 0 && tvNsec == 0) + { + WriteRemainingTime(ctx, remainAddress, 0, 0); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + GuestThreadExecution.Scheduler?.Pump(ctx, posix ? "nanosleep" : "sceKernelNanosleep"); + var totalTicks = tvSec * TimeSpan.TicksPerSecond + Math.Max(tvNsec / 100L, 1L); + try + { + Thread.Sleep(TimeSpan.FromTicks(totalTicks)); + } + catch (ArgumentOutOfRangeException) + { + Thread.Sleep(TimeSpan.FromMilliseconds(int.MaxValue)); + } + + WriteRemainingTime(ctx, remainAddress, 0, 0); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + private static int NanosleepFailure( + CpuContext ctx, + bool posix, + int errnoValue, + OrbisGen2Result sceResult) + { + if (posix) + { + TrySetErrno(ctx, errnoValue); + ctx[CpuRegister.Rax] = unchecked((ulong)(-1L)); + } + else + { + ctx[CpuRegister.Rax] = unchecked((ulong)errnoValue); + } + + return (int)sceResult; + } + + private static void WriteRemainingTime(CpuContext ctx, ulong remainAddress, long seconds, long nanoseconds) + { + if (remainAddress == 0) + { + return; + } + + Span remainBuffer = stackalloc byte[16]; + BinaryPrimitives.WriteInt64LittleEndian(remainBuffer, seconds); + BinaryPrimitives.WriteInt64LittleEndian(remainBuffer[sizeof(long)..], nanoseconds); + ctx.Memory.TryWrite(remainAddress, remainBuffer); + } + + [SysAbiExport( + Nid = "QcteRwbsnV0", + ExportName = "usleep", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixUsleep(CpuContext ctx) => KernelUsleep(ctx); + + [SysAbiExport( + Nid = "HoLVWNanBBc", + ExportName = "getpid", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int GetProcessId(CpuContext ctx) + { + var processId = Environment.ProcessId; + ctx[CpuRegister.Rax] = unchecked((uint)processId); + return processId; + } } diff --git a/src/SharpEmu.Libs/Kernel/KernelSemaphoreCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelSemaphoreCompatExports.cs index 4fc12f8..097a550 100644 --- a/src/SharpEmu.Libs/Kernel/KernelSemaphoreCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelSemaphoreCompatExports.cs @@ -1,8 +1,9 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Buffers.Binary; using System.Collections.Concurrent; -using System.Diagnostics; +using System.Text; using SharpEmu.HLE; namespace SharpEmu.Libs.Kernel; @@ -13,9 +14,6 @@ public static class KernelSemaphoreCompatExports private static readonly ConcurrentDictionary _semaphores = new(); private static int _nextSemaphoreHandle = 1; - [ThreadStatic] - private static int _semaPollBackoffCount; - private sealed class KernelSemaphoreState { public required string Name { get; init; } @@ -25,35 +23,9 @@ public static class KernelSemaphoreCompatExports public required int MaxCount { get; init; } public int Count { get; set; } public int WaitingThreads { get; set; } - public int CancelEpoch { get; set; } - public bool Deleted { get; set; } public object Gate { get; } = new(); } - private sealed class SemaphoreWaiter : IGuestThreadBlockWaiter - { - public required KernelSemaphoreState Semaphore { get; init; } - public required int NeedCount { get; init; } - public required int CancelEpochAtBlock { get; init; } - public bool Timed { get; init; } - - // Timed-wait completion state; unused when Timed is false. - public CpuContext? Ctx { get; init; } - public ulong TimeoutAddress { get; init; } - public long DeadlineTimestamp { get; init; } - - // Written and read only under the owning semaphore's Gate. - public int? Result { get; set; } - - public int Resume() => Timed - ? CompleteBlockedTimedSemaWait(Ctx!, Semaphore, this, TimeoutAddress, DeadlineTimestamp) - : CompleteBlockedSemaWait(Semaphore, this); - - public bool TryWake() => TryConsumeBlockedSemaWait(Semaphore, this); - } - - private static string GetSemaphoreWakeKey(uint handle) => $"kernel_sema:0x{handle:X8}"; - [SysAbiExport( Nid = "188x57JYp0g", ExportName = "sceKernelCreateSema", @@ -76,12 +48,12 @@ public static class KernelSemaphoreCompatExports initialCount > maxCount || optionAddress != 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadNullTerminatedUtf8(nameAddress, MaxSemaphoreNameLength, out var name)) + if (!TryReadNullTerminatedUtf8(ctx, nameAddress, MaxSemaphoreNameLength, out var name)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } var handle = unchecked((uint)Interlocked.Increment(ref _nextSemaphoreHandle)); @@ -90,7 +62,7 @@ public static class KernelSemaphoreCompatExports handle = unchecked((uint)Interlocked.Increment(ref _nextSemaphoreHandle)); } - var state = new KernelSemaphoreState + _semaphores[handle] = new KernelSemaphoreState { Name = name, WakeKey = GetSemaphoreWakeKey(handle), @@ -98,28 +70,15 @@ public static class KernelSemaphoreCompatExports MaxCount = maxCount, Count = initialCount, }; - _semaphores[handle] = state; - if (!ctx.TryWriteUInt32(semaphoreAddress, handle)) + if (!TryWriteUInt32(ctx, semaphoreAddress, handle)) { _semaphores.TryRemove(handle, out _); - // Handles are sequential and guest-predictable, so a hostile guest can - // race a WaitSema onto the handle between publication above and this - // rollback. Strand-proof that waiter exactly like DeleteSema does. - lock (state.Gate) - { - state.Deleted = true; - } - - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(state.WakeKey); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - if (_traceSema) - { - TraceSemaphore($"create handle=0x{handle:X8} name='{name}' attr=0x{attr:X} init={initialCount} max={maxCount}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceSemaphore($"create handle=0x{handle:X8} name='{name}' attr=0x{attr:X} init={initialCount} max={maxCount}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -135,125 +94,144 @@ public static class KernelSemaphoreCompatExports if (!_semaphores.TryGetValue(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (needCount < 1 || needCount > semaphore.MaxCount) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + uint timeoutUsec = 0; + if (timeoutAddress != 0 && !TryReadUInt32(ctx, timeoutAddress, out timeoutUsec)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - var pollTimedOut = false; lock (semaphore.Gate) { if (semaphore.Count >= needCount) { semaphore.Count -= needCount; - if (_traceSema) + if (timeoutAddress != 0) { - TraceSemaphore($"wait handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + _ = TryWriteUInt32(ctx, timeoutAddress, timeoutUsec); } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + + TraceSemaphore($"wait handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} timeout={(timeoutAddress == 0 ? "infinite" : timeoutUsec)}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } + semaphore.WaitingThreads++; + } + + // Block cooperatively: the wake predicate atomically acquires the + // tokens (so a wake commits the acquisition), while the resume + // handler distinguishes a real acquisition from a deadline expiry. + var acquired = false; + var deadline = timeoutAddress != 0 + ? GuestThreadExecution.ComputeDeadlineTimestamp(TimeSpan.FromMicroseconds(timeoutUsec)) + : 0; + + bool WakePredicate() + { + lock (semaphore.Gate) + { + if (semaphore.Count >= needCount) + { + semaphore.Count -= needCount; + semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); + acquired = true; + return true; + } + + return false; + } + } + + int ResumeWait() + { if (timeoutAddress != 0) { - if (!ctx.TryReadUInt32(timeoutAddress, out var timeoutMicros)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - if (timeoutMicros == 0) - { - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - if (_traceSema) - { - TraceSemaphore($"wait-timeout handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - pollTimedOut = true; - } - else - { - var deadline = GuestThreadExecution.ComputeDeadlineTimestamp(TimeSpan.FromTicks((long)timeoutMicros * 10L)); - var timedWaiter = new SemaphoreWaiter - { - Semaphore = semaphore, - NeedCount = needCount, - CancelEpochAtBlock = semaphore.CancelEpoch, - Timed = true, - Ctx = ctx, - TimeoutAddress = timeoutAddress, - DeadlineTimestamp = deadline, - }; - if (GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - "sceKernelWaitSema", - semaphore.WakeKey, - timedWaiter, - blockDeadlineTimestamp: deadline)) - { - semaphore.WaitingThreads++; - if (_traceSema) - { - TraceSemaphore($"wait-block-timed handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} timeout_us={timeoutMicros} waiters={semaphore.WaitingThreads}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - // Host-owned threads cannot park in the guest scheduler; degrade to the - // immediate-timeout poll the callers already tolerate. - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - if (_traceSema) - { - TraceSemaphore($"wait-timeout handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - pollTimedOut = true; - } + _ = TryWriteUInt32(ctx, timeoutAddress, 0); } - if (!pollTimedOut) + if (acquired) { - var waiter = new SemaphoreWaiter - { - Semaphore = semaphore, - NeedCount = needCount, - CancelEpochAtBlock = semaphore.CancelEpoch, - }; - if (!GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - "sceKernelWaitSema", - semaphore.WakeKey, - waiter)) - { - if (_traceSema) - { - TraceSemaphore($"wait-would-block handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); - } - - semaphore.WaitingThreads++; - if (_traceSema) - { - TraceSemaphore($"wait-block handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} waiters={semaphore.WaitingThreads}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceSemaphore($"wait-wake handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + + lock (semaphore.Gate) + { + semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); + } + + TraceSemaphore($"wait-timeout handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; } - GuestThreadExecution.Scheduler?.Pump(ctx, "sceKernelWaitSema"); - if ((++_semaPollBackoffCount & 255) == 0) + if (GuestThreadExecution.RequestCurrentThreadBlock( + ctx, + "sceKernelWaitSema", + GetSemaphoreWakeKey(handle), + ResumeWait, + WakePredicate, + deadline)) { - Thread.Sleep(0); - } - else - { - Thread.Yield(); + TraceSemaphore($"wait-block handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} timeout={(timeoutAddress == 0 ? "infinite" : timeoutUsec)} waiters={semaphore.WaitingThreads} {FormatCallSite(ctx)}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); + // Not a guest thread (or no scheduler): fall back to a host-thread + // wait so the semantics still hold on non-cooperative callers. + return WaitSemaphoreOnHostThread(ctx, semaphore, handle, needCount, timeoutAddress, timeoutUsec); } + private static int WaitSemaphoreOnHostThread( + CpuContext ctx, + KernelSemaphoreState semaphore, + uint handle, + int needCount, + ulong timeoutAddress, + uint timeoutUsec) + { + var deadlineMs = timeoutAddress != 0 + ? Environment.TickCount64 + Math.Max(1L, timeoutUsec / 1000L) + : long.MaxValue; + lock (semaphore.Gate) + { + TraceSemaphore( + $"wait-host-block handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} " + + $"count={semaphore.Count} timeout={(timeoutAddress == 0 ? "infinite" : timeoutUsec)} {FormatCallSite(ctx)}"); + while (semaphore.Count < needCount) + { + var remaining = deadlineMs - Environment.TickCount64; + if (timeoutAddress != 0 && remaining <= 0) + { + semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); + _ = TryWriteUInt32(ctx, timeoutAddress, 0); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); + } + + Monitor.Wait(semaphore.Gate, (int)Math.Min(remaining, 100)); + } + + semaphore.Count -= needCount; + semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); + TraceSemaphore( + $"wait-host-wake handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} {FormatCallSite(ctx)}"); + if (timeoutAddress != 0) + { + _ = TryWriteUInt32(ctx, timeoutAddress, 0); + } + + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + } + + private static string GetSemaphoreWakeKey(uint handle) => $"sceKernelWaitSema:{handle:X8}"; + [SysAbiExport( Nid = "12wOHk8ywb0", ExportName = "sceKernelPollSema", @@ -263,31 +241,25 @@ public static class KernelSemaphoreCompatExports { if (!_semaphores.TryGetValue(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (needCount < 1 || needCount > semaphore.MaxCount) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } lock (semaphore.Gate) { if (semaphore.Count < needCount) { - if (_traceSema) - { - TraceSemaphore($"poll-busy handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + TraceSemaphore($"poll-busy handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); } semaphore.Count -= needCount; - if (_traceSema) - { - TraceSemaphore($"poll handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceSemaphore($"poll handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } } @@ -300,34 +272,31 @@ public static class KernelSemaphoreCompatExports { if (!_semaphores.TryGetValue(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (signalCount <= 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } lock (semaphore.Gate) { if (semaphore.Count > semaphore.MaxCount - signalCount) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } semaphore.Count += signalCount; - if (_traceSema) - { - TraceSemaphore($"signal handle=0x{handle:X8} name='{semaphore.Name}' signal={signalCount} count={semaphore.Count} waiters={semaphore.WaitingThreads}"); - } + // Wake host-thread waiters parked in the fallback path. + Monitor.PulseAll(semaphore.Gate); + TraceSemaphore($"signal handle=0x{handle:X8} name='{semaphore.Name}' signal={signalCount} count={semaphore.Count} waiters={semaphore.WaitingThreads} {FormatCallSite(ctx)}"); } - // Wake after releasing the gate (lock order: scheduler gate -> semaphore gate). - // Wake everyone; the wake handler consumes the count per waiter, so a waiter - // whose needCount exceeds the remaining count stays parked while a smaller - // waiter can proceed. - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(semaphore.WakeKey); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + // Wake cooperatively-blocked guest threads; their wake predicate + // acquires the tokens atomically, so this respects the new count. + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(GetSemaphoreWakeKey(handle)); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -339,35 +308,29 @@ public static class KernelSemaphoreCompatExports { if (!_semaphores.TryGetValue(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (setCount > semaphore.MaxCount) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } lock (semaphore.Gate) { - if (waitingThreadsAddress != 0 && !ctx.TryWriteUInt32(waitingThreadsAddress, unchecked((uint)semaphore.WaitingThreads))) + if (waitingThreadsAddress != 0 && !TryWriteUInt32(ctx, waitingThreadsAddress, unchecked((uint)semaphore.WaitingThreads))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } semaphore.Count = setCount < 0 ? semaphore.InitialCount : setCount; - semaphore.CancelEpoch++; - // WaitingThreads is NOT zeroed here: each canceled waiter decrements it - // exactly once in its wake handler. Zeroing here as well would double-count - // and silently absorb the increment of a waiter that parks between this - // gate release and the wake-all below. - if (_traceSema) - { - TraceSemaphore($"cancel handle=0x{handle:X8} name='{semaphore.Name}' set={setCount} count={semaphore.Count}"); - } + semaphore.WaitingThreads = 0; + Monitor.PulseAll(semaphore.Gate); + TraceSemaphore($"cancel handle=0x{handle:X8} name='{semaphore.Name}' set={setCount} count={semaphore.Count}"); } - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(semaphore.WakeKey); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(GetSemaphoreWakeKey(handle)); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -380,138 +343,238 @@ public static class KernelSemaphoreCompatExports var handle = unchecked((uint)ctx[CpuRegister.Rdi]); if (!_semaphores.TryRemove(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } - // Delete succeeds even with blocked waiters; they wake with the deleted - // result (the SCE kernel wakes waiters with the EACCES-class code). - lock (semaphore.Gate) - { - semaphore.Deleted = true; - } - - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(semaphore.WakeKey); - if (_traceSema) - { - TraceSemaphore($"delete handle=0x{handle:X8} name='{semaphore.Name}'"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceSemaphore($"delete handle=0x{handle:X8} name='{semaphore.Name}'"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } - // Wake handler: runs under the scheduler's guest-thread gate (lock order: - // scheduler gate -> semaphore gate). Returns true iff the waiter has a final - // result and should be re-readied; false leaves it parked. - private static bool TryConsumeBlockedSemaWait(KernelSemaphoreState semaphore, SemaphoreWaiter waiter) + [SysAbiExport( + Nid = "pDuPEf3m4fI", + ExportName = "sem_init", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemInit(CpuContext ctx) { - lock (semaphore.Gate) + var semaphoreAddress = ctx[CpuRegister.Rdi]; + var initialCountValue = ctx[CpuRegister.Rdx]; + if (semaphoreAddress == 0 || initialCountValue > int.MaxValue) { - return TryConsumeBlockedSemaWaitLocked(semaphore, waiter); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } + + var handle = unchecked((uint)Interlocked.Increment(ref _nextSemaphoreHandle)); + if (handle == 0) + { + handle = unchecked((uint)Interlocked.Increment(ref _nextSemaphoreHandle)); + } + + var initialCount = unchecked((int)initialCountValue); + _semaphores[handle] = new KernelSemaphoreState + { + Name = $"posix@0x{semaphoreAddress:X16}", + WakeKey = GetSemaphoreWakeKey(handle), + InitialCount = initialCount, + MaxCount = int.MaxValue, + Count = initialCount, + }; + if (!TryWriteUInt32(ctx, semaphoreAddress, handle)) + { + _semaphores.TryRemove(handle, out _); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + TraceSemaphore($"posix-init address=0x{semaphoreAddress:X16} handle=0x{handle:X8} count={initialCount}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } - private static bool TryConsumeBlockedSemaWaitLocked(KernelSemaphoreState semaphore, SemaphoreWaiter waiter) + [SysAbiExport( + Nid = "YCV5dGGBcCo", + ExportName = "sem_wait", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemWait(CpuContext ctx) { - if (waiter.Result is not null) + if (!TryGetPosixSemaphoreHandle(ctx, ctx[CpuRegister.Rdi], out var handle)) { - return true; + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (semaphore.Deleted) - { - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DELETED; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - if (_traceSema) - { - TraceSemaphore($"wake-deleted name='{semaphore.Name}' need={waiter.NeedCount}"); - } - return true; - } - - if (semaphore.CancelEpoch != waiter.CancelEpochAtBlock) - { - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_CANCELED; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - if (_traceSema) - { - TraceSemaphore($"wake-canceled name='{semaphore.Name}' need={waiter.NeedCount}"); - } - return true; - } - - if (semaphore.Count >= waiter.NeedCount) - { - semaphore.Count -= waiter.NeedCount; - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_OK; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - if (_traceSema) - { - TraceSemaphore($"wake-consume name='{semaphore.Name}' need={waiter.NeedCount} count={semaphore.Count} waiters={semaphore.WaitingThreads}"); - } - return true; - } - - return false; + ctx[CpuRegister.Rdi] = handle; + ctx[CpuRegister.Rsi] = 1; + ctx[CpuRegister.Rdx] = 0; + return KernelWaitSema(ctx); } - // Resume handler: runs on the woken guest thread outside the scheduler gate; - // its return value becomes the guest's RAX for the resumed sceKernelWaitSema. - private static int CompleteBlockedSemaWait(KernelSemaphoreState semaphore, SemaphoreWaiter waiter) + [SysAbiExport( + Nid = "WBWzsRifCEA", + ExportName = "sem_trywait", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemTryWait(CpuContext ctx) { - lock (semaphore.Gate) + if (!TryGetPosixSemaphoreHandle(ctx, ctx[CpuRegister.Rdi], out var handle)) { - if (waiter.Result is null && !TryConsumeBlockedSemaWaitLocked(semaphore, waiter)) - { - // Nothing readies a parked semaphore waiter without the wake handler - // resolving it, so reaching here means the scheduler contract changed. - Console.Error.WriteLine( - $"[LOADER][GAP] sema.resume-no-outcome name='{semaphore.Name}' need={waiter.NeedCount} count={semaphore.Count}"); - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - } - - return waiter.Result!.Value; + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } + + ctx[CpuRegister.Rdi] = handle; + ctx[CpuRegister.Rsi] = 1; + return KernelPollSema(ctx, handle, 1); } - private static int CompleteBlockedTimedSemaWait( - CpuContext ctx, - KernelSemaphoreState semaphore, - SemaphoreWaiter waiter, - ulong timeoutAddress, - long deadlineTimestamp) + [SysAbiExport( + Nid = "w5IHyvahg-o", + ExportName = "sem_timedwait", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemTimedWait(CpuContext ctx) { - int result; - lock (semaphore.Gate) + var timeoutAddress = ctx[CpuRegister.Rsi]; + if (!TryGetPosixSemaphoreHandle(ctx, ctx[CpuRegister.Rdi], out var handle)) { - if (waiter.Result is null && !TryConsumeBlockedSemaWaitLocked(semaphore, waiter)) - { - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - if (_traceSema) - { - TraceSemaphore($"wake-timeout name='{semaphore.Name}' need={waiter.NeedCount} count={semaphore.Count} waiters={semaphore.WaitingThreads}"); - } - } - - result = waiter.Result!.Value; + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } + ctx[CpuRegister.Rdi] = handle; + ctx[CpuRegister.Rsi] = 1; + ctx[CpuRegister.Rdx] = timeoutAddress; + return KernelWaitSema(ctx); + } + + [SysAbiExport( + Nid = "IKP8typ0QUk", + ExportName = "sem_post", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemPost(CpuContext ctx) + { + if (!TryGetPosixSemaphoreHandle(ctx, ctx[CpuRegister.Rdi], out var handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + ctx[CpuRegister.Rdi] = handle; + ctx[CpuRegister.Rsi] = 1; + return KernelSignalSema(ctx, handle, 1); + } + + [SysAbiExport( + Nid = "Bq+LRV-N6Hk", + ExportName = "sem_getvalue", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemGetValue(CpuContext ctx) + { + var semaphoreAddress = ctx[CpuRegister.Rdi]; + var valueAddress = ctx[CpuRegister.Rsi]; + if (valueAddress == 0 || + !TryGetPosixSemaphoreHandle(ctx, semaphoreAddress, out var handle) || + !_semaphores.TryGetValue(handle, out var semaphore)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + int count; + lock (semaphore.Gate) + { + count = semaphore.Count; + } + + return TryWriteUInt32(ctx, valueAddress, unchecked((uint)count)) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "cDW233RAwWo", + ExportName = "sem_destroy", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemDestroy(CpuContext ctx) + { + var semaphoreAddress = ctx[CpuRegister.Rdi]; + if (!TryGetPosixSemaphoreHandle(ctx, semaphoreAddress, out var handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + ctx[CpuRegister.Rdi] = handle; + var result = KernelDeleteSema(ctx); if (result == (int)OrbisGen2Result.ORBIS_GEN2_OK) { - var remainingTicks = deadlineTimestamp - Stopwatch.GetTimestamp(); - var remainingMicros = remainingTicks <= 0 - ? 0u - : (uint)Math.Min(uint.MaxValue, remainingTicks / (double)Stopwatch.Frequency * 1_000_000d); - _ = ctx.TryWriteUInt32(timeoutAddress, remainingMicros); - } - else - { - _ = ctx.TryWriteUInt32(timeoutAddress, 0); + _ = TryWriteUInt32(ctx, semaphoreAddress, 0); } return result; } + private static bool TryGetPosixSemaphoreHandle(CpuContext ctx, ulong semaphoreAddress, out uint handle) + { + handle = 0; + return semaphoreAddress != 0 && + TryReadUInt32(ctx, semaphoreAddress, out handle) && + handle != 0; + } + + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + var value = (int)result; + ctx[CpuRegister.Rax] = unchecked((ulong)value); + return value; + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadNullTerminatedUtf8(CpuContext ctx, ulong address, int maxLength, out string value) + { + value = string.Empty; + if (address == 0 || maxLength <= 0) + { + return false; + } + + var bytes = new byte[Math.Min(maxLength, 4096)]; + Span current = stackalloc byte[1]; + for (var i = 0; i < bytes.Length; i++) + { + if (!ctx.Memory.TryRead(address + (ulong)i, current)) + { + return false; + } + + if (current[0] == 0) + { + value = Encoding.UTF8.GetString(bytes, 0, i); + return true; + } + + bytes[i] = current[0]; + } + + value = Encoding.UTF8.GetString(bytes); + return true; + } + // Call sites must check this before building the interpolated message; the trace // strings would otherwise be allocated on every semaphore op even with tracing off. private static readonly bool _traceSema = @@ -521,4 +584,10 @@ public static class KernelSemaphoreCompatExports { Console.Error.WriteLine($"[LOADER][TRACE] sema.{message}"); } + + private static string FormatCallSite(CpuContext ctx) + { + _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnAddress); + return $"guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} ret=0x{returnAddress:X16}"; + } } diff --git a/src/SharpEmu.Libs/LibcStdioExports.cs b/src/SharpEmu.Libs/LibcStdioExports.cs index b49c5db..b9a17af 100644 --- a/src/SharpEmu.Libs/LibcStdioExports.cs +++ b/src/SharpEmu.Libs/LibcStdioExports.cs @@ -14,9 +14,9 @@ public static class LibcStdioExports private const int MaxPathLength = 4096; private const int MaxModeLength = 16; private const int ReadChunkSize = 1024 * 1024; + private const ulong GuestFileObjectSize = 0x100; private static readonly ConcurrentDictionary _fileHandles = new(); - private static long _nextHandle = 0x1000 - 8; private static readonly bool _traceStdio = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_STDIO"), "1", StringComparison.Ordinal); @@ -96,7 +96,21 @@ public static class LibcStdioExports stream.Seek(0, SeekOrigin.End); } - var handle = unchecked((ulong)Interlocked.Add(ref _nextHandle, 8)); + // A FILE* can cross between HLE stdio and a title's bundled libc. + // Back the handle with zeroed guest memory so native helpers such + // as _Lockfilelock can safely inspect the FILE object instead of + // dereferencing the old small integer token (0x1000, 0x1008...). + if (!KernelMemoryCompatExports.TryAllocateHleData( + ctx, + GuestFileObjectSize, + alignment: 0x10, + out var handle)) + { + stream.Dispose(); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN; + } + _fileHandles[handle] = stream; if (_traceStdio) diff --git a/src/SharpEmu.Libs/Network/NetExports.cs b/src/SharpEmu.Libs/Network/NetExports.cs index 65b2dc3..c80a7fc 100644 --- a/src/SharpEmu.Libs/Network/NetExports.cs +++ b/src/SharpEmu.Libs/Network/NetExports.cs @@ -3,6 +3,9 @@ using System.Buffers.Binary; using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using System.Runtime.InteropServices; using System.Text; using SharpEmu.HLE; @@ -12,13 +15,29 @@ public static class NetExports { private const int NetErrorBadFileDescriptor = unchecked((int)0x80410109); private const int NetErrorInvalidArgument = unchecked((int)0x80410116); + private const int NetErrorWouldBlock = unchecked((int)0x80410123); + private const int NetErrorAddressInUse = unchecked((int)0x80410130); + private const int NetErrorNotInitialized = unchecked((int)0x804101C8); + private const int NetErrnoBadFileDescriptor = 9; + private const int NetErrnoInvalidArgument = 22; + private const int NetErrnoWouldBlock = 35; + private const int NetErrnoAddressInUse = 48; + private const int NetErrnoNotInitialized = 200; private const int MaxNameLength = 256; private static readonly ConcurrentDictionary _pools = new(); private static readonly ConcurrentDictionary _resolvers = new(); + private static readonly ConcurrentDictionary _sockets = new(); private static int _nextPoolId; private static int _nextResolverId = 0x2000; - private static bool _initialized; + private static int _nextSocketId = 0x4000; + // The platform networking module is usable immediately after it is loaded. + // Games and middleware (notably FMOD) can create internal sockets before an + // explicit sceNetInit call reaches application code. + private static bool _initialized = true; + + [ThreadStatic] + private static nint _errnoAddress; private sealed record NetPool(string Name, int Size, int Flags); @@ -46,10 +65,232 @@ public static class NetExports _initialized = false; _pools.Clear(); _resolvers.Clear(); + foreach (var socket in _sockets.Values) + { + socket.Dispose(); + } + _sockets.Clear(); TraceNet("term", 0, 0, 0, 0); return ctx.SetReturn(0); } + [SysAbiExport( + Nid = "Q4qBuN-c0ZM", + ExportName = "sceNetSocket", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetSocket(CpuContext ctx) + { + if (!_initialized) + { + return SetNetError(ctx, NetErrorNotInitialized, NetErrnoNotInitialized); + } + + var nameAddress = ctx[CpuRegister.Rdi]; + var family = unchecked((int)ctx[CpuRegister.Rsi]); + var type = unchecked((int)ctx[CpuRegister.Rdx]); + var protocol = unchecked((int)ctx[CpuRegister.Rcx]); + var name = TryReadUtf8Z(ctx, nameAddress, MaxNameLength, out var value) + ? value + : string.Empty; + + if (!TryTranslateSocketParameters(family, type, protocol, out var addressFamily, out var socketType, out var protocolType)) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + try + { + var socket = new Socket(addressFamily, socketType, protocolType); + var id = Interlocked.Increment(ref _nextSocketId); + _sockets[id] = socket; + TraceNet("socket.create", id, unchecked((ulong)family), unchecked((ulong)type), unchecked((ulong)protocol)); + ctx[CpuRegister.Rax] = unchecked((ulong)id); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + catch (SocketException) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + } + + [SysAbiExport( + Nid = "45ggEzakPJQ", + ExportName = "sceNetSocketClose", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetSocketClose(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + if (!_sockets.TryRemove(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + + socket.Dispose(); + TraceNet("socket.close", id, 0, 0, 0); + return ctx.SetReturn(0); + } + + [SysAbiExport( + Nid = "2mKX2Spso7I", + ExportName = "sceNetSetsockopt", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetSetsockopt(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + var level = unchecked((int)ctx[CpuRegister.Rsi]); + var option = unchecked((int)ctx[CpuRegister.Rdx]); + var valueAddress = ctx[CpuRegister.Rcx]; + var valueLength = unchecked((int)ctx[CpuRegister.R8]); + if (!_sockets.TryGetValue(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + + // ORBIS_NET_SOL_SOCKET / ORBIS_NET_SO_NBIO. This is the first option + // used by FMOD's discovery socket and maps directly to host blocking. + if (level == 0xFFFF && option == 0x1200) + { + Span value = stackalloc byte[sizeof(int)]; + if (valueLength < value.Length || valueAddress == 0 || !ctx.Memory.TryRead(valueAddress, value)) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + socket.Blocking = BinaryPrimitives.ReadInt32LittleEndian(value) == 0; + TraceNet("socket.nonblocking", id, socket.Blocking ? 0UL : 1UL, 0, 0); + return ctx.SetReturn(0); + } + + // ORBIS_NET_SO_REUSEADDR uses the BSD value 0x0004. + if (level == 0xFFFF && option == 0x0004) + { + Span value = stackalloc byte[sizeof(int)]; + if (valueLength < value.Length || valueAddress == 0 || !ctx.Memory.TryRead(valueAddress, value)) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + socket.SetSocketOption( + SocketOptionLevel.Socket, + SocketOptionName.ReuseAddress, + BinaryPrimitives.ReadInt32LittleEndian(value) != 0); + TraceNet("socket.reuseaddr", id, BinaryPrimitives.ReadUInt32LittleEndian(value), 0, 0); + return ctx.SetReturn(0); + } + + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + [SysAbiExport( + Nid = "bErx49PgxyY", + ExportName = "sceNetBind", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetBind(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + if (!_sockets.TryGetValue(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + if (!TryReadSocketAddress(ctx, ctx[CpuRegister.Rsi], unchecked((int)ctx[CpuRegister.Rdx]), out var endpoint)) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + try + { + socket.Bind(endpoint); + TraceNet("socket.bind", id, unchecked((ulong)endpoint.Port), 0, 0); + return ctx.SetReturn(0); + } + catch (SocketException ex) when (ex.SocketErrorCode == SocketError.AddressAlreadyInUse) + { + return SetNetError(ctx, NetErrorAddressInUse, NetErrnoAddressInUse); + } + catch (SocketException) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + } + + [SysAbiExport( + Nid = "kOj1HiAGE54", + ExportName = "sceNetListen", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetListen(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + if (!_sockets.TryGetValue(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + + try + { + socket.Listen(Math.Max(0, unchecked((int)ctx[CpuRegister.Rsi]))); + TraceNet("socket.listen", id, ctx[CpuRegister.Rsi], 0, 0); + return ctx.SetReturn(0); + } + catch (SocketException) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + } + + [SysAbiExport( + Nid = "PIWqhn9oSxc", + ExportName = "sceNetAccept", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetAccept(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + if (!_sockets.TryGetValue(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + + try + { + var accepted = socket.Accept(); + var acceptedId = Interlocked.Increment(ref _nextSocketId); + _sockets[acceptedId] = accepted; + TraceNet("socket.accept", acceptedId, unchecked((ulong)id), 0, 0); + ctx[CpuRegister.Rax] = unchecked((ulong)acceptedId); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + catch (SocketException ex) when (ex.SocketErrorCode is SocketError.WouldBlock or SocketError.IOPending) + { + return SetNetError(ctx, NetErrorWouldBlock, NetErrnoWouldBlock); + } + catch (SocketException) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + } + + [SysAbiExport( + Nid = "HQOwnfMGipQ", + ExportName = "sceNetErrnoLoc", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetErrnoLoc(CpuContext ctx) + { + if (_errnoAddress == 0) + { + _errnoAddress = Marshal.AllocHGlobal(sizeof(int)); + Marshal.WriteInt32(_errnoAddress, 0); + } + + ctx[CpuRegister.Rax] = unchecked((ulong)_errnoAddress); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + [SysAbiExport( Nid = "dgJBaeJnGpo", ExportName = "sceNetPoolCreate", @@ -207,6 +448,69 @@ public static class NetExports : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + private static int SetNetError(CpuContext ctx, int result, int errno) + { + if (_errnoAddress == 0) + { + _errnoAddress = Marshal.AllocHGlobal(sizeof(int)); + } + Marshal.WriteInt32(_errnoAddress, errno); + return ctx.SetReturn(result); + } + + private static bool TryTranslateSocketParameters( + int family, + int type, + int protocol, + out AddressFamily addressFamily, + out SocketType socketType, + out ProtocolType protocolType) + { + addressFamily = family switch + { + 2 => AddressFamily.InterNetwork, + 28 => AddressFamily.InterNetworkV6, + _ => AddressFamily.Unspecified, + }; + socketType = type switch + { + 1 => SocketType.Stream, + 2 => SocketType.Dgram, + _ => SocketType.Unknown, + }; + protocolType = protocol switch + { + 0 when socketType == SocketType.Stream => ProtocolType.Tcp, + 0 when socketType == SocketType.Dgram => ProtocolType.Udp, + 6 => ProtocolType.Tcp, + 17 => ProtocolType.Udp, + _ => ProtocolType.Unknown, + }; + + return addressFamily != AddressFamily.Unspecified && + socketType != SocketType.Unknown && + protocolType != ProtocolType.Unknown; + } + + private static bool TryReadSocketAddress(CpuContext ctx, ulong address, int length, out IPEndPoint endpoint) + { + endpoint = new IPEndPoint(IPAddress.Any, 0); + if (address == 0 || length < 16) + { + return false; + } + + Span bytes = stackalloc byte[16]; + if (!ctx.Memory.TryRead(address, bytes) || bytes[1] != 2) + { + return false; + } + + var port = BinaryPrimitives.ReadUInt16BigEndian(bytes[2..4]); + endpoint = new IPEndPoint(new IPAddress(bytes[4..8]), port); + return true; + } + private static bool TryReadUtf8Z(CpuContext ctx, ulong address, int maxLength, out string value) { value = string.Empty; diff --git a/src/SharpEmu.Libs/Ngs2/Ngs2Exports.cs b/src/SharpEmu.Libs/Ngs2/Ngs2Exports.cs index dc3af01..f93c047 100644 --- a/src/SharpEmu.Libs/Ngs2/Ngs2Exports.cs +++ b/src/SharpEmu.Libs/Ngs2/Ngs2Exports.cs @@ -4,6 +4,7 @@ using SharpEmu.HLE; using SharpEmu.Libs.Kernel; using System.Buffers.Binary; +using System.Threading; namespace SharpEmu.Libs.Ngs2; @@ -13,7 +14,7 @@ public static class Ngs2Exports private const int OrbisNgs2ErrorInvalidSystemHandle = unchecked((int)0x804A0230); private const int OrbisNgs2ErrorInvalidRackHandle = unchecked((int)0x804A0261); private const int OrbisNgs2ErrorInvalidVoiceHandle = unchecked((int)0x804A0300); - private const ulong HandleStorageSize = 0x1000; + private const ulong HandleStorageSize = 0x20; private const int RenderBufferInfoSize = 0x18; private const ulong MaximumRenderBufferSize = 16 * 1024 * 1024; @@ -38,13 +39,13 @@ public static class Ngs2Exports var outHandleAddress = ctx[CpuRegister.Rdx]; if (outHandleAddress == 0) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidOutAddress); + return SetReturn(ctx, OrbisNgs2ErrorInvalidOutAddress); } if (!TryCreateHandle(ctx, type: 1, ownerHandle: 0, out var handle) || !ctx.TryWriteUInt64(outHandleAddress, handle)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } lock (StateGate) @@ -52,7 +53,7 @@ public static class Ngs2Exports Systems[handle] = new SystemState(unchecked((uint)Interlocked.Increment(ref _nextUid))); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -67,7 +68,7 @@ public static class Ngs2Exports { if (!Systems.Remove(handle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidSystemHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidSystemHandle); } var rackHandles = Racks @@ -80,7 +81,7 @@ public static class Ngs2Exports } } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -97,19 +98,19 @@ public static class Ngs2Exports { if (!Systems.ContainsKey(systemHandle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidSystemHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidSystemHandle); } } if (outHandleAddress == 0) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidOutAddress); + return SetReturn(ctx, OrbisNgs2ErrorInvalidOutAddress); } if (!TryCreateHandle(ctx, type: 2, systemHandle, out var handle) || !ctx.TryWriteUInt64(outHandleAddress, handle)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } lock (StateGate) @@ -117,7 +118,7 @@ public static class Ngs2Exports Racks[handle] = new RackState(systemHandle, rackId); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -132,13 +133,13 @@ public static class Ngs2Exports { if (!Racks.ContainsKey(handle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidRackHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidRackHandle); } RemoveRackLocked(handle); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -155,7 +156,7 @@ public static class Ngs2Exports { if (!Racks.ContainsKey(rackHandle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidRackHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidRackHandle); } var existing = Voices.FirstOrDefault( @@ -163,20 +164,20 @@ public static class Ngs2Exports if (existing.Key != 0) { return ctx.TryWriteUInt64(outHandleAddress, existing.Key) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } if (outHandleAddress == 0) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidOutAddress); + return SetReturn(ctx, OrbisNgs2ErrorInvalidOutAddress); } if (!TryCreateHandle(ctx, type: 4, rackHandle, out var handle) || !ctx.TryWriteUInt64(outHandleAddress, handle)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } lock (StateGate) @@ -184,7 +185,7 @@ public static class Ngs2Exports Voices[handle] = new VoiceState(rackHandle, voiceIndex); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -196,19 +197,12 @@ public static class Ngs2Exports { lock (StateGate) { - return ctx.SetReturn( - Voices.ContainsKey(ctx[CpuRegister.Rdi]) - ? 0 - : OrbisNgs2ErrorInvalidVoiceHandle); + return SetReturn( + ctx, + Voices.ContainsKey(ctx[CpuRegister.Rdi]) ? 0 : OrbisNgs2ErrorInvalidVoiceHandle); } } - // The earlier "alt NID" guesses here were wrong: an NID is the aerolib hash of one - // symbol name, and hashing scripts/ps5_names.txt shows the previous alt bindings - // actually belonged to sceImeUpdate (-4GCfYdNF1s), sceMouseRead (x8qnXqh-tiM) and - // sceSystemGestureUpdateAllTouchRecognizer (wPJGwI2RM2I) — Quake's audio thread was - // receiving an NGS2 error from what it thought was sceImeUpdate. Those now live in - // their real libraries; the NIDs below are verified against the hash. [SysAbiExport( Nid = "AbYvTOZ8Pts", ExportName = "sceNgs2VoiceRunCommands", @@ -216,48 +210,6 @@ public static class Ngs2Exports LibraryName = "libSceNgs2")] public static int Ngs2VoiceRunCommands(CpuContext ctx) => Ngs2VoiceControl(ctx); - [SysAbiExport( - Nid = "-TOuuAQ-buE", - ExportName = "sceNgs2VoiceGetState", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2VoiceGetState(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "rEh728kXk3w", - ExportName = "sceNgs2VoiceGetStateFlags", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2VoiceGetStateFlags(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "xa8oL9dmXkM", - ExportName = "sceNgs2PanInit", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2PanInit(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "1WsleK-MTkE", - ExportName = "sceNgs2GeomCalcListener", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2GeomCalcListener(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "0lbbayqDNoE", - ExportName = "sceNgs2GeomResetSourceParam", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2GeomResetSourceParam(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "7Lcfo8SmpsU", - ExportName = "sceNgs2GeomResetListenerParam", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2GeomResetListenerParam(CpuContext ctx) => ctx.SetReturn(0); - [SysAbiExport( Nid = "i0VnXM-C9fc", ExportName = "sceNgs2SystemRender", @@ -272,13 +224,13 @@ public static class Ngs2Exports { if (!Systems.ContainsKey(systemHandle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidSystemHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidSystemHandle); } } if (bufferInfoCount != 0 && bufferInfoAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } for (uint i = 0; i < bufferInfoCount; i++) @@ -287,14 +239,14 @@ public static class Ngs2Exports if (!ctx.TryReadUInt64(entryAddress, out var bufferAddress) || !ctx.TryReadUInt64(entryAddress + 8, out var bufferSize)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (bufferAddress != 0 && bufferSize != 0) { if (bufferSize > MaximumRenderBufferSize || !TryClearGuestBuffer(ctx, bufferAddress, bufferSize)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } } @@ -306,11 +258,138 @@ public static class Ngs2Exports $"[LOADER][TRACE] ngs2.render#{count} system=0x{systemHandle:X16} buffers={bufferInfoCount}"); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "pgFAiLR5qT4", + ExportName = "sceNgs2SystemQueryBufferSize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemQueryBufferSize(CpuContext ctx) => WriteBufferSize(ctx, ctx[CpuRegister.Rsi]); + + [SysAbiExport( + Nid = "0eFLVCfWVds", + ExportName = "sceNgs2RackQueryBufferSize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2RackQueryBufferSize(CpuContext ctx) => WriteBufferSize(ctx, ctx[CpuRegister.Rdx]); + + // Report a fixed working-memory footprint for the requested object. The + // out struct (SceNgs2BufferAllocator-style) begins with the size field. + private static int WriteBufferSize(CpuContext ctx, ulong outAddress) + { + if (outAddress == 0) + { + return SetReturn(ctx, OrbisNgs2ErrorInvalidOutAddress); + } + + Span info = stackalloc byte[RenderBufferInfoSize]; + info.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(info[0..8], 0x10000); + BinaryPrimitives.WriteUInt64LittleEndian(info[8..16], 0x100); + return ctx.Memory.TryWrite(outAddress, info) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "l4Q2dWEH6UM", + ExportName = "sceNgs2SystemSetGrainSamples", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemSetGrainSamples(CpuContext ctx) => ValidateSystem(ctx); + + [SysAbiExport( + Nid = "-tbc2SxQD60", + ExportName = "sceNgs2SystemSetSampleRate", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemSetSampleRate(CpuContext ctx) => ValidateSystem(ctx); + + [SysAbiExport( + Nid = "gThZqM5PYlQ", + ExportName = "sceNgs2SystemLock", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemLock(CpuContext ctx) => ValidateSystem(ctx); + + [SysAbiExport( + Nid = "JXRC5n0RQls", + ExportName = "sceNgs2SystemUnlock", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemUnlock(CpuContext ctx) => ValidateSystem(ctx); + + [SysAbiExport( + Nid = "-TOuuAQ-buE", + ExportName = "sceNgs2VoiceGetState", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2VoiceGetState(CpuContext ctx) + { + var voiceHandle = ctx[CpuRegister.Rdi]; + var stateAddress = ctx[CpuRegister.Rsi]; + var stateSize = (int)Math.Min(ctx[CpuRegister.Rdx], 0x400); + lock (StateGate) + { + if (!Voices.ContainsKey(voiceHandle)) + { + return SetReturn(ctx, OrbisNgs2ErrorInvalidVoiceHandle); + } + } + + // Report an idle (not-in-use) voice: all-zero state block. + if (stateAddress != 0 && stateSize > 0) + { + if (!TryClearGuestBuffer(ctx, stateAddress, (ulong)stateSize)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + } + + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "rEh728kXk3w", + ExportName = "sceNgs2VoiceGetStateFlags", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2VoiceGetStateFlags(CpuContext ctx) + { + var voiceHandle = ctx[CpuRegister.Rdi]; + var flagsAddress = ctx[CpuRegister.Rsi]; + lock (StateGate) + { + if (!Voices.ContainsKey(voiceHandle)) + { + return SetReturn(ctx, OrbisNgs2ErrorInvalidVoiceHandle); + } + } + + // No flags set: voice is idle. + if (flagsAddress != 0 && !ctx.TryWriteUInt64(flagsAddress, 0)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetReturn(ctx, 0); + } + + private static int ValidateSystem(CpuContext ctx) + { + lock (StateGate) + { + return SetReturn( + ctx, + Systems.ContainsKey(ctx[CpuRegister.Rdi]) ? 0 : OrbisNgs2ErrorInvalidSystemHandle); + } } private static bool TryCreateHandle(CpuContext ctx, uint type, ulong ownerHandle, out ulong handle) { + handle = 0; if (!KernelMemoryCompatExports.TryAllocateHleData(ctx, HandleStorageSize, 16, out handle)) { return false; @@ -318,18 +397,11 @@ public static class Ngs2Exports Span data = stackalloc byte[(int)HandleStorageSize]; data.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(data[0..8], handle); BinaryPrimitives.WriteUInt64LittleEndian(data[8..16], ownerHandle); BinaryPrimitives.WriteUInt32LittleEndian(data[16..20], 1); BinaryPrimitives.WriteUInt32LittleEndian(data[24..28], type); - if (!ctx.Memory.TryWrite(handle, data)) - { - return false; - } - - // Offset 0 doubles as a vtable slot for guest code that treats this handle as a C++ - // object; stamp it with the shared dummy vtable instead of a self-referential value. - KernelMemoryCompatExports.TryWriteDummyVtable(ctx, handle); - return true; + return ctx.Memory.TryWrite(handle, data); } private static bool TryClearGuestBuffer(CpuContext ctx, ulong address, ulong length) @@ -367,4 +439,37 @@ public static class Ngs2Exports Environment.GetEnvironmentVariable("SHARPEMU_LOG_NGS2"), "1", StringComparison.Ordinal); + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + [SysAbiExport( + Nid = "xa8oL9dmXkM", + ExportName = "sceNgs2PanInit", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2PanInit(CpuContext ctx) => ctx.SetReturn(0); + + [SysAbiExport( + Nid = "1WsleK-MTkE", + ExportName = "sceNgs2GeomCalcListener", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2GeomCalcListener(CpuContext ctx) => ctx.SetReturn(0); + + [SysAbiExport( + Nid = "0lbbayqDNoE", + ExportName = "sceNgs2GeomResetSourceParam", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2GeomResetSourceParam(CpuContext ctx) => ctx.SetReturn(0); + + [SysAbiExport( + Nid = "7Lcfo8SmpsU", + ExportName = "sceNgs2GeomResetListenerParam", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2GeomResetListenerParam(CpuContext ctx) => ctx.SetReturn(0); } diff --git a/src/SharpEmu.Libs/Np/NpManagerExports.cs b/src/SharpEmu.Libs/Np/NpManagerExports.cs index f4e6c08..e5d44fe 100644 --- a/src/SharpEmu.Libs/Np/NpManagerExports.cs +++ b/src/SharpEmu.Libs/Np/NpManagerExports.cs @@ -10,7 +10,7 @@ public static class NpManagerExports { private const int NpTitleIdSize = 16; private const int NpTitleSecretSize = 128; - private const int NpReachabilityStateUnavailable = 0; + private const int NpErrorInvalidArgument = unchecked((int)0x80550003); [SysAbiExport( Nid = "3Zl8BePTh9Y", @@ -58,35 +58,6 @@ public static class NpManagerExports return WriteOfflineOnlineId(ctx, ctx[CpuRegister.Rsi]); } - [SysAbiExport( - Nid = "rbknaUjpqWo", - ExportName = "sceNpGetAccountIdA", - Target = Generation.Gen5, - LibraryName = "libSceNpManager")] - public static int NpGetOnlineIdA(CpuContext ctx) - { - // The A variant takes a user ID before OnlineId. - return WriteOfflineOnlineId(ctx, ctx[CpuRegister.Rsi]); - } - - [SysAbiExport( - Nid = "e-ZuhGEoeC4", - ExportName = "sceNpGetNpReachabilityState", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNpManager")] - public static int NpGetNpReachabilityState(CpuContext ctx) - { - var stateAddress = ctx[CpuRegister.Rsi]; - if (stateAddress == 0) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - return ctx.TryWriteInt32(stateAddress, NpReachabilityStateUnavailable) - ? ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK) - : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - [SysAbiExport( Nid = "VfRSmPmj8Q8", ExportName = "sceNpRegisterStateCallback", @@ -140,6 +111,77 @@ public static class NpManagerExports : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "rbknaUjpqWo", + ExportName = "sceNpGetAccountIdA", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNpManager")] + public static int NpGetAccountIdA(CpuContext ctx) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var accountIdAddress = ctx[CpuRegister.Rsi]; + if (userId == -1 || accountIdAddress == 0) + { + return SetReturn(ctx, NpErrorInvalidArgument); + } + + // The offline profile exposed by sceNpGetState is signed in. Keep the + // account query consistent with that state: Unity's PSN integration + // treats SIGNED_OUT as an exceptional state and retries it every frame. + // A stable local-only id is sufficient for titles which only use the + // value as a profile key. + Span accountId = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(accountId, 1); + return ctx.Memory.TryWrite(accountIdAddress, accountId) + ? SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "JT+t00a3TxA", + ExportName = "sceNpGetAccountCountryA", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNpManager")] + public static int NpGetAccountCountryA(CpuContext ctx) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var countryAddress = ctx[CpuRegister.Rsi]; + if (userId == -1 || countryAddress == 0) + { + return SetReturn(ctx, NpErrorInvalidArgument); + } + + Span country = stackalloc byte[4]; + country[0] = (byte)'U'; + country[1] = (byte)'S'; + country[2] = 0; + country[3] = 0; + return ctx.Memory.TryWrite(countryAddress, country) + ? SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "e-ZuhGEoeC4", + ExportName = "sceNpGetNpReachabilityState", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNpManager")] + public static int NpGetNpReachabilityState(CpuContext ctx) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var stateAddress = ctx[CpuRegister.Rsi]; + if (userId == -1 || stateAddress == 0) + { + return SetReturn(ctx, NpErrorInvalidArgument); + } + + Span state = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(state, 0); // Unavailable while offline. + return ctx.Memory.TryWrite(stateAddress, state) + ? SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + [SysAbiExport( Nid = "Ec63y59l9tw", ExportName = "sceNpSetNpTitleId", @@ -166,6 +208,12 @@ public static class NpManagerExports return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + private static string ReadTitleId(ReadOnlySpan bytes) { var length = 0; diff --git a/src/SharpEmu.Libs/Pad/HostWindowInput.cs b/src/SharpEmu.Libs/Pad/HostWindowInput.cs index 9a609b6..7494cba 100644 --- a/src/SharpEmu.Libs/Pad/HostWindowInput.cs +++ b/src/SharpEmu.Libs/Pad/HostWindowInput.cs @@ -40,6 +40,11 @@ public static class HostWindowInput { keyboard.KeyDown += (_, key, _) => { + if (key == Key.F1) + { + VideoOut.PerfOverlay.Toggle(); + } + lock (Gate) { Pressed.Add(key); diff --git a/src/SharpEmu.Libs/Pad/ImeExports.cs b/src/SharpEmu.Libs/Pad/ImeExports.cs new file mode 100644 index 0000000..9cbee88 --- /dev/null +++ b/src/SharpEmu.Libs/Pad/ImeExports.cs @@ -0,0 +1,41 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; + +namespace SharpEmu.Libs.Pad; + +public static class ImeExports +{ + private const int PrimaryUserId = 0x10000000; + private const int ImeErrorInvalidAddress = unchecked((int)0x80BC0001); + private const int ImeErrorInvalidUserId = unchecked((int)0x80BC0010); + private const int ImeErrorNotOpened = unchecked((int)0x80BC0005); + + private static bool _keyboardOpen; + public static int ImeKeyboardOpen(CpuContext ctx) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var parameterAddress = ctx[CpuRegister.Rsi]; + if (parameterAddress == 0) + { + return SetReturn(ctx, ImeErrorInvalidAddress); + } + + if (userId != PrimaryUserId) + { + return SetReturn(ctx, ImeErrorInvalidUserId); + } + + _keyboardOpen = true; + return SetReturn(ctx, 0); + } + public static int ImeUpdate(CpuContext ctx) => + SetReturn(ctx, _keyboardOpen ? 0 : ImeErrorNotOpened); + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } +} diff --git a/src/SharpEmu.Libs/Pad/MouseExports.cs b/src/SharpEmu.Libs/Pad/MouseExports.cs new file mode 100644 index 0000000..f87ace5 --- /dev/null +++ b/src/SharpEmu.Libs/Pad/MouseExports.cs @@ -0,0 +1,100 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Diagnostics; +using SharpEmu.HLE; + +namespace SharpEmu.Libs.Pad; + +public static class MouseExports +{ + private const int PrimaryUserId = 0x10000000; + private const int MouseDataSize = 0x28; + private const int MouseErrorInvalidArgument = unchecked((int)0x80020002); + private const int MouseErrorInvalidHandle = unchecked((int)0x80020003); + private const int MouseErrorNotInitialized = unchecked((int)0x80020005); + private const int MouseErrorAlreadyOpened = unchecked((int)0x80020008); + + private static readonly bool[] OpenHandles = new bool[2]; + private static bool _initialized; + public static int MouseInit(CpuContext ctx) + { + _initialized = true; + Array.Clear(OpenHandles); + return SetReturn(ctx, 0); + } + public static int MouseOpen(CpuContext ctx) + { + if (!_initialized) + { + return SetReturn(ctx, MouseErrorNotInitialized); + } + + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var type = unchecked((int)ctx[CpuRegister.Rsi]); + var index = unchecked((int)ctx[CpuRegister.Rdx]); + if (userId != PrimaryUserId || type != 0 || index is < 0 or > 1) + { + return SetReturn(ctx, MouseErrorInvalidArgument); + } + + if (OpenHandles[index]) + { + return SetReturn(ctx, MouseErrorAlreadyOpened); + } + + OpenHandles[index] = true; + return SetReturn(ctx, index); + } + public static int MouseRead(CpuContext ctx) + { + var handle = unchecked((int)ctx[CpuRegister.Rdi]); + var dataAddress = ctx[CpuRegister.Rsi]; + var count = unchecked((int)ctx[CpuRegister.Rdx]); + if (dataAddress == 0 || count is < 1 or > 64) + { + return SetReturn(ctx, MouseErrorInvalidArgument); + } + + if (handle is < 0 or > 1 || !OpenHandles[handle]) + { + return SetReturn(ctx, MouseErrorInvalidHandle); + } + + Span data = stackalloc byte[MouseDataSize]; + data.Clear(); + var ticks = Stopwatch.GetTimestamp(); + var timestamp = + ((ulong)(ticks / Stopwatch.Frequency) * 1_000_000UL) + + ((ulong)(ticks % Stopwatch.Frequency) * 1_000_000UL / (ulong)Stopwatch.Frequency); + BinaryPrimitives.WriteUInt64LittleEndian(data, timestamp); + data[0x08] = 0; // No host mouse is exposed to the guest yet. + return ctx.Memory.TryWrite(dataAddress, data) + ? SetReturn(ctx, 1) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "cAnT0Rw-IwU", + ExportName = "sceMouseClose", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceMouse")] + public static int MouseClose(CpuContext ctx) + { + var handle = unchecked((int)ctx[CpuRegister.Rdi]); + if (handle is < 0 or > 1 || !OpenHandles[handle]) + { + return SetReturn(ctx, MouseErrorInvalidHandle); + } + + OpenHandles[handle] = false; + return SetReturn(ctx, 0); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } +} diff --git a/src/SharpEmu.Libs/Pad/PadExports.cs b/src/SharpEmu.Libs/Pad/PadExports.cs index 54a595f..f5253ac 100644 --- a/src/SharpEmu.Libs/Pad/PadExports.cs +++ b/src/SharpEmu.Libs/Pad/PadExports.cs @@ -14,7 +14,11 @@ public static class PadExports private const int OrbisPadErrorNotInitialized = unchecked((int)0x80920005); private const int OrbisPadErrorDeviceNotConnected = unchecked((int)0x80920007); private const int OrbisPadErrorDeviceNoHandle = unchecked((int)0x80920008); - private const int PrimaryUserId = 1000; + // Keep the pad session on the same retail user id returned by + // libSceUserService. A mismatched emulator-local id makes games pass a + // valid 0x10000000 user to scePadOpen and receive DEVICE_NOT_CONNECTED, + // leaving every later keyboard/gamepad read on an invalid handle. + private const int PrimaryUserId = 0x10000000; private const int StandardPortType = 0; private const int PrimaryPadHandle = 1; private const int ControllerInformationSize = 0x1C; diff --git a/src/SharpEmu.Libs/SaveData/SaveDataExports.cs b/src/SharpEmu.Libs/SaveData/SaveDataExports.cs index b05a3ab..9d06104 100644 --- a/src/SharpEmu.Libs/SaveData/SaveDataExports.cs +++ b/src/SharpEmu.Libs/SaveData/SaveDataExports.cs @@ -30,19 +30,15 @@ public static class SaveDataExports private const uint MountModeCreate2 = 1u << 5; private const int MountResultSize = 0x40; private static readonly object _stateGate = new(); - private static readonly HashSet _transactionResources = []; private static readonly HashSet _preparedTransactionResources = []; private static string? _titleId; - private static int _nextTransactionResource; public static void ConfigureApplicationInfo(string? titleId) { lock (_stateGate) { _titleId = string.IsNullOrWhiteSpace(titleId) ? null : SanitizePathSegment(titleId.Trim()); - _transactionResources.Clear(); _preparedTransactionResources.Clear(); - _nextTransactionResource = 0; } } @@ -56,15 +52,15 @@ public static class SaveDataExports try { Directory.CreateDirectory(ResolveSaveDataRoot()); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } catch (IOException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } catch (UnauthorizedAccessException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } } @@ -79,18 +75,18 @@ public static class SaveDataExports var resultAddress = ctx[CpuRegister.Rsi]; if (condAddress == 0 || resultAddress == 0) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } if (!TryReadSearchCond(ctx, condAddress, out var cond) || !TryReadSearchResult(ctx, resultAddress, out var result)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (cond.UserId < 0 || cond.SortKey > SortKeyFreeBlocks || cond.SortOrder > SortOrderDescent) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } try @@ -102,7 +98,7 @@ public static class SaveDataExports } else if (!TryReadFixedAscii(ctx, cond.TitleIdAddress, SaveDataTitleIdSize, out titleId)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } var root = ResolveTitleSaveRoot(cond.UserId, titleId); @@ -114,21 +110,21 @@ public static class SaveDataExports var setNum = result.DirNamesNum == 0 ? 0 : Math.Min(result.DirNamesNum, entries.Count); - if (!ctx.TryWriteUInt32(resultAddress + ResultHitNumOffset, checked((uint)entries.Count)) || - !ctx.TryWriteUInt32(resultAddress + ResultSetNumOffset, checked((uint)setNum))) + if (!TryWriteUInt32(ctx, resultAddress + ResultHitNumOffset, checked((uint)entries.Count)) || + !TryWriteUInt32(ctx, resultAddress + ResultSetNumOffset, checked((uint)setNum))) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (setNum == 0) { TraceSaveData($"dir_name_search user={cond.UserId} title={titleId} hits={entries.Count} set=0 root='{root}'"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } if (result.DirNamesAddress == 0) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } for (var i = 0; i < setNum; i++) @@ -144,20 +140,20 @@ public static class SaveDataExports (result.InfosAddress != 0 && !TryWriteSearchInfo(ctx, result.InfosAddress + ((ulong)i * SaveDataSearchInfoSize), entry))) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } TraceSaveData($"dir_name_search user={cond.UserId} title={titleId} hits={entries.Count} set={setNum} root='{root}'"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } catch (IOException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } catch (UnauthorizedAccessException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } } @@ -172,25 +168,25 @@ public static class SaveDataExports var resultAddress = ctx[CpuRegister.Rsi]; if (mountAddress == 0 || resultAddress == 0) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } - if (!ctx.TryReadInt32(mountAddress, out var userId) || + if (!TryReadInt32(ctx, mountAddress, out var userId) || !ctx.TryReadUInt64(mountAddress + 0x08, out var dirNameAddress) || !ctx.TryReadUInt64(mountAddress + 0x10, out var blocks) || !ctx.TryReadUInt64(mountAddress + 0x18, out var systemBlocks) || - !ctx.TryReadUInt32(mountAddress + 0x20, out var mountMode) || - !ctx.TryReadUInt32(mountAddress + 0x24, out var resource) || - !ctx.TryReadUInt32(mountAddress + 0x28, out var mode) || + !TryReadUInt32(ctx, mountAddress + 0x20, out var mountMode) || + !TryReadUInt32(ctx, mountAddress + 0x24, out var resource) || + !TryReadUInt32(ctx, mountAddress + 0x28, out var mode) || dirNameAddress == 0 || !TryReadFixedAscii(ctx, dirNameAddress, SaveDataDirNameSize, out var dirName)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (userId < 0 || string.IsNullOrWhiteSpace(dirName)) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } try @@ -205,12 +201,12 @@ public static class SaveDataExports if (!existed && !create && !createIfMissing) { - return ctx.SetReturn(OrbisSaveDataErrorNotFound); + return SetReturn(ctx, OrbisSaveDataErrorNotFound); } if (existed && create) { - return ctx.SetReturn(OrbisSaveDataErrorExists); + return SetReturn(ctx, OrbisSaveDataErrorExists); } if (!existed) @@ -227,29 +223,30 @@ public static class SaveDataExports BinaryPrimitives.WriteUInt32LittleEndian(result[0x1C..], createIfMissing && !existed ? 1u : 0u); if (!ctx.Memory.TryWrite(resultAddress, result)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceSaveData( $"mount3 user={userId} title={titleId} dir={dirName} blocks={blocks} " + $"system_blocks={systemBlocks} mount_mode=0x{mountMode:X} resource={resource} mode={mode} " + $"mount_point={mountPoint} created={!existed} root='{savePath}'"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } catch (IOException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } catch (UnauthorizedAccessException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } catch (ArgumentException) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } } + private static int _nextTransactionResource; [SysAbiExport( Nid = "gjRZNnw0JPE", ExportName = "sceSaveDataCreateTransactionResource", @@ -257,16 +254,39 @@ public static class SaveDataExports LibraryName = "libSceSaveData")] public static int SaveDataCreateTransactionResource(CpuContext ctx) { - var memorySize = ctx[CpuRegister.Rdi]; - int resource; - lock (_stateGate) + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var reserved = ctx[CpuRegister.Rsi]; + + var id = (uint)Interlocked.Increment(ref _nextTransactionResource); + + // The resource-out pointer's argument slot varies by SDK revision: some + // callers pass it in rdx, others in rcx (a 4-arg form where rdx holds a + // count/flag). Void Terrarium passes rdx=0x1 (not a pointer) and the + // real out-pointer in rcx. Probe the plausible candidates and write the + // handle to the first writable one instead of faulting on a bad rdx. + // This is a stub-level create (matches shadPS4's return-OK semantics); + // never return MEMORY_FAULT for it, or the guest treats savedata init as + // failed and never advances. + var resourceAddress = 0UL; + foreach (var candidate in new[] + { + ctx[CpuRegister.Rdx], + ctx[CpuRegister.Rcx], + ctx[CpuRegister.R8], + ctx[CpuRegister.R9], + }) { - resource = ++_nextTransactionResource; - _transactionResources.Add(resource); + if (candidate != 0 && TryWriteUInt32(ctx, candidate, id)) + { + resourceAddress = candidate; + break; + } } - TraceSaveData($"create_transaction_resource memory_size=0x{memorySize:X} resource={resource}"); - return ctx.SetReturn(resource); + TraceSaveData( + $"create_transaction_resource user={userId} reserved=0x{reserved:X} resource_addr=0x{resourceAddress:X} id={id}"); + + return SetReturn(ctx, 0); } [SysAbiExport( @@ -279,70 +299,11 @@ public static class SaveDataExports var resource = unchecked((int)ctx[CpuRegister.Rdi]); lock (_stateGate) { - _transactionResources.Remove(resource); _preparedTransactionResources.Remove(resource); } TraceSaveData($"delete_transaction_resource resource={resource}"); - return ctx.SetReturn(0); - } - - [SysAbiExport( - Nid = "sDCBrmc61XU", - ExportName = "sceSaveDataPrepare", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceSaveData")] - public static int SaveDataPrepare(CpuContext ctx) - { - var mountPointAddress = ctx[CpuRegister.Rdi]; - var resource = unchecked((int)ctx[CpuRegister.Rdx]); - if (mountPointAddress == 0) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - if (!TryReadFixedAscii(ctx, mountPointAddress, 16, out var mountPoint)) - { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - if (string.IsNullOrWhiteSpace(mountPoint)) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - lock (_stateGate) - { - if (resource != 0) - { - _preparedTransactionResources.Add(resource); - } - } - - TraceSaveData($"prepare mount_point={mountPoint} resource={resource}"); - return ctx.SetReturn(0); - } - - [SysAbiExport( - Nid = "ie7qhZ4X0Cc", - ExportName = "sceSaveDataCommit", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceSaveData")] - public static int SaveDataCommit(CpuContext ctx) - { - var commitAddress = ctx[CpuRegister.Rdi]; - if (commitAddress == 0) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - lock (_stateGate) - { - _preparedTransactionResources.Clear(); - } - - TraceSaveData($"commit commit=0x{commitAddress:X16}"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -352,40 +313,21 @@ public static class SaveDataExports LibraryName = "libSceSaveData")] public static int SaveDataUmount2(CpuContext ctx) { - var mountPointAddress = ctx[CpuRegister.Rdi]; - if (mountPointAddress == 0) - { - mountPointAddress = ctx[CpuRegister.Rsi]; - } - - if (mountPointAddress == 0) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - if (!TryReadFixedAscii(ctx, mountPointAddress, 16, out var mountPoint)) - { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - if (string.IsNullOrWhiteSpace(mountPoint)) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - var unmounted = KernelMemoryCompatExports.TryUnregisterGuestPathMount(mountPoint); - TraceSaveData($"umount2 mount_point={mountPoint} unregistered={unmounted}"); - return ctx.SetReturn(0); + // Unmounting a save directory always succeeds in the stub filesystem; + // returning an error here makes the game's save flow stall before it + // hands control to the title/gameplay state. + TraceSaveData($"umount2 user={unchecked((int)ctx[CpuRegister.Rdi])}"); + return SetReturn(ctx, 0); } private static bool TryReadSearchCond(CpuContext ctx, ulong address, out SearchCond cond) { cond = default; - if (!ctx.TryReadInt32(address, out var userId) || + if (!TryReadInt32(ctx, address, out var userId) || !ctx.TryReadUInt64(address + 0x08, out var titleIdAddress) || !ctx.TryReadUInt64(address + 0x10, out var dirNameAddress) || - !ctx.TryReadUInt32(address + 0x18, out var sortKey) || - !ctx.TryReadUInt32(address + 0x1C, out var sortOrder)) + !TryReadUInt32(ctx, address + 0x18, out var sortKey) || + !TryReadUInt32(ctx, address + 0x1C, out var sortOrder)) { return false; } @@ -408,7 +350,7 @@ public static class SaveDataExports { result = default; if (!ctx.TryReadUInt64(address + ResultDirNamesOffset, out var dirNamesAddress) || - !ctx.TryReadUInt32(address + ResultDirNamesNumOffset, out var dirNamesNum) || + !TryReadUInt32(ctx, address + ResultDirNamesNumOffset, out var dirNamesNum) || !ctx.TryReadUInt64(address + ResultParamsOffset, out var paramsAddress) || !ctx.TryReadUInt64(address + ResultInfosOffset, out var infosAddress)) { @@ -609,6 +551,45 @@ public static class SaveDataExports } } + private static bool TryReadInt32(CpuContext ctx, ulong address, out int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt32LittleEndian(bytes); + return true; + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + private static void TraceSaveData(string message) { if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_SAVEDATA"), "1", StringComparison.Ordinal)) @@ -631,4 +612,62 @@ public static class SaveDataExports ulong InfosAddress); private readonly record struct SaveEntry(string Name, string Path, DateTime LastWriteUtc); + + [SysAbiExport( + Nid = "sDCBrmc61XU", + ExportName = "sceSaveDataPrepare", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceSaveData")] + public static int SaveDataPrepare(CpuContext ctx) + { + var mountPointAddress = ctx[CpuRegister.Rdi]; + var resource = unchecked((int)ctx[CpuRegister.Rdx]); + if (mountPointAddress == 0) + { + return ctx.SetReturn(OrbisSaveDataErrorParameter); + } + + if (!TryReadFixedAscii(ctx, mountPointAddress, 16, out var mountPoint)) + { + return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + if (string.IsNullOrWhiteSpace(mountPoint)) + { + return ctx.SetReturn(OrbisSaveDataErrorParameter); + } + + lock (_stateGate) + { + if (resource != 0) + { + _preparedTransactionResources.Add(resource); + } + } + + TraceSaveData($"prepare mount_point={mountPoint} resource={resource}"); + return ctx.SetReturn(0); + } + + [SysAbiExport( + Nid = "ie7qhZ4X0Cc", + ExportName = "sceSaveDataCommit", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceSaveData")] + public static int SaveDataCommit(CpuContext ctx) + { + var commitAddress = ctx[CpuRegister.Rdi]; + if (commitAddress == 0) + { + return ctx.SetReturn(OrbisSaveDataErrorParameter); + } + + lock (_stateGate) + { + _preparedTransactionResources.Clear(); + } + + TraceSaveData($"commit commit=0x{commitAddress:X16}"); + return ctx.SetReturn(0); + } } diff --git a/src/SharpEmu.Libs/Stubs/GameServiceStubs.cs b/src/SharpEmu.Libs/Stubs/GameServiceStubs.cs new file mode 100644 index 0000000..4915bcd --- /dev/null +++ b/src/SharpEmu.Libs/Stubs/GameServiceStubs.cs @@ -0,0 +1,62 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; + +namespace SharpEmu.Libs.Stubs; + +/// +/// Success stubs for trophy, character-encoding and telemetry ABI calls that +/// Void Terrarium (and other titles) invoke during startup. They were +/// previously unresolved and returned NOT_FOUND, and a title that gates its +/// UI/text initialization on these succeeding then skips ahead and never draws +/// its content (a black screen with only a clear pass). These return success +/// (and a non-zero handle where an out pointer is expected) so init proceeds. +/// +public static class GameServiceStubs +{ + private static int Ok(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + // Writes a small non-zero handle to the pointer in the given register so + // the caller treats the object as created; returns success. + private static int OkWithHandle(CpuContext ctx, CpuRegister outPointerRegister) + { + var outAddress = ctx[outPointerRegister]; + if (outAddress != 0) + { + Span handle = stackalloc byte[sizeof(int)]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(handle, 1); + _ = ctx.Memory.TryWrite(outAddress, handle); + } + + return Ok(ctx); + } + + // ---- NpTrophy2: trophy context/handle registration at boot ---- + public static int NpTrophy2CreateContext(CpuContext ctx) => OkWithHandle(ctx, CpuRegister.Rdi); + public static int NpTrophy2CreateHandle(CpuContext ctx) => OkWithHandle(ctx, CpuRegister.Rdi); + public static int NpTrophy2RegisterContext(CpuContext ctx) => Ok(ctx); + + [SysAbiExport(Nid = "4IzqhhUQ3nk", ExportName = "sceNpTrophy2GetGameInfo", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceNpTrophy2")] + public static int NpTrophy2GetGameInfo(CpuContext ctx) => Ok(ctx); + + // ---- CES: Shift-JIS <-> Unicode conversion setup (Japanese text) ---- + + [SysAbiExport(Nid = "ZiDCxUUGbec", ExportName = "sceCesUcsProfileInitSJis1997Cp932", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceLibcInternal")] + public static int CesUcsProfileInitSJis1997Cp932(CpuContext ctx) => Ok(ctx); + + [SysAbiExport(Nid = "538bRGc6Zo8", ExportName = "sceCesMbcsUcsContextInit", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceLibcInternal")] + public static int CesMbcsUcsContextInit(CpuContext ctx) => Ok(ctx); + + // ---- NpUniversalDataSystem: gameplay telemetry events ---- + public static int NpUniversalDataSystemCreateEvent(CpuContext ctx) => OkWithHandle(ctx, CpuRegister.Rdi); + public static int NpUniversalDataSystemPostEvent(CpuContext ctx) => Ok(ctx); + public static int NpUniversalDataSystemDestroyEvent(CpuContext ctx) => Ok(ctx); +} diff --git a/src/SharpEmu.Libs/SystemService/SystemServiceExports.cs b/src/SharpEmu.Libs/SystemService/SystemServiceExports.cs index f172c48..d0a35b0 100644 --- a/src/SharpEmu.Libs/SystemService/SystemServiceExports.cs +++ b/src/SharpEmu.Libs/SystemService/SystemServiceExports.cs @@ -13,6 +13,7 @@ public static class SystemServiceExports private const int OrbisSystemServiceErrorParameter = unchecked((int)0x80A10003); private const int SystemServiceStatusSize = 0x0C; private const int DisplaySafeAreaInfoSize = sizeof(float) + 128; + private const int HdrToneMapLuminanceSize = sizeof(float) * 3; [SysAbiExport( Nid = "fZo48un7LK4", @@ -42,6 +43,35 @@ public static class SystemServiceExports : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "SsC-m-S9JTA", + ExportName = "sceSystemServiceParamGetString", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceSystemService")] + public static int SystemServiceParamGetString(CpuContext ctx) + { + _ = unchecked((int)ctx[CpuRegister.Rdi]); // parameter id (nickname, etc.) + var bufferAddress = ctx[CpuRegister.Rsi]; + var bufferSize = unchecked((int)ctx[CpuRegister.Rdx]); + if (bufferAddress == 0 || bufferSize <= 0) + { + return ctx.SetReturn(OrbisSystemServiceErrorParameter); + } + + // String params are typically the user nickname. Callers that gate UI + // or text setup on a successful read (and skip it on failure) stall on + // a black screen when this returns NOT_FOUND, so return a neutral + // non-empty default and success. + var value = System.Text.Encoding.UTF8.GetBytes("SharpEmu"); + var writeLength = Math.Min(value.Length, bufferSize - 1); + Span output = stackalloc byte[writeLength + 1]; + value.AsSpan(0, writeLength).CopyTo(output); + output[writeLength] = 0; + return ctx.Memory.TryWrite(bufferAddress, output) + ? ctx.SetReturn(0) + : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + [SysAbiExport( Nid = "rPo6tV8D9bM", ExportName = "sceSystemServiceGetStatus", @@ -87,6 +117,28 @@ public static class SystemServiceExports : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "mPpPxv5CZt4", + ExportName = "sceSystemServiceGetHdrToneMapLuminance", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceSystemService")] + public static int SystemServiceGetHdrToneMapLuminance(CpuContext ctx) + { + var luminanceAddress = ctx[CpuRegister.Rdi]; + if (luminanceAddress == 0) + { + return ctx.SetReturn(OrbisSystemServiceErrorParameter); + } + + Span luminance = stackalloc byte[HdrToneMapLuminanceSize]; + BinaryPrimitives.WriteSingleLittleEndian(luminance, 1000.0f); + BinaryPrimitives.WriteSingleLittleEndian(luminance[sizeof(float)..], 1000.0f); + BinaryPrimitives.WriteSingleLittleEndian(luminance[(sizeof(float) * 2)..], 0.01f); + return ctx.Memory.TryWrite(luminanceAddress, luminance) + ? ctx.SetReturn(0) + : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + [SysAbiExport( Nid = "Vo5V8KAwCmk", ExportName = "sceSystemServiceHideSplashScreen", diff --git a/src/SharpEmu.Libs/UserService/UserServiceExports.cs b/src/SharpEmu.Libs/UserService/UserServiceExports.cs index a8c5ce1..2a4724d 100644 --- a/src/SharpEmu.Libs/UserService/UserServiceExports.cs +++ b/src/SharpEmu.Libs/UserService/UserServiceExports.cs @@ -4,6 +4,7 @@ using SharpEmu.HLE; using System.Buffers.Binary; using System.Text; +using System.Threading; namespace SharpEmu.Libs.UserService; @@ -13,15 +14,14 @@ public static class UserServiceExports private const int OrbisUserServiceErrorNoEvent = unchecked((int)0x80960007); private const int OrbisUserServiceErrorInvalidParameter = unchecked((int)0x80960009); private const int OrbisUserServiceErrorBufferTooShort = unchecked((int)0x8096000A); - // Retail user-service IDs begin at 0x3E8. - private const int PrimaryUserId = 1000; + // Retail user ids encode their local user slot in the 0x10000000 range; + // the first signed-in user maps to slot 0. Small emulator-local ids do + // not pass Unity's user-id-to-slot conversion and become slot -1. + private const int PrimaryUserId = 0x10000000; private const int InvalidUserId = -1; private const string PrimaryUserName = "SharpEmu"; private static int _loginEventDelivered; - private static readonly bool _traceUserService = - string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_USER_SERVICE"), "1", StringComparison.Ordinal); - [SysAbiExport( Nid = "j3YMu1MVNNo", ExportName = "sceUserServiceInitialize", @@ -29,6 +29,7 @@ public static class UserServiceExports LibraryName = "libSceUserService")] public static int UserServiceInitialize(CpuContext ctx) { + Trace("initialize"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -43,14 +44,12 @@ public static class UserServiceExports var userIdAddress = ctx[CpuRegister.Rdi]; if (userIdAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } - var result = ctx.TryWriteInt32(userIdAddress, PrimaryUserId) - ? 0 - : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - TraceUserService($"get_initial_user out=0x{userIdAddress:X16} value={PrimaryUserId} result=0x{result:X8}"); - return ctx.SetReturn(result); + return TryWriteInt32(ctx, userIdAddress, PrimaryUserId) + ? SetReturnWithTrace(ctx, 0, $"get_initial_user user={PrimaryUserId} out=0x{userIdAddress:X16}") + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -63,7 +62,7 @@ public static class UserServiceExports var userIdListAddress = ctx[CpuRegister.Rdi]; if (userIdListAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } Span userIds = stackalloc byte[sizeof(int) * 4]; @@ -72,8 +71,12 @@ public static class UserServiceExports BinaryPrimitives.WriteInt32LittleEndian(userIds[0x08..], InvalidUserId); BinaryPrimitives.WriteInt32LittleEndian(userIds[0x0C..], InvalidUserId); return ctx.Memory.TryWrite(userIdListAddress, userIds) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturnWithTrace( + ctx, + 0, + $"get_login_user_id_list users=[{PrimaryUserId},{InvalidUserId},{InvalidUserId},{InvalidUserId}] " + + $"out=0x{userIdListAddress:X16}") + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -86,20 +89,23 @@ public static class UserServiceExports var eventAddress = ctx[CpuRegister.Rdi]; if (eventAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } if (Interlocked.Exchange(ref _loginEventDelivered, 1) != 0) { - return ctx.SetReturn(OrbisUserServiceErrorNoEvent); + return SetReturn(ctx, OrbisUserServiceErrorNoEvent); } Span payload = stackalloc byte[sizeof(int) * 2]; BinaryPrimitives.WriteInt32LittleEndian(payload[0..], 0); BinaryPrimitives.WriteInt32LittleEndian(payload[sizeof(int)..], PrimaryUserId); return ctx.Memory.TryWrite(eventAddress, payload) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturnWithTrace( + ctx, + 0, + $"get_event type=login user={PrimaryUserId} out=0x{eventAddress:X16}") + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -112,53 +118,30 @@ public static class UserServiceExports var userId = unchecked((int)ctx[CpuRegister.Rdi]); var nameAddress = ctx[CpuRegister.Rsi]; var capacity = ctx[CpuRegister.Rdx]; - // Zero selects the current user. - if (userId != 0 && userId != PrimaryUserId) + if (userId != PrimaryUserId) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidParameter); + return SetReturn(ctx, OrbisUserServiceErrorInvalidParameter); } if (nameAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } var nameBytes = Encoding.UTF8.GetBytes(PrimaryUserName); if (capacity <= (ulong)nameBytes.Length) { - return ctx.SetReturn(OrbisUserServiceErrorBufferTooShort); + return SetReturn(ctx, OrbisUserServiceErrorBufferTooShort); } Span output = stackalloc byte[nameBytes.Length + 1]; nameBytes.CopyTo(output); - var result = ctx.Memory.TryWrite(nameAddress, output) - ? 0 - : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - TraceUserService( - $"get_user_name user={userId} out=0x{nameAddress:X16} capacity=0x{capacity:X} value='{PrimaryUserName}' result=0x{result:X8}"); - return ctx.SetReturn(result); - } - - [SysAbiExport( - Nid = "-sD02mFDBh4", - ExportName = "sceUserServiceGetGamePresets", - Target = Generation.Gen5, - LibraryName = "libSceUserService")] - public static int UserServiceGetGamePresets(CpuContext ctx) - { - // Return deterministic defaults for the offline profile. - var userId = unchecked((int)ctx[CpuRegister.Rdi]); - var resultAddress = ctx[CpuRegister.Rcx]; - Span defaults = stackalloc byte[0x18]; - if (resultAddress == 0 || !ctx.Memory.TryWrite(resultAddress, defaults)) - { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); - } - - TraceUserService( - $"get_game_presets user={userId} title=0x{ctx[CpuRegister.Rsi]:X16} " + - $"key=0x{ctx[CpuRegister.R9]:X} out=0x{resultAddress:X16} result=0x00000000"); - return ctx.SetReturn(0); + return ctx.Memory.TryWrite(nameAddress, output) + ? SetReturnWithTrace( + ctx, + 0, + $"get_user_name user={userId} name='{PrimaryUserName}' out=0x{nameAddress:X16}") + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } // Name not yet in ps5_names.txt and the NID was captured from titles; revisit when the symbol is catalogued. @@ -174,50 +157,165 @@ public static class UserServiceExports var valueAddress = ctx[CpuRegister.Rsi]; if (parameterId != 1000) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidParameter); + return SetReturn(ctx, OrbisUserServiceErrorInvalidParameter); } if (valueAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } - return ctx.TryWriteInt32(valueAddress, 0) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return TryWriteInt32(ctx, valueAddress, 0) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } #pragma warning restore SHEM006 [SysAbiExport( Nid = "woNpu+45RLk", ExportName = "sceUserServiceGetAgeLevel", - Target = Generation.Gen4 | Generation.Gen5, + Target = Generation.Gen5, LibraryName = "libSceUserService")] - public static int UserServiceGetAgeLevel(CpuContext ctx) + public static int UserServiceGetAgeLevel(CpuContext ctx) => + WriteUserSettingInt32(ctx, 18, "get_age_level"); + + [SysAbiExport( + Nid = "-sD02mFDBh4", + ExportName = "sceUserServiceGetGamePresets", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetGamePresets(CpuContext ctx) { var userId = unchecked((int)ctx[CpuRegister.Rdi]); - var ageLevelAddress = ctx[CpuRegister.Rsi]; - if (userId != 1000) + var presetsAddress = ctx[CpuRegister.Rsi]; + if (userId != PrimaryUserId) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidParameter); + return SetReturn(ctx, OrbisUserServiceErrorInvalidParameter); } - if (ageLevelAddress == 0) + if (presetsAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } - // Report an adult account so titles skip parental-restriction paths. - return ctx.TryWriteInt32(ageLevelAddress, 21) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + Span presets = stackalloc byte[0x28]; + presets.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(presets, (ulong)presets.Length); + if (!ctx.Memory.TryWrite(presetsAddress, presets)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetReturnWithTrace( + ctx, + 0, + $"get_game_presets user={userId} out=0x{presetsAddress:X16}"); } - private static void TraceUserService(string message) + [SysAbiExport( + Nid = "rnEhHqG-4xo", + ExportName = "sceUserServiceGetAccessibilityChatTranscription", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityChatTranscription(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_chat_transcription"); + + [SysAbiExport( + Nid = "ZKJtxdgvzwg", + ExportName = "sceUserServiceGetAccessibilityPressAndHoldDelay", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityPressAndHoldDelay(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_press_and_hold_delay"); + + [SysAbiExport( + Nid = "-3Y5GO+-i78", + ExportName = "sceUserServiceGetAccessibilityTriggerEffect", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityTriggerEffect(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_trigger_effect"); + + [SysAbiExport( + Nid = "qWYHOFwqCxY", + ExportName = "sceUserServiceGetAccessibilityVibration", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityVibration(CpuContext ctx) => + WriteUserSettingInt32(ctx, 1, "get_accessibility_vibration"); + + [SysAbiExport( + Nid = "hD-H81EN9Vg", + ExportName = "sceUserServiceGetAccessibilityZoomEnabled", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityZoomEnabled(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_zoom_enabled"); + + [SysAbiExport( + Nid = "O6IW1-Dwm-w", + ExportName = "sceUserServiceGetAccessibilityZoomFollowFocus", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityZoomFollowFocus(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_zoom_follow_focus"); + + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) { - if (_traceUserService) + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + + private static int WriteUserSettingInt32(CpuContext ctx, int value, string operation) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var valueAddress = ctx[CpuRegister.Rsi]; + if (userId != PrimaryUserId) { - Console.Error.WriteLine($"[LOADER][TRACE] user_service.{message}"); + return SetReturn(ctx, OrbisUserServiceErrorInvalidParameter); + } + + if (valueAddress == 0) + { + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); + } + + if (!TryWriteInt32(ctx, valueAddress, value)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetReturnWithTrace( + ctx, + 0, + $"{operation} user={userId} value={value} out=0x{valueAddress:X16}"); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + + private static int SetReturnWithTrace(CpuContext ctx, int result, string message) + { + Trace(message); + return SetReturn(ctx, result); + } + + private static void Trace(string message) + { + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_USER_SERVICE"), + "1", + StringComparison.Ordinal)) + { + var returnRip = GuestThreadExecution.TryGetCurrentImportCallFrame(out var frame) + ? frame.ReturnRip + : 0; + Console.Error.WriteLine( + $"[LOADER][TRACE] user_service.{message} ret=0x{returnRip:X16}"); } } } diff --git a/src/SharpEmu.Libs/VideoOut/PerfOverlay.cs b/src/SharpEmu.Libs/VideoOut/PerfOverlay.cs new file mode 100644 index 0000000..fa7d8a2 --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/PerfOverlay.cs @@ -0,0 +1,363 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; + +namespace SharpEmu.Libs.VideoOut; + +/// +/// In-window performance HUD. The panel is rasterized on the CPU into a +/// small BGRA buffer each frame (embedded 5x7 font, no assets) and the +/// presenter blits it onto the swapchain image, so it needs no pipelines, +/// descriptors or blending state. Toggled with F1; SHARPEMU_OVERLAY=0 +/// starts it hidden. +/// +public static class PerfOverlay +{ + public const int PanelWidth = 376; + public const int PanelHeight = 176; + + private const int GlyphColumns = 5; + private const int GlyphRows = 7; + private const int Scale = 2; + private const int CellWidth = (GlyphColumns + 1) * Scale; + private const int LineHeight = (GlyphRows + 2) * Scale; + private const int FrameHistorySize = 128; + + private static volatile bool _enabled = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_OVERLAY"), + "0", + StringComparison.Ordinal); + + private static long _lastPresentTimestamp; + private static readonly double[] _frameMilliseconds = new double[FrameHistorySize]; + private static int _frameHistoryIndex; + private static long _presentedInWindow; + private static long _submittedInWindow; + private static long _drawsInWindow; + + // Refreshed once per second so per-frame fills never allocate. + private static long _statsWindowStart = Stopwatch.GetTimestamp(); + private static double _fps; + private static double _submittedFps; + private static double _drawsPerSecond; + private static double _averageFrameMs; + private static double _allocatedMbPerSecond; + private static long _lastAllocatedBytes = GC.GetTotalAllocatedBytes(precise: false); + private static int _lastGen0 = GC.CollectionCount(0); + private static int _lastGen1 = GC.CollectionCount(1); + private static int _lastGen2 = GC.CollectionCount(2); + private static int _gen0PerWindow; + private static int _gen1PerWindow; + private static int _gen2PerWindow; + private static double _cpuPercent; + private static TimeSpan _lastCpuTime = GetProcessCpuTime(); + private static string _line1 = string.Empty; + private static string _line2 = string.Empty; + private static string _line3 = string.Empty; + private static string _line4 = string.Empty; + + public static bool Enabled => _enabled; + + public static void Toggle() => _enabled = !_enabled; + + /// Called by the presenter after each successful present. + public static void RecordPresent() + { + var now = Stopwatch.GetTimestamp(); + var last = _lastPresentTimestamp; + _lastPresentTimestamp = now; + Interlocked.Increment(ref _presentedInWindow); + if (last != 0) + { + var milliseconds = (now - last) * 1000.0 / Stopwatch.Frequency; + if (milliseconds < 1000.0) + { + _frameMilliseconds[_frameHistoryIndex] = milliseconds; + _frameHistoryIndex = (_frameHistoryIndex + 1) % FrameHistorySize; + } + } + } + + /// Called on every guest flip submission. + public static void RecordSubmit() => Interlocked.Increment(ref _submittedInWindow); + + /// Called per translated draw/dispatch executed. + public static void RecordDraw() => Interlocked.Increment(ref _drawsInWindow); + + /// + /// Rasterizes the panel into a BGRA byte span of PanelWidth x PanelHeight. + /// Runs on the render thread. + /// + public static void Fill(Span bgra, int pendingWork, int inFlightSubmissions) + { + RefreshStatsIfDue(pendingWork, inFlightSubmissions); + + // Background: opaque dark slate. + for (var i = 0; i < bgra.Length; i += 4) + { + bgra[i] = 0x20; + bgra[i + 1] = 0x18; + bgra[i + 2] = 0x14; + bgra[i + 3] = 0xFF; + } + + var y = 6; + DrawString(bgra, 8, y, _line1, 0x60, 0xFF, 0x60); + y += LineHeight; + DrawString(bgra, 8, y, _line2, 0xFF, 0xFF, 0xFF); + y += LineHeight; + DrawString(bgra, 8, y, _line3, 0xB0, 0xD0, 0xFF); + y += LineHeight; + DrawString(bgra, 8, y, _line4, 0xB0, 0xB0, 0xB0); + y += LineHeight + 4; + DrawFrameGraph(bgra, 8, y, PanelWidth - 16, PanelHeight - y - 6); + } + + private static void RefreshStatsIfDue(int pendingWork, int inFlightSubmissions) + { + var now = Stopwatch.GetTimestamp(); + var elapsedTicks = now - _statsWindowStart; + if (elapsedTicks >= Stopwatch.Frequency) + { + var seconds = (double)elapsedTicks / Stopwatch.Frequency; + _statsWindowStart = now; + _fps = Interlocked.Exchange(ref _presentedInWindow, 0) / seconds; + _submittedFps = Interlocked.Exchange(ref _submittedInWindow, 0) / seconds; + _drawsPerSecond = Interlocked.Exchange(ref _drawsInWindow, 0) / seconds; + + double totalMs = 0; + var samples = 0; + foreach (var ms in _frameMilliseconds) + { + if (ms > 0) + { + totalMs += ms; + samples++; + } + } + + _averageFrameMs = samples > 0 ? totalMs / samples : 0; + + var allocated = GC.GetTotalAllocatedBytes(precise: false); + _allocatedMbPerSecond = (allocated - _lastAllocatedBytes) / seconds / (1024.0 * 1024.0); + _lastAllocatedBytes = allocated; + + var gen0 = GC.CollectionCount(0); + var gen1 = GC.CollectionCount(1); + var gen2 = GC.CollectionCount(2); + _gen0PerWindow = gen0 - _lastGen0; + _gen1PerWindow = gen1 - _lastGen1; + _gen2PerWindow = gen2 - _lastGen2; + _lastGen0 = gen0; + _lastGen1 = gen1; + _lastGen2 = gen2; + + var cpuTime = GetProcessCpuTime(); + _cpuPercent = (cpuTime - _lastCpuTime).TotalSeconds / seconds * 100.0; + _lastCpuTime = cpuTime; + + var drawsPerFrame = _fps > 0.5 ? _drawsPerSecond / _fps : 0; + _line1 = $"FPS {_fps:0.0} FLIP {_submittedFps:0.0} {_averageFrameMs:0.0} MS"; + _line2 = $"DRAWS {_drawsPerSecond:0}/S {drawsPerFrame:0}/F Q {pendingWork}+{inFlightSubmissions}"; + _line3 = $"ALLOC {_allocatedMbPerSecond:0.0} MB/S GC {_gen0PerWindow}/{_gen1PerWindow}/{_gen2PerWindow}"; + _line4 = $"CPU {_cpuPercent:0}% HEAP {GC.GetTotalMemory(false) / (1024 * 1024)} MB F1 HIDE"; + } + } + + private static TimeSpan GetProcessCpuTime() + { + var usage = Environment.CpuUsage; + return usage.UserTime + usage.PrivilegedTime; + } + + private static void DrawFrameGraph(Span bgra, int x, int y, int width, int height) + { + if (height < 8) + { + return; + } + + // Reference lines at 16.7ms and 33.3ms of a 40ms-tall scale. + const double maxMs = 40.0; + DrawHorizontalLine(bgra, x, y + height - (int)(16.7 / maxMs * height), width, 0x30, 0x60, 0x30); + DrawHorizontalLine(bgra, x, y + height - (int)(33.3 / maxMs * height), width, 0x30, 0x30, 0x60); + + var barWidth = Math.Max(1, width / FrameHistorySize); + for (var i = 0; i < FrameHistorySize; i++) + { + var ms = _frameMilliseconds[(_frameHistoryIndex + i) % FrameHistorySize]; + if (ms <= 0) + { + continue; + } + + var barHeight = (int)Math.Min(height, ms / maxMs * height); + var barX = x + i * barWidth; + if (barX + barWidth > x + width) + { + break; + } + + byte r, g, b; + if (ms <= 17.5) + { + (r, g, b) = ((byte)0x40, (byte)0xE0, (byte)0x40); + } + else if (ms <= 34.0) + { + (r, g, b) = ((byte)0xE0, (byte)0xC0, (byte)0x30); + } + else + { + (r, g, b) = ((byte)0xE8, (byte)0x40, (byte)0x40); + } + + for (var py = 0; py < barHeight; py++) + { + var rowY = y + height - 1 - py; + for (var px = 0; px < barWidth; px++) + { + SetPixel(bgra, barX + px, rowY, r, g, b); + } + } + } + } + + private static void DrawHorizontalLine(Span bgra, int x, int y, int width, byte r, byte g, byte b) + { + if (y < 0 || y >= PanelHeight) + { + return; + } + + for (var px = 0; px < width; px++) + { + SetPixel(bgra, x + px, y, r, g, b); + } + } + + private static void DrawString(Span bgra, int x, int y, string text, byte r, byte g, byte b) + { + var penX = x; + foreach (var rawChar in text) + { + var c = char.ToUpperInvariant(rawChar); + if (c < ' ' || c > 'Z') + { + c = '?'; + } + + var glyph = Font.Slice((c - ' ') * GlyphColumns, GlyphColumns); + for (var column = 0; column < GlyphColumns; column++) + { + var bits = glyph[column]; + for (var row = 0; row < GlyphRows; row++) + { + if ((bits & (1 << row)) == 0) + { + continue; + } + + for (var sy = 0; sy < Scale; sy++) + { + for (var sx = 0; sx < Scale; sx++) + { + SetPixel( + bgra, + penX + column * Scale + sx, + y + row * Scale + sy, + r, + g, + b); + } + } + } + } + + penX += CellWidth; + if (penX + CellWidth > PanelWidth) + { + break; + } + } + } + + private static void SetPixel(Span bgra, int x, int y, byte r, byte g, byte b) + { + if ((uint)x >= PanelWidth || (uint)y >= PanelHeight) + { + return; + } + + var offset = (y * PanelWidth + x) * 4; + bgra[offset] = b; + bgra[offset + 1] = g; + bgra[offset + 2] = r; + bgra[offset + 3] = 0xFF; + } + + // Classic 5x7 column-encoded font (bit 0 = top row), ASCII 32..90. + private static ReadOnlySpan Font => + [ + 0x00, 0x00, 0x00, 0x00, 0x00, // ' ' + 0x00, 0x00, 0x5F, 0x00, 0x00, // '!' + 0x00, 0x07, 0x00, 0x07, 0x00, // '"' + 0x14, 0x7F, 0x14, 0x7F, 0x14, // '#' + 0x24, 0x2A, 0x7F, 0x2A, 0x12, // '$' + 0x23, 0x13, 0x08, 0x64, 0x62, // '%' + 0x36, 0x49, 0x55, 0x22, 0x50, // '&' + 0x00, 0x05, 0x03, 0x00, 0x00, // ''' + 0x00, 0x1C, 0x22, 0x41, 0x00, // '(' + 0x00, 0x41, 0x22, 0x1C, 0x00, // ')' + 0x08, 0x2A, 0x1C, 0x2A, 0x08, // '*' + 0x08, 0x08, 0x3E, 0x08, 0x08, // '+' + 0x00, 0x50, 0x30, 0x00, 0x00, // ',' + 0x08, 0x08, 0x08, 0x08, 0x08, // '-' + 0x00, 0x60, 0x60, 0x00, 0x00, // '.' + 0x20, 0x10, 0x08, 0x04, 0x02, // '/' + 0x3E, 0x51, 0x49, 0x45, 0x3E, // '0' + 0x00, 0x42, 0x7F, 0x40, 0x00, // '1' + 0x42, 0x61, 0x51, 0x49, 0x46, // '2' + 0x21, 0x41, 0x45, 0x4B, 0x31, // '3' + 0x18, 0x14, 0x12, 0x7F, 0x10, // '4' + 0x27, 0x45, 0x45, 0x45, 0x39, // '5' + 0x3C, 0x4A, 0x49, 0x49, 0x30, // '6' + 0x01, 0x71, 0x09, 0x05, 0x03, // '7' + 0x36, 0x49, 0x49, 0x49, 0x36, // '8' + 0x06, 0x49, 0x49, 0x29, 0x1E, // '9' + 0x00, 0x36, 0x36, 0x00, 0x00, // ':' + 0x00, 0x56, 0x36, 0x00, 0x00, // ';' + 0x00, 0x08, 0x14, 0x22, 0x41, // '<' + 0x14, 0x14, 0x14, 0x14, 0x14, // '=' + 0x41, 0x22, 0x14, 0x08, 0x00, // '>' + 0x02, 0x01, 0x51, 0x09, 0x06, // '?' + 0x32, 0x49, 0x79, 0x41, 0x3E, // '@' + 0x7E, 0x11, 0x11, 0x11, 0x7E, // 'A' + 0x7F, 0x49, 0x49, 0x49, 0x36, // 'B' + 0x3E, 0x41, 0x41, 0x41, 0x22, // 'C' + 0x7F, 0x41, 0x41, 0x22, 0x1C, // 'D' + 0x7F, 0x49, 0x49, 0x49, 0x41, // 'E' + 0x7F, 0x09, 0x09, 0x09, 0x01, // 'F' + 0x3E, 0x41, 0x49, 0x49, 0x7A, // 'G' + 0x7F, 0x08, 0x08, 0x08, 0x7F, // 'H' + 0x00, 0x41, 0x7F, 0x41, 0x00, // 'I' + 0x20, 0x40, 0x41, 0x3F, 0x01, // 'J' + 0x7F, 0x08, 0x14, 0x22, 0x41, // 'K' + 0x7F, 0x40, 0x40, 0x40, 0x40, // 'L' + 0x7F, 0x02, 0x0C, 0x02, 0x7F, // 'M' + 0x7F, 0x04, 0x08, 0x10, 0x7F, // 'N' + 0x3E, 0x41, 0x41, 0x41, 0x3E, // 'O' + 0x7F, 0x09, 0x09, 0x09, 0x06, // 'P' + 0x3E, 0x41, 0x51, 0x21, 0x5E, // 'Q' + 0x7F, 0x09, 0x19, 0x29, 0x46, // 'R' + 0x46, 0x49, 0x49, 0x49, 0x31, // 'S' + 0x01, 0x01, 0x7F, 0x01, 0x01, // 'T' + 0x3F, 0x40, 0x40, 0x40, 0x3F, // 'U' + 0x1F, 0x20, 0x40, 0x20, 0x1F, // 'V' + 0x3F, 0x40, 0x38, 0x40, 0x3F, // 'W' + 0x63, 0x14, 0x08, 0x14, 0x63, // 'X' + 0x07, 0x08, 0x70, 0x08, 0x07, // 'Y' + 0x61, 0x51, 0x49, 0x45, 0x43, // 'Z' + ]; +} diff --git a/src/SharpEmu.Libs/VideoOut/VideoOutExports.cs b/src/SharpEmu.Libs/VideoOut/VideoOutExports.cs index 290a169..f01f8f2 100644 --- a/src/SharpEmu.Libs/VideoOut/VideoOutExports.cs +++ b/src/SharpEmu.Libs/VideoOut/VideoOutExports.cs @@ -6,7 +6,6 @@ using SharpEmu.HLE.Host; using SharpEmu.Libs.Gpu; using SharpEmu.Libs.Audio; using SharpEmu.Libs.Kernel; -using SharpEmu.Logging; using System.Buffers; using System.Buffers.Binary; using System.Diagnostics; @@ -36,221 +35,63 @@ public static class VideoOutExports private const int VideoOutBufferAttribute2Size = 0x50; private const int VideoOutBuffersEntrySize = 0x20; private const int VideoOutOutputStatusSize = 0x30; + private const int VideoOutVblankStatusSize = 0x28; private const ulong SceVideoOutPixelFormatA8R8G8B8Srgb = 0x80000000; private const ulong SceVideoOutPixelFormatA8B8G8R8Srgb = 0x80002200; - private const ulong SceVideoOutPixelFormatB8G8R8A8Unorm = 0x8100000000000000; - private const ulong SceVideoOutPixelFormatR8G8B8A8Unorm = 0x8100000022000000; private const ulong SceVideoOutPixelFormatA2R10G10B10 = 0x88060000; private const ulong SceVideoOutPixelFormatA2R10G10B10Srgb = 0x88000000; private const ulong SceVideoOutPixelFormatA2R10G10B10Bt2020Pq = 0x88740000; - private const ulong SceVideoOutInternalEventVblank = 0x5; + // Prospero/PS5 format2 values are 64-bit encodings. The 0x22000000 field + // selects R-first component order; notably, the 0x81000000... family is + // packed 10:10:10:2 and must not be mistaken for an 8-bit RGBA format. + private const ulong SceVideoOutPixelFormat2R8G8B8A8Srgb = 0x8000000022000000; + private const ulong SceVideoOutPixelFormat2B8G8R8A8Srgb = 0x8000000000000000; + private const ulong SceVideoOutPixelFormat2R10G10B10A2 = 0x8100000622000000; + private const ulong SceVideoOutPixelFormat2B10G10R10A2 = 0x8100000600000000; + private const ulong SceVideoOutPixelFormat2R10G10B10A2Srgb = 0x8100000022000000; + private const ulong SceVideoOutPixelFormat2B10G10R10A2Srgb = 0x8100000000000000; + private const ulong SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq = 0x8100070422000000; + private const ulong SceVideoOutPixelFormat2B10G10R10A2Bt2100Pq = 0x8100070400000000; private const ulong SceVideoOutInternalEventFlip = 0x6; + // Distinct internal ident for vblank events. Games interpret events through + // sceVideoOutGetEventId (mapped below), so the exact value is internal; only + // its distinctness from the flip ident matters for GetEventId/GetEventData. + private const ulong SceVideoOutInternalEventVblank = 0x40; private const short OrbisKernelEventFilterVideoOut = -13; private static readonly object _stateGate = new(); private static readonly object _frameDumpGate = new(); private static readonly Dictionary _ports = new(); - - // Hardware raises vblank autonomously; UE blocks its frame loop on it. - private const double VblankHz = 60.0; - private const int VblankWaitTimeoutMilliseconds = 100; - private static Thread? _vblankPumpThread; - private static int _vblankPumpStarted; - private static volatile int _vblankPumpStopRequested; - private static volatile int _presentationWindowCloseNotified; - - private static readonly object _vblankEdgeGate = new(); - private static ulong _vblankEdgeSequence; - private static long _vblankMissedEdges; - - private static void EnsureVblankPumpStarted() - { - if (Interlocked.Exchange(ref _vblankPumpStarted, 1) != 0) - { - return; - } - - HostPlatform.Current.Threading.RequestTimerResolution(); - - _vblankPumpThread = new Thread(VblankPumpLoop) - { - IsBackground = true, - Name = "SharpEmu VideoOut vblank", - Priority = ThreadPriority.AboveNormal, - }; - _vblankPumpThread.Start(); - } - - private static void VblankPumpLoop() - { - var intervalTicks = Math.Max(1L, (long)(Stopwatch.Frequency / VblankHz)); - var nextEdge = Stopwatch.GetTimestamp() + intervalTicks; - - while (_vblankPumpStopRequested == 0) - { - WaitUntilTimestamp(nextEdge); - PumpVblanks(); - - nextEdge += intervalTicks; - - var now = Stopwatch.GetTimestamp(); - if (nextEdge < now) - { - var missed = (now - nextEdge) / intervalTicks + 1; - Interlocked.Add(ref _vblankMissedEdges, missed); - nextEdge = now + intervalTicks; - } - } - } - - public static void NotifyPresentationWindowClosed() - { - if (Interlocked.Exchange(ref _presentationWindowCloseNotified, 1) != 0) - { - return; - } - - Console.Error.WriteLine("[LOADER][INFO] VideoOut presentation window closed"); - RequestHostShutdown("videoout-window-closed"); - } - - public static void NotifyHostInterrupt() - { - if (Interlocked.Exchange(ref _presentationWindowCloseNotified, 1) != 0) - { - return; - } - - Console.Error.WriteLine("[LOADER][INFO] Host interrupt requested"); - RequestHostShutdown("host-interrupt"); - } - - private static void RequestHostShutdown(string reason) - { - AudioOutExports.ShutdownAllPorts(); - StopVblankPump(); - HostSessionControl.RequestShutdown(reason); - ScheduleProcessExitIfGuestDoesNotStop(); - } - - private static void ScheduleProcessExitIfGuestDoesNotStop() - { - ThreadPool.QueueUserWorkItem(static _ => - { - Thread.Sleep(2000); - Environment.Exit(0); - }); - } - - public static void StopVblankPump() - { - if (Interlocked.Exchange(ref _vblankPumpStopRequested, 1) != 0) - { - return; - } - - var thread = _vblankPumpThread; - if (thread is { IsAlive: true }) - { - thread.Join(TimeSpan.FromSeconds(2)); - } - } - - private static void WaitUntilTimestamp(long deadlineTicks) - { - var spinThresholdTicks = Stopwatch.Frequency * 2L / 1000L; - - while (true) - { - var remaining = deadlineTicks - Stopwatch.GetTimestamp(); - if (remaining <= 0) - { - return; - } - - if (remaining > spinThresholdTicks) - { - var sleepMilliseconds = - (int)((remaining - spinThresholdTicks) * 1000L / Stopwatch.Frequency); - if (sleepMilliseconds > 0) - { - Thread.Sleep(sleepMilliseconds); - continue; - } - } - - Thread.SpinWait(64); - } - } - - // Only ever touched by the vblank pump thread; reused across edges so the 60 Hz - // pump does not allocate a fresh snapshot per edge. - private static readonly List _vblankPumpPorts = new(); - - private static void PumpVblanks() - { - lock (_vblankEdgeGate) - { - _vblankEdgeSequence++; - Monitor.PulseAll(_vblankEdgeGate); - } - - _vblankPumpPorts.Clear(); - lock (_stateGate) - { - if (_ports.Count == 0) - { - return; - } - - // Signalling reaches WakeBlockedThreads -> Pump(), which serialises on one global - // flag. Waking an unwatched queue would hold it 60x/sec and starve guest threads. - foreach (var port in _ports.Values) - { - if (port.VblankEvents.Count != 0) - { - _vblankPumpPorts.Add(port); - } - } - } - - foreach (var port in _vblankPumpPorts) - { - SignalVblank(port); - } - } - + private static int _presentationWindowCloseNotified; + private static int _vblankStopRequested; private static readonly Dictionary<(int Handle, int BufferIndex, ulong Address), ulong> _lastFrameFingerprints = new(); private static int _nextHandle = 1; private static int _frameDumpCount; private static long _nextFrameDumpIndex; - private static string _windowTitleBase = "SharpEmu VideoOut"; private static string _windowTitle = "SharpEmu VideoOut"; private static readonly bool _logFrameRate = string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIDEOOUT_FPS"), "1", StringComparison.Ordinal); - private static readonly bool _logVideoOutSync = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIDEOOUT_SYNC"), - "1", - StringComparison.Ordinal); - // Call sites must check this before building the interpolated message; the trace - // strings would otherwise be allocated on the per-frame flip path even with tracing off. - private static readonly bool _logVideoOut = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIDEOOUT"), - "1", - StringComparison.Ordinal); - private static readonly bool _dumpVideoOut = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_DUMP_VIDEOOUT"), - "1", - StringComparison.Ordinal); private static long _frameRateWindowStart = Stopwatch.GetTimestamp(); private static long _submittedFrameCount; + private static int _diagnosticFlipCount; + private static readonly int _holdFirstFlipMilliseconds = + int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_HOLD_FIRST_FLIP_MS"), out var holdMs) + ? Math.Clamp(holdMs, 0, 60_000) + : 0; + private static readonly int _holdFlipNumber = + int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_HOLD_FLIP_NUMBER"), out var holdFlip) + ? Math.Max(1, holdFlip) + : 1; private static long _presentedFrameCount; - private static long _vblankSignalCount; - private static long _flipSubmitCount; - public static void ConfigureApplicationInfo(string? title, string? titleId, string? version, string? emulatorCommitSha) + static VideoOutExports() + { + RunPixelFormatSelfChecks(); + } + + public static void ConfigureApplicationInfo(string? title, string? titleId, string? version) { var parts = new List(); if (!string.IsNullOrWhiteSpace(title)) @@ -265,12 +106,17 @@ public static class VideoOutExports var application = parts.Count == 0 ? "VideoOut" : string.Join(' ', parts); var versionSuffix = string.IsNullOrWhiteSpace(version) ? string.Empty : $" v{version.Trim()}"; - var commitSuffix = string.IsNullOrWhiteSpace(emulatorCommitSha) ? string.Empty : $" \u00b7 {emulatorCommitSha.Trim()}"; - var hardwareSuffix = $" \u00b7 {HostSystemInfo.CpuName}"; lock (_stateGate) { - _windowTitleBase = $"SharpEmu{commitSuffix} - {application}{versionSuffix}{hardwareSuffix}"; - _windowTitle = $"{_windowTitleBase} \u00b7 {HostSystemInfo.GpuName}"; + _windowTitle = $"SharpEmu - {application}{versionSuffix}"; + } + } + + internal static string GetWindowTitle() + { + lock (_stateGate) + { + return _windowTitle; } } @@ -283,16 +129,42 @@ public static class VideoOutExports lock (_stateGate) { - _windowTitle = $"{_windowTitleBase} \u00b7 {gpuName.Trim()}"; + _windowTitle = $"{_windowTitle} · {gpuName.Trim()}"; } } - internal static string GetWindowTitle() + public static void NotifyPresentationWindowClosed() { - lock (_stateGate) + if (Interlocked.Exchange(ref _presentationWindowCloseNotified, 1) != 0) { - return _windowTitle; + return; } + + RequestHostShutdown("videoout-window-closed"); + } + + public static void NotifyHostInterrupt() + { + if (Interlocked.Exchange(ref _presentationWindowCloseNotified, 1) != 0) + { + return; + } + + RequestHostShutdown("host-interrupt"); + } + + private static void RequestHostShutdown(string reason) + { + Console.Error.WriteLine($"[LOADER][INFO] Host shutdown requested: {reason}"); + VulkanVideoPresenter.RequestClose(); + AudioOutExports.ShutdownAllPorts(); + Interlocked.Exchange(ref _vblankStopRequested, 1); + HostSessionControl.RequestShutdown(reason); + ThreadPool.QueueUserWorkItem(static _ => + { + Thread.Sleep(2000); + Environment.Exit(0); + }); } private sealed class VideoOutPortState @@ -308,8 +180,10 @@ public static class VideoOutExports public float Gamma { get; set; } = 1.0f; public VideoOutBufferGroup?[] Groups { get; } = new VideoOutBufferGroup?[MaxDisplayBufferGroups]; public VideoOutBufferSlot[] BufferSlots { get; } = CreateBufferSlots(); - public List VblankEvents { get; } = new(); public List FlipEvents { get; } = new(); + public List VblankEvents { get; } = new(); + public long OpenTimestamp; + public long LastVblankTimestamp; } private sealed class VideoOutBufferGroup @@ -374,11 +248,13 @@ public static class VideoOutExports } var handle = _nextHandle++; + var openedAt = Stopwatch.GetTimestamp(); _ports[handle] = new VideoOutPortState { Handle = handle, + OpenTimestamp = openedAt, + LastVblankTimestamp = openedAt, }; - EnsureVblankPumpStarted(); return handle; } } @@ -422,16 +298,6 @@ public static class VideoOutExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "+I4K03i3EL0", - ExportName = "sceVideoOutInitializeOutputOptions", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceVideoOut")] - public static int VideoOutInitializeOutputOptions(CpuContext ctx) - { - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "w0hLuNarQxY", ExportName = "sceVideoOutConfigureOutput", @@ -439,14 +305,19 @@ public static class VideoOutExports LibraryName = "libSceVideoOut")] public static int VideoOutConfigureOutput(CpuContext ctx) { - // Accept the requested output configuration; the presenter always renders - // at the display buffer's native size. var handle = unchecked((int)ctx[CpuRegister.Rdi]); - if (!TryGetPort(handle, out _)) - { - return OrbisVideoOutErrorInvalidHandle; - } + return TryGetPort(handle, out _) + ? (int)OrbisGen2Result.ORBIS_GEN2_OK + : OrbisVideoOutErrorInvalidHandle; + } + [SysAbiExport( + Nid = "+I4K03i3EL0", + ExportName = "sceVideoOutInitializeOutputOptions", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceVideoOut")] + public static int VideoOutInitializeOutputOptions(CpuContext ctx) + { return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -550,66 +421,75 @@ public static class VideoOutExports return OrbisVideoOutErrorInvalidHandle; } - EnsureVblankPumpStarted(); - - lock (_vblankEdgeGate) + // Wait to the next boundary of the emulated display refresh rather + // than a raw Thread.Sleep(1): coarse sleeps overshoot to the + // scheduler quantum, which mis-paces games that spin on vblank. A + // caller that arrives past the boundary already missed the vblank: + // report it immediately instead of charging a full extra interval. + var intervalTicks = Stopwatch.Frequency / Math.Max(1, (long)port.RefreshRate); + var now = Stopwatch.GetTimestamp(); + var last = Interlocked.Read(ref port.LastVblankTimestamp); + var target = last + intervalTicks; + if (target <= now || target > now + intervalTicks) { - var entryEdge = _vblankEdgeSequence; - while (_vblankEdgeSequence == entryEdge) - { - if (!Monitor.Wait(_vblankEdgeGate, VblankWaitTimeoutMilliseconds)) - { - break; - } - } + Interlocked.CompareExchange(ref port.LastVblankTimestamp, now, last); + } + else + { + HostTiming.SleepUntil(target); + Interlocked.CompareExchange(ref port.LastVblankTimestamp, target, last); + } + lock (_stateGate) + { + port.VblankCount++; } - - SignalVblank(port); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } [SysAbiExport( - Nid = "Xru92wHJRmg", - ExportName = "sceVideoOutAddVblankEvent", + Nid = "1FZBKy8HeNU", + ExportName = "sceVideoOutGetVblankStatus", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideoOut")] - public static int VideoOutAddVblankEvent(CpuContext ctx) + public static int VideoOutGetVblankStatus(CpuContext ctx) { - var equeue = ctx[CpuRegister.Rdi]; - var handle = unchecked((int)ctx[CpuRegister.Rsi]); + var handle = unchecked((int)ctx[CpuRegister.Rdi]); + var statusAddress = ctx[CpuRegister.Rsi]; + if (statusAddress == 0) + { + return OrbisVideoOutErrorInvalidAddress; + } + if (!TryGetPort(handle, out var port)) { return OrbisVideoOutErrorInvalidHandle; } - if (!KernelEventQueueCompatExports.IsValidEqueue(equeue)) - { - return OrbisVideoOutErrorInvalidEventQueue; - } - - var userData = ctx[CpuRegister.Rdx]; + var now = Stopwatch.GetTimestamp(); + ulong count; + long openedAt; lock (_stateGate) { - var existingIndex = port.VblankEvents.FindIndex(registration => registration.Equeue == equeue); - if (existingIndex >= 0) - { - port.VblankEvents[existingIndex] = new FlipEventRegistration(equeue, userData); - } - else - { - port.VblankEvents.Add(new FlipEventRegistration(equeue, userData)); - } + openedAt = port.OpenTimestamp; + var elapsedTicks = Math.Max(now - openedAt, 0); + var elapsedCount = unchecked((ulong)(elapsedTicks * + Math.Max(1L, (long)port.RefreshRate) / Stopwatch.Frequency)); + port.VblankCount = Math.Max(port.VblankCount, elapsedCount); + count = port.VblankCount; } - // Some engines wait on this queue before issuing their first flip. Provide a first - // edge now; later calls to WaitVblank advance the same notification sequence. - SignalVblank(port); - if (_logVideoOut) - { - TraceVideoOut($"videoout.add_vblank_event eq=0x{equeue:X16} handle={handle} udata=0x{userData:X16}"); - } - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + var elapsedMicroseconds = unchecked((ulong)(Math.Max(now - openedAt, 0) * + 1_000_000L / Stopwatch.Frequency)); + Span status = stackalloc byte[VideoOutVblankStatusSize]; + status.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(status, count); + BinaryPrimitives.WriteUInt64LittleEndian(status[0x08..], elapsedMicroseconds); + BinaryPrimitives.WriteUInt64LittleEndian(status[0x10..], unchecked((ulong)now)); + status[0x20] = 0; + return ctx.Memory.TryWrite(statusAddress, status) + ? (int)OrbisGen2Result.ORBIS_GEN2_OK + : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } [SysAbiExport( @@ -645,13 +525,76 @@ public static class VideoOutExports } } - if (_logVideoOut) + if (_traceVideoOut) { TraceVideoOut($"videoout.add_flip_event eq=0x{equeue:X16} handle={handle} udata=0x{userData:X16}"); } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + [SysAbiExport( + Nid = "Xru92wHJRmg", + ExportName = "sceVideoOutAddVblankEvent", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceVideoOut")] + public static int VideoOutAddVblankEvent(CpuContext ctx) + { + var equeue = ctx[CpuRegister.Rdi]; + var handle = unchecked((int)ctx[CpuRegister.Rsi]); + var userData = ctx[CpuRegister.Rdx]; + if (!TryGetPort(handle, out var port)) + { + return OrbisVideoOutErrorInvalidHandle; + } + + if (!KernelEventQueueCompatExports.IsValidEqueue(equeue)) + { + return OrbisVideoOutErrorInvalidEventQueue; + } + + lock (_stateGate) + { + var existingIndex = port.VblankEvents.FindIndex(registration => registration.Equeue == equeue); + if (existingIndex >= 0) + { + port.VblankEvents[existingIndex] = new FlipEventRegistration(equeue, userData); + } + else + { + port.VblankEvents.Add(new FlipEventRegistration(equeue, userData)); + } + } + + // A guest that parks its main/render loop on a vblank event needs a + // steady tick to advance; start the emulated vblank cadence on demand. + StartVblankThreadOnce(); + TraceVideoOut($"videoout.add_vblank_event eq=0x{equeue:X16} handle={handle} udata=0x{userData:X16}"); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "oNOQn3knW6s", + ExportName = "sceVideoOutDeleteVblankEvent", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceVideoOut")] + public static int VideoOutDeleteVblankEvent(CpuContext ctx) + { + var equeue = ctx[CpuRegister.Rdi]; + var handle = unchecked((int)ctx[CpuRegister.Rsi]); + if (!TryGetPort(handle, out var port)) + { + return OrbisVideoOutErrorInvalidHandle; + } + + lock (_stateGate) + { + port.VblankEvents.RemoveAll(registration => registration.Equeue == equeue); + } + + TraceVideoOut($"videoout.delete_vblank_event eq=0x{equeue:X16} handle={handle}"); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + [SysAbiExport( Nid = "U46NwOiJpys", ExportName = "sceVideoOutSubmitFlip", @@ -666,8 +609,6 @@ public static class VideoOutExports return SubmitFlip(ctx, handle, bufferIndex, flipMode, flipArg, submitGpuImage: true); } - // Struct layout matches the classic SceVideoOutFlipStatus (40 bytes): - // count, processTime, tsc, flipArg, currentBuffer, flipPendingNum. [SysAbiExport( Nid = "SbU3dwp80lQ", ExportName = "sceVideoOutGetFlipStatus", @@ -682,37 +623,24 @@ public static class VideoOutExports return OrbisVideoOutErrorInvalidAddress; } - VideoOutPortState? port; - lock (_stateGate) - { - _ports.TryGetValue(handle, out port); - } - - if (port is null) + if (!TryGetPort(handle, out var port)) { return OrbisVideoOutErrorInvalidHandle; } ulong count; - long flipArg; uint currentBuffer; lock (_stateGate) { count = port.FlipCount; - flipArg = 0; currentBuffer = unchecked((uint)port.CurrentBuffer); } KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x00, count); KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x08, 0); KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x10, 0); - KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x18, unchecked((ulong)flipArg)); + KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x18, 0); KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x20, currentBuffer); - - if (_logVideoOut) - { - TraceVideoOut($"videoout.get_flip_status handle={handle} count={count} currentBuffer={currentBuffer}"); - } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -752,13 +680,23 @@ public static class VideoOutExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (filter != OrbisKernelEventFilterVideoOut || - ident is not (SceVideoOutInternalEventVblank or SceVideoOutInternalEventFlip)) + if (filter != OrbisKernelEventFilterVideoOut) { return OrbisVideoOutErrorInvalidEvent; } - return 0; + // sceVideoOutGetEventId reports the event kind: 0 = flip, 1 = vblank. + if (ident == SceVideoOutInternalEventFlip) + { + return 0; + } + + if (ident == SceVideoOutInternalEventVblank) + { + return 1; + } + + return OrbisVideoOutErrorInvalidEvent; } [SysAbiExport( @@ -783,7 +721,7 @@ public static class VideoOutExports } if (filter != OrbisKernelEventFilterVideoOut || - ident is not (SceVideoOutInternalEventVblank or SceVideoOutInternalEventFlip)) + (ident != SceVideoOutInternalEventFlip && ident != SceVideoOutInternalEventVblank)) { return OrbisVideoOutErrorInvalidEvent; } @@ -1103,58 +1041,6 @@ public static class VideoOutExports return groupIndex < 0 ? groupIndex : setIndex; } - private static void SignalVblank(VideoOutPortState port) - { - // Snapshot the registrations into a pooled rental so the triggers can run outside - // _stateGate without copying the list into a fresh allocation on every edge. - // A per-port reusable buffer would race: the pump thread and a guest thread's - // first-edge signal (AddVblankEvent) can signal the same port concurrently. - FlipEventRegistration[]? vblankEvents = null; - int vblankEventCount; - ulong eventHint; - lock (_stateGate) - { - port.VblankCount++; - eventHint = SceVideoOutInternalEventVblank | - ((port.VblankCount & 0x0000_FFFF_FFFF_FFFFUL) << 16); - vblankEventCount = port.VblankEvents.Count; - if (vblankEventCount != 0) - { - vblankEvents = ArrayPool.Shared.Rent(vblankEventCount); - port.VblankEvents.CopyTo(vblankEvents); - } - } - - var signalCount = Interlocked.Increment(ref _vblankSignalCount); - - if (vblankEvents is not null) - { - try - { - for (var i = 0; i < vblankEventCount; i++) - { - _ = KernelEventQueueCompatExports.TriggerDisplayEvent( - vblankEvents[i].Equeue, - SceVideoOutInternalEventVblank, - OrbisKernelEventFilterVideoOut, - eventHint, - vblankEvents[i].UserData); - } - } - finally - { - ArrayPool.Shared.Return(vblankEvents); - } - } - - if (_logVideoOutSync && (signalCount <= 8 || signalCount % 60 == 0)) - { - Console.Error.WriteLine( - $"[LOADER][SYNC] vblank#{signalCount} handle={port.Handle} count={port.VblankCount} " + - $"queues={vblankEventCount} hint=0x{eventHint:X16}"); - } - } - private static int SubmitFlip( CpuContext ctx, int handle, @@ -1197,6 +1083,9 @@ public static class VideoOutExports } } + PaceFlip(port.FlipRate); + PerfOverlay.RecordSubmit(); + var guestImageSubmitted = false; ulong guestImageAddress = 0; if (submitGpuImage && @@ -1216,8 +1105,13 @@ public static class VideoOutExports _ = TryDumpFrame(ctx, port, bufferIndex, flipMode, flipArg); } - if (flipEvents is not null) + void TriggerFlipEvents() { + if (flipEvents is null) + { + return; + } + try { for (var i = 0; i < flipEventCount; i++) @@ -1233,23 +1127,34 @@ public static class VideoOutExports finally { ArrayPool.Shared.Return(flipEvents); + flipEvents = null; } } - var flipCount = Interlocked.Increment(ref _flipSubmitCount); - if (_logVideoOutSync && (flipCount <= 8 || flipCount % 60 == 0)) + if (submitGpuImage) { - Console.Error.WriteLine( - $"[LOADER][SYNC] flip#{flipCount} handle={handle} buffer={bufferIndex} " + - $"addr=0x{guestImageAddress:X16} submitted={guestImageSubmitted} " + - $"flipQueues={flipEventCount}"); + TriggerFlipEvents(); + } + else if (VulkanVideoPresenter.SubmitOrderedGuestAction( + TriggerFlipEvents, + $"videoout flip complete handle={handle} index={bufferIndex}") == 0) + { + // Headless startup has no render queue to order against. + TriggerFlipEvents(); } - if (_logVideoOut) - { - TraceVideoOut($"videoout.submit_flip handle={handle} index={bufferIndex} mode={flipMode} arg={flipArg} events={flipEventCount}"); - } + TraceVideoOut( + $"videoout.submit_flip handle={handle} index={bufferIndex} mode={flipMode} " + + $"arg={flipArg} addr=0x{guestImageAddress:X16} submitted={guestImageSubmitted} " + + $"events={flipEventCount} ordered_completion={!submitGpuImage}"); ReportFrameRate(presented: false); + var diagnosticFlipNumber = Interlocked.Increment(ref _diagnosticFlipCount); + if (_holdFirstFlipMilliseconds > 0 && diagnosticFlipNumber == _holdFlipNumber) + { + Console.Error.WriteLine( + $"[LOADER][INFO] Holding guest flip #{diagnosticFlipNumber} for {_holdFirstFlipMilliseconds} ms for visual verification."); + Thread.Sleep(_holdFirstFlipMilliseconds); + } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1284,11 +1189,140 @@ public static class VideoOutExports var elapsedSeconds = (double)elapsedTicks / Stopwatch.Frequency; var submitted = Interlocked.Exchange(ref _submittedFrameCount, 0); var presentedCount = Interlocked.Exchange(ref _presentedFrameCount, 0); - var missedEdges = Interlocked.Exchange(ref _vblankMissedEdges, 0); + var (draws, drawMs, pipelines, spirvCompiles) = VulkanVideoPresenter.ReadAndResetPerfCounters(); Console.Error.WriteLine( $"[LOADER][PERF] videoout submitted_fps={submitted / elapsedSeconds:F1} " + $"presented_fps={presentedCount / elapsedSeconds:F1} " + - $"vblank_missed={missedEdges}"); + $"draws={draws} draw_ms={drawMs:F0} pipelines={pipelines} spirv={spirvCompiles}"); + } + + private static readonly bool _flipPacingDisabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_NO_FLIP_PACING"), + "1", + StringComparison.Ordinal); + private static long _lastFlipPacingTimestamp; + + private static Thread? _vblankThread; + private static readonly object _vblankThreadGate = new(); + + /// + /// Starts the emulated vblank tick once a guest registers interest in vblank + /// events. The tick fires the registered vblank events on their event queues + /// at the display refresh cadence so guests that park their main/render loop + /// on a vblank equeue keep advancing. + /// + private static void StartVblankThreadOnce() + { + if (Volatile.Read(ref _vblankThread) is not null) + { + return; + } + + lock (_vblankThreadGate) + { + if (_vblankThread is not null) + { + return; + } + + var thread = new Thread(VblankTickLoop) + { + IsBackground = true, + Name = "SharpEmu-Vblank", + }; + _vblankThread = thread; + thread.Start(); + } + } + + private static void VblankTickLoop() + { + var pending = new List<(ulong Equeue, ulong DataHint, ulong UserData)>(); + var next = Stopwatch.GetTimestamp(); + while (Volatile.Read(ref _vblankStopRequested) == 0) + { + uint refresh = 60; + pending.Clear(); + lock (_stateGate) + { + foreach (var port in _ports.Values) + { + if (port.VblankEvents.Count == 0) + { + continue; + } + + refresh = port.RefreshRate == 0 ? 60 : port.RefreshRate; + port.VblankCount++; + var dataHint = (port.VblankCount & 0x0000_FFFF_FFFF_FFFFUL) << 16; + foreach (var registration in port.VblankEvents) + { + pending.Add((registration.Equeue, dataHint, registration.UserData)); + } + } + } + + foreach (var (equeue, dataHint, userData) in pending) + { + _ = KernelEventQueueCompatExports.TriggerDisplayEvent( + equeue, + SceVideoOutInternalEventVblank, + OrbisKernelEventFilterVideoOut, + dataHint, + userData); + } + + var interval = Stopwatch.Frequency / Math.Max(1, (long)refresh); + next += interval; + var now = Stopwatch.GetTimestamp(); + if (next < now) + { + next = now; + } + + HostTiming.SleepUntil(next); + } + } + + /// + /// Emulates the display vblank cadence: hardware completes flips at the + /// requested rate, which is what paces the game's main loop. Without this + /// the guest runs as fast as the GPU pipeline drains, so frame delivery + /// is bursty and animation judders. When the emulator runs slower than + /// the target rate the sleep never engages. + /// + private static void PaceFlip(int flipRate) + { + if (_flipPacingDisabled) + { + return; + } + + var refreshRate = flipRate switch + { + 1 => 30, + 2 => 20, + _ => 60, + }; + var intervalTicks = Stopwatch.Frequency / refreshRate; + var now = Stopwatch.GetTimestamp(); + var last = Interlocked.Read(ref _lastFlipPacingTimestamp); + var target = last + intervalTicks; + if (target <= now) + { + Interlocked.CompareExchange(ref _lastFlipPacingTimestamp, now, last); + return; + } + + var waitMilliseconds = (target - now) * 1000 / Stopwatch.Frequency; + if (waitMilliseconds is >= 0 and < 100) + { + // Precise wait: Thread.Sleep alone overshoots by a scheduler + // quantum, which caps the flip rate below the target cadence. + HostTiming.SleepUntil(target); + } + + Interlocked.CompareExchange(ref _lastFlipPacingTimestamp, target, last); } private static int RegisterBufferRange(VideoOutPortState port, int startIndex, ReadOnlySpan addresses, BufferAttribute attribute, int requestedGroupIndex = -1) @@ -1330,11 +1364,10 @@ public static class VideoOutExports slot.AddressRight = 0; } - if (_logVideoOut) - { - TraceVideoOut( - $"videoout.register_buffers handle={port.Handle} group={groupIndex} start={startIndex} count={addresses.Length} fmt=0x{attribute.PixelFormat:X} tile={attribute.TilingMode} {attribute.Width}x{attribute.Height} pitch={attribute.PitchInPixel}"); - } + TraceVideoOut( + $"videoout.register_buffers handle={port.Handle} group={groupIndex} start={startIndex} count={addresses.Length} " + + $"addresses=[{string.Join(',', addresses.ToArray().Select(static address => $"0x{address:X16}"))}] " + + $"fmt=0x{attribute.PixelFormat:X} tile={attribute.TilingMode} {attribute.Width}x{attribute.Height} pitch={attribute.PitchInPixel}"); GuestGpu.Current.EnsureStarted(attribute.Width, attribute.Height); var guestFormat = MapPixelFormatToGuestTextureFormat(attribute.PixelFormat); @@ -1366,9 +1399,9 @@ public static class VideoOutExports private static bool TryReadBufferAttribute(CpuContext ctx, ulong attributeAddress, bool attribute2, out BufferAttribute attribute) { attribute = default; - if (!ctx.TryReadUInt32(attributeAddress + 0x04, out var tilingMode) || - !ctx.TryReadUInt32(attributeAddress + 0x0C, out var width) || - !ctx.TryReadUInt32(attributeAddress + 0x10, out var height)) + if (!TryReadUInt32(ctx, attributeAddress + 0x04, out var tilingMode) || + !TryReadUInt32(ctx, attributeAddress + 0x0C, out var width) || + !TryReadUInt32(ctx, attributeAddress + 0x10, out var height)) { return false; } @@ -1385,10 +1418,10 @@ public static class VideoOutExports return true; } - if (!ctx.TryReadUInt32(attributeAddress + 0x00, out var pixelFormat32) || - !ctx.TryReadUInt32(attributeAddress + 0x08, out var aspectRatio) || - !ctx.TryReadUInt32(attributeAddress + 0x14, out var pitchInPixel) || - !ctx.TryReadUInt32(attributeAddress + 0x18, out var option32)) + if (!TryReadUInt32(ctx, attributeAddress + 0x00, out var pixelFormat32) || + !TryReadUInt32(ctx, attributeAddress + 0x08, out var aspectRatio) || + !TryReadUInt32(ctx, attributeAddress + 0x14, out var pitchInPixel) || + !TryReadUInt32(ctx, attributeAddress + 0x18, out var option32)) { return false; } @@ -1500,7 +1533,7 @@ public static class VideoOutExports var basePath = GetFrameDumpBasePath(frameIndex, port.Handle, bufferIndex); WriteBmp(basePath + ".bmp", attribute.Width, attribute.Height, rgb); WriteFrameMetadata(basePath + ".txt", slot.AddressLeft, attribute, bufferIndex, flipMode, flipArg, "bmp-linear-read", fingerprint); - if (_logVideoOut) + if (_traceVideoOut) { TraceVideoOut($"videoout.dump_frame path={basePath}.bmp addr=0x{slot.AddressLeft:X16} {attribute.Width}x{attribute.Height} fmt=0x{attribute.PixelFormat:X} fingerprint=0x{fingerprint:X16}"); } @@ -1542,7 +1575,7 @@ public static class VideoOutExports var basePath = GetFrameDumpBasePath(frameIndex, handle, bufferIndex); File.WriteAllBytes(basePath + ".raw", bytes); WriteFrameMetadata(basePath + ".txt", address, attribute, bufferIndex, flipMode, flipArg, reason, fingerprint); - if (_logVideoOut) + if (_traceVideoOut) { TraceVideoOut($"videoout.dump_frame path={basePath}.raw addr=0x{address:X16} bytes={byteCount} reason={reason} fingerprint=0x{fingerprint:X16}"); } @@ -1565,14 +1598,35 @@ public static class VideoOutExports private static uint GetBytesPerPixel(ulong pixelFormat) => pixelFormat is SceVideoOutPixelFormatA8R8G8B8Srgb or SceVideoOutPixelFormatA8B8G8R8Srgb or - SceVideoOutPixelFormatB8G8R8A8Unorm or - SceVideoOutPixelFormatR8G8B8A8Unorm or SceVideoOutPixelFormatA2R10G10B10 or SceVideoOutPixelFormatA2R10G10B10Srgb or - SceVideoOutPixelFormatA2R10G10B10Bt2020Pq + SceVideoOutPixelFormatA2R10G10B10Bt2020Pq or + SceVideoOutPixelFormat2R8G8B8A8Srgb or + SceVideoOutPixelFormat2B8G8R8A8Srgb or + SceVideoOutPixelFormat2R10G10B10A2 or + SceVideoOutPixelFormat2B10G10R10A2 or + SceVideoOutPixelFormat2R10G10B10A2Srgb or + SceVideoOutPixelFormat2B10G10R10A2Srgb or + SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq or + SceVideoOutPixelFormat2B10G10R10A2Bt2100Pq ? 4u : 0u; + internal static bool IsPacked10BitPixelFormat(ulong pixelFormat) => + IsPacked10BitPixelFormatNormalized(NormalizePixelFormat(pixelFormat)); + + private static bool IsPacked10BitPixelFormatNormalized(ulong pixelFormat) => + pixelFormat is + SceVideoOutPixelFormatA2R10G10B10 or + SceVideoOutPixelFormatA2R10G10B10Srgb or + SceVideoOutPixelFormatA2R10G10B10Bt2020Pq or + SceVideoOutPixelFormat2R10G10B10A2 or + SceVideoOutPixelFormat2B10G10R10A2 or + SceVideoOutPixelFormat2R10G10B10A2Srgb or + SceVideoOutPixelFormat2B10G10R10A2Srgb or + SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq or + SceVideoOutPixelFormat2B10G10R10A2Bt2100Pq; + // Maps the PS5 VideoOut pixel format space to the AGC "guest texture format" tags // the backend keys its guest-image registry on (see VulkanVideoPresenter. // GetGuestTextureFormat: format=10 => 56 for 8-bit RGBA variants, format=9 => 9 for 10-bit). @@ -1581,14 +1635,101 @@ public static class VideoOutExports { SceVideoOutPixelFormatA8R8G8B8Srgb or SceVideoOutPixelFormatA8B8G8R8Srgb or - SceVideoOutPixelFormatB8G8R8A8Unorm or - SceVideoOutPixelFormatR8G8B8A8Unorm => 56u, + SceVideoOutPixelFormat2R8G8B8A8Srgb or + SceVideoOutPixelFormat2B8G8R8A8Srgb => 56u, SceVideoOutPixelFormatA2R10G10B10 or SceVideoOutPixelFormatA2R10G10B10Srgb or - SceVideoOutPixelFormatA2R10G10B10Bt2020Pq => 9u, + SceVideoOutPixelFormatA2R10G10B10Bt2020Pq or + SceVideoOutPixelFormat2R10G10B10A2 or + SceVideoOutPixelFormat2B10G10R10A2 or + SceVideoOutPixelFormat2R10G10B10A2Srgb or + SceVideoOutPixelFormat2B10G10R10A2Srgb or + SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq or + SceVideoOutPixelFormat2B10G10R10A2Bt2100Pq => 9u, _ => 0u, }; + internal static bool TryPackRgba8Pixel( + ulong pixelFormat, + byte red, + byte green, + byte blue, + byte alpha, + out uint packed) + { + pixelFormat = NormalizePixelFormat(pixelFormat); + if (!IsPacked10BitPixelFormatNormalized(pixelFormat)) + { + packed = 0; + return false; + } + + packed = PackRgba8PixelNormalized(pixelFormat, red, green, blue, alpha); + return true; + } + + private static uint PackRgba8PixelNormalized( + ulong pixelFormat, + byte red, + byte green, + byte blue, + byte alpha) + { + var red10 = ExpandUnorm8To10(red); + var green10 = ExpandUnorm8To10(green); + var blue10 = ExpandUnorm8To10(blue); + var alpha2 = ((uint)alpha * 3u + 127u) / 255u; + return HasRedInLeastSignificantBits(pixelFormat) + ? red10 | (green10 << 10) | (blue10 << 20) | (alpha2 << 30) + : blue10 | (green10 << 10) | (red10 << 20) | (alpha2 << 30); + } + + internal static bool TryConvertPacked10ToRgba8( + uint packed, + ulong pixelFormat, + Span rgba) + { + pixelFormat = NormalizePixelFormat(pixelFormat); + if (rgba.Length < 4 || !IsPacked10BitPixelFormatNormalized(pixelFormat)) + { + return false; + } + + ConvertPacked10ToRgba8Normalized(packed, pixelFormat, rgba); + return true; + } + + private static void ConvertPacked10ToRgba8Normalized( + uint packed, + ulong pixelFormat, + Span rgba) + { + var least = packed & 0x3FFu; + var green = (packed >> 10) & 0x3FFu; + var most = (packed >> 20) & 0x3FFu; + var redIsLeast = HasRedInLeastSignificantBits(pixelFormat); + var red = redIsLeast ? least : most; + var blue = redIsLeast ? most : least; + rgba[0] = ReduceUnorm10To8(red); + rgba[1] = ReduceUnorm10To8(green); + rgba[2] = ReduceUnorm10To8(blue); + rgba[3] = (byte)((((packed >> 30) & 0x3u) * 255u + 1u) / 3u); + } + + private static bool HasRedInLeastSignificantBits(ulong pixelFormat) => + pixelFormat is + SceVideoOutPixelFormat2R10G10B10A2 or + SceVideoOutPixelFormat2R10G10B10A2Srgb or + SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq; + + private static uint ExpandUnorm8To10(byte value) => + ((uint)value * 1023u + 127u) / 255u; + + // Preserve both UNORM endpoints and round to nearest. A plain >> 2 is a + // biased truncation because the 10-bit maximum is 1023, not 1020. + private static byte ReduceUnorm10To8(uint value) => + (byte)((value * 255u + 511u) / 1023u); + private static ulong NormalizePixelFormat(ulong pixelFormat) { if (GetBytesPerPixel(pixelFormat) != 0) @@ -1614,22 +1755,28 @@ public static class VideoOutExports private static void ConvertRowToRgb(ReadOnlySpan source, Span destination, ulong pixelFormat) { + pixelFormat = NormalizePixelFormat(pixelFormat); var dst = 0; + Span rgba = stackalloc byte[4]; + var packed10 = IsPacked10BitPixelFormatNormalized(pixelFormat); for (var src = 0; src + 3 < source.Length; src += 4) { - if (pixelFormat is SceVideoOutPixelFormatA8B8G8R8Srgb or SceVideoOutPixelFormatR8G8B8A8Unorm) + if (packed10) + { + var packed = BinaryPrimitives.ReadUInt32LittleEndian(source[src..(src + 4)]); + ConvertPacked10ToRgba8Normalized(packed, pixelFormat, rgba); + destination[dst++] = rgba[0]; + destination[dst++] = rgba[1]; + destination[dst++] = rgba[2]; + } + else if (pixelFormat is + SceVideoOutPixelFormatA8B8G8R8Srgb or + SceVideoOutPixelFormat2R8G8B8A8Srgb) { destination[dst++] = source[src + 0]; destination[dst++] = source[src + 1]; destination[dst++] = source[src + 2]; } - else if (pixelFormat is SceVideoOutPixelFormatA2R10G10B10 or SceVideoOutPixelFormatA2R10G10B10Srgb or SceVideoOutPixelFormatA2R10G10B10Bt2020Pq) - { - var value = BinaryPrimitives.ReadUInt32LittleEndian(source[src..(src + 4)]); - destination[dst++] = (byte)(((value >> 20) & 0x3FF) >> 2); - destination[dst++] = (byte)(((value >> 10) & 0x3FF) >> 2); - destination[dst++] = (byte)((value & 0x3FF) >> 2); - } else { destination[dst++] = source[src + 2]; @@ -1639,6 +1786,36 @@ public static class VideoOutExports } } + [Conditional("DEBUG")] + private static void RunPixelFormatSelfChecks() + { + Span rgba = stackalloc byte[4]; + Debug.Assert(TryPackRgba8Pixel( + SceVideoOutPixelFormat2R10G10B10A2Srgb, + 255, 0, 0, 255, + out var rFirst)); + Debug.Assert(rFirst == 0xC00003FFu); + Debug.Assert(TryConvertPacked10ToRgba8( + rFirst, + SceVideoOutPixelFormat2R10G10B10A2Srgb, + rgba)); + Debug.Assert(rgba.SequenceEqual(new byte[] { 255, 0, 0, 255 })); + + Debug.Assert(TryPackRgba8Pixel( + SceVideoOutPixelFormat2B10G10R10A2Srgb, + 255, 0, 0, 255, + out var bFirst)); + Debug.Assert(bFirst == 0xFFF00000u); + Debug.Assert(TryConvertPacked10ToRgba8( + bFirst, + SceVideoOutPixelFormat2B10G10R10A2Srgb, + rgba)); + Debug.Assert(rgba.SequenceEqual(new byte[] { 255, 0, 0, 255 })); + Debug.Assert(ReduceUnorm10To8(0) == 0); + Debug.Assert(ReduceUnorm10To8(512) == 128); + Debug.Assert(ReduceUnorm10To8(1023) == 255); + } + private static string GetFrameDumpBasePath(long frameIndex, int handle, int bufferIndex) { var directory = GetLogsDirectory(); @@ -1757,6 +1934,19 @@ public static class VideoOutExports return true; } + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + private static bool TryReadInt16(CpuContext ctx, ulong address, out short value) { Span buffer = stackalloc byte[sizeof(short)]; @@ -1770,8 +1960,22 @@ public static class VideoOutExports return true; } + private static readonly bool _traceVideoOut = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIDEOOUT"), + "1", + StringComparison.Ordinal); + private static readonly bool _dumpVideoOut = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DUMP_VIDEOOUT"), + "1", + StringComparison.Ordinal); + private static void TraceVideoOut(string message) { + if (!_traceVideoOut) + { + return; + } + Console.Error.WriteLine($"[LOADER][TRACE] {message}"); } } diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs index a766825..29efe6c 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs @@ -6,6 +6,7 @@ using Silk.NET.Core.Native; using Silk.NET.Maths; using SharpEmu.HLE; using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Bink; using Silk.NET.Input; using SharpEmu.Libs.Gpu; using SharpEmu.ShaderCompiler; @@ -48,7 +49,9 @@ internal sealed record VulkanTranslatedGuestDraw( internal sealed record VulkanOffscreenGuestDraw( VulkanTranslatedGuestDraw Draw, IReadOnlyList Targets, - bool PublishTarget); + GuestDepthTarget? DepthTarget, + bool PublishTarget, + ulong ShaderAddress); internal sealed record VulkanComputeGuestDispatch( ulong ShaderAddress, @@ -57,33 +60,205 @@ internal sealed record VulkanComputeGuestDispatch( IReadOnlyList GlobalMemoryBuffers, uint GroupCountX, uint GroupCountY, - uint GroupCountZ); + uint GroupCountZ, + uint BaseGroupX, + uint BaseGroupY, + uint BaseGroupZ, + uint LocalSizeX, + uint LocalSizeY, + uint LocalSizeZ, + bool IsIndirect, + bool WritesGlobalMemory, + uint ThreadCountX = uint.MaxValue, + uint ThreadCountY = uint.MaxValue, + uint ThreadCountZ = uint.MaxValue); + +internal sealed record VulkanOrderedGuestAction( + Action Action, + string DebugName); + +internal sealed record VulkanOrderedGuestFlip( + long Version, + int VideoOutHandle, + int DisplayBufferIndex, + ulong Address, + uint Width, + uint Height, + uint PitchInPixel); + +internal sealed record VulkanOrderedGuestFlipWait( + long Version, + int VideoOutHandle, + int DisplayBufferIndex); + +internal readonly record struct VulkanGuestQueueIdentity( + string Name, + ulong SubmissionId) +{ + public static VulkanGuestQueueIdentity Default { get; } = new("host.default", 0); +} internal static unsafe class VulkanVideoPresenter { private const uint DefaultWindowWidth = 1280; private const uint DefaultWindowHeight = 720; - private const int MaxPendingGuestWork = 16; - // A single guest frame commonly contains 30-50 translated draws. Limiting - // this to 16 split one frame across several 60 Hz window callbacks and - // unnecessarily throttled the producer behind the bounded work queue. - private const int MaxGuestWorkPerRender = 128; + // Vulkan's portable upper bound for minStorageBufferOffsetAlignment is + // 256 bytes. Using that fixed power of two (instead of racing the render + // thread's physical-device query) gives shader translation and descriptor + // creation one stable aliasing contract on every conformant device. + internal const ulong GuestStorageBufferOffsetAlignment = 256; + // Guest draw snapshots churn through a small set of 128 KiB-16 MiB size + // classes thousands of times per second. The process-wide shared pool + // trims and repartitions those large arrays aggressively under GC load, + // causing hundreds of MiB/s of replacement byte[] allocations. Keep a + // bounded, non-shared pool for AGC-to-presenter ownership transfers. + internal static System.Buffers.ArrayPool GuestDataPool { get; } = + System.Buffers.ArrayPool.Create( + maxArrayLength: 16 * 1024 * 1024, + maxArraysPerBucket: 96); + // The pending queue and per-render drain budget bound how much guest GPU + // work can be buffered ahead of the presenter. Draws are batched into + // shared command buffers, so draining a large batch per render tick is + // cheap; small caps here throttle games that issue more than a handful + // of draws per frame to a fraction of the display rate. The pending cap + // stays tighter than the drain budget because queued draws pin their + // pooled guest-data arrays until the render thread uploads them. + private const int MaxPendingGuestWork = 64; + // A captured 4K flip can consume tens of MiB of device-local memory. + // Retain only a short presentation queue while always preserving the + // newest generation; older immutable versions are retired immediately. + private const int MaxPendingGuestFlipVersions = 4; + // A count-only queue bound is not a memory bound: one compute dispatch can + // carry dozens of full-resolution texture snapshots. At 4K, 64 queued + // dispatches retained more than 12 GiB of managed byte arrays before the + // render thread could upload them. Keep the count cap for small work and + // independently apply backpressure to the actual retained payload. + private static readonly ulong _maxPendingGuestWorkBytes = + (ulong.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_PENDING_GUEST_WORK_MB"), + out var pendingGuestWorkMb) && pendingGuestWorkMb > 0 + ? pendingGuestWorkMb + : 256UL) * 1024UL * 1024UL; + private const int MaxGuestWorkPerRender = 256; + // On macOS the whole window loop — including Render() and its guest-work + // drain — runs on the process main thread, so draining a large backlog of + // slow guest work (heavy compute) blocks the Cocoa event pump and marks the + // window "Not Responding" while starving the swapchain present. Cap the + // wall-clock time spent draining per Render() call; leftover work stays + // queued for the next frame. SHARPEMU_RENDER_WORK_BUDGET_MS overrides + // (0 disables the cap); default 12ms keeps the window interactive at ~60Hz. + private static readonly long _renderWorkBudgetTicks = + (long.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_RENDER_WORK_BUDGET_MS"), + out var renderBudgetMs) && renderBudgetMs >= 0 + ? renderBudgetMs + : 12L) * System.Diagnostics.Stopwatch.Frequency / 1000L; + // Max time the main-thread Render() will block waiting for a frame slot's + // GPU fence before skipping the frame and returning to the event pump. + // Prevents the window freezing behind a slow-compute GPU backlog. + // SHARPEMU_FRAME_WAIT_BUDGET_MS overrides; default 8ms. + private static readonly ulong _frameSlotWaitBudgetNs = + (ulong.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_FRAME_WAIT_BUDGET_MS"), + out var frameWaitMs) && frameWaitMs > 0 + ? frameWaitMs + : 8UL) * 1_000_000UL; + // Cap the guest-submission fence wait so a GPU submission whose fence never + // signals (a mistranslated compute shader that hangs the Metal queue) cannot + // freeze the render thread forever and starve the swapchain present. + // SHARPEMU_FENCE_WAIT_TIMEOUT_MS overrides; default 3s. + private static readonly ulong _guestFenceWaitTimeoutNs = + ulong.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_FENCE_WAIT_TIMEOUT_MS"), out var fenceMs) && fenceMs > 0 + ? fenceMs * 1_000_000UL + : 3_000_000_000UL; + // When making room in the in-flight submission queue from the macOS MAIN + // thread (Render() -> guest-work drain), block only this long per attempt + // instead of the full fence timeout. If a slow/capped compute submission + // isn't done yet, proceed anyway: the in-flight cap is soft, the fence and + // command-buffer pools are dynamic so a brief overshoot is safe, and the + // queue drains as GPU completions land on later frames. This keeps the + // window responsive (event pump runs) under a heavy compute backlog instead + // of the main thread sitting in vkWaitForFences for up to 3s per chunk. + // SHARPEMU_SUBMISSION_CAPACITY_WAIT_MS overrides; default 100ms; 0 restores + // the full blocking wait. + private static readonly ulong _submissionCapacityWaitNs = + ulong.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_SUBMISSION_CAPACITY_WAIT_MS"), out var capMs) + ? capMs * 1_000_000UL + : 100_000_000UL; + private static readonly HashSet _tracedFenceTimeouts = new(); + private static long _guestQueueBackpressureTraceCount; + // Diagnostic: skip every compute dispatch (mistranslated compute shaders + // run long / GPU-hang and starve the present). Isolates whether the + // geometry+composite path renders on its own. + private static readonly bool _skipAllCompute = + Environment.GetEnvironmentVariable("SHARPEMU_SKIP_ALL_COMPUTE") == "1"; + // Diagnostic: skip compute dispatches whose GroupCountZ is at least this, + // to isolate a specific tall dispatch (e.g. Demon's Souls' 27x15x72 froxel + // shader that hangs the Metal queue) without needing its ASLR-varying + // address. 0 disables. + private static readonly uint _skipTallComputeZ = + uint.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_SKIP_TALL_COMPUTE_Z"), out var z) + ? z + : 0; private const uint GuestPrimitiveRectList = 0x11; - private const uint GuestFormatR32Uint = 0x10004; - private const uint GuestFormatR32Sint = 0x20004; - private const uint GuestFormatR32Sfloat = 0x30004; - private const uint GuestFormatR16G16Uint = 0x10005; - private const uint GuestFormatR16G16Sint = 0x20005; - private const uint GuestFormatR16G16Sfloat = 0x30005; - private const uint GuestFormatR8G8B8A8Uint = 0x1000A; - private const uint GuestFormatR8G8B8A8Sint = 0x2000A; - private const uint GuestFormatR16G16B16A16Uint = 0x1000C; - private const uint GuestFormatR16G16B16A16Sint = 0x2000C; private static readonly object _gate = new(); - private static readonly Queue _pendingGuestWork = new(); + private readonly record struct PendingGuestWork( + object Work, + ulong PayloadBytes, + long Sequence, + long RequiredSequence, + long EnqueuedTicks, + VulkanGuestQueueIdentity Queue); + + // PS5 exposes independent graphics and asynchronous-compute queues. A + // single host FIFO adds dependencies that do not exist in the guest: one + // slow ACB dispatch can delay a graphics clear until the CPU has reused + // that heap. Keep FIFO order within each logical guest queue and schedule + // ready queues round-robin. Explicit WAIT_REG_MEM packets remain the only + // mechanism that orders one logical queue behind another. + private static readonly Dictionary> + _pendingGuestWorkByQueue = new(StringComparer.Ordinal); + private static readonly List _pendingGuestQueueSchedule = []; + private static int _pendingGuestQueueCursor; + private static int _pendingGuestWorkCount; + private static ulong _pendingGuestWorkBytes; + // A flip names an image that was rendered earlier in the command stream. + // Keep a small FIFO of those flips instead of replacing an incomplete one + // with the next frame: the guest can enqueue the next frame before the + // render thread reaches the previous image, which otherwise starves + // presentation indefinitely. + private static readonly Queue _pendingGuestImagePresentations = new(); + private static readonly Dictionary _guestImageWorkSequences = new(); private static readonly Dictionary _availableGuestImages = new(); - private static readonly Dictionary _gpuGuestImages = new(); + private static readonly Dictionary<(int Handle, int BufferIndex), long> + _lastOrderedGuestFlipVersions = new(); + private static long _orderedGuestFlipVersionSequence; + // Storage-image initialization is copied only by the first queued writer. + // Later dispatches targeting the same image must not each retain another + // multi-megabyte guest-memory snapshot while waiting for that first writer + // to reach the presenter. Reference counts let failed/completed work + // retire its reservation without leaving a permanent false cache hit. + private readonly record struct PendingGuestImageUpload(int Count, long OwnerSequence); + private static readonly Dictionary<(ulong Address, uint Format), PendingGuestImageUpload> + _pendingGuestImageUploads = new(); + private static readonly Dictionary _pendingGuestImageInitialData = new(); + private static readonly Dictionary + _guestImageExtents = new(); + private static readonly bool _traceGuestImageEvents = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_DRAWS"), + "1", + StringComparison.Ordinal) || + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_EVENTS"), + "1", + StringComparison.Ordinal); + private static readonly bool _traceGuestWorkCompletion = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_WORK_COMPLETION"), + "1", + StringComparison.Ordinal); private static readonly HashSet<(ulong Address, uint Width, uint Height)> _tracedGuestImageSubmissions = []; private static Thread? _thread; @@ -97,15 +272,66 @@ internal static unsafe class VulkanVideoPresenter private const uint NvidiaVendorId = 0x10DE; private const string PortabilityEnumerationExtensionName = "VK_KHR_portability_enumeration"; private const string PortabilitySubsetExtensionName = "VK_KHR_portability_subset"; + private const int GlfwPlatformHint = 0x00050003; + private const int GlfwPlatformWin32 = 0x00060001; + private const int GlfwPlatformCocoa = 0x00060002; + private const int GlfwPlatformWayland = 0x00060003; + private const int GlfwPlatformX11 = 0x00060004; + private const int GlfwPlatformNull = 0x00060005; private static bool _splashHidden; private static long _enqueuedGuestWorkSequence; + // Largest contiguous completed sequence, retained for compact diagnostics. + // Per-queue scheduling can complete a later global id first, so correctness + // checks use IsGuestWorkCompletedLocked rather than numeric <= comparisons. private static long _completedGuestWorkSequence; + private static readonly HashSet _completedGuestWorkOutOfOrder = []; + private static readonly Dictionary _lastEnqueuedGuestWorkByQueue = + new(StringComparer.Ordinal); + private static long _executingGuestWorkSequence; + [ThreadStatic] + private static VulkanGuestQueueIdentity? _submittingGuestQueue; + [ThreadStatic] + private static bool _enqueueAsImmediateQueueFollowup; + [ThreadStatic] + private static LinkedListNode? _immediateFollowupTail; - private static bool ShouldTracePresentedGuestImageContentsForDiagnostics() + private sealed class GuestQueueScope : IDisposable { - var mode = Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"); - return string.Equals(mode, "1", StringComparison.Ordinal) || - string.Equals(mode, "present", StringComparison.OrdinalIgnoreCase); + private readonly VulkanGuestQueueIdentity? _previous; + private bool _disposed; + + public GuestQueueScope(VulkanGuestQueueIdentity queue) + { + _previous = _submittingGuestQueue; + _submittingGuestQueue = queue; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _submittingGuestQueue = _previous; + } + } + + public static IDisposable EnterGuestQueue( + string queueName, + ulong submissionId) => + new GuestQueueScope(new VulkanGuestQueueIdentity( + string.IsNullOrWhiteSpace(queueName) ? "guest.unknown" : queueName, + submissionId)); + + private static long CurrentSubmittingQueueTailLocked() + { + var queue = _submittingGuestQueue; + return queue is { } identity && + _lastEnqueuedGuestWorkByQueue.TryGetValue(identity.Name, out var tail) + ? tail + : 0; } private static bool ShouldTraceGuestImageSubmissionsForDiagnostics() @@ -167,7 +393,7 @@ internal static unsafe class VulkanVideoPresenter 1, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: false) : hasSplash ? new Presentation( @@ -177,7 +403,7 @@ internal static unsafe class VulkanVideoPresenter 1, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: true) : new Presentation( null, @@ -186,7 +412,7 @@ internal static unsafe class VulkanVideoPresenter 0, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: false); StartPresenterLocked(); } @@ -210,9 +436,8 @@ internal static unsafe class VulkanVideoPresenter sequence, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: false); - System.Threading.Monitor.PulseAll(_gate); Console.Error.WriteLine("[LOADER][INFO] Vulkan VideoOut hid splash"); } } @@ -224,11 +449,6 @@ internal static unsafe class VulkanVideoPresenter return; } - if (ShouldTraceGuestImageSubmissionsForDiagnostics()) - { - Console.Error.WriteLine($"[LOADER][TRACE] vk.submit_call kind=Submit {width}x{height}"); - } - lock (_gate) { if (_closed) @@ -244,9 +464,8 @@ internal static unsafe class VulkanVideoPresenter sequence, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: false); - System.Threading.Monitor.PulseAll(_gate); if (_thread is not null) { return; @@ -265,11 +484,6 @@ internal static unsafe class VulkanVideoPresenter return; } - if (ShouldTraceGuestImageSubmissionsForDiagnostics()) - { - Console.Error.WriteLine($"[LOADER][TRACE] vk.submit_call kind=SubmitGuestDraw({drawKind}) {width}x{height}"); - } - lock (_gate) { if (_closed || @@ -289,9 +503,8 @@ internal static unsafe class VulkanVideoPresenter sequence, drawKind, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: CurrentSubmittingQueueTailLocked(), IsSplash: false); - System.Threading.Monitor.PulseAll(_gate); if (_thread is not null) { return; @@ -323,12 +536,6 @@ internal static unsafe class VulkanVideoPresenter return; } - if (ShouldTraceGuestImageSubmissionsForDiagnostics()) - { - Console.Error.WriteLine( - $"[LOADER][TRACE] vk.submit_call kind=SubmitTranslatedDraw {width}x{height} textures={textures.Count}"); - } - lock (_gate) { if (_closed) @@ -355,9 +562,8 @@ internal static unsafe class VulkanVideoPresenter primitiveType, indexBuffer, renderState ?? GuestRenderState.Default), - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: CurrentSubmittingQueueTailLocked(), IsSplash: false); - System.Threading.Monitor.PulseAll(_gate); if (_thread is not null) { return; @@ -381,7 +587,9 @@ internal static unsafe class VulkanVideoPresenter uint primitiveType = 4, GuestIndexBuffer? indexBuffer = null, IReadOnlyList? vertexBuffers = null, - GuestRenderState? renderState = null) + GuestRenderState? renderState = null, + GuestDepthTarget? depthTarget = null, + ulong shaderAddress = 0) { SubmitOffscreenTranslatedDraw( pixelSpirv, @@ -395,7 +603,9 @@ internal static unsafe class VulkanVideoPresenter primitiveType, indexBuffer, vertexBuffers, - renderState); + renderState, + depthTarget, + shaderAddress); } public static void SubmitOffscreenTranslatedDraw( @@ -410,7 +620,9 @@ internal static unsafe class VulkanVideoPresenter uint primitiveType = 4, GuestIndexBuffer? indexBuffer = null, IReadOnlyList? vertexBuffers = null, - GuestRenderState? renderState = null) + GuestRenderState? renderState = null, + GuestDepthTarget? depthTarget = null, + ulong shaderAddress = 0) { if (pixelSpirv.Length == 0 || targets.Count == 0 || @@ -447,7 +659,6 @@ internal static unsafe class VulkanVideoPresenter Blends = Enumerable.Repeat(effectiveRenderState.Blends[0], targets.Count).ToArray(), }; } - lock (_gate) { if (_closed) @@ -466,7 +677,7 @@ internal static unsafe class VulkanVideoPresenter } } - EnqueueGuestWorkLocked( + var workSequence = EnqueueGuestWorkLocked( new VulkanOffscreenGuestDraw( new VulkanTranslatedGuestDraw( vertexSpirv ?? [], @@ -481,7 +692,160 @@ internal static unsafe class VulkanVideoPresenter indexBuffer, effectiveRenderState), targets.ToArray(), - PublishTarget: true)); + depthTarget, + PublishTarget: true, + shaderAddress)); + foreach (var target in targets) + { + _guestImageWorkSequences[target.Address] = workSequence; + } + } + } + + public static void SubmitDepthOnlyTranslatedDraw( + byte[] pixelSpirv, + IReadOnlyList textures, + IReadOnlyList globalMemoryBuffers, + uint attributeCount, + GuestDepthTarget depthTarget, + byte[]? vertexSpirv = null, + uint vertexCount = 3, + uint instanceCount = 1, + uint primitiveType = 4, + GuestIndexBuffer? indexBuffer = null, + IReadOnlyList? vertexBuffers = null, + GuestRenderState? renderState = null, + ulong shaderAddress = 0) + { + if (pixelSpirv.Length == 0 || + depthTarget.Address == 0 || + depthTarget.Width == 0 || + depthTarget.Height == 0) + { + return; + } + + lock (_gate) + { + if (_closed) + { + return; + } + + EnqueueGuestWorkLocked( + new VulkanOffscreenGuestDraw( + new VulkanTranslatedGuestDraw( + vertexSpirv ?? [], + pixelSpirv, + textures.ToArray(), + globalMemoryBuffers.ToArray(), + vertexBuffers?.ToArray() ?? [], + attributeCount, + vertexCount, + instanceCount, + primitiveType, + indexBuffer, + renderState ?? GuestRenderState.Default), + [new GuestRenderTarget( + Address: 0, + depthTarget.Width, + depthTarget.Height, + Format: 10, + NumberType: 0)], + depthTarget, + PublishTarget: false, + shaderAddress)); + } + } + + private sealed record VulkanGuestImageWrite( + ulong Address, + byte[]? Pixels, + uint FillValue); + + /// + /// Reports the extent of a live guest image so DMA writes to its backing + /// memory can be mirrored into the Vulkan image (PS5 render targets alias + /// guest memory, so CP DMA fills/copies are visible to later GPU reads). + /// + internal static bool TryGetGuestImageExtent( + ulong address, + out uint width, + out uint height, + out ulong byteCount) + { + lock (_gate) + { + if (_guestImageExtents.TryGetValue(address, out var extent)) + { + (width, height, byteCount) = extent; + return true; + } + } + + width = 0; + height = 0; + byteCount = 0; + return false; + } + + internal static void SubmitGuestImageFill(ulong address, uint fillValue) + { + lock (_gate) + { + if (_closed || !_guestImageExtents.ContainsKey(address)) + { + return; + } + + _guestImageWorkSequences[address] = EnqueueGuestWorkLocked( + new VulkanGuestImageWrite(address, null, fillValue)); + } + } + + internal static void SubmitGuestImageWrite(ulong address, byte[] pixels) + { + lock (_gate) + { + if (_closed || !_guestImageExtents.ContainsKey(address)) + { + return; + } + + _guestImageWorkSequences[address] = EnqueueGuestWorkLocked( + new VulkanGuestImageWrite(address, pixels, 0)); + } + } + + private static long _perfDrawCount; + private static long _perfDrawTicks; + private static long _perfPipelineCreations; + private static long _perfSpirvCompilations; + + internal static (long Draws, double DrawMs, long Pipelines, long SpirvCompilations) + ReadAndResetPerfCounters() + { + var draws = Interlocked.Exchange(ref _perfDrawCount, 0); + var ticks = Interlocked.Exchange(ref _perfDrawTicks, 0); + var pipelines = Interlocked.Exchange(ref _perfPipelineCreations, 0); + var spirv = Interlocked.Exchange(ref _perfSpirvCompilations, 0); + return (draws, ticks * 1000.0 / System.Diagnostics.Stopwatch.Frequency, pipelines, spirv); + } + + internal static void CountSpirvCompilation() => + Interlocked.Increment(ref _perfSpirvCompilations); + + internal static IReadOnlyList<(ulong Address, uint Width, uint Height, ulong ByteCount)> GetGuestImageExtents() + { + lock (_gate) + { + return _guestImageExtents + .Select(entry => ( + entry.Key, + entry.Value.Width, + entry.Value.Height, + entry.Value.ByteCount)) + .ToArray(); } } @@ -491,7 +855,8 @@ internal static unsafe class VulkanVideoPresenter IReadOnlyList globalMemoryBuffers, uint attributeCount, uint width, - uint height) + uint height, + ulong shaderAddress = 0) { if (pixelSpirv.Length == 0 || width == 0 || @@ -528,36 +893,51 @@ internal static unsafe class VulkanVideoPresenter height, Format: 12, NumberType: 7)], - PublishTarget: false)); + DepthTarget: null, + PublishTarget: false, + shaderAddress)); } } - public static void SubmitComputeDispatch( + public static long SubmitComputeDispatch( ulong shaderAddress, byte[] computeSpirv, IReadOnlyList textures, IReadOnlyList globalMemoryBuffers, uint groupCountX, uint groupCountY, - uint groupCountZ) + uint groupCountZ, + uint baseGroupX, + uint baseGroupY, + uint baseGroupZ, + uint localSizeX, + uint localSizeY, + uint localSizeZ, + bool isIndirect, + bool writesGlobalMemory, + uint threadCountX = uint.MaxValue, + uint threadCountY = uint.MaxValue, + uint threadCountZ = uint.MaxValue) { if (computeSpirv.Length == 0 || groupCountX == 0 || groupCountY == 0 || groupCountZ == 0 || - textures.All(texture => !texture.IsStorage)) + textures.All(texture => !texture.IsStorage) && + !writesGlobalMemory) { - return; + return 0; } + long workSequence; lock (_gate) { if (_closed) { - return; + return 0; } - EnqueueGuestWorkLocked( + workSequence = EnqueueGuestWorkLocked( new VulkanComputeGuestDispatch( shaderAddress, computeSpirv, @@ -565,7 +945,127 @@ internal static unsafe class VulkanVideoPresenter globalMemoryBuffers.ToArray(), groupCountX, groupCountY, - groupCountZ)); + groupCountZ, + baseGroupX, + baseGroupY, + baseGroupZ, + localSizeX, + localSizeY, + localSizeZ, + isIndirect, + writesGlobalMemory, + threadCountX, + threadCountY, + threadCountZ)); + foreach (var key in GetStorageImageUploadKeys(textures)) + { + _pendingGuestImageUploads[key] = + _pendingGuestImageUploads.TryGetValue(key, out var pendingUpload) + ? pendingUpload with { Count = checked(pendingUpload.Count + 1) } + : new PendingGuestImageUpload(1, workSequence); + } + + foreach (var texture in textures) + { + if (texture.IsStorage && texture.Address != 0) + { + _guestImageWorkSequences[texture.Address] = workSequence; + } + } + } + + return workSequence; + } + + /// + /// Enqueues a CPU-visible PM4 side effect behind all GPU work submitted + /// before it. The render thread flushes its open batch and waits for the + /// corresponding guest fences before invoking the action. + /// + public static long SubmitOrderedGuestAction(Action action, string debugName) + { + ArgumentNullException.ThrowIfNull(action); + lock (_gate) + { + return _closed || _thread is null + ? 0 + : EnqueueGuestWorkLocked(new VulkanOrderedGuestAction(action, debugName)); + } + } + + /// + /// Sequence currently being executed by the single guest-work consumer. + /// Intended only for address-filtered lifetime diagnostics emitted from a + /// guest-work callback before advances it. + /// + public static long CurrentGuestWorkSequenceForDiagnostics => + Volatile.Read(ref _executingGuestWorkSequence); + + private static bool IsGuestWorkCompletedLocked(long sequence) => + sequence <= 0 || + sequence <= _completedGuestWorkSequence || + _completedGuestWorkOutOfOrder.Contains(sequence); + + public static bool WaitForGuestWork( + long workSequence, + int timeoutMilliseconds = System.Threading.Timeout.Infinite) + { + if (workSequence <= 0) + { + return false; + } + + var waitIndefinitely = timeoutMilliseconds == System.Threading.Timeout.Infinite; + var deadline = waitIndefinitely + ? long.MaxValue + : Environment.TickCount64 + Math.Max(timeoutMilliseconds, 1); + lock (_gate) + { + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_wait_enter sequence={workSequence} " + + $"contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + } + while (!_closed && !IsGuestWorkCompletedLocked(workSequence)) + { + if (!waitIndefinitely) + { + var remaining = deadline - Environment.TickCount64; + if (remaining <= 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan guest work wait timed out " + + $"sequence={workSequence} contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + return false; + } + + System.Threading.Monitor.Wait( + _gate, + checked((int)Math.Min(remaining, 1_000))); + continue; + } + + // CPU-visible GPU writes are ordering points in the guest + // command stream. First-use shader compilation can take more + // than a minute on MoltenVK; timing out would let the guest + // consume stale zero-filled buffers and permanently corrupt + // the frame. Closing the presenter pulses this monitor, so an + // unbounded correctness wait remains interruptible. + System.Threading.Monitor.Wait(_gate, 1_000); + } + + var completed = IsGuestWorkCompletedLocked(workSequence); + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_wait_exit sequence={workSequence} " + + $"completed={completed} contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + } + return completed; } } @@ -578,64 +1078,40 @@ internal static unsafe class VulkanVideoPresenter var traceSubmission = false; lock (_gate) { - // VideoOut registration does not imply a rendered Vulkan image. - var known = _gpuGuestImages.ContainsKey(address); - if (ShouldTraceGuestImageSubmissionsForDiagnostics()) + if (_closed || + !_availableGuestImages.ContainsKey(address)) { - Console.Error.WriteLine( - $"[LOADER][TRACE] vk.submit_call kind=TrySubmitGuestImage addr=0x{address:X16} " + - $"{width}x{height} known={known}"); - } - - if (_closed) - { - return false; - } - - // The caller (VideoOutExports.SubmitFlip) reports the flip as successful either - // way, so an unregistered address means the frame is dropped silently; warn once - // per address so that shows up in the log. - if (!known) - { - if (_tracedGuestImageSubmissions.Add((address, width, height))) - { - Console.Error.WriteLine( - $"[LOADER][WARN] vk.submit_guest_image_unknown addr=0x{address:X16} " + - $"{width}x{height} - flip target has no completed GPU render output"); - } - return false; } traceSubmission = _tracedGuestImageSubmissions.Add((address, width, height)); var sequence = (_latestPresentation?.Sequence ?? 0) + 1; - _latestPresentation = new Presentation( + var requiredWorkSequence = _guestImageWorkSequences.TryGetValue( + address, + out var imageWorkSequence) + ? imageWorkSequence + : _completedGuestWorkSequence; + var presentation = new Presentation( null, width, height, sequence, GuestDrawKind.None, TranslatedDraw: null, - // A flip targets the image produced by all work already queued - // for this frame. Do not expose it until those draws finish. - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + // Wait only for the work that last wrote this image, not for + // every later command the guest has already queued. Requiring + // the global tail makes a fast guest permanently outrun the + // renderer and turns every flip into a dropped black frame. + RequiredGuestWorkSequence: requiredWorkSequence, IsSplash: false, GuestImageAddress: address); - System.Threading.Monitor.PulseAll(_gate); - if (_thread is not null) + _latestPresentation = presentation; + _pendingGuestImagePresentations.Enqueue(presentation); + while (_pendingGuestImagePresentations.Count > MaxPendingGuestWork) { - return true; + _pendingGuestImagePresentations.Dequeue(); } - - _windowWidth = width; - _windowHeight = height; - _thread = new Thread(Run) - { - IsBackground = true, - Name = "SharpEmu Vulkan VideoOut", - }; - _thread.Start(); } if (traceSubmission) @@ -649,8 +1125,176 @@ internal static unsafe class VulkanVideoPresenter return true; } - // Display buffers registered through sceVideoOutRegisterBuffers are valid flip targets - // even when no AGC render-target write to them was ever observed. + /// + /// Enqueues an AGC flip at its exact position in the logical guest queue. + /// The presenter captures the named image into an immutable Vulkan image + /// before it executes later work from the same queue. Presentation then + /// consumes that captured generation rather than the mutable render target. + /// + public static bool TrySubmitOrderedGuestImageFlip( + int videoOutHandle, + int displayBufferIndex, + ulong address, + uint width, + uint height, + uint pitchInPixel) + { + lock (_gate) + { + if (_closed || + _thread is null || + !_availableGuestImages.ContainsKey(address)) + { + return false; + } + + var version = ++_orderedGuestFlipVersionSequence; + _lastOrderedGuestFlipVersions[(videoOutHandle, displayBufferIndex)] = version; + return EnqueueGuestWorkLocked( + new VulkanOrderedGuestFlip( + version, + videoOutHandle, + displayBufferIndex, + address, + width, + height, + pitchInPixel)) > 0; + } + } + + /// + /// Preserves sceAgcDcbWaitUntilSafeForRendering in queue order. Because an + /// ordered flip first copies the mutable render target into an immutable + /// generation on the same Vulkan queue, reaching this marker proves later + /// rendering cannot change the frame selected by that flip. No CPU wait or + /// event-loop stall is required. + /// + public static long SubmitOrderedGuestFlipWait( + int videoOutHandle, + int displayBufferIndex) + { + lock (_gate) + { + var version = _lastOrderedGuestFlipVersions.TryGetValue( + (videoOutHandle, displayBufferIndex), + out var lastVersion) + ? lastVersion + : 0; + return _closed || _thread is null + ? 0 + : EnqueueGuestWorkLocked( + new VulkanOrderedGuestFlipWait( + version, + videoOutHandle, + displayBufferIndex)); + } + } + + /// + /// On PS5 a render target aliases guest memory, so CPU-prefilled pixels are + /// visible before the first draw. Our Vulkan images start undefined, so the + /// first draw into a new address must seed the image from guest memory. + /// + internal static bool GuestImageWantsInitialData(ulong address) + { + if (address == 0) + { + return false; + } + + lock (_gate) + { + return !_availableGuestImages.ContainsKey(address) && + !_pendingGuestImageInitialData.ContainsKey(address); + } + } + + internal static void ProvideGuestImageInitialData(ulong address, byte[] rgbaPixels) + { + lock (_gate) + { + _pendingGuestImageInitialData[address] = rgbaPixels; + } + } + + private static byte[]? TakeGuestImageInitialData(ulong address) + { + lock (_gate) + { + if (!_pendingGuestImageInitialData.TryGetValue(address, out var data)) + { + return null; + } + + _pendingGuestImageInitialData.Remove(address); + return data; + } + } + + // Mirror of the render thread's texture-cache identities, readable from + // the guest submit thread. The AGC translator used to allocate and copy + // every referenced texture's texels out of guest memory on every draw, + // only for the presenter to discard the bytes on a cache hit — for a + // scene sampling large textures this was by far the dominant CPU cost + // (gigabytes/second of allocation, page faults and GC pressure). + // ConcurrentDictionary keyed set: reads happen per texture per draw on + // the guest submit thread and must not contend with the render thread's + // mutations. + private static readonly System.Collections.Concurrent.ConcurrentDictionary< + TextureContentIdentity, byte> _cachedTextureIdentities = new(); + + internal readonly record struct TextureContentIdentity( + ulong Address, + uint Width, + uint Height, + uint Format, + uint NumberType, + uint DstSelect, + uint TileMode, + uint Pitch, + GuestSampler Sampler); + + // Guest memory handle for render-thread self-healing: when a draw whose + // texel copy was skipped misses the texture cache (eviction, cache + // clear, or any other race), the presenter re-reads the texels itself + // instead of showing a fallback pattern. + private static volatile SharpEmu.HLE.ICpuMemory? _guestMemory; + + internal static void AttachGuestMemory(SharpEmu.HLE.ICpuMemory memory) => + _guestMemory = memory; + + internal static bool IsTextureContentCached(in TextureContentIdentity identity) => + _cachedTextureIdentities.ContainsKey(identity); + + private static void MarkTextureContentCached(in TextureContentIdentity identity) => + _cachedTextureIdentities.TryAdd(identity, 0); + + private static void UnmarkTextureContentCached(in TextureContentIdentity identity) => + _cachedTextureIdentities.TryRemove(identity, out _); + + private static void ClearCachedTextureIdentities() => + _cachedTextureIdentities.Clear(); + + internal static bool IsGuestImageAvailable( + ulong address, + uint format, + uint numberType) + { + var guestFormat = GetGuestTextureFormat(format, numberType); + if (address == 0 || guestFormat == 0) + { + return false; + } + + lock (_gate) + { + return _availableGuestImages.TryGetValue(address, out var availableFormat) && + availableFormat == guestFormat; + } + } + + // Display buffers registered through sceVideoOutRegisterBuffers remain + // valid flip targets even before AGC has rendered into them. internal static void RegisterKnownDisplayBuffer(ulong address, uint guestFormat) { if (address == 0 || guestFormat == 0) @@ -665,6 +1309,19 @@ internal static unsafe class VulkanVideoPresenter } internal static bool IsGpuGuestImageAvailable( + ulong address, + uint format, + uint numberType) => + IsGuestImageAvailable(address, format, numberType); + + /// + /// Returns whether a storage image already exists on the presenter or an + /// earlier queued dispatch owns its one-time guest-memory initialization. + /// This is intentionally separate from : + /// a pending image may skip a duplicate upload but is not yet safe for a + /// flip/presentation lookup. + /// + internal static bool IsGuestImageUploadKnown( ulong address, uint format, uint numberType) @@ -677,8 +1334,9 @@ internal static unsafe class VulkanVideoPresenter lock (_gate) { - return _gpuGuestImages.TryGetValue(address, out var availableFormat) && - availableFormat == guestFormat; + return _availableGuestImages.TryGetValue(address, out var availableFormat) && + availableFormat == guestFormat || + _pendingGuestImageUploads.ContainsKey((address, guestFormat)); } } @@ -687,10 +1345,12 @@ internal static unsafe class VulkanVideoPresenter uint sourceWidth, uint sourceHeight, uint sourceFormat, + uint sourceNumberType, ulong destinationAddress, uint destinationWidth, uint destinationHeight, - uint destinationFormat) + uint destinationFormat, + uint destinationNumberType) { if (sourceAddress == 0 || destinationAddress == 0 || @@ -721,7 +1381,7 @@ internal static unsafe class VulkanVideoPresenter sourceWidth, sourceHeight, sourceFormat, - NumberType: 0, + sourceNumberType, [], IsFallback: false, IsStorage: false), @@ -733,7 +1393,7 @@ internal static unsafe class VulkanVideoPresenter destinationWidth, destinationHeight, destinationFormat, - NumberType: 0)); + destinationNumberType)); return true; } @@ -759,39 +1419,10 @@ internal static unsafe class VulkanVideoPresenter return true; } - private static uint GetGuestTextureFormat(uint format, uint numberType) - { - if (!TryDecodeRenderTargetFormat(format, numberType, out var decoded)) - { - return 0; - } - - return decoded.Format switch - { - Format.R8Unorm => 36, - Format.R8Uint => 49, - Format.R8G8Unorm => 3, - Format.A2R10G10B10UnormPack32 => 9, - Format.B10G11R11UfloatPack32 => 7, - Format.R32Uint => GuestFormatR32Uint, - Format.R32Sint => GuestFormatR32Sint, - Format.R32Sfloat => GuestFormatR32Sfloat, - Format.R16G16Unorm => 5, - Format.R16G16Uint => GuestFormatR16G16Uint, - Format.R16G16Sint => GuestFormatR16G16Sint, - Format.R16G16Sfloat => GuestFormatR16G16Sfloat, - Format.R32G32Sfloat => 75, - Format.R8G8B8A8Unorm => 56, - Format.R8G8B8A8Uint => GuestFormatR8G8B8A8Uint, - Format.R8G8B8A8Sint => GuestFormatR8G8B8A8Sint, - Format.R16G16B16A16Unorm => 12, - Format.R16G16B16A16Uint => GuestFormatR16G16B16A16Uint, - Format.R16G16B16A16Sint => GuestFormatR16G16B16A16Sint, - Format.R16G16B16A16Sfloat => 71, - Format.R32G32B32A32Sfloat => 14, - _ => 0, - }; - } + private static uint GetGuestTextureFormat(uint format, uint numberType) => + IsKnownGuestTextureFormat(format) + ? 0x8000_0000u | ((format & 0x1FFu) << 8) | (numberType & 0xFFu) + : 0; internal static bool TryDecodeRenderTargetFormat( uint dataFormat, @@ -806,23 +1437,19 @@ internal static unsafe class VulkanVideoPresenter (5, 4) => Format.R16G16Uint, (5, 5) => Format.R16G16Sint, (5, 7) => Format.R16G16Sfloat, + (6, 7) or (7, 7) => Format.B10G11R11UfloatPack32, (9, _) => Format.A2R10G10B10UnormPack32, (10, 4) => Format.R8G8B8A8Uint, (10, 5) => Format.R8G8B8A8Sint, + (10, 9) => Format.R8G8B8A8Srgb, (10, _) => Format.R8G8B8A8Unorm, + (11, 7) => Format.R32G32Sfloat, (12, 4) => Format.R16G16B16A16Uint, (12, 5) => Format.R16G16B16A16Sint, (12, 7) => Format.R16G16B16A16Sfloat, - (GuestFormatR32Uint, _) or (20, 0) => Format.R32Uint, - (GuestFormatR32Sint, _) => Format.R32Sint, - (GuestFormatR32Sfloat, _) or (29, 0) or (4, 0) => Format.R32Sfloat, - (GuestFormatR16G16Uint, _) => Format.R16G16Uint, - (GuestFormatR16G16Sint, _) => Format.R16G16Sint, - (GuestFormatR16G16Sfloat, _) => Format.R16G16Sfloat, - (GuestFormatR8G8B8A8Uint, _) => Format.R8G8B8A8Uint, - (GuestFormatR8G8B8A8Sint, _) => Format.R8G8B8A8Sint, - (GuestFormatR16G16B16A16Uint, _) => Format.R16G16B16A16Uint, - (GuestFormatR16G16B16A16Sint, _) => Format.R16G16B16A16Sint, + (13, 7) or (14, 7) => Format.R32G32B32A32Sfloat, + (20, 0) => Format.R32Uint, + (29, 0) or (4, 0) => Format.R32Sfloat, (1, 0) or (36, 0) => Format.R8Unorm, (49, 0) => Format.R8Uint, (3, 0) => Format.R8G8Unorm, @@ -855,7 +1482,7 @@ internal static unsafe class VulkanVideoPresenter } private static bool IsKnownGuestTextureFormat(uint format) => - format is 4 or 5 or 7 or 9 or 13 or 14 or 22 or 29 or 36 or 56 or 62 or 64 or 71; + format is >= 1 and <= 19 or 34 or >= 169 and <= 182; private static byte[] CreateBlackFrame(uint width, uint height) { @@ -879,11 +1506,10 @@ internal static unsafe class VulkanVideoPresenter { if (HostMainThread.IsAvailable) { - // GLFW windowing must run on the process main thread (AppKit on - // macOS, X11's single event queue on Linux), so hand the whole - // window loop to the main-thread pump the CLI parked for us. - // _thread only marks the presenter as running; Run() clears it on - // exit either way. + // AppKit (and therefore GLFW) traps when touched off the process + // main thread on macOS, so hand the whole window loop to the + // main-thread pump the CLI parked for us. _thread only marks the + // presenter as running; Run() clears it on exit either way. _thread = Thread.CurrentThread; HostMainThread.SetShutdownRequestHandler(RequestClose); HostMainThread.Post(Run); @@ -961,25 +1587,6 @@ internal static unsafe class VulkanVideoPresenter } } - // GLFW platform enum (GLFW 3.4): glfwInitHint(GLFW_PLATFORM, ...) selects a - // backend, glfwGetPlatform() reports the one in use. - private const int GlfwPlatformHint = 0x00050003; - private const int GlfwPlatformWin32 = 0x00060001; - private const int GlfwPlatformCocoa = 0x00060002; - private const int GlfwPlatformWayland = 0x00060003; - private const int GlfwPlatformX11 = 0x00060004; - private const int GlfwPlatformNull = 0x00060005; - - /// - /// GLFW's native Wayland backend does not reliably map the Vulkan window - /// with some drivers (notably NVIDIA): the surface presents frames but the - /// window never becomes visible, so the game runs with no picture while - /// audio works. XWayland is dependable, so on a Wayland session that also - /// exposes an X server (DISPLAY set) we force GLFW's X11 backend through - /// its GLFW_PLATFORM init hint before GLFW initializes — the supported way - /// to pick a backend, applied by calling into the same libglfw GLFW loads. - /// Opt back into native Wayland with SHARPEMU_ENABLE_WAYLAND=1. - /// private static unsafe void PreferX11OnLinuxWayland() { if (!OperatingSystem.IsLinux() || @@ -991,10 +1598,6 @@ internal static unsafe class VulkanVideoPresenter return; } - // Only steer on a Wayland session (WAYLAND_DISPLAY set). Forcing X11 - // needs an X server to fall back to (XWayland, DISPLAY set); without - // one, forcing it would make glfwInit fail outright, so leave GLFW - // alone and say why the window may not appear. if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("WAYLAND_DISPLAY"))) { return; @@ -1004,8 +1607,7 @@ internal static unsafe class VulkanVideoPresenter { Console.Error.WriteLine( "[LOADER][WARN] Wayland session without an X server (DISPLAY unset); " + - "cannot steer GLFW to XWayland. If the window does not appear, install " + - "XWayland, or run natively with SHARPEMU_ENABLE_WAYLAND=1."); + "cannot steer GLFW to XWayland. Set SHARPEMU_ENABLE_WAYLAND=1 to use native Wayland."); return; } @@ -1020,8 +1622,7 @@ internal static unsafe class VulkanVideoPresenter System.Runtime.InteropServices.NativeLibrary.GetExport(glfw, "glfwInitHint"); initHint(GlfwPlatformHint, GlfwPlatformX11); Console.Error.WriteLine( - "[LOADER][INFO] Wayland session detected; requested GLFW X11/XWayland " + - "backend (set SHARPEMU_ENABLE_WAYLAND=1 to force native Wayland)."); + "[LOADER][INFO] Wayland session detected; requested GLFW X11/XWayland backend."); } catch (Exception exception) { @@ -1030,8 +1631,6 @@ internal static unsafe class VulkanVideoPresenter } } - /// Logs the backend GLFW actually selected, so a "no window" - /// report shows Wayland vs X11 at a glance. private static unsafe void LogGlfwPlatformInUse() { if (OperatingSystem.IsWindows() || !TryLoadGlfw(out var glfw)) @@ -1079,8 +1678,8 @@ internal static unsafe class VulkanVideoPresenter height = _windowHeight == 0 ? _latestPresentation?.Height ?? 720 : _windowHeight; } - InitializeMacVulkanLoader(); PreferX11OnLinuxWayland(); + InitializeMacVulkanLoader(); try { @@ -1106,53 +1705,476 @@ internal static unsafe class VulkanVideoPresenter { lock (_gate) { + // Guest flips are retained in submission order. The renderer is + // deliberately allowed to lag a frame or two behind the guest + // while it drains expensive work, so use the first completed flip + // rather than repeatedly asking only for the newest one. + while (_pendingGuestImagePresentations.Count > 0 && + _pendingGuestImagePresentations.Peek().Sequence <= presentedSequence) + { + _pendingGuestImagePresentations.Dequeue(); + } + + if (_pendingGuestImagePresentations.Count > 0) + { + var pending = _pendingGuestImagePresentations.Peek(); + if (IsGuestWorkCompletedLocked(pending.RequiredGuestWorkSequence)) + { + presentation = _pendingGuestImagePresentations.Dequeue(); + TryReplaceWithBinkFrame(ref presentation); + return true; + } + + presentation = default; + return false; + } + if (_latestPresentation is not { } latest || latest.Sequence == presentedSequence || - latest.RequiredGuestWorkSequence > _completedGuestWorkSequence) + !IsGuestWorkCompletedLocked(latest.RequiredGuestWorkSequence)) { + if (_latestPresentation is { } rej && + rej.GuestImageAddress != 0 && + rej.Sequence != presentedSequence && + _tracedGuestImagePresentRejections.Add(rej.Sequence)) + { + var reason = rej.Sequence == presentedSequence + ? "already-presented(seq==presented)" + : !IsGuestWorkCompletedLocked(rej.RequiredGuestWorkSequence) + ? $"work-not-done(req={rej.RequiredGuestWorkSequence}>" + + $"contiguous_done={_completedGuestWorkSequence})" + : "unknown"; + Console.Error.WriteLine( + $"[LOADER][WARN] vk.guest_present_rejected addr=0x{rej.GuestImageAddress:X16} " + + $"seq={rej.Sequence} presentedSeq={presentedSequence} reason={reason}"); + } + presentation = default; return false; } presentation = latest; + TryReplaceWithBinkFrame(ref presentation); return true; } } - private static void EnqueueGuestWorkLocked(object work) + private static void TryReplaceWithBinkFrame(ref Presentation presentation) { - while (!_closed && - _thread is not null && - _pendingGuestWork.Count >= MaxPendingGuestWork) + if (!Bink2MovieBridge.TryDecodeNextFrame(out var pixels, out var width, out var height)) { + return; + } + + presentation = new Presentation( + pixels, + width, + height, + presentation.Sequence, + GuestDrawKind.None, + TranslatedDraw: null, + presentation.RequiredGuestWorkSequence, + IsSplash: false); + } + + private static readonly HashSet _tracedGuestImagePresentRejections = new(); + + private static bool HasPendingGuestPresentation(long presentedSequence) + { + lock (_gate) + { + return _pendingGuestImagePresentations.Count > 0 || + _latestPresentation is { } latest && latest.Sequence > presentedSequence; + } + } + + private static long EnqueueGuestWorkLocked(object work) + { + var payloadBytes = GetGuestWorkPayloadBytes(work); + var backpressureLogged = false; + // Work executed by the render-thread consumer can enqueue an ordered + // same-queue completion marker. Blocking that consumer on the normal + // producer backpressure limit deadlocks a full queue: no other thread + // can drain an item to make room for the marker. The consumer has + // already removed the current item, and each immediate follow-up is + // bounded by that item, so admitting it cannot cause unbounded growth. + while (!_enqueueAsImmediateQueueFollowup && + !_closed && + _thread is not null && + (_pendingGuestWorkCount >= MaxPendingGuestWork || + // Always admit one item when no payload is outstanding, even + // when that single item exceeds the configured budget. This + // avoids an impossible wait while still bounding the normal + // multi-item backlog. + (_pendingGuestWorkBytes != 0 && + payloadBytes > _maxPendingGuestWorkBytes - + Math.Min(_pendingGuestWorkBytes, _maxPendingGuestWorkBytes)))) + { + if (!backpressureLogged) + { + backpressureLogged = true; + var traceCount = Interlocked.Increment( + ref _guestQueueBackpressureTraceCount); + if (traceCount <= 16 || (traceCount & (traceCount - 1)) == 0) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_queue_backpressure " + + $"count={traceCount} " + + $"queued={_pendingGuestWorkCount} " + + $"logical_queues={_pendingGuestWorkByQueue.Count} " + + $"retained_mb={_pendingGuestWorkBytes / (1024 * 1024)} " + + $"incoming_mb={payloadBytes / (1024 * 1024)} " + + $"budget_mb={_maxPendingGuestWorkBytes / (1024 * 1024)} " + + $"work={work.GetType().Name}" + + GetGuestWorkPayloadBreakdown(work)); + } + } + System.Threading.Monitor.Wait(_gate); } if (_closed) + { + return 0; + } + + var queue = _submittingGuestQueue ?? VulkanGuestQueueIdentity.Default; + var sequence = ++_enqueuedGuestWorkSequence; + _lastEnqueuedGuestWorkByQueue[queue.Name] = sequence; + var requiredSequence = GetGuestWorkDependencyLocked(work); + if (!_pendingGuestWorkByQueue.TryGetValue(queue.Name, out var pendingQueue)) + { + pendingQueue = new LinkedList(); + _pendingGuestWorkByQueue.Add(queue.Name, pendingQueue); + _pendingGuestQueueSchedule.Add(queue.Name); + } + + var pending = new PendingGuestWork( + work, + payloadBytes, + sequence, + requiredSequence, + System.Diagnostics.Stopwatch.GetTimestamp(), + queue); + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_enqueue sequence={sequence} " + + $"required={requiredSequence} queue={queue.Name} " + + $"immediate={_enqueueAsImmediateQueueFollowup} " + + $"work={work.GetType().Name}"); + } + if (_enqueueAsImmediateQueueFollowup && + _immediateFollowupTail is { List: not null } tail && + ReferenceEquals(tail.List, pendingQueue)) + { + _immediateFollowupTail = pendingQueue.AddAfter(tail, pending); + } + else if (_enqueueAsImmediateQueueFollowup) + { + _immediateFollowupTail = pendingQueue.AddFirst(pending); + } + else + { + pendingQueue.AddLast(pending); + } + RecordGuestImageWritersLocked(work, sequence); + _pendingGuestWorkCount++; + _pendingGuestWorkBytes = SaturatingAdd(_pendingGuestWorkBytes, payloadBytes); + return sequence; + } + + private static long GetGuestWorkDependencyLocked(object work) + { + IReadOnlyList textures = work switch + { + VulkanOffscreenGuestDraw draw => draw.Draw.Textures, + VulkanComputeGuestDispatch compute => compute.Textures, + _ => Array.Empty(), + }; + var required = 0L; + foreach (var texture in textures) + { + if (!texture.IsStorage || + texture.Address == 0 || + texture.RgbaPixels.Length != 0) + { + continue; + } + + var format = GetGuestTextureFormat(texture.Format, texture.NumberType); + if (_pendingGuestImageUploads.TryGetValue( + (texture.Address, format), + out var pendingUpload)) + { + required = Math.Max(required, pendingUpload.OwnerSequence); + } + } + + return required; + } + + private static void RecordGuestImageWritersLocked(object work, long sequence) + { + static IEnumerable StorageAddresses( + IReadOnlyList textures) => + textures + .Where(static texture => texture.IsStorage && texture.Address != 0) + .Select(static texture => texture.Address); + + IEnumerable addresses = work switch + { + VulkanOffscreenGuestDraw draw => + (draw.PublishTarget + ? draw.Targets + .Where(static target => target.Address != 0) + .Select(static target => target.Address) + : Enumerable.Empty()) + .Concat(StorageAddresses(draw.Draw.Textures)), + VulkanComputeGuestDispatch compute => StorageAddresses(compute.Textures), + VulkanGuestImageWrite imageWrite when imageWrite.Address != 0 => + new[] { imageWrite.Address }, + _ => Array.Empty(), + }; + foreach (var address in addresses.Distinct()) + { + _guestImageWorkSequences[address] = sequence; + } + } + + private static bool TryTakeGuestWork(out PendingGuestWork work) + { + lock (_gate) + { + var queuesToProbe = _pendingGuestQueueSchedule.Count; + while (_pendingGuestQueueSchedule.Count > 0 && queuesToProbe > 0) + { + if (_pendingGuestQueueCursor >= _pendingGuestQueueSchedule.Count) + { + _pendingGuestQueueCursor = 0; + } + + var queueName = _pendingGuestQueueSchedule[_pendingGuestQueueCursor]; + if (!_pendingGuestWorkByQueue.TryGetValue(queueName, out var queue) || + queue.First is not { } first) + { + _pendingGuestWorkByQueue.Remove(queueName); + _pendingGuestQueueSchedule.RemoveAt(_pendingGuestQueueCursor); + queuesToProbe = Math.Min( + queuesToProbe, + _pendingGuestQueueSchedule.Count); + continue; + } + + work = first.Value; + if (!IsGuestWorkCompletedLocked(work.RequiredSequence)) + { + _pendingGuestQueueCursor = + (_pendingGuestQueueCursor + 1) % _pendingGuestQueueSchedule.Count; + queuesToProbe--; + continue; + } + + queue.RemoveFirst(); + _pendingGuestWorkCount--; + if (queue.Count == 0) + { + _pendingGuestWorkByQueue.Remove(queueName); + _pendingGuestQueueSchedule.RemoveAt(_pendingGuestQueueCursor); + } + else + { + _pendingGuestQueueCursor = + (_pendingGuestQueueCursor + 1) % _pendingGuestQueueSchedule.Count; + } + + return true; + } + + work = default; + return false; + } + } + + private static void CompleteGuestWork(in PendingGuestWork pending) + { + SharpEmu.HLE.GuestImageWriteTracker.FlushPendingDiagnostics(); + lock (_gate) + { + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_complete_enter " + + $"sequence={pending.Sequence} work={pending.Work.GetType().Name} " + + $"contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + } + _pendingGuestWorkBytes = pending.PayloadBytes >= _pendingGuestWorkBytes + ? 0 + : _pendingGuestWorkBytes - pending.PayloadBytes; + ReleasePendingGuestImageUploadsLocked(pending.Work); + if (pending.Sequence == _completedGuestWorkSequence + 1) + { + _completedGuestWorkSequence = pending.Sequence; + while (_completedGuestWorkOutOfOrder.Remove( + _completedGuestWorkSequence + 1)) + { + _completedGuestWorkSequence++; + } + } + else if (pending.Sequence > _completedGuestWorkSequence) + { + // Debug.Assert calls are compiled out of Release builds, so + // never put the state mutation inside its argument. Doing so + // discarded every out-of-order completion in normal runs and + // left the first immediate follow-up as a permanent sequence + // hole, blocking all later CPU-visible GPU waits. + var added = _completedGuestWorkOutOfOrder.Add(pending.Sequence); + System.Diagnostics.Debug.Assert( + added, + "A guest work sequence must complete exactly once."); + } + System.Threading.Monitor.PulseAll(_gate); + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_complete_exit " + + $"sequence={pending.Sequence} contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + } + } + } + + private static ulong GetGuestWorkPayloadBytes(object work) => work switch + { + VulkanComputeGuestDispatch compute => SaturatingAdd( + GetTexturePayloadBytes(compute.Textures), + GetGlobalBufferPayloadBytes(compute.GlobalMemoryBuffers)), + VulkanOffscreenGuestDraw offscreen => GetDrawPayloadBytes(offscreen.Draw), + VulkanGuestImageWrite { Pixels: { } pixels } => (ulong)pixels.LongLength, + _ => 0, + }; + + private static string GetGuestWorkPayloadBreakdown(object work) + { + static ulong SumTextures(IReadOnlyList textures) => + GetTexturePayloadBytes(textures) / (1024 * 1024); + static ulong SumGlobals(IReadOnlyList buffers) => + GetGlobalBufferPayloadBytes(buffers) / (1024 * 1024); + + return work switch + { + VulkanOffscreenGuestDraw offscreen => + $" textures_mb={SumTextures(offscreen.Draw.Textures)}" + + $" globals_mb={SumGlobals(offscreen.Draw.GlobalMemoryBuffers)}" + + $" vertex_mb={offscreen.Draw.VertexBuffers.Aggregate(0UL, static (sum, buffer) => SaturatingAdd(sum, (ulong)buffer.Data.LongLength)) / (1024 * 1024)}" + + $" index_mb={(ulong)(offscreen.Draw.IndexBuffer?.Data.LongLength ?? 0) / (1024 * 1024)}" + + $" vertex_lengths=[{string.Join(',', offscreen.Draw.VertexBuffers.Select(static buffer => $"{buffer.Length}/{buffer.Data.LongLength}:s{buffer.Stride}:o{buffer.OffsetBytes}"))}]" + + $" global_lengths=[{string.Join(',', offscreen.Draw.GlobalMemoryBuffers.Select(static buffer => buffer.Length))}]", + VulkanComputeGuestDispatch compute => + $" textures_mb={SumTextures(compute.Textures)}" + + $" globals_mb={SumGlobals(compute.GlobalMemoryBuffers)}" + + $" global_lengths=[{string.Join(',', compute.GlobalMemoryBuffers.Select(static buffer => buffer.Length))}]", + _ => string.Empty, + }; + } + + private static ulong GetDrawPayloadBytes(VulkanTranslatedGuestDraw draw) + { + var bytes = GetTexturePayloadBytes(draw.Textures); + bytes = SaturatingAdd(bytes, GetGlobalBufferPayloadBytes(draw.GlobalMemoryBuffers)); + var uniqueVertexData = new HashSet( + System.Collections.Generic.ReferenceEqualityComparer.Instance); + foreach (var vertex in draw.VertexBuffers) + { + if (uniqueVertexData.Add(vertex.Data)) + { + bytes = SaturatingAdd(bytes, (ulong)vertex.Data.LongLength); + } + } + + if (draw.IndexBuffer is { } index) + { + bytes = SaturatingAdd(bytes, (ulong)index.Data.LongLength); + } + + return bytes; + } + + private static ulong GetTexturePayloadBytes( + IReadOnlyList textures) + { + var bytes = 0UL; + foreach (var texture in textures) + { + bytes = SaturatingAdd(bytes, (ulong)texture.RgbaPixels.LongLength); + } + + return bytes; + } + + private static ulong GetGlobalBufferPayloadBytes( + IReadOnlyList buffers) + { + var bytes = 0UL; + foreach (var buffer in buffers) + { + bytes = SaturatingAdd(bytes, (ulong)buffer.Data.LongLength); + } + + return bytes; + } + + private static ulong SaturatingAdd(ulong left, ulong right) => + ulong.MaxValue - left < right ? ulong.MaxValue : left + right; + + private static void ReleasePendingGuestImageUploadsLocked(object work) + { + if (work is not VulkanComputeGuestDispatch compute) { return; } - _pendingGuestWork.Enqueue(work); - _enqueuedGuestWorkSequence++; - System.Threading.Monitor.PulseAll(_gate); - } - - private static bool TryTakeGuestWork(out object work) - { - lock (_gate) + foreach (var key in GetStorageImageUploadKeys(compute.Textures)) { - return _pendingGuestWork.TryDequeue(out work!); + if (!_pendingGuestImageUploads.TryGetValue(key, out var pendingUpload)) + { + continue; + } + + if (pendingUpload.Count <= 1) + { + _pendingGuestImageUploads.Remove(key); + } + else + { + _pendingGuestImageUploads[key] = pendingUpload with + { + Count = pendingUpload.Count - 1, + }; + } } } - private static void CompleteGuestWork() + private static HashSet<(ulong Address, uint Format)> GetStorageImageUploadKeys( + IReadOnlyList textures) { - lock (_gate) + var keys = new HashSet<(ulong Address, uint Format)>(); + foreach (var texture in textures) { - _completedGuestWorkSequence++; - System.Threading.Monitor.PulseAll(_gate); + if (!texture.IsStorage || texture.Address == 0) + { + continue; + } + + var format = GetGuestTextureFormat(texture.Format, texture.NumberType); + if (format != 0) + { + keys.Add((texture.Address, format)); + } } + + return keys; } private readonly record struct Presentation( @@ -1164,7 +2186,8 @@ internal static unsafe class VulkanVideoPresenter VulkanTranslatedGuestDraw? TranslatedDraw, long RequiredGuestWorkSequence, bool IsSplash, - ulong GuestImageAddress = 0); + ulong GuestImageAddress = 0, + long GuestImageVersion = 0); private sealed class Presenter : IDisposable { @@ -1176,13 +2199,6 @@ internal static unsafe class VulkanVideoPresenter private readonly IWindow _window; private const int MaxInFlightGuestSubmissions = 8; - private const double PerformanceHudSampleSeconds = 0.5; - private const uint ThreadQueryLimitedInformation = 0x0800; - private static readonly bool _performanceHudEnabled = - !string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_PERF_HUD"), - "0", - StringComparison.Ordinal); private Vk _vk = null!; private KhrSurface _surfaceApi = null!; private KhrSwapchain _swapchainApi = null!; @@ -1194,11 +2210,21 @@ internal static unsafe class VulkanVideoPresenter private DebugUtilsMessengerEXT _debugMessenger; private ExtDebugUtils? _debugUtils; private PhysicalDevice _physicalDevice; + private uint _maxComputeWorkGroupCountX; + private uint _maxComputeWorkGroupCountY; + private uint _maxComputeWorkGroupCountZ; + private uint _maxComputeWorkGroupSizeX; + private uint _maxComputeWorkGroupSizeY; + private uint _maxComputeWorkGroupSizeZ; + private uint _maxComputeWorkGroupInvocations; + private ulong _minStorageBufferOffsetAlignment = 1; private bool _supportsIndependentBlend; private uint _maxColorAttachments; private Device _device; private PipelineCache _pipelineCache; private string? _pipelineCachePath; + private bool _pipelineCacheDirty; + private long _lastPipelineCacheSaveTick; private Queue _queue; private uint _queueFamilyIndex; private SwapchainKHR _swapchain; @@ -1214,26 +2240,54 @@ internal static unsafe class VulkanVideoPresenter private CommandPool _commandPool; private CommandBuffer _commandBuffer; private CommandBuffer _presentationCommandBuffer; - private VkSemaphore _imageAvailable; - private VkSemaphore _renderFinished; - private Fence _presentationFence; - private bool _presentationInFlight; - private TranslatedDrawResources? _pendingPresentationResources; + // Presentation runs with multiple frames in flight: each frame slot + // owns a command buffer, an acquire semaphore and a fence, so the CPU + // can record frame N+1 while the GPU still executes frame N. The + // per-present vkQueueWaitIdle this replaces made CPU and GPU costs + // strictly additive. Render-finished semaphores are per swapchain + // image because the presentation engine may still wait on them after + // the frame fence has signaled. + private const int MaxFramesInFlight = 2; + private CommandBuffer[] _frameCommandBuffers = []; + private VkSemaphore[] _frameImageAvailable = []; + private VkSemaphore[] _renderFinishedPerImage = []; + private Fence[] _frameFences = []; + private bool[] _frameFencePending = []; + private ulong[] _frameTimelines = []; + private TranslatedDrawResources?[] _frameTranslatedResources = []; + private GuestImageResource?[] _frameGuestImageVersions = []; + private int _currentFrameSlot; + // Monotonic submission/completion counters across every queue submit + // (guest batches, compute chunks and presents). Fences on a single + // queue signal in submission order, so "timeline <= completed" means + // the GPU is done with everything submitted up to that point; this + // lets evicted resources be destroyed without a queue drain. + private ulong _submitTimeline; + private ulong _completedTimeline; + private readonly Queue<(TextureResource Texture, ulong RetireTimeline)> + _deferredTextureDestroys = new(); + private readonly Queue<(TranslatedDrawResources Resources, ulong RetireTimeline)> + _deferredResourceDestroys = new(); + private readonly Queue<(GuestImageResource Image, ulong RetireTimeline)> + _deferredGuestImageVersionDestroys = new(); + private readonly Stack _recycledGuestFences = new(); + private readonly Stack _recycledGuestCommandBuffers = new(); + private readonly List<(VkBuffer Buffer, DeviceMemory Memory)> _batchRetireBuffers = new(); + private const int MaxRecycledGuestFences = 32; + private const int MaxRecycledGuestCommandBuffers = 32; private VkBuffer _stagingBuffer; private DeviceMemory _stagingMemory; private ulong _stagingSize; + // Perf overlay: CPU-rasterized panel copied through per-slot staging + // buffers into one image, then blitted onto the swapchain. + private Image _overlayImage; + private DeviceMemory _overlayImageMemory; + private bool _overlayImageInitialized; + private VkBuffer[] _overlayStagingBuffers = []; + private DeviceMemory[] _overlayStagingMemory = []; + private nint[] _overlayStagingMapped = []; private long _presentedSequence; - private long _performanceHudLastTimestamp; - private TimeSpan _performanceHudLastProcessCpu; - private long _performanceHudPresentedFrames; - private long _performanceHudLastPresentedFrames; - private long _performanceHudLastReadCount; - private long _performanceHudLastReadBytes; - private long _performanceHudLastReadHits; - private long _performanceHudLastReadPvmBytes; - private long _performanceHudLastReadLibcBytes; - private readonly Dictionary _performanceHudThreadCpu = []; - private readonly Dictionary _performanceHudThreadNames = []; + private long _presentNotTakenLoggedSequence = long.MinValue; private bool _vulkanReady; private bool _firstFramePresented; private bool _firstGuestDrawPresented; @@ -1241,25 +2295,62 @@ internal static unsafe class VulkanVideoPresenter private bool _swapchainRecreateDeferred; private bool _tracedPresentedSwapchain; private bool _swapchainReadbackPending; + private long _presentedSwapchainCount; private static int _guestImageDumpSequence; private readonly System.Collections.Concurrent.ConcurrentQueue _pendingAliasImageDumps = new(); private bool _deviceLost; private bool _deviceLostLogged; private int _directPresentationCount; + private readonly Dictionary _presentedGuestImageTraceCounts = new(); private readonly Dictionary _guestImages = new(); + private readonly record struct GuestImageVariantKey( + ulong Address, + uint Width, + uint Height, + uint MipLevels, + uint GuestFormat, + Format Format); + + // A single guest allocation may be rebound through several RT descriptors. + // Keep the inactive Vulkan images instead of destroying their contents each + // time the guest switches size or format at the same address. + private readonly Dictionary + _guestImageVariants = new(); + private readonly Dictionary _guestImageVersions = new(); + private readonly HashSet _capturedGuestFlipVersions = []; + private readonly record struct GuestDepthKey( + ulong Address, + ulong ReadAddress, + uint Width, + uint Height, + uint GuestFormat, + uint SwizzleMode); + + private readonly Dictionary _guestDepthImages = new(); + private readonly Dictionary _depthOnlyColorAddresses = new(); + private ulong _nextDepthOnlyColorAddress = 0xFFFF_FF00_0000_0000UL; private readonly HashSet<(ulong Address, uint Width, uint Height, Format Format)> _tracedTextureCacheHits = new(); - private static readonly bool _dumpTextures = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_DUMP_TEXTURES"), - "1", - StringComparison.Ordinal); + private readonly HashSet<(ulong Address, uint Width, uint Height, uint DstSelect)> _tracedDepthTextureAliases = new(); + private readonly HashSet<(ulong Address, uint Width, uint Height)> _tracedDepthExtentFallbacks = new(); private readonly HashSet<(ulong Address, uint Width, uint Height, Format Format)> _tracedTextureUploads = new(); private readonly HashSet<(ulong Address, uint Width, uint Height, uint Format)> _dumpedTextures = new(); + private readonly HashSet<(ulong Address, uint Width, uint Height, uint Format)> _tracedTextureUploadContents = new(); private readonly HashSet<(ulong Address, int Size)> _tracedGlobalBuffers = new(); + private readonly HashSet<(ulong Address, ulong Size)> _tracedGlobalWritebacks = new(); + private readonly HashSet<(ulong Shader, uint X, uint Y, uint Z, string Reason)> + _rejectedComputeDispatches = new(); + private int _tracedSmallGlobalWritebackEvents; + private int _tracedLargeGlobalWritebackEvents; private readonly HashSet _tracedGuestImageContents = new(); private readonly Dictionary _tracedGuestWriteCounts = new(); + private readonly Dictionary _pixelSpirvWriteCounts = new(); private int _tracedVertexBufferCount; - private readonly Dictionary _computePipelines = - new(ReferenceEqualityComparer.Instance); + private bool _tracedTitleDraw; + // Compute translation can produce an equivalent new byte array on a + // later submit. Reference identity turns that into an expensive new + // MoltenVK pipeline compilation every frame, so key the cache by the + // program content and descriptor-layout shape instead. + private readonly Dictionary _computePipelines = new(); private readonly Dictionary _graphicsPipelines = new(); private readonly Dictionary _samplers = new(); private readonly Dictionary _shaderDigests = @@ -1269,16 +2360,26 @@ internal static unsafe class VulkanVideoPresenter private readonly Dictionary> _hostBufferPool = new(); private readonly Dictionary _hostBufferAllocations = new(); + private readonly List _guestBufferAllocations = []; private readonly Queue _pendingGuestSubmissions = new(); + private readonly Dictionary _lastSubmittedTimelineByGuestQueue = + new(StringComparer.Ordinal); + private readonly Stack _recycledDescriptorPools = new(); + private VulkanGuestQueueIdentity _activeGuestQueue = + VulkanGuestQueueIdentity.Default; + private long _activeGuestWorkSequence; private readonly record struct GraphicsPipelineKey( string VertexShader, string FragmentShader, string RenderTargetLayout, + bool HasDepthAttachment, PrimitiveTopology Topology, string BlendLayout, string ResourceLayout, - string VertexLayout); + string VertexLayout, + GuestRasterState Raster, + GuestDepthState Depth); private readonly record struct HostBufferPoolKey( BufferUsageFlags Usage, @@ -1288,6 +2389,10 @@ internal static unsafe class VulkanVideoPresenter ShaderStageFlags Stages, string Resources); + private readonly record struct ComputePipelineKey( + string ShaderDigest, + string Resources); + private sealed record DescriptorLayoutBundle( DescriptorSetLayout DescriptorSetLayout, PipelineLayout PipelineLayout); @@ -1295,7 +2400,25 @@ internal static unsafe class VulkanVideoPresenter private sealed record HostBufferAllocation( VkBuffer Buffer, DeviceMemory Memory, - HostBufferPoolKey Key); + HostBufferPoolKey Key, + nint Mapped); + + private readonly record struct DirtyGuestBufferRange(ulong Offset, ulong Length); + + private sealed class GuestBufferAllocation + { + public string QueueName = VulkanGuestQueueIdentity.Default.Name; + public ulong BaseAddress; + public ulong Size; + public VkBuffer Buffer; + public DeviceMemory Memory; + public nint Mapped; + public byte[] Shadow = []; + public ulong LastUseTimeline; + public List DirtyRanges { get; } = []; + } + + private const string SharedReadOnlyGuestBufferQueue = "shared.readonly"; private sealed class TranslatedDrawResources { @@ -1317,8 +2440,20 @@ internal static unsafe class VulkanVideoPresenter public uint InstanceCount = 1; public PrimitiveTopology Topology = PrimitiveTopology.TriangleList; public GuestBlendState[] Blends = [GuestBlendState.Default]; + // Vulkan format of this draw's color target. Needed to suppress + // blending on formats Metal cannot blend (integer / 32-bit float), + // which otherwise makes vkCreateGraphicsPipelines fail and can + // trip a Metal validation assertion. + public Format[] TargetFormats = [Format.Undefined]; public GuestRect? Scissor; public GuestViewport? Viewport; + public GuestRasterState Raster = GuestRasterState.Default; + public GuestDepthState Depth = GuestDepthState.Default; + public bool HasDepthAttachment; + // Layout keys are needed twice per draw (pipeline lookup and + // descriptor-layout lookup); cache the built strings. + public string? ResourceLayoutKey; + public string? VertexLayoutKey; public RenderPass TransientRenderPass; public Framebuffer TransientFramebuffer; } @@ -1338,24 +2473,46 @@ internal static unsafe class VulkanVideoPresenter public bool NeedsUpload; public bool OwnsStorage; public bool IsStorage; + public bool Cached; + public ulong CpuContentFingerprint; + public bool UpdatesCpuContent; public GuestSampler SamplerState; public Sampler Sampler; public GuestImageResource? GuestImage; - public ulong CpuContentFingerprint; - public bool UpdatesCpuContent; + public GuestDepthResource? GuestDepth; + // A sampled render-target alias cannot remain bound to the same + // image while that image is a color attachment. The per-draw + // snapshot uses this source to copy the target's pre-draw contents + // before the render pass begins. The snapshot itself is owned by + // this TextureResource and retires with the draw fence. + public GuestImageResource? FeedbackSource; + public GuestDepthResource? DepthFeedbackSource; } private sealed class GlobalBufferResource { + public ulong BaseAddress; + public bool Writable; + public bool WriteBackToGuest; public VkBuffer Buffer; public DeviceMemory Memory; + public nint Mapped; + // DescriptorOffset/Size include the shader-visible byte bias. + public ulong Offset; public ulong Size; + // GuestOffset/Size identify only the original guest resource and + // are used for dirty writeback; descriptor padding must not be + // published over unrelated guest bytes. + public ulong GuestOffset; + public ulong GuestSize; + public GuestBufferAllocation? Allocation; } private sealed class VertexBufferResource { public VkBuffer Buffer; public DeviceMemory Memory; + public bool OwnsBuffer; public ulong Size; public uint Location; public uint ComponentCount; @@ -1365,12 +2522,47 @@ internal static unsafe class VulkanVideoPresenter public uint OffsetBytes; } + private const Format DepthFormat = Format.D32Sfloat; + + private sealed class GuestDepthResource + { + public GuestDepthKey Key; + public ulong Address; + public ulong ReadAddress; + public ulong WriteAddress; + public uint Width; + public uint Height; + public uint GuestFormat; + public uint SwizzleMode; + public Image Image; + public DeviceMemory Memory; + public ImageView View; + public Dictionary SampleViews { get; } = new(); + public bool Initialized; + public ImageLayout Layout = ImageLayout.Undefined; + public float GuestClearDepth = 1f; + public float ClearDepth = 1f; + public string InitializationSource = "none"; + } + + private sealed class DepthFramebufferResource + { + public required GuestDepthResource Depth; + public RenderPass LoadRenderPass; + public RenderPass ColorClearRenderPass; + public RenderPass DepthClearRenderPass; + public RenderPass BothClearRenderPass; + public Framebuffer Framebuffer; + } + private sealed class GuestImageResource { public ulong Address; + public long FlipVersion; public uint Width; public uint Height; public uint MipLevels; + public uint GuestFormat; public Format Format; public Image Image; public DeviceMemory Memory; @@ -1378,7 +2570,9 @@ internal static unsafe class VulkanVideoPresenter public ImageView[] MipViews = []; public Dictionary<(Format Format, uint MipLevel, uint LevelCount, uint DstSelect), ImageView> FormatViews { get; } = new(); public RenderPass RenderPass; + public RenderPass InitialRenderPass; public Framebuffer Framebuffer; + public Dictionary DepthFramebuffers { get; } = new(); public bool Initialized; public bool InitialUploadPending; public bool IsCpuBacked; @@ -1388,9 +2582,13 @@ internal static unsafe class VulkanVideoPresenter private sealed record PendingGuestSubmission( Fence Fence, CommandBuffer CommandBuffer, - TranslatedDrawResources Resources, + IReadOnlyList Resources, IReadOnlyList TraceImages, - string DebugName); + IReadOnlyList<(VkBuffer Buffer, DeviceMemory Memory)> RetireBuffers, + ulong Timeline, + string DebugName, + VulkanGuestQueueIdentity Queue, + long WorkSequence); public Presenter(uint width, uint height) { @@ -1398,21 +2596,21 @@ internal static unsafe class VulkanVideoPresenter options.Size = new Vector2D((int)DefaultWindowWidth, (int)DefaultWindowHeight); options.Title = VideoOutExports.GetWindowTitle(); options.WindowBorder = WindowBorder.Fixed; - // FIFO already provides the presentation clock. Throttling Silk's render loop - // as well can miss alternating vblanks and collapse delivery to 30 FPS or less. - options.VSync = false; - options.FramesPerSecond = 0; - options.UpdatesPerSecond = 0; + options.VSync = true; + options.FramesPerSecond = 60; + options.UpdatesPerSecond = 60; _window = Window.Create(options); _window.Load += Initialize; _window.Render += Render; - _window.Closing += OnWindowClosing; - } - - private void OnWindowClosing() - { - VideoOutExports.NotifyPresentationWindowClosed(); - DisposeVulkan(); + _window.Closing += () => + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan VideoOut window closing; " + + $"requested={Volatile.Read(ref _presenterCloseRequested)} " + + $"deviceLost={_deviceLost}"); + VideoOutExports.NotifyPresentationWindowClosed(); + DisposeVulkan(); + }; } public void Run() => _window.Run(); @@ -1434,12 +2632,6 @@ internal static unsafe class VulkanVideoPresenter private void Initialize() { - if (PngSplashLoader.TryLoadIcon(out var iconPixels, out var iconWidth, out var iconHeight)) - { - var icon = new RawImage((int)iconWidth, (int)iconHeight, iconPixels); - _window.SetWindowIcon(ref icon); - } - LogGlfwPlatformInUse(); if (!OperatingSystem.IsWindows()) { @@ -1870,13 +3062,9 @@ internal static unsafe class VulkanVideoPresenter Check(_vk.EnumeratePhysicalDevices(_instance, &deviceCount, devicePointer), "vkEnumeratePhysicalDevices"); } - // Hybrid laptops enumerate the iGPU first, and AMD's integrated driver - // segfaults compiling some translated shaders, so rank rather than take - // the first hit. SHARPEMU_VK_DEVICE= pins an adapter by name. var deviceOverride = Environment.GetEnvironmentVariable("SHARPEMU_VK_DEVICE"); var bestScore = int.MinValue; var found = false; - foreach (var device in devices) { uint queueCount = 0; @@ -1901,7 +3089,6 @@ internal static unsafe class VulkanVideoPresenter var score = ScorePhysicalDevice(properties, name, deviceOverride); Console.Error.WriteLine( $"[LOADER][INFO] Vulkan candidate: {name} ({properties.DeviceType}) score={score}"); - if (score > bestScore) { bestScore = score; @@ -1919,15 +3106,12 @@ internal static unsafe class VulkanVideoPresenter throw new InvalidOperationException("No Vulkan graphics/present queue was found."); } + LoadComputeDeviceLimits(); _vk.GetPhysicalDeviceProperties(_physicalDevice, out var selected); _maxColorAttachments = selected.Limits.MaxColorAttachments; var selectedName = SilkMarshal.PtrToString((nint)selected.DeviceName) ?? "unknown"; - var apiMajor = (selected.ApiVersion >> 22) & 0x7F; - var apiMinor = (selected.ApiVersion >> 12) & 0x3FF; - var apiPatch = selected.ApiVersion & 0xFFF; Console.Error.WriteLine( - $"[LOADER][INFO] Vulkan device: {selectedName} " + - $"(type={selected.DeviceType}, api={apiMajor}.{apiMinor}.{apiPatch})"); + $"[LOADER][INFO] Vulkan device: {selectedName} ({selected.DeviceType})"); VideoOutExports.SetSelectedGpuName(selectedName); _window.Title = VideoOutExports.GetWindowTitle(); } @@ -1947,7 +3131,6 @@ internal static unsafe class VulkanVideoPresenter PhysicalDeviceType.DiscreteGpu => 300, PhysicalDeviceType.VirtualGpu => 100, PhysicalDeviceType.Cpu => 50, - // Last resort: only picked when nothing else can present. PhysicalDeviceType.IntegratedGpu => -100, _ => 10, }; @@ -1960,6 +3143,59 @@ internal static unsafe class VulkanVideoPresenter return score; } + private void LoadComputeDeviceLimits() + { + _vk.GetPhysicalDeviceProperties(_physicalDevice, out var properties); + var subgroupSizeControl = new PhysicalDeviceSubgroupSizeControlProperties + { + SType = StructureType.PhysicalDeviceSubgroupSizeControlProperties, + }; + var subgroup = new PhysicalDeviceSubgroupProperties + { + SType = StructureType.PhysicalDeviceSubgroupProperties, + PNext = &subgroupSizeControl, + }; + var properties2 = new PhysicalDeviceProperties2 + { + SType = StructureType.PhysicalDeviceProperties2, + PNext = &subgroup, + }; + _vk.GetPhysicalDeviceProperties2(_physicalDevice, &properties2); + _maxComputeWorkGroupCountX = properties.Limits.MaxComputeWorkGroupCount[0]; + _maxComputeWorkGroupCountY = properties.Limits.MaxComputeWorkGroupCount[1]; + _maxComputeWorkGroupCountZ = properties.Limits.MaxComputeWorkGroupCount[2]; + _maxComputeWorkGroupSizeX = properties.Limits.MaxComputeWorkGroupSize[0]; + _maxComputeWorkGroupSizeY = properties.Limits.MaxComputeWorkGroupSize[1]; + _maxComputeWorkGroupSizeZ = properties.Limits.MaxComputeWorkGroupSize[2]; + _maxComputeWorkGroupInvocations = properties.Limits.MaxComputeWorkGroupInvocations; + _minStorageBufferOffsetAlignment = Math.Max( + properties.Limits.MinStorageBufferOffsetAlignment, + 1UL); + if (GuestStorageBufferOffsetAlignment % + _minStorageBufferOffsetAlignment != 0) + { + throw new InvalidOperationException( + $"Vulkan storage-buffer alignment " + + $"{_minStorageBufferOffsetAlignment} is not compatible with " + + $"the portable alias alignment " + + $"{GuestStorageBufferOffsetAlignment}"); + } + Console.Error.WriteLine( + $"[LOADER][INFO] Vulkan compute limits groups=" + + $"{_maxComputeWorkGroupCountX}x{_maxComputeWorkGroupCountY}x{_maxComputeWorkGroupCountZ} " + + $"local={_maxComputeWorkGroupSizeX}x{_maxComputeWorkGroupSizeY}x" + + $"{_maxComputeWorkGroupSizeZ} invocations={_maxComputeWorkGroupInvocations} " + + $"storage_alignment={_minStorageBufferOffsetAlignment}"); + Console.Error.WriteLine( + $"[LOADER][INFO] Vulkan subgroup default={subgroup.SubgroupSize} " + + $"stages={subgroup.SupportedStages} ops={subgroup.SupportedOperations} " + + $"size_control={subgroupSizeControl.MinSubgroupSize}-" + + $"{subgroupSizeControl.MaxSubgroupSize} " + + $"required_stages={subgroupSizeControl.RequiredSubgroupSizeStages} " + + $"max_compute_subgroups=" + + $"{subgroupSizeControl.MaxComputeWorkgroupSubgroups}"); + } + private void CreateDevice() { var priority = 1.0f; @@ -2126,26 +3362,69 @@ internal static unsafe class VulkanVideoPresenter private void CreatePipelineCache() { - _vk.GetPhysicalDeviceProperties(_physicalDevice, out var properties); - var directory = Path.Combine(AppContext.BaseDirectory, "pipeline-cache"); - _pipelineCachePath = Path.Combine( - directory, - $"vulkan-{properties.VendorID:X4}-{properties.DeviceID:X4}-{properties.DriverVersion:X8}.bin"); - + var cacheMode = Environment.GetEnvironmentVariable("SHARPEMU_VK_PIPELINE_CACHE"); + // Vulkan cache blobs carry the implementation's compatibility + // header and are rejected/rebuilt below when the device or driver + // changes. MoltenVK compilation of a large translated shader can + // take ten seconds, so discarding a valid cache at every launch is + // much more harmful than using Vulkan's normal persistence path. + // Keep an explicit opt-out for diagnostics and read-only systems. + var persistentCacheEnabled = + !string.Equals(cacheMode, "0", StringComparison.Ordinal); + _pipelineCachePath = persistentCacheEnabled ? GetPipelineCachePath() : null; byte[] initialData = []; try { - if (File.Exists(_pipelineCachePath)) + if (_pipelineCachePath is not null && File.Exists(_pipelineCachePath)) { initialData = File.ReadAllBytes(_pipelineCachePath); } } - catch (IOException exception) + catch (Exception exception) { Console.Error.WriteLine( - $"[LOADER][WARN] Vulkan pipeline cache could not be read: {exception.Message}"); + $"[LOADER][WARN] Vulkan pipeline cache read failed: {exception.Message}"); } + var result = TryCreatePipelineCache(initialData, out _pipelineCache); + if (result != Result.Success && initialData.Length != 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan pipeline cache rejected ({result}); rebuilding it."); + result = TryCreatePipelineCache([], out _pipelineCache); + } + + if (result != Result.Success) + { + _pipelineCache = default; + _pipelineCachePath = null; + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan pipeline cache unavailable: {result}"); + return; + } + + SetDebugName( + ObjectType.PipelineCache, + _pipelineCache.Handle, + _pipelineCachePath is null + ? "SharpEmu in-memory pipeline cache" + : "SharpEmu persistent pipeline cache"); + _lastPipelineCacheSaveTick = Environment.TickCount64; + if (_pipelineCachePath is null) + { + Console.Error.WriteLine( + "[LOADER][INFO] Vulkan pipeline cache ready: memory-only " + + "(persistence disabled with SHARPEMU_VK_PIPELINE_CACHE=0)."); + } + else + { + Console.Error.WriteLine( + $"[LOADER][INFO] Vulkan pipeline cache ready: path={_pipelineCachePath} initial={initialData.Length} bytes"); + } + } + + private Result TryCreatePipelineCache(byte[] initialData, out PipelineCache pipelineCache) + { fixed (byte* initialDataPointer = initialData) { var createInfo = new PipelineCacheCreateInfo @@ -2154,63 +3433,98 @@ internal static unsafe class VulkanVideoPresenter InitialDataSize = (nuint)initialData.Length, PInitialData = initialData.Length == 0 ? null : initialDataPointer, }; - var result = _vk.CreatePipelineCache( + return _vk.CreatePipelineCache( _device, &createInfo, null, - out _pipelineCache); - if (result != Result.Success && initialData.Length != 0) - { - createInfo.InitialDataSize = 0; - createInfo.PInitialData = null; - Check( - _vk.CreatePipelineCache( - _device, - &createInfo, - null, - out _pipelineCache), - "vkCreatePipelineCache(empty)"); - initialData = []; - } - else - { - Check(result, "vkCreatePipelineCache"); - } + out pipelineCache); } - - Console.Error.WriteLine( - $"[LOADER][INFO] Vulkan pipeline cache: " + - $"{(initialData.Length == 0 ? "new" : $"loaded {initialData.Length} bytes")}"); } - private void SavePipelineCache() + private static string GetPipelineCachePath() + { + var configured = Environment.GetEnvironmentVariable("SHARPEMU_VK_PIPELINE_CACHE_PATH"); + if (!string.IsNullOrWhiteSpace(configured)) + { + return Path.GetFullPath( + Environment.ExpandEnvironmentVariables(configured)); + } + + var root = OperatingSystem.IsMacOS() + ? Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "Library", + "Caches") + : Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); + return Path.Combine(root, "SharpEmu", "vulkan-pipeline-cache.bin"); + } + + private void MarkPipelineCacheDirty() + { + if (_pipelineCache.Handle == 0) + { + return; + } + + _pipelineCacheDirty = true; + // Exporting MoltenVK's cache can itself serialize the compiler. + // Gameplay may discover dozens of expensive pipelines in one + // frame, so saving after every slow creation compounds a warm-up + // hitch into a multi-minute stall. Coalesce all creations into one + // periodic snapshot; shutdown still forces a final save. + if (Environment.TickCount64 - _lastPipelineCacheSaveTick >= 30_000) + { + SavePipelineCache(force: false); + } + } + + private void SavePipelineCache(bool force) { if (_pipelineCache.Handle == 0 || string.IsNullOrWhiteSpace(_pipelineCachePath)) { return; } + if (!force && !_pipelineCacheDirty) + { + return; + } + try { nuint size = 0; - Check( - _vk.GetPipelineCacheData(_device, _pipelineCache, &size, null), - "vkGetPipelineCacheData(size)"); - if (size == 0 || size > int.MaxValue) + var result = _vk.GetPipelineCacheData( + _device, + _pipelineCache, + &size, + null); + if (result != Result.Success || size == 0 || size > 256u * 1024u * 1024u) { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan pipeline cache query failed: result={result} size={size}"); return; } - var data = new byte[(int)size]; + var data = new byte[checked((int)size)]; fixed (byte* dataPointer = data) { - Check( - _vk.GetPipelineCacheData( - _device, - _pipelineCache, - &size, - dataPointer), - "vkGetPipelineCacheData"); + result = _vk.GetPipelineCacheData( + _device, + _pipelineCache, + &size, + dataPointer); + } + + if (result != Result.Success) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan pipeline cache export failed: {result}"); + return; + } + + if (size != (nuint)data.Length) + { + Array.Resize(ref data, checked((int)size)); } var directory = Path.GetDirectoryName(_pipelineCachePath); @@ -2218,13 +3532,19 @@ internal static unsafe class VulkanVideoPresenter { Directory.CreateDirectory(directory); } - File.WriteAllBytes(_pipelineCachePath, data.AsSpan(0, (int)size).ToArray()); + + var temporaryPath = _pipelineCachePath + $".{Environment.ProcessId}.tmp"; + File.WriteAllBytes(temporaryPath, data); + File.Move(temporaryPath, _pipelineCachePath, overwrite: true); + _pipelineCacheDirty = false; + _lastPipelineCacheSaveTick = Environment.TickCount64; + Console.Error.WriteLine( + $"[LOADER][INFO] Vulkan pipeline cache saved: path={_pipelineCachePath} bytes={data.Length}"); } - catch (Exception exception) when ( - exception is IOException or UnauthorizedAccessException or InvalidOperationException) + catch (Exception exception) { Console.Error.WriteLine( - $"[LOADER][WARN] Vulkan pipeline cache could not be saved: {exception.Message}"); + $"[LOADER][WARN] Vulkan pipeline cache save failed: {exception.Message}"); } } @@ -2249,30 +3569,7 @@ internal static unsafe class VulkanVideoPresenter var surfaceFormat = ChooseSurfaceFormat(formats); _swapchainFormat = surfaceFormat.Format; _extent = ChooseExtent(capabilities); - uint presentModeCount = 0; - Check( - _surfaceApi.GetPhysicalDeviceSurfacePresentModes( - _physicalDevice, - _surface, - &presentModeCount, - null), - "vkGetPhysicalDeviceSurfacePresentModesKHR"); - var presentModes = new PresentModeKHR[presentModeCount]; - fixed (PresentModeKHR* presentModePointer = presentModes) - { - Check( - _surfaceApi.GetPhysicalDeviceSurfacePresentModes( - _physicalDevice, - _surface, - &presentModeCount, - presentModePointer), - "vkGetPhysicalDeviceSurfacePresentModesKHR"); - } - - var presentMode = presentModes.Contains(PresentModeKHR.MailboxKhr) - ? PresentModeKHR.MailboxKhr - : PresentModeKHR.FifoKhr; - Console.Error.WriteLine($"[LOADER][INFO] Vulkan present mode: {presentMode}"); + var presentMode = ChoosePresentMode(); var imageCount = capabilities.MinImageCount + 1; if (capabilities.MaxImageCount != 0) { @@ -2317,6 +3614,45 @@ internal static unsafe class VulkanVideoPresenter _imageInitialized = new bool[swapchainImageCount]; } + private PresentModeKHR ChoosePresentMode() + { + // MAILBOX never blocks vkQueuePresentKHR on vblank, so a slow + // frame does not quantize the frame rate down to 30/20 fps the + // way FIFO does; the guest side is already paced by PaceFlip. + // FIFO is the only mode guaranteed by the spec and remains the + // fallback (MoltenVK typically exposes FIFO + IMMEDIATE only). + uint modeCount = 0; + if (_surfaceApi.GetPhysicalDeviceSurfacePresentModes( + _physicalDevice, + _surface, + &modeCount, + null) != Result.Success || + modeCount == 0) + { + return PresentModeKHR.FifoKhr; + } + + var modes = stackalloc PresentModeKHR[(int)modeCount]; + if (_surfaceApi.GetPhysicalDeviceSurfacePresentModes( + _physicalDevice, + _surface, + &modeCount, + modes) != Result.Success) + { + return PresentModeKHR.FifoKhr; + } + + for (var index = 0u; index < modeCount; index++) + { + if (modes[index] == PresentModeKHR.MailboxKhr) + { + return PresentModeKHR.MailboxKhr; + } + } + + return PresentModeKHR.FifoKhr; + } + private void CreateCommandResources() { var poolInfo = new CommandPoolCreateInfo @@ -2332,58 +3668,246 @@ internal static unsafe class VulkanVideoPresenter SType = StructureType.CommandBufferAllocateInfo, CommandPool = _commandPool, Level = CommandBufferLevel.Primary, - CommandBufferCount = 1, + CommandBufferCount = MaxFramesInFlight, }; - Check(_vk.AllocateCommandBuffers(_device, &allocateInfo, out _commandBuffer), "vkAllocateCommandBuffers"); - _presentationCommandBuffer = _commandBuffer; + _frameCommandBuffers = new CommandBuffer[MaxFramesInFlight]; + fixed (CommandBuffer* frameCommandBuffers = _frameCommandBuffers) + { + Check( + _vk.AllocateCommandBuffers(_device, &allocateInfo, frameCommandBuffers), + "vkAllocateCommandBuffers"); + } var semaphoreInfo = new SemaphoreCreateInfo { SType = StructureType.SemaphoreCreateInfo, }; - Check(_vk.CreateSemaphore(_device, &semaphoreInfo, null, out _imageAvailable), "vkCreateSemaphore"); - Check(_vk.CreateSemaphore(_device, &semaphoreInfo, null, out _renderFinished), "vkCreateSemaphore"); - var fenceInfo = new FenceCreateInfo { SType = StructureType.FenceCreateInfo, }; - Check( - _vk.CreateFence(_device, &fenceInfo, null, out _presentationFence), - "vkCreateFence(presentation)"); + _frameImageAvailable = new VkSemaphore[MaxFramesInFlight]; + _frameFences = new Fence[MaxFramesInFlight]; + _frameFencePending = new bool[MaxFramesInFlight]; + _frameTimelines = new ulong[MaxFramesInFlight]; + _frameTranslatedResources = new TranslatedDrawResources?[MaxFramesInFlight]; + _frameGuestImageVersions = new GuestImageResource?[MaxFramesInFlight]; + for (var slot = 0; slot < MaxFramesInFlight; slot++) + { + Check( + _vk.CreateSemaphore(_device, &semaphoreInfo, null, out _frameImageAvailable[slot]), + "vkCreateSemaphore"); + Check( + _vk.CreateFence(_device, &fenceInfo, null, out _frameFences[slot]), + "vkCreateFence(frame)"); + } + + _renderFinishedPerImage = new VkSemaphore[_swapchainImages.Length]; + for (var image = 0; image < _renderFinishedPerImage.Length; image++) + { + Check( + _vk.CreateSemaphore(_device, &semaphoreInfo, null, out _renderFinishedPerImage[image]), + "vkCreateSemaphore"); + } + + _currentFrameSlot = 0; + _commandBuffer = _frameCommandBuffers[0]; + _presentationCommandBuffer = _commandBuffer; CreateStagingBuffer((ulong)_extent.Width * _extent.Height * 4); + CreateOverlayResources(); } - private void CompletePendingPresentation(bool wait) + private void CreateOverlayResources() { - if (!_presentationInFlight) + const ulong overlayBytes = PerfOverlay.PanelWidth * PerfOverlay.PanelHeight * 4; + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + ImageType = ImageType.Type2D, + Format = Format.B8G8R8A8Unorm, + Extent = new Extent3D(PerfOverlay.PanelWidth, PerfOverlay.PanelHeight, 1), + MipLevels = 1, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = ImageUsageFlags.TransferDstBit | ImageUsageFlags.TransferSrcBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check(_vk.CreateImage(_device, &imageInfo, null, out _overlayImage), "vkCreateImage(overlay)"); + _vk.GetImageMemoryRequirements(_device, _overlayImage, out var requirements); + var memoryInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType( + requirements.MemoryTypeBits, + MemoryPropertyFlags.DeviceLocalBit), + }; + Check( + _vk.AllocateMemory(_device, &memoryInfo, null, out _overlayImageMemory), + "vkAllocateMemory(overlay)"); + Check( + _vk.BindImageMemory(_device, _overlayImage, _overlayImageMemory, 0), + "vkBindImageMemory(overlay)"); + _overlayImageInitialized = false; + + _overlayStagingBuffers = new VkBuffer[MaxFramesInFlight]; + _overlayStagingMemory = new DeviceMemory[MaxFramesInFlight]; + _overlayStagingMapped = new nint[MaxFramesInFlight]; + for (var slot = 0; slot < MaxFramesInFlight; slot++) + { + _overlayStagingBuffers[slot] = CreateBuffer( + overlayBytes, + BufferUsageFlags.TransferSrcBit, + MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, + out _overlayStagingMemory[slot]); + void* mapped; + Check( + _vk.MapMemory(_device, _overlayStagingMemory[slot], 0, overlayBytes, 0, &mapped), + "vkMapMemory(overlay staging)"); + _overlayStagingMapped[slot] = (nint)mapped; + } + } + + private void RecordOverlayBlit(uint imageIndex, int frameSlot) + { + if (_overlayImage.Handle == 0 || _overlayStagingMapped.Length <= frameSlot) { return; } - var fence = _presentationFence; - var result = wait - ? _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue) - : _vk.GetFenceStatus(_device, fence); - if (!wait && result == Result.NotReady) + int pendingWork; + lock (_gate) { - return; + pendingWork = _pendingGuestWorkCount; } - Check(result, "vkWaitForFences(presentation)"); - _presentationInFlight = false; - if (_pendingPresentationResources is not null) + var pixels = new Span( + (void*)_overlayStagingMapped[frameSlot], + PerfOverlay.PanelWidth * PerfOverlay.PanelHeight * 4); + PerfOverlay.Fill(pixels, pendingWork, _pendingGuestSubmissions.Count); + + var toTransferDst = new ImageMemoryBarrier { - MarkSampledImagesInitialized(_pendingPresentationResources); - MarkStorageImagesInitialized(_pendingPresentationResources); - DestroyTranslatedDrawResources(_pendingPresentationResources); - _pendingPresentationResources = null; - } + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = _overlayImageInitialized ? AccessFlags.TransferReadBit : 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = _overlayImageInitialized + ? ImageLayout.TransferSrcOptimal + : ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = _overlayImage, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.TransferBit, + 0, 0, null, 0, null, 1, &toTransferDst); + + var copyRegion = new BufferImageCopy + { + ImageSubresource = new ImageSubresourceLayers(ImageAspectFlags.ColorBit, 0, 0, 1), + ImageExtent = new Extent3D(PerfOverlay.PanelWidth, PerfOverlay.PanelHeight, 1), + }; + _vk.CmdCopyBufferToImage( + _commandBuffer, + _overlayStagingBuffers[frameSlot], + _overlayImage, + ImageLayout.TransferDstOptimal, + 1, + ©Region); + + var toTransferSrc = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.TransferReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.TransferSrcOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = _overlayImage, + SubresourceRange = ColorSubresourceRange(), + }; + var swapchainToDst = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = ImageLayout.PresentSrcKhr, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = _swapchainImages[imageIndex], + SubresourceRange = ColorSubresourceRange(), + }; + var preBlitBarriers = stackalloc ImageMemoryBarrier[2] { toTransferSrc, swapchainToDst }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit | PipelineStageFlags.ColorAttachmentOutputBit, + PipelineStageFlags.TransferBit, + 0, 0, null, 0, null, 2, preBlitBarriers); + + const int margin = 12; + var panelWidth = (int)Math.Min(PerfOverlay.PanelWidth, _extent.Width - margin); + var panelHeight = (int)Math.Min(PerfOverlay.PanelHeight, _extent.Height - margin); + // Source and destination are both B8G8R8A8 and the panel is not + // scaled. MoltenVK has corrupted pixels outside the blit region + // for this transfer-on-swapchain path (horizontal red/yellow + // scanlines across the entire window). An exact image copy has + // the required semantics and avoids the driver's blit conversion + // path altogether. + var copy = new ImageCopy + { + SrcSubresource = new ImageSubresourceLayers(ImageAspectFlags.ColorBit, 0, 0, 1), + DstSubresource = new ImageSubresourceLayers(ImageAspectFlags.ColorBit, 0, 0, 1), + SrcOffset = new Offset3D(0, 0, 0), + DstOffset = new Offset3D(margin, margin, 0), + Extent = new Extent3D((uint)panelWidth, (uint)panelHeight, 1), + }; + _vk.CmdCopyImage( + _commandBuffer, + _overlayImage, + ImageLayout.TransferSrcOptimal, + _swapchainImages[imageIndex], + ImageLayout.TransferDstOptimal, + 1, + ©); + + var swapchainToPresent = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = 0, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.PresentSrcKhr, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = _swapchainImages[imageIndex], + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.BottomOfPipeBit, + 0, 0, null, 0, null, 1, &swapchainToPresent); + _overlayImageInitialized = true; } private CommandBuffer AllocateGuestCommandBuffer() { + // The pool has ResetCommandBufferBit, so vkBeginCommandBuffer + // implicitly resets recycled buffers. + if (_recycledGuestCommandBuffers.TryPop(out var recycled)) + { + return recycled; + } + var allocateInfo = new CommandBufferAllocateInfo { SType = StructureType.CommandBufferAllocateInfo, @@ -2401,11 +3925,14 @@ internal static unsafe class VulkanVideoPresenter return commandBuffer; } - private void SubmitGuestCommandBuffer( - CommandBuffer commandBuffer, - TranslatedDrawResources resources, - IReadOnlyList traceImages) + private Fence AcquireGuestFence() { + // Recycled fences were reset when they were collected. + if (_recycledGuestFences.TryPop(out var recycled)) + { + return recycled; + } + var fenceInfo = new FenceCreateInfo { SType = StructureType.FenceCreateInfo, @@ -2414,6 +3941,163 @@ internal static unsafe class VulkanVideoPresenter Check( _vk.CreateFence(_device, &fenceInfo, null, out fence), "vkCreateFence(guest)"); + return fence; + } + + private void ReleaseGuestCommandBuffer(CommandBuffer commandBuffer) + { + if (_recycledGuestCommandBuffers.Count < MaxRecycledGuestCommandBuffers) + { + _recycledGuestCommandBuffers.Push(commandBuffer); + return; + } + + _vk.FreeCommandBuffers(_device, _commandPool, 1, &commandBuffer); + } + + private void ReleaseGuestFence(Fence fence, bool needsReset) + { + if (_recycledGuestFences.Count < MaxRecycledGuestFences) + { + if (needsReset) + { + Check(_vk.ResetFences(_device, 1, &fence), "vkResetFences(guest)"); + } + + _recycledGuestFences.Push(fence); + return; + } + + _vk.DestroyFence(_device, fence, null); + } + + // Translated draws are recorded into a shared command buffer and + // submitted once per drained work batch: on MoltenVK every + // vkQueueSubmit is a Metal command-buffer commit (~0.8ms), which used + // to be paid per draw and dominated the frame time. + private CommandBuffer _batchCommandBuffer; + private bool _batchOpen; + private int _batchDrawCount; + private readonly List _batchResources = new(); + private readonly List _batchTraceImages = new(); + + // Consecutive draws into the same target stay inside one render pass: + // on MoltenVK every render pass is a Metal render encoder, and one + // encoder per draw was the dominant per-draw fixed cost after submit + // batching. The pass closes when the target changes, when a draw + // needs transfer/storage work outside a pass, or when the batch + // flushes. + private GuestImageResource? _openPassTarget; + + private void CloseOpenTranslatedRenderPass() + { + if (_openPassTarget is not { } target) + { + return; + } + + _openPassTarget = null; + _vk.CmdEndRenderPass(_batchCommandBuffer); + var toShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.ColorAttachmentWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.ColorAttachmentOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _batchCommandBuffer, + PipelineStageFlags.ColorAttachmentOutputBit, + PipelineStageFlags.FragmentShaderBit, + 0, + 0, + null, + 0, + null, + 1, + &toShaderRead); + } + + private CommandBuffer BeginBatchedGuestCommands() + { + if (_batchOpen) + { + return _batchCommandBuffer; + } + + _batchCommandBuffer = AllocateGuestCommandBuffer(); + var beginInfo = new CommandBufferBeginInfo + { + SType = StructureType.CommandBufferBeginInfo, + Flags = CommandBufferUsageFlags.OneTimeSubmitBit, + }; + Check( + _vk.BeginCommandBuffer(_batchCommandBuffer, &beginInfo), + "vkBeginCommandBuffer(batch)"); + _batchOpen = true; + _batchDrawCount = 0; + return _batchCommandBuffer; + } + + private void FlushBatchedGuestCommands() + { + if (!_batchOpen) + { + return; + } + + CloseOpenTranslatedRenderPass(); + _batchOpen = false; + try + { + Check(_vk.EndCommandBuffer(_batchCommandBuffer), "vkEndCommandBuffer(batch)"); + SubmitGuestCommandBuffer( + _batchCommandBuffer, + _batchResources.ToArray(), + _batchTraceImages.ToArray(), + _batchRetireBuffers.Count > 0 ? _batchRetireBuffers.ToArray() : []); + } + catch + { + // The batch never reached the queue: release everything it + // owned here so the stale lists cannot ride into the next + // batch's submission. + foreach (var resources in _batchResources) + { + DestroyTranslatedDrawResources(resources); + } + + foreach (var (buffer, memory) in _batchRetireBuffers) + { + _vk.DestroyBuffer(_device, buffer, null); + _vk.FreeMemory(_device, memory, null); + } + + ReleaseGuestCommandBuffer(_batchCommandBuffer); + throw; + } + finally + { + _batchResources.Clear(); + _batchTraceImages.Clear(); + _batchRetireBuffers.Clear(); + _batchCommandBuffer = default; + } + } + + private void SubmitGuestCommandBuffer( + CommandBuffer commandBuffer, + IReadOnlyList resources, + IReadOnlyList traceImages, + IReadOnlyList<(VkBuffer Buffer, DeviceMemory Memory)>? retireBuffers = null, + IReadOnlyList? referencedResources = null) + { + var fence = AcquireGuestFence(); try { var submitInfo = new SubmitInfo @@ -2428,50 +4112,46 @@ internal static unsafe class VulkanVideoPresenter } catch { - _vk.DestroyFence(_device, fence, null); + ReleaseGuestFence(fence, needsReset: false); throw; } + _submitTimeline++; + foreach (var referenced in referencedResources ?? resources) + { + foreach (var globalBuffer in referenced.GlobalMemoryBuffers) + { + if (globalBuffer.Allocation is not { } allocation) + { + continue; + } + + allocation.LastUseTimeline = Math.Max( + allocation.LastUseTimeline, + _submitTimeline); + if (globalBuffer.Writable && globalBuffer.WriteBackToGuest) + { + MarkGuestBufferDirty( + allocation, + globalBuffer.GuestOffset, + globalBuffer.GuestSize); + } + } + } + _pendingGuestSubmissions.Enqueue( new PendingGuestSubmission( fence, commandBuffer, resources, traceImages, - resources.DebugName)); - } - - private void SubmitGuestCommandBufferAndWait(CommandBuffer commandBuffer) - { - var fenceInfo = new FenceCreateInfo - { - SType = StructureType.FenceCreateInfo, - }; - Fence fence; - Check( - _vk.CreateFence(_device, &fenceInfo, null, out fence), - "vkCreateFence(guest chunk)"); - try - { - var submitInfo = new SubmitInfo - { - SType = StructureType.SubmitInfo, - CommandBufferCount = 1, - PCommandBuffers = &commandBuffer, - }; - Check( - _vk.QueueSubmit(_queue, 1, &submitInfo, fence), - "vkQueueSubmit(guest chunk)"); - Check( - _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue), - "vkWaitForFences(guest chunk)"); - } - finally - { - _vk.DestroyFence(_device, fence, null); - } - - _vk.FreeCommandBuffers(_device, _commandPool, 1, &commandBuffer); + retireBuffers ?? [], + _submitTimeline, + resources.Count > 0 ? resources[0].DebugName : "batch", + _activeGuestQueue, + _activeGuestWorkSequence)); + _lastSubmittedTimelineByGuestQueue[_activeGuestQueue.Name] = + _submitTimeline; } private void EnsureGuestSubmissionCapacity() @@ -2479,7 +4159,14 @@ internal static unsafe class VulkanVideoPresenter CollectCompletedGuestSubmissions(waitForOldest: false); if (_pendingGuestSubmissions.Count >= MaxInFlightGuestSubmissions) { - CollectCompletedGuestSubmissions(waitForOldest: true); + // Bounded wait so the macOS main thread returns to its event + // pump promptly under a slow-compute backlog; if the oldest + // isn't done yet we proceed (soft cap, dynamic pools). + CollectCompletedGuestSubmissions( + waitForOldest: true, + maxWaitNs: _submissionCapacityWaitNs == 0 + ? _guestFenceWaitTimeoutNs + : _submissionCapacityWaitNs); } } @@ -2491,17 +4178,42 @@ internal static unsafe class VulkanVideoPresenter } } - private void CollectCompletedGuestSubmissions(bool waitForOldest) + private void CollectCompletedGuestSubmissions(bool waitForOldest, ulong maxWaitNs = 0) { if (waitForOldest && _pendingGuestSubmissions.TryPeek(out var oldest)) { var fence = oldest.Fence; + // maxWaitNs==0 => the full "is this submission hung" timeout, + // which also emits the one-shot hang warning below. A shorter + // capacity-probe wait (maxWaitNs>0) must NOT report a hang: the + // submission is still tracked and will be collected once the GPU + // finishes it on a later frame. + var isProbeWait = maxWaitNs != 0 && maxWaitNs < _guestFenceWaitTimeoutNs; + var waitNs = maxWaitNs != 0 ? maxWaitNs : _guestFenceWaitTimeoutNs; var result = _vk.WaitForFences( _device, 1, &fence, true, - ulong.MaxValue); + waitNs); + if (result == Result.Timeout) + { + // A GPU submission whose fence never signals (typically a + // mistranslated compute shader that hangs the Metal queue) + // would otherwise block the render thread forever, starving + // the swapchain present (black screen). Log the culprit and + // continue so at least the last good frame can be shown. + if (!isProbeWait && _tracedFenceTimeouts.Add(oldest.DebugName)) + { + Console.Error.WriteLine( + $"[LOADER][WARN] vk.fence_wait_timeout submission='{oldest.DebugName}' " + + $"— GPU work not completing after {_guestFenceWaitTimeoutNs / 1_000_000}ms; " + + "render thread continuing (present not blocked)."); + } + + return; + } + Check(result, $"vkWaitForFences(guest: {oldest.DebugName})"); } @@ -2521,41 +4233,575 @@ internal static unsafe class VulkanVideoPresenter TraceGuestImageContents(image); } - DestroyTranslatedDrawResources(submission.Resources); - var commandBuffer = submission.CommandBuffer; - _vk.FreeCommandBuffers( - _device, - _commandPool, - 1, - &commandBuffer); - _vk.DestroyFence(_device, submission.Fence, null); + foreach (var resources in submission.Resources) + { + DestroyTranslatedDrawResources(resources); + } + + foreach (var (buffer, memory) in submission.RetireBuffers) + { + _vk.DestroyBuffer(_device, buffer, null); + _vk.FreeMemory(_device, memory, null); + } + + ReleaseGuestCommandBuffer(submission.CommandBuffer); + ReleaseGuestFence(submission.Fence, needsReset: true); + if (submission.Timeline > _completedTimeline) + { + _completedTimeline = submission.Timeline; + } } + + ProcessDeferredTextureDestroys(); + } + + private void WaitForAllGuestSubmissionsForCpuVisibility() + { + FlushBatchedGuestCommands(); + while (_pendingGuestSubmissions.TryPeek(out var oldest)) + { + var fence = oldest.Fence; + Check( + _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue), + $"vkWaitForFences(cpu visibility: {oldest.DebugName})"); + CollectCompletedGuestSubmissions(waitForOldest: false); + } + } + + private void WaitForActiveGuestQueueSubmissionsForCpuVisibility() + { + FlushBatchedGuestCommands(); + if (!_lastSubmittedTimelineByGuestQueue.TryGetValue( + _activeGuestQueue.Name, + out var targetTimeline) || + targetTimeline <= _completedTimeline) + { + return; + } + + PendingGuestSubmission? target = null; + foreach (var submission in _pendingGuestSubmissions) + { + if (submission.Timeline == targetTimeline) + { + target = submission; + break; + } + } + + if (target is null) + { + throw new InvalidOperationException( + $"Guest queue '{_activeGuestQueue.Name}' lost pending timeline " + + $"{targetTimeline} (completed={_completedTimeline})."); + } + + var waitStart = System.Diagnostics.Stopwatch.GetTimestamp(); + var fence = target.Fence; + Check( + _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue), + $"vkWaitForFences(queue visibility: {_activeGuestQueue.Name})"); + CollectCompletedGuestSubmissions(waitForOldest: false); + var waitedMs = (System.Diagnostics.Stopwatch.GetTimestamp() - waitStart) * + 1000.0 / System.Diagnostics.Stopwatch.Frequency; + TraceVulkanShader( + $"vk.queue_visibility queue={_activeGuestQueue.Name} " + + $"submission={_activeGuestQueue.SubmissionId} " + + $"target_timeline={targetTimeline} completed_timeline={_completedTimeline} " + + $"waited_ms={waitedMs:F3}"); + } + + private void ExecuteOrderedGuestAction(VulkanOrderedGuestAction work) + { + WaitForActiveGuestQueueSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(_activeGuestQueue.Name); + work.Action(); + TraceVulkanShader( + $"vk.ordered_action queue={_activeGuestQueue.Name} " + + $"submission={_activeGuestQueue.SubmissionId} " + + $"work_sequence={_activeGuestWorkSequence} name='{work.DebugName}'"); + } + + private void ExecuteOrderedGuestFlip(VulkanOrderedGuestFlip work) + { + FlushBatchedGuestCommands(); + _guestImages.TryGetValue(work.Address, out var source); + if (_deviceLost || + source is null || + !source.Initialized) + { + Console.Error.WriteLine( + $"[LOADER][WARN] vk.flip_capture_failed version={work.Version} " + + $"queue={_activeGuestQueue.Name} addr=0x{work.Address:X16} " + + $"found={(source is not null)} initialized={(source?.Initialized ?? false)}"); + return; + } + + EnsureGuestSubmissionCapacity(); + var snapshot = CreateGuestFlipSnapshot(source, work.Version); + var commandBuffer = AllocateGuestCommandBuffer(); + var submitted = false; + try + { + var beginInfo = new CommandBufferBeginInfo + { + SType = StructureType.CommandBufferBeginInfo, + Flags = CommandBufferUsageFlags.OneTimeSubmitBit, + }; + Check( + _vk.BeginCommandBuffer(commandBuffer, &beginInfo), + "vkBeginCommandBuffer(flip capture)"); + + var barriers = stackalloc ImageMemoryBarrier[2]; + barriers[0] = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.ShaderReadBit | + AccessFlags.ShaderWriteBit | + AccessFlags.ColorAttachmentWriteBit | + AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.TransferReadBit, + OldLayout = ImageLayout.ShaderReadOnlyOptimal, + NewLayout = ImageLayout.TransferSrcOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = ColorSubresourceRange(), + }; + barriers[1] = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = snapshot.Image, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.AllCommandsBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 2, + barriers); + + var copy = new ImageCopy + { + SrcSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, 0, 0, 1), + DstSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, 0, 0, 1), + Extent = new Extent3D(source.Width, source.Height, 1), + }; + _vk.CmdCopyImage( + commandBuffer, + source.Image, + ImageLayout.TransferSrcOptimal, + snapshot.Image, + ImageLayout.TransferDstOptimal, + 1, + ©); + + barriers[0] = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferReadBit, + DstAccessMask = AccessFlags.ShaderReadBit | + AccessFlags.ShaderWriteBit | + AccessFlags.ColorAttachmentWriteBit, + OldLayout = ImageLayout.TransferSrcOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = ColorSubresourceRange(), + }; + barriers[1] = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = snapshot.Image, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.AllCommandsBit, + 0, + 0, + null, + 0, + null, + 2, + barriers); + + Check( + _vk.EndCommandBuffer(commandBuffer), + "vkEndCommandBuffer(flip capture)"); + SubmitGuestCommandBuffer(commandBuffer, [], []); + submitted = true; + snapshot.Initialized = true; + _guestImageVersions.Add(work.Version, snapshot); + _capturedGuestFlipVersions.Add(work.Version); + + lock (_gate) + { + var sequence = (_latestPresentation?.Sequence ?? 0) + 1; + var presentation = new Presentation( + null, + work.Width, + work.Height, + sequence, + GuestDrawKind.None, + TranslatedDraw: null, + RequiredGuestWorkSequence: _activeGuestWorkSequence, + IsSplash: false, + GuestImageAddress: work.Address, + GuestImageVersion: work.Version); + _latestPresentation = presentation; + _pendingGuestImagePresentations.Enqueue(presentation); + while (_pendingGuestImagePresentations.Count > MaxPendingGuestFlipVersions) + { + _pendingGuestImagePresentations.Dequeue(); + } + } + + CollectAbandonedGuestImageVersions(); + + var effectivePitch = work.PitchInPixel == 0 + ? work.Width + : work.PitchInPixel; + TraceVulkanShader( + $"vk.flip_capture version={work.Version} " + + $"queue={_activeGuestQueue.Name} submission={_activeGuestQueue.SubmissionId} " + + $"work_sequence={_activeGuestWorkSequence} addr=0x{work.Address:X16} " + + $"size={work.Width}x{work.Height} pitch={effectivePitch}"); + } + finally + { + if (!submitted) + { + ReleaseGuestCommandBuffer(commandBuffer); + DestroyGuestImage(snapshot); + } + } + } + + private void ExecuteOrderedGuestFlipWait(VulkanOrderedGuestFlipWait work) + { + var captured = work.Version != 0 && + _capturedGuestFlipVersions.Contains(work.Version); + TraceVulkanShader( + $"vk.flip_wait_safe version={work.Version} " + + $"queue={_activeGuestQueue.Name} submission={_activeGuestQueue.SubmissionId} " + + $"handle={work.VideoOutHandle} index={work.DisplayBufferIndex} " + + $"capture_complete={(captured ? 1 : 0)}"); +#if DEBUG + System.Diagnostics.Debug.Assert( + work.Version == 0 || captured, + "An ordered wait-safe marker must execute after its flip capture."); +#endif + } + + private GuestImageResource CreateGuestFlipSnapshot( + GuestImageResource source, + long version) + { + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + ImageType = ImageType.Type2D, + Format = source.Format, + Extent = new Extent3D(source.Width, source.Height, 1), + MipLevels = 1, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = ImageUsageFlags.TransferSrcBit | + ImageUsageFlags.TransferDstBit | + ImageUsageFlags.SampledBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check( + _vk.CreateImage(_device, &imageInfo, null, out var image), + "vkCreateImage(flip snapshot)"); + _vk.GetImageMemoryRequirements(_device, image, out var requirements); + var allocationInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType( + requirements.MemoryTypeBits, + MemoryPropertyFlags.DeviceLocalBit), + }; + DeviceMemory memory = default; + try + { + Check( + _vk.AllocateMemory(_device, &allocationInfo, null, out memory), + "vkAllocateMemory(flip snapshot)"); + Check( + _vk.BindImageMemory(_device, image, memory, 0), + "vkBindImageMemory(flip snapshot)"); + } + catch + { + if (memory.Handle != 0) + { + _vk.FreeMemory(_device, memory, null); + } + _vk.DestroyImage(_device, image, null); + throw; + } + + SetDebugName( + ObjectType.Image, + image.Handle, + $"guest flip v{version} source 0x{source.Address:X16}"); + return new GuestImageResource + { + Address = source.Address, + FlipVersion = version, + Width = source.Width, + Height = source.Height, + MipLevels = 1, + GuestFormat = source.GuestFormat, + Format = source.Format, + Image = image, + Memory = memory, + }; + } + + private static byte[]? TryReadGuestTexturePixels(GuestDrawTexture texture) + { + var memory = _guestMemory; + if (memory is null || texture.Address == 0) + { + return null; + } + + var width = Math.Max(texture.Width, 1); + var height = Math.Max(texture.Height, 1); + var rowLength = texture.TileMode == 0 + ? Math.Max(texture.Pitch, width) + : width; + var byteCount = GetTextureByteCount(texture.Format, rowLength, height); + if (byteCount == 0 || byteCount > int.MaxValue) + { + return null; + } + + var pixels = new byte[(int)byteCount]; + return memory.TryRead(texture.Address, pixels) ? pixels : null; + } + + /// + /// Returns a skipped draw's pooled data arrays: draws dropped before + /// resource creation would otherwise strand their rented buffers. + /// + private static void ReturnPooledGuestData(VulkanTranslatedGuestDraw draw) + { + foreach (var buffer in draw.GlobalMemoryBuffers) + { + if (buffer.Pooled) + { + GuestDataPool.Return(buffer.Data); + } + } + + foreach (var buffer in draw.VertexBuffers) + { + if (buffer.Pooled) + { + GuestDataPool.Return(buffer.Data); + } + } + + if (draw.IndexBuffer is { Pooled: true } indexBuffer) + { + GuestDataPool.Return(indexBuffer.Data); + } + } + + private void ProcessDeferredTextureDestroys() + { + while (_deferredTextureDestroys.TryPeek(out var entry) && + entry.RetireTimeline <= _completedTimeline) + { + _deferredTextureDestroys.Dequeue(); + DestroyCachedTextureResource(entry.Texture); + } + + while (_deferredResourceDestroys.TryPeek(out var resourceEntry) && + resourceEntry.RetireTimeline <= _completedTimeline) + { + _deferredResourceDestroys.Dequeue(); + DestroyTranslatedDrawResources(resourceEntry.Resources); + } + + while (_deferredGuestImageVersionDestroys.TryPeek(out var imageEntry) && + imageEntry.RetireTimeline <= _completedTimeline) + { + _deferredGuestImageVersionDestroys.Dequeue(); + DestroyGuestImage(imageEntry.Image); + TraceVulkanShader( + $"vk.flip_retired version={imageEntry.Image.FlipVersion} " + + $"timeline={imageEntry.RetireTimeline} reason=presentation-dropped"); + } + } + + private void WaitFrameSlot(int slot) => TryWaitFrameSlot(slot, ulong.MaxValue); + + // Returns false when the slot's fence is still unsignaled after + // timeoutNs (the GPU is behind, e.g. a slow-compute backlog). Callers + // on the macOS main thread must NOT wait forever here or the Cocoa + // event pump stalls and the window goes "Not Responding" (F1 overlay / + // close stop working). A bounded wait lets Render() skip the frame and + // return to the pump; the fence still signals later and the frame is + // retried. + private bool TryWaitFrameSlot(int slot, ulong timeoutNs) + { + if (_frameFencePending.Length <= slot || !_frameFencePending[slot]) + { + if (_frameGuestImageVersions.Length > slot && + _frameGuestImageVersions[slot] is { } unsubmittedVersion) + { + _frameGuestImageVersions[slot] = null; + _capturedGuestFlipVersions.Remove(unsubmittedVersion.FlipVersion); + DestroyGuestImage(unsubmittedVersion); + TraceVulkanShader( + $"vk.flip_retired version={unsubmittedVersion.FlipVersion} " + + $"frame_slot={slot} reason=frame-not-submitted"); + } + return true; + } + + var fence = _frameFences[slot]; + var waitResult = _vk.WaitForFences(_device, 1, &fence, true, timeoutNs); + if (waitResult == Result.Timeout) + { + return false; + } + + Check(waitResult, "vkWaitForFences(frame)"); + Check(_vk.ResetFences(_device, 1, &fence), "vkResetFences(frame)"); + _frameFencePending[slot] = false; + if (_frameTimelines[slot] > _completedTimeline) + { + _completedTimeline = _frameTimelines[slot]; + } + + if (_frameTranslatedResources[slot] is { } translated) + { + _frameTranslatedResources[slot] = null; + DestroyTranslatedDrawResources(translated); + } + + if (_frameGuestImageVersions[slot] is { } guestImageVersion) + { + _frameGuestImageVersions[slot] = null; + _capturedGuestFlipVersions.Remove(guestImageVersion.FlipVersion); + DestroyGuestImage(guestImageVersion); + TraceVulkanShader( + $"vk.flip_retired version={guestImageVersion.FlipVersion} " + + $"frame_slot={slot} timeline={_frameTimelines[slot]}"); + } + + ProcessDeferredTextureDestroys(); + return true; + } + + private void WaitAllFrameSlots() + { + for (var slot = 0; slot < _frameFencePending.Length; slot++) + { + WaitFrameSlot(slot); + } + } + + private void CollectAbandonedGuestImageVersions() + { + if (_guestImageVersions.Count == 0) + { + return; + } + + HashSet referencedVersions; + lock (_gate) + { + referencedVersions = _pendingGuestImagePresentations + .Select(static presentation => presentation.GuestImageVersion) + .Where(static version => version != 0) + .ToHashSet(); + if (_latestPresentation is { GuestImageVersion: not 0 } latest) + { + referencedVersions.Add(latest.GuestImageVersion); + } + } + + foreach (var entry in _guestImageVersions.ToArray()) + { + if (referencedVersions.Contains(entry.Key)) + { + continue; + } + + _guestImageVersions.Remove(entry.Key); + _capturedGuestFlipVersions.Remove(entry.Key); + _deferredGuestImageVersionDestroys.Enqueue( + (entry.Value, _submitTimeline)); + TraceVulkanShader( + $"vk.flip_retire_deferred version={entry.Key} " + + $"timeline={_submitTimeline} reason=presentation-dropped"); + } + } + + // Used on teardown paths that already drained the queue (device + // wait-idle): releases per-slot state without touching fences that + // were never submitted. + private void DrainFrameSlots() + { + WaitAllFrameSlots(); + _completedTimeline = _submitTimeline; + ProcessDeferredTextureDestroys(); } private IReadOnlyList GetTraceImages( TranslatedDrawResources resources, - IReadOnlyList? renderTargets = null) + IReadOnlyList? renderTargets = null, + ulong shaderAddress = 0) { - var images = new HashSet(); + var candidates = new HashSet(); foreach (var renderTarget in renderTargets ?? []) { - if (ShouldTraceGuestImageContents(renderTarget)) - { - images.Add(renderTarget); - } + candidates.Add(renderTarget); } foreach (var texture in resources.Textures) { - if (texture.IsStorage && - texture.GuestImage is { } image && - ShouldTraceGuestImageContents(image)) + if ((texture.IsStorage || _traceGuestImageAddressFilterEnabled) && + texture.GuestImage is { } image) { - images.Add(image); + candidates.Add(image); } } - return images.ToArray(); + return candidates + .Where(image => ShouldTraceGuestImageContents(image, shaderAddress)) + .ToArray(); } private void CreateGuestDrawResources() @@ -2744,6 +4990,7 @@ internal static unsafe class VulkanVideoPresenter null, out _barycentricPipeline), "vkCreateGraphicsPipelines"); + MarkPipelineCacheDirty(); } finally { @@ -2774,15 +5021,71 @@ internal static unsafe class VulkanVideoPresenter VulkanTranslatedGuestDraw draw, RenderPass renderPass, IReadOnlyList renderTargetFormats, - Extent2D extent) + Extent2D extent, + IReadOnlyList? feedbackTargets = null, + bool hasDepthAttachment = false, + GuestDepthResource? feedbackDepth = null) { - var vertexSpirv = draw.VertexSpirv; + var isTitleDraw = draw.VertexBuffers.Any(buffer => + buffer.Location == 0 && + buffer.ComponentCount == 4 && + buffer.DataFormat == 10 && + buffer.NumberFormat == 0 && + buffer.Stride == 16 && + buffer.OffsetBytes == 12 && + buffer.Length == 67568); + var forceFullscreenPipeline = Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_FULLSCREEN_PIPELINE") == "1"; + var forceFullscreenVertex = forceFullscreenPipeline || + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_FULLSCREEN_VERTEX") == "1" || + isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_FULLSCREEN_VERTEX") == "1" || + feedbackTargets?.Any(target => AddressListContains( + "SHARPEMU_FORCE_FULLSCREEN_VERTEX_TARGETS", + target.Address)) == true; + var forceRasterState = forceFullscreenPipeline || + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_DEFAULT_RASTER_STATE") == "1" || + isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DEFAULT_RASTER_STATE") == "1" || + feedbackTargets?.Any(target => AddressListContains( + "SHARPEMU_FORCE_DEFAULT_RASTER_STATE_TARGETS", + target.Address)) == true; + var forceTitleSolidFragment = + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_SOLID_FRAGMENT") == "1" && + isTitleDraw; + var forceSolidFragment = forceTitleSolidFragment || forceFullscreenPipeline || + Environment.GetEnvironmentVariable("SHARPEMU_FORCE_SOLID_FRAGMENT") == "1" || + feedbackTargets?.Any(target => AddressListContains( + "SHARPEMU_FORCE_SOLID_FRAGMENT_TARGETS", + target.Address)) == true; + var forceAttributeFragment = uint.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_FORCE_ATTRIBUTE_FRAGMENT"), + out var attributeFragmentLocation) && + feedbackTargets?.Any(target => AddressListContains( + "SHARPEMU_FORCE_ATTRIBUTE_FRAGMENT_TARGETS", + target.Address)) == true; + var vertexSpirv = forceFullscreenVertex + ? SpirvFixedShaders.CreateFullscreenVertex(0) + : draw.VertexSpirv; + var fragmentSpirv = forceSolidFragment + ? SpirvFixedShaders.CreateSolidFragment(1f, 0f, 1f, 1f) + : forceAttributeFragment + ? SpirvFixedShaders.CreateAttributeFragment(attributeFragmentLocation) + : draw.PixelSpirv; + var fixedFragmentDumpPath = Environment.GetEnvironmentVariable( + "SHARPEMU_DUMP_FIXED_SOLID_FRAGMENT"); + if (forceSolidFragment && !string.IsNullOrWhiteSpace(fixedFragmentDumpPath)) + { + File.WriteAllBytes(fixedFragmentDumpPath, fragmentSpirv); + } if (draw.RenderState.Blends.Count != renderTargetFormats.Count) { throw new InvalidOperationException( "color attachment formats and blend states must have matching counts"); } - if (vertexSpirv.Length == 0 && !TryCompileFullscreenVertexShader( draw.AttributeCount, @@ -2805,13 +5108,74 @@ internal static unsafe class VulkanVideoPresenter Blends = draw.RenderState.Blends.ToArray(), Scissor = draw.RenderState.Scissor, Viewport = draw.RenderState.Viewport, + Raster = draw.RenderState.Raster, + Depth = draw.RenderState.Depth, + HasDepthAttachment = hasDepthAttachment, + TargetFormats = renderTargetFormats.ToArray(), }; + if (forceFullscreenVertex) + { + resources.VertexCount = 3; + resources.InstanceCount = 1; + resources.Topology = PrimitiveTopology.TriangleList; + } + if (forceRasterState) + { + resources.Blends = Enumerable.Repeat( + GuestBlendState.Default, + renderTargetFormats.Count).ToArray(); + resources.Scissor = null; + resources.Viewport = null; + resources.Raster = GuestRasterState.Default; + resources.Depth = GuestDepthState.Default; + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DEFAULT_BLEND") == "1") + { + resources.Blends = Enumerable.Repeat( + GuestBlendState.Default, + renderTargetFormats.Count).ToArray(); + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DEFAULT_VIEWPORT_SCISSOR") == "1") + { + resources.Scissor = null; + resources.Viewport = null; + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DISABLE_CULL") == "1") + { + resources.Raster = resources.Raster with + { + CullFront = false, + CullBack = false, + }; + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DISABLE_DEPTH") == "1") + { + resources.Depth = GuestDepthState.Default; + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_TITLE_STATE") == "1") + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.title_state " + + $"viewport={resources.Viewport} scissor={resources.Scissor} " + + $"raster={resources.Raster} depth={resources.Depth} " + + $"blends=[{string.Join(',', resources.Blends)}]"); + } try { foreach (var texture in draw.Textures) { - if (texture.IsStorage) + // Skip address-0 storage bindings here: the real resolution + // path uses a scratch image for those, but this warm-up pass + // called ResolveStorageGuestImage directly, which throws on + // address 0 and dropped the whole draw (Demon's Souls G-buffer + // normals/IDs passes -> lighting had no input -> black). + if (texture.IsStorage && texture.Address != 0) { _ = ResolveStorageGuestImage(texture); } @@ -2819,28 +5183,70 @@ internal static unsafe class VulkanVideoPresenter for (var index = 0; index < draw.Textures.Count; index++) { - resources.Textures[index] = ResolveTextureResource(draw.Textures[index]); + var texture = draw.Textures[index]; + var resolved = ResolveTextureResource(texture); + var feedbackTarget = !texture.IsStorage + ? feedbackTargets?.FirstOrDefault(target => + ReferenceEquals(resolved.GuestImage, target)) + : null; + // ResolveTextureResource may deliberately decline an + // address alias when the descriptor is incompatible with + // the render-target image. Only snapshot an alias which + // actually resolved to the target; a separately uploaded + // texture has no Vulkan attachment feedback hazard. + resources.Textures[index] = + feedbackDepth is not null && + !texture.IsStorage && + ReferenceEquals(resolved.GuestDepth, feedbackDepth) + ? CreateDepthFeedbackSnapshot(texture, feedbackDepth) + : + feedbackTarget is not null && + !texture.IsStorage && + ReferenceEquals(resolved.GuestImage, feedbackTarget) + ? CreateRenderTargetFeedbackSnapshot(texture, feedbackTarget) + : resolved; } + PrepareGuestBufferAllocations(draw.GlobalMemoryBuffers); for (var index = 0; index < draw.GlobalMemoryBuffers.Count; index++) { resources.GlobalMemoryBuffers[index] = CreateGlobalBufferResource(draw.GlobalMemoryBuffers[index]); } + var sharedVertexResources = new Dictionary< + byte[], VertexBufferResource>( + System.Collections.Generic.ReferenceEqualityComparer.Instance); for (var index = 0; index < draw.VertexBuffers.Count; index++) { - resources.VertexBuffers[index] = - CreateVertexBufferResource(draw.VertexBuffers[index]); + var guestVertex = draw.VertexBuffers[index]; + if (sharedVertexResources.TryGetValue( + guestVertex.Data, + out var sharedVertex)) + { + resources.VertexBuffers[index] = + CreateVertexBufferAlias(sharedVertex, guestVertex); + } + else + { + var vertexResource = + CreateVertexBufferResource(guestVertex); + resources.VertexBuffers[index] = vertexResource; + sharedVertexResources.Add(guestVertex.Data, vertexResource); + } } - if (draw.IndexBuffer is { Data.Length: > 0 } indexBuffer) + if (draw.IndexBuffer is { Length: > 0 } indexBuffer) { resources.IndexBuffer = CreateHostBuffer( - indexBuffer.Data, + indexBuffer.Data.AsSpan(0, indexBuffer.Length), BufferUsageFlags.IndexBufferBit, out resources.IndexMemory); resources.Index32Bit = indexBuffer.Is32Bit; + if (indexBuffer.Pooled) + { + GuestDataPool.Return(indexBuffer.Data); + } } CreateTranslatedDescriptorResources( @@ -2849,7 +5255,7 @@ internal static unsafe class VulkanVideoPresenter CreateTranslatedPipeline( resources, vertexSpirv, - draw.PixelSpirv, + fragmentSpirv, renderPass, renderTargetFormats, extent); @@ -2860,6 +5266,18 @@ internal static unsafe class VulkanVideoPresenter DestroyTranslatedDrawResources(resources); throw; } + finally + { + var returnedVertexData = new HashSet( + System.Collections.Generic.ReferenceEqualityComparer.Instance); + foreach (var vertex in draw.VertexBuffers) + { + if (vertex.Pooled && returnedVertexData.Add(vertex.Data)) + { + GuestDataPool.Return(vertex.Data); + } + } + } } [MethodImpl(MethodImplOptions.NoInlining)] @@ -2887,7 +5305,11 @@ internal static unsafe class VulkanVideoPresenter for (var index = 0; index < dispatch.Textures.Count; index++) { var texture = dispatch.Textures[index]; - if (texture.IsStorage) + // Address-zero storage descriptors are valid scratch bindings. + // ResolveTextureResource creates their transient image below; + // pre-resolving them as guest-backed images throws and drops + // the entire compute dispatch before that path can run. + if (texture.IsStorage && texture.Address != 0) { if (traceResources) { @@ -2895,10 +5317,12 @@ internal static unsafe class VulkanVideoPresenter $"vk.compute_resources storage[{index}] begin " + $"addr=0x{texture.Address:X16} fmt={texture.Format} " + $"size={texture.Width}x{texture.Height} " + - $"mips={texture.MipLevels} level={texture.MipLevel}"); + $"view_mips={texture.BaseMipLevel}+{texture.MipLevels} " + + $"resource_mips={texture.ResourceMipLevels} " + + $"relative_level={texture.MipLevel}"); } - _ = ResolveStorageImageResource(texture); + _ = ResolveStorageGuestImage(texture); if (traceResources) { TraceVulkanShader($"vk.compute_resources storage[{index}] ready"); @@ -2922,6 +5346,7 @@ internal static unsafe class VulkanVideoPresenter TraceVulkanShader("vk.compute_resources resolve ready"); } + PrepareGuestBufferAllocations(dispatch.GlobalMemoryBuffers); for (var index = 0; index < dispatch.GlobalMemoryBuffers.Count; index++) { resources.GlobalMemoryBuffers[index] = @@ -3034,14 +5459,42 @@ internal static unsafe class VulkanVideoPresenter }; } - fixed (DescriptorPoolSize* poolSizePointer = poolSizes) + if (_recycledDescriptorPools.TryPop(out var recycledPool)) { + Check( + _vk.ResetDescriptorPool(_device, recycledPool, 0), + "vkResetDescriptorPool"); + resources.DescriptorPool = recycledPool; + } + else + { + // Generously sized so any draw's set fits, making the pool + // recyclable regardless of the draw's binding mix. AAA titles + // (e.g. Demon's Souls) bind well over 32 textures in a single + // descriptor set, so the sampled-image budget in particular + // must be large enough to avoid a per-draw dynamic fallback. + var genericPoolSizes = stackalloc DescriptorPoolSize[3]; + genericPoolSizes[0] = new DescriptorPoolSize + { + Type = DescriptorType.CombinedImageSampler, + DescriptorCount = 256, + }; + genericPoolSizes[1] = new DescriptorPoolSize + { + Type = DescriptorType.StorageImage, + DescriptorCount = 64, + }; + genericPoolSizes[2] = new DescriptorPoolSize + { + Type = DescriptorType.StorageBuffer, + DescriptorCount = 64, + }; var poolInfo = new DescriptorPoolCreateInfo { SType = StructureType.DescriptorPoolCreateInfo, MaxSets = 1, - PoolSizeCount = (uint)poolSizes.Length, - PPoolSizes = poolSizePointer, + PoolSizeCount = 3, + PPoolSizes = genericPoolSizes, }; DescriptorPool descriptorPool; Check( @@ -3082,7 +5535,7 @@ internal static unsafe class VulkanVideoPresenter bufferInfoPointer[index] = new DescriptorBufferInfo { Buffer = resources.GlobalMemoryBuffers[index].Buffer, - Offset = 0, + Offset = resources.GlobalMemoryBuffers[index].Offset, Range = resources.GlobalMemoryBuffers[index].Size, }; } @@ -3155,13 +5608,16 @@ internal static unsafe class VulkanVideoPresenter GetShaderDigest(vertexSpirv), GetShaderDigest(fragmentSpirv), string.Join(',', renderTargetFormats.Select(format => (uint)format)), + resources.HasDepthAttachment, resources.Topology, string.Join(';', resources.Blends.Select(blend => $"{(blend.Enable ? 1 : 0)}:{blend.ColorSrcFactor}:{blend.ColorDstFactor}:" + $"{blend.ColorFunc}:{blend.AlphaSrcFactor}:{blend.AlphaDstFactor}:" + $"{blend.AlphaFunc}:{(blend.SeparateAlphaBlend ? 1 : 0)}:{blend.WriteMask}")), GetResourceLayoutKey(resources), - GetVertexLayoutKey(resources)); + GetVertexLayoutKey(resources), + resources.Raster, + resources.HasDepthAttachment ? resources.Depth : GuestDepthState.Default); if (_graphicsPipelines.TryGetValue(pipelineKey, out var cachedPipeline)) { resources.Pipeline = cachedPipeline; @@ -3236,6 +5692,14 @@ internal static unsafe class VulkanVideoPresenter { SType = StructureType.PipelineInputAssemblyStateCreateInfo, Topology = resources.Topology, + // Metal always applies primitive restart to strip/fan + // topologies with the max index as the cut value, so + // match that here. Enabling it for lists is what makes + // MoltenVK warn ("Metal does not support disabling + // primitive restart"); lists never carry a restart + // index, so leaving it off for them is both correct + // and warning-free. + PrimitiveRestartEnable = RequiresPrimitiveRestart(resources.Topology), }; var viewport = new Viewport(0, 0, extent.Width, extent.Height, 0, 1); var scissor = new Rect2D(new Offset2D(0, 0), extent); @@ -3247,12 +5711,29 @@ internal static unsafe class VulkanVideoPresenter ScissorCount = 1, PScissors = &scissor, }; + var raster = resources.Raster; + var cullMode = CullModeFlags.None; + if (raster.CullFront) + { + cullMode |= CullModeFlags.FrontBit; + } + + if (raster.CullBack) + { + cullMode |= CullModeFlags.BackBit; + } + var rasterization = new PipelineRasterizationStateCreateInfo { SType = StructureType.PipelineRasterizationStateCreateInfo, + // Wireframe (PolygonMode.Line) needs the fillModeNonSolid + // device feature and is effectively unused by shipping + // titles, so fall back to a solid fill. PolygonMode = PolygonMode.Fill, - CullMode = CullModeFlags.None, - FrontFace = FrontFace.CounterClockwise, + CullMode = cullMode, + FrontFace = raster.FrontFaceClockwise + ? FrontFace.Clockwise + : FrontFace.CounterClockwise, LineWidth = 1, }; var multisample = new PipelineMultisampleStateCreateInfo @@ -3266,7 +5747,8 @@ internal static unsafe class VulkanVideoPresenter var blend = resources.Blends[index]; colorBlendAttachments[index] = new PipelineColorBlendAttachmentState { - BlendEnable = blend.Enable, + BlendEnable = blend.Enable && + IsBlendableFormat(renderTargetFormats[index]), SrcColorBlendFactor = ToVkBlendFactor(blend.ColorSrcFactor), DstColorBlendFactor = ToVkBlendFactor(blend.ColorDstFactor), ColorBlendOp = ToVkBlendOp(blend.ColorFunc), @@ -3297,6 +5779,16 @@ internal static unsafe class VulkanVideoPresenter DynamicStateCount = 2, PDynamicStates = dynamicStateValues, }; + var depth = resources.Depth; + var depthStencil = new PipelineDepthStencilStateCreateInfo + { + SType = StructureType.PipelineDepthStencilStateCreateInfo, + DepthTestEnable = depth.TestEnable, + DepthWriteEnable = depth.WriteEnable, + DepthCompareOp = ToVkCompareOp(depth.CompareOp), + DepthBoundsTestEnable = false, + StencilTestEnable = false, + }; var pipelineInfo = new GraphicsPipelineCreateInfo { SType = StructureType.GraphicsPipelineCreateInfo, @@ -3308,6 +5800,7 @@ internal static unsafe class VulkanVideoPresenter PRasterizationState = &rasterization, PMultisampleState = &multisample, PColorBlendState = &colorBlend, + PDepthStencilState = resources.HasDepthAttachment ? &depthStencil : null, PDynamicState = &dynamicState, Layout = resources.PipelineLayout, RenderPass = renderPass, @@ -3320,12 +5813,14 @@ internal static unsafe class VulkanVideoPresenter _pipelineCache, 1, &pipelineInfo, - null, - out pipeline), - "vkCreateGraphicsPipelines(translated)"); + null, + out pipeline), + "vkCreateGraphicsPipelines(translated)"); + MarkPipelineCacheDirty(); resources.Pipeline = pipeline; resources.PipelineCached = true; _graphicsPipelines.Add(pipelineKey, pipeline); + Interlocked.Increment(ref _perfPipelineCreations); SetDebugName( ObjectType.Pipeline, pipeline.Handle, @@ -3408,6 +5903,18 @@ internal static unsafe class VulkanVideoPresenter pipelineInfo.PSetLayouts = &descriptorSetLayout; } + var computePushConstantRange = new PushConstantRange + { + StageFlags = ShaderStageFlags.ComputeBit, + Offset = 0, + Size = 3 * sizeof(uint), + }; + if ((stageFlags & ShaderStageFlags.ComputeBit) != 0) + { + pipelineInfo.PushConstantRangeCount = 1; + pipelineInfo.PPushConstantRanges = &computePushConstantRange; + } + PipelineLayout pipelineLayout; Check( _vk.CreatePipelineLayout( @@ -3433,7 +5940,13 @@ internal static unsafe class VulkanVideoPresenter return digest; } - private static string GetResourceLayoutKey(TranslatedDrawResources resources) + private static string GetResourceLayoutKey(TranslatedDrawResources resources) => + resources.ResourceLayoutKey ??= BuildResourceLayoutKey(resources); + + private static string GetVertexLayoutKey(TranslatedDrawResources resources) => + resources.VertexLayoutKey ??= BuildVertexLayoutKey(resources); + + private static string BuildResourceLayoutKey(TranslatedDrawResources resources) { var key = new StringBuilder(); key.Append(resources.GlobalMemoryBuffers.Length).Append(':'); @@ -3445,7 +5958,7 @@ internal static unsafe class VulkanVideoPresenter return key.ToString(); } - private static string GetVertexLayoutKey(TranslatedDrawResources resources) + private static string BuildVertexLayoutKey(TranslatedDrawResources resources) { var key = new StringBuilder(); foreach (var buffer in resources.VertexBuffers) @@ -3468,7 +5981,10 @@ internal static unsafe class VulkanVideoPresenter TranslatedDrawResources resources, byte[] computeSpirv) { - if (_computePipelines.TryGetValue(computeSpirv, out var cachedPipeline)) + var pipelineKey = new ComputePipelineKey( + GetShaderDigest(computeSpirv), + GetResourceLayoutKey(resources)); + if (_computePipelines.TryGetValue(pipelineKey, out var cachedPipeline)) { resources.Pipeline = cachedPipeline; resources.PipelineCached = true; @@ -3503,13 +6019,14 @@ internal static unsafe class VulkanVideoPresenter null, out pipeline), "vkCreateComputePipelines(translated)"); + MarkPipelineCacheDirty(); resources.Pipeline = pipeline; resources.PipelineCached = true; SetDebugName( ObjectType.Pipeline, pipeline.Handle, $"SharpEmu compute cs={computeSpirv.Length}b"); - _computePipelines.Add(computeSpirv, pipeline); + _computePipelines.Add(pipelineKey, pipeline); } finally { @@ -3526,16 +6043,20 @@ internal static unsafe class VulkanVideoPresenter return ResolveStorageImageResource(texture); } + if (texture.Address != 0 && + TryResolveGuestDepthTexture(texture, out var depthTexture)) + { + return depthTexture; + } + var vkFormat = GetTextureFormat(texture.Format, texture.NumberType); if (texture.Address != 0 && - _guestImages.TryGetValue(texture.Address, out var guestImage) && - IsCompatibleGuestImageAlias(texture, guestImage) && - IsCompatibleViewFormat(guestImage.Format, vkFormat) && + TryResolveGuestImageAlias(texture, vkFormat, out var guestImage) && TryGetOrCreateGuestImageView( guestImage, vkFormat, - mipLevel: 0, - levelCount: guestImage.MipLevels, + mipLevel: texture.BaseMipLevel, + levelCount: texture.MipLevels, dstSelect: texture.DstSelect, out var view)) { @@ -3570,9 +6091,13 @@ internal static unsafe class VulkanVideoPresenter _pendingAliasImageDumps.Enqueue(guestImage); } - if (TryCreateCpuTextureRefreshResource(texture, guestImage, view, out var refresh)) + if (TryCreateCpuTextureRefreshResource( + texture, + guestImage, + view, + out var refreshResource)) { - return refresh; + return refreshResource; } return new TextureResource @@ -3589,7 +6114,26 @@ internal static unsafe class VulkanVideoPresenter }; } - return CreateTextureResource(texture); + if (ShouldTraceVulkanResources() && texture.Address != 0) + { + if (_guestImages.TryGetValue(texture.Address, out var missImage)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.alias_miss addr=0x{texture.Address:X16} " + + $"reason={(IsCompatibleGuestImageAlias(texture, missImage) ? "format" : "size")} " + + $"tex={texture.Width}x{texture.Height}/f{texture.Format}/n{texture.NumberType}/vk{vkFormat} " + + $"img={missImage.Width}x{missImage.Height}/imgfmt{missImage.Format} " + + $"init={missImage.Initialized}"); + } + else + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.alias_miss addr=0x{texture.Address:X16} " + + $"reason=absent tex={texture.Width}x{texture.Height}/f{texture.Format}/n{texture.NumberType}"); + } + } + + return GetOrCreateCachedTextureResource(texture); } private bool TryCreateCpuTextureRefreshResource( @@ -3657,6 +6201,318 @@ internal static unsafe class VulkanVideoPresenter return true; } + private bool TryResolveGuestImageAlias( + GuestDrawTexture texture, + Format viewFormat, + out GuestImageResource guestImage) + { + GuestImageResource? best = null; + var bestScore = int.MinValue; + var guestFormat = GetGuestTextureFormat(texture.Format, texture.NumberType); + + void Consider(GuestImageResource candidate, bool isActive) + { + if (!IsUsableGuestImageAlias(texture, viewFormat, candidate)) + { + return; + } + + // Prefer an exact descriptor match. Initialization is important, + // but it must not make a differently sized alias outrank the image + // that the texture descriptor actually names. + var score = 0; + if (candidate.Width == texture.Width && + candidate.Height == texture.Height) + { + score += 32; + } + if (candidate.Format == viewFormat) + { + score += 16; + } + if (candidate.GuestFormat == guestFormat) + { + score += 8; + } + if (candidate.Initialized) + { + score += 4; + } + if (candidate.MipLevels == texture.ResourceMipLevels) + { + score += 2; + } + if (isActive) + { + score += 1; + } + + if (score > bestScore) + { + best = candidate; + bestScore = score; + } + } + + if (_guestImages.TryGetValue(texture.Address, out var active)) + { + Consider(active, isActive: true); + } + + foreach (var (key, candidate) in _guestImageVariants) + { + if (key.Address == texture.Address) + { + Consider(candidate, isActive: false); + } + } + + if (best is not null) + { + guestImage = best; + if (ShouldTraceVulkanResources()) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.texture_variant_hit " + + $"addr=0x{texture.Address:X16} " + + $"tex={texture.Width}x{texture.Height}/{viewFormat} " + + $"image={best.Width}x{best.Height}/{best.Format} " + + $"initialized={best.Initialized}"); + } + return true; + } + + guestImage = null!; + return false; + } + + private static bool IsUsableGuestImageAlias( + GuestDrawTexture texture, + Format viewFormat, + GuestImageResource guestImage) => + texture.BaseMipLevel < guestImage.MipLevels && + IsCompatibleGuestImageAlias(texture, guestImage) && + IsCompatibleViewFormat(guestImage.Format, viewFormat); + + private bool TryResolveGuestDepthTexture( + GuestDrawTexture texture, + out TextureResource resource) + { + foreach (var depth in _guestDepthImages.Values) + { + if (texture.Address != depth.Address && + texture.Address != depth.ReadAddress && + texture.Address != depth.WriteAddress) + { + continue; + } + + if (texture.Width > depth.Width || texture.Height > depth.Height) + { + continue; + } + + if (!depth.SampleViews.TryGetValue(texture.DstSelect, out var view)) + { + var viewInfo = new ImageViewCreateInfo + { + SType = StructureType.ImageViewCreateInfo, + Image = depth.Image, + ViewType = ImageViewType.Type2D, + Format = DepthFormat, + Components = ToVkComponentMapping(texture.DstSelect), + SubresourceRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1), + }; + Check( + _vk.CreateImageView(_device, &viewInfo, null, out view), + "vkCreateImageView(depth sample)"); + SetDebugName( + ObjectType.ImageView, + view.Handle, + $"SharpEmu guest depth sample 0x{depth.Address:X16} " + + $"dst=0x{texture.DstSelect:X3}"); + depth.SampleViews.Add(texture.DstSelect, view); + } + + if (_tracedDepthTextureAliases.Add( + (texture.Address, texture.Width, texture.Height, texture.DstSelect))) + { + TraceVulkanShader( + $"vk.depth_texture_alias addr=0x{texture.Address:X16} " + + $"depth=0x{depth.Address:X16} " + + $"texture={texture.Width}x{texture.Height} " + + $"surface={depth.Width}x{depth.Height} dst=0x{texture.DstSelect:X3}"); + } + resource = new TextureResource + { + Address = texture.Address, + Image = depth.Image, + View = view, + Width = depth.Width, + Height = depth.Height, + RowLength = depth.Width, + DstSelect = texture.DstSelect, + SamplerState = texture.Sampler, + GuestDepth = depth, + }; + return true; + } + + resource = null!; + return false; + } + + private readonly Dictionary _textureCache = new(); + + /// + /// Guest textures are static assets in the common case, but every draw + /// used to restage and reupload them (a fresh image, device memory and + /// staging buffer per draw). Cache the uploaded resource per descriptor + /// identity and invalidate through the CPU write tracker, so animated + /// or streamed texture memory still refreshes. + /// + private TextureResource GetOrCreateCachedTextureResource(GuestDrawTexture texture) + { + if (texture.Address == 0) + { + return CreateTextureResource(texture); + } + + var key = new TextureContentIdentity( + texture.Address, + texture.Width, + texture.Height, + texture.Format, + texture.NumberType, + texture.DstSelect, + texture.TileMode, + texture.Pitch, + texture.Sampler); + if (_textureCache.TryGetValue(key, out var cached)) + { + return cached; + } + + // Empty pixels mean the submit thread skipped the guest-memory + // copy because this identity was marked cached; a miss here is + // an invalidation race (eviction, cache clear). Self-heal by + // reading the texels directly rather than rendering a fallback. + if (texture.RgbaPixels.Length == 0) + { + var refreshed = TryReadGuestTexturePixels(texture); + if (refreshed is null) + { + return CreateTextureResource(texture); + } + + texture = texture with { RgbaPixels = refreshed }; + } + + var resource = CreateTextureResource(texture); + if (resource.OwnsStorage) + { + resource.Cached = true; + _textureCache[key] = resource; + MarkTextureContentCached(key); + SharpEmu.HLE.GuestImageWriteTracker.Track( + texture.Address, + (ulong)texture.RgbaPixels.Length, + CurrentGuestWorkSequenceForDiagnostics, + "vulkan.texture-cache"); + } + + return resource; + } + + private void EvictDirtyCachedTextures() + { + if (_textureCache.Count == 0) + { + return; + } + + List? evicted = null; + foreach (var entry in _textureCache) + { + if (SharpEmu.HLE.GuestImageWriteTracker.ConsumeDirty(entry.Key.Address)) + { + (evicted ??= []).Add(entry.Key); + } + } + + if (evicted is null && _textureCache.Count <= 2048) + { + return; + } + + // Destruction is deferred until every submission that may still + // reference the texture has completed (fences signal in queue + // order), so eviction never has to drain the GPU. An open batch + // is flushed first so the retire timeline exactly covers every + // recorded reference (nothing may guess which submission lands + // next on the shared queue). + if (_batchOpen) + { + FlushBatchedGuestCommands(); + } + + var retireTimeline = _submitTimeline; + if (_textureCache.Count > 2048) + { + foreach (var entry in _textureCache) + { + _deferredTextureDestroys.Enqueue((entry.Value, retireTimeline)); + } + + _textureCache.Clear(); + ClearCachedTextureIdentities(); + return; + } + + foreach (var key in evicted!) + { + if (_textureCache.Remove(key, out var resource)) + { + UnmarkTextureContentCached(key); + _deferredTextureDestroys.Enqueue((resource, retireTimeline)); + SharpEmu.HLE.GuestImageWriteTracker.Rearm(key.Address); + } + } + } + + private void DestroyCachedTextureResource(TextureResource texture) + { + texture.Cached = false; + if (texture.View.Handle != 0) + { + _vk.DestroyImageView(_device, texture.View, null); + } + + if (texture.Image.Handle != 0 && texture.GuestImage is null) + { + _vk.DestroyImage(_device, texture.Image, null); + if (texture.ImageMemory.Handle != 0) + { + _vk.FreeMemory(_device, texture.ImageMemory, null); + } + } + + if (texture.StagingBuffer.Handle != 0) + { + _vk.DestroyBuffer(_device, texture.StagingBuffer, null); + } + + if (texture.StagingMemory.Handle != 0) + { + _vk.FreeMemory(_device, texture.StagingMemory, null); + } + } + private static bool IsCompatibleGuestImageAlias( GuestDrawTexture texture, GuestImageResource guestImage) @@ -3688,10 +6544,11 @@ internal static unsafe class VulkanVideoPresenter var guestImage = ResolveStorageGuestImage(texture); var vkFormat = GetTextureFormat(texture.Format, texture.NumberType); + var selectedMipLevel = GetStorageMipLevel(texture); var view = GetOrCreateGuestImageView( guestImage, vkFormat, - texture.MipLevel, + selectedMipLevel, levelCount: 1); var resource = new TextureResource { @@ -3718,8 +6575,12 @@ internal static unsafe class VulkanVideoPresenter if ((ulong)texture.RgbaPixels.Length == expectedSize && texture.RgbaPixels.AsSpan().IndexOfAnyExcept((byte)0) >= 0) { + var uploadPixels = texture.Format == 13 + ? ExpandRgb32Pixels(texture.RgbaPixels) + : texture.RgbaPixels; + var uploadSize = (ulong)uploadPixels.Length; resource.StagingBuffer = CreateBuffer( - expectedSize, + uploadSize, BufferUsageFlags.TransferSrcBit, MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, @@ -3731,17 +6592,17 @@ internal static unsafe class VulkanVideoPresenter _device, resource.StagingMemory, 0, - expectedSize, + uploadSize, 0, &mapped), "vkMapMemory(storage texture)"); - fixed (byte* source = texture.RgbaPixels) + fixed (byte* source = uploadPixels) { System.Buffer.MemoryCopy( source, mapped, - texture.RgbaPixels.Length, - texture.RgbaPixels.Length); + uploadPixels.Length, + uploadPixels.Length); } _vk.UnmapMemory(_device, resource.StagingMemory); @@ -3749,7 +6610,8 @@ internal static unsafe class VulkanVideoPresenter guestImage.InitialUploadPending = true; TraceVulkanShader( $"vk.storage_upload addr=0x{texture.Address:X16} " + - $"size={texture.Width}x{texture.Height} bytes={expectedSize}"); + $"size={texture.Width}x{texture.Height} " + + $"logical_bytes={expectedSize} upload_bytes={uploadSize}"); } } @@ -3823,6 +6685,7 @@ internal static unsafe class VulkanVideoPresenter Width = width, Height = height, MipLevels = 1, + GuestFormat = GetGuestTextureFormat(texture.Format, texture.NumberType), Format = vkFormat, Image = image, Memory = memory, @@ -3861,17 +6724,36 @@ internal static unsafe class VulkanVideoPresenter texture.Height, texture.Format, texture.NumberType, - texture.MipLevels), + texture.ResourceMipLevels), format); - if (texture.MipLevel >= guestImage.MipLevels) + var selectedMipLevel = GetStorageMipLevel(texture); + if (selectedMipLevel >= guestImage.MipLevels) { throw new InvalidOperationException( - $"Storage mip {texture.MipLevel} exceeds image mip count {guestImage.MipLevels}."); + $"Storage mip {selectedMipLevel} (base {texture.BaseMipLevel} + relative " + + $"{texture.MipLevel}) exceeds image mip count {guestImage.MipLevels}."); } return guestImage; } + private static uint GetStorageMipLevel(GuestDrawTexture texture) + { + // IMAGE_STORE targets BASE_LEVEL and IMAGE_STORE_MIP's operand is + // expressed in resource-view space, so Vulkan's absolute image + // subresource is descriptor base plus the instruction-relative + // mip. Sampled views achieve the same mapping through their view + // base in ResolveTextureResource. + var selectedMipLevel = (ulong)texture.BaseMipLevel + texture.MipLevel; + if (selectedMipLevel > uint.MaxValue) + { + throw new InvalidOperationException( + $"Storage mip overflow (base {texture.BaseMipLevel} + relative {texture.MipLevel})."); + } + + return (uint)selectedMipLevel; + } + private TextureResource CreateTextureResource(GuestDrawTexture texture) { var width = Math.Max(texture.Width, 1); @@ -3894,7 +6776,16 @@ internal static unsafe class VulkanVideoPresenter var pixels = texture.RgbaPixels.Length == (int)expectedSize ? texture.RgbaPixels : CreateFallbackTexturePixels(texture.Format, rowLength, height, expectedSize); + if (AddressListContains("SHARPEMU_FORCE_WHITE_TEXTURE_TARGETS", texture.Address)) + { + pixels = pixels.ToArray(); + pixels.AsSpan().Fill(0xFF); + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.texture_force_white addr=0x{texture.Address:X16} " + + $"size={width}x{height} bytes={pixels.Length}"); + } DumpTextureUpload(texture, pixels, rowLength, width, height); + TraceTextureUploadContents(texture, pixels, rowLength, width, height, vkFormat); var uploadPixels = texture.Format == 13 ? ExpandRgb32Pixels(pixels) : pixels; @@ -3976,12 +6867,14 @@ internal static unsafe class VulkanVideoPresenter if (texture.Address != 0 && !_guestImages.ContainsKey(texture.Address)) { + var guestFormat = GetGuestTextureFormat(texture.Format, texture.NumberType); var guestImage = new GuestImageResource { Address = texture.Address, Width = width, Height = height, MipLevels = 1, + GuestFormat = guestFormat, Format = vkFormat, Image = image, Memory = imageMemory, @@ -3995,14 +6888,13 @@ internal static unsafe class VulkanVideoPresenter resource.GuestImage = guestImage; lock (_gate) { - var guestFormat = VulkanVideoPresenter.GetGuestTextureFormat( - texture.Format, - texture.NumberType); if (guestFormat != 0) { _availableGuestImages[texture.Address] = guestFormat; - _gpuGuestImages.Remove(texture.Address); } + + _guestImageExtents[texture.Address] = + (width, height, expectedSize); } } @@ -4069,6 +6961,229 @@ internal static unsafe class VulkanVideoPresenter return hash; } + /// + /// Creates a draw-local sampled image containing the render target's + /// value immediately before the draw. RDNA permits a pixel shader to + /// read an attachment while producing its replacement value, whereas + /// core Vulkan does not permit the same subresource to be both a + /// sampled image and a color attachment. Keeping the two images + /// distinct preserves the guest's read-before-write behavior without + /// a queue idle or an undefined Vulkan feedback loop. + /// + private TextureResource CreateRenderTargetFeedbackSnapshot( + GuestDrawTexture texture, + GuestImageResource source) + { + var image = default(Image); + var memory = default(DeviceMemory); + var view = default(ImageView); + try + { + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + Flags = + ImageCreateFlags.CreateMutableFormatBit | + ImageCreateFlags.CreateExtendedUsageBit, + ImageType = ImageType.Type2D, + Format = source.Format, + Extent = new Extent3D(source.Width, source.Height, 1), + MipLevels = source.MipLevels, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = + ImageUsageFlags.TransferDstBit | + ImageUsageFlags.SampledBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check( + _vk.CreateImage(_device, &imageInfo, null, out image), + "vkCreateImage(render-target feedback snapshot)"); + _vk.GetImageMemoryRequirements(_device, image, out var requirements); + var memoryInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType( + requirements.MemoryTypeBits, + MemoryPropertyFlags.DeviceLocalBit), + }; + Check( + _vk.AllocateMemory(_device, &memoryInfo, null, out memory), + "vkAllocateMemory(render-target feedback snapshot)"); + Check( + _vk.BindImageMemory(_device, image, memory, 0), + "vkBindImageMemory(render-target feedback snapshot)"); + + var viewFormat = GetTextureFormat(texture.Format, texture.NumberType); + if (!IsCompatibleViewFormat(source.Format, viewFormat)) + { + throw new InvalidOperationException( + $"Feedback view format {viewFormat} is incompatible with " + + $"render-target format {source.Format}."); + } + + var viewInfo = new ImageViewCreateInfo + { + SType = StructureType.ImageViewCreateInfo, + Image = image, + ViewType = ImageViewType.Type2D, + Format = viewFormat, + Components = ToVkComponentMapping(texture.DstSelect), + SubresourceRange = ColorSubresourceRange(0, source.MipLevels), + }; + Check( + _vk.CreateImageView(_device, &viewInfo, null, out view), + "vkCreateImageView(render-target feedback snapshot)"); + + var debugName = + $"SharpEmu feedback 0x{source.Address:X16} " + + $"{source.Width}x{source.Height} {source.Format}->{viewFormat}"; + SetDebugName(ObjectType.Image, image.Handle, $"{debugName} image"); + SetDebugName(ObjectType.ImageView, view.Handle, $"{debugName} view"); + TraceVulkanShader( + $"vk.feedback_snapshot_create addr=0x{source.Address:X16} " + + $"size={source.Width}x{source.Height} mips={source.MipLevels} " + + $"image_format={source.Format} view_format={viewFormat} " + + $"dst=0x{texture.DstSelect:X3}"); + + return new TextureResource + { + Address = texture.Address, + Image = image, + ImageMemory = memory, + View = view, + Width = source.Width, + Height = source.Height, + RowLength = source.Width, + DstSelect = texture.DstSelect, + OwnsStorage = true, + SamplerState = texture.Sampler, + FeedbackSource = source, + }; + } + catch + { + if (view.Handle != 0) + { + _vk.DestroyImageView(_device, view, null); + } + + if (image.Handle != 0) + { + _vk.DestroyImage(_device, image, null); + } + + if (memory.Handle != 0) + { + _vk.FreeMemory(_device, memory, null); + } + + throw; + } + } + + private TextureResource CreateDepthFeedbackSnapshot( + GuestDrawTexture texture, + GuestDepthResource source) + { + var image = default(Image); + var memory = default(DeviceMemory); + var view = default(ImageView); + try + { + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + ImageType = ImageType.Type2D, + Format = DepthFormat, + Extent = new Extent3D(source.Width, source.Height, 1), + MipLevels = 1, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = ImageUsageFlags.TransferDstBit | ImageUsageFlags.SampledBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check( + _vk.CreateImage(_device, &imageInfo, null, out image), + "vkCreateImage(depth feedback snapshot)"); + _vk.GetImageMemoryRequirements(_device, image, out var requirements); + var memoryInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType( + requirements.MemoryTypeBits, + MemoryPropertyFlags.DeviceLocalBit), + }; + Check( + _vk.AllocateMemory(_device, &memoryInfo, null, out memory), + "vkAllocateMemory(depth feedback snapshot)"); + Check( + _vk.BindImageMemory(_device, image, memory, 0), + "vkBindImageMemory(depth feedback snapshot)"); + var viewInfo = new ImageViewCreateInfo + { + SType = StructureType.ImageViewCreateInfo, + Image = image, + ViewType = ImageViewType.Type2D, + Format = DepthFormat, + Components = ToVkComponentMapping(texture.DstSelect), + SubresourceRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1), + }; + Check( + _vk.CreateImageView(_device, &viewInfo, null, out view), + "vkCreateImageView(depth feedback snapshot)"); + SetDebugName( + ObjectType.Image, + image.Handle, + $"SharpEmu depth feedback 0x{source.Address:X16} image"); + SetDebugName( + ObjectType.ImageView, + view.Handle, + $"SharpEmu depth feedback 0x{source.Address:X16} view"); + return new TextureResource + { + Address = texture.Address, + Image = image, + ImageMemory = memory, + View = view, + Width = source.Width, + Height = source.Height, + RowLength = source.Width, + DstSelect = texture.DstSelect, + OwnsStorage = true, + SamplerState = texture.Sampler, + DepthFeedbackSource = source, + }; + } + catch + { + if (view.Handle != 0) + { + _vk.DestroyImageView(_device, view, null); + } + if (image.Handle != 0) + { + _vk.DestroyImage(_device, image, null); + } + if (memory.Handle != 0) + { + _vk.FreeMemory(_device, memory, null); + } + throw; + } + } + private void DumpTextureUpload( GuestDrawTexture texture, byte[] pixels, @@ -4076,7 +7191,10 @@ internal static unsafe class VulkanVideoPresenter uint width, uint height) { - if (!_dumpTextures || + if (!string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DUMP_TEXTURES"), + "1", + StringComparison.Ordinal) || texture.IsFallback || texture.IsStorage || GetTextureBytesPerPixel(texture.Format) != 4 || @@ -4102,6 +7220,63 @@ internal static unsafe class VulkanVideoPresenter WriteRgbaBmp(path, pixels, rowBytes, visibleRowBytes, (int)width, (int)height); } + private void TraceTextureUploadContents( + GuestDrawTexture texture, + byte[] pixels, + uint rowLength, + uint width, + uint height, + Format format) + { + if (!_traceGuestImageAddressFilterEnabled || + !AddressListContains("SHARPEMU_TRACE_GUEST_IMAGE_ADDRS", texture.Address) || + !_tracedTextureUploadContents.Add( + (texture.Address, width, height, texture.Format))) + { + return; + } + + var bytesPerPixel = checked((uint)GetTextureBytesPerPixel(texture.Format)); + var nonzeroBytes = 0L; + ulong hash = 14695981039346656037UL; + foreach (var value in pixels) + { + nonzeroBytes += value == 0 ? 0 : 1; + hash = (hash ^ value) * 1099511628211UL; + } + + var centerOffset = checked( + ((int)(height / 2) * (int)rowLength + (int)(width / 2)) * + (int)bytesPerPixel); + var center = centerOffset + bytesPerPixel <= pixels.Length + ? Convert.ToHexString( + pixels.AsSpan(centerOffset, checked((int)bytesPerPixel))) + : "out-of-range"; + Console.Error.WriteLine( + "[LOADER][TRACE] " + + $"vk.texture_upload_contents addr=0x{texture.Address:X16} " + + $"size={width}x{height} row={rowLength} format={format} " + + $"guest_format={texture.Format} nonzero_bytes={nonzeroBytes}/{pixels.Length} " + + $"nonblack_pixels={CountNonblackPixels(pixels, format, bytesPerPixel)}/{(ulong)rowLength * height} " + + $"center={center} sample_unique={CountSampledUniquePixels(pixels, bytesPerPixel)} " + + $"hash=0x{hash:X16}"); + + var directory = + Environment.GetEnvironmentVariable("SHARPEMU_GUEST_IMAGE_DUMP_DIR"); + if (string.IsNullOrWhiteSpace(directory)) + { + return; + } + + Directory.CreateDirectory(directory); + var sequence = Interlocked.Increment(ref _guestImageDumpSequence); + var path = Path.Combine( + directory, + $"{sequence:D4}-texture-0x{texture.Address:X16}-{width}x{height}-" + + $"row{rowLength}-fmt{texture.Format}-{format}.rgba"); + File.WriteAllBytes(path, pixels); + } + private static void WriteRgbaBmp( string path, byte[] rgba, @@ -4201,11 +7376,6 @@ internal static unsafe class VulkanVideoPresenter private static ComponentMapping ToVkComponentMapping(uint dstSelect) { - if (dstSelect == 0) - { - dstSelect = 0xFAC; - } - return new ComponentMapping( ToVkComponentSwizzle(dstSelect & 0x7), ToVkComponentSwizzle((dstSelect >> 3) & 0x7), @@ -4242,45 +7412,374 @@ internal static unsafe class VulkanVideoPresenter private GlobalBufferResource CreateGlobalBufferResource( GuestMemoryBuffer guestBuffer) { - var buffer = CreateHostBuffer( - guestBuffer.Data, - BufferUsageFlags.StorageBufferBit, - out var memory); - var size = (ulong)Math.Max(guestBuffer.Data.Length, sizeof(uint)); + if (guestBuffer.BaseAddress == 0) + { + return CreateTransientGlobalBufferResource(guestBuffer); + } + + var size = (ulong)Math.Max(guestBuffer.Length, sizeof(uint)); + var endAddress = checked(guestBuffer.BaseAddress + size); + var allocation = _guestBufferAllocations + .Where(candidate => + candidate.BaseAddress <= guestBuffer.BaseAddress && + candidate.BaseAddress + candidate.Size >= endAddress) + .OrderByDescending(candidate => + string.Equals( + candidate.QueueName, + _activeGuestQueue.Name, + StringComparison.Ordinal)) + .ThenByDescending(candidate => + string.Equals( + candidate.QueueName, + SharedReadOnlyGuestBufferQueue, + StringComparison.Ordinal)) + .First(); + var guestOffset = guestBuffer.BaseAddress - allocation.BaseAddress; + var descriptorOffset = guestOffset & + ~(GuestStorageBufferOffsetAlignment - 1); + var byteBias = guestOffset - descriptorOffset; + if (descriptorOffset % _minStorageBufferOffsetAlignment != 0) + { + throw new InvalidOperationException( + $"guest buffer alias offset 0x{descriptorOffset:X} is not aligned to Vulkan's " + + $"minStorageBufferOffsetAlignment={_minStorageBufferOffsetAlignment}"); + } + + var expectedBias = guestBuffer.BaseAddress & + (GuestStorageBufferOffsetAlignment - 1); + if (byteBias != expectedBias) + { + throw new InvalidOperationException( + $"guest buffer allocation base 0x{allocation.BaseAddress:X16} " + + $"does not satisfy alias alignment " + + $"{GuestStorageBufferOffsetAlignment}"); + } + + var source = guestBuffer.Data.AsSpan(0, guestBuffer.Length); + var shadow = allocation.Shadow.AsSpan(checked((int)guestOffset), guestBuffer.Length); + if (!source.SequenceEqual(shadow)) + { + // HOST_COHERENT does not permit racing a mapped CPU write with + // an in-flight shader access. Retire prior users, publish their + // dirty ranges to guest memory, then upload the current guest + // bytes (which may be newer than the parser's captured array). + if (string.Equals( + allocation.QueueName, + SharedReadOnlyGuestBufferQueue, + StringComparison.Ordinal)) + { + // Shared read-only storage can still be referenced by any + // logical guest queue, so replacing its mapped contents + // requires retiring every prior reader. + WaitForAllGuestSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(); + } + else + { + // Writable aliases are private to one logical guest queue. + // Retiring unrelated queues here recreates the global FIFO + // and turns routine buffer refreshes into queue-wide stalls. + WaitForActiveGuestQueueSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(_activeGuestQueue.Name); + } + var live = new byte[guestBuffer.Length]; + if (_guestMemory?.TryRead(guestBuffer.BaseAddress, live) == true) + { + source = live; + } + + source.CopyTo(new Span( + (void*)(allocation.Mapped + checked((nint)guestOffset)), + source.Length)); + source.CopyTo(shadow); + } if (ShouldTraceVulkanResources() && - _tracedGlobalBuffers.Add((guestBuffer.BaseAddress, guestBuffer.Data.Length))) + _tracedGlobalBuffers.Add((guestBuffer.BaseAddress, guestBuffer.Length))) { Console.Error.WriteLine( $"[LOADER][TRACE] vk.global_buffer base=0x{guestBuffer.BaseAddress:X16} " + - $"bytes={guestBuffer.Data.Length}"); + $"bytes={guestBuffer.Length}"); + } + if (_setDebugUtilsObjectName is not null) + { + SetDebugName( + ObjectType.Buffer, + allocation.Buffer.Handle, + $"SharpEmu global 0x{guestBuffer.BaseAddress:X16} {guestBuffer.Length}b"); + } + + if (guestBuffer.Pooled) + { + GuestDataPool.Return(guestBuffer.Data); } - SetDebugName( - ObjectType.Buffer, - buffer.Handle, - $"SharpEmu global 0x{guestBuffer.BaseAddress:X16} {guestBuffer.Data.Length}b"); return new GlobalBufferResource { + BaseAddress = guestBuffer.BaseAddress, + Writable = guestBuffer.Writable, + WriteBackToGuest = guestBuffer.WriteBackToGuest, + Buffer = allocation.Buffer, + Memory = allocation.Memory, + Mapped = allocation.Mapped + checked((nint)guestOffset), + Offset = descriptorOffset, + Size = checked((size + byteBias + 3) & ~3UL), + GuestOffset = guestOffset, + GuestSize = size, + Allocation = allocation, + }; + } + + private GlobalBufferResource CreateTransientGlobalBufferResource( + GuestMemoryBuffer guestBuffer) + { + var buffer = CreateHostBuffer( + guestBuffer.Data.AsSpan(0, guestBuffer.Length), + BufferUsageFlags.StorageBufferBit, + out var memory); + var allocation = _hostBufferAllocations[buffer.Handle]; + if (guestBuffer.Pooled) + { + GuestDataPool.Return(guestBuffer.Data); + } + + return new GlobalBufferResource + { + BaseAddress = 0, + Writable = false, + WriteBackToGuest = false, Buffer = buffer, Memory = memory, - Size = size, + Mapped = allocation.Mapped, + Offset = 0, + Size = (ulong)Math.Max(guestBuffer.Length, sizeof(uint)), + GuestOffset = 0, + GuestSize = (ulong)Math.Max(guestBuffer.Length, sizeof(uint)), }; } + private void PrepareGuestBufferAllocations( + IReadOnlyList buffers) + { + if (buffers.Count == 0) + { + return; + } + + var ranges = new List<(ulong Start, ulong End, bool Writable)>(buffers.Count); + foreach (var buffer in buffers) + { + if (buffer.BaseAddress == 0) + { + continue; + } + + var size = (ulong)Math.Max(buffer.Length, sizeof(uint)); + var alignedStart = buffer.BaseAddress & + ~(GuestStorageBufferOffsetAlignment - 1); + var paddedEnd = checked(buffer.BaseAddress + size + 3) & ~3UL; + ranges.Add(( + alignedStart, + paddedEnd, + buffer.Writable && buffer.WriteBackToGuest)); + } + + if (ranges.Count == 0) + { + return; + } + + ranges.Sort(static (left, right) => left.Start.CompareTo(right.Start)); + var merged = new List<(ulong Start, ulong End, bool Writable)>(ranges.Count); + foreach (var range in ranges) + { + if (merged.Count == 0 || range.Start > merged[^1].End) + { + merged.Add(range); + continue; + } + + var previous = merged[^1]; + merged[^1] = ( + previous.Start, + Math.Max(previous.End, range.End), + previous.Writable || range.Writable); + } + + foreach (var range in merged) + { + EnsureGuestBufferAllocation( + range.Start, + range.End, + range.Writable + ? _activeGuestQueue.Name + : SharedReadOnlyGuestBufferQueue); + } + } + + private void EnsureGuestBufferAllocation( + ulong requestedStart, + ulong requestedEnd, + string queueName) + { + var start = requestedStart; + var end = requestedEnd; + List overlaps; + do + { + overlaps = _guestBufferAllocations + .Where(allocation => + string.Equals( + allocation.QueueName, + queueName, + StringComparison.Ordinal) && + allocation.BaseAddress < end && + start < allocation.BaseAddress + allocation.Size) + .ToList(); + var expandedStart = overlaps.Aggregate( + start, + static (value, allocation) => Math.Min(value, allocation.BaseAddress)); + var expandedEnd = overlaps.Aggregate( + end, + static (value, allocation) => + Math.Max(value, allocation.BaseAddress + allocation.Size)); + if (expandedStart == start && expandedEnd == end) + { + break; + } + + start = expandedStart; + end = expandedEnd; + } + while (true); + + if (overlaps.Count == 1 && + overlaps[0].BaseAddress <= requestedStart && + overlaps[0].BaseAddress + overlaps[0].Size >= requestedEnd) + { + return; + } + + if (overlaps.Count > 0) + { + // Growing/merging an aliased allocation is rare. Synchronize + // only this structural transition so no in-flight descriptor + // can observe storage being replaced underneath it. + WaitForAllGuestSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(); + } + + var replacement = CreateGuestBufferAllocation(start, end, queueName); + foreach (var overlap in overlaps) + { + _guestBufferAllocations.Remove(overlap); + DestroyGuestBufferAllocation(overlap); + } + + _guestBufferAllocations.Add(replacement); + _guestBufferAllocations.Sort(static (left, right) => + { + var queueOrder = string.CompareOrdinal(left.QueueName, right.QueueName); + return queueOrder != 0 + ? queueOrder + : left.BaseAddress.CompareTo(right.BaseAddress); + }); + TraceVulkanShader( + $"vk.guest_buffer_allocation queue={queueName} " + + $"base=0x{start:X16} bytes={replacement.Size} " + + $"merged={overlaps.Count}"); + } + + private GuestBufferAllocation CreateGuestBufferAllocation( + ulong start, + ulong end, + string queueName) + { + var size = checked(end - start); + if (size == 0 || size > int.MaxValue) + { + throw new InvalidOperationException( + $"guest buffer allocation is outside the supported host span: " + + $"base=0x{start:X16} bytes={size}"); + } + + var buffer = CreateBuffer( + size, + BufferUsageFlags.StorageBufferBit, + MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, + out var memory); + void* mapped; + Check(_vk.MapMemory(_device, memory, 0, size, 0, &mapped), "vkMapMemory(guest buffer)"); + var shadow = new byte[checked((int)size)]; + _ = _guestMemory?.TryRead(start, shadow); + shadow.CopyTo(new Span(mapped, shadow.Length)); + SetDebugName( + ObjectType.Buffer, + buffer.Handle, + $"SharpEmu guest VA 0x{start:X16}-0x{end:X16}"); + return new GuestBufferAllocation + { + QueueName = queueName, + BaseAddress = start, + Size = size, + Buffer = buffer, + Memory = memory, + Mapped = (nint)mapped, + Shadow = shadow, + }; + } + + private void DestroyGuestBufferAllocation(GuestBufferAllocation allocation) + { + if (allocation.Mapped != 0) + { + _vk.UnmapMemory(_device, allocation.Memory); + } + + _vk.DestroyBuffer(_device, allocation.Buffer, null); + _vk.FreeMemory(_device, allocation.Memory, null); + } + private VertexBufferResource CreateVertexBufferResource( GuestVertexBuffer guestBuffer) { + ReadOnlySpan source = guestBuffer.Data.AsSpan(0, guestBuffer.Length); + byte[]? forcedVertexColors = null; + if (Environment.GetEnvironmentVariable("SHARPEMU_FORCE_TITLE_VERTEX_COLOR_WHITE") == "1" && + guestBuffer.Location == 0 && + guestBuffer.ComponentCount == 4 && + guestBuffer.DataFormat == 10 && + guestBuffer.NumberFormat == 0 && + guestBuffer.Stride == 16 && + guestBuffer.OffsetBytes == 12 && + guestBuffer.Length == 67568) + { + forcedVertexColors = source.ToArray(); + for (var offset = 12; offset + 3 < forcedVertexColors.Length; offset += 16) + { + forcedVertexColors[offset] = 0xFF; + forcedVertexColors[offset + 1] = 0xFF; + forcedVertexColors[offset + 2] = 0xFF; + } + + source = forcedVertexColors; + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.vertex_force_title_color_white " + + $"base=0x{guestBuffer.BaseAddress:X16} bytes={guestBuffer.Length}"); + } var buffer = CreateHostBuffer( - guestBuffer.Data, + source, BufferUsageFlags.VertexBufferBit, out var memory); - var size = (ulong)Math.Max(guestBuffer.Data.Length, sizeof(uint)); - SetDebugName( - ObjectType.Buffer, - buffer.Handle, - $"SharpEmu vertex loc{guestBuffer.Location} " + - $"0x{guestBuffer.BaseAddress:X16} {guestBuffer.Data.Length}b"); + var size = (ulong)Math.Max(guestBuffer.Length, sizeof(uint)); + if (_setDebugUtilsObjectName is not null) + { + SetDebugName( + ObjectType.Buffer, + buffer.Handle, + $"SharpEmu vertex loc{guestBuffer.Location} " + + $"0x{guestBuffer.BaseAddress:X16} {guestBuffer.Length}b"); + } if (_tracedVertexBufferCount++ < 64) { TraceVulkanShader( @@ -4288,13 +7787,29 @@ internal static unsafe class VulkanVideoPresenter $"base=0x{guestBuffer.BaseAddress:X16} stride={guestBuffer.Stride} " + $"offset={guestBuffer.OffsetBytes} comps={guestBuffer.ComponentCount} " + $"fmt={guestBuffer.DataFormat}/num={guestBuffer.NumberFormat} " + - $"bytes={guestBuffer.Data.Length}"); + $"bytes={guestBuffer.Length}"); } + return CreateVertexBufferResource( + buffer, + memory, + size, + guestBuffer, + ownsBuffer: true); + } + + private static VertexBufferResource CreateVertexBufferResource( + VkBuffer buffer, + DeviceMemory memory, + ulong size, + GuestVertexBuffer guestBuffer, + bool ownsBuffer) + { return new VertexBufferResource { Buffer = buffer, Memory = memory, + OwnsBuffer = ownsBuffer, Size = size, Location = guestBuffer.Location, ComponentCount = guestBuffer.ComponentCount, @@ -4305,6 +7820,22 @@ internal static unsafe class VulkanVideoPresenter }; } + private static VertexBufferResource CreateVertexBufferAlias( + VertexBufferResource shared, + GuestVertexBuffer guestBuffer) => new() + { + Buffer = shared.Buffer, + Memory = shared.Memory, + OwnsBuffer = false, + Size = shared.Size, + Location = guestBuffer.Location, + ComponentCount = guestBuffer.ComponentCount, + DataFormat = guestBuffer.DataFormat, + NumberFormat = guestBuffer.NumberFormat, + Stride = guestBuffer.Stride, + OffsetBytes = guestBuffer.OffsetBytes, + }; + private VkBuffer CreateHostBuffer( ReadOnlySpan data, BufferUsageFlags usage, @@ -4331,27 +7862,29 @@ internal static unsafe class VulkanVideoPresenter usage, MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, out var allocatedMemory); - allocation = new HostBufferAllocation(buffer, allocatedMemory, key); + // Persistently mapped: map/unmap per draw was a measurable + // share of the per-draw fixed cost, and HOST_COHERENT memory + // may legally stay mapped for its lifetime. + void* persistentMapping; + Check( + _vk.MapMemory(_device, allocatedMemory, 0, capacity, 0, &persistentMapping), + "vkMapMemory(host persistent)"); + allocation = new HostBufferAllocation( + buffer, + allocatedMemory, + key, + (nint)persistentMapping); _hostBufferAllocations.Add(buffer.Handle, allocation); } memory = allocation.Memory; - void* mapped; - Check(_vk.MapMemory(_device, memory, 0, size, 0, &mapped), "vkMapMemory(host)"); - try + fixed (byte* source = data) { - fixed (byte* source = data) - { - System.Buffer.MemoryCopy( - source, - mapped, - checked((long)size), - data.Length); - } - } - finally - { - _vk.UnmapMemory(_device, memory); + System.Buffer.MemoryCopy( + source, + (void*)allocation.Mapped, + checked((long)allocation.Key.Capacity), + data.Length); } return allocation.Buffer; @@ -4390,6 +7923,13 @@ internal static unsafe class VulkanVideoPresenter _ => PrimitiveTopology.TriangleList, }; + // Strip and fan topologies are the ones for which a restart index + // splits primitives; list topologies never restart. + private static bool RequiresPrimitiveRestart(PrimitiveTopology topology) => + topology is PrimitiveTopology.LineStrip + or PrimitiveTopology.TriangleStrip + or PrimitiveTopology.TriangleFan; + private static Format ToVkVertexFormat( uint dataFormat, uint numberFormat, @@ -4429,12 +7969,15 @@ internal static unsafe class VulkanVideoPresenter (8, 3) => Format.A2B10G10R10SscaledPack32, (8, 4) => Format.A2B10G10R10UintPack32, (8, 5) => Format.A2B10G10R10SintPack32, - (9, 0) => Format.A2R10G10B10UnormPack32, - (9, 1) => Format.A2R10G10B10SNormPack32, - (9, 2) => Format.A2R10G10B10UscaledPack32, - (9, 3) => Format.A2R10G10B10SscaledPack32, - (9, 4) => Format.A2R10G10B10UintPack32, - (9, 5) => Format.A2R10G10B10SintPack32, + // RDNA COLOR_2_10_10_10 stores component 0 (R) in bits + // 0..9 and A in 30..31. Vulkan names that exact bit layout + // A2B10G10R10_PACK32 (the packed name is MSB-to-LSB). + (9, 0) => Format.A2B10G10R10UnormPack32, + (9, 1) => Format.A2B10G10R10SNormPack32, + (9, 2) => Format.A2B10G10R10UscaledPack32, + (9, 3) => Format.A2B10G10R10SscaledPack32, + (9, 4) => Format.A2B10G10R10UintPack32, + (9, 5) => Format.A2B10G10R10SintPack32, (10, 0) => Format.R8G8B8A8Unorm, (10, 1) => Format.R8G8B8A8SNorm, (10, 2) => Format.R8G8B8A8Uscaled, @@ -4702,7 +8245,7 @@ internal static unsafe class VulkanVideoPresenter private static byte[] CreateFallbackTexturePixels(uint format, uint width, uint height, ulong expectedSize) { - if (format is 9 or 10 or 56 or 62 or 64) + if (format is 9 or 10) { return CreateBlackFrame(width, height); } @@ -4726,15 +8269,6 @@ internal static unsafe class VulkanVideoPresenter 12 => 8UL, 13 => 12UL, 14 => 16UL, - 20 => 4UL, - 22 => 8UL, - 29 => 4UL, - 36 => 1UL, - 49 => 1UL, - 56 => 4UL, - 62 => 4UL, - 64 => 4UL, - 71 => 8UL, _ => 4UL, }; @@ -4742,8 +8276,10 @@ internal static unsafe class VulkanVideoPresenter { var blockBytes = format switch { - 169 or 170 => 8UL, - 171 or 172 or 173 or 174 or 175 or 176 or + // BC1 (169/170) and BC4 (175/176) are 8 bytes per 4x4 block; + // BC2/BC3/BC5/BC6H/BC7 are 16 bytes per block. + 169 or 170 or 175 or 176 => 8UL, + 171 or 172 or 173 or 174 or 177 or 178 or 179 or 180 or 181 or 182 => 16UL, _ => 0UL, }; @@ -4761,20 +8297,26 @@ internal static unsafe class VulkanVideoPresenter private static Format GetTextureFormat(uint format, uint numberType) => (format, numberType) switch { - (9, _) => Format.A2R10G10B10UnormPack32, - (GuestFormatR32Uint, _) => Format.R32Uint, - (GuestFormatR32Sint, _) => Format.R32Sint, - (GuestFormatR32Sfloat, _) => Format.R32Sfloat, - (GuestFormatR16G16Uint, _) => Format.R16G16Uint, - (GuestFormatR16G16Sint, _) => Format.R16G16Sint, - (GuestFormatR16G16Sfloat, _) => Format.R16G16Sfloat, - (GuestFormatR8G8B8A8Uint, _) => Format.R8G8B8A8Uint, - (GuestFormatR8G8B8A8Sint, _) => Format.R8G8B8A8Sint, - (GuestFormatR16G16B16A16Uint, _) => Format.R16G16B16A16Uint, - (GuestFormatR16G16B16A16Sint, _) => Format.R16G16B16A16Sint, + (9, _) => Format.A2B10G10R10UnormPack32, (1, 0) => Format.R8Unorm, + (1, 1) => Format.R8SNorm, + (1, 2) => Format.R8Uscaled, + (1, 3) => Format.R8Sscaled, + (1, 4) => Format.R8Uint, + (1, 5) => Format.R8Sint, (2, 7) => Format.R16Sfloat, + (2, 0) => Format.R16Unorm, + (2, 1) => Format.R16SNorm, + (2, 2) => Format.R16Uscaled, + (2, 3) => Format.R16Sscaled, + (2, 4) => Format.R16Uint, + (2, 5) => Format.R16Sint, (3, 0) => Format.R8G8Unorm, + (3, 1) => Format.R8G8SNorm, + (3, 2) => Format.R8G8Uscaled, + (3, 3) => Format.R8G8Sscaled, + (3, 4) => Format.R8G8Uint, + (3, 5) => Format.R8G8Sint, (4, 4) => Format.R32Uint, (4, 5) => Format.R32Sint, (4, 7) => Format.R32Sfloat, @@ -4783,9 +8325,19 @@ internal static unsafe class VulkanVideoPresenter (5, 5) => Format.R16G16Sint, (5, 7) => Format.R16G16Sfloat, (6, 7) => Format.B10G11R11UfloatPack32, + (7, 7) => Format.B10G11R11UfloatPack32, + (8, 0) => Format.A2B10G10R10UnormPack32, + (8, 1) => Format.A2B10G10R10SNormPack32, + (8, 2) => Format.A2B10G10R10UscaledPack32, + (8, 3) => Format.A2B10G10R10SscaledPack32, + (8, 4) => Format.A2B10G10R10UintPack32, + (8, 5) => Format.A2B10G10R10SintPack32, (10, 0) => Format.R8G8B8A8Unorm, (10, 4) => Format.R8G8B8A8Uint, (10, 5) => Format.R8G8B8A8Sint, + (10, 9) => Format.R8G8B8A8Srgb, + (1, 9) => Format.R8Srgb, + (3, 9) => Format.R8G8Srgb, (11, 4) => Format.R32G32Uint, (11, 5) => Format.R32G32Sint, (11, 7) => Format.R32G32Sfloat, @@ -4799,30 +8351,68 @@ internal static unsafe class VulkanVideoPresenter (14, 4) => Format.R32G32B32A32Uint, (14, 5) => Format.R32G32B32A32Sint, (14, 7) => Format.R32G32B32A32Sfloat, - (20, _) => Format.R32Uint, - (4, _) => Format.R32Sfloat, - (5, _) => Format.R16G16Sfloat, - (7, _) => Format.B10G11R11UfloatPack32, - (14, _) => Format.R32G32B32A32Sfloat, - (22, _) => Format.R16G16B16A16Sfloat, - (29, _) => Format.R32Sfloat, - (36, _) => Format.R8Unorm, - (49, _) => Format.R8Uint, - (56, _) => Format.R8G8B8A8Unorm, - (62, _) => Format.R8G8B8A8Unorm, - (64, _) => Format.R8G8B8A8Unorm, - (71, _) => Format.R16G16B16A16Sfloat, - (75, _) => Format.R32G32Sfloat, + (16, 0) => Format.B5G6R5UnormPack16, + (17, 0) => Format.R5G5B5A1UnormPack16, + (19, 0) => Format.R4G4B4A4UnormPack16, + (34, 7) => Format.E5B9G9R9UfloatPack32, (169, _) => Format.BC1RgbaUnormBlock, (170, _) => Format.BC1RgbaSrgbBlock, + (171, _) => Format.BC2UnormBlock, + (172, _) => Format.BC2SrgbBlock, + (173, _) => Format.BC3UnormBlock, + (174, _) => Format.BC3SrgbBlock, + (175, 1) => Format.BC4SNormBlock, + (175, _) => Format.BC4UnormBlock, + (176, _) => Format.BC4SNormBlock, + (177, 1) => Format.BC5SNormBlock, + (177, _) => Format.BC5UnormBlock, + (178, _) => Format.BC5SNormBlock, + (179, _) => Format.BC6HUfloatBlock, + (180, _) => Format.BC6HSfloatBlock, (181, _) => Format.BC7UnormBlock, (182, _) => Format.BC7SrgbBlock, _ => Format.R8G8B8A8Unorm, }; + private static Format GetRenderTargetFormat(uint format, uint numberType) => + (format, numberType) switch + { + (4, 4) => Format.R32Uint, + (4, 5) => Format.R32Sint, + (4, 7) => Format.R32Sfloat, + (5, 4) => Format.R16G16Uint, + (5, 5) => Format.R16G16Sint, + (5, 7) => Format.R16G16Sfloat, + (6, 7) => Format.B10G11R11UfloatPack32, + (7, 7) => Format.B10G11R11UfloatPack32, + (9, _) => Format.A2B10G10R10UnormPack32, + (10, 9) => Format.R8G8B8A8Srgb, + (10, 4) => Format.R8G8B8A8Uint, + (10, 5) => Format.R8G8B8A8Sint, + (10, _) => Format.R8G8B8A8Unorm, + (11, 7) => Format.R32G32Sfloat, + (12, 4) => Format.R16G16B16A16Uint, + (12, 5) => Format.R16G16B16A16Sint, + (12, 7) => Format.R16G16B16A16Sfloat, + (13, 7) => Format.R32G32B32A32Sfloat, + (14, 7) => Format.R32G32B32A32Sfloat, + (_, 0) => GetTextureFormat(format, numberType), + (_, 9) => GetTextureFormat(format, numberType), + _ => Format.Undefined, + }; private static bool IsBlockCompressedFormat(Format format) => format is Format.BC1RgbaUnormBlock or Format.BC1RgbaSrgbBlock or + Format.BC2UnormBlock or + Format.BC2SrgbBlock or + Format.BC3UnormBlock or + Format.BC3SrgbBlock or + Format.BC4UnormBlock or + Format.BC4SNormBlock or + Format.BC5UnormBlock or + Format.BC5SNormBlock or + Format.BC6HUfloatBlock or + Format.BC6HSfloatBlock or Format.BC7UnormBlock or Format.BC7SrgbBlock; @@ -4880,15 +8470,43 @@ internal static unsafe class VulkanVideoPresenter } private const uint MaxComputeZSlicesPerSubmission = 8; + // An indirect guest dispatch above this size is not credible frame work + // (at the minimum 64-thread group used by the captured title this is + // already over one billion invocations). Treat it as poisoned + // indirect-command data and quarantine it instead of feeding a host + // API a multi-billion-workgroup command. This is validation, not a + // clamp: the raw dimensions remain visible in the trace so the + // producer can be fixed without changing the guest value. + private const ulong MaxCredibleGuestWorkgroupsPerDispatch = 16UL * 1024 * 1024; private void ExecuteComputeDispatch(VulkanComputeGuestDispatch work) { + var perfStart = Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _perfDrawCount); + PerfOverlay.RecordDraw(); + try + { + ExecuteComputeDispatchCore(work); + } + finally + { + Interlocked.Add( + ref _perfDrawTicks, + Stopwatch.GetTimestamp() - perfStart); + } + } + + private void ExecuteComputeDispatchCore(VulkanComputeGuestDispatch work) + { + FlushBatchedGuestCommands(); if (_deviceLost) { return; } - if (AddressListContains("SHARPEMU_SKIP_COMPUTE_CS", work.ShaderAddress)) + if (_skipAllCompute || + AddressListContains("SHARPEMU_SKIP_COMPUTE_CS", work.ShaderAddress) || + (_skipTallComputeZ > 0 && work.GroupCountZ >= _skipTallComputeZ)) { TraceVulkanShader( $"vk.compute_skip cs=0x{work.ShaderAddress:X16} " + @@ -4897,9 +8515,16 @@ internal static unsafe class VulkanVideoPresenter return; } + if (!TryValidateComputeDispatch(work, out var validationError)) + { + LogRejectedComputeDispatch(work, validationError); + return; + } + TranslatedDrawResources? resources = null; CommandBuffer commandBuffer = default; var submitted = false; + var chunksSubmitted = 0; try { EnsureGuestSubmissionCapacity(); @@ -4908,6 +8533,12 @@ internal static unsafe class VulkanVideoPresenter var batchCount = Math.Max( 1u, (uint)Math.Ceiling(work.GroupCountZ / (double)MaxComputeZSlicesPerSubmission)); + var threadLimits = stackalloc uint[3] + { + work.ThreadCountX, + work.ThreadCountY, + work.ThreadCountZ, + }; for (var batchIndex = 0u; batchIndex < batchCount; batchIndex++) { @@ -4916,6 +8547,14 @@ internal static unsafe class VulkanVideoPresenter var isFirstBatch = batchIndex == 0; var isLastBatch = batchIndex == batchCount - 1; + if (!isFirstBatch) + { + // Each chunk is its own queue submission; without + // this the in-flight submission cap only applies to + // the first chunk of a tall dispatch. + EnsureGuestSubmissionCapacity(); + } + commandBuffer = AllocateGuestCommandBuffer(); _commandBuffer = commandBuffer; var beginInfo = new CommandBufferBeginInfo @@ -4930,9 +8569,36 @@ internal static unsafe class VulkanVideoPresenter BeginDebugLabel(_commandBuffer, resources.DebugName); if (isFirstBatch) { + RecordGlobalBufferVisibilityBarrier( + _commandBuffer, + resources, + PipelineStageFlags.ComputeShaderBit); RecordTextureUploads(resources, PipelineStageFlags.ComputeShaderBit); RecordStorageImagesForWrite(resources, PipelineStageFlags.ComputeShaderBit); } + else + { + // Chunks are submitted without CPU waits; this + // barrier orders them against the previous chunk's + // shader writes on the same queue. + var chunkBarrier = new MemoryBarrier + { + SType = StructureType.MemoryBarrier, + SrcAccessMask = AccessFlags.ShaderWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.ComputeShaderBit, + PipelineStageFlags.ComputeShaderBit, + 0, + 1, + &chunkBarrier, + 0, + null, + 0, + null); + } _vk.CmdBindPipeline( _commandBuffer, @@ -4952,6 +8618,14 @@ internal static unsafe class VulkanVideoPresenter null); } + _vk.CmdPushConstants( + _commandBuffer, + resources.PipelineLayout, + ShaderStageFlags.ComputeBit, + 0, + 3 * sizeof(uint), + threadLimits); + RecordChunkedComputeDispatch(_commandBuffer, work, zStart, zCount); if (isLastBatch) @@ -4969,22 +8643,37 @@ internal static unsafe class VulkanVideoPresenter { SubmitGuestCommandBuffer( commandBuffer, - resources, - GetTraceImages(resources)); + [resources], + GetTraceImages(resources, shaderAddress: work.ShaderAddress)); submitted = true; } else { - SubmitGuestCommandBufferAndWait(commandBuffer); + SubmitGuestCommandBuffer( + commandBuffer, + [], + [], + referencedResources: [resources]); + chunksSubmitted++; commandBuffer = default; } } MarkSampledImagesInitialized(resources); MarkStorageImagesInitialized(resources, traceContents: false); + if (work.WritesGlobalMemory) + { + // The CPU submit thread may immediately consume an indirect + // argument written by this dispatch. Wait for the specific + // guest fences and publish only dirty ranges; a queue-wide + // idle unnecessarily serialized presentation work too. + WaitForActiveGuestQueueSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(_activeGuestQueue.Name); + } TraceVulkanShader( $"vk.compute_dispatch groups={work.GroupCountX}x" + $"{work.GroupCountY}x{work.GroupCountZ} " + + $"base={work.BaseGroupX}x{work.BaseGroupY}x{work.BaseGroupZ} " + $"textures={work.Textures.Count} cs=0x{work.ShaderAddress:X16} " + $"batches={batchCount}"); } @@ -5013,7 +8702,444 @@ internal static unsafe class VulkanVideoPresenter if (!submitted && resources is not null) { - DestroyTranslatedDrawResources(resources); + if (chunksSubmitted > 0) + { + // Earlier chunks were submitted with empty resource + // lists and may still execute against these + // pipelines/images; destroy only after every + // submission issued so far has completed. + _deferredResourceDestroys.Enqueue((resources, _submitTimeline)); + } + else + { + DestroyTranslatedDrawResources(resources); + } + } + } + } + + private bool TryValidateComputeDispatch( + VulkanComputeGuestDispatch work, + out string error) + { + if (work.LocalSizeX == 0 || work.LocalSizeY == 0 || work.LocalSizeZ == 0) + { + error = "zero-local-size"; + return false; + } + + if (work.LocalSizeX > _maxComputeWorkGroupSizeX || + work.LocalSizeY > _maxComputeWorkGroupSizeY || + work.LocalSizeZ > _maxComputeWorkGroupSizeZ) + { + error = + $"local-size-exceeds-device({work.LocalSizeX}x{work.LocalSizeY}x{work.LocalSizeZ}>" + + $"{_maxComputeWorkGroupSizeX}x{_maxComputeWorkGroupSizeY}x{_maxComputeWorkGroupSizeZ})"; + return false; + } + + var localInvocations = + (ulong)work.LocalSizeX * work.LocalSizeY * work.LocalSizeZ; + if (localInvocations > _maxComputeWorkGroupInvocations) + { + error = + $"local-invocations-exceed-device({localInvocations}>" + + $"{_maxComputeWorkGroupInvocations})"; + return false; + } + + if ((ulong)work.BaseGroupX + work.GroupCountX > _maxComputeWorkGroupCountX || + (ulong)work.BaseGroupY + work.GroupCountY > _maxComputeWorkGroupCountY || + (ulong)work.BaseGroupZ + work.GroupCountZ > _maxComputeWorkGroupCountZ) + { + error = + $"group-range-exceeds-device(base={work.BaseGroupX}x{work.BaseGroupY}x{work.BaseGroupZ}," + + $"count={work.GroupCountX}x{work.GroupCountY}x{work.GroupCountZ}," + + $"limit={_maxComputeWorkGroupCountX}x{_maxComputeWorkGroupCountY}x" + + $"{_maxComputeWorkGroupCountZ})"; + return false; + } + + if (work.IsIndirect) + { + ulong totalWorkgroups; + try + { + totalWorkgroups = checked( + (ulong)work.GroupCountX * work.GroupCountY * work.GroupCountZ); + } + catch (OverflowException) + { + error = "indirect-workgroup-count-overflow"; + return false; + } + + if (totalWorkgroups > MaxCredibleGuestWorkgroupsPerDispatch) + { + error = + $"poisoned-indirect-workgroup-count({totalWorkgroups}>" + + $"{MaxCredibleGuestWorkgroupsPerDispatch})"; + return false; + } + } + + error = string.Empty; + return true; + } + + private void LogRejectedComputeDispatch( + VulkanComputeGuestDispatch work, + string reason) + { + if (_rejectedComputeDispatches.Count >= 256 || + !_rejectedComputeDispatches.Add( + (work.ShaderAddress, + work.GroupCountX, + work.GroupCountY, + work.GroupCountZ, + reason))) + { + return; + } + + Console.Error.WriteLine( + $"[LOADER][WARN] vk.compute_reject cs=0x{work.ShaderAddress:X16} " + + $"source={(work.IsIndirect ? "indirect" : "direct")} " + + $"groups={work.GroupCountX}x{work.GroupCountY}x{work.GroupCountZ} " + + $"base={work.BaseGroupX}x{work.BaseGroupY}x{work.BaseGroupZ} " + + $"local={work.LocalSizeX}x{work.LocalSizeY}x{work.LocalSizeZ} " + + $"reason={reason}"); + } + + private void RecordGlobalBufferVisibilityBarrier( + CommandBuffer commandBuffer, + TranslatedDrawResources resources, + PipelineStageFlags destinationStages) + { + if (resources.GlobalMemoryBuffers.Length == 0) + { + return; + } + + // Queue submission order alone is not a shader-memory dependency. + // This makes stores through any aliased guest view available to + // later vertex/fragment/compute reads and writes on the same queue. + var barrier = new MemoryBarrier + { + SType = StructureType.MemoryBarrier, + SrcAccessMask = AccessFlags.ShaderWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit, + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.AllCommandsBit, + destinationStages, + 0, + 1, + &barrier, + 0, + null, + 0, + null); + } + + private static void MarkGuestBufferDirty( + GuestBufferAllocation allocation, + ulong offset, + ulong length) + { + if (length == 0) + { + return; + } + + var start = offset; + var end = checked(offset + length); + for (var index = allocation.DirtyRanges.Count - 1; index >= 0; index--) + { + var existing = allocation.DirtyRanges[index]; + var existingEnd = existing.Offset + existing.Length; + if (end < existing.Offset || existingEnd < start) + { + continue; + } + + start = Math.Min(start, existing.Offset); + end = Math.Max(end, existingEnd); + allocation.DirtyRanges.RemoveAt(index); + } + + allocation.DirtyRanges.Add(new DirtyGuestBufferRange(start, end - start)); + } + + private void WriteBackAllDirtyGuestBuffers(string? queueName = null) + { + var memory = _guestMemory; + if (memory is null) + { + return; + } + + foreach (var allocation in _guestBufferAllocations) + { + if (queueName is not null && + !string.Equals(allocation.QueueName, queueName, StringComparison.Ordinal)) + { + continue; + } + + if (allocation.DirtyRanges.Count != 0 && + allocation.LastUseTimeline > _completedTimeline) + { + // A mapped HOST_COHERENT allocation still cannot be read + // by the CPU while a shader may be writing it. Callers + // normally retire the relevant fences first; keep this + // helper fail-closed if a future path forgets to do so. + TraceVulkanShader( + $"vk.global_writeback_deferred base=0x{allocation.BaseAddress:X16} " + + $"last_use={allocation.LastUseTimeline} completed={_completedTimeline}"); + continue; + } + + for (var index = allocation.DirtyRanges.Count - 1; index >= 0; index--) + { + var range = allocation.DirtyRanges[index]; + if (range.Length == 0 || range.Length > int.MaxValue) + { + continue; + } + + var mappedBytes = new ReadOnlySpan( + (void*)(allocation.Mapped + checked((nint)range.Offset)), + checked((int)range.Length)); + var shadowBytes = allocation.Shadow.AsSpan( + checked((int)range.Offset), + mappedBytes.Length); + var guestAddress = allocation.BaseAddress + range.Offset; + var changedBytes = 0UL; + var changedRuns = 0; + var changedPages = 0; + var writtenRuns = 0; + var writtenPages = 0; + var failedRuns = 0; + var unreadablePages = 0; + var fallbackWrites = 0; + var firstChangedOffset = -1; + allocation.DirtyRanges.RemoveAt(index); + + // A writable descriptor only identifies a potential write + // range. Publishing the entire mapped view would overwrite + // unrelated live CPU data with its old snapshot. Compare + // against the last synchronized image. For each changed + // page, start with current guest bytes and overlay only the + // shader changes before one bounded write. This preserves + // live CPU changes in unchanged bytes without degenerating + // into millions of writes for alternating output patterns. + const int pageSize = 4096; + const int unreadableMergeGap = 16; + var livePageBuffer = GuestDataPool.Rent(pageSize); + var pageRuns = new List<(int Start, int Length)>(64); + try + { + for (var pageStart = 0; + pageStart < mappedBytes.Length; + pageStart += pageSize) + { + var pageEnd = Math.Min(pageStart + pageSize, mappedBytes.Length); + pageRuns.Clear(); + var cursor = pageStart; + while (cursor < pageEnd) + { + while (cursor < pageEnd && + mappedBytes[cursor] == shadowBytes[cursor]) + { + cursor++; + } + + if (cursor == pageEnd) + { + break; + } + + var runStart = cursor; + while (cursor < pageEnd && + mappedBytes[cursor] != shadowBytes[cursor]) + { + cursor++; + } + + var runLength = cursor - runStart; + pageRuns.Add((runStart, runLength)); + changedRuns++; + changedBytes += (ulong)runLength; + if (firstChangedOffset < 0) + { + firstChangedOffset = runStart; + } + } + + if (pageRuns.Count == 0) + { + continue; + } + + changedPages++; + var pageLength = pageEnd - pageStart; + var livePage = livePageBuffer.AsSpan(0, pageLength); + if (memory.TryRead(guestAddress + (ulong)pageStart, livePage)) + { + foreach (var run in pageRuns) + { + mappedBytes.Slice(run.Start, run.Length).CopyTo( + livePage.Slice(run.Start - pageStart, run.Length)); + } + + if (memory.TryWrite(guestAddress + (ulong)pageStart, livePage)) + { + foreach (var run in pageRuns) + { + mappedBytes.Slice(run.Start, run.Length).CopyTo( + shadowBytes.Slice(run.Start, run.Length)); + } + + writtenPages++; + writtenRuns += pageRuns.Count; + continue; + } + + foreach (var run in pageRuns) + { + failedRuns++; + MarkGuestBufferDirty( + allocation, + range.Offset + (ulong)run.Start, + (ulong)run.Length); + } + + continue; + } + + // A partial/unreadable edge cannot be safely + // reconstructed as a page. Fall back to bounded + // changed spans, coalescing only tiny gaps. + unreadablePages++; + for (var runIndex = 0; runIndex < pageRuns.Count; runIndex++) + { + var firstRunIndex = runIndex; + var mergedStart = pageRuns[runIndex].Start; + var mergedEnd = mergedStart + pageRuns[runIndex].Length; + while (runIndex + 1 < pageRuns.Count && + pageRuns[runIndex + 1].Start - mergedEnd <= + unreadableMergeGap) + { + runIndex++; + mergedEnd = pageRuns[runIndex].Start + + pageRuns[runIndex].Length; + } + + var lastRunIndex = runIndex; + var mergedLength = mergedEnd - mergedStart; + var mergedLive = livePageBuffer.AsSpan(0, mergedLength); + if (memory.TryRead( + guestAddress + (ulong)mergedStart, + mergedLive)) + { + for (var overlayIndex = firstRunIndex; + overlayIndex <= lastRunIndex; + overlayIndex++) + { + var run = pageRuns[overlayIndex]; + mappedBytes.Slice(run.Start, run.Length).CopyTo( + mergedLive.Slice( + run.Start - mergedStart, + run.Length)); + } + + fallbackWrites++; + if (memory.TryWrite( + guestAddress + (ulong)mergedStart, + mergedLive)) + { + for (var overlayIndex = firstRunIndex; + overlayIndex <= lastRunIndex; + overlayIndex++) + { + var run = pageRuns[overlayIndex]; + mappedBytes.Slice(run.Start, run.Length).CopyTo( + shadowBytes.Slice(run.Start, run.Length)); + } + + writtenRuns += lastRunIndex - firstRunIndex + 1; + continue; + } + } + + // Even the merged span crosses an unreadable + // edge. Exact changed runs remain safe because + // they never carry stale gap bytes. + for (var exactIndex = firstRunIndex; + exactIndex <= lastRunIndex; + exactIndex++) + { + var run = pageRuns[exactIndex]; + var changed = mappedBytes.Slice(run.Start, run.Length); + fallbackWrites++; + if (memory.TryWrite( + guestAddress + (ulong)run.Start, + changed)) + { + changed.CopyTo(shadowBytes.Slice( + run.Start, + run.Length)); + writtenRuns++; + } + else + { + failedRuns++; + MarkGuestBufferDirty( + allocation, + range.Offset + (ulong)run.Start, + (ulong)run.Length); + } + } + } + } + } + finally + { + GuestDataPool.Return(livePageBuffer); + } + + var probe = mappedBytes[..Math.Min(mappedBytes.Length, 256)]; + var nonzero = 0; + foreach (var value in probe) + { + nonzero += value == 0 ? 0 : 1; + } + + var firstForRange = _tracedGlobalWritebacks.Count < 256 && + _tracedGlobalWritebacks.Add((guestAddress, range.Length)); + var traceSmallMutation = range.Length <= 4096 && + _tracedSmallGlobalWritebackEvents++ < 1024; + var traceLargeMutation = range.Length >= 1024 * 1024 && + _tracedLargeGlobalWritebackEvents++ < 256; + if (firstForRange || traceSmallMutation || traceLargeMutation) + { + var head = firstChangedOffset >= 0 + ? mappedBytes.Slice( + firstChangedOffset, + Math.Min(mappedBytes.Length - firstChangedOffset, 32)) + : ReadOnlySpan.Empty; + TraceVulkanShader( + $"vk.global_writeback base=0x{guestAddress:X16} " + + $"potential_bytes={mappedBytes.Length} changed_bytes={changedBytes} " + + $"changed_runs={changedRuns} changed_pages={changedPages} " + + $"written_pages={writtenPages} written_runs={writtenRuns} " + + $"unreadable_pages={unreadablePages} " + + $"fallback_writes={fallbackWrites} failed_runs={failedRuns} " + + $"probe_nonzero={nonzero}/{probe.Length} " + + $"changed_head={Convert.ToHexString(head)}"); + } } } } @@ -5025,28 +9151,49 @@ internal static unsafe class VulkanVideoPresenter uint zCount) { const uint maxWorkgroupsPerCommand = 4096; - var yChunk = Math.Max( + ulong commandCount = 0; + var maxXChunk = Math.Max( 1u, Math.Min( - work.GroupCountY, - maxWorkgroupsPerCommand / Math.Max(work.GroupCountX, 1u))); - var commandCount = 0u; - - for (var z = zStart; z < zStart + zCount; z++) + work.GroupCountX, + Math.Min(_maxComputeWorkGroupCountX, maxWorkgroupsPerCommand))); + for (var x = 0u; x < work.GroupCountX;) { - for (var y = 0u; y < work.GroupCountY; y += yChunk) + var countX = Math.Min(maxXChunk, work.GroupCountX - x); + var xyBudget = Math.Max(maxWorkgroupsPerCommand / countX, 1u); + var maxYChunk = Math.Max( + 1u, + Math.Min( + work.GroupCountY, + Math.Min(_maxComputeWorkGroupCountY, xyBudget))); + for (var y = 0u; y < work.GroupCountY;) { - var countY = Math.Min(yChunk, work.GroupCountY - y); - _vk.CmdDispatchBase( - commandBuffer, - 0, - y, - z, - work.GroupCountX, - countY, - 1); - commandCount++; + var countY = Math.Min(maxYChunk, work.GroupCountY - y); + var xyzBudget = Math.Max(xyBudget / countY, 1u); + var maxZChunk = Math.Max( + 1u, + Math.Min( + zCount, + Math.Min(_maxComputeWorkGroupCountZ, xyzBudget))); + for (var z = 0u; z < zCount;) + { + var countZ = Math.Min(maxZChunk, zCount - z); + _vk.CmdDispatchBase( + commandBuffer, + checked(work.BaseGroupX + x), + checked(work.BaseGroupY + y), + checked(work.BaseGroupZ + zStart + z), + countX, + countY, + countZ); + commandCount++; + z += countZ; + } + + y += countY; } + + x += countX; } if (commandCount > 1) @@ -5054,7 +9201,11 @@ internal static unsafe class VulkanVideoPresenter TraceVulkanShader( $"vk.compute_chunked cs=0x{work.ShaderAddress:X16} " + $"groups={work.GroupCountX}x{work.GroupCountY}x{work.GroupCountZ} " + - $"z_range={zStart}..{zStart + zCount} commands={commandCount} y_chunk={yChunk}"); + $"base={work.BaseGroupX}x{work.BaseGroupY}x{work.BaseGroupZ} " + + $"z_range={zStart}..{zStart + zCount} commands={commandCount} " + + $"command_budget={maxWorkgroupsPerCommand} " + + $"device_limit={_maxComputeWorkGroupCountX}x" + + $"{_maxComputeWorkGroupCountY}x{_maxComputeWorkGroupCountZ}"); } } @@ -5065,11 +9216,32 @@ internal static unsafe class VulkanVideoPresenter return; } + var perfStart = System.Diagnostics.Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _perfDrawCount); + PerfOverlay.RecordDraw(); + try + { + ExecuteOffscreenDrawCore(work); + } + finally + { + // Single atomic add per draw: staging -start/+end separately + // let the stats window reset land between them and report + // huge negative draw_ms values. + Interlocked.Add( + ref _perfDrawTicks, + System.Diagnostics.Stopwatch.GetTimestamp() - perfStart); + } + } + + private void ExecuteOffscreenDrawCore(VulkanOffscreenGuestDraw work) + { if (work.Targets.Count > _maxColorAttachments) { Console.Error.WriteLine( $"[LOADER][WARN] Vulkan skipped MRT draw requesting {work.Targets.Count} color attachments; " + $"the selected device supports {_maxColorAttachments}."); + ReturnPooledGuestData(work.Draw); return; } @@ -5083,6 +9255,7 @@ internal static unsafe class VulkanVideoPresenter Console.Error.WriteLine( $"[LOADER][WARN] Vulkan skipped MRT draw with unsupported color target " + $"format={target.Format} number_type={target.NumberType}."); + ReturnPooledGuestData(work.Draw); return; } } @@ -5091,6 +9264,7 @@ internal static unsafe class VulkanVideoPresenter { Console.Error.WriteLine( "[LOADER][WARN] Vulkan skipped MRT draw with mismatched attachment/blend counts."); + ReturnPooledGuestData(work.Draw); return; } @@ -5100,6 +9274,7 @@ internal static unsafe class VulkanVideoPresenter { Console.Error.WriteLine( "[LOADER][WARN] Vulkan skipped MRT draw with blending enabled for an integer attachment."); + ReturnPooledGuestData(work.Draw); return; } @@ -5108,6 +9283,7 @@ internal static unsafe class VulkanVideoPresenter { Console.Error.WriteLine( "[LOADER][WARN] Vulkan skipped MRT draw requiring unsupported independentBlend."); + ReturnPooledGuestData(work.Draw); return; } @@ -5118,11 +9294,13 @@ internal static unsafe class VulkanVideoPresenter .Select(target => target.Address) .ToHashSet(); if (work.Draw.Textures.Any(texture => - targetAddresses.Contains(texture.Address))) + texture.IsStorage && targetAddresses.Contains(texture.Address))) { Console.Error.WriteLine( - $"[LOADER][WARN] Vulkan skipped render-target feedback loop " + - $"targets={string.Join(',', targetAddresses.Select(address => $"0x{address:X16}"))}"); + $"[LOADER][WARN] Vulkan skipped storage render-target feedback loop " + + $"targets={string.Join(',', targetAddresses.Select(address => $"0x{address:X16}"))}; " + + "sampled aliases use ordered snapshots"); + ReturnPooledGuestData(work.Draw); return; } @@ -5130,7 +9308,19 @@ internal static unsafe class VulkanVideoPresenter EnsureGuestSubmissionCapacity(); for (var index = 0; index < targets.Length; index++) { - targets[index] = GetOrCreateGuestImage(work.Targets[index], formats[index]); + var targetDescriptor = work.Targets[index].Address == 0 && + work.DepthTarget is { } depthOnlyTarget + ? GetDepthOnlyColorTarget(depthOnlyTarget) + : work.Targets[index]; + targets[index] = GetOrCreateGuestImage(targetDescriptor, formats[index]); + if (work.Targets[index].Address != 0 && + TakeGuestImageInitialData(work.Targets[index].Address) is { } initialData && + !targets[index].Initialized && + (ulong)initialData.Length == + (ulong)targets[index].Width * targets[index].Height * 4) + { + UploadGuestImageInitialData(targets[index], initialData); + } } var firstTarget = targets[0]; @@ -5142,24 +9332,111 @@ internal static unsafe class VulkanVideoPresenter try { var extent = new Extent2D(firstTarget.Width, firstTarget.Height); - var renderPass = firstTarget.RenderPass; - var framebuffer = firstTarget.Framebuffer; + var draw = work.Draw; + if (work.DepthTarget?.ReadOnly == true && draw.RenderState.Depth.WriteEnable) + { + draw = draw with + { + RenderState = draw.RenderState with + { + Depth = draw.RenderState.Depth with { WriteEnable = false }, + }, + }; + } + GuestDepthResource? depth = null; + DepthFramebufferResource? depthFramebuffer = null; + if (work.DepthTarget is { } depthTarget && + (draw.RenderState.Depth.TestEnable || + draw.RenderState.Depth.WriteEnable)) + { + var effectiveDepthTarget = depthTarget; + if (depthTarget.Width < firstTarget.Width || depthTarget.Height < firstTarget.Height) + { + var matchingTexture = work.Draw.Textures.FirstOrDefault(texture => + texture.Address == depthTarget.Address && + texture.Width >= firstTarget.Width && + texture.Height >= firstTarget.Height); + if (matchingTexture is not null) + { + effectiveDepthTarget = depthTarget with + { + Width = matchingTexture.Width, + Height = matchingTexture.Height, + }; + } + else if (depthTarget.Width == 1 && depthTarget.Height == 1) + { + // Some Gen5 streams leave DB_DEPTH_SIZE_XY at its + // clear-state value while binding a full-size DB + // surface. The color attachment and viewport still + // define the render extent; treating that stale 1x1 + // value literally discards the entire draw. + effectiveDepthTarget = depthTarget with + { + Width = firstTarget.Width, + Height = firstTarget.Height, + }; + } + + if (effectiveDepthTarget != depthTarget && + _tracedDepthExtentFallbacks.Add( + (depthTarget.Address, + effectiveDepthTarget.Width, + effectiveDepthTarget.Height))) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan repaired stale guest depth extent " + + $"addr=0x{depthTarget.Address:X16} " + + $"{depthTarget.Width}x{depthTarget.Height} -> " + + $"{effectiveDepthTarget.Width}x{effectiveDepthTarget.Height}"); + } + } + + depth = GetOrCreateGuestDepth(effectiveDepthTarget); + PrepareFirstUseDepth(depth, draw.RenderState.Depth); + if (targets.Length == 1) + { + depthFramebuffer = GetOrCreateDepthFramebuffer(firstTarget, depth); + } + } + + var renderPass = depthFramebuffer is null + ? firstTarget.Initialized + ? firstTarget.RenderPass + : firstTarget.InitialRenderPass + : firstTarget.Initialized + ? depth!.Initialized + ? depthFramebuffer.LoadRenderPass + : depthFramebuffer.DepthClearRenderPass + : depth!.Initialized + ? depthFramebuffer.ColorClearRenderPass + : depthFramebuffer.BothClearRenderPass; + var framebuffer = depthFramebuffer?.Framebuffer ?? firstTarget.Framebuffer; if (targets.Length > 1) { (renderPass, framebuffer) = CreateRenderPassAndFramebuffer( formats, - targets.Select(target => target.MipViews[0]).ToArray(), + targets.Select(target => target.MipViews.Length > 0 + ? target.MipViews[0] + : target.View).ToArray(), firstTarget.Width, - firstTarget.Height); + firstTarget.Height, + targets.Select(target => + target.Initialized || target.InitialUploadPending).ToArray(), + depth, + depth?.Initialized == true); transientRenderPass = renderPass; transientFramebuffer = framebuffer; } resources = CreateTranslatedDrawResources( - work.Draw, + draw, renderPass, formats, - extent); + extent, + targets, + hasDepthAttachment: depth is not null, + feedbackDepth: depth); resources.TransientRenderPass = transientRenderPass; resources.TransientFramebuffer = transientFramebuffer; transientRenderPass = default; @@ -5169,18 +9446,39 @@ internal static unsafe class VulkanVideoPresenter $"first=0x{work.Targets[0].Address:X16} " + $"{firstTarget.Width}x{firstTarget.Height}"; - commandBuffer = AllocateGuestCommandBuffer(); + commandBuffer = BeginBatchedGuestCommands(); _commandBuffer = commandBuffer; - var beginInfo = new CommandBufferBeginInfo - { - SType = StructureType.CommandBufferBeginInfo, - Flags = CommandBufferUsageFlags.OneTimeSubmitBit, - }; - Check( - _vk.BeginCommandBuffer(_commandBuffer, &beginInfo), - "vkBeginCommandBuffer(offscreen)"); + + // Lifetime: recorded commands reference these resources, so + // they join the batch before recording and are destroyed only + // after the batch's fence signals. + _batchResources.Add(resources); + submitted = true; BeginDebugLabel(_commandBuffer, resources.DebugName); + var hasStorageImages = false; + foreach (var texture in resources.Textures) + { + if (texture is null) + { + continue; + } + + hasStorageImages |= texture.IsStorage; + } + + CloseOpenTranslatedRenderPass(); + RecordGlobalBufferVisibilityBarrier( + _commandBuffer, + resources, + PipelineStageFlags.VertexShaderBit | + PipelineStageFlags.FragmentShaderBit); + RecordRenderTargetFeedbackSnapshots( + resources, + PipelineStageFlags.FragmentShaderBit); + RecordDepthFeedbackSnapshots( + resources, + PipelineStageFlags.FragmentShaderBit); RecordTextureUploads(resources, PipelineStageFlags.FragmentShaderBit); RecordStorageImagesForWrite(resources, PipelineStageFlags.FragmentShaderBit); @@ -5188,7 +9486,8 @@ internal static unsafe class VulkanVideoPresenter var anyPriorContents = false; for (var index = 0; index < targets.Length; index++) { - var hasPriorContents = targets[index].Initialized || targets[index].InitialUploadPending; + var hasPriorContents = + targets[index].Initialized || targets[index].InitialUploadPending; anyPriorContents |= hasPriorContents; toColorAttachments[index] = new ImageMemoryBarrier { @@ -5219,12 +9518,48 @@ internal static unsafe class VulkanVideoPresenter (uint)targets.Length, toColorAttachments); - RecordTranslatedGraphicsPass( - resources, + if (depth is not null && + depth.Layout == ImageLayout.ShaderReadOnlyOptimal) + { + var toDepthAttachment = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.ShaderReadBit, + DstAccessMask = + AccessFlags.DepthStencilAttachmentReadBit | + AccessFlags.DepthStencilAttachmentWriteBit, + OldLayout = ImageLayout.ShaderReadOnlyOptimal, + NewLayout = ImageLayout.DepthStencilAttachmentOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = depth.Image, + SubresourceRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, 0, 1, 0, 1), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.FragmentShaderBit | + PipelineStageFlags.ComputeShaderBit, + PipelineStageFlags.EarlyFragmentTestsBit | + PipelineStageFlags.LateFragmentTestsBit, + 0, + 0, + null, + 0, + null, + 1, + &toDepthAttachment); + } + + BeginTranslatedRenderPass( renderPass, framebuffer, - extent); - RecordStorageImagesForRead(resources, PipelineStageFlags.FragmentShaderBit); + extent, + colorAttachmentCount: targets.Length, + hasDepthAttachment: depth is not null, + clearDepth: depth?.ClearDepth ?? 1f); + RecordTranslatedDrawInPass(resources, extent); + _vk.CmdEndRenderPass(_commandBuffer); var toShaderRead = stackalloc ImageMemoryBarrier[targets.Length]; for (var index = 0; index < targets.Length; index++) @@ -5253,17 +9588,34 @@ internal static unsafe class VulkanVideoPresenter null, (uint)targets.Length, toShaderRead); + + if (hasStorageImages) + { + RecordStorageImagesForRead(resources, PipelineStageFlags.FragmentShaderBit); + } + EndDebugLabel(_commandBuffer); - Check(_vk.EndCommandBuffer(_commandBuffer), "vkEndCommandBuffer(offscreen)"); - SubmitGuestCommandBuffer( - commandBuffer, - resources, - GetTraceImages(resources, targets)); - submitted = true; + var traceImages = GetTraceImages(resources, targets, work.ShaderAddress); + _batchTraceImages.AddRange(traceImages); + if (++_batchDrawCount >= 64 || + (_traceGuestImageShaderFilterEnabled && traceImages.Count != 0)) + { + FlushBatchedGuestCommands(); + } + foreach (var target in targets) { target.Initialized = true; + target.InitialUploadPending = false; + } + if (depth is not null) + { + depth.Initialized = true; + if (draw.RenderState.Depth.WriteEnable) + { + depth.InitializationSource = "translated-depth-write"; + } } MarkSampledImagesInitialized(resources); MarkStorageImagesInitialized(resources, traceContents: false); @@ -5283,16 +9635,71 @@ internal static unsafe class VulkanVideoPresenter lock (_gate) { _availableGuestImages[targets[index].Address] = guestTextureFormat; - _gpuGuestImages[targets[index].Address] = guestTextureFormat; } } } + + var guestWritesMode = Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_GUEST_WRITES"); + var traceAddressWriteOrdinal = 0L; + _ = long.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_WRITE_ORDINAL"), + out traceAddressWriteOrdinal); + var traceLargeWriteOrdinal = 0L; + if (guestWritesMode is not null && + guestWritesMode.StartsWith("large@", StringComparison.Ordinal) && + long.TryParse( + guestWritesMode.AsSpan("large@".Length), + out var parsedTraceLargeWriteOrdinal) && + parsedTraceLargeWriteOrdinal > 0) + { + traceLargeWriteOrdinal = parsedTraceLargeWriteOrdinal; + } + + var tracePixelSpirv = false; + if (int.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_PIXEL_SPIRV_BYTES"), + out var tracePixelSpirvBytes) && + tracePixelSpirvBytes == work.Draw.PixelSpirv.Length) + { + var pixelWriteCount = _pixelSpirvWriteCounts.TryGetValue( + tracePixelSpirvBytes, + out var previousPixelWriteCount) + ? previousPixelWriteCount + 1 + : 1; + _pixelSpirvWriteCounts[tracePixelSpirvBytes] = pixelWriteCount; + var tracePixelSpirvOccurrence = 1; + _ = int.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_PIXEL_SPIRV_OCCURRENCE"), + out tracePixelSpirvOccurrence); + tracePixelSpirv = + pixelWriteCount == Math.Max(tracePixelSpirvOccurrence, 1); + } + var traceTitleDraw = + !_tracedTitleDraw && + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TITLE_DRAW") == "1" && + work.Draw.VertexBuffers.Any(buffer => + buffer.Location == 0 && + buffer.ComponentCount == 4 && + buffer.DataFormat == 10 && + buffer.NumberFormat == 0 && + buffer.Stride == 16 && + buffer.OffsetBytes == 12 && + buffer.Length == 67568); + _tracedTitleDraw |= traceTitleDraw; + foreach (var target in targets) { - var traceSmallWrites = - Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_WRITES") == "small" && + var traceAddressWrite = + ShouldTraceGuestImageWriteForDiagnostics(target.Address); + var traceSmallWrites = guestWritesMode == "small" && target.Width <= 512 && target.Height <= 256; - if (ShouldTraceGuestImageWriteForDiagnostics(target.Address) || traceSmallWrites) + var traceLargeWrites = + (guestWritesMode == "large" || traceLargeWriteOrdinal != 0) && + target.Width >= 2560 && target.Height >= 1440; + if (traceAddressWrite || traceSmallWrites || + traceLargeWrites || tracePixelSpirv || traceTitleDraw) { var writeCount = _tracedGuestWriteCounts.TryGetValue( target.Address, @@ -5300,16 +9707,41 @@ internal static unsafe class VulkanVideoPresenter ? previousCount + 1 : 1; _tracedGuestWriteCounts[target.Address] = writeCount; - if (writeCount <= (traceSmallWrites ? 48 : 3)) + var shouldTraceWrite = tracePixelSpirv || traceTitleDraw + ? true + : traceAddressWrite && traceAddressWriteOrdinal > 0 + ? writeCount == traceAddressWriteOrdinal + : traceLargeWriteOrdinal != 0 + ? writeCount == traceLargeWriteOrdinal + : writeCount <= + (traceLargeWrites ? 2 : traceSmallWrites ? 48 : 3); + if (traceAddressWrite || shouldTraceWrite) { - _commandBuffer = _presentationCommandBuffer; - Check( - _vk.QueueWaitIdle(_queue), - "vkQueueWaitIdle(guest write trace)"); + var sampledTextures = string.Join( + ',', + work.Draw.Textures.Select(texture => + $"0x{texture.Address:X}:{texture.Width}x{texture.Height}:" + + $"f{texture.Format}:n{texture.NumberType}:" + + $"storage={(texture.IsStorage ? 1 : 0)}")); + var pixelDigest = Convert.ToHexString( + SHA256.HashData(work.Draw.PixelSpirv).AsSpan(0, 4)); Console.Error.WriteLine( $"[LOADER][TRACE] vk.guest_write_sample " + $"addr=0x{target.Address:X16} write={writeCount} " + - $"ps_bytes={work.Draw.PixelSpirv.Length}"); + $"vs_bytes={work.Draw.VertexSpirv.Length} " + + $"ps_bytes={work.Draw.PixelSpirv.Length} ps_hash={pixelDigest} " + + $"vertices={work.Draw.VertexCount} instances={work.Draw.InstanceCount} " + + $"primitive=0x{work.Draw.PrimitiveType:X} " + + $"readback={(shouldTraceWrite ? 1 : 0)} textures=[{sampledTextures}]"); + } + + if (shouldTraceWrite) + { + _commandBuffer = _presentationCommandBuffer; + FlushBatchedGuestCommands(); + Check( + _vk.QueueWaitIdle(_queue), + "vkQueueWaitIdle(guest write trace)"); TraceGuestImageContents(target); } } @@ -5334,7 +9766,6 @@ internal static unsafe class VulkanVideoPresenter !failedTarget.Initialized) { _availableGuestImages.Remove(target.Address); - _gpuGuestImages.Remove(target.Address); } } } @@ -5346,15 +9777,10 @@ internal static unsafe class VulkanVideoPresenter finally { _commandBuffer = _presentationCommandBuffer; - if (!submitted && commandBuffer.Handle != 0) - { - _vk.FreeCommandBuffers( - _device, - _commandPool, - 1, - &commandBuffer); - } - + // The command buffer is the shared batch; it is submitted and + // freed by FlushBatchedGuestCommands. Resources joined the + // batch list before recording, so only pre-recording failures + // (submitted still false) own their cleanup here. if (!submitted && resources is not null) { DestroyTranslatedDrawResources(resources); @@ -5373,16 +9799,240 @@ internal static unsafe class VulkanVideoPresenter } [MethodImpl(MethodImplOptions.NoInlining)] + private void ExecuteGuestImageWrite(VulkanGuestImageWrite work) + { + if (_deviceLost || !_guestImages.TryGetValue(work.Address, out var target)) + { + return; + } + + if (work.Pixels is { } pixels) + { + if (pixels.Length > 0) + { + UploadGuestImageInitialData(target, pixels); + } + + return; + } + + // Recorded into the shared batch command buffer: recording order + // preserves queue-order semantics against earlier batched draws, + // and the fill no longer costs a submit + full queue drain. + var commandBuffer = BeginBatchedGuestCommands(); + CloseOpenTranslatedRenderPass(); + var toTransferDst = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = target.Initialized ? AccessFlags.ShaderReadBit : 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = target.Initialized + ? ImageLayout.ShaderReadOnlyOptimal + : ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(0, target.MipLevels), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + target.Initialized + ? PipelineStageFlags.FragmentShaderBit + : PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &toTransferDst); + + var clearValue = new ClearColorValue( + (work.FillValue & 0xFF) / 255f, + ((work.FillValue >> 8) & 0xFF) / 255f, + ((work.FillValue >> 16) & 0xFF) / 255f, + ((work.FillValue >> 24) & 0xFF) / 255f); + var range = ColorSubresourceRange(0, target.MipLevels); + _vk.CmdClearColorImage( + commandBuffer, + target.Image, + ImageLayout.TransferDstOptimal, + &clearValue, + 1, + &range); + + var toShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(0, target.MipLevels), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.FragmentShaderBit, + 0, + 0, + null, + 0, + null, + 1, + &toShaderRead); + target.Initialized = true; + } + + private void UploadGuestImageInitialData(GuestImageResource target, byte[] pixels) + { + var byteCount = (ulong)pixels.Length; + var staging = CreateBuffer( + byteCount, + BufferUsageFlags.TransferSrcBit, + MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, + out var stagingMemory); + try + { + void* mapped; + Check( + _vk.MapMemory(_device, stagingMemory, 0, byteCount, 0, &mapped), + "vkMapMemory(guest image init)"); + fixed (byte* source = pixels) + { + System.Buffer.MemoryCopy(source, mapped, pixels.Length, pixels.Length); + } + + _vk.UnmapMemory(_device, stagingMemory); + + // Recorded into the shared batch; the staging buffer joins + // the batch's retire list and is destroyed when the batch + // fence signals, so the upload costs no queue drain. + var commandBuffer = BeginBatchedGuestCommands(); + CloseOpenTranslatedRenderPass(); + + var toTransferDst = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = target.Initialized ? AccessFlags.ShaderReadBit : 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = target.Initialized + ? ImageLayout.ShaderReadOnlyOptimal + : ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(0, target.MipLevels), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + target.Initialized + ? PipelineStageFlags.FragmentShaderBit + : PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &toTransferDst); + + var copyRegion = new BufferImageCopy + { + BufferOffset = 0, + BufferRowLength = 0, + BufferImageHeight = 0, + ImageSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, + 0, + 0, + 1), + ImageOffset = default, + ImageExtent = new Extent3D(target.Width, target.Height, 1), + }; + _vk.CmdCopyBufferToImage( + commandBuffer, + staging, + target.Image, + ImageLayout.TransferDstOptimal, + 1, + ©Region); + + var toShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(0, target.MipLevels), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.FragmentShaderBit, + 0, + 0, + null, + 0, + null, + 1, + &toShaderRead); + + target.Initialized = true; + _batchRetireBuffers.Add((staging, stagingMemory)); + staging = default; + stagingMemory = default; + if (_traceGuestImageEvents) + { + Console.Error.WriteLine( + $"[GIMG] seeded addr=0x{target.Address:X} " + + $"{target.Width}x{target.Height}"); + } + } + finally + { + if (staging.Handle != 0) + { + _vk.DestroyBuffer(_device, staging, null); + } + + if (stagingMemory.Handle != 0) + { + _vk.FreeMemory(_device, stagingMemory, null); + } + } + } + private GuestImageResource GetOrCreateGuestImage( GuestRenderTarget target, Format format) { var mipLevels = ClampMipLevels(target.Width, target.Height, target.MipLevels); + var guestFormat = GetGuestTextureFormat(target.Format, target.NumberType); + var requestedKey = new GuestImageVariantKey( + target.Address, + target.Width, + target.Height, + mipLevels, + guestFormat, + format); if (_guestImages.TryGetValue(target.Address, out var existing)) { if (existing.Width == target.Width && existing.Height == target.Height && existing.MipLevels == mipLevels && + existing.GuestFormat == guestFormat && existing.Format == format) { existing.IsCpuBacked = false; @@ -5392,30 +10042,90 @@ internal static unsafe class VulkanVideoPresenter var attachmentView = existing.MipViews.Length > 0 ? existing.MipViews[0] : existing.View; - var (promotedRenderPass, promotedFramebuffer) = CreateRenderPassAndFramebuffer( + var promoted = CreateRenderPassAndFramebuffer( existing.Format, attachmentView, existing.Width, existing.Height); - existing.RenderPass = promotedRenderPass; - existing.Framebuffer = promotedFramebuffer; + existing.RenderPass = promoted.RenderPass; + existing.InitialRenderPass = promoted.InitialRenderPass; + existing.Framebuffer = promoted.Framebuffer; + var promotedRenderPass = promoted.RenderPass; + var promotedInitialRenderPass = promoted.InitialRenderPass; + var promotedFramebuffer = promoted.Framebuffer; var promotedName = GuestImageDebugName(target, format); SetDebugName(ObjectType.RenderPass, promotedRenderPass.Handle, $"{promotedName} renderpass"); + SetDebugName(ObjectType.RenderPass, promotedInitialRenderPass.Handle, $"{promotedName} initial-renderpass"); SetDebugName(ObjectType.Framebuffer, promotedFramebuffer.Handle, $"{promotedName} framebuffer"); } return existing; } - CompletePendingPresentation(wait: true); - WaitForAllGuestSubmissions(); - DestroyGuestImage(existing); + if (_traceGuestImageEvents) + { + Console.Error.WriteLine( + $"[GIMG] recreate addr=0x{target.Address:X} " + + $"old={existing.Width}x{existing.Height}/{existing.Format}/m{existing.MipLevels} " + + $"new={target.Width}x{target.Height}/{format}/m{mipLevels} " + + $"initialized={existing.Initialized}"); + } + + _guestImageVariants.Add( + new GuestImageVariantKey( + existing.Address, + existing.Width, + existing.Height, + existing.MipLevels, + existing.GuestFormat, + existing.Format), + existing); _guestImages.Remove(target.Address); lock (_gate) { _availableGuestImages.Remove(target.Address); - _gpuGuestImages.Remove(target.Address); + _guestImageExtents.Remove(target.Address); } + + // Address-filtered readback diagnostics should inspect the + // replacement image too; a resized/reformatted allocation at + // the same guest address is a different piece of evidence. + _tracedGuestImageContents.Remove(target.Address); + + SharpEmu.HLE.GuestImageWriteTracker.Untrack(target.Address); + } + + if (_guestImageVariants.Remove(requestedKey, out var retained)) + { + retained.IsCpuBacked = false; + retained.CpuContentFingerprint = 0; + _guestImages.Add(target.Address, retained); + lock (_gate) + { + _guestImageExtents[target.Address] = ( + target.Width, + target.Height, + GetTextureByteCount(target.Format, target.Width, target.Height)); + } + + if (target.Width <= 1920 && target.Height <= 1080) + { + SharpEmu.HLE.GuestImageWriteTracker.Track( + target.Address, + (ulong)target.Width * target.Height * GetTextureBytesPerPixel(target.Format), + CurrentGuestWorkSequenceForDiagnostics, + "vulkan.render-target"); + } + + if (_traceGuestImageEvents) + { + Console.Error.WriteLine( + $"[GIMG] retained addr=0x{target.Address:X} " + + $"{target.Width}x{target.Height} fmt={format} " + + $"initialized={retained.Initialized}"); + } + + return retained; } var imageInfo = new ImageCreateInfo @@ -5487,11 +10197,12 @@ internal static unsafe class VulkanVideoPresenter mipViews[mipLevel] = mipView; } - var (renderPass, framebuffer) = CreateRenderPassAndFramebuffer( - format, - mipViews[0], - target.Width, - target.Height); + var (renderPass, initialRenderPass, framebuffer) = + CreateRenderPassAndFramebuffer( + format, + mipViews[0], + target.Width, + target.Height); var resource = new GuestImageResource { @@ -5499,12 +10210,14 @@ internal static unsafe class VulkanVideoPresenter Width = target.Width, Height = target.Height, MipLevels = mipLevels, + GuestFormat = guestFormat, Format = format, Image = image, Memory = memory, View = view, MipViews = mipViews, RenderPass = renderPass, + InitialRenderPass = initialRenderPass, Framebuffer = framebuffer, }; var debugName = GuestImageDebugName(target, format); @@ -5518,39 +10231,95 @@ internal static unsafe class VulkanVideoPresenter $"{debugName} mip{mipLevel}"); } SetDebugName(ObjectType.RenderPass, renderPass.Handle, $"{debugName} renderpass"); + SetDebugName(ObjectType.RenderPass, initialRenderPass.Handle, $"{debugName} initial-renderpass"); SetDebugName(ObjectType.Framebuffer, framebuffer.Handle, $"{debugName} framebuffer"); _guestImages.Add(target.Address, resource); + lock (_gate) + { + _guestImageExtents[target.Address] = ( + target.Width, + target.Height, + GetTextureByteCount(target.Format, target.Width, target.Height)); + } + + if (target.Width <= 1920 && target.Height <= 1080) + { + SharpEmu.HLE.GuestImageWriteTracker.Track( + target.Address, + (ulong)target.Width * target.Height * GetTextureBytesPerPixel(target.Format), + CurrentGuestWorkSequenceForDiagnostics, + "vulkan.render-target"); + } + + if (_traceGuestImageEvents) + { + Console.Error.WriteLine( + $"[GIMG] created-as-rt addr=0x{target.Address:X} " + + $"{target.Width}x{target.Height} fmt={format}"); + } + return resource; } - private (RenderPass RenderPass, Framebuffer Framebuffer) CreateRenderPassAndFramebuffer( + private (RenderPass RenderPass, RenderPass InitialRenderPass, Framebuffer Framebuffer) + CreateRenderPassAndFramebuffer( Format format, ImageView attachmentView, uint width, - uint height) => - CreateRenderPassAndFramebuffer([format], [attachmentView], width, height); + uint height) + { + var load = CreateRenderPassAndFramebuffer( + [format], [attachmentView], width, height, [true], null, false); + var initial = CreateRenderPassAndFramebuffer( + [format], [attachmentView], width, height, [false], null, false); + _vk.DestroyFramebuffer(_device, initial.Framebuffer, null); + return (load.RenderPass, initial.RenderPass, load.Framebuffer); + } private (RenderPass RenderPass, Framebuffer Framebuffer) CreateRenderPassAndFramebuffer( IReadOnlyList formats, IReadOnlyList attachmentViews, uint width, - uint height) + uint height) => + CreateRenderPassAndFramebuffer( + formats, + attachmentViews, + width, + height, + Enumerable.Repeat(true, formats.Count).ToArray(), + null, + false); + + private (RenderPass RenderPass, Framebuffer Framebuffer) CreateRenderPassAndFramebuffer( + IReadOnlyList formats, + IReadOnlyList attachmentViews, + uint width, + uint height, + IReadOnlyList initialized, + GuestDepthResource? depth, + bool depthInitialized) { - if (formats.Count == 0 || formats.Count != attachmentViews.Count) + if (formats.Count == 0 || + formats.Count != attachmentViews.Count || + formats.Count != initialized.Count) { - throw new InvalidOperationException("render target formats and views must have matching counts"); + throw new InvalidOperationException( + "render target formats, views, and initialization states must have matching counts"); } - var colorAttachments = stackalloc AttachmentDescription[formats.Count]; + var attachmentCount = formats.Count + (depth is null ? 0 : 1); + var attachments = stackalloc AttachmentDescription[attachmentCount]; var colorReferences = stackalloc AttachmentReference[formats.Count]; - var views = stackalloc ImageView[formats.Count]; + var views = stackalloc ImageView[attachmentCount]; for (var index = 0; index < formats.Count; index++) { - colorAttachments[index] = new AttachmentDescription + attachments[index] = new AttachmentDescription { Format = formats[index], Samples = SampleCountFlags.Count1Bit, - LoadOp = AttachmentLoadOp.Load, + LoadOp = initialized[index] + ? AttachmentLoadOp.Load + : AttachmentLoadOp.Clear, StoreOp = AttachmentStoreOp.Store, StencilLoadOp = AttachmentLoadOp.DontCare, StencilStoreOp = AttachmentStoreOp.DontCare, @@ -5565,17 +10334,44 @@ internal static unsafe class VulkanVideoPresenter views[index] = attachmentViews[index]; } + AttachmentReference depthReference = default; + if (depth is not null) + { + attachments[formats.Count] = new AttachmentDescription + { + Format = DepthFormat, + Samples = SampleCountFlags.Count1Bit, + LoadOp = depthInitialized + ? AttachmentLoadOp.Load + : AttachmentLoadOp.Clear, + StoreOp = AttachmentStoreOp.Store, + StencilLoadOp = AttachmentLoadOp.DontCare, + StencilStoreOp = AttachmentStoreOp.DontCare, + InitialLayout = depthInitialized + ? ImageLayout.DepthStencilAttachmentOptimal + : ImageLayout.Undefined, + FinalLayout = ImageLayout.DepthStencilAttachmentOptimal, + }; + depthReference = new AttachmentReference + { + Attachment = (uint)formats.Count, + Layout = ImageLayout.DepthStencilAttachmentOptimal, + }; + views[formats.Count] = depth.View; + } + var subpass = new SubpassDescription { PipelineBindPoint = PipelineBindPoint.Graphics, ColorAttachmentCount = (uint)formats.Count, PColorAttachments = colorReferences, + PDepthStencilAttachment = depth is null ? null : &depthReference, }; var renderPassInfo = new RenderPassCreateInfo { SType = StructureType.RenderPassCreateInfo, - AttachmentCount = (uint)formats.Count, - PAttachments = colorAttachments, + AttachmentCount = (uint)attachmentCount, + PAttachments = attachments, SubpassCount = 1, PSubpasses = &subpass, }; @@ -5587,7 +10383,7 @@ internal static unsafe class VulkanVideoPresenter { SType = StructureType.FramebufferCreateInfo, RenderPass = renderPass, - AttachmentCount = (uint)formats.Count, + AttachmentCount = (uint)attachmentCount, PAttachments = views, Width = width, Height = height, @@ -5600,6 +10396,333 @@ internal static unsafe class VulkanVideoPresenter return (renderPass, framebuffer); } + private (Image Image, DeviceMemory Memory, ImageView View) CreateDepthAttachment( + uint width, + uint height) + { + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + ImageType = ImageType.Type2D, + Format = DepthFormat, + Extent = new Extent3D(Math.Max(width, 1), Math.Max(height, 1), 1), + MipLevels = 1, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = + ImageUsageFlags.DepthStencilAttachmentBit | + ImageUsageFlags.SampledBit | + ImageUsageFlags.TransferSrcBit | + ImageUsageFlags.TransferDstBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check(_vk.CreateImage(_device, &imageInfo, null, out var image), "vkCreateImage(depth)"); + _vk.GetImageMemoryRequirements(_device, image, out var requirements); + var memoryInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType(requirements.MemoryTypeBits, MemoryPropertyFlags.DeviceLocalBit), + }; + Check(_vk.AllocateMemory(_device, &memoryInfo, null, out var memory), "vkAllocateMemory(depth)"); + Check(_vk.BindImageMemory(_device, image, memory, 0), "vkBindImageMemory(depth)"); + + var viewInfo = new ImageViewCreateInfo + { + SType = StructureType.ImageViewCreateInfo, + Image = image, + ViewType = ImageViewType.Type2D, + Format = DepthFormat, + SubresourceRange = new ImageSubresourceRange(ImageAspectFlags.DepthBit, 0, 1, 0, 1), + }; + Check(_vk.CreateImageView(_device, &viewInfo, null, out var view), "vkCreateImageView(depth)"); + return (image, memory, view); + } + + private GuestDepthResource GetOrCreateGuestDepth(GuestDepthTarget target) + { + var key = new GuestDepthKey( + target.Address, + target.ReadAddress, + target.Width, + target.Height, + target.GuestFormat, + target.SwizzleMode); + if (_guestDepthImages.TryGetValue(key, out var existing)) + { + existing.GuestClearDepth = target.ClearDepth; + if (!existing.Initialized && existing.InitializationSource == "none") + { + existing.ClearDepth = target.ClearDepth; + } + return existing; + } + + var (image, memory, view) = CreateDepthAttachment(target.Width, target.Height); + var resource = new GuestDepthResource + { + Key = key, + Address = target.Address, + ReadAddress = target.ReadAddress, + WriteAddress = target.WriteAddress, + Width = target.Width, + Height = target.Height, + GuestFormat = target.GuestFormat, + SwizzleMode = target.SwizzleMode, + Image = image, + Memory = memory, + View = view, + GuestClearDepth = target.ClearDepth, + ClearDepth = target.ClearDepth, + }; + SetDebugName( + ObjectType.Image, + image.Handle, + $"SharpEmu guest depth 0x{target.Address:X16} {target.Width}x{target.Height}"); + SetDebugName( + ObjectType.ImageView, + view.Handle, + $"SharpEmu guest depth view 0x{target.Address:X16}"); + _guestDepthImages.Add(key, resource); + if (_traceGuestImageEvents || _traceVulkanShaderEnabled) + { + Console.Error.WriteLine( + $"[GIMG] created-depth addr=0x{target.Address:X} " + + $"read=0x{target.ReadAddress:X} write=0x{target.WriteAddress:X} " + + $"{target.Width}x{target.Height} zfmt={target.GuestFormat} " + + $"sw={target.SwizzleMode} clear={target.ClearDepth:0.######}"); + } + + return resource; + } + + private static void PrepareFirstUseDepth( + GuestDepthResource depth, + GuestDepthState state) + { + if (depth.Initialized || depth.InitializationSource != "none") + { + return; + } + + var effectiveClear = depth.GuestClearDepth; + var source = "guest-clear"; + if (state.TestEnable && !state.WriteEnable) + { + switch (state.CompareOp) + { + case 1: // Less + case 3: // LessOrEqual + effectiveClear = 1f; + source = "neutral-first-use"; + break; + case 4: // Greater + case 6: // GreaterOrEqual + effectiveClear = 0f; + source = "neutral-first-use"; + break; + case 7: // Always + break; + default: // Never, Equal, NotEqual + source = "guest-clear-ambiguous"; + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan has no neutral first-use depth clear " + + $"addr=0x{depth.Address:X16} compare={state.CompareOp}; " + + $"using guest clear {depth.GuestClearDepth:0.######}."); + break; + } + } + + depth.ClearDepth = effectiveClear; + depth.InitializationSource = source; + if (_traceDepthInitialization) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.depth_init " + + $"addr=0x{depth.Address:X16} size={depth.Width}x{depth.Height} " + + $"source={source} guest_clear={depth.GuestClearDepth:0.######} " + + $"effective_clear={effectiveClear:0.######} compare={state.CompareOp} " + + $"test={(state.TestEnable ? 1 : 0)} write={(state.WriteEnable ? 1 : 0)} " + + $"initialized=0"); + } + } + + private GuestRenderTarget GetDepthOnlyColorTarget(GuestDepthTarget depth) + { + var key = new GuestDepthKey( + depth.Address, + depth.ReadAddress, + depth.Width, + depth.Height, + depth.GuestFormat, + depth.SwizzleMode); + if (!_depthOnlyColorAddresses.TryGetValue(key, out var address)) + { + address = _nextDepthOnlyColorAddress; + _nextDepthOnlyColorAddress = checked(_nextDepthOnlyColorAddress + 0x1000_0000UL); + _depthOnlyColorAddresses.Add(key, address); + } + + // The translated fragment module still declares a color output, + // even for a guest depth-only pass. A private, never-published + // color attachment keeps that output legal while the persistent + // guest DB surface remains the only observable result. + return new GuestRenderTarget( + address, + depth.Width, + depth.Height, + Format: 10, + NumberType: 0); + } + + private DepthFramebufferResource GetOrCreateDepthFramebuffer( + GuestImageResource color, + GuestDepthResource depth) + { + if (color.DepthFramebuffers.TryGetValue(depth.Key, out var existing)) + { + return existing; + } + + if (depth.Width < color.Width || depth.Height < color.Height) + { + throw new InvalidOperationException( + $"guest depth 0x{depth.Address:X16} extent {depth.Width}x{depth.Height} " + + $"is smaller than color target 0x{color.Address:X16} " + + $"{color.Width}x{color.Height}"); + } + + var attachmentView = color.MipViews.Length > 0 ? color.MipViews[0] : color.View; + var loadRenderPass = CreateDepthRenderPass( + color.Format, + clearColor: false, + clearDepth: false); + var colorClearRenderPass = CreateDepthRenderPass( + color.Format, + clearColor: true, + clearDepth: false); + var depthClearRenderPass = CreateDepthRenderPass( + color.Format, + clearColor: false, + clearDepth: true); + var bothClearRenderPass = CreateDepthRenderPass( + color.Format, + clearColor: true, + clearDepth: true); + var attachments = stackalloc ImageView[2]; + attachments[0] = attachmentView; + attachments[1] = depth.View; + var framebufferInfo = new FramebufferCreateInfo + { + SType = StructureType.FramebufferCreateInfo, + RenderPass = loadRenderPass, + AttachmentCount = 2, + PAttachments = attachments, + Width = color.Width, + Height = color.Height, + Layers = 1, + }; + Check( + _vk.CreateFramebuffer(_device, &framebufferInfo, null, out var framebuffer), + "vkCreateFramebuffer(offscreen depth)"); + var resource = new DepthFramebufferResource + { + Depth = depth, + LoadRenderPass = loadRenderPass, + ColorClearRenderPass = colorClearRenderPass, + DepthClearRenderPass = depthClearRenderPass, + BothClearRenderPass = bothClearRenderPass, + Framebuffer = framebuffer, + }; + var name = $"SharpEmu color 0x{color.Address:X16} depth 0x{depth.Address:X16}"; + SetDebugName(ObjectType.RenderPass, loadRenderPass.Handle, $"{name} load"); + SetDebugName(ObjectType.RenderPass, colorClearRenderPass.Handle, $"{name} color-clear"); + SetDebugName(ObjectType.RenderPass, depthClearRenderPass.Handle, $"{name} depth-clear"); + SetDebugName(ObjectType.RenderPass, bothClearRenderPass.Handle, $"{name} both-clear"); + SetDebugName(ObjectType.Framebuffer, framebuffer.Handle, $"{name} framebuffer"); + color.DepthFramebuffers.Add(depth.Key, resource); + return resource; + } + + private RenderPass CreateDepthRenderPass( + Format colorFormat, + bool clearColor, + bool clearDepth) + { + var attachments = stackalloc AttachmentDescription[2]; + attachments[0] = new AttachmentDescription + { + Format = colorFormat, + Samples = SampleCountFlags.Count1Bit, + LoadOp = clearColor ? AttachmentLoadOp.Clear : AttachmentLoadOp.Load, + StoreOp = AttachmentStoreOp.Store, + StencilLoadOp = AttachmentLoadOp.DontCare, + StencilStoreOp = AttachmentStoreOp.DontCare, + InitialLayout = ImageLayout.ColorAttachmentOptimal, + FinalLayout = ImageLayout.ColorAttachmentOptimal, + }; + attachments[1] = new AttachmentDescription + { + Format = DepthFormat, + Samples = SampleCountFlags.Count1Bit, + LoadOp = clearDepth ? AttachmentLoadOp.Clear : AttachmentLoadOp.Load, + StoreOp = AttachmentStoreOp.Store, + StencilLoadOp = AttachmentLoadOp.DontCare, + StencilStoreOp = AttachmentStoreOp.DontCare, + InitialLayout = clearDepth + ? ImageLayout.Undefined + : ImageLayout.DepthStencilAttachmentOptimal, + FinalLayout = ImageLayout.DepthStencilAttachmentOptimal, + }; + var colorReference = new AttachmentReference + { + Attachment = 0, + Layout = ImageLayout.ColorAttachmentOptimal, + }; + var depthReference = new AttachmentReference + { + Attachment = 1, + Layout = ImageLayout.DepthStencilAttachmentOptimal, + }; + var subpass = new SubpassDescription + { + PipelineBindPoint = PipelineBindPoint.Graphics, + ColorAttachmentCount = 1, + PColorAttachments = &colorReference, + PDepthStencilAttachment = &depthReference, + }; + var dependency = new SubpassDependency + { + SrcSubpass = Vk.SubpassExternal, + DstSubpass = 0, + SrcStageMask = PipelineStageFlags.LateFragmentTestsBit, + DstStageMask = + PipelineStageFlags.EarlyFragmentTestsBit | + PipelineStageFlags.LateFragmentTestsBit, + SrcAccessMask = AccessFlags.DepthStencilAttachmentWriteBit, + DstAccessMask = + AccessFlags.DepthStencilAttachmentReadBit | + AccessFlags.DepthStencilAttachmentWriteBit, + DependencyFlags = DependencyFlags.ByRegionBit, + }; + var createInfo = new RenderPassCreateInfo + { + SType = StructureType.RenderPassCreateInfo, + AttachmentCount = 2, + PAttachments = attachments, + SubpassCount = 1, + PSubpasses = &subpass, + DependencyCount = 1, + PDependencies = &dependency, + }; + Check( + _vk.CreateRenderPass(_device, &createInfo, null, out var renderPass), + "vkCreateRenderPass(offscreen depth)"); + return renderPass; + } + private static uint ClampMipLevels(uint width, uint height, uint requestedMipLevels) { var largestDimension = Math.Max(width, height); @@ -5615,6 +10738,12 @@ internal static unsafe class VulkanVideoPresenter private void DestroyGuestImage(GuestImageResource resource) { + foreach (var depthFramebuffer in resource.DepthFramebuffers.Values) + { + DestroyDepthFramebuffer(depthFramebuffer); + } + resource.DepthFramebuffers.Clear(); + foreach (var view in resource.FormatViews.Values) { if (view.Handle != 0) @@ -5634,6 +10763,11 @@ internal static unsafe class VulkanVideoPresenter _vk.DestroyRenderPass(_device, resource.RenderPass, null); } + if (resource.InitialRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.InitialRenderPass, null); + } + if (resource.View.Handle != 0) { _vk.DestroyImageView(_device, resource.View, null); @@ -5656,28 +10790,58 @@ internal static unsafe class VulkanVideoPresenter { _vk.FreeMemory(_device, resource.Memory, null); } + } - private static uint GetGuestTextureFormat(Format format) => - format switch + private void DestroyDepthFramebuffer(DepthFramebufferResource resource) + { + if (resource.Framebuffer.Handle != 0) { - Format.A2R10G10B10UnormPack32 => 9, - Format.R8G8B8A8Unorm => 56, - Format.R16G16Unorm => 5, - Format.R16G16B16A16Unorm => 12, - Format.R32Uint => GuestFormatR32Uint, - Format.R32Sint => GuestFormatR32Sint, - Format.R32Sfloat => GuestFormatR32Sfloat, - Format.R16G16Uint => GuestFormatR16G16Uint, - Format.R16G16Sint => GuestFormatR16G16Sint, - Format.R16G16Sfloat => GuestFormatR16G16Sfloat, - Format.R8G8B8A8Uint => GuestFormatR8G8B8A8Uint, - Format.R8G8B8A8Sint => GuestFormatR8G8B8A8Sint, - Format.R16G16B16A16Uint => GuestFormatR16G16B16A16Uint, - Format.R16G16B16A16Sint => GuestFormatR16G16B16A16Sint, - Format.R16G16B16A16Sfloat => 71, - _ => 0, - }; + _vk.DestroyFramebuffer(_device, resource.Framebuffer, null); + } + + if (resource.LoadRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.LoadRenderPass, null); + } + if (resource.ColorClearRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.ColorClearRenderPass, null); + } + if (resource.DepthClearRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.DepthClearRenderPass, null); + } + if (resource.BothClearRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.BothClearRenderPass, null); + } + } + + private void DestroyGuestDepth(GuestDepthResource resource) + { + foreach (var sampleView in resource.SampleViews.Values) + { + if (sampleView.Handle != 0) + { + _vk.DestroyImageView(_device, sampleView, null); + } + } + resource.SampleViews.Clear(); + + if (resource.View.Handle != 0) + { + _vk.DestroyImageView(_device, resource.View, null); + } + if (resource.Image.Handle != 0) + { + _vk.DestroyImage(_device, resource.Image, null); + } + if (resource.Memory.Handle != 0) + { + _vk.FreeMemory(_device, resource.Memory, null); + } + } private bool TryGetOrCreateGuestImageView( GuestImageResource resource, @@ -5793,9 +10957,11 @@ internal static unsafe class VulkanVideoPresenter Format.R16G16Sint or Format.R16G16Sfloat or Format.R8G8B8A8Unorm or + Format.R8G8B8A8Srgb or Format.R8G8B8A8Uint or Format.R8G8B8A8Sint or Format.A2R10G10B10UnormPack32 or + Format.A2B10G10R10UnormPack32 or Format.B10G11R11UfloatPack32 => 32, Format.R32G32Uint or Format.R32G32Sint or @@ -5811,214 +10977,14 @@ internal static unsafe class VulkanVideoPresenter _ => 0, }; - private void UpdatePerformanceHud() - { - if (!_performanceHudEnabled) - { - return; - } - - var now = Stopwatch.GetTimestamp(); - if (_performanceHudLastTimestamp != 0 && - Stopwatch.GetElapsedTime(_performanceHudLastTimestamp, now).TotalSeconds < - PerformanceHudSampleSeconds) - { - return; - } - - try - { - using var process = Process.GetCurrentProcess(); - var processCpu = process.TotalProcessorTime; - var currentThreadCpu = new Dictionary(); - var currentThreadIds = new HashSet(); - var hottestThreadId = 0; - var hottestThreadCpuSeconds = 0.0; - - // Per-thread CPU times and thread names come from Windows-only - // APIs; on POSIX the HUD reports process totals with an "idle" - // hottest-thread slot. - if (OperatingSystem.IsWindows()) - { - foreach (ProcessThread thread in process.Threads) - { - using (thread) - { - try - { - var threadId = thread.Id; - var cpu = thread.TotalProcessorTime; - currentThreadIds.Add(threadId); - currentThreadCpu[threadId] = cpu; - if (_performanceHudThreadCpu.TryGetValue(threadId, out var previousCpu)) - { - var deltaSeconds = Math.Max(0.0, (cpu - previousCpu).TotalSeconds); - if (deltaSeconds > hottestThreadCpuSeconds) - { - hottestThreadCpuSeconds = deltaSeconds; - hottestThreadId = threadId; - } - } - } - catch (InvalidOperationException) - { - } - } - } - } - - if (_performanceHudLastTimestamp != 0) - { - var elapsedSeconds = Math.Max( - Stopwatch.GetElapsedTime(_performanceHudLastTimestamp, now).TotalSeconds, - 0.001); - var processCpuPercent = Math.Max( - 0.0, - (processCpu - _performanceHudLastProcessCpu).TotalSeconds / - elapsedSeconds / - Math.Max(Environment.ProcessorCount, 1) * - 100.0); - var hottestThreadPercent = hottestThreadCpuSeconds / elapsedSeconds * 100.0; - var presentedFrames = _performanceHudPresentedFrames; - var fps = (presentedFrames - _performanceHudLastPresentedFrames) / elapsedSeconds; - var hotName = hottestThreadId == 0 - ? "idle" - : GetPerformanceThreadName(hottestThreadId); - long guestBacklog; - int queuedGuestWork; - lock (_gate) - { - guestBacklog = Math.Max( - 0, - _enqueuedGuestWorkSequence - _completedGuestWorkSequence); - queuedGuestWork = _pendingGuestWork.Count; - } - - var gpuInFlight = _pendingGuestSubmissions.Count + - (_presentationInFlight ? 1 : 0); - var readCount = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadCount); - var readBytes = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadBytes); - var readHits = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadCacheHits); - var readPvmBytes = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadPvmBytes); - var readLibcBytes = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadLibcBytes); - var readsPerSecond = - (readCount - _performanceHudLastReadCount) / elapsedSeconds; - var readMbPerSecond = - (readBytes - _performanceHudLastReadBytes) / - elapsedSeconds / - (1024.0 * 1024.0); - var readHitsPerSecond = - (readHits - _performanceHudLastReadHits) / elapsedSeconds; - var readPvmMbPerSecond = - (readPvmBytes - _performanceHudLastReadPvmBytes) / - elapsedSeconds / - (1024.0 * 1024.0); - var readLibcMbPerSecond = - (readLibcBytes - _performanceHudLastReadLibcBytes) / - elapsedSeconds / - (1024.0 * 1024.0); - _performanceHudLastReadCount = readCount; - _performanceHudLastReadBytes = readBytes; - _performanceHudLastReadHits = readHits; - _performanceHudLastReadPvmBytes = readPvmBytes; - _performanceHudLastReadLibcBytes = readLibcBytes; - _window.Title = - $"FPS {fps:0.0} CPU {processCpuPercent:0}% | " + - $"HOT {hotName}#{hottestThreadId} {hottestThreadPercent:0}% | " + - $"WORK {guestBacklog} (q{queuedGuestWork}/gpu{gpuInFlight}) | " + - $"RD {readsPerSecond:0}/s {readMbPerSecond:0}MB/s h{readHitsPerSecond:0}/s " + - $"P{readPvmMbPerSecond:0} L{readLibcMbPerSecond:0} | " + - VideoOutExports.GetWindowTitle(); - _performanceHudLastPresentedFrames = presentedFrames; - } - - _performanceHudThreadCpu.Clear(); - foreach (var (threadId, cpu) in currentThreadCpu) - { - _performanceHudThreadCpu[threadId] = cpu; - } - - foreach (var staleThreadId in _performanceHudThreadNames.Keys - .Where(threadId => !currentThreadIds.Contains(threadId)) - .ToArray()) - { - _performanceHudThreadNames.Remove(staleThreadId); - } - - _performanceHudLastProcessCpu = processCpu; - _performanceHudLastTimestamp = now; - } - catch (Exception exception) when ( - exception is InvalidOperationException or System.ComponentModel.Win32Exception) - { - _performanceHudLastTimestamp = now; - } - } - - private string GetPerformanceThreadName(int threadId) - { - if (_performanceHudThreadNames.TryGetValue(threadId, out var cached)) - { - return cached; - } - - var name = "tid"; - var handle = OpenThread(ThreadQueryLimitedInformation, false, (uint)threadId); - if (handle != 0) - { - try - { - if (GetThreadDescription(handle, out var description) >= 0 && description != 0) - { - try - { - var described = Marshal.PtrToStringUni(description); - if (!string.IsNullOrWhiteSpace(described)) - { - name = described.Length <= 28 ? described : described[..28]; - } - } - finally - { - LocalFree(description); - } - } - } - finally - { - CloseHandle(handle); - } - } - - _performanceHudThreadNames[threadId] = name; - return name; - } - - [DllImport("kernel32.dll", SetLastError = true)] - private static extern nint OpenThread(uint desiredAccess, bool inheritHandle, uint threadId); - - [DllImport("kernel32.dll")] - private static extern int GetThreadDescription(nint thread, out nint description); - - [DllImport("kernel32.dll")] - private static extern nint LocalFree(nint memory); - - [DllImport("kernel32.dll", SetLastError = true)] - [return: MarshalAs(System.Runtime.InteropServices.UnmanagedType.Bool)] - private static extern bool CloseHandle(nint handle); - private void WaitForRenderWork() { - var gpuWorkInFlight = _pendingGuestSubmissions.Count > 0 || _presentationInFlight; + var gpuWorkInFlight = _pendingGuestSubmissions.Count > 0 || + Array.Exists(_frameFencePending, static pending => pending); lock (_gate) { if (_closed || - _pendingGuestWork.Count > 0 || + _pendingGuestWorkCount > 0 || (_latestPresentation is { } latest && latest.Sequence != _presentedSequence && latest.RequiredGuestWorkSequence <= _completedGuestWorkSequence)) @@ -6034,6 +11000,7 @@ internal static unsafe class VulkanVideoPresenter { if (Volatile.Read(ref _presenterCloseRequested)) { + Console.Error.WriteLine("[LOADER][WARN] Vulkan VideoOut closing on host shutdown request."); _window.Close(); return; } @@ -6043,19 +11010,84 @@ internal static unsafe class VulkanVideoPresenter return; } - WaitForRenderWork(); - UpdatePerformanceHud(); + // Reuse of a frame slot waits only on that slot's fence, keeping + // up to MaxFramesInFlight frames pipelined between CPU and GPU. + var frameSlot = _currentFrameSlot; + if (!TryWaitFrameSlot(frameSlot, _frameSlotWaitBudgetNs)) + { + // The GPU is still finishing this slot's previous frame (slow + // compute backlog). Don't block the macOS main thread — return + // to the Cocoa event pump so the window keeps handling input + // (F1 overlay, drag, close) and redrawing. The frame is retried + // next Render(); the fence signals once the GPU catches up. + return; + } + _presentationCommandBuffer = _frameCommandBuffers[frameSlot]; _commandBuffer = _presentationCommandBuffer; if (!_deviceLost) { CollectCompletedGuestSubmissions(waitForOldest: false); } + EvictDirtyCachedTextures(); var completedWork = 0; - while (completedWork < MaxGuestWorkPerRender && - TryTakeGuestWork(out var work)) + var renderWorkDeadline = _renderWorkBudgetTicks > 0 + ? System.Diagnostics.Stopwatch.GetTimestamp() + _renderWorkBudgetTicks + : long.MaxValue; + while (completedWork < MaxGuestWorkPerRender) { + // Never block the macOS main thread waiting for in-flight GPU + // work to drain. If submission is at capacity (a slow-compute + // backlog), stop processing and let the event pump run; the + // remaining queued work is picked up on later frames as the GPU + // completions free up capacity (collected non-blockingly here). + CollectCompletedGuestSubmissions(waitForOldest: false); + if (_pendingGuestSubmissions.Count >= MaxInFlightGuestSubmissions) + { + break; + } + + if (!TryTakeGuestWork(out var pendingGuestWork)) + { + break; + } + + if (!string.Equals( + _activeGuestQueue.Name, + pendingGuestWork.Queue.Name, + StringComparison.Ordinal)) + { + // A host command buffer must never contain commands from + // two independent guest queues: an ordered action fences + // only its own queue's predecessor submissions. + FlushBatchedGuestCommands(); + } + + _activeGuestQueue = pendingGuestWork.Queue; + _activeGuestWorkSequence = pendingGuestWork.Sequence; + Volatile.Write( + ref _executingGuestWorkSequence, + pendingGuestWork.Sequence); + using var guestQueueScope = EnterGuestQueue( + pendingGuestWork.Queue.Name, + pendingGuestWork.Queue.SubmissionId); + _enqueueAsImmediateQueueFollowup = true; + _immediateFollowupTail = null; + var work = pendingGuestWork.Work; + + var traceWork = ShouldTracePresentedGuestImageContentsForDiagnostics(); + var workStart = traceWork ? System.Diagnostics.Stopwatch.GetTimestamp() : 0L; + if (traceWork && work is VulkanComputeGuestDispatch or VulkanOffscreenGuestDraw) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.render_work_enter #{completedWork} " + + $"sequence={pendingGuestWork.Sequence} " + + $"queue={pendingGuestWork.Queue.Name} " + + $"submission={pendingGuestWork.Queue.SubmissionId} " + + $"queued_ms={(System.Diagnostics.Stopwatch.GetTimestamp() - pendingGuestWork.EnqueuedTicks) * 1000.0 / System.Diagnostics.Stopwatch.Frequency:F3} " + + work.GetType().Name); + } try { switch (work) @@ -6066,21 +11098,92 @@ internal static unsafe class VulkanVideoPresenter case VulkanComputeGuestDispatch computeDispatch: ExecuteComputeDispatch(computeDispatch); break; + case VulkanGuestImageWrite guestImageWrite: + ExecuteGuestImageWrite(guestImageWrite); + break; + case VulkanOrderedGuestAction orderedAction: + ExecuteOrderedGuestAction(orderedAction); + break; + case VulkanOrderedGuestFlip orderedFlip: + ExecuteOrderedGuestFlip(orderedFlip); + break; + case VulkanOrderedGuestFlipWait flipWait: + ExecuteOrderedGuestFlipWait(flipWait); + break; } } finally { - CompleteGuestWork(); + CompleteGuestWork(pendingGuestWork); + _enqueueAsImmediateQueueFollowup = false; + _immediateFollowupTail = null; + Volatile.Write(ref _executingGuestWorkSequence, 0); + } + + if (workStart != 0) + { + var elapsedMs = (System.Diagnostics.Stopwatch.GetTimestamp() - workStart) + * 1000.0 / System.Diagnostics.Stopwatch.Frequency; + if (elapsedMs > 250.0) + { + var desc = work switch + { + VulkanComputeGuestDispatch c => $"compute cs=0x{c.ShaderAddress:X16} groups={c.GroupCountX}x{c.GroupCountY}x{c.GroupCountZ}", + VulkanOffscreenGuestDraw d => + $"draw mrt={d.Targets.Count} " + + $"rt=0x{d.Targets[0].Address:X16} " + + $"{d.Targets[0].Width}x{d.Targets[0].Height}", + _ => work.GetType().Name, + }; + Console.Error.WriteLine( + $"[LOADER][WARN] vk.slow_render_work {elapsedMs:F0}ms " + + $"queue={pendingGuestWork.Queue.Name} " + + $"submission={pendingGuestWork.Queue.SubmissionId} " + + $"sequence={pendingGuestWork.Sequence}: {desc}"); + } } completedWork++; + + // Return to the main-thread event pump + present once the + // per-frame budget is spent; remaining guest work is drained + // on subsequent Render() calls. Without this a compute-heavy + // backlog freezes the window (macOS "Not Responding"). + if (System.Diagnostics.Stopwatch.GetTimestamp() >= renderWorkDeadline) + { + break; + } } + FlushBatchedGuestCommands(); + CollectAbandonedGuestImageVersions(); + if (!TryTakePresentation(_presentedSequence, out var presentation)) { + // A render-loop tick with no newer flip is normal. Warn only when + // an actual queued presentation is waiting on unfinished guest work. + if (ShouldTracePresentedGuestImageContentsForDiagnostics() && + HasPendingGuestPresentation(_presentedSequence) && + _presentNotTakenLoggedSequence != _presentedSequence) + { + _presentNotTakenLoggedSequence = _presentedSequence; + Console.Error.WriteLine( + $"[LOADER][WARN] vk.present_not_taken seq={_presentedSequence} " + + "— presentation submitted but its required guest work isn't complete; nothing shown."); + } + return; } + if (ShouldTracePresentedGuestImageContentsForDiagnostics()) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.present_taken addr=0x{presentation.GuestImageAddress:X16} " + + $"version={presentation.GuestImageVersion} " + + $"drawKind={presentation.DrawKind} hasPixels={presentation.Pixels is not null} " + + $"hasTranslatedDraw={presentation.TranslatedDraw is not null}"); + } + if (presentation.Pixels is null && presentation.DrawKind != GuestDrawKind.FullscreenBarycentric && presentation.TranslatedDraw is null && @@ -6089,8 +11192,6 @@ internal static unsafe class VulkanVideoPresenter return; } - CompletePendingPresentation(wait: true); - byte[]? pixels = null; if (presentation.Pixels is { } sourcePixels) { @@ -6110,22 +11211,57 @@ internal static unsafe class VulkanVideoPresenter TranslatedDrawResources? translatedResources = null; GuestImageResource? presentedGuestImage = null; - var tracePresentedGuestImage = false; - if (presentation.GuestImageAddress != 0 && - (!_guestImages.TryGetValue( - presentation.GuestImageAddress, - out presentedGuestImage) || - !presentedGuestImage.Initialized)) + var ownsPresentedGuestImageVersion = false; + if (presentation.GuestImageVersion != 0) { + ownsPresentedGuestImageVersion = _guestImageVersions.Remove( + presentation.GuestImageVersion, + out presentedGuestImage); + } + else if (presentation.GuestImageAddress != 0) + { + _guestImages.TryGetValue( + presentation.GuestImageAddress, + out presentedGuestImage); + } + + if (presentation.GuestImageAddress != 0 && + (presentedGuestImage is null || !presentedGuestImage.Initialized)) + { + if (ShouldTracePresentedGuestImageContentsForDiagnostics()) + { + Console.Error.WriteLine( + $"[LOADER][WARN] vk.present_dropped addr=0x{presentation.GuestImageAddress:X16} " + + $"version={presentation.GuestImageVersion} " + + $"found={(presentedGuestImage is not null)} " + + $"initialized={(presentedGuestImage?.Initialized ?? false)} " + + $"— no swapchain present this frame (black)."); + } + + if (ownsPresentedGuestImageVersion && presentedGuestImage is not null) + { + DestroyGuestImage(presentedGuestImage); + } + return; } + if (ownsPresentedGuestImageVersion) + { + System.Diagnostics.Debug.Assert( + _frameGuestImageVersions[frameSlot] is null, + "A reusable frame slot cannot still own a flip version."); + _frameGuestImageVersions[frameSlot] = presentedGuestImage; + } if (presentedGuestImage is not null) { _directPresentationCount++; - if (ShouldSamplePresentedGuestImageForDiagnostics( - _directPresentationCount)) + var traceAddressedPresentation = + ShouldTraceAddressedPresentedGuestImage(presentedGuestImage); + if (traceAddressedPresentation || + ShouldTracePresentedGuestImageContentsForDiagnostics() && + (_directPresentationCount is 1 or 30 or 120 || + _directPresentationCount % 600 == 0)) { - tracePresentedGuestImage = true; Console.Error.WriteLine( $"[LOADER][TRACE] vk.present_sample frame={_directPresentationCount} " + $"addr=0x{presentedGuestImage.Address:X16}"); @@ -6142,19 +11278,14 @@ internal static unsafe class VulkanVideoPresenter [_swapchainFormat], _extent); if (ShouldTracePresentedGuestImageContentsForDiagnostics() && - !_firstGuestDrawPresented) + !_firstGuestDrawPresented && + translatedResources.Textures is + [ + { GuestImage: { } guestImage }, + ] && + _tracedGuestImageContents.Add(guestImage.Address)) { - Console.Error.WriteLine( - $"[LOADER][TRACE] vk.translated_draw kind={presentation.DrawKind} " + - $"textures={translatedResources.Textures.Length}"); - foreach (var boundTexture in translatedResources.Textures) - { - if (boundTexture.GuestImage is { } guestImage && - _tracedGuestImageContents.Add(guestImage.Address)) - { - TraceGuestImageContents(guestImage); - } - } + TraceGuestImageContents(guestImage); } } catch (Exception exception) @@ -6171,7 +11302,7 @@ internal static unsafe class VulkanVideoPresenter _device, _swapchain, ulong.MaxValue, - _imageAvailable, + _frameImageAvailable[frameSlot], default, &imageIndex); if (acquireResult == Result.ErrorOutOfDateKhr) @@ -6181,6 +11312,19 @@ internal static unsafe class VulkanVideoPresenter { DestroyTranslatedDrawResources(translatedResources); } + if (ownsPresentedGuestImageVersion && presentedGuestImage is not null) + { + if (_frameGuestImageVersions.Length > frameSlot && + ReferenceEquals( + _frameGuestImageVersions[frameSlot], + presentedGuestImage)) + { + _frameGuestImageVersions[frameSlot] = null; + _capturedGuestFlipVersions.Remove( + presentedGuestImage.FlipVersion); + DestroyGuestImage(presentedGuestImage); + } + } return; } @@ -6190,6 +11334,10 @@ internal static unsafe class VulkanVideoPresenter if (pixels is not null) { + // The staging buffer is shared across frame slots; a CPU + // pixel upload (splash / host frames) degrades to serial + // presentation rather than corrupting an in-flight copy. + WaitAllFrameSlots(); void* mapped; Check( _vk.MapMemory(_device, _stagingMemory, 0, (ulong)pixels.Length, 0, &mapped), @@ -6255,11 +11403,16 @@ internal static unsafe class VulkanVideoPresenter $"Unsupported translated guest draw: {presentation.DrawKind}."); } + if (PerfOverlay.Enabled) + { + RecordOverlayBlit(imageIndex, frameSlot); + } + Check(_vk.EndCommandBuffer(_commandBuffer), "vkEndCommandBuffer"); - var imageAvailable = _imageAvailable; + var imageAvailable = _frameImageAvailable[frameSlot]; var commandBuffer = _commandBuffer; - var renderFinished = _renderFinished; + var renderFinished = _renderFinishedPerImage[imageIndex]; var submitInfo = new SubmitInfo { SType = StructureType.SubmitInfo, @@ -6271,16 +11424,21 @@ internal static unsafe class VulkanVideoPresenter SignalSemaphoreCount = 1, PSignalSemaphores = &renderFinished, }; - var presentationFence = _presentationFence; Check( - _vk.ResetFences(_device, 1, &presentationFence), - "vkResetFences(presentation)"); - Check( - _vk.QueueSubmit(_queue, 1, &submitInfo, presentationFence), + _vk.QueueSubmit(_queue, 1, &submitInfo, _frameFences[frameSlot]), "vkQueueSubmit"); - _presentationInFlight = true; - _pendingPresentationResources = translatedResources; - translatedResources = null; + _submitTimeline++; + _frameTimelines[frameSlot] = _submitTimeline; + _frameFencePending[frameSlot] = true; + _frameTranslatedResources[frameSlot] = translatedResources; + if (translatedResources is not null) + { + // CPU-side layout bookkeeping only; later command buffers are + // recorded after this submission, so queue order makes the + // flags valid before any dependent GPU work runs. + MarkSampledImagesInitialized(translatedResources); + MarkStorageImagesInitialized(translatedResources); + } var swapchain = _swapchain; var presentInfo = new PresentInfoKHR @@ -6295,14 +11453,8 @@ internal static unsafe class VulkanVideoPresenter var presentResult = _swapchainApi.QueuePresent(_queue, &presentInfo); if (presentResult == Result.ErrorOutOfDateKhr) { - Check(_vk.QueueWaitIdle(_queue), "vkQueueWaitIdle"); - CompletePendingPresentation(wait: false); - CollectCompletedGuestSubmissions(waitForOldest: false); - if (translatedResources is not null) - { - DestroyTranslatedDrawResources(translatedResources); - } - + // The submitted frame still executes; RecreateSwapchainResources + // drains it (and every frame slot) before destroying anything. RecreateSwapchainResources("vkQueuePresentKHR", presentResult); return; } @@ -6310,28 +11462,25 @@ internal static unsafe class VulkanVideoPresenter CheckSwapchainResult(presentResult, "vkQueuePresentKHR"); recreateAfterPresent |= presentResult == Result.SuboptimalKhr; VideoOutExports.ReportPresentedFrame(); - _performanceHudPresentedFrames++; - if (_swapchainReadbackPending) + PerfOverlay.RecordPresent(); + if (_swapchainReadbackPending || !_pendingAliasImageDumps.IsEmpty) { - CompletePendingPresentation(wait: true); - TraceSwapchainReadback(); - } - // Report the actual presented pixels before starting the larger - // source-image readback. If a guest draw wedges the GPU, the - // source probe can block in vkQueueWaitIdle; doing it first used - // to hide whether the swapchain itself was black and made the - // diagnostic run stop immediately after vk.present_sample. - if (tracePresentedGuestImage && presentedGuestImage is not null) - { - TraceGuestImageContents(presentedGuestImage); - } - while (_pendingAliasImageDumps.TryDequeue(out var aliasImage)) - { - TraceGuestImageContents(aliasImage); - } - CollectCompletedGuestSubmissions(waitForOldest: false); + // Diagnostics read back GPU memory and need this frame done. + WaitFrameSlot(frameSlot); + if (_swapchainReadbackPending) + { + TraceSwapchainReadback(); + } + while (_pendingAliasImageDumps.TryDequeue(out var aliasImage)) + { + TraceGuestImageContents(aliasImage); + } + } + + CollectCompletedGuestSubmissions(waitForOldest: false); _imageInitialized[imageIndex] = true; + _currentFrameSlot = (frameSlot + 1) % MaxFramesInFlight; _presentedSequence = presentation.Sequence; if (presentation.IsSplash && !_splashPresented) { @@ -6490,6 +11639,34 @@ internal static unsafe class VulkanVideoPresenter try { var bytes = new ReadOnlySpan(mapped, checked((int)byteCount)); + if (GuestImageTraceInterval() is not null && bytesPerPixel == 4) + { + long r = 0, g = 0, b = 0, a = 0, samples = 0; + for (var offset = 0; offset + 4 <= bytes.Length; offset += 4 * 251) + { + r += bytes[offset]; + g += bytes[offset + 1]; + b += bytes[offset + 2]; + a += bytes[offset + 3]; + samples++; + } + + if (samples > 0) + { + Console.Error.WriteLine( + $"[RB] addr=0x{image.Address:X} " + + $"mean={r / samples},{g / samples},{b / samples},A{a / samples} " + + $"sample_unique={CountSampledUniquePixels(bytes, bytesPerPixel)}"); + } + + if (++_intervalReadbackCount % 25 == 0) + { + DumpGuestImageBytes(image, bytes); + } + + return; + } + var nonzeroBytes = 0L; ulong hash = 14695981039346656037UL; foreach (var value in bytes) @@ -6514,7 +11691,8 @@ internal static unsafe class VulkanVideoPresenter $"size={image.Width}x{image.Height} format={image.Format} " + $"nonzero_bytes={nonzeroBytes}/{byteCount} " + $"nonblack_pixels={nonblackPixels}/{(ulong)image.Width * image.Height} " + - $"center={center} hash=0x{hash:X16}"); + $"center={center} sample_unique={CountSampledUniquePixels(bytes, bytesPerPixel)} " + + $"hash=0x{hash:X16}"); DumpGuestImageBytes(image, bytes); } finally @@ -6529,6 +11707,37 @@ internal static unsafe class VulkanVideoPresenter } } + private static int CountSampledUniquePixels( + ReadOnlySpan bytes, + uint bytesPerPixel) + { + if (bytesPerPixel == 0) + { + return 0; + } + + var unique = new HashSet(); + var stride = checked((int)bytesPerPixel * 251); + for (var offset = 0; + offset + bytesPerPixel <= bytes.Length; + offset += stride) + { + ulong hash = 14695981039346656037UL; + for (var index = 0; index < bytesPerPixel; index++) + { + hash = (hash ^ bytes[offset + index]) * 1099511628211UL; + } + + unique.Add(hash); + if (unique.Count > 256) + { + return 257; + } + } + + return unique.Count; + } + private static void DumpGuestImageBytes( GuestImageResource image, ReadOnlySpan bytes) @@ -6548,22 +11757,49 @@ internal static unsafe class VulkanVideoPresenter File.WriteAllBytes(path, bytes.ToArray()); } + // Metal cannot blend into integer render targets or 32-bit-per-channel + // float targets (unsupported on Apple-family GPUs). Enabling blend on + // one makes vkCreateGraphicsPipelines fail with ErrorInitializationFailed + // (and trips a Metal "not blendable" validation assertion), so the draw + // is silently dropped. Force blend off for those; blending on an integer + // target is meaningless on real hardware anyway. + private static bool IsBlendableFormat(Format format) => + format switch + { + Format.R8Uint or Format.R8Sint or + Format.R8G8B8A8Uint or Format.R8G8B8A8Sint or + Format.R16G16Uint or Format.R16G16Sint or + Format.R16G16B16A16Uint or Format.R16G16B16A16Sint or + Format.R32Uint or Format.R32Sint or + Format.R32G32Uint or Format.R32G32Sint or + Format.R32G32B32A32Uint or Format.R32G32B32A32Sint or + Format.R32Sfloat or + Format.R32G32Sfloat or + Format.R32G32B32A32Sfloat => false, + _ => true, + }; + private static uint GetReadbackBytesPerPixel(Format format) => format switch { Format.R8Unorm or Format.R8Uint or Format.R8Sint => 1, + Format.R8G8Unorm or + Format.R8G8Uint or + Format.R8G8Sint => 2, Format.R32Uint or Format.R32Sint or Format.R32Sfloat or + Format.B10G11R11UfloatPack32 or Format.R16G16Uint or Format.R16G16Sint or Format.R16G16Sfloat or Format.R8G8B8A8Uint or Format.R8G8B8A8Sint or Format.R8G8B8A8Unorm or - Format.A2R10G10B10UnormPack32 => 4, + Format.A2R10G10B10UnormPack32 or + Format.A2B10G10R10UnormPack32 => 4, Format.R16G16B16A16Uint or Format.R16G16B16A16Sint or Format.R16G16B16A16Sfloat => 8, @@ -6587,7 +11823,8 @@ internal static unsafe class VulkanVideoPresenter var pixel = bytes.Slice(offset, (int)bytesPerPixel); var hasColor = format switch { - Format.A2R10G10B10UnormPack32 => + Format.A2R10G10B10UnormPack32 or + Format.A2B10G10R10UnormPack32 => (BitConverter.ToUInt32(pixel) & 0x3FFFFFFFu) != 0, Format.R8G8B8A8Uint or Format.R8G8B8A8Sint or @@ -6608,6 +11845,11 @@ internal static unsafe class VulkanVideoPresenter private void RecordTranslatedDraw(uint imageIndex, TranslatedDrawResources resources) { BeginDebugLabel(_commandBuffer, "SharpEmu swapchain draw"); + RecordGlobalBufferVisibilityBarrier( + _commandBuffer, + resources, + PipelineStageFlags.VertexShaderBit | + PipelineStageFlags.FragmentShaderBit); RecordTextureUploads(resources, PipelineStageFlags.FragmentShaderBit); RecordStorageImagesForWrite(resources, PipelineStageFlags.FragmentShaderBit); RecordTranslatedGraphicsPass( @@ -6625,29 +11867,21 @@ internal static unsafe class VulkanVideoPresenter { foreach (var texture in resources.Textures) { + if (texture.GuestDepth is { } depth) + { + RecordGuestDepthForSampling(depth, shaderStage); + } + if (!texture.NeedsUpload) { continue; } - // InitialUploadPending means this upload still has to perform - // the image's first layout transition. Treating it as prior - // contents records ShaderReadOnlyOptimal as oldLayout even - // though a freshly created image is still Undefined. Linux - // validation reports VUID-vkCmdDraw-None-09600 and NVIDIA - // samples the uninitialized (black) image in that case. - var hasPriorContents = - texture.GuestImage is { Initialized: true }; var toTransfer = new ImageMemoryBarrier { SType = StructureType.ImageMemoryBarrier, - SrcAccessMask = hasPriorContents - ? AccessFlags.ShaderReadBit - : 0, DstAccessMask = AccessFlags.TransferWriteBit, - OldLayout = hasPriorContents - ? ImageLayout.ShaderReadOnlyOptimal - : ImageLayout.Undefined, + OldLayout = ImageLayout.Undefined, NewLayout = ImageLayout.TransferDstOptimal, SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, DstQueueFamilyIndex = Vk.QueueFamilyIgnored, @@ -6656,9 +11890,7 @@ internal static unsafe class VulkanVideoPresenter }; _vk.CmdPipelineBarrier( _commandBuffer, - hasPriorContents - ? PipelineStageFlags.AllCommandsBit - : PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TopOfPipeBit, PipelineStageFlags.TransferBit, 0, 0, @@ -6711,6 +11943,426 @@ internal static unsafe class VulkanVideoPresenter null, 1, &toShaderRead); + if (texture.Cached) + { + // The queue executes command buffers in submission order, + // so once this upload is recorded every later draw can + // reuse the image without restaging it. + texture.NeedsUpload = false; + } + } + } + + private void RecordGuestDepthForSampling( + GuestDepthResource depth, + PipelineStageFlags shaderStage) + { + if (depth.Layout == ImageLayout.ShaderReadOnlyOptimal) + { + return; + } + + if (!depth.Initialized) + { + var depthRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1); + var toTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = depth.Layout, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = depth.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &toTransfer); + var clearValue = new ClearDepthStencilValue(depth.ClearDepth, 0); + _vk.CmdClearDepthStencilImage( + _commandBuffer, + depth.Image, + ImageLayout.TransferDstOptimal, + &clearValue, + 1, + &depthRange); + depth.Initialized = true; + if (depth.InitializationSource == "none") + { + depth.InitializationSource = "sample-clear"; + } + depth.Layout = ImageLayout.TransferDstOptimal; + } + + var barrier = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = depth.Layout == ImageLayout.TransferDstOptimal + ? AccessFlags.TransferWriteBit + : AccessFlags.DepthStencilAttachmentWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = depth.Layout, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = depth.Image, + SubresourceRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + depth.Layout == ImageLayout.TransferDstOptimal + ? PipelineStageFlags.TransferBit + : PipelineStageFlags.LateFragmentTestsBit, + shaderStage, + 0, + 0, + null, + 0, + null, + 1, + &barrier); + depth.Layout = ImageLayout.ShaderReadOnlyOptimal; + } + + private void RecordRenderTargetFeedbackSnapshots( + TranslatedDrawResources resources, + PipelineStageFlags shaderStage) + { + foreach (var texture in resources.Textures) + { + if (texture.FeedbackSource is not { } source) + { + continue; + } + + // Initialize every destination mip to a deterministic zero. + // Render-target writes currently populate mip 0; leaving the + // remaining sampled mips undefined turns guest LOD selection + // into driver-dependent colored garbage. + var destinationToTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = texture.Image, + SubresourceRange = ColorSubresourceRange(0, source.MipLevels), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &destinationToTransfer); + + var clearValue = new ClearColorValue(0f, 0f, 0f, 0f); + // Avoid overlapping a clear and copy on mip 0: without an + // intervening dependency two transfer writes to one + // subresource are not ordered merely because they were + // recorded in that order. Initialized sources overwrite mip + // 0 directly and clear only the otherwise undefined tail. + var clearBaseMip = source.Initialized ? 1u : 0u; + if (clearBaseMip < source.MipLevels) + { + var destinationRange = ColorSubresourceRange( + clearBaseMip, + source.MipLevels - clearBaseMip); + _vk.CmdClearColorImage( + _commandBuffer, + texture.Image, + ImageLayout.TransferDstOptimal, + &clearValue, + 1, + &destinationRange); + } + + if (source.Initialized) + { + var sourceToTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.ShaderReadBit, + DstAccessMask = AccessFlags.TransferReadBit, + OldLayout = ImageLayout.ShaderReadOnlyOptimal, + NewLayout = ImageLayout.TransferSrcOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + // Only mip 0 has defined render-target contents. + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + shaderStage | + PipelineStageFlags.ColorAttachmentOutputBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &sourceToTransfer); + + var copy = new ImageCopy + { + SrcSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, + 0, + 0, + 1), + DstSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, + 0, + 0, + 1), + Extent = new Extent3D(source.Width, source.Height, 1), + }; + _vk.CmdCopyImage( + _commandBuffer, + source.Image, + ImageLayout.TransferSrcOptimal, + texture.Image, + ImageLayout.TransferDstOptimal, + 1, + ©); + + var sourceToShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferReadBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferSrcOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + shaderStage, + 0, + 0, + null, + 0, + null, + 1, + &sourceToShaderRead); + } + + var destinationToShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = texture.Image, + SubresourceRange = ColorSubresourceRange(0, source.MipLevels), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + shaderStage, + 0, + 0, + null, + 0, + null, + 1, + &destinationToShaderRead); + + TraceVulkanShader( + $"vk.feedback_snapshot_copy addr=0x{source.Address:X16} " + + $"size={source.Width}x{source.Height} initialized={source.Initialized}"); + } + } + + private void RecordDepthFeedbackSnapshots( + TranslatedDrawResources resources, + PipelineStageFlags shaderStage) + { + foreach (var texture in resources.Textures) + { + if (texture.DepthFeedbackSource is not { } source) + { + continue; + } + + var depthRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1); + var destinationToTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = texture.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &destinationToTransfer); + + if (source.Initialized) + { + var sourceToTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = source.Layout == ImageLayout.ShaderReadOnlyOptimal + ? AccessFlags.ShaderReadBit + : AccessFlags.DepthStencilAttachmentWriteBit, + DstAccessMask = AccessFlags.TransferReadBit, + OldLayout = source.Layout, + NewLayout = ImageLayout.TransferSrcOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + shaderStage | + PipelineStageFlags.EarlyFragmentTestsBit | + PipelineStageFlags.LateFragmentTestsBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &sourceToTransfer); + var copy = new ImageCopy + { + SrcSubresource = new ImageSubresourceLayers( + ImageAspectFlags.DepthBit, + 0, + 0, + 1), + DstSubresource = new ImageSubresourceLayers( + ImageAspectFlags.DepthBit, + 0, + 0, + 1), + Extent = new Extent3D(source.Width, source.Height, 1), + }; + _vk.CmdCopyImage( + _commandBuffer, + source.Image, + ImageLayout.TransferSrcOptimal, + texture.Image, + ImageLayout.TransferDstOptimal, + 1, + ©); + + var sourceToAttachment = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferReadBit, + DstAccessMask = + AccessFlags.DepthStencilAttachmentReadBit | + AccessFlags.DepthStencilAttachmentWriteBit, + OldLayout = ImageLayout.TransferSrcOptimal, + NewLayout = ImageLayout.DepthStencilAttachmentOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.EarlyFragmentTestsBit | + PipelineStageFlags.LateFragmentTestsBit, + 0, + 0, + null, + 0, + null, + 1, + &sourceToAttachment); + source.Layout = ImageLayout.DepthStencilAttachmentOptimal; + } + else + { + var clearValue = new ClearDepthStencilValue(source.ClearDepth, 0); + _vk.CmdClearDepthStencilImage( + _commandBuffer, + texture.Image, + ImageLayout.TransferDstOptimal, + &clearValue, + 1, + &depthRange); + } + + var destinationToShader = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = texture.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + shaderStage, + 0, + 0, + null, + 0, + null, + 1, + &destinationToShader); } } @@ -6824,11 +12476,9 @@ internal static unsafe class VulkanVideoPresenter guestImage.Initialized = true; guestImage.InitialUploadPending = false; - var format = GetGuestTextureFormat(guestImage.Format); - if (format != 0) + if (guestImage.GuestFormat != 0) { - _availableGuestImages[texture.Address] = format; - _gpuGuestImages[texture.Address] = format; + _availableGuestImages[texture.Address] = guestImage.GuestFormat; } if (traceContents && @@ -6876,27 +12526,200 @@ internal static unsafe class VulkanVideoPresenter } } - private bool ShouldTraceGuestImageContents(GuestImageResource image) + private bool ShouldTraceGuestImageContents( + GuestImageResource image, + ulong shaderAddress = 0) { if (image.Address == 0) { return false; } + if (_traceGuestImageShaderFilterEnabled && + !AddressListContains( + "SHARPEMU_TRACE_GUEST_IMAGE_SHADER_ADDRS", + shaderAddress)) + { + return false; + } + + if ((_traceGuestImageWidth > 0 && image.Width != _traceGuestImageWidth) || + (_traceGuestImageHeight > 0 && image.Height != _traceGuestImageHeight)) + { + return false; + } + + if (!string.IsNullOrWhiteSpace(_traceGuestImageFormat) && + (!Enum.TryParse( + _traceGuestImageFormat, + ignoreCase: true, + out var expectedFormat) || + image.Format != expectedFormat)) + { + return false; + } + var addressMatched = ShouldTraceGuestImageAddressForDiagnostics(image.Address); + if (addressMatched && _traceGuestImageOccurrence > 0) + { + var count = _guestImageTraceCounts.TryGetValue(image.Address, out var previous) + ? previous + 1 + : 1; + _guestImageTraceCounts[image.Address] = count; + return count == _traceGuestImageOccurrence; + } + var broadTrace = ShouldTraceGuestImageContentsForDiagnostics() && image.Width >= 1280 && image.Height >= 720; + if (GuestImageTraceInterval() is { } interval) + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_GUEST_IMAGE_ADDRS"); + if (!string.IsNullOrWhiteSpace(addressFilter) && !addressMatched) + { + return false; + } + + if (image.Width < 1280 || image.Height < 720) + { + return false; + } + + _globalGuestImageDrawCount++; + if (_globalGuestImageDrawCount < GuestImageTraceStartAfter() || + _intervalReadbackCount > 3000) + { + return false; + } + + var count = _guestImageTraceCounts.TryGetValue(image.Address, out var previous) + ? previous + 1 + : 1; + _guestImageTraceCounts[image.Address] = count; + return count % interval == 0; + } + return (addressMatched || broadTrace) && _tracedGuestImageContents.Add(image.Address); } - private static bool ShouldTraceGuestImageContentsForDiagnostics() => + private readonly Dictionary _guestImageTraceCounts = new(); + private long _globalGuestImageDrawCount; + private long _intervalReadbackCount; + + private static long? _cachedGuestImageTraceInterval = long.MinValue; + private static long _cachedGuestImageTraceStartAfter; + + // SHARPEMU_TRACE_GUEST_IMAGES=every:N[@M] — read back 1280x720+ guest + // images every Nth draw into each, starting after M total such draws. + private static long? GuestImageTraceInterval() + { + if (_cachedGuestImageTraceInterval == long.MinValue) + { + var mode = Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"); + long? interval = null; + if (mode is not null && mode.StartsWith("every:", StringComparison.Ordinal)) + { + var spec = mode["every:".Length..]; + var at = spec.IndexOf('@'); + var intervalText = at < 0 ? spec : spec[..at]; + if (long.TryParse(intervalText, out var parsed) && parsed > 0) + { + interval = parsed; + } + + if (at >= 0 && long.TryParse(spec[(at + 1)..], out var after) && after > 0) + { + _cachedGuestImageTraceStartAfter = after; + } + } + + _cachedGuestImageTraceInterval = interval; + } + + return _cachedGuestImageTraceInterval; + } + + private static long GuestImageTraceStartAfter() + { + _ = GuestImageTraceInterval(); + return _cachedGuestImageTraceStartAfter; + } + + // Diagnostics toggles are read once: these run per draw / per cached + // texture hit, and env lookups plus string parsing are far too + // expensive there (and non-trivially so under Rosetta 2). + private static readonly string? _traceGuestImagesMode = + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"); + private static readonly bool _traceGuestImagesEnabled = + string.Equals(_traceGuestImagesMode, "1", StringComparison.Ordinal); + private static readonly bool _tracePresentedGuestImagesEnabled = + _traceGuestImagesEnabled || + string.Equals(_traceGuestImagesMode, "present", StringComparison.OrdinalIgnoreCase); + private static readonly bool _traceVulkanResourcesEnabled = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"), + Environment.GetEnvironmentVariable("SHARPEMU_LOG_VK_RESOURCES"), "1", StringComparison.Ordinal); + private static readonly bool _traceVulkanShaderEnabled = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), + "1", + StringComparison.Ordinal) || + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC_SHADER"), + "1", + StringComparison.Ordinal); + private static readonly bool _traceDepthInitialization = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_DEPTH_INIT"), + "1", + StringComparison.Ordinal); + private static readonly long _traceGuestImageOccurrence = + long.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_OCCURRENCE"), + out var traceGuestImageOccurrence) && + traceGuestImageOccurrence > 0 + ? traceGuestImageOccurrence + : 0; + private static readonly uint _traceGuestImageWidth = + uint.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_WIDTH"), + out var traceGuestImageWidth) + ? traceGuestImageWidth + : 0; + private static readonly uint _traceGuestImageHeight = + uint.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_HEIGHT"), + out var traceGuestImageHeight) + ? traceGuestImageHeight + : 0; + private static readonly string? _traceGuestImageFormat = + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_FORMAT"); + private static readonly long _tracePresentedGuestImageOccurrence = + long.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_PRESENTED_GUEST_IMAGE_OCCURRENCE"), + out var tracePresentedGuestImageOccurrence) && + tracePresentedGuestImageOccurrence > 0 + ? tracePresentedGuestImageOccurrence + : 0; + private static readonly bool _traceGuestImageShaderFilterEnabled = + !string.IsNullOrWhiteSpace( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_GUEST_IMAGE_SHADER_ADDRS")); + private static readonly bool _traceGuestImageAddressFilterEnabled = + !string.IsNullOrWhiteSpace( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_GUEST_IMAGE_ADDRS")); + private static readonly System.Collections.Concurrent.ConcurrentDictionary< + string, + (bool Wildcard, ulong[] Addresses)> _cachedAddressLists = new(); + + private static bool ShouldTraceGuestImageContentsForDiagnostics() => + _traceGuestImagesEnabled; private static bool ShouldTraceGuestImageAddressForDiagnostics(ulong address) { @@ -6916,19 +12739,27 @@ internal static unsafe class VulkanVideoPresenter string environmentVariable, ulong address) { - var addresses = Environment.GetEnvironmentVariable(environmentVariable); + var (wildcard, addresses) = _cachedAddressLists.GetOrAdd( + environmentVariable, + static name => ParseAddressList(Environment.GetEnvironmentVariable(name))); + return wildcard || Array.IndexOf(addresses, address) >= 0; + } + + private static (bool Wildcard, ulong[] Addresses) ParseAddressList(string? addresses) + { if (string.IsNullOrWhiteSpace(addresses)) { - return false; + return (false, []); } + var parsedAddresses = new List(); foreach (var token in addresses.Split( [',', ';', ' ', '\t'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) { if (token == "*") { - return true; + return (true, []); } var span = token.AsSpan(); @@ -6941,21 +12772,40 @@ internal static unsafe class VulkanVideoPresenter span, System.Globalization.NumberStyles.HexNumber, System.Globalization.CultureInfo.InvariantCulture, - out var parsed) && - parsed == address) + out var parsed)) { - return true; + parsedAddresses.Add(parsed); } } - return false; + return (false, parsedAddresses.ToArray()); + } + + private static bool ShouldTracePresentedGuestImageContentsForDiagnostics() => + _tracePresentedGuestImagesEnabled; + + private bool ShouldTraceAddressedPresentedGuestImage(GuestImageResource image) + { + if (!AddressListContains( + "SHARPEMU_TRACE_PRESENTED_GUEST_IMAGE_ADDRS", + image.Address)) + { + return false; + } + + var count = _presentedGuestImageTraceCounts.TryGetValue( + image.Address, + out var previous) + ? previous + 1 + : 1; + _presentedGuestImageTraceCounts[image.Address] = count; + return _tracePresentedGuestImageOccurrence == 0 + ? count == 1 + : count == _tracePresentedGuestImageOccurrence; } private static bool ShouldTraceVulkanResources() => - string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_LOG_VK_RESOURCES"), - "1", - StringComparison.Ordinal); + _traceVulkanResourcesEnabled; private void RecordTranslatedGraphicsPass( TranslatedDrawResources resources, @@ -6963,20 +12813,54 @@ internal static unsafe class VulkanVideoPresenter Framebuffer framebuffer, Extent2D extent) { - var clearValue = default(ClearValue); + BeginTranslatedRenderPass(renderPass, framebuffer, extent); + RecordTranslatedDrawInPass(resources, extent); + _vk.CmdEndRenderPass(_commandBuffer); + } + + private void BeginTranslatedRenderPass( + RenderPass renderPass, + Framebuffer framebuffer, + Extent2D extent, + int colorAttachmentCount = 1, + bool hasDepthAttachment = false, + float clearDepth = 1f) + { + colorAttachmentCount = Math.Max(colorAttachmentCount, 1); + var clearValueCount = colorAttachmentCount + (hasDepthAttachment ? 1 : 0); + var clearValues = stackalloc ClearValue[clearValueCount]; + for (var index = 0; index < colorAttachmentCount; index++) + { + clearValues[index] = default; + } + // Reverse-Z is not assumed; clear depth to 1.0 (far) so a standard + // LessOrEqual/Less test keeps the nearest fragment. + if (hasDepthAttachment) + { + clearValues[colorAttachmentCount] = new ClearValue + { + DepthStencil = new ClearDepthStencilValue(clearDepth, 0), + }; + } var renderPassInfo = new RenderPassBeginInfo { SType = StructureType.RenderPassBeginInfo, RenderPass = renderPass, Framebuffer = framebuffer, RenderArea = new Rect2D(new Offset2D(0, 0), extent), - ClearValueCount = 1, - PClearValues = &clearValue, + ClearValueCount = (uint)clearValueCount, + PClearValues = clearValues, }; _vk.CmdBeginRenderPass( _commandBuffer, &renderPassInfo, SubpassContents.Inline); + } + + private void RecordTranslatedDrawInPass( + TranslatedDrawResources resources, + Extent2D extent) + { _vk.CmdBindPipeline( _commandBuffer, PipelineBindPoint.Graphics, @@ -6998,7 +12882,6 @@ internal static unsafe class VulkanVideoPresenter var drawScissor = ClampScissor(resources.Scissor, extent); if (drawScissor.Width == 0 || drawScissor.Height == 0) { - _vk.CmdEndRenderPass(_commandBuffer); return; } @@ -7027,36 +12910,68 @@ internal static unsafe class VulkanVideoPresenter offsets); } - var scissor = new Rect2D( - new Offset2D(drawScissor.X, drawScissor.Y), - new Extent2D(drawScissor.Width, drawScissor.Height)); - _vk.CmdSetScissor(_commandBuffer, 0, 1, &scissor); + // Replaying a full-screen primitive once per 512x512 scissor tile + // multiplies an ordinary 4K composite into 32 complete draws. On + // MoltenVK this starves the render thread and makes the guest fall + // behind its own flip queue. Vulkan clips a normal fullscreen draw + // efficiently; keep tiling only as an explicit driver diagnostic. + var maxPixelsPerDraw = Environment.GetEnvironmentVariable( + "SHARPEMU_ENABLE_CHUNKED_DRAWS") == "1" + ? 512u * 512u + : uint.MaxValue; + var rowsPerDraw = Math.Max( + 1u, + Math.Min(drawScissor.Height, maxPixelsPerDraw / Math.Max(drawScissor.Width, 1u))); + var drawCount = 0u; + for (var y = 0u; y < drawScissor.Height; y += rowsPerDraw) + { + var scissor = new Rect2D( + new Offset2D( + drawScissor.X, + checked(drawScissor.Y + (int)y)), + new Extent2D( + drawScissor.Width, + Math.Min(rowsPerDraw, drawScissor.Height - y))); + _vk.CmdSetScissor(_commandBuffer, 0, 1, &scissor); - if (resources.IndexBuffer.Handle != 0) - { - _vk.CmdBindIndexBuffer( - _commandBuffer, - resources.IndexBuffer, - 0, - resources.Index32Bit ? IndexType.Uint32 : IndexType.Uint16); - _vk.CmdDrawIndexed( - _commandBuffer, - resources.VertexCount, - resources.InstanceCount, - 0, - 0, - 0); + if (resources.IndexBuffer.Handle != 0) + { + _vk.CmdBindIndexBuffer( + _commandBuffer, + resources.IndexBuffer, + 0, + resources.Index32Bit ? IndexType.Uint32 : IndexType.Uint16); + _vk.CmdDrawIndexed( + _commandBuffer, + resources.VertexCount, + resources.InstanceCount, + 0, + 0, + 0); + } + else + { + _vk.CmdDraw( + _commandBuffer, + resources.VertexCount, + resources.InstanceCount, + 0, + 0); + } + + drawCount++; } - else + + if (drawCount > 1) { - _vk.CmdDraw( - _commandBuffer, - resources.VertexCount, - resources.InstanceCount, - 0, - 0); + TraceVulkanShader( + $"vk.graphics_chunked target={extent.Width}x{extent.Height} " + + $"draws={drawCount} rows={rowsPerDraw} " + + $"scissor={drawScissor.X},{drawScissor.Y},{drawScissor.Width}x{drawScissor.Height} " + + $"viewport={drawViewport.X:0.###},{drawViewport.Y:0.###}," + + $"{drawViewport.Width:0.###}x{drawViewport.Height:0.###} " + + $"name={resources.DebugName}"); } - _vk.CmdEndRenderPass(_commandBuffer); } private void DestroyTranslatedDrawResources(TranslatedDrawResources resources) @@ -7073,7 +12988,7 @@ internal static unsafe class VulkanVideoPresenter foreach (var texture in resources.Textures) { - if (texture is null) + if (texture is null || texture.Cached) { continue; } @@ -7112,7 +13027,7 @@ internal static unsafe class VulkanVideoPresenter foreach (var globalBuffer in resources.GlobalMemoryBuffers) { - if (globalBuffer is null) + if (globalBuffer is null || globalBuffer.Allocation is not null) { continue; } @@ -7122,7 +13037,7 @@ internal static unsafe class VulkanVideoPresenter foreach (var vertexBuffer in resources.VertexBuffers) { - if (vertexBuffer is null) + if (vertexBuffer is null || !vertexBuffer.OwnsBuffer) { continue; } @@ -7139,7 +13054,14 @@ internal static unsafe class VulkanVideoPresenter if (resources.DescriptorPool.Handle != 0) { - _vk.DestroyDescriptorPool(_device, resources.DescriptorPool, null); + if (_recycledDescriptorPools.Count < 256) + { + _recycledDescriptorPools.Push(resources.DescriptorPool); + } + else + { + _vk.DestroyDescriptorPool(_device, resources.DescriptorPool, null); + } } if (!resources.DescriptorLayoutCached && @@ -7232,9 +13154,12 @@ internal static unsafe class VulkanVideoPresenter uint imageIndex, GuestImageResource source) { + var presentedCount = Interlocked.Increment(ref _presentedSwapchainCount); + var periodicDumpInterval = SwapchainDumpInterval(); var traceDestination = ShouldTracePresentedGuestImageContentsForDiagnostics() && - !_tracedPresentedSwapchain; + (!_tracedPresentedSwapchain || + periodicDumpInterval > 0 && presentedCount % periodicDumpInterval == 0); _tracedPresentedSwapchain |= traceDestination; BeginDebugLabel( _commandBuffer, @@ -7465,6 +13390,27 @@ internal static unsafe class VulkanVideoPresenter $"format={_swapchainFormat} nonzero_bytes={nonzeroBytes}/{byteCount} " + $"nonblack_pixels={nonblackPixels}/{(ulong)_extent.Width * _extent.Height} " + $"hash=0x{hash:X16}"); + + var dumpDir = Environment.GetEnvironmentVariable("SHARPEMU_GUEST_IMAGE_DUMP_DIR"); + if (!string.IsNullOrWhiteSpace(dumpDir)) + { + Directory.CreateDirectory(dumpDir); + var seq = Interlocked.Increment(ref _guestImageDumpSequence); + var path = Path.Combine( + dumpDir, + $"present-{seq:D4}-{_extent.Width}x{_extent.Height}-{_swapchainFormat}.bgra"); + File.WriteAllBytes(path, bytes.ToArray()); + Console.Error.WriteLine($"[LOADER][TRACE] vk.swapchain_dump path={path}"); + // Continuous readback is intentionally opt-in: each 1080p frame + // is several megabytes and synchronously waits for the GPU. + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_GUEST_IMAGE_DUMP_CONTINUOUS"), + "1", + StringComparison.Ordinal)) + { + _tracedPresentedSwapchain = false; + } + } } finally { @@ -7472,6 +13418,16 @@ internal static unsafe class VulkanVideoPresenter } } + private static readonly long _swapchainDumpInterval = ParseSwapchainDumpInterval(); + + private static long SwapchainDumpInterval() => _swapchainDumpInterval; + + private static long ParseSwapchainDumpInterval() + { + var raw = Environment.GetEnvironmentVariable("SHARPEMU_SWAPCHAIN_DUMP_EVERY"); + return long.TryParse(raw, out var interval) && interval > 0 ? interval : 0; + } + private Extent2D ChooseExtent(SurfaceCapabilitiesKHR capabilities) { if (capabilities.CurrentExtent.Width != uint.MaxValue) @@ -7598,9 +13554,10 @@ internal static unsafe class VulkanVideoPresenter } _vulkanReady = false; _vk.DeviceWaitIdle(_device); - CompletePendingPresentation(wait: false); + SavePipelineCache(force: true); + DrainFrameSlots(); CollectCompletedGuestSubmissions(waitForOldest: false); - SavePipelineCache(); + ClearCachedTextureIdentities(); foreach (var pipeline in _computePipelines.Values) { _vk.DestroyPipeline(_device, pipeline, null); @@ -7611,11 +13568,6 @@ internal static unsafe class VulkanVideoPresenter _vk.DestroyPipeline(_device, pipeline, null); } _graphicsPipelines.Clear(); - if (_pipelineCache.Handle != 0) - { - _vk.DestroyPipelineCache(_device, _pipelineCache, null); - _pipelineCache = default; - } foreach (var layout in _descriptorLayouts.Values) { _vk.DestroyPipelineLayout(_device, layout.PipelineLayout, null); @@ -7634,6 +13586,12 @@ internal static unsafe class VulkanVideoPresenter } _samplers.Clear(); _shaderDigests.Clear(); + WriteBackAllDirtyGuestBuffers(); + foreach (var allocation in _guestBufferAllocations) + { + DestroyGuestBufferAllocation(allocation); + } + _guestBufferAllocations.Clear(); foreach (var allocation in _hostBufferAllocations.Values) { _vk.DestroyBuffer(_device, allocation.Buffer, null); @@ -7646,14 +13604,39 @@ internal static unsafe class VulkanVideoPresenter DestroyGuestImage(guestImage); } _guestImages.Clear(); + foreach (var guestImageVariant in _guestImageVariants.Values) + { + DestroyGuestImage(guestImageVariant); + } + _guestImageVariants.Clear(); + foreach (var guestImageVersion in _guestImageVersions.Values) + { + DestroyGuestImage(guestImageVersion); + } + _guestImageVersions.Clear(); + _capturedGuestFlipVersions.Clear(); + while (_deferredGuestImageVersionDestroys.TryDequeue(out var deferredVersion)) + { + DestroyGuestImage(deferredVersion.Image); + } + foreach (var guestDepth in _guestDepthImages.Values) + { + DestroyGuestDepth(guestDepth); + } + _guestDepthImages.Clear(); lock (_gate) { _availableGuestImages.Clear(); - _gpuGuestImages.Clear(); + _lastOrderedGuestFlipVersions.Clear(); } DestroySwapchainResources(); if (_device.Handle != 0) { + if (_pipelineCache.Handle != 0) + { + _vk.DestroyPipelineCache(_device, _pipelineCache, null); + _pipelineCache = default; + } _vk.DestroyDevice(_device, null); _device = default; } @@ -7707,7 +13690,7 @@ internal static unsafe class VulkanVideoPresenter Console.Error.WriteLine( $"[LOADER][INFO] Vulkan VideoOut recreating swapchain after {operation}: {result}"); _vk.DeviceWaitIdle(_device); - CompletePendingPresentation(wait: false); + DrainFrameSlots(); CollectCompletedGuestSubmissions(waitForOldest: false); DestroySwapchainResources(); CreateSwapchain(); @@ -7731,20 +13714,62 @@ internal static unsafe class VulkanVideoPresenter _stagingMemory = default; _stagingSize = 0; } - if (_imageAvailable.Handle != 0) + foreach (var semaphore in _frameImageAvailable) { - _vk.DestroySemaphore(_device, _imageAvailable, null); - _imageAvailable = default; + if (semaphore.Handle != 0) + { + _vk.DestroySemaphore(_device, semaphore, null); + } } - if (_renderFinished.Handle != 0) + _frameImageAvailable = []; + foreach (var semaphore in _renderFinishedPerImage) { - _vk.DestroySemaphore(_device, _renderFinished, null); - _renderFinished = default; + if (semaphore.Handle != 0) + { + _vk.DestroySemaphore(_device, semaphore, null); + } } - if (_presentationFence.Handle != 0) + _renderFinishedPerImage = []; + if (_overlayImage.Handle != 0) { - _vk.DestroyFence(_device, _presentationFence, null); - _presentationFence = default; + _vk.DestroyImage(_device, _overlayImage, null); + _overlayImage = default; + } + if (_overlayImageMemory.Handle != 0) + { + _vk.FreeMemory(_device, _overlayImageMemory, null); + _overlayImageMemory = default; + } + for (var slot = 0; slot < _overlayStagingBuffers.Length; slot++) + { + if (_overlayStagingBuffers[slot].Handle != 0) + { + _vk.DestroyBuffer(_device, _overlayStagingBuffers[slot], null); + } + if (_overlayStagingMemory[slot].Handle != 0) + { + _vk.FreeMemory(_device, _overlayStagingMemory[slot], null); + } + } + _overlayStagingBuffers = []; + _overlayStagingMemory = []; + _overlayStagingMapped = []; + _overlayImageInitialized = false; + foreach (var fence in _frameFences) + { + if (fence.Handle != 0) + { + _vk.DestroyFence(_device, fence, null); + } + } + _frameFences = []; + _frameFencePending = []; + _frameTimelines = []; + _frameTranslatedResources = []; + _frameGuestImageVersions = []; + while (_recycledGuestFences.TryPop(out var recycledFence)) + { + _vk.DestroyFence(_device, recycledFence, null); } if (_barycentricPipeline.Handle != 0) { @@ -7777,6 +13802,10 @@ internal static unsafe class VulkanVideoPresenter } if (_commandPool.Handle != 0) { + // Destroying the pool frees every command buffer allocated + // from it, including recycled and per-frame ones. + _recycledGuestCommandBuffers.Clear(); + _frameCommandBuffers = []; _vk.DestroyCommandPool(_device, _commandPool, null); _commandPool = default; _commandBuffer = default; @@ -7833,8 +13862,7 @@ internal static unsafe class VulkanVideoPresenter private static void TraceVulkanShader(string message) { - if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal) && - !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC_SHADER"), "1", StringComparison.Ordinal)) + if (!_traceVulkanShaderEnabled) { return; } diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs index f5a1d52..5b28d7a 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs @@ -19,11 +19,77 @@ public static partial class Gen5SpirvTranslator return true; } + if (instruction.Control is Gen5SdwaControl sdwa && + (sdwa.Source0Select == 7 || + sdwa.Source1Select == 7 || + sdwa.DestinationSelect == 7 || + sdwa.DestinationUnused == 3)) + { + error = $"reserved SDWA selector/modifier in {instruction.Opcode}"; + return false; + } + + if (instruction.Control is Gen5DppControl dppControl && + !IsSupportedDppControl(dppControl.Control)) + { + error = $"unsupported DPP16 control 0x{dppControl.Control:X3}"; + return false; + } + if (instruction.Opcode.StartsWith("VCmp", StringComparison.Ordinal)) { return TryEmitVectorCompare(instruction, out error); } + if (instruction.Opcode == "VReadfirstlaneB32") + { + if (instruction.Destinations.Count == 0 || + instruction.Destinations[0].Kind != Gen5OperandKind.ScalarRegister || + instruction.Sources.Count == 0) + { + error = "invalid read-first-lane operands"; + return false; + } + + var value = GetRawSource(instruction, 0); + if (_subgroupInvocationIdInput != 0) + { + if (_emulateWave64) + { + value = BroadcastFirstWave64Active(value); + } + else + { + // SPIR-V's BroadcastFirst uses the first host-active + // invocation. Guest EXEC is modeled as data, so obtain the + // guest-active mask explicitly and broadcast from its first + // set lane instead. This also updates the private SGPR copy + // for lanes that are currently disabled and may be restored + // by a later saveexec sequence. + var activeLanes = _module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + _uvec4Type, + UInt(3), + Load(_boolType, _exec)); + var activeLow = _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + activeLanes, + 0); + var firstActiveLane = Ext(73, _uintType, activeLow); + value = _module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _uintType, + UInt(3), + value, + firstActiveLane); + } + } + + StoreS(instruction.Destinations[0].Value, value); + return true; + } + if (instruction.Opcode == "VReadlaneB32") { return TryEmitReadlane(instruction, out error); @@ -39,7 +105,6 @@ public static partial class Gen5SpirvTranslator switch (instruction.Opcode) { case "VMovB32": - case "VReadfirstlaneB32": result = GetRawSource(instruction, 0); break; case "VWritelaneB32": @@ -288,8 +353,8 @@ public static partial class Gen5SpirvTranslator case "VFmaF32": case "VMadMkF32": case "VMadAkF32": - case "VFmamkF32": - case "VFmaakF32": + case "VFmaMkF32": + case "VFmaAkF32": result = EmitFloatResult( instruction, Ext( @@ -328,6 +393,12 @@ public static partial class Gen5SpirvTranslator case "VXorB32": result = EmitIntegerBinary(instruction, SpirvOp.BitwiseXor); break; + case "VXnorB32": + { + var xor = EmitIntegerBinary(instruction, SpirvOp.BitwiseXor); + result = _module.AddInstruction(SpirvOp.Not, _uintType, xor); + break; + } case "VNotB32": result = _module.AddInstruction( SpirvOp.Not, @@ -353,6 +424,7 @@ public static partial class Gen5SpirvTranslator result = EmitIntegerBinary(instruction, SpirvOp.IAdd); break; case "VAddcU32": + case "VAddCoCiU32": result = EmitAddWithCarry(instruction); break; case "VSubI32": @@ -421,12 +493,16 @@ public static partial class Gen5SpirvTranslator _longType, wideLeft, wideRight); - result = _module.AddInstruction( - SpirvOp.UConvert, + result = Bitcast( _uintType, - ShiftRightLogical64( - Bitcast(_ulongType, product), - _module.Constant64(_ulongType, 32))); + _module.AddInstruction( + SpirvOp.SConvert, + _intType, + _module.AddInstruction( + SpirvOp.ShiftRightArithmetic, + _longType, + product, + _module.Constant64(_longType, 32)))); break; } case "VBcntU32B32": @@ -437,6 +513,72 @@ public static partial class Gen5SpirvTranslator GetRawSource(instruction, 0)), GetRawSource(instruction, 1)); break; + case "VMbcntHiU32B32": + { + var guestLane = GuestWaveLane(); + var lane = BitwiseAnd(guestLane, UInt(31)); + var partialMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), lane), + UInt(1)); + var countedBits = _module.AddInstruction( + SpirvOp.BitCount, + _uintType, + BitwiseAnd(GetRawSource(instruction, 0), partialMask)); + var isUpperHalf = _module.AddInstruction( + SpirvOp.UGreaterThanEqual, + _boolType, + guestLane, + UInt(32)); + result = IAdd( + GetRawSource(instruction, 1), + _module.AddInstruction( + SpirvOp.Select, + _uintType, + isUpperHalf, + countedBits, + UInt(0))); + break; + } + case "VMbcntLoU32B32": + { + var guestLane = GuestWaveLane(); + var lane = BitwiseAnd(guestLane, UInt(31)); + var partialMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), lane), + UInt(1)); + var lowBitsMask = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.UGreaterThanEqual, + _boolType, + guestLane, + UInt(32)), + UInt(uint.MaxValue), + partialMask); + result = IAdd( + GetRawSource(instruction, 1), + _module.AddInstruction( + SpirvOp.BitCount, + _uintType, + BitwiseAnd(GetRawSource(instruction, 0), lowBitsMask))); + break; + } + case "VBfmB32": + { + var width = BitwiseAnd(GetRawSource(instruction, 0), UInt(31)); + var lowMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), width), + UInt(1)); + result = ShiftLeftLogical(lowMask, GetRawSource(instruction, 1)); + break; + } case "VMadU32U24": { var left = BitwiseAnd( @@ -634,18 +776,6 @@ public static partial class Gen5SpirvTranslator Ext(38, _uintType, high, right)); break; } - case "VSadU32": - { - var left = GetRawSource(instruction, 0); - var right = GetRawSource(instruction, 1); - var difference = _module.AddInstruction( - SpirvOp.ISub, - _uintType, - Ext(41, _uintType, left, right), - Ext(38, _uintType, left, right)); - result = IAdd(difference, GetRawSource(instruction, 2)); - break; - } case "VMed3I32": { var left = Bitcast(_intType, GetRawSource(instruction, 0)); @@ -718,6 +848,50 @@ public static partial class Gen5SpirvTranslator StoreCarryOut(instruction, borrow); break; } + case "VMadU64U32": + { + // V_MAD_U64_U32 writes a 64-bit VGPR pair. The first two + // sources are 32-bit factors; the third is a 64-bit addend + // held in a VGPR or SGPR pair. Its SDST receives the carry + // mask for the unsigned 64-bit addition. + var wideLeft = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + GetRawSource(instruction, 0)); + var wideRight = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + GetRawSource(instruction, 1)); + var product = _module.AddInstruction( + SpirvOp.IMul, + _ulongType, + wideLeft, + wideRight); + var addend = GetRawSource64(instruction, 2); + var wideResult = _module.AddInstruction( + SpirvOp.IAdd, + _ulongType, + product, + addend); + var carry = _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + wideResult, + addend); + result = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + wideResult); + var high = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + ShiftRightLogical64( + wideResult, + _module.Constant64(_ulongType, 32))); + StoreV(destination + 1, high); + StoreCarryOut(instruction, carry); + break; + } case "VBfeU32": { var width = BitwiseAnd(GetRawSource(instruction, 2), UInt(31)); @@ -753,21 +927,51 @@ public static partial class Gen5SpirvTranslator first, second); result = Ext(58, _uintType, vector); + StorePackedHalf( + destination, + vector); break; } - case "VCvtPkU16U32": - case "VCvtPkI16I32": - result = BitwiseOr( - BitwiseAnd(GetRawSource(instruction, 0), UInt(0xFFFF)), - ShiftLeftLogical( - BitwiseAnd(GetRawSource(instruction, 1), UInt(0xFFFF)), - UInt(16))); + case "VCvtPknormI16F32": + case "VCvtPknormU16F32": + { + var vector = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + GetFloatSource(instruction, 0), + GetFloatSource(instruction, 1)); + // GLSL.std.450 PackSnorm2x16 / PackUnorm2x16 match the + // RDNA saturated normalized conversion and packing rules. + result = Ext( + instruction.Opcode == "VCvtPknormI16F32" ? 56u : 57u, + _uintType, + vector); break; + } default: error = $"unsupported vector opcode {instruction.Opcode}"; return false; } + if (instruction.Control is Gen5DppControl dpp) + { + result = _module.AddInstruction( + SpirvOp.Select, + _uintType, + IsDppWriteEnabled(dpp), + result, + LoadV(destination)); + } + + if (instruction.Control is Gen5SdwaControl destinationControl && + destinationControl.ScalarDestination is null) + { + result = ApplySdwaDestination( + destinationControl, + result, + LoadV(destination)); + } + StoreV(destination, result); return true; } @@ -894,21 +1098,17 @@ public static partial class Gen5SpirvTranslator { condition = _module.ConstantBool(true); } - else if (opcode is "VCmpOF32" or "VCmpxOF32" or "VCmpUF32" or "VCmpxUF32") + else if (opcode is + "VCmpOF32" or "VCmpxOF32" or + "VCmpUF32" or "VCmpxUF32") { - // The ordered/unordered predicates only test whether either - // operand is NaN. SPIR-V's OpOrdered/OpUnordered are Kernel-only, - // so build the same result from OpIsNan, which needs no extra - // capability: unordered = isnan(a) || isnan(b), ordered = !that. var left = GetFloatSource(instruction, 0); var right = GetFloatSource(instruction, 1); - var nanLeft = _module.AddInstruction(SpirvOp.IsNan, _boolType, left); - var nanRight = _module.AddInstruction(SpirvOp.IsNan, _boolType, right); var unordered = _module.AddInstruction( SpirvOp.LogicalOr, _boolType, - nanLeft, - nanRight); + _module.AddInstruction(SpirvOp.IsNan, _boolType, left), + _module.AddInstruction(SpirvOp.IsNan, _boolType, right)); condition = opcode is "VCmpUF32" or "VCmpxUF32" ? unordered : _module.AddInstruction(SpirvOp.LogicalNot, _boolType, unordered); @@ -927,11 +1127,11 @@ public static partial class Gen5SpirvTranslator "VCmpLgF32" or "VCmpxLgF32" => SpirvOp.FOrdNotEqual, "VCmpGeF32" or "VCmpxGeF32" => SpirvOp.FOrdGreaterThanEqual, "VCmpNeqF32" or "VCmpxNeqF32" => SpirvOp.FUnordNotEqual, - "VCmpNlgF32" or "VCmpxNlgF32" => SpirvOp.FUnordEqual, "VCmpNltF32" or "VCmpxNltF32" => SpirvOp.FUnordGreaterThanEqual, "VCmpNleF32" or "VCmpxNleF32" => SpirvOp.FUnordGreaterThan, "VCmpNgtF32" or "VCmpxNgtF32" => SpirvOp.FUnordLessThanEqual, "VCmpNgeF32" or "VCmpxNgeF32" => SpirvOp.FUnordLessThan, + "VCmpNlgF32" or "VCmpxNlgF32" => SpirvOp.FUnordEqual, _ => SpirvOp.Nop, }; if (operation == SpirvOp.Nop) @@ -978,20 +1178,52 @@ public static partial class Gen5SpirvTranslator condition = _module.AddInstruction(operation, _boolType, left, right); } - // On gfx10, VCmpx writes EXEC only and preserves VCC; the sdst - // operand was removed from the cmpx encodings on this generation. + if (_state.Program.Address == 0x0000000500781200ul && + ((instruction.Pc == 0x4D4 && + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_COMPARE_4D4") == "1") || + (instruction.Pc == 0x540 && + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_COMPARE_540") == "1"))) + { + condition = _module.ConstantBool(true); + } + + // Vector compares fully overwrite the destination mask, but only + // lanes enabled by EXEC can pass the test: VCC = EXEC & condition. + // Balloting the raw condition leaks results from disabled lanes + // into later saveexec/branch sequences. + var activeCondition = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + Load(_boolType, _exec), + condition); + if (instruction.Control is Gen5DppControl compareDpp) + { + activeCondition = _module.AddInstruction( + SpirvOp.Select, + _boolType, + IsDppWriteEnabled(compareDpp), + activeCondition, + Load(_boolType, _vcc)); + } + if (opcode.StartsWith("VCmpx", StringComparison.Ordinal)) { - var active = _module.AddInstruction( - SpirvOp.LogicalAnd, - _boolType, - Load(_boolType, _exec), - condition); - StoreWaveMask(126, active); + // GFX10 VCMPX is EXEC-only. The SDWA bits that older + // generations exposed as an explicit scalar destination must + // not overwrite VCC/an SGPR pair here. Guest shaders rely on + // those registers retaining a saved EXEC mask for later + // reconvergence. + StoreWaveMask(126, activeCondition); } else { - StoreWaveMask(106, condition); + var compareDestination = instruction.Control is Gen5SdwaControl + { ScalarDestination: { } scalarDestination } + ? scalarDestination + : 106u; + StoreWaveMask(compareDestination, activeCondition); } return true; @@ -1062,6 +1294,56 @@ public static partial class Gen5SpirvTranslator } var left = GetRawSource(instruction, 0); + if (instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal)) + { + var oldExec64 = BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + oldExec64); + var notLeft = _module.AddInstruction(SpirvOp.Not, _uintType, left); + var notOldExec = _module.AddInstruction(SpirvOp.Not, _uintType, oldExec); + var newExec = instruction.Opcode switch + { + "SAndSaveexecB32" => BitwiseAnd(oldExec, left), + "SOrSaveexecB32" => BitwiseOr(oldExec, left), + "SXorSaveexecB32" => _module.AddInstruction( + SpirvOp.BitwiseXor, _uintType, oldExec, left), + "SAndn1SaveexecB32" => BitwiseAnd(notLeft, oldExec), + "SAndn2SaveexecB32" => BitwiseAnd(left, notOldExec), + "SOrn1SaveexecB32" => BitwiseOr(notLeft, oldExec), + "SOrn2SaveexecB32" => BitwiseOr(left, notOldExec), + "SNandSaveexecB32" => _module.AddInstruction( + SpirvOp.Not, _uintType, BitwiseAnd(left, oldExec)), + "SNorSaveexecB32" => _module.AddInstruction( + SpirvOp.Not, _uintType, BitwiseOr(left, oldExec)), + "SXnorSaveexecB32" => _module.AddInstruction( + SpirvOp.Not, + _uintType, + _module.AddInstruction( + SpirvOp.BitwiseXor, + _uintType, + left, + oldExec)), + _ => 0u, + }; + if (newExec == 0) + { + error = $"unsupported scalar 32-bit saveexec opcode {instruction.Opcode}"; + return false; + } + + StoreS(destination, oldExec); + // B32 saveexec is the Wave32 form; EXEC_HI is zero. + var newExec64 = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + newExec); + StoreS64(126, newExec64); + Store(_scc, IsNotZero(newExec)); + return true; + } + uint result; switch (instruction.Opcode) { @@ -1224,19 +1506,11 @@ public static partial class Gen5SpirvTranslator break; case "SMulHiU32": { - var wideLeft = _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - left); - var wideRight = _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - right); var product = _module.AddInstruction( SpirvOp.IMul, _ulongType, - wideLeft, - wideRight); + _module.AddInstruction(SpirvOp.UConvert, _ulongType, left), + _module.AddInstruction(SpirvOp.UConvert, _ulongType, right)); result = _module.AddInstruction( SpirvOp.UConvert, _uintType, @@ -1245,29 +1519,6 @@ public static partial class Gen5SpirvTranslator _module.Constant64(_ulongType, 32))); break; } - case "SMulHiI32": - { - var wideLeft = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, left)); - var wideRight = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, right)); - var product = _module.AddInstruction( - SpirvOp.IMul, - _longType, - wideLeft, - wideRight); - result = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - ShiftRightLogical64( - Bitcast(_ulongType, product), - _module.Constant64(_ulongType, 32))); - break; - } case "SAndB32": result = BitwiseAnd(left, right); Store(_scc, IsNotZero(result)); @@ -1389,43 +1640,6 @@ public static partial class Gen5SpirvTranslator Store(_scc, IsNotZero(result)); break; } - case "SAbsdiffI32": - { - var wideLeft = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, left)); - var wideRight = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, right)); - var difference = _module.AddInstruction( - SpirvOp.ISub, - _longType, - wideLeft, - wideRight); - result = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - Ext(5, _longType, difference)); - Store(_scc, IsNotZero(result)); - break; - } - case "SPackLlB32B16": - result = BitwiseOr( - BitwiseAnd(left, UInt(0xFFFF)), - ShiftLeftLogical(right, UInt(16))); - break; - case "SPackLhB32B16": - result = BitwiseOr( - BitwiseAnd(left, UInt(0xFFFF)), - BitwiseAnd(right, UInt(0xFFFF0000))); - break; - case "SPackHhB32B16": - result = BitwiseOr( - ShiftRightLogical(left, UInt(16)), - BitwiseAnd(right, UInt(0xFFFF0000))); - break; case "SCselectB32": result = _module.AddInstruction( SpirvOp.Select, @@ -1489,6 +1703,21 @@ public static partial class Gen5SpirvTranslator right); break; } + case "SPackLlB32B16": + result = BitwiseOr( + BitwiseAnd(left, UInt(0xFFFF)), + ShiftLeftLogical(right, UInt(16))); + break; + case "SPackLhB32B16": + result = BitwiseOr( + BitwiseAnd(left, UInt(0xFFFF)), + BitwiseAnd(right, UInt(0xFFFF0000))); + break; + case "SPackHhB32B16": + result = BitwiseOr( + ShiftRightLogical(left, UInt(16)), + BitwiseAnd(right, UInt(0xFFFF0000))); + break; default: error = $"unsupported scalar opcode {instruction.Opcode}"; return false; @@ -1610,7 +1839,7 @@ public static partial class Gen5SpirvTranslator var left = GetRawSource64(instruction, 0); if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal)) { - var oldExec = BooleanToLaneMask(Load(_boolType, _exec)); + var oldExec = BooleanToWaveMask(Load(_boolType, _exec)); var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left); var newExec = instruction.Opcode switch { @@ -1798,11 +2027,70 @@ public static partial class Gen5SpirvTranslator return true; } + if (instruction.Opcode == "SBfmB64") + { + if (instruction.Sources.Count < 2) + { + error = "missing scalar 64-bit bitfield-mask source"; + return false; + } + + var width = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + BitwiseAnd(GetRawSource(instruction, 0), UInt(63))); + var offset = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + BitwiseAnd(GetRawSource(instruction, 1), UInt(63))); + // Width is masked to 0..63, so (1 << width) never invokes an + // undefined 64-bit shift. This naturally yields zero for a + // zero-width mask and avoids OpBitFieldInsert, which the + // MoltenVK/SPIRV-Cross path rejects for 64-bit integers. + var lowMask = _module.AddInstruction( + SpirvOp.ISub, + _ulongType, + ShiftLeftLogical64( + _module.Constant64(_ulongType, 1), + width), + _module.Constant64(_ulongType, 1)); + var maskValue = ShiftLeftLogical64(lowMask, offset); + StoreS64(destination, maskValue); + Store(_scc, IsNotZero64(maskValue)); + return true; + } + uint value; - if (instruction.Opcode is "SMovB64" or "SWqmB64") + if (instruction.Opcode == "SMovB64") { value = left; } + else if (instruction.Opcode == "SWqmB64") + { + var quadAny = _module.AddInstruction( + SpirvOp.BitwiseOr, + _ulongType, + left, + _module.AddInstruction( + SpirvOp.BitwiseOr, + _ulongType, + ShiftRightLogical64(left, _module.Constant64(_ulongType, 1)), + _module.AddInstruction( + SpirvOp.BitwiseOr, + _ulongType, + ShiftRightLogical64(left, _module.Constant64(_ulongType, 2)), + ShiftRightLogical64(left, _module.Constant64(_ulongType, 3))))); + quadAny = _module.AddInstruction( + SpirvOp.BitwiseAnd, + _ulongType, + quadAny, + _module.Constant64(_ulongType, 0x1111_1111_1111_1111UL)); + value = _module.AddInstruction( + SpirvOp.IMul, + _ulongType, + quadAny, + _module.Constant64(_ulongType, 0xFUL)); + } else if (instruction.Opcode == "SNotB64") { value = _module.AddInstruction(SpirvOp.Not, _ulongType, left); @@ -1896,7 +2184,8 @@ public static partial class Gen5SpirvTranslator private uint GetRawSource( Gen5ShaderInstruction instruction, - int sourceIndex) + int sourceIndex, + bool applySdwaIntegerModifiers = true) { if ((uint)sourceIndex >= instruction.Sources.Count) { @@ -1909,12 +2198,48 @@ public static partial class Gen5SpirvTranslator Gen5OperandKind.VectorRegister => LoadV(operand.Value), Gen5OperandKind.ScalarRegister => LoadS(operand.Value), Gen5OperandKind.LiteralConstant => UInt(operand.Value), + Gen5OperandKind.EncodedConstant when operand.Value == 251 => + _module.AddInstruction( + SpirvOp.Select, + _uintType, + LogicalNot(SubgroupAny(Load(_boolType, _vcc))), + UInt(1), + UInt(0)), + Gen5OperandKind.EncodedConstant when operand.Value == 252 => + _module.AddInstruction( + SpirvOp.Select, + _uintType, + LogicalNot(SubgroupAny(Load(_boolType, _exec))), + UInt(1), + UInt(0)), + Gen5OperandKind.EncodedConstant when operand.Value == 253 => + _module.AddInstruction( + SpirvOp.Select, + _uintType, + Load(_boolType, _scc), + UInt(1), + UInt(0)), Gen5OperandKind.EncodedConstant when TryDecodeInlineConstant( operand.Value, out var inline) => UInt(inline), _ => throw new InvalidOperationException($"unsupported source {operand}"), }; + // DPP16 remaps src0 across lanes before the VALU operation. The IR + // has always preserved this control, but treating it as an ordinary + // local VGPR read breaks every wave reduction used by the XPR + // renderer (min/max/OR scans become value-with-self operations). + if (sourceIndex == 0 && + instruction.Control is Gen5DppControl dpp) + { + value = ApplyDppSource(dpp, value); + } + else if (sourceIndex == 0 && + instruction.Control is Gen5Dpp8Control dpp8) + { + value = ApplyDpp8Source(dpp8, value); + } + if (instruction.Control is Gen5SdwaControl sdwa) { var selector = sourceIndex switch @@ -1933,11 +2258,326 @@ public static partial class Gen5SpirvTranslator 5 => BitwiseAnd(ShiftRightLogical(value, UInt(16)), UInt(0xFFFF)), _ => value, }; + var signExtend = sourceIndex switch + { + 0 => sdwa.Source0SignExtend, + 1 => sdwa.Source1SignExtend, + _ => false, + }; + if (signExtend && selector != 6) + { + var width = selector <= 3 ? 8u : 16u; + value = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.BitFieldSExtract, + _intType, + Bitcast(_intType, value), + UInt(0), + UInt(width))); + } + + if (applySdwaIntegerModifiers) + { + if ((sdwa.AbsoluteMask & (1u << sourceIndex)) != 0) + { + value = Bitcast( + _uintType, + Ext(5, _intType, Bitcast(_intType, value))); + } + + if ((sdwa.NegateMask & (1u << sourceIndex)) != 0) + { + value = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + UInt(0), + value); + } + } } return value; } + private uint ApplyDpp8Source(Gen5Dpp8Control control, uint value) + { + var lane = GuestWaveLane(); + var laneInGroup = BitwiseAnd(lane, UInt(7)); + var selector = UInt(control.LaneSelectors & 7); + for (var index = 1u; index < 8; index++) + { + selector = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + laneInGroup, + UInt(index)), + UInt((control.LaneSelectors >> checked((int)(index * 3))) & 7), + selector); + } + + var targetLane = IAdd(BitwiseAnd(lane, UInt(0xFFFF_FFF8)), selector); + targetLane = BitwiseAnd(targetLane, UInt(31)); + var shuffled = _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + value, + targetLane); + if (control.FetchInactive) + { + return shuffled; + } + + var activeWord = _module.AddInstruction( + SpirvOp.Select, + _uintType, + Load(_boolType, _exec), + UInt(1), + UInt(0)); + var sourceActive = IsNotZero( + _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + activeWord, + targetLane)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + sourceActive, + shuffled, + UInt(0)); + } + + private uint ApplyDppSource(Gen5DppControl control, uint value) + { + GetDppSourceLane(control, out var targetLane, out var inRange); + var lane = GuestWaveLane(); + var safeTarget = _module.AddInstruction( + SpirvOp.Select, + _uintType, + inRange, + targetLane, + lane); + safeTarget = BitwiseAnd(safeTarget, UInt(31)); + var shuffled = _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + value, + safeTarget); + + var sourceAvailable = inRange; + if (!control.FetchInactive) + { + var activeWord = _module.AddInstruction( + SpirvOp.Select, + _uintType, + Load(_boolType, _exec), + UInt(1), + UInt(0)); + var shuffledActive = _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + activeWord, + safeTarget); + sourceAvailable = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + sourceAvailable, + IsNotZero(shuffledActive)); + } + + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + sourceAvailable, + shuffled, + UInt(0)); + } + + private uint ApplySdwaDestination( + Gen5SdwaControl control, + uint value, + uint previous) + { + var (shift, width) = control.DestinationSelect switch + { + 0 => (0u, 8u), + 1 => (8u, 8u), + 2 => (16u, 8u), + 3 => (24u, 8u), + 4 => (0u, 16u), + 5 => (16u, 16u), + _ => (0u, 32u), + }; + if (width == 32) + { + return value; + } + + var lowMask = width == 8 ? 0xFFu : 0xFFFFu; + var fieldMask = lowMask << checked((int)shift); + var upperStart = shift + width; + var upperMask = upperStart == 32 + ? 0u + : uint.MaxValue << checked((int)upperStart); + var positioned = ShiftLeftLogical( + BitwiseAnd(value, UInt(lowMask)), + UInt(shift)); + return control.DestinationUnused switch + { + 0 => positioned, + 1 => BitwiseOr( + positioned, + _module.AddInstruction( + SpirvOp.Select, + _uintType, + IsNotZero(BitwiseAnd(positioned, UInt(1u << checked((int)(shift + width - 1))))), + UInt(upperMask), + UInt(0))), + 2 => BitwiseOr( + BitwiseAnd(previous, UInt(~fieldMask)), + positioned), + _ => throw new InvalidOperationException("reserved SDWA destination-unused mode"), + }; + } + + private static bool IsSupportedDppControl(uint control) => + control <= 0xFF || + control is >= 0x101 and <= 0x10F or + >= 0x111 and <= 0x11F or + >= 0x121 and <= 0x12F or + 0x140 or 0x141 or + >= 0x150 and <= 0x15F or + >= 0x160 and <= 0x16F; + + private void GetDppSourceLane( + Gen5DppControl control, + out uint targetLane, + out uint inRange) + { + var lane = GuestWaveLane(); + var rowBase = BitwiseAnd(lane, UInt(0xFFFF_FFF0)); + var rowLane = BitwiseAnd(lane, UInt(15)); + var dpp = control.Control; + inRange = _module.ConstantBool(true); + + if (dpp <= 0xFF) + { + var quadLane = BitwiseAnd(lane, UInt(3)); + var selected = UInt(dpp & 3); + for (var index = 1u; index < 4; index++) + { + selected = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + quadLane, + UInt(index)), + UInt((dpp >> checked((int)(index * 2))) & 3), + selected); + } + + targetLane = IAdd(BitwiseAnd(lane, UInt(0xFFFF_FFFC)), selected); + return; + } + + if (dpp is >= 0x101 and <= 0x10F) + { + var shift = UInt(dpp & 15); + var shifted = IAdd(rowLane, shift); + inRange = _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + shifted, + UInt(16)); + targetLane = IAdd(rowBase, BitwiseAnd(shifted, UInt(15))); + return; + } + + if (dpp is >= 0x111 and <= 0x11F) + { + var shift = UInt(dpp & 15); + inRange = _module.AddInstruction( + SpirvOp.UGreaterThanEqual, + _boolType, + rowLane, + shift); + targetLane = IAdd( + rowBase, + BitwiseAnd( + _module.AddInstruction(SpirvOp.ISub, _uintType, rowLane, shift), + UInt(15))); + return; + } + + if (dpp is >= 0x121 and <= 0x12F) + { + targetLane = IAdd( + rowBase, + BitwiseAnd( + _module.AddInstruction( + SpirvOp.ISub, + _uintType, + rowLane, + UInt(dpp & 15)), + UInt(15))); + return; + } + + targetLane = dpp switch + { + 0x140 => IAdd(rowBase, _module.AddInstruction( + SpirvOp.ISub, _uintType, UInt(15), rowLane)), + 0x141 => IAdd( + BitwiseAnd(lane, UInt(0xFFFF_FFF8)), + _module.AddInstruction( + SpirvOp.ISub, + _uintType, + UInt(7), + BitwiseAnd(lane, UInt(7)))), + >= 0x150 and <= 0x15F => IAdd(rowBase, UInt(dpp & 15)), + >= 0x160 and <= 0x16F => IAdd( + rowBase, + BitwiseXor(rowLane, UInt(dpp & 15))), + _ => lane, + }; + } + + private uint IsDppWriteEnabled(Gen5DppControl control) + { + GetDppSourceLane(control, out _, out var inRange); + var lane = GuestWaveLane(); + var row = ShiftRightLogical(lane, UInt(4)); + var bank = BitwiseAnd(lane, UInt(3)); + var rowEnabled = IsNotZero(BitwiseAnd( + UInt(control.RowMask), + ShiftLeftLogical(UInt(1), row))); + var bankEnabled = IsNotZero(BitwiseAnd( + UInt(control.BankMask), + ShiftLeftLogical(UInt(1), bank))); + var sourceAllowsWrite = control.BoundControl + ? _module.ConstantBool(true) + : inRange; + return _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + rowEnabled, + _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + bankEnabled, + sourceAllowsWrite)); + } + private uint GetFloatSource( Gen5ShaderInstruction instruction, int sourceIndex) @@ -1956,7 +2596,12 @@ public static partial class Gen5SpirvTranslator } else { - value = Bitcast(_floatType, GetRawSource(instruction, sourceIndex)); + value = Bitcast( + _floatType, + GetRawSource( + instruction, + sourceIndex, + applySdwaIntegerModifiers: false)); } uint absoluteMask = 0; @@ -2000,6 +2645,35 @@ public static partial class Gen5SpirvTranslator return LoadS64(operand.Value); } + if (operand.Kind == Gen5OperandKind.VectorRegister) + { + var vectorLow = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + LoadV(operand.Value)); + var high = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + LoadV(operand.Value + 1)); + high = ShiftLeftLogical64( + high, + _module.Constant64(_ulongType, 32)); + return _module.AddInstruction( + SpirvOp.BitwiseOr, + _ulongType, + vectorLow, + high); + } + + // Scalar inline negative constants are signed immediates. B64 + // consumers sign-extend them, so -1 denotes a full 64-bit mask. + if (operand.Kind == Gen5OperandKind.EncodedConstant && + operand.Value is >= 193 and <= 208) + { + var signed = -(long)(operand.Value - 192); + return _module.Constant64(_ulongType, unchecked((ulong)signed)); + } + var low = GetRawSource(instruction, sourceIndex); return _module.AddInstruction(SpirvOp.UConvert, _ulongType, low); } @@ -2296,10 +2970,13 @@ public static partial class Gen5SpirvTranslator { var left = GetRawSource(instruction, 0); var right = GetRawSource(instruction, 1); + var carryMask = instruction.Sources.Count > 2 + ? IsCurrentLaneSet(GetRawSource64(instruction, 2)) + : Load(_boolType, _vcc); var carryIn = _module.AddInstruction( SpirvOp.Select, _uintType, - Load(_boolType, _vcc), + carryMask, UInt(1), UInt(0)); var partial = IAdd(left, right); @@ -2309,7 +2986,7 @@ public static partial class Gen5SpirvTranslator _boolType, _module.AddInstruction(SpirvOp.ULessThan, _boolType, partial, left), _module.AddInstruction(SpirvOp.ULessThan, _boolType, result, partial)); - StoreWaveMask(106, carry); + StoreCarryOut(instruction, carry); return result; } @@ -2319,10 +2996,13 @@ public static partial class Gen5SpirvTranslator { var left = GetRawSource(instruction, reverse ? 1 : 0); var right = GetRawSource(instruction, reverse ? 0 : 1); + var borrowMask = instruction.Sources.Count > 2 + ? IsCurrentLaneSet(GetRawSource64(instruction, 2)) + : Load(_boolType, _vcc); var borrowIn = _module.AddInstruction( SpirvOp.Select, _uintType, - Load(_boolType, _vcc), + borrowMask, UInt(1), UInt(0)); var partial = _module.AddInstruction(SpirvOp.ISub, _uintType, left, right); @@ -2340,7 +3020,67 @@ public static partial class Gen5SpirvTranslator _boolType, partial, borrowIn)); - StoreWaveMask(106, borrow); + StoreCarryOut(instruction, borrow); + return result; + } + + private uint BroadcastFirstWave64Active(uint value) + { + var lane = GuestWaveLane(); + EmitConditional( + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + lane, + UInt(0)), + () => Store(WaveBroadcastScratchPointer(), UInt(0))); + EmitWave64Barrier(); + + var activeMask = BooleanToWaveMask(Load(_boolType, _exec)); + var lowMask = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + activeMask); + var highMask = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + ShiftRightLogical64( + activeMask, + _module.Constant64(_ulongType, 32))); + var hasLow = IsNotZero(lowMask); + var hasHigh = IsNotZero(highMask); + var firstLow = Ext(73, _uintType, lowMask); + var firstHigh = IAdd(UInt(32), Ext(73, _uintType, highMask)); + var firstLane = _module.AddInstruction( + SpirvOp.Select, + _uintType, + hasLow, + firstLow, + _module.AddInstruction( + SpirvOp.Select, + _uintType, + hasHigh, + firstHigh, + UInt(0))); + var isFirst = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + lane, + firstLane); + EmitConditional( + _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + isFirst, + _module.AddInstruction( + SpirvOp.LogicalOr, + _boolType, + hasLow, + hasHigh)), + () => Store(WaveBroadcastScratchPointer(), value)); + EmitWave64Barrier(); + var result = Load(_uintType, WaveBroadcastScratchPointer()); + EmitWave64Barrier(); return result; } @@ -2348,25 +3088,18 @@ public static partial class Gen5SpirvTranslator Gen5ShaderInstruction instruction, uint carry) { + var activeCarry = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + Load(_boolType, _exec), + carry); if (instruction.Control is Gen5Vop3Control { ScalarDestination: { } register }) { - StoreS( - register, - _module.AddInstruction( - SpirvOp.Select, - _uintType, - carry, - UInt(1), - UInt(0))); - if (register == 106) - { - StoreWaveMask(106, carry); - } - + StoreWaveMask(register, activeCarry); return; } - StoreWaveMask(106, carry); + StoreWaveMask(106, activeCarry); } private bool TryEmitReadlane( @@ -2409,10 +3142,21 @@ public static partial class Gen5SpirvTranslator Gen5ShaderInstruction instruction, bool exchangeRows) { + if (instruction.Control is not Gen5Vop3Control control || + (control.OperandSelect & ~3u) != 0 || + control.AbsoluteMask != 0 || + control.NegateMask != 0 || + control.OutputModifier != 0 || + control.Clamp) + { + throw new InvalidOperationException( + $"invalid permlane modifiers for {instruction.Opcode}"); + } + var value = GetRawSource(instruction, 0); var selectorLow = GetRawSource(instruction, 1); var selectorHigh = GetRawSource(instruction, 2); - var lane = Load(_uintType, _subgroupInvocationIdInput); + var lane = GuestWaveLane(); var localLane = BitwiseAnd(lane, UInt(15)); var lowHalf = _module.AddInstruction( SpirvOp.ULessThan, @@ -2445,12 +3189,38 @@ public static partial class Gen5SpirvTranslator } var targetLane = IAdd(rowBase, selector); - return _module.AddInstruction( + targetLane = BitwiseAnd(targetLane, UInt(31)); + var shuffled = _module.AddInstruction( SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), value, targetLane); + var fetchInactive = (control.OperandSelect & 1) != 0; + if (fetchInactive) + { + return shuffled; + } + + var activeWord = _module.AddInstruction( + SpirvOp.Select, + _uintType, + Load(_boolType, _exec), + UInt(1), + UInt(0)); + var sourceActive = IsNotZero( + _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + activeWord, + targetLane)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + sourceActive, + shuffled, + UInt(0)); } private uint EmitFloatResult( diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs index c2cbebe..60722e2 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs @@ -10,6 +10,12 @@ public static partial class Gen5SpirvTranslator private const uint ScalarRegisterCount = 256; private const uint VectorRegisterCount = 512; private const uint LdsDwordCount = 8192; + // Graphics stages model LDS as a per-invocation Private array rather than + // real workgroup-shared memory. A full 32 KB Private array per vertex/pixel + // invocation is wasteful and risks Metal compile limits, and per-invocation + // write-then-read correctness only needs deterministic address→slot masking, + // so a smaller array is safe. + private const uint PrivateLdsDwordCount = 2048; private const uint RdnaWaveLaneCount = 32; public static bool TryCompilePixelShader( @@ -21,17 +27,24 @@ public static partial class Gen5SpirvTranslator int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) => + int initialScalarBufferIndex = -1, + int pixelRenderTargetSlot = 0, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + ulong storageBufferOffsetAlignment = 1) => TryCompilePixelShader( state, evaluation, - [new Gen5PixelOutputBinding(0, 0, outputKind)], + [new Gen5PixelOutputBinding((uint)pixelRenderTargetSlot, 0, outputKind)], out shader, out error, globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex); + initialScalarBufferIndex, + pixelInputEnable, + pixelInputAddress, + storageBufferOffsetAlignment); public static bool TryCompilePixelShader( Gen5ShaderState state, @@ -42,7 +55,10 @@ public static partial class Gen5SpirvTranslator int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) + int initialScalarBufferIndex = -1, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + ulong storageBufferOffsetAlignment = 1) { if (outputs.Count > 8 || outputs.Any(output => output.GuestSlot > 7)) { @@ -80,7 +96,10 @@ public static partial class Gen5SpirvTranslator globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex); + initialScalarBufferIndex, + pixelInputEnable: pixelInputEnable, + pixelInputAddress: pixelInputAddress, + storageBufferOffsetAlignment: storageBufferOffsetAlignment); return context.TryCompile(out shader, out error); } @@ -92,7 +111,9 @@ public static partial class Gen5SpirvTranslator int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) + int initialScalarBufferIndex = -1, + int requiredVertexOutputCount = 0, + ulong storageBufferOffsetAlignment = 1) { var context = new CompilationContext( Gen5SpirvStage.Vertex, @@ -105,7 +126,9 @@ public static partial class Gen5SpirvTranslator globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex); + initialScalarBufferIndex, + requiredVertexOutputCount: requiredVertexOutputCount, + storageBufferOffsetAlignment: storageBufferOffsetAlignment); return context.TryCompile(out shader, out error); } @@ -116,7 +139,11 @@ public static partial class Gen5SpirvTranslator uint localSizeY, uint localSizeZ, out Gen5SpirvShader shader, - out string error) + out string error, + int totalGlobalBufferCount = -1, + int initialScalarBufferIndex = -1, + uint waveLaneCount = 32, + ulong storageBufferOffsetAlignment = 1) { var context = new CompilationContext( Gen5SpirvStage.Compute, @@ -127,26 +154,79 @@ public static partial class Gen5SpirvTranslator Math.Max(localSizeY, 1), Math.Max(localSizeZ, 1), 0, - -1, + totalGlobalBufferCount, 0, - -1); + initialScalarBufferIndex, + waveLaneCount: waveLaneCount, + storageBufferOffsetAlignment: storageBufferOffsetAlignment); return context.TryCompile(out shader, out error); } private sealed partial class CompilationContext { + private const uint ImageDescriptorDwords = 8; + private const uint SamplerDescriptorDwords = 4; + private const int ScalarRegisterCount = 128; + private const long InitialScalarDefinition = -1; + private const long ConflictingScalarDefinition = -2; + private const long UnreachableScalarDefinition = -3; + private readonly SpirvModuleBuilder _module = new(); private readonly Gen5SpirvStage _stage; private readonly Gen5ShaderState _state; private readonly Gen5ShaderEvaluation _evaluation; private readonly IReadOnlyList _pixelOutputBindings; + private readonly uint _waveLaneCount; + private readonly bool _emulateWave64; + + // Safety valve for the PC-dispatcher loop. Each iteration executes one + // GCN basic block; a correctly-translated shader always reaches its + // terminal block (pc out of range -> default -> exit) well within any + // real control flow. A mistranslated shader whose loop-exit condition is + // wrong would otherwise spin the dispatcher forever, hanging the single + // Metal queue and freezing every later submission (black screen, no + // recovery). Bounding the iteration count guarantees the invocation + // terminates instead: the effect may be wrong for that shader, but the + // GPU never wedges. 0 disables the guard (original unbounded behaviour). + private static readonly int _maxDispatcherSteps = + int.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_SHADER_MAX_STEPS"), + out var maxSteps) && maxSteps >= 0 + ? maxSteps + : 100_000; + + // Diagnostic coverage probe. When enabled, every selected MRT export + // writes opaque magenta while preserving the shader's control flow, + // EXEC mask, geometry and raster state. This separates missing + // rasterization from valid fragments whose translated values are zero. + private static readonly bool _forcePixelMagenta = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_FORCE_PIXEL_MAGENTA"), + "1", + StringComparison.Ordinal); + + // Which pixel-shader MRT export target (EXP_MRT0..7 == render-target + // slot) is routed to the single fragment output. The offscreen draw + // path renders one bound color target per pass, so a multi-render-target + // (deferred G-buffer) draw compiles one pixel variant per slot, each + // selecting that slot's export here. + // Vertex stage only: the fragment shader paired with this vertex shader + // declares interpolated inputs for locations 0..(this-1). Metal requires + // every fragment input location to be written by the vertex shader, so + // the vertex stage must export at least this many param outputs (any it + // does not naturally export are zero-filled) or pipeline creation fails + // with "Fragment input(s) `user(locnN)` ... not written by vertex shader". + private readonly int _requiredVertexOutputCount; private readonly uint _localSizeX; private readonly uint _localSizeY; private readonly uint _localSizeZ; private readonly int _globalBufferBase; private readonly int _totalGlobalBufferCount; private readonly int _imageBindingBase; - private readonly int _scalarRegisterBufferIndex; + private readonly int _initialScalarBufferIndex; + private readonly uint _pixelInputEnable; + private readonly uint _pixelInputAddress; + private readonly ulong _storageBufferOffsetAlignment; private readonly List _interfaces = []; private readonly Dictionary _pixelInputs = []; private readonly Dictionary _pixelOutputs = []; @@ -155,6 +235,7 @@ public static partial class Gen5SpirvTranslator private readonly List _imageResources = []; private readonly Dictionary _imageBindingByPc = []; private readonly Dictionary _bufferBindingByPc = []; + private readonly Dictionary _scalarDefinitionsBeforePc = []; private uint _voidType; private uint _boolType; private uint _uintType; @@ -169,25 +250,40 @@ public static partial class Gen5SpirvTranslator private uint _uvec3Type; private uint _uvec4Type; private uint _privateUintPointer; + private uint _privateVec2Pointer; private uint _privateBoolPointer; + private uint _runtimeBufferBiases; private uint _scalarRegisters; private uint _vectorRegisters; + private uint _packedHalfRegisters; private uint _scc; private uint _vcc; private uint _exec; + private uint _reachedPixelExport; private uint _programCounter; private uint _programActive; + private uint _iterationGuard; private uint _globalBuffers; + private uint _gfx10BufferFormatTable; + private uint _storageBlockPointer; private uint _storageUintPointer; private uint _lds; - private uint _workgroupUintPointer; + private uint _ldsElementPointer; + private uint _ldsDwordMask; private uint _positionOutput; private uint _vertexIndexInput; private uint _instanceIndexInput; private uint _fragCoordInput; private uint _localInvocationIdInput; + private uint _localInvocationIndexInput; private uint _workGroupIdInput; + private uint _computeDispatchLimit; + private uint _pushConstantUintPointer; private uint _subgroupInvocationIdInput; + private uint _waveMaskScratch; + private uint _waveMaskScratchElementPointer; + private uint _waveBroadcastScratch; + private bool _waveScratchInLds; private uint _glsl; private enum ImageComponentKind @@ -227,12 +323,23 @@ public static partial class Gen5SpirvTranslator int globalBufferBase, int totalGlobalBufferCount, int imageBindingBase, - int scalarRegisterBufferIndex) + int initialScalarBufferIndex, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + int requiredVertexOutputCount = 0, + uint waveLaneCount = 32, + ulong storageBufferOffsetAlignment = 1) { _stage = stage; + _requiredVertexOutputCount = requiredVertexOutputCount; _state = state; _evaluation = evaluation; _pixelOutputBindings = pixelOutputBindings; + _waveLaneCount = waveLaneCount == 64 ? 64u : 32u; + _emulateWave64 = + stage == Gen5SpirvStage.Compute && + _waveLaneCount == 64 && + (ulong)localSizeX * localSizeY * localSizeZ == 64; _localSizeX = localSizeX; _localSizeY = localSizeY; _localSizeZ = localSizeZ; @@ -241,7 +348,20 @@ public static partial class Gen5SpirvTranslator ? evaluation.GlobalMemoryBindings.Count : totalGlobalBufferCount; _imageBindingBase = imageBindingBase; - _scalarRegisterBufferIndex = scalarRegisterBufferIndex; + _initialScalarBufferIndex = initialScalarBufferIndex; + _pixelInputEnable = pixelInputEnable; + _pixelInputAddress = pixelInputAddress; + if (storageBufferOffsetAlignment == 0 || + (storageBufferOffsetAlignment & (storageBufferOffsetAlignment - 1)) != 0 || + storageBufferOffsetAlignment > uint.MaxValue) + { + throw new ArgumentOutOfRangeException( + nameof(storageBufferOffsetAlignment), + storageBufferOffsetAlignment, + "storage-buffer offset alignment must be a uint-sized power of two"); + } + + _storageBufferOffsetAlignment = storageBufferOffsetAlignment; } public bool TryCompile(out Gen5SpirvShader shader, out string error) @@ -250,6 +370,75 @@ public static partial class Gen5SpirvTranslator error = string.Empty; try { + if (Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_TITLE_INTERFACE") == "1" && + _state.Program.Address is 0x0000000500780000ul or + 0x0000000500781200ul) + { + Console.Error.WriteLine( + $"[AGC][TITLE-INTERFACE] stage={_stage} " + + $"address=0x{_state.Program.Address:X16} " + + $"required_vertex_outputs={_requiredVertexOutputCount} " + + $"ps_ena=0x{_pixelInputEnable:X8} ps_addr=0x{_pixelInputAddress:X8}"); + foreach (var instruction in _state.Program.Instructions) + { + if (instruction.Control is Gen5ExportControl export) + { + Console.Error.WriteLine( + $"[AGC][TITLE-INTERFACE] pc=0x{instruction.Pc:X4} " + + $"export_target={export.Target} mask=0x{export.EnableMask:X} " + + $"compressed={export.Compressed} src=[" + + string.Join(',', instruction.Sources) + "]"); + } + else if (instruction.Control is Gen5InterpolationControl interpolation) + { + Console.Error.WriteLine( + $"[AGC][TITLE-INTERFACE] pc=0x{instruction.Pc:X4} " + + $"attribute={interpolation.Attribute} " + + $"channel={interpolation.Channel} dst=[" + + string.Join(',', instruction.Destinations) + "]"); + } + else if (instruction.Control is Gen5ScalarMemoryControl scalarMemory) + { + var bindingIndex = -1; + for (var index = 0; + index < _evaluation.GlobalMemoryBindings.Count; + index++) + { + if (_evaluation.GlobalMemoryBindings[index] + .InstructionPcs.Contains(instruction.Pc)) + { + bindingIndex = index; + break; + } + } + + var binding = bindingIndex >= 0 + ? _evaluation.GlobalMemoryBindings[bindingIndex] + : null; + var byteOffset = scalarMemory.ImmediateOffsetBytes; + var sample = binding is not null && + scalarMemory.DynamicOffsetRegister is null && + byteOffset >= 0 && + byteOffset < binding.DataLength + ? Convert.ToHexString( + binding.Data.AsSpan( + byteOffset, + Math.Min( + checked((int)scalarMemory.DestinationCount * 4), + binding.DataLength - byteOffset))) + : "dynamic-or-unavailable"; + Console.Error.WriteLine( + $"[AGC][TITLE-INTERFACE] pc=0x{instruction.Pc:X4} " + + $"scalar_load={instruction.Opcode} binding={bindingIndex} " + + $"scalar=s{binding?.ScalarAddress} " + + $"base=0x{binding?.BaseAddress:X16} length={binding?.DataLength} " + + $"offset={byteOffset} dynamic={scalarMemory.DynamicOffsetRegister} " + + $"bytes={sample}"); + } + } + } + DeclareModule(); var blocks = BuildBasicBlocks(_state.Program.Instructions); if (blocks.Count == 0) @@ -258,10 +447,39 @@ public static partial class Gen5SpirvTranslator return false; } + BuildScalarDefinitionInfo(blocks, _state.Program.Instructions); + var functionType = _module.TypeFunction(_voidType); var main = _module.BeginFunction(_voidType, functionType); _module.AddName(main, "main"); _module.AddLabel(); + if (_stage == Gen5SpirvStage.Pixel && + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_EARLY_COLOR") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + var earlyOutput = _pixelOutputs + .OrderBy(static pair => pair.Key) + .Select(static pair => pair.Value) + .First(); + var earlyColor = earlyOutput.Kind switch + { + Gen5PixelOutputKind.Float => + _module.AddInstruction( + SpirvOp.CompositeConstruct, + earlyOutput.Type, + Float(1f), + Float(0f), + Float(1f), + Float(1f)), + Gen5PixelOutputKind.Sint => + _module.ConstantNull(earlyOutput.Type), + _ => _module.ConstantNull(earlyOutput.Type), + }; + Store(earlyOutput.Variable, earlyColor); + _module.AddStatement(SpirvOp.Return); + _module.AddLabel(); + } EmitInitialState(); var loopHeader = _module.AllocateId(); @@ -314,12 +532,91 @@ public static partial class Gen5SpirvTranslator _module.AddStatement(SpirvOp.Branch, loopContinue); _module.AddLabel(loopContinue); var active = Load(_boolType, _programActive); + if (_maxDispatcherSteps > 0) + { + var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); + Store(_iterationGuard, steps); + var withinLimit = _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + steps, + UInt((uint)_maxDispatcherSteps)); + active = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + active, + withinLimit); + } + _module.AddStatement( SpirvOp.BranchConditional, active, loopHeader, loopMerge); _module.AddLabel(loopMerge); + if (_stage == Gen5SpirvStage.Pixel && + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_TITLE_SHADER_STATE") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + var stateOutput = _pixelOutputs + .OrderBy(static pair => pair.Key) + .Select(static pair => pair.Value) + .FirstOrDefault(static output => + output.Kind == Gen5PixelOutputKind.Float); + if (stateOutput.Variable != 0) + { + uint EncodeBool(uint condition) => + _module.AddInstruction( + SpirvOp.Select, + _floatType, + condition, + Float(1f), + Float(0f)); + + Store( + stateOutput.Variable, + _module.AddInstruction( + SpirvOp.CompositeConstruct, + stateOutput.Type, + EncodeBool(Load(_boolType, _exec)), + EncodeBool(IsWaveMaskActive(LoadS64(52))), + EncodeBool(Load(_boolType, _reachedPixelExport)), + Float(1f))); + } + + StoreS64( + 126, + _module.Constant64(_ulongType, 1)); + } + if (_stage == Gen5SpirvStage.Pixel) + { + // A fragment lane removed from EXEC is not a request to + // write the output variable's zero initializer. It is a + // killed fragment and must not participate in color, + // depth, or blend operations. Keep EXEC masking during + // translation, then terminate lanes that remain inactive + // when the guest pixel shader exits. + var returnLabel = _module.AllocateId(); + var killLabel = _module.AllocateId(); + // Materialize the condition before SelectionMerge: SPIR-V + // requires the merge instruction to be immediately followed + // by its structured branch terminator. + var laneActive = Load(_boolType, _exec); + _module.AddStatement( + SpirvOp.SelectionMerge, + returnLabel, + 0); + _module.AddStatement( + SpirvOp.BranchConditional, + laneActive, + returnLabel, + killLabel); + _module.AddLabel(killLabel); + _module.AddStatement(SpirvOp.Kill); + _module.AddLabel(returnLabel); + } + _module.AddStatement(SpirvOp.Return); _module.EndFunction(); @@ -394,6 +691,11 @@ public static partial class Gen5SpirvTranslator { _module.AddCapability(SpirvCapability.GroupNonUniformVote); } + + if (UsesSubgroupBroadcast() || UsesWaveControl()) + { + _module.AddCapability(SpirvCapability.GroupNonUniformBallot); + } } _glsl = _module.ImportExtInst("GLSL.std.450"); @@ -412,15 +714,20 @@ public static partial class Gen5SpirvTranslator _uvec4Type = _module.TypeVector(_uintType, 4); _privateUintPointer = _module.TypePointer(SpirvStorageClass.Private, _uintType); + _privateVec2Pointer = + _module.TypePointer(SpirvStorageClass.Private, _vec2Type); _privateBoolPointer = _module.TypePointer(SpirvStorageClass.Private, _boolType); var scalarArrayType = _module.TypeArray(_uintType, ScalarRegisterCount); var vectorArrayType = _module.TypeArray(_uintType, VectorRegisterCount); + var packedHalfArrayType = _module.TypeArray(_vec2Type, VectorRegisterCount); var privateScalarArrayPointer = _module.TypePointer(SpirvStorageClass.Private, scalarArrayType); var privateVectorArrayPointer = _module.TypePointer(SpirvStorageClass.Private, vectorArrayType); + var privatePackedHalfArrayPointer = + _module.TypePointer(SpirvStorageClass.Private, packedHalfArrayType); _scalarRegisters = _module.AddGlobalVariable( privateScalarArrayPointer, SpirvStorageClass.Private, @@ -429,6 +736,10 @@ public static partial class Gen5SpirvTranslator privateVectorArrayPointer, SpirvStorageClass.Private, _module.ConstantNull(vectorArrayType)); + _packedHalfRegisters = _module.AddGlobalVariable( + privatePackedHalfArrayPointer, + SpirvStorageClass.Private, + _module.ConstantNull(packedHalfArrayType)); _scc = _module.AddGlobalVariable( _privateBoolPointer, SpirvStorageClass.Private, @@ -441,6 +752,10 @@ public static partial class Gen5SpirvTranslator _privateBoolPointer, SpirvStorageClass.Private, _module.ConstantBool(true)); + _reachedPixelExport = _module.AddGlobalVariable( + _privateBoolPointer, + SpirvStorageClass.Private, + _module.ConstantBool(false)); _programCounter = _module.AddGlobalVariable( _privateUintPointer, SpirvStorageClass.Private, @@ -449,37 +764,155 @@ public static partial class Gen5SpirvTranslator _privateBoolPointer, SpirvStorageClass.Private, _module.ConstantBool(true)); + if (_maxDispatcherSteps > 0) + { + _iterationGuard = _module.AddGlobalVariable( + _privateUintPointer, + SpirvStorageClass.Private, + _module.Constant(_uintType, 0)); + _interfaces.Add(_iterationGuard); + _module.AddName(_iterationGuard, "pcGuard"); + } + _interfaces.Add(_scalarRegisters); _interfaces.Add(_vectorRegisters); + _interfaces.Add(_packedHalfRegisters); _interfaces.Add(_scc); _interfaces.Add(_vcc); _interfaces.Add(_exec); + _interfaces.Add(_reachedPixelExport); _interfaces.Add(_programCounter); _interfaces.Add(_programActive); _module.AddName(_scalarRegisters, "sgpr"); _module.AddName(_vectorRegisters, "vgpr"); + _module.AddName(_packedHalfRegisters, "vgprPackedHalf"); + + var runtimeBufferBiasCount = + _globalBufferBase + _evaluation.GlobalMemoryBindings.Count; + if (_initialScalarBufferIndex >= 0 && runtimeBufferBiasCount > 0) + { + var biasArrayType = _module.TypeArray( + _uintType, + (uint)runtimeBufferBiasCount); + var privateBiasArrayPointer = _module.TypePointer( + SpirvStorageClass.Private, + biasArrayType); + _runtimeBufferBiases = _module.AddGlobalVariable( + privateBiasArrayPointer, + SpirvStorageClass.Private, + _module.ConstantNull(biasArrayType)); + _module.AddName(_runtimeBufferBiases, "guestBufferByteBias"); + _interfaces.Add(_runtimeBufferBiases); + } DeclareBuffers(); DeclareImages(); DeclareLds(); + DeclareWave64Scratch(); DeclareStageInterface(); + DeclareComputeDispatchLimit(); } - private void DeclareLds() + private void DeclareComputeDispatchLimit() { - if (_stage != Gen5SpirvStage.Compute || !UsesLds()) + if (_stage != Gen5SpirvStage.Compute) { return; } - var ldsArrayType = _module.TypeArray(_uintType, LdsDwordCount); - var ldsPointer = - _module.TypePointer(SpirvStorageClass.Workgroup, ldsArrayType); - _workgroupUintPointer = + // RDNA DISPATCH_* can express exact thread dimensions, including + // a final partially populated workgroup. Vulkan dispatches whole + // workgroups, so the command path supplies the exact exclusive + // thread bounds through a small push-constant block and excess + // invocations are disabled before any guest instruction executes. + var block = _module.TypeStruct(_uvec3Type); + _module.AddDecoration(block, SpirvDecoration.Block); + _module.AddMemberDecoration(block, 0, SpirvDecoration.Offset, 0); + var blockPointer = + _module.TypePointer(SpirvStorageClass.PushConstant, block); + _pushConstantUintPointer = + _module.TypePointer(SpirvStorageClass.PushConstant, _uintType); + _computeDispatchLimit = _module.AddGlobalVariable( + blockPointer, + SpirvStorageClass.PushConstant); + _module.AddName(_computeDispatchLimit, "dispatchThreadLimit"); + _interfaces.Add(_computeDispatchLimit); + } + + private void DeclareWave64Scratch() + { + if (!_emulateWave64 || !UsesSubgroupOperations()) + { + return; + } + + // Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we + // target. Some PS5 compute shaders legitimately request all of it, + // so allocating another workgroup variable for the wave64 bridge + // makes pipeline creation fail. Reuse the final three dwords of the + // existing LDS allocation in that case. The translator already + // bounds guest LDS accesses to this fixed allocation; keeping the + // bridge inside it preserves the host limit and still provides the + // cross-subgroup rendezvous needed to model one 64-lane guest wave. + if (_lds != 0) + { + _waveScratchInLds = true; + _waveMaskScratchElementPointer = _ldsElementPointer; + return; + } + + var maskArrayType = _module.TypeArray(_uintType, 2); + var maskArrayPointer = + _module.TypePointer(SpirvStorageClass.Workgroup, maskArrayType); + _waveMaskScratchElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); - _lds = _module.AddGlobalVariable( - ldsPointer, + _waveMaskScratch = _module.AddGlobalVariable( + maskArrayPointer, SpirvStorageClass.Workgroup); + _module.AddName(_waveMaskScratch, "wave64MaskScratch"); + _interfaces.Add(_waveMaskScratch); + + var uintPointer = + _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); + _waveBroadcastScratch = _module.AddGlobalVariable( + uintPointer, + SpirvStorageClass.Workgroup); + _module.AddName(_waveBroadcastScratch, "wave64BroadcastScratch"); + _interfaces.Add(_waveBroadcastScratch); + } + + private void DeclareLds() + { + if (!UsesLds()) + { + return; + } + + // Compute shaders get genuine workgroup-shared LDS. Graphics stages + // (NGG export/vertex, pixel) cannot use the Workgroup storage class + // in SPIR-V, but they still emit ds_write/ds_read — typically as + // per-invocation scratch/spill or as NGG staging whose cross-lane + // reads don't feed this stage's exports. Model those as a + // per-invocation Private array so the shader is valid SPIR-V and its + // draw stops being dropped. Index masking in LdsPointer keeps the + // arbitrary computed addresses inside the array. + var storageClass = _stage == Gen5SpirvStage.Compute + ? SpirvStorageClass.Workgroup + : SpirvStorageClass.Private; + var dwordCount = _stage == Gen5SpirvStage.Compute + ? LdsDwordCount + : PrivateLdsDwordCount; + _ldsDwordMask = dwordCount - 1; + + var ldsArrayType = _module.TypeArray(_uintType, dwordCount); + var ldsPointer = _module.TypePointer(storageClass, ldsArrayType); + _ldsElementPointer = _module.TypePointer(storageClass, _uintType); + _lds = storageClass == SpirvStorageClass.Workgroup + ? _module.AddGlobalVariable(ldsPointer, storageClass) + : _module.AddGlobalVariable( + ldsPointer, + storageClass, + _module.ConstantNull(ldsArrayType)); _module.AddName(_lds, "lds"); _interfaces.Add(_lds); } @@ -509,6 +942,8 @@ public static partial class Gen5SpirvTranslator (uint)_totalGlobalBufferCount); var descriptorsPointer = _module.TypePointer(SpirvStorageClass.StorageBuffer, descriptors); + _storageBlockPointer = + _module.TypePointer(SpirvStorageClass.StorageBuffer, block); _storageUintPointer = _module.TypePointer(SpirvStorageClass.StorageBuffer, _uintType); _globalBuffers = _module.AddGlobalVariable( @@ -600,8 +1035,15 @@ public static partial class Gen5SpirvTranslator return (SpirvImageFormat.Unknown, ImageComponentKind.Float); } - var dataFormat = (descriptor[1] >> 20) & 0x1FFu; - var numberType = (descriptor[1] >> 26) & 0xFu; + var unifiedFormat = (descriptor[1] >> 20) & 0x1FFu; + if (!Gfx10UnifiedFormat.TryDecode( + unifiedFormat, + out var dataFormat, + out var numberType)) + { + return (SpirvImageFormat.Unknown, ImageComponentKind.Float); + } + return (dataFormat, numberType) switch { (1, _) => (SpirvImageFormat.R8, ImageComponentKind.Float), @@ -668,6 +1110,18 @@ public static partial class Gen5SpirvTranslator SpirvDecoration.BuiltIn, (uint)SpirvBuiltIn.SubgroupLocalInvocationId); _interfaces.Add(_subgroupInvocationIdInput); + + if (_waveLaneCount == 64) + { + _localInvocationIndexInput = _module.AddGlobalVariable( + subgroupPointer, + SpirvStorageClass.Input); + _module.AddDecoration( + _localInvocationIndexInput, + SpirvDecoration.BuiltIn, + (uint)SpirvBuiltIn.LocalInvocationIndex); + _interfaces.Add(_localInvocationIndexInput); + } } if (_stage == Gen5SpirvStage.Vertex) @@ -710,6 +1164,13 @@ public static partial class Gen5SpirvTranslator .OfType() .Where(export => export.Target is >= 32 and < 64) .Select(export => export.Target - 32) + // Cover every location the paired fragment shader reads, even + // ones this vertex program never exports, so Metal's exact + // vertex-out/fragment-in interface match succeeds. Extras are + // zero-filled in EmitInitialState. + .Concat(Enumerable + .Range(0, Math.Max(_requiredVertexOutputCount, 0)) + .Select(location => (uint)location)) .Distinct() .Order() .ToArray(); @@ -753,7 +1214,13 @@ public static partial class Gen5SpirvTranslator (uint)SpirvBuiltIn.FragCoord); _interfaces.Add(_fragCoordInput); - foreach (var binding in _pixelOutputBindings) + var declaredPixelOutputs = + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_SINGLE_MRT") == "1" && + _state.Program.Address == 0x0000000500781200ul + ? _pixelOutputBindings.Take(1) + : _pixelOutputBindings; + foreach (var binding in declaredPixelOutputs) { var outputType = GetPixelOutputType(binding.Kind); var outputPointer = @@ -833,11 +1300,37 @@ public static partial class Gen5SpirvTranslator private void EmitInitialState() { - if (_scalarRegisterBufferIndex >= 0) + if (_initialScalarBufferIndex >= 0) { - for (uint index = 0; index < ScalarRegisterCount; index++) + // Initial scalar registers arrive in a per-draw buffer instead + // of being baked as constants, so animated user data (colors, + // scroll offsets) reuses one translation and pipeline. Only + // registers the program can observe need loading. + var consumed = Gen5ShaderTranslator.ComputeConsumedScalarMask(_state.Program); + for (uint index = 0; + index < _evaluation.InitialScalarRegisters.Count && + index < ScalarRegisterCount; + index++) { - StoreS(index, LoadBufferWord(_scalarRegisterBufferIndex, UInt(index))); + if (Gen5ShaderTranslator.IsScalarConsumed(consumed, index)) + { + StoreS( + index, + LoadBufferWord(_initialScalarBufferIndex, UInt(index))); + } + } + + var runtimeBufferBiasCount = + _globalBufferBase + _evaluation.GlobalMemoryBindings.Count; + for (var binding = 0; + binding < runtimeBufferBiasCount; + binding++) + { + Store( + RuntimeBufferBiasPointer(binding), + LoadBufferWord( + _initialScalarBufferIndex, + UInt(checked(256u + (uint)binding)))); } } else @@ -856,6 +1349,7 @@ public static partial class Gen5SpirvTranslator } Store(_scc, _module.ConstantBool(false)); + Store(_reachedPixelExport, _module.ConstantBool(false)); if (_subgroupInvocationIdInput != 0) { StoreWaveMask(106, _module.ConstantBool(false)); @@ -863,8 +1357,14 @@ public static partial class Gen5SpirvTranslator } else { - Store(_vcc, _module.ConstantBool(false)); - Store(_exec, _module.ConstantBool(true)); + // Graphics stages emulate one logical wave lane. Keep the + // guest-visible VCC/EXEC scalar pairs synchronized with the + // internal booleans: shaders commonly save EXEC from s126:s127 + // and restore it after divergent work. Initializing only _exec + // left those registers at zero, so the first restore disabled + // every fragment before its color export. + StoreS64(106, _module.Constant64(_ulongType, 0)); + StoreS64(126, _module.Constant64(_ulongType, 1)); } Store(_programCounter, UInt(0)); Store(_programActive, _module.ConstantBool(true)); @@ -873,22 +1373,21 @@ public static partial class Gen5SpirvTranslator { StoreV(5, Load(_uintType, _vertexIndexInput), guardWithExec: false); StoreV(8, Load(_uintType, _instanceIndexInput), guardWithExec: false); + + // Give every declared param output a defined starting value. + // Outputs the program actually exports overwrite this; the + // extras that only exist to satisfy the fragment interface stay + // zero. The explicit store also keeps SPIRV-Cross from pruning + // an unexported output (which would re-break the interface). + foreach (var output in _vertexOutputs.Values) + { + Store(output, _module.ConstantNull(_vec4Type)); + } } else if (_stage == Gen5SpirvStage.Pixel) { var fragCoord = Load(_vec4Type, _fragCoordInput); - var x = _module.AddInstruction( - SpirvOp.CompositeExtract, - _floatType, - fragCoord, - 0); - var y = _module.AddInstruction( - SpirvOp.CompositeExtract, - _floatType, - fragCoord, - 1); - StoreV(2, Bitcast(_uintType, x), guardWithExec: false); - StoreV(3, Bitcast(_uintType, y), guardWithExec: false); + EmitPixelInputState(fragCoord); foreach (var output in _pixelOutputs.Values) { Store(output.Variable, _module.ConstantNull(output.Type)); @@ -897,19 +1396,57 @@ public static partial class Gen5SpirvTranslator else { var localId = Load(_uvec3Type, _localInvocationIdInput); + var workGroupId = Load(_uvec3Type, _workGroupIdInput); + var invocationInBounds = _module.ConstantBool(true); for (uint component = 0; component < 3; component++) { - var value = _module.AddInstruction( + var localComponent = _module.AddInstruction( SpirvOp.CompositeExtract, _uintType, localId, component); - StoreV(component, value, guardWithExec: false); + StoreV(component, localComponent, guardWithExec: false); + + var groupComponent = _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + workGroupId, + component); + var localSize = component switch + { + 0 => _localSizeX, + 1 => _localSizeY, + _ => _localSizeZ, + }; + var globalComponent = IAdd( + _module.AddInstruction( + SpirvOp.IMul, + _uintType, + groupComponent, + UInt(localSize)), + localComponent); + var limitPointer = _module.AddInstruction( + SpirvOp.AccessChain, + _pushConstantUintPointer, + _computeDispatchLimit, + UInt(0), + UInt(component)); + var componentInBounds = _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + globalComponent, + Load(_uintType, limitPointer)); + invocationInBounds = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + invocationInBounds, + componentInBounds); } + Store(_programActive, invocationInBounds); + if (_state.ComputeSystemRegisters is { } registers) { - var workGroupId = Load(_uvec3Type, _workGroupIdInput); StoreComputeSystemRegister( registers.WorkGroupXRegister, workGroupId, @@ -932,6 +1469,70 @@ public static partial class Gen5SpirvTranslator } } + private void EmitPixelInputState(uint fragCoord) + { + uint vgpr = 0; + + // Pixel input VGPRs are compacted in SPI_PS_INPUT_ADDR order. The + // interpolation inputs occupy register slots even though V_INTERP + // is lowered directly from SPIR-V interpolants; position inputs + // following them must still land in the hardware-selected VGPRs. + AdvancePixelInput(0, 2, ref vgpr); // PERSP_SAMPLE + AdvancePixelInput(1, 2, ref vgpr); // PERSP_CENTER + AdvancePixelInput(2, 2, ref vgpr); // PERSP_CENTROID + AdvancePixelInput(3, 3, ref vgpr); // PERSP_PULL_MODEL + AdvancePixelInput(4, 2, ref vgpr); // LINEAR_SAMPLE + AdvancePixelInput(5, 2, ref vgpr); // LINEAR_CENTER + AdvancePixelInput(6, 2, ref vgpr); // LINEAR_CENTROID + AdvancePixelInput(7, 1, ref vgpr); // LINE_STIPPLE + + EmitPixelPositionInput(8, 0, fragCoord, ref vgpr); // POS_X_FLOAT + EmitPixelPositionInput(9, 1, fragCoord, ref vgpr); // POS_Y_FLOAT + EmitPixelPositionInput(10, 2, fragCoord, ref vgpr); // POS_Z_FLOAT + EmitPixelPositionInput(11, 3, fragCoord, ref vgpr); // POS_W_FLOAT + + // FRONT_FACE, ANCILLARY, SAMPLE_COVERAGE and POS_FIXED_PT follow + // position inputs. Reserve their compact slots until their SPIR-V + // builtins are needed by a guest shader. + AdvancePixelInput(12, 1, ref vgpr); + AdvancePixelInput(13, 1, ref vgpr); + AdvancePixelInput(14, 1, ref vgpr); + AdvancePixelInput(15, 1, ref vgpr); + } + + private void AdvancePixelInput(int bit, uint dwordCount, ref uint vgpr) + { + if ((_pixelInputAddress & (1u << bit)) != 0) + { + vgpr += dwordCount; + } + } + + private void EmitPixelPositionInput( + int bit, + uint component, + uint fragCoord, + ref uint vgpr) + { + var mask = 1u << bit; + if ((_pixelInputAddress & mask) == 0) + { + return; + } + + if ((_pixelInputEnable & mask) != 0) + { + var value = _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + fragCoord, + component); + StoreV(vgpr, Bitcast(_uintType, value), guardWithExec: false); + } + + vgpr++; + } + private void StoreComputeSystemRegister( uint? register, uint workGroupId, @@ -970,6 +1571,11 @@ public static partial class Gen5SpirvTranslator error = $"pc=0x{instruction.Pc:X} {instruction.Opcode}: {error}"; return false; } + + CapturePixelVgprs(instruction); + CapturePixelVgprPoints(instruction); + MarkPixelPath(instruction); + CapturePixelExec(instruction); } var terminator = _state.Program.Instructions[block.EndIndex - 1]; @@ -1180,9 +1786,8 @@ public static partial class Gen5SpirvTranslator out string error) { error = string.Empty; - if (_stage != Gen5SpirvStage.Compute || - _lds == 0 || - _workgroupUintPointer == 0 || + if (_lds == 0 || + _ldsElementPointer == 0 || instruction.Control is not Gen5DataShareControl control) { error = "invalid LDS instruction"; @@ -1197,6 +1802,24 @@ public static partial class Gen5SpirvTranslator switch (instruction.Opcode) { + case "DsAddU32": + { + if (instruction.Sources.Count < 2) + { + error = "missing LDS atomic-add source"; + return false; + } + + var address = GetRawSource(instruction, 0); + _module.AddInstruction( + SpirvOp.AtomicIAdd, + _uintType, + LdsPointer(address, control.Offset0), + UInt(2), // Workgroup scope. + UInt(0x108), // AcquireRelease | WorkgroupMemory. + GetRawSource(instruction, 1)); + return true; + } case "DsWriteB32": { if (instruction.Sources.Count < 2) @@ -1207,7 +1830,7 @@ public static partial class Gen5SpirvTranslator var address = GetRawSource(instruction, 0); StoreLds( - LdsPointer(address, EffectiveDsOffsetBytes(control.Offset0)), + LdsPointer(address, control.Offset0), GetRawSource(instruction, 1)); return true; } @@ -1220,13 +1843,36 @@ public static partial class Gen5SpirvTranslator } var address = GetRawSource(instruction, 0); - var offset = EffectiveDsOffsetBytes(control.Offset0); + var offset = control.Offset0; StoreLds(LdsPointer(address, offset), GetRawSource(instruction, 1)); StoreLds( LdsPointer(address, offset + sizeof(uint)), GetRawSource(instruction, 2)); return true; } + case "DsWriteB96": + case "DsWriteB128": + { + // ds_write_b96 stores 3 consecutive dwords, ds_write_b128 + // stores 4, from data0..data0+N at the address's offset. + var dwordCount = instruction.Opcode == "DsWriteB128" ? 4 : 3; + if (instruction.Sources.Count < 1 + dwordCount) + { + error = "missing LDS write128 source"; + return false; + } + + var address = GetRawSource(instruction, 0); + var offset = control.Offset0; + for (var dword = 0; dword < dwordCount; dword++) + { + StoreLds( + LdsPointer(address, offset + (uint)(dword * sizeof(uint))), + GetRawSource(instruction, 1 + dword)); + } + + return true; + } case "DsWrite2B32": case "DsWrite2St64B32": { @@ -1241,12 +1887,12 @@ public static partial class Gen5SpirvTranslator StoreLds( LdsPointer( address, - EffectiveDsOffsetBytes(control.Offset0, st64)), + EffectiveDsPairOffsetBytes(control.Offset0, st64)), GetRawSource(instruction, 1)); StoreLds( LdsPointer( address, - EffectiveDsOffsetBytes(control.Offset1, st64)), + EffectiveDsPairOffsetBytes(control.Offset1, st64)), GetRawSource(instruction, 2)); return true; } @@ -1262,10 +1908,35 @@ public static partial class Gen5SpirvTranslator var address = GetRawSource(instruction, 0); var value = Load( _uintType, - LdsPointer(address, EffectiveDsOffsetBytes(control.Offset0))); + LdsPointer(address, control.Offset0)); StoreV(instruction.Destinations[0].Value, value); return true; } + case "DsReadB96": + case "DsReadB128": + { + // ds_read_b96 loads 3 consecutive dwords, ds_read_b128 loads + // 4, into dest..dest+N from the address's offset. + var dwordCount = instruction.Opcode == "DsReadB128" ? 4 : 3; + if (instruction.Destinations.Count < dwordCount || + instruction.Sources.Count < 1) + { + error = "missing LDS read128 operand"; + return false; + } + + var address = GetRawSource(instruction, 0); + var offset = control.Offset0; + for (var dword = 0; dword < dwordCount; dword++) + { + var value = Load( + _uintType, + LdsPointer(address, offset + (uint)(dword * sizeof(uint)))); + StoreV(instruction.Destinations[dword].Value, value); + } + + return true; + } case "DsRead2B32": case "DsRead2St64B32": { @@ -1282,12 +1953,12 @@ public static partial class Gen5SpirvTranslator _uintType, LdsPointer( address, - EffectiveDsOffsetBytes(control.Offset0, st64))); + EffectiveDsPairOffsetBytes(control.Offset0, st64))); var second = Load( _uintType, LdsPointer( address, - EffectiveDsOffsetBytes(control.Offset1, st64))); + EffectiveDsPairOffsetBytes(control.Offset1, st64))); StoreV(instruction.Destinations[0].Value, first); StoreV(instruction.Destinations[1].Value, second); return true; @@ -1298,7 +1969,7 @@ public static partial class Gen5SpirvTranslator } } - private static uint EffectiveDsOffsetBytes(uint offset, bool st64 = false) => + private static uint EffectiveDsPairOffsetBytes(uint offset, bool st64 = false) => offset * (st64 ? 256u : sizeof(uint)); private uint LdsPointer(uint address, uint offsetBytes) @@ -1306,10 +1977,16 @@ public static partial class Gen5SpirvTranslator var addressWithOffset = offsetBytes == 0 ? address : IAdd(address, UInt(offsetBytes)); - var index = ShiftRightLogical(addressWithOffset, UInt(2)); + // Mask the dword index into the array bounds. LDS is a power-of-two + // dword count, so this is a no-op for in-range compute addresses but + // prevents out-of-bounds access when a graphics-stage scratch write + // uses an arbitrary computed byte address. + var index = BitwiseAnd( + ShiftRightLogical(addressWithOffset, UInt(2)), + UInt(_ldsDwordMask)); return _module.AddInstruction( SpirvOp.AccessChain, - _workgroupUintPointer, + _ldsElementPointer, _lds, index); } @@ -1357,7 +2034,17 @@ public static partial class Gen5SpirvTranslator out string error) { error = string.Empty; - if (!_bufferBindingByPc.TryGetValue(instruction.Pc, out var bindingIndex)) + var scalarAddress = instruction.Sources.Count != 0 && + instruction.Sources[0].Kind == Gen5OperandKind.ScalarRegister + ? instruction.Sources[0].Value + : uint.MaxValue; + if (!TryResolveDominatingBufferBinding( + instruction.Pc, + scalarAddress, + registerCount: instruction.Opcode.StartsWith( + "SBufferLoad", + StringComparison.Ordinal) ? 4u : 2u, + out var bindingIndex)) { foreach (var destination in instruction.Destinations) { @@ -1376,6 +2063,7 @@ public static partial class Gen5SpirvTranslator var byteAddress = IAdd( dynamicOffset, UInt(unchecked((uint)control.ImmediateOffsetBytes))); + byteAddress = ApplyGuestBufferByteBias(bindingIndex, byteAddress); var dwordAddress = ShiftRightLogical(byteAddress, UInt(2)); for (var index = 0; index < instruction.Destinations.Count; index++) { @@ -1401,7 +2089,11 @@ public static partial class Gen5SpirvTranslator out string error) { error = string.Empty; - if (!_bufferBindingByPc.TryGetValue(instruction.Pc, out var bindingIndex)) + if (!TryResolveDominatingBufferBinding( + instruction.Pc, + control.ScalarAddress, + registerCount: 2, + out var bindingIndex)) { error = "missing global-memory binding"; return false; @@ -1410,15 +2102,95 @@ public static partial class Gen5SpirvTranslator var byteAddress = IAdd( LoadV(control.VectorAddress), UInt(unchecked((uint)control.OffsetBytes))); + byteAddress = ApplyGuestBufferByteBias(bindingIndex, byteAddress); var dwordAddress = ShiftRightLogical(byteAddress, UInt(2)); + + if (instruction.Opcode is "GlobalAtomicAdd" or "GlobalAtomicUMax") + { + EmitExecConditional(() => + { + EmitConditional(IsBufferWordInRange(bindingIndex, dwordAddress), () => + { + var original = _module.AddInstruction( + instruction.Opcode == "GlobalAtomicAdd" + ? SpirvOp.AtomicIAdd + : SpirvOp.AtomicUMax, + _uintType, + BufferWordPointer(bindingIndex, dwordAddress), + UInt(1), + UInt(0x48), + LoadV(control.VectorData)); + if (control.Glc) + { + StoreV(control.VectorData, original); + } + }); + }); + return true; + } + + if (instruction.Opcode.StartsWith("GlobalStore", StringComparison.Ordinal)) + { + EmitExecConditional(() => + { + if (TryGetSubdwordStoreInfo( + instruction.Opcode, + out var byteCount, + out var sourceShift)) + { + StoreBufferBytes( + bindingIndex, + byteAddress, + LoadV(control.VectorData), + byteCount, + sourceShift); + return; + } + + for (uint index = 0; index < control.DwordCount; index++) + { + var address = index == 0 + ? byteAddress + : IAdd(byteAddress, UInt(index * sizeof(uint))); + StoreBufferBytes( + bindingIndex, + address, + LoadV(control.VectorData + index), + sizeof(uint), + 0); + } + }); + return true; + } + + if (TryGetSubdwordLoadInfo( + instruction.Opcode, + out var loadByteCount, + out var signExtend, + out var d16, + out var d16High)) + { + StoreV( + control.VectorData, + LoadSubdwordBufferValue( + bindingIndex, + byteAddress, + LoadV(control.VectorData), + loadByteCount, + signExtend, + d16, + d16High)); + return true; + } + for (uint index = 0; index < control.DwordCount; index++) { var address = index == 0 - ? dwordAddress - : IAdd(dwordAddress, UInt(index)); + ? byteAddress + : IAdd(byteAddress, UInt(index * sizeof(uint))); StoreV( control.VectorData + index, - LoadBufferWord(bindingIndex, address)); + LoadUnalignedBufferWord(bindingIndex, address)); } return true; @@ -1436,14 +2208,11 @@ public static partial class Gen5SpirvTranslator return TryEmitVertexInputFetch(control, vertexInput, out error); } - if (_stage == Gen5SpirvStage.Vertex && - IsFormatBufferLoad(instruction.Opcode)) - { - error = $"missing vertex input for {instruction.Opcode} pc=0x{instruction.Pc:X}"; - return false; - } - - if (!_bufferBindingByPc.TryGetValue(instruction.Pc, out var bindingIndex)) + if (!TryResolveDominatingBufferBinding( + instruction.Pc, + control.ScalarResource, + registerCount: 4, + out var bindingIndex)) { error = "missing buffer-memory binding"; return false; @@ -1467,44 +2236,90 @@ public static partial class Gen5SpirvTranslator byteAddress = IAdd( byteAddress, _module.AddInstruction(SpirvOp.IMul, _uintType, vectorIndex, stride)); + byteAddress = ApplyGuestBufferByteBias(bindingIndex, byteAddress); var dwordAddress = ShiftRightLogical(byteAddress, UInt(2)); - if (instruction.Opcode == "BufferAtomicAdd") + if (instruction.Opcode is "BufferAtomicAdd" or "BufferAtomicUMax") { EmitExecConditional(() => { - var original = _module.AddInstruction( - SpirvOp.AtomicIAdd, - _uintType, - BufferWordPointer(bindingIndex, dwordAddress), - UInt(1), - UInt(0x48), - LoadV(control.VectorData)); - if (control.Glc) + var inRange = IsBufferWordInRange(bindingIndex, dwordAddress); + EmitConditional(inRange, () => { - StoreV(control.VectorData, original); + var original = _module.AddInstruction( + instruction.Opcode == "BufferAtomicAdd" + ? SpirvOp.AtomicIAdd + : SpirvOp.AtomicUMax, + _uintType, + BufferWordPointer(bindingIndex, dwordAddress), + UInt(1), + UInt(0x48), + LoadV(control.VectorData)); + if (control.Glc) + { + StoreV(control.VectorData, original); + } + }); + }); + + return true; + } + + if (instruction.Opcode.StartsWith("BufferStoreDword", StringComparison.Ordinal) || + instruction.Opcode.StartsWith("BufferStoreFormat", StringComparison.Ordinal) || + instruction.Opcode.StartsWith("BufferStoreByte", StringComparison.Ordinal) || + instruction.Opcode.StartsWith("BufferStoreShort", StringComparison.Ordinal)) + { + EmitExecConditional(() => + { + if (TryGetSubdwordStoreInfo( + instruction.Opcode, + out var byteCount, + out var sourceShift)) + { + StoreBufferBytes( + bindingIndex, + byteAddress, + LoadV(control.VectorData), + byteCount, + sourceShift); + return; + } + + for (uint index = 0; index < control.DwordCount; index++) + { + var address = index == 0 + ? byteAddress + : IAdd(byteAddress, UInt(index * sizeof(uint))); + StoreBufferBytes( + bindingIndex, + address, + LoadV(control.VectorData + index), + sizeof(uint), + 0); } }); return true; } - if (instruction.Opcode.StartsWith("BufferStoreDword", StringComparison.Ordinal)) + if (TryGetSubdwordLoadInfo( + instruction.Opcode, + out var loadByteCount, + out var signExtend, + out var d16, + out var d16High)) { - EmitExecConditional(() => - { - for (uint index = 0; index < control.DwordCount; index++) - { - var address = index == 0 - ? dwordAddress - : IAdd(dwordAddress, UInt(index)); - StoreBufferWord( - bindingIndex, - address, - LoadV(control.VectorData + index)); - } - }); - + StoreV( + control.VectorData, + LoadSubdwordBufferValue( + bindingIndex, + byteAddress, + LoadV(control.VectorData), + loadByteCount, + signExtend, + d16, + d16High)); return true; } @@ -1515,23 +2330,715 @@ public static partial class Gen5SpirvTranslator return false; } + // MUBUF format loads take their element format and destination + // swizzle from the GFX10 buffer descriptor. Keep raw dword loads + // on the byte-address >> 2 path below: unlike typed loads they do + // not perform component conversion or dst_sel processing. + // Vertex shaders normally expose indexed format loads as Vulkan + // attributes. Loads with an additional per-lane byte offset cannot + // be represented by a fixed attribute description, so the scalar + // evaluator captures their descriptor as storage instead. Preserve + // typed conversion for both MUBUF and MTBUF in that fallback path. + if (IsFormatBufferLoad(instruction.Opcode)) + { + EmitBufferFormatLoad( + bindingIndex, + byteAddress, + control.ScalarResource, + control.VectorData, + control.DwordCount); + return true; + } + for (uint index = 0; index < control.DwordCount; index++) { var address = index == 0 - ? dwordAddress - : IAdd(dwordAddress, UInt(index)); + ? byteAddress + : IAdd(byteAddress, UInt(index * sizeof(uint))); StoreV( control.VectorData + index, - LoadBufferWord(bindingIndex, address)); + LoadUnalignedBufferWord(bindingIndex, address)); } return true; } + private void EmitBufferFormatLoad( + int bindingIndex, + uint byteAddress, + uint scalarResource, + uint vectorData, + uint componentCount) + { + var descriptorWord3 = LoadS(scalarResource + 3); + var unifiedFormat = BitwiseAnd( + ShiftRightLogical(descriptorWord3, UInt(12)), + UInt(0x7F)); + var (dataFormat, numberFormat) = DecodeGfx10BufferFormat(unifiedFormat); + + var canonical = new uint[4]; + for (var component = 0; component < canonical.Length; component++) + { + canonical[component] = LoadGfx10BufferFormatComponent( + bindingIndex, + byteAddress, + dataFormat, + numberFormat, + component); + } + + var one = Gfx10FormatOne(numberFormat); + for (uint destination = 0; destination < componentCount; destination++) + { + var selector = BitwiseAnd( + ShiftRightLogical(descriptorWord3, UInt(destination * 3)), + UInt(7)); + var value = UInt(0); + value = SelectUInt(selector, 1, one, value); + value = SelectUInt(selector, 4, canonical[0], value); + value = SelectUInt(selector, 5, canonical[1], value); + value = SelectUInt(selector, 6, canonical[2], value); + value = SelectUInt(selector, 7, canonical[3], value); + StoreV(vectorData + destination, value); + } + } + + private (uint DataFormat, uint NumberFormat) DecodeGfx10BufferFormat( + uint unifiedFormat) + { + // The descriptor is loaded at execution time, so format decoding + // must remain dynamic too. Generate one module-level lookup table + // from the same authoritative decoder used by descriptor + // evaluation rather than specializing the shader to the SRD seen + // at compile time (compiled compute shaders may be reused with new + // SRDs). A table also avoids emitting 77 compares at every format + // load site, which matters in buffer-heavy compute kernels. + if (_gfx10BufferFormatTable == 0) + { + const uint formatCount = 128; + var entries = new uint[formatCount]; + for (uint format = 0; format < formatCount; format++) + { + Gfx10UnifiedFormat.TryDecode( + format, + out var decodedDataFormat, + out var decodedNumberFormat); + entries[format] = UInt( + decodedDataFormat | (decodedNumberFormat << 8)); + } + + var tableType = _module.TypeArray(_uintType, formatCount); + var tablePointer = _module.TypePointer( + SpirvStorageClass.Private, + tableType); + _gfx10BufferFormatTable = _module.AddGlobalVariable( + tablePointer, + SpirvStorageClass.Private, + _module.ConstantComposite(tableType, entries)); + _module.AddName(_gfx10BufferFormatTable, "gfx10BufferFormats"); + _interfaces.Add(_gfx10BufferFormatTable); + } + + var entryPointer = _module.AddInstruction( + SpirvOp.AccessChain, + _privateUintPointer, + _gfx10BufferFormatTable, + unifiedFormat); + var entry = Load(_uintType, entryPointer); + return ( + BitwiseAnd(entry, UInt(0xFF)), + BitwiseAnd( + ShiftRightLogical(entry, UInt(8)), + UInt(0xFF))); + } + + private uint LoadGfx10BufferFormatComponent( + int bindingIndex, + uint elementAddress, + uint dataFormat, + uint numberFormat, + int component) + { + var byteOffset = UInt(0); + var bitOffset = UInt(0); + var bitCount = UInt(0); + + void SetLayout(uint format, uint bytes, uint bits, uint count) + { + var matches = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + dataFormat, + UInt(format)); + byteOffset = _module.AddInstruction( + SpirvOp.Select, + _uintType, + matches, + UInt(bytes), + byteOffset); + bitOffset = _module.AddInstruction( + SpirvOp.Select, + _uintType, + matches, + UInt(bits), + bitOffset); + bitCount = _module.AddInstruction( + SpirvOp.Select, + _uintType, + matches, + UInt(count), + bitCount); + } + + // Legacy DATA_FORMAT layouts selected by the GFX10 unified format. + // Packed formats keep their bit offset in the first dword; byte + // offsets are used for naturally aligned vector components. + switch (component) + { + case 0: + SetLayout(1, 0, 0, 8); // 8 + SetLayout(2, 0, 0, 16); // 16 + SetLayout(3, 0, 0, 8); // 8_8 + SetLayout(4, 0, 0, 32); // 32 + SetLayout(5, 0, 0, 16); // 16_16 + SetLayout(6, 0, 0, 10); // 10_11_11 + SetLayout(7, 0, 0, 11); // 11_11_10 + SetLayout(8, 0, 0, 10); // 10_10_10_2 + SetLayout(9, 0, 0, 2); // 2_10_10_10 + SetLayout(10, 0, 0, 8); // 8_8_8_8 + SetLayout(11, 0, 0, 32); // 32_32 + SetLayout(12, 0, 0, 16); // 16_16_16_16 + SetLayout(13, 0, 0, 32); // 32_32_32 + SetLayout(14, 0, 0, 32); // 32_32_32_32 + break; + case 1: + SetLayout(3, 1, 0, 8); + SetLayout(5, 2, 0, 16); + SetLayout(6, 0, 10, 11); + SetLayout(7, 0, 11, 11); + SetLayout(8, 0, 10, 10); + SetLayout(9, 0, 2, 10); + SetLayout(10, 1, 0, 8); + SetLayout(11, 4, 0, 32); + SetLayout(12, 2, 0, 16); + SetLayout(13, 4, 0, 32); + SetLayout(14, 4, 0, 32); + break; + case 2: + SetLayout(6, 0, 21, 11); + SetLayout(7, 0, 22, 10); + SetLayout(8, 0, 20, 10); + SetLayout(9, 0, 12, 10); + SetLayout(10, 2, 0, 8); + SetLayout(12, 4, 0, 16); + SetLayout(13, 8, 0, 32); + SetLayout(14, 8, 0, 32); + break; + case 3: + SetLayout(8, 0, 30, 2); + SetLayout(9, 0, 22, 10); + SetLayout(10, 3, 0, 8); + SetLayout(12, 6, 0, 16); + SetLayout(14, 12, 0, 32); + break; + } + + var packed = LoadUnalignedBufferWord( + bindingIndex, + IAdd(elementAddress, byteOffset)); + var raw = _module.AddInstruction( + SpirvOp.BitFieldUExtract, + _uintType, + packed, + bitOffset, + bitCount); + var converted = ConvertGfx10BufferComponent( + raw, + bitCount, + numberFormat, + dataFormat); + var valid = _module.AddInstruction( + SpirvOp.INotEqual, + _boolType, + bitCount, + UInt(0)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + valid, + converted, + component == 3 ? Gfx10FormatOne(numberFormat) : UInt(0)); + } + + private uint ConvertGfx10BufferComponent( + uint raw, + uint bitCount, + uint numberFormat, + uint dataFormat) + { + var widthIs32 = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + bitCount, + UInt(32)); + var lowMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), bitCount), + UInt(1)); + lowMask = _module.AddInstruction( + SpirvOp.Select, + _uintType, + widthIs32, + UInt(uint.MaxValue), + lowMask); + + var signedRaw = _module.AddInstruction( + SpirvOp.BitFieldSExtract, + _intType, + Bitcast(_intType, raw), + UInt(0), + bitCount); + var signedBits = Bitcast(_uintType, signedRaw); + var unsignedFloat = _module.AddInstruction( + SpirvOp.ConvertUToF, + _floatType, + raw); + var signedFloat = _module.AddInstruction( + SpirvOp.ConvertSToF, + _floatType, + signedRaw); + + var unorm = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.FDiv, + _floatType, + unsignedFloat, + _module.AddInstruction( + SpirvOp.ConvertUToF, + _floatType, + lowMask))); + var signedMaximum = ShiftRightLogical(lowMask, UInt(1)); + var snormFloat = _module.AddInstruction( + SpirvOp.FDiv, + _floatType, + signedFloat, + _module.AddInstruction( + SpirvOp.ConvertUToF, + _floatType, + signedMaximum)); + snormFloat = _module.AddInstruction( + SpirvOp.Select, + _floatType, + _module.AddInstruction( + SpirvOp.FOrdLessThan, + _boolType, + snormFloat, + Float(-1f)), + Float(-1f), + snormFloat); + var snorm = Bitcast(_uintType, snormFloat); + var uscaled = Bitcast(_uintType, unsignedFloat); + var sscaled = Bitcast(_uintType, signedFloat); + + var unpackedHalf = Ext(62, _vec2Type, BitwiseAnd(raw, UInt(0xFFFF))); + var half = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedHalf, + 0)); + var floating = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + bitCount, + UInt(16)), + half, + raw); + + // DATA_FORMAT 10_11_11 and 11_11_10 use unsigned mini-floats + // when NUM_FORMAT is FLOAT, not ordinary integer bit patterns. + var isPackedFloat = _module.AddInstruction( + SpirvOp.LogicalOr, + _boolType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + dataFormat, + UInt(6)), + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + dataFormat, + UInt(7))); + floating = _module.AddInstruction( + SpirvOp.Select, + _uintType, + isPackedFloat, + DecodeUnsignedMiniFloat(raw, bitCount), + floating); + + var result = raw; + result = SelectUInt(numberFormat, 0, unorm, result); + result = SelectUInt(numberFormat, 1, snorm, result); + result = SelectUInt(numberFormat, 2, uscaled, result); + result = SelectUInt(numberFormat, 3, sscaled, result); + result = SelectUInt(numberFormat, 4, raw, result); + result = SelectUInt(numberFormat, 5, signedBits, result); + result = SelectUInt(numberFormat, 7, floating, result); + return result; + } + + private uint DecodeUnsignedMiniFloat(uint raw, uint bitCount) + { + var mantissaBits = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + bitCount, + UInt(5)); + var mantissaMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), mantissaBits), + UInt(1)); + var mantissa = BitwiseAnd(raw, mantissaMask); + var exponent = BitwiseAnd( + ShiftRightLogical(raw, mantissaBits), + UInt(0x1F)); + var mantissaShift = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + UInt(23), + mantissaBits); + var normalBits = BitwiseOr( + ShiftLeftLogical(IAdd(exponent, UInt(112)), UInt(23)), + ShiftLeftLogical(mantissa, mantissaShift)); + var subnormal = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.FMul, + _floatType, + _module.AddInstruction( + SpirvOp.ConvertUToF, + _floatType, + mantissa), + _module.AddInstruction( + SpirvOp.Select, + _floatType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + mantissaBits, + UInt(6)), + Float(1f / 1_048_576f), // 2^-20 for 11-bit UFLOAT + Float(1f / 524_288f)))); // 2^-19 for 10-bit UFLOAT + var special = BitwiseOr( + UInt(0x7F800000), + ShiftLeftLogical(mantissa, mantissaShift)); + var result = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + exponent, + UInt(0)), + subnormal, + normalBits); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + exponent, + UInt(31)), + special, + result); + } + + private uint Gfx10FormatOne(uint numberFormat) + { + var isUint = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + numberFormat, + UInt(4)); + var isSint = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + numberFormat, + UInt(5)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.LogicalOr, + _boolType, + isUint, + isSint), + UInt(1), + UInt(0x3F800000)); + } + + private uint SelectUInt( + uint selector, + uint expected, + uint whenTrue, + uint whenFalse) => + _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + selector, + UInt(expected)), + whenTrue, + whenFalse); + + private uint LoadUnalignedBufferWord(int bindingIndex, uint byteAddress) + { + var result = UInt(0); + for (uint index = 0; index < 4; index++) + { + var address = index == 0 + ? byteAddress + : IAdd(byteAddress, UInt(index)); + var dwordAddress = ShiftRightLogical(address, UInt(2)); + var bitOffset = ShiftLeftLogical(BitwiseAnd(address, UInt(3)), UInt(3)); + var value = BitwiseAnd( + ShiftRightLogical(LoadBufferWord(bindingIndex, dwordAddress), bitOffset), + UInt(0xFF)); + result = BitwiseOr(result, ShiftLeftLogical(value, UInt(index * 8))); + } + + return result; + } + + private uint LoadSubdwordBufferValue( + int bindingIndex, + uint byteAddress, + uint previous, + uint byteCount, + bool signExtend, + bool d16, + bool d16High) + { + var width = byteCount * 8; + var raw = BitwiseAnd( + LoadUnalignedBufferWord(bindingIndex, byteAddress), + UInt(byteCount == 1 ? 0xFFu : 0xFFFFu)); + if (signExtend) + { + raw = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.BitFieldSExtract, + _intType, + Bitcast(_intType, raw), + UInt(0), + UInt(width))); + } + + if (!d16) + { + return raw; + } + + var half = BitwiseAnd(raw, UInt(0xFFFF)); + return d16High + ? BitwiseOr( + BitwiseAnd(previous, UInt(0x0000_FFFF)), + ShiftLeftLogical(half, UInt(16))) + : BitwiseOr( + BitwiseAnd(previous, UInt(0xFFFF_0000)), + half); + } + + private void StoreBufferBytes( + int bindingIndex, + uint byteAddress, + uint value, + uint byteCount, + uint sourceShift) + { + value = ShiftRightLogical(value, UInt(sourceShift)); + for (uint index = 0; index < byteCount; index++) + { + var address = index == 0 + ? byteAddress + : IAdd(byteAddress, UInt(index)); + var dwordAddress = ShiftRightLogical(address, UInt(2)); + var shift = ShiftLeftLogical(BitwiseAnd(address, UInt(3)), UInt(3)); + var oldValue = LoadBufferWord(bindingIndex, dwordAddress); + var byteMask = ShiftLeftLogical(UInt(0xFF), shift); + var sourceByte = BitwiseAnd( + ShiftRightLogical(value, UInt(index * 8)), + UInt(0xFF)); + var updated = BitwiseOr( + BitwiseAnd( + oldValue, + _module.AddInstruction(SpirvOp.Not, _uintType, byteMask)), + ShiftLeftLogical(sourceByte, shift)); + StoreBufferWord(bindingIndex, dwordAddress, updated); + } + } + + private static bool TryGetSubdwordLoadInfo( + string opcode, + out uint byteCount, + out bool signExtend, + out bool d16, + out bool d16High) + { + byteCount = opcode.Contains("byte", StringComparison.OrdinalIgnoreCase) ? 1u : 2u; + signExtend = opcode.Contains("Sbyte", StringComparison.Ordinal) || + opcode.Contains("Sshort", StringComparison.Ordinal); + d16 = opcode.Contains("D16", StringComparison.Ordinal); + d16High = opcode.EndsWith("D16Hi", StringComparison.Ordinal); + return opcode.Contains("LoadUbyte", StringComparison.Ordinal) || + opcode.Contains("LoadSbyte", StringComparison.Ordinal) || + opcode.Contains("LoadUshort", StringComparison.Ordinal) || + opcode.Contains("LoadSshort", StringComparison.Ordinal) || + opcode.Contains("LoadShortD16", StringComparison.Ordinal); + } + + private static bool TryGetSubdwordStoreInfo( + string opcode, + out uint byteCount, + out uint sourceShift) + { + byteCount = opcode.Contains("StoreByte", StringComparison.Ordinal) ? 1u : 2u; + sourceShift = opcode.EndsWith("D16Hi", StringComparison.Ordinal) ? 16u : 0u; + return opcode.Contains("StoreByte", StringComparison.Ordinal) || + opcode.Contains("StoreShort", StringComparison.Ordinal); + } + private static bool IsFormatBufferLoad(string opcode) => opcode.StartsWith("BufferLoadFormat", StringComparison.Ordinal) || opcode.StartsWith("TBufferLoadFormat", StringComparison.Ordinal); + private static bool UsesSampler(string opcode) => + opcode.StartsWith("ImageSample", StringComparison.Ordinal) || + opcode.StartsWith("ImageGather", StringComparison.Ordinal); + + private bool TryResolveDominatingBufferBinding( + uint pc, + uint scalarRegister, + uint registerCount, + out int bindingIndex) + { + if (_bufferBindingByPc.TryGetValue(pc, out bindingIndex)) + { + return true; + } + + for (var index = 0; index < _evaluation.GlobalMemoryBindings.Count; index++) + { + var binding = _evaluation.GlobalMemoryBindings[index]; + if (binding.ScalarAddress != scalarRegister) + { + continue; + } + + foreach (var candidatePc in binding.InstructionPcs) + { + if (!HasSameScalarDefinitions( + candidatePc, + pc, + scalarRegister, + registerCount)) + { + continue; + } + + bindingIndex = _globalBufferBase + index; + _bufferBindingByPc.Add(pc, bindingIndex); + return true; + } + } + + bindingIndex = -1; + return false; + } + + private bool TryResolveDominatingImageBinding( + Gen5ShaderInstruction instruction, + Gen5ImageControl control, + out int bindingIndex) + { + if (_imageBindingByPc.TryGetValue(instruction.Pc, out bindingIndex) && + bindingIndex < _imageResources.Count) + { + return true; + } + + var storage = Gen5ShaderTranslator.IsStorageImageOperation(instruction.Opcode); + for (var index = 0; index < _evaluation.ImageBindings.Count; index++) + { + var candidate = _evaluation.ImageBindings[index]; + if (candidate.Control.ScalarResource != control.ScalarResource || + candidate.Control.ScalarSampler != control.ScalarSampler || + Gen5ShaderTranslator.IsStorageImageOperation(candidate.Opcode) != storage || + !HasSameScalarDefinitions( + candidate.Pc, + instruction.Pc, + control.ScalarResource, + ImageDescriptorDwords) || + UsesSampler(instruction.Opcode) && + !HasSameScalarDefinitions( + candidate.Pc, + instruction.Pc, + control.ScalarSampler, + SamplerDescriptorDwords)) + { + continue; + } + + bindingIndex = index; + _imageBindingByPc.Add(instruction.Pc, index); + return true; + } + + bindingIndex = -1; + return false; + } + + private bool HasSameScalarDefinitions( + uint candidatePc, + uint targetPc, + uint firstRegister, + uint registerCount) + { + if (firstRegister + registerCount > ScalarRegisterCount || + !_scalarDefinitionsBeforePc.TryGetValue(candidatePc, out var candidate) || + !_scalarDefinitionsBeforePc.TryGetValue(targetPc, out var target)) + { + return false; + } + + for (var register = firstRegister; + register < firstRegister + registerCount; + register++) + { + var definition = candidate[register]; + if (definition is ConflictingScalarDefinition or + UnreachableScalarDefinition || + target[register] != definition) + { + return false; + } + } + + return true; + } + private bool TryEmitVertexInputFetch( Gen5BufferMemoryControl control, SpirvVertexInput input, @@ -1569,10 +3076,21 @@ public static partial class Gen5SpirvTranslator out string error) { error = string.Empty; - if (!_imageBindingByPc.TryGetValue(instruction.Pc, out var bindingIndex) || - bindingIndex >= _imageResources.Count) + if (!TryResolveDominatingImageBinding(instruction, image, out var bindingIndex)) { - error = "unresolved image binding"; + var candidates = _evaluation.ImageBindings + .Where(binding => + binding.Control.ScalarResource == image.ScalarResource && + binding.Control.ScalarSampler == image.ScalarSampler) + .Take(16) + .Select(binding => + $"{binding.Opcode}@0x{binding.Pc:X}" + + $"/r={HasSameScalarDefinitions(binding.Pc, instruction.Pc, image.ScalarResource, ImageDescriptorDwords)}" + + $"/s={!UsesSampler(instruction.Opcode) || HasSameScalarDefinitions(binding.Pc, instruction.Pc, image.ScalarSampler, SamplerDescriptorDwords)}"); + error = + $"unresolved image binding t=s{image.ScalarResource} " + + $"s=s{image.ScalarSampler} " + + $"candidates=[{string.Join(',', candidates)}]"; return false; } @@ -1638,7 +3156,10 @@ public static partial class Gen5SpirvTranslator { if ((image.Dmask & (1u << component)) != 0) { - var raw = LoadV(image.VectorData + sourceIndex++); + var raw = LoadImageStoreComponent( + image, + resource, + sourceIndex++); components[component] = resource.ComponentKind switch { ImageComponentKind.Sint => Bitcast(_intType, raw), @@ -1662,32 +3183,21 @@ public static partial class Gen5SpirvTranslator SpirvOp.CompositeConstruct, resource.VectorType, components); - if (TryGetImageBounds( - _evaluation.ImageBindings[bindingIndex].ResourceDescriptor, - out var width, - out var height)) - { - EmitBoundsCheckedImageWrite( - coordinates, - width, - height, - imageObject, - texel); - } - else - { - EmitExecConditional( - () => _module.AddStatement( - SpirvOp.ImageWrite, - imageObject, - coordinates, - texel)); - } + var imageSize = _module.AddInstruction( + SpirvOp.ImageQuerySize, + _module.TypeVector(_intType, 2), + imageObject); + EmitBoundsCheckedImageWrite( + coordinates, + imageSize, + imageObject, + texel); return true; } - if (resource.IsStorage) + if (resource.IsStorage && + instruction.Opcode is not ("ImageLoad" or "ImageLoadMip")) { error = $"unsupported storage image opcode {instruction.Opcode}"; return false; @@ -1697,24 +3207,46 @@ public static partial class Gen5SpirvTranslator var writeAllComponents = false; if (instruction.Opcode is "ImageLoad" or "ImageLoadMip") { - var coordinates = TryGetImageBounds( - _evaluation.ImageBindings[bindingIndex].ResourceDescriptor, - out var width, - out var height) - ? BuildClampedIntegerCoordinates(image, 0, width, height) - : BuildIntegerCoordinates(image, 0); - var mipLevel = _evaluation.ImageBindings[bindingIndex].MipLevel ?? 0; - var fetchedImage = _module.AddInstruction( - SpirvOp.Image, - resource.ImageType, - imageObject); - sampled = _module.AddInstruction( - SpirvOp.ImageFetch, - resource.VectorType, - fetchedImage, - coordinates, - 2, - UInt(mipLevel)); + if (resource.IsStorage) + { + var imageSize = _module.AddInstruction( + SpirvOp.ImageQuerySize, + _module.TypeVector(_intType, 2), + imageObject); + var coordinates = BuildClampedIntegerCoordinates( + image, + 0, + imageSize); + sampled = _module.AddInstruction( + SpirvOp.ImageRead, + resource.VectorType, + imageObject, + coordinates); + } + else + { + var mipLevel = _evaluation.ImageBindings[bindingIndex].MipLevel ?? 0; + var fetchedImage = _module.AddInstruction( + SpirvOp.Image, + resource.ImageType, + imageObject); + var imageSize = _module.AddInstruction( + SpirvOp.ImageQuerySizeLod, + _module.TypeVector(_intType, 2), + fetchedImage, + UInt(mipLevel)); + var coordinates = BuildClampedIntegerCoordinates( + image, + 0, + imageSize); + sampled = _module.AddInstruction( + SpirvOp.ImageFetch, + resource.VectorType, + fetchedImage, + coordinates, + 2, + UInt(mipLevel)); + } } else if (instruction.Opcode.StartsWith( "ImageSample", @@ -1724,22 +3256,86 @@ public static partial class Gen5SpirvTranslator instruction.Opcode.EndsWith("O", StringComparison.Ordinal); var hasCompare = instruction.Opcode.Contains("SampleC", StringComparison.Ordinal); - var start = (hasOffset ? 1 : 0) + (hasCompare ? 1 : 0); - var coordinates = BuildFloatCoordinates(image, start); - var explicitLod = - instruction.Opcode.Contains("Lz", StringComparison.Ordinal) || + var hasGradients = + instruction.Opcode.Contains("SampleD", StringComparison.Ordinal); + var hasZeroLod = + instruction.Opcode.Contains("Lz", StringComparison.Ordinal); + var hasLod = !hasZeroLod && instruction.Opcode.Contains("SampleL", StringComparison.Ordinal); - var lod = instruction.Opcode.Contains("Lz", StringComparison.Ordinal) - ? Float(0) - : Bitcast( - _floatType, - LoadV(image.GetAddressRegister(start + 2))); - var offset = hasOffset ? BuildImageOffset(image, 0) : 0u; - var imageOperands = - (explicitLod ? 2u : 0u) | (hasOffset ? 0x10u : 0u); - var reference = hasCompare - ? Bitcast(_floatType, LoadV(image.GetAddressRegister(hasOffset ? 1 : 0))) + var hasBias = + instruction.Opcode.Contains("SampleB", StringComparison.Ordinal); + + // RDNA MIMG address operands are ordered as + // {offset}{bias/lod}{z-compare}{derivatives}{body}. The old + // lowering treated SAMPLE_D as body-first and consequently + // sampled gradients as coordinates in every captured + // derivative operation. + var addressCursor = 0; + var offset = 0u; + if (hasOffset) + { + addressCursor = AlignFullImageAddress(image, addressCursor); + offset = BuildImageOffset(image, addressCursor); + addressCursor += ImageFullAddressSlots(image); + } + + // SAMPLE_B prefixes the body with a bias. SAMPLE_L instead + // carries LOD as the final body component (x, y, lod for 2D), + // per the RDNA image-address table. + var lodOrBias = hasBias + ? LoadImageFloatAddress(image, addressCursor++) : 0u; + var reference = 0u; + if (hasCompare) + { + // PCF references remain full-width even when A16 packs the + // ordinary address components two per VGPR. + addressCursor = AlignFullImageAddress(image, addressCursor); + reference = Bitcast( + _floatType, + LoadV(image.GetAddressRegister( + ImageAddressRegister(image, addressCursor)))); + addressCursor += ImageFullAddressSlots(image); + } + + var gradientX = hasGradients + ? BuildFloatCoordinates(image, addressCursor) + : 0u; + var gradientY = hasGradients + ? BuildFloatCoordinates(image, addressCursor + 2) + : 0u; + if (hasGradients) + { + addressCursor += 4; + } + + var coordinates = BuildFloatCoordinates(image, addressCursor); + var explicitLod = hasGradients || hasZeroLod || hasLod; + var lod = hasZeroLod + ? Float(0) + : hasLod + ? LoadImageFloatAddress(image, addressCursor + 2) + : lodOrBias; + if (hasOffset) + { + // Vulkan before maintenance8 forbids the dynamic Offset + // image operand on non-gather sampling operations. RDNA + // offsets are per-lane VGPR values, so ConstOffset is not + // equivalent. Fold the texel offset into normalized sample + // coordinates using the queried mip extent instead. + var offsetLod = explicitLod && !hasGradients + ? lod + : Float(0); + coordinates = ApplyDynamicSampleOffset( + resource, + imageObject, + coordinates, + offset, + offsetLod); + } + + var imageOperands = + hasGradients ? 4u : explicitLod ? 2u : hasBias ? 1u : 0u; var operands = new List { imageObject, @@ -1749,15 +3345,20 @@ public static partial class Gen5SpirvTranslator if (imageOperands != 0) { operands.Add(imageOperands); - if (explicitLod) + if (hasGradients) + { + operands.Add(gradientX); + operands.Add(gradientY); + } + else if (explicitLod) { operands.Add(lod); } - - if (hasOffset) + else if (hasBias) { - operands.Add(offset); + operands.Add(lodOrBias); } + } sampled = _module.AddInstruction( @@ -1779,12 +3380,26 @@ public static partial class Gen5SpirvTranslator instruction.Opcode.EndsWith("O", StringComparison.Ordinal); var hasCompare = instruction.Opcode.Contains("Gather4C", StringComparison.Ordinal); - var start = (hasOffset ? 1 : 0) + (hasCompare ? 1 : 0); - var coordinates = BuildFloatCoordinates(image, start); - var offset = hasOffset ? BuildImageOffset(image, 0) : 0u; - var reference = hasCompare - ? Bitcast(_floatType, LoadV(image.GetAddressRegister(hasOffset ? 1 : 0))) - : 0u; + var addressCursor = 0; + var offset = 0u; + if (hasOffset) + { + offset = BuildImageOffset(image, addressCursor); + addressCursor += ImageFullAddressSlots(image); + } + + var reference = 0u; + if (hasCompare) + { + addressCursor = AlignFullImageAddress(image, addressCursor); + reference = Bitcast( + _floatType, + LoadV(image.GetAddressRegister( + ImageAddressRegister(image, addressCursor)))); + addressCursor += ImageFullAddressSlots(image); + } + + var coordinates = BuildFloatCoordinates(image, addressCursor); var operands = new List { imageObject, @@ -1843,7 +3458,7 @@ public static partial class Gen5SpirvTranslator return false; } - uint output = 0; + var outputValues = new List(4); for (uint component = 0; component < 4; component++) { if (!writeAllComponents && @@ -1862,7 +3477,55 @@ public static partial class Gen5SpirvTranslator ImageComponentKind.Uint => value, _ => Bitcast(_uintType, value), }; - StoreV(image.VectorData + output++, raw); + outputValues.Add(raw); + } + + if (_stage == Gen5SpirvStage.Pixel && + PixelImageCaptureAddressMatches() && + uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_IMAGE_PC"), + out var captureImagePc) && + instruction.Pc == captureImagePc) + { + var captureBase = 248u; + if (uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_IMAGE_VGPR_BASE"), + out var requestedCaptureBase)) + { + captureBase = requestedCaptureBase; + } + captureBase = captureBase <= 252 ? captureBase : 248u; + for (var component = 0; component < 4; component++) + { + StoreV( + captureBase + (uint)component, + component < outputValues.Count + ? outputValues[component] + : Bitcast(_uintType, Float(1))); + } + } + + if (image.D16) + { + for (var index = 0; index < outputValues.Count; index += 2) + { + var low = outputValues[index]; + var high = index + 1 < outputValues.Count + ? outputValues[index + 1] + : UInt(0); + StoreV( + image.VectorData + (uint)(index / 2), + PackImageD16(resource, low, high)); + } + } + else + { + for (var index = 0; index < outputValues.Count; index++) + { + StoreV(image.VectorData + (uint)index, outputValues[index]); + } } return true; @@ -1931,12 +3594,8 @@ public static partial class Gen5SpirvTranslator private uint BuildFloatCoordinates(Gen5ImageControl image, int start) { - var x = Bitcast( - _floatType, - LoadV(image.GetAddressRegister(start))); - var y = Bitcast( - _floatType, - LoadV(image.GetAddressRegister(start + 1))); + var x = LoadImageFloatAddress(image, start); + var y = LoadImageFloatAddress(image, start + 1); return _module.AddInstruction( SpirvOp.CompositeConstruct, _vec2Type, @@ -1944,15 +3603,113 @@ public static partial class Gen5SpirvTranslator y); } + private static int ImageAddressRegister( + Gen5ImageControl image, + int component) => image.A16 ? component / 2 : component; + + private static int ImageFullAddressSlots(Gen5ImageControl image) => + image.A16 ? 2 : 1; + + private static int AlignFullImageAddress( + Gen5ImageControl image, + int component) => image.A16 ? (component + 1) & ~1 : component; + + private uint LoadImageFloatAddress(Gen5ImageControl image, int component) + { + var raw = LoadV(image.GetAddressRegister( + ImageAddressRegister(image, component))); + if (!image.A16) + { + return Bitcast(_floatType, raw); + } + + var unpacked = Ext(62, _vec2Type, raw); + return _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpacked, + (uint)(component & 1)); + } + + private uint LoadImageIntegerAddress(Gen5ImageControl image, int component) + { + var raw = LoadV(image.GetAddressRegister( + ImageAddressRegister(image, component))); + if (!image.A16) + { + return raw; + } + + return BitwiseAnd( + ShiftRightLogical(raw, UInt((uint)((component & 1) * 16))), + UInt(0xFFFF)); + } + + private uint LoadImageStoreComponent( + Gen5ImageControl image, + SpirvImageResource resource, + uint component) + { + if (!image.D16) + { + return LoadV(image.VectorData + component); + } + + var packed = LoadV(image.VectorData + component / 2); + if (resource.ComponentKind == ImageComponentKind.Float) + { + var unpacked = Ext(62, _vec2Type, packed); + return Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpacked, + component & 1)); + } + + var shifted = ShiftRightLogical(packed, UInt((component & 1) * 16)); + var low = BitwiseAnd(shifted, UInt(0xFFFF)); + if (resource.ComponentKind != ImageComponentKind.Sint) + { + return low; + } + + return Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.BitFieldSExtract, + _intType, + Bitcast(_intType, low), + UInt(0), + UInt(16))); + } + + private uint PackImageD16( + SpirvImageResource resource, + uint low, + uint high) + { + if (resource.ComponentKind == ImageComponentKind.Float) + { + var pair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + Bitcast(_floatType, low), + Bitcast(_floatType, high)); + return Ext(58, _uintType, pair); + } + + return BitwiseOr( + BitwiseAnd(low, UInt(0xFFFF)), + ShiftLeftLogical(BitwiseAnd(high, UInt(0xFFFF)), UInt(16))); + } + private uint BuildIntegerCoordinates(Gen5ImageControl image, int start) { var ivec2 = _module.TypeVector(_intType, 2); - var x = Bitcast( - _intType, - LoadV(image.GetAddressRegister(start))); - var y = Bitcast( - _intType, - LoadV(image.GetAddressRegister(start + 1))); + var x = Bitcast(_intType, LoadImageIntegerAddress(image, start)); + var y = Bitcast(_intType, LoadImageIntegerAddress(image, start + 1)); return _module.AddInstruction( SpirvOp.CompositeConstruct, ivec2, @@ -1963,20 +3720,27 @@ public static partial class Gen5SpirvTranslator private uint BuildClampedIntegerCoordinates( Gen5ImageControl image, int start, - uint width, - uint height) + uint imageSize) { var ivec2 = _module.TypeVector(_intType, 2); var x = ClampSignedCoordinate( Bitcast( _intType, - LoadV(image.GetAddressRegister(start))), - width); + LoadImageIntegerAddress(image, start)), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _intType, + imageSize, + 0)); var y = ClampSignedCoordinate( Bitcast( _intType, - LoadV(image.GetAddressRegister(start + 1))), - height); + LoadImageIntegerAddress(image, start + 1)), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _intType, + imageSize, + 1)); return _module.AddInstruction( SpirvOp.CompositeConstruct, ivec2, @@ -1987,7 +3751,11 @@ public static partial class Gen5SpirvTranslator private uint ClampSignedCoordinate(uint value, uint extent) { var zero = _module.Constant(_intType, 0); - var max = _module.Constant(_intType, Math.Max(extent, 1) - 1); + var max = _module.AddInstruction( + SpirvOp.ISub, + _intType, + extent, + _module.Constant(_intType, 1)); var belowZero = _module.AddInstruction( SpirvOp.SLessThan, _boolType, @@ -2014,8 +3782,7 @@ public static partial class Gen5SpirvTranslator private void EmitBoundsCheckedImageWrite( uint coordinates, - uint width, - uint height, + uint imageSize, uint imageObject, uint texel) { @@ -2029,6 +3796,16 @@ public static partial class Gen5SpirvTranslator _intType, coordinates, 1); + var width = _module.AddInstruction( + SpirvOp.CompositeExtract, + _intType, + imageSize, + 0); + var height = _module.AddInstruction( + SpirvOp.CompositeExtract, + _intType, + imageSize, + 1); var zero = _module.Constant(_intType, 0); var xNonNegative = _module.AddInstruction( SpirvOp.SGreaterThanEqual, @@ -2044,12 +3821,12 @@ public static partial class Gen5SpirvTranslator SpirvOp.SLessThan, _boolType, x, - _module.Constant(_intType, width)); + width); var yInRange = _module.AddInstruction( SpirvOp.SLessThan, _boolType, y, - _module.Constant(_intType, height)); + height); var lowerInRange = _module.AddInstruction( SpirvOp.LogicalAnd, _boolType, @@ -2088,30 +3865,13 @@ public static partial class Gen5SpirvTranslator _module.AddLabel(mergeLabel); } - private static bool TryGetImageBounds( - IReadOnlyList descriptor, - out uint width, - out uint height) - { - width = 0; - height = 0; - if (descriptor.Count < 3) - { - return false; - } - - width = (((descriptor[1] >> 30) & 0x3u) | - ((descriptor[2] & 0xFFFu) << 2)) + 1; - height = ((descriptor[2] >> 14) & 0x3FFFu) + 1; - return width != 0 && height != 0 && width <= 16384 && height <= 16384; - } - private uint BuildImageOffset(Gen5ImageControl image, int component) { var ivec2 = _module.TypeVector(_intType, 2); var packed = Bitcast( _intType, - LoadV(image.GetAddressRegister(component))); + LoadV(image.GetAddressRegister( + ImageAddressRegister(image, component)))); var x = _module.AddInstruction( SpirvOp.BitFieldSExtract, _intType, @@ -2131,6 +3891,58 @@ public static partial class Gen5SpirvTranslator y); } + private uint ApplyDynamicSampleOffset( + SpirvImageResource resource, + uint sampledImage, + uint coordinates, + uint texelOffset, + uint lod) + { + var ivec2 = _module.TypeVector(_intType, 2); + var image = _module.AddInstruction( + SpirvOp.Image, + resource.ImageType, + sampledImage); + var signedLod = _module.AddInstruction( + SpirvOp.ConvertFToS, + _intType, + lod); + var lodIsNegative = _module.AddInstruction( + SpirvOp.SLessThan, + _boolType, + signedLod, + _module.Constant(_intType, 0)); + var clampedLod = _module.AddInstruction( + SpirvOp.Select, + _intType, + lodIsNegative, + _module.Constant(_intType, 0), + signedLod); + var size = _module.AddInstruction( + SpirvOp.ImageQuerySizeLod, + ivec2, + image, + clampedLod); + var sizeFloat = _module.AddInstruction( + SpirvOp.ConvertSToF, + _vec2Type, + size); + var offsetFloat = _module.AddInstruction( + SpirvOp.ConvertSToF, + _vec2Type, + texelOffset); + var normalizedOffset = _module.AddInstruction( + SpirvOp.FDiv, + _vec2Type, + offsetFloat, + sizeFloat); + return _module.AddInstruction( + SpirvOp.FAdd, + _vec2Type, + coordinates, + normalizedOffset); + } + private bool TryEmitExport( Gen5ShaderInstruction instruction, Gen5ExportControl export, @@ -2150,6 +3962,8 @@ public static partial class Gen5SpirvTranslator return true; } + Store(_reachedPixelExport, _module.ConstantBool(true)); + var values = new uint[4]; for (var component = 0; component < 4; component++) { @@ -2202,6 +4016,105 @@ public static partial class Gen5SpirvTranslator SpirvOp.CompositeConstruct, output.Type, values); + if (output.Kind == Gen5PixelOutputKind.Float && + PixelExportVgprAddressMatches() && + uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PIXEL_EXPORT_VGPR_BASE"), + out var debugVgprBase)) + { + var registerBase = debugVgprBase + export.Target * 4; + vector = _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + Bitcast(_floatType, LoadV(registerBase)), + Bitcast(_floatType, LoadV(registerBase + 1)), + Bitcast(_floatType, LoadV(registerBase + 2)), + Bitcast(_floatType, LoadV(registerBase + 3))); + } + if (output.Kind == Gen5PixelOutputKind.Float && + PixelExportVgprAddressMatches() && + uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PIXEL_EXPORT_PACK_VGPR_BASE"), + out var debugPackVgprBase)) + { + var registerBase = debugPackVgprBase + export.Target * 4; + var lowPair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase))), + TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase + 1)))); + var highPair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase + 2))), + TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase + 3)))); + var unpackedLow = Ext(62, _vec2Type, Ext(58, _uintType, lowPair)); + var unpackedHigh = Ext(62, _vec2Type, Ext(58, _uintType, highPair)); + vector = _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedLow, + 0), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedLow, + 1), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedHigh, + 0), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedHigh, + 1)); + } + if (_forcePixelMagenta && PixelExportDebugAddressMatches()) + { + vector = output.Kind switch + { + Gen5PixelOutputKind.Float => + _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + Float(1f), + Float(0f), + Float(1f), + Float(1f)), + Gen5PixelOutputKind.Sint => + _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + Bitcast(_intType, UInt(1)), + Bitcast(_intType, UInt(0)), + Bitcast(_intType, UInt(1)), + Bitcast(_intType, UInt(1))), + _ => + _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + UInt(1), + UInt(0), + UInt(1), + UInt(1)), + }; + } + if (Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_EXPORT_EXEC") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + Store(_exec, _module.ConstantBool(true)); + StoreS64( + 126, + _module.Constant64(_ulongType, 1)); + } vector = _module.AddInstruction( SpirvOp.Select, output.Type, @@ -2253,6 +4166,19 @@ public static partial class Gen5SpirvTranslator SpirvOp.CompositeConstruct, _vec4Type, components); + if (_state.Program.Address == 0x0000000500780000ul && + export.Target is >= 32 and < 36 && + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_VERTEX_OUTPUTS_ONE") == "1") + { + outputValue = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec4Type, + Float(1f), + Float(1f), + Float(1f), + Float(1f)); + } outputValue = _module.AddInstruction( SpirvOp.Select, _vec4Type, @@ -2263,10 +4189,288 @@ public static partial class Gen5SpirvTranslator return true; } + private bool PixelExportDebugAddressMatches() + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PIXEL_EXPORT_ADDRESS"); + if (string.IsNullOrWhiteSpace(addressFilter)) + { + return true; + } + + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) && + _state.Program.Address == address; + } + + private bool PixelImageCaptureAddressMatches() + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_IMAGE_ADDRESS"); + if (string.IsNullOrWhiteSpace(addressFilter)) + { + return false; + } + + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) && + _state.Program.Address == address; + } + + private void CapturePixelVgprs(Gen5ShaderInstruction instruction) + { + if (_stage != Gen5SpirvStage.Pixel || + !PixelVgprCaptureAddressMatches() || + !uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_PC"), + out var capturePc) || + instruction.Pc != capturePc) + { + return; + } + + var sourceText = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_SOURCES"); + if (string.IsNullOrWhiteSpace(sourceText)) + { + return; + } + + var destinationBase = 248u; + if (uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_DEST_BASE"), + out var requestedDestinationBase)) + { + destinationBase = requestedDestinationBase; + } + + var sources = sourceText.Split( + ',', + StringSplitOptions.RemoveEmptyEntries | + StringSplitOptions.TrimEntries); + if (sources.Length is 0 or > 4 || + destinationBase > 252 || + destinationBase + (uint)sources.Length > 256) + { + return; + } + + for (var index = 0; index < sources.Length; index++) + { + if (!uint.TryParse(sources[index], out var source) || + source >= 256) + { + return; + } + } + + for (var index = 0; index < sources.Length; index++) + { + _ = uint.TryParse(sources[index], out var source); + StoreV( + destinationBase + (uint)index, + LoadV(source), + guardWithExec: + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_IGNORE_EXEC") != "1"); + } + } + + private bool PixelVgprCaptureAddressMatches() + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_ADDRESS"); + if (string.IsNullOrWhiteSpace(addressFilter)) + { + return false; + } + + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) && + _state.Program.Address == address; + } + + private void CapturePixelVgprPoints(Gen5ShaderInstruction instruction) + { + if (_stage != Gen5SpirvStage.Pixel || + !PixelVgprCaptureAddressMatches()) + { + return; + } + + var captureText = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_POINTS"); + if (string.IsNullOrWhiteSpace(captureText)) + { + return; + } + + foreach (var capture in captureText.Split( + ',', + StringSplitOptions.RemoveEmptyEntries | + StringSplitOptions.TrimEntries)) + { + var fields = capture.Split(':'); + if (fields.Length != 3 || + !uint.TryParse(fields[0], out var pc) || + !uint.TryParse(fields[1], out var source) || + !uint.TryParse(fields[2], out var destination) || + pc != instruction.Pc || source >= 256 || destination >= 256) + { + continue; + } + + StoreV( + destination, + LoadV(source), + guardWithExec: + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_IGNORE_EXEC") != "1"); + } + } + + private void MarkPixelPath(Gen5ShaderInstruction instruction) + { + if (_stage != Gen5SpirvStage.Pixel || + !PixelVgprCaptureAddressMatches()) + { + return; + } + + var markerText = Environment.GetEnvironmentVariable( + "SHARPEMU_MARK_PIXEL_PCS"); + if (string.IsNullOrWhiteSpace(markerText)) + { + return; + } + + foreach (var marker in markerText.Split( + ',', + StringSplitOptions.RemoveEmptyEntries | + StringSplitOptions.TrimEntries)) + { + var separator = marker.IndexOf(':'); + if (separator <= 0 || separator == marker.Length - 1 || + !uint.TryParse(marker.AsSpan(0, separator), out var pc) || + !uint.TryParse(marker.AsSpan(separator + 1), out var register) || + pc != instruction.Pc || register >= 256) + { + continue; + } + + StoreV( + register, + Bitcast(_uintType, Float(1)), + guardWithExec: false); + } + } + + private void CapturePixelExec(Gen5ShaderInstruction instruction) + { + if (_stage != Gen5SpirvStage.Pixel || + !PixelVgprCaptureAddressMatches()) + { + return; + } + + var captureText = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_EXEC_PCS"); + if (string.IsNullOrWhiteSpace(captureText)) + { + return; + } + + foreach (var capture in captureText.Split( + ',', + StringSplitOptions.RemoveEmptyEntries | + StringSplitOptions.TrimEntries)) + { + var separator = capture.IndexOf(':'); + if (separator <= 0 || separator == capture.Length - 1 || + !uint.TryParse(capture.AsSpan(0, separator), out var pc) || + !uint.TryParse(capture.AsSpan(separator + 1), out var register) || + pc != instruction.Pc || register >= 256) + { + continue; + } + + var value = _module.AddInstruction( + SpirvOp.Select, + _floatType, + Load(_boolType, _exec), + Float(1), + Float(0)); + StoreV( + register, + Bitcast(_uintType, value), + guardWithExec: false); + } + } + + private bool PixelExportVgprAddressMatches() + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PIXEL_EXPORT_VGPR_ADDRESS"); + if (string.IsNullOrWhiteSpace(addressFilter)) + { + return PixelExportDebugAddressMatches(); + } + + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) && + _state.Program.Address == address; + } + private uint LoadCompressedExportComponent( Gen5ShaderInstruction instruction, int component) { + if (TryLoadPackedHalfExportComponent( + instruction, + component, + out var shadowValue)) + { + return shadowValue; + } + var packed = LoadV(instruction.Sources[component >> 1].Value); var unpacked = Ext(62, _vec2Type, packed); return _module.AddInstruction( @@ -2276,6 +4480,132 @@ public static partial class Gen5SpirvTranslator (uint)(component & 1)); } + private bool TryLoadPackedHalfExportComponent( + Gen5ShaderInstruction exportInstruction, + int component, + out uint value) + { + value = 0; + var packedSource = exportInstruction.Sources[component >> 1]; + var tracePackedExport = + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_PACKED_EXPORT") == "1" && + _state.Program.Address == 0x0000000500781200ul; + if (tracePackedExport) + { + Console.Error.WriteLine( + $"[AGC][PACKED-EXPORT] exp_pc=0x{exportInstruction.Pc:X} " + + $"component={component} source={packedSource.Kind}:" + + $"{packedSource.Value}"); + if (component == 0 && exportInstruction.Pc == 0x630) + { + foreach (var decoded in _state.Program.Instructions.Where( + static decoded => decoded.Pc <= 0x640)) + { + Console.Error.WriteLine( + $"[AGC][TITLE-IR] 0x{decoded.Pc:X4} " + + $"{decoded.Opcode} dst=[" + + string.Join(',', decoded.Destinations) + + "] src=[" + + string.Join(',', decoded.Sources) + "] words=[" + + string.Join(',', decoded.Words.Select(static word => $"{word:X8}")) + + "] ctrl=" + decoded.Control); + } + } + } + if (packedSource.Kind != Gen5OperandKind.VectorRegister) + { + if (tracePackedExport) + { + Console.Error.WriteLine( + "[AGC][PACKED-EXPORT] rejected: source is not a VGPR"); + } + return false; + } + + for (var index = _state.Program.Instructions.Count - 1; index >= 0; index--) + { + var candidate = _state.Program.Instructions[index]; + if (candidate.Pc >= exportInstruction.Pc) + { + continue; + } + + if (exportInstruction.Pc - candidate.Pc > 128) + { + break; + } + + if (!candidate.Destinations.Any(destination => + destination.Kind == Gen5OperandKind.VectorRegister && + destination.Value == packedSource.Value)) + { + continue; + } + + if (tracePackedExport) + { + Console.Error.WriteLine( + $"[AGC][PACKED-EXPORT] nearest_pc=0x{candidate.Pc:X} " + + $"opcode={candidate.Opcode} distance=" + + $"{exportInstruction.Pc - candidate.Pc}"); + } + + if (candidate.Opcode != "VCvtPkrtzF16F32" || + candidate.Sources.Count < 2) + { + if (tracePackedExport) + { + Console.Error.WriteLine( + "[AGC][PACKED-EXPORT] rejected: nearest writer is " + + candidate.Opcode); + } + return false; + } + + var packedPointer = PackedHalfPointer(packedSource.Value); + if (Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PACKED_EXPORT_STORE_ONE") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + Store( + packedPointer, + _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + Float(1f), + Float(1f))); + } + + var packedPair = Load( + _vec2Type, + packedPointer); + value = _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + packedPair, + (uint)(component & 1)); + if (Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PACKED_EXPORT_ONE") == "1") + { + value = Float(1f); + } + if (tracePackedExport) + { + Console.Error.WriteLine( + "[AGC][PACKED-EXPORT] selected shadow pair"); + } + return true; + } + + if (tracePackedExport) + { + Console.Error.WriteLine( + "[AGC][PACKED-EXPORT] rejected: no nearby writer"); + } + return false; + } + private uint GetPixelOutputType(Gen5PixelOutputKind kind) => kind switch { @@ -2286,14 +4616,87 @@ public static partial class Gen5SpirvTranslator private uint LoadBufferWord(int binding, uint dwordAddress) { - var pointer = BufferWordPointer(binding, dwordAddress); - return Load(_uintType, pointer); + var inRange = IsBufferWordInRange(binding, dwordAddress); + var safeAddress = _module.AddInstruction( + SpirvOp.Select, + _uintType, + inRange, + dwordAddress, + UInt(0)); + var value = Load(_uintType, BufferWordPointer(binding, safeAddress)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + inRange, + value, + UInt(0)); + } + + private uint ApplyGuestBufferByteBias(int binding, uint byteAddress) + { + var evaluationBinding = binding - _globalBufferBase; + if ((uint)evaluationBinding >= + (uint)_evaluation.GlobalMemoryBindings.Count) + { + // Runtime SGPR blocks and other synthetic descriptors do not + // alias guest virtual memory and are always bound at offset 0. + return byteAddress; + } + + if (_initialScalarBufferIndex >= 0) + { + // Descriptor offsets must satisfy Vulkan's storage-buffer + // alignment. The presenter therefore rounds the shared guest + // allocation offset down and packs the discarded low address + // bits after the 256 initial SGPRs in the per-dispatch runtime + // block. Keeping this value runtime-stable prevents rotating + // guest allocations from producing a new multi-megabyte SPIR-V + // module and Metal pipeline while preserving exact byte access. + var runtimeByteBias = Load( + _uintType, + RuntimeBufferBiasPointer(binding)); + return IAdd(byteAddress, runtimeByteBias); + } + + // The presenter binds the shared allocation at the largest aligned + // offset not greater than this guest resource's offset. Because the + // allocation base is aligned to the same power of two, the bytes + // discarded from the descriptor offset are exactly the low address + // bits below. Adding them here keeps scalar, MUBUF and GLOBAL paths + // byte-exact, including atomics and resources that overlap another + // descriptor at an unaligned guest address. + var byteBias = + _evaluation.GlobalMemoryBindings[evaluationBinding].BaseAddress & + (_storageBufferOffsetAlignment - 1); + return byteBias == 0 + ? byteAddress + : IAdd(byteAddress, UInt(checked((uint)byteBias))); } private void StoreBufferWord(int binding, uint dwordAddress, uint value) { - var pointer = BufferWordPointer(binding, dwordAddress); - Store(pointer, value); + EmitConditional( + IsBufferWordInRange(binding, dwordAddress), + () => Store(BufferWordPointer(binding, dwordAddress), value)); + } + + private uint IsBufferWordInRange(int binding, uint dwordAddress) + { + var buffer = _module.AddInstruction( + SpirvOp.AccessChain, + _storageBlockPointer, + _globalBuffers, + UInt((uint)binding)); + var length = _module.AddInstruction( + SpirvOp.ArrayLength, + _uintType, + buffer, + 0); + return _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + dwordAddress, + length); } private uint BufferWordPointer(int binding, uint dwordAddress) => @@ -2312,6 +4715,13 @@ public static partial class Gen5SpirvTranslator _scalarRegisters, UInt(register)); + private uint RuntimeBufferBiasPointer(int binding) => + _module.AddInstruction( + SpirvOp.AccessChain, + _privateUintPointer, + _runtimeBufferBiases, + UInt(checked((uint)binding))); + private uint VectorPointer(uint register) => _module.AddInstruction( SpirvOp.AccessChain, @@ -2319,6 +4729,13 @@ public static partial class Gen5SpirvTranslator _vectorRegisters, UInt(register)); + private uint PackedHalfPointer(uint register) => + _module.AddInstruction( + SpirvOp.AccessChain, + _privateVec2Pointer, + _packedHalfRegisters, + UInt(register)); + private uint LoadS(uint register) => Load(_uintType, ScalarPointer(register)); private uint LoadV(uint register) => Load(_uintType, VectorPointer(register)); @@ -2353,8 +4770,52 @@ public static partial class Gen5SpirvTranslator Store(VectorPointer(register), value); } - private uint Load(uint type, uint pointer) => - _module.AddInstruction(SpirvOp.Load, type, pointer); + private void StorePackedHalf(uint register, uint value) + { + var active = Load(_boolType, _exec); + if (Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PACKED_STORE_EXEC_VALUES") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + var activePair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + Float(1f), + Float(1f)); + var inactivePair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + Float(0.5f), + Float(0.5f)); + value = _module.AddInstruction( + SpirvOp.Select, + _vec2Type, + active, + activePair, + inactivePair); + Store(PackedHalfPointer(register), value); + return; + } + + value = _module.AddInstruction( + SpirvOp.Select, + _vec2Type, + active, + value, + Load(_vec2Type, PackedHalfPointer(register))); + Store(PackedHalfPointer(register), value); + } + + private uint Load(uint type, uint pointer) + { + if (pointer == 0) + { + throw new InvalidOperationException( + "SPIR-V generator attempted OpLoad from id 0."); + } + + return _module.AddInstruction(SpirvOp.Load, type, pointer); + } private void Store(uint pointer, uint value) => _module.AddStatement(SpirvOp.Store, pointer, value); @@ -2412,6 +4873,9 @@ public static partial class Gen5SpirvTranslator private uint BitwiseAnd64(uint left, uint right) => _module.AddInstruction(SpirvOp.BitwiseAnd, _ulongType, left, right); + private uint BitwiseOr64(uint left, uint right) => + _module.AddInstruction(SpirvOp.BitwiseOr, _ulongType, left, right); + private uint BitwiseOr(uint left, uint right) => _module.AddInstruction(SpirvOp.BitwiseOr, _uintType, left, right); @@ -2424,12 +4888,35 @@ public static partial class Gen5SpirvTranslator private uint SubgroupAny(uint condition) => _subgroupInvocationIdInput == 0 ? condition + : _emulateWave64 + ? IsNotZero64(BooleanToWaveMask(condition)) : _module.AddInstruction( SpirvOp.GroupNonUniformAny, _boolType, UInt(3), condition); + private uint GuestWaveLane() + { + if (_waveLaneCount == 64 && _localInvocationIndexInput != 0) + { + return BitwiseAnd( + Load(_uintType, _localInvocationIndexInput), + UInt(63)); + } + + if (_subgroupInvocationIdInput != 0) + { + return BitwiseAnd( + Load(_uintType, _subgroupInvocationIdInput), + UInt(31)); + } + + // Graphics stages without subgroup support have one logical lane; + // they must not emit OpLoad for absent SPIR-V input ID zero. + return UInt(0); + } + private uint CurrentLaneBit() { if (_subgroupInvocationIdInput == 0) @@ -2437,8 +4924,7 @@ public static partial class Gen5SpirvTranslator return _module.Constant64(_ulongType, 1); } - var lane = Load(_uintType, _subgroupInvocationIdInput); - var maskedLane = BitwiseAnd(lane, UInt(RdnaWaveLaneCount - 1)); + var maskedLane = GuestWaveLane(); var shifted = ShiftLeftLogical64( _module.Constant64(_ulongType, 1), _module.AddInstruction( @@ -2458,7 +4944,7 @@ public static partial class Gen5SpirvTranslator SpirvOp.ULessThan, _boolType, Load(_uintType, _subgroupInvocationIdInput), - UInt(RdnaWaveLaneCount)); + UInt(32)); private uint BooleanToLaneMask(uint condition) => _module.AddInstruction( @@ -2468,6 +4954,105 @@ public static partial class Gen5SpirvTranslator CurrentLaneBit(), _module.Constant64(_ulongType, 0)); + private uint BooleanToWaveMask(uint condition) + { + if (_subgroupInvocationIdInput == 0) + { + return BooleanToLaneMask(condition); + } + + var ballot = _module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + _uvec4Type, + UInt(3), + condition); + var low = _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + ballot, + 0); + if (_emulateWave64) + { + var subgroupLane = + Load(_uintType, _subgroupInvocationIdInput); + var firstLane = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + subgroupLane, + UInt(0)); + var half = ShiftRightLogical(GuestWaveLane(), UInt(5)); + EmitConditional(firstLane, () => + { + Store(WaveMaskScratchPointer(half), low); + }); + EmitWave64Barrier(); + var lowMask = Load( + _uintType, + WaveMaskScratchPointer(UInt(0))); + var highMask = Load( + _uintType, + WaveMaskScratchPointer(UInt(1))); + var combined = BitwiseOr64( + _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + lowMask), + ShiftLeftLogical64( + _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + highMask), + _module.Constant64(_ulongType, 32))); + EmitWave64Barrier(); + return combined; + } + + var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low); + if (_waveLaneCount != 64) + { + return widened; + } + + return _module.AddInstruction( + SpirvOp.Select, + _ulongType, + _module.AddInstruction( + SpirvOp.UGreaterThanEqual, + _boolType, + GuestWaveLane(), + UInt(32)), + ShiftLeftLogical64( + widened, + _module.Constant64(_ulongType, 32)), + widened); + } + + private uint WaveMaskScratchPointer(uint index) => + _module.AddInstruction( + SpirvOp.AccessChain, + _waveMaskScratchElementPointer, + _waveScratchInLds ? _lds : _waveMaskScratch, + _waveScratchInLds ? IAdd(UInt(LdsDwordCount - 3), index) : index); + + private uint WaveBroadcastScratchPointer() => + _waveScratchInLds + ? _module.AddInstruction( + SpirvOp.AccessChain, + _ldsElementPointer, + _lds, + UInt(LdsDwordCount - 1)) + : _waveBroadcastScratch; + + private void EmitWave64Barrier() + { + var workgroup = UInt(2); + _module.AddStatement( + SpirvOp.ControlBarrier, + workgroup, + workgroup, + UInt(0x108)); + } + private uint IsWaveMaskActive(uint mask) => _subgroupInvocationIdInput == 0 ? IsNotZero64(mask) @@ -2482,17 +5067,22 @@ public static partial class Gen5SpirvTranslator CurrentLaneBit())); private void StoreWaveMask(uint register, uint condition) => - StoreS64(register, BooleanToLaneMask(condition)); + StoreS64(register, BooleanToWaveMask(condition)); private void EmitExecConditional(Action emit) + { + var active = Load(_boolType, _exec); + EmitConditional(active, emit); + } + + private void EmitConditional(uint condition, Action emit) { var activeLabel = _module.AllocateId(); var mergeLabel = _module.AllocateId(); - var active = Load(_boolType, _exec); _module.AddStatement(SpirvOp.SelectionMerge, mergeLabel, 0); _module.AddStatement( SpirvOp.BranchConditional, - active, + condition, activeLabel, mergeLabel); _module.AddLabel(activeLabel); @@ -2507,8 +5097,13 @@ public static partial class Gen5SpirvTranslator private bool UsesSubgroupShuffle() => _state.Program.Instructions.Any(instruction => + instruction.Control is Gen5DppControl or Gen5Dpp8Control || instruction.Opcode is "VPermlane16B32" or "VPermlanex16B32" or "VReadlaneB32"); + private bool UsesSubgroupBroadcast() => + _state.Program.Instructions.Any(instruction => + instruction.Opcode == "VReadfirstlaneB32"); + private bool UsesWaveControl() => _state.Program.Instructions.Any(instruction => instruction.Opcode.Contains("Saveexec", StringComparison.Ordinal) || @@ -2520,7 +5115,11 @@ public static partial class Gen5SpirvTranslator private bool UsesSubgroupOperations() => _stage == Gen5SpirvStage.Compute && - (UsesSubgroupShuffle() || UsesWaveControl()); + (UsesSubgroupShuffle() || + UsesSubgroupBroadcast() || + UsesWaveControl() || + _state.Program.Instructions.Any(static instruction => + instruction.Opcode is "VMbcntLoU32B32" or "VMbcntHiU32B32")); private static bool IsWaveMaskOperand(Gen5Operand operand) => operand.Kind == Gen5OperandKind.ScalarRegister && @@ -2613,6 +5212,198 @@ public static partial class Gen5SpirvTranslator return blocks; } + private void BuildScalarDefinitionInfo( + IReadOnlyList blocks, + IReadOnlyList instructions) + { + var predecessors = new HashSet[blocks.Count]; + for (var index = 0; index < blocks.Count; index++) + { + predecessors[index] = []; + } + + void AddEdge(int source, int destination) + { + if (destination < 0 || destination >= blocks.Count) + { + return; + } + + predecessors[destination].Add(source); + } + + for (var blockIndex = 0; blockIndex < blocks.Count; blockIndex++) + { + var block = blocks[blockIndex]; + var terminator = instructions[block.EndIndex - 1]; + var hasFallthrough = blockIndex + 1 < blocks.Count; + if (terminator.Opcode == "SEndpgm") + { + continue; + } + + if (terminator.Opcode == "SBranch") + { + if (TryGetBranchTargetPc(terminator, out var targetPc) && + TryFindBlock(blocks, targetPc, out var targetBlock)) + { + AddEdge(blockIndex, targetBlock); + } + + continue; + } + + if (terminator.Opcode.StartsWith("SCbranch", StringComparison.Ordinal)) + { + if (TryGetBranchTargetPc(terminator, out var targetPc) && + TryFindBlock(blocks, targetPc, out var targetBlock)) + { + AddEdge(blockIndex, targetBlock); + } + + if (hasFallthrough) + { + AddEdge(blockIndex, blockIndex + 1); + } + + continue; + } + + if (hasFallthrough) + { + AddEdge(blockIndex, blockIndex + 1); + } + } + + var blockInputs = new long[blocks.Count][]; + var blockOutputs = new long[blocks.Count][]; + var hasOutput = new bool[blocks.Count]; + var initialDefinitions = Enumerable.Repeat( + InitialScalarDefinition, + ScalarRegisterCount).ToArray(); + + static void MergeDefinitions( + long[] destination, + long[] source, + ref bool hasInput) + { + if (!hasInput) + { + Array.Copy(source, destination, ScalarRegisterCount); + hasInput = true; + return; + } + + for (var register = 0; register < ScalarRegisterCount; register++) + { + if (destination[register] != source[register]) + { + destination[register] = ConflictingScalarDefinition; + } + } + } + + static void ApplyScalarDefinitions( + long[] definitions, + ShaderBlock block, + IReadOnlyList blockInstructions) + { + for (var instructionIndex = block.StartIndex; + instructionIndex < block.EndIndex; + instructionIndex++) + { + var instruction = blockInstructions[instructionIndex]; + foreach (var destination in instruction.Destinations) + { + if (destination.Kind == Gen5OperandKind.ScalarRegister && + destination.Value < ScalarRegisterCount) + { + definitions[destination.Value] = instruction.Pc + 1L; + } + } + } + } + + var changed = true; + while (changed) + { + changed = false; + for (var blockIndex = 0; blockIndex < blocks.Count; blockIndex++) + { + var input = Enumerable.Repeat( + UnreachableScalarDefinition, + ScalarRegisterCount).ToArray(); + var hasInput = false; + if (blockIndex == 0) + { + MergeDefinitions(input, initialDefinitions, ref hasInput); + } + + foreach (var predecessor in predecessors[blockIndex]) + { + if (hasOutput[predecessor]) + { + MergeDefinitions( + input, + blockOutputs[predecessor], + ref hasInput); + } + } + + if (!hasInput) + { + continue; + } + + var output = (long[])input.Clone(); + ApplyScalarDefinitions(output, blocks[blockIndex], instructions); + if (!hasOutput[blockIndex] || + !blockInputs[blockIndex].AsSpan().SequenceEqual(input) || + !blockOutputs[blockIndex].AsSpan().SequenceEqual(output)) + { + blockInputs[blockIndex] = input; + blockOutputs[blockIndex] = output; + hasOutput[blockIndex] = true; + changed = true; + } + } + } + + _scalarDefinitionsBeforePc.Clear(); + for (var blockIndex = 0; blockIndex < blocks.Count; blockIndex++) + { + if (!hasOutput[blockIndex]) + { + continue; + } + + var definitions = (long[])blockInputs[blockIndex].Clone(); + var block = blocks[blockIndex]; + for (var instructionIndex = block.StartIndex; + instructionIndex < block.EndIndex; + instructionIndex++) + { + var instruction = instructions[instructionIndex]; + if (instruction.Control is Gen5ImageControl or + Gen5ScalarMemoryControl or + Gen5GlobalMemoryControl or + Gen5BufferMemoryControl) + { + _scalarDefinitionsBeforePc[instruction.Pc] = + (long[])definitions.Clone(); + } + foreach (var destination in instruction.Destinations) + { + if (destination.Kind == Gen5OperandKind.ScalarRegister && + destination.Value < ScalarRegisterCount) + { + definitions[destination.Value] = instruction.Pc + 1L; + } + } + } + } + } + private static int FindInstructionIndex( IReadOnlyList instructions, uint pc) diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvFixedShaders.cs b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvFixedShaders.cs index 7626886..479a27e 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvFixedShaders.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvFixedShaders.cs @@ -164,4 +164,100 @@ public static class SpirvFixedShaders module.AddExecutionMode(main, SpirvExecutionMode.OriginUpperLeft); return module.Build(); } + + public static byte[] CreateSolidFragment(float red, float green, float blue, float alpha) + { + var module = new SpirvModuleBuilder(); + module.AddCapability(SpirvCapability.Shader); + + var voidType = module.TypeVoid(); + var floatType = module.TypeFloat(32); + var vec4Type = module.TypeVector(floatType, 4); + var outputVec4Pointer = module.TypePointer(SpirvStorageClass.Output, vec4Type); + var output = module.AddGlobalVariable(outputVec4Pointer, SpirvStorageClass.Output); + module.AddName(output, "outColor"); + module.AddDecoration(output, SpirvDecoration.Location, 0); + + var functionType = module.TypeFunction(voidType); + var main = module.BeginFunction(voidType, functionType); + module.AddName(main, "main"); + module.AddLabel(); + var color = module.ConstantComposite( + vec4Type, + module.ConstantFloat(floatType, red), + module.ConstantFloat(floatType, green), + module.ConstantFloat(floatType, blue), + module.ConstantFloat(floatType, alpha)); + module.AddStatement(SpirvOp.Store, output, color); + module.AddStatement(SpirvOp.Return); + module.EndFunction(); + + module.AddEntryPoint(SpirvExecutionModel.Fragment, main, "main", [output]); + module.AddExecutionMode(main, SpirvExecutionMode.OriginUpperLeft); + return module.Build(); + } + + /// + /// Diagnostic fragment stage that exposes one interpolated vertex output + /// directly as color. This keeps the real guest vertex/index/depth path + /// intact while isolating fragment-shader translation from interface data. + /// + public static byte[] CreateAttributeFragment(uint location) + { + var module = new SpirvModuleBuilder(); + module.AddCapability(SpirvCapability.Shader); + + var voidType = module.TypeVoid(); + var floatType = module.TypeFloat(32); + var vec4Type = module.TypeVector(floatType, 4); + var inputPointer = module.TypePointer(SpirvStorageClass.Input, vec4Type); + var outputPointer = module.TypePointer(SpirvStorageClass.Output, vec4Type); + var input = module.AddGlobalVariable(inputPointer, SpirvStorageClass.Input); + module.AddName(input, $"attr{location}"); + module.AddDecoration(input, SpirvDecoration.Location, location); + var output = module.AddGlobalVariable(outputPointer, SpirvStorageClass.Output); + module.AddName(output, "outColor"); + module.AddDecoration(output, SpirvDecoration.Location, 0); + + var functionType = module.TypeFunction(voidType); + var main = module.BeginFunction(voidType, functionType); + module.AddName(main, "main"); + module.AddLabel(); + var value = module.AddInstruction(SpirvOp.Load, vec4Type, input); + module.AddStatement(SpirvOp.Store, output, value); + module.AddStatement(SpirvOp.Return); + module.EndFunction(); + + module.AddEntryPoint( + SpirvExecutionModel.Fragment, + main, + "main", + [input, output]); + module.AddExecutionMode(main, SpirvExecutionMode.OriginUpperLeft); + return module.Build(); + } + + /// + /// Minimal fragment stage for fixed-function depth-only passes. The + /// guest has no pixel shader and therefore cannot export colour; keeping + /// this stage output-free preserves that contract while allowing Vulkan + /// to run early/late depth tests for the translated vertex shader. + /// + public static byte[] CreateDepthOnlyFragment() + { + var module = new SpirvModuleBuilder(); + module.AddCapability(SpirvCapability.Shader); + + var voidType = module.TypeVoid(); + var functionType = module.TypeFunction(voidType); + var main = module.BeginFunction(voidType, functionType); + module.AddName(main, "main"); + module.AddLabel(); + module.AddStatement(SpirvOp.Return); + module.EndFunction(); + + module.AddEntryPoint(SpirvExecutionModel.Fragment, main, "main", []); + module.AddExecutionMode(main, SpirvExecutionMode.OriginUpperLeft); + return module.Build(); + } } diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs index bbb7b0b..6295010 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs @@ -143,6 +143,7 @@ public enum SpirvOp : ushort ControlBarrier = 224, MemoryBarrier = 225, AtomicIAdd = 234, + AtomicUMax = 239, Phi = 245, LoopMerge = 246, SelectionMerge = 247, diff --git a/src/SharpEmu.ShaderCompiler/Gen5ShaderIr.cs b/src/SharpEmu.ShaderCompiler/Gen5ShaderIr.cs index ba0db21..01734ed 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5ShaderIr.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5ShaderIr.cs @@ -175,7 +175,9 @@ public sealed record Gen5ImageControl( uint Dimension, bool IsArray, bool Glc, - bool Slc) : Gen5InstructionControl + bool Slc, + bool A16, + bool D16) : Gen5InstructionControl { public uint GetAddressRegister(int component) => component < AddressRegisters.Count @@ -219,16 +221,21 @@ public sealed record Gen5Vop3Control( uint NegateMask, uint OutputModifier, bool Clamp, + uint OperandSelect, uint? ScalarDestination) : Gen5InstructionControl; public sealed record Gen5SdwaControl( uint DestinationSelect, + uint DestinationUnused, uint Source0Select, uint Source1Select, + bool Source0SignExtend, + bool Source1SignExtend, uint AbsoluteMask, uint NegateMask, uint OutputModifier, - bool Clamp) : Gen5InstructionControl; + bool Clamp, + uint? ScalarDestination) : Gen5InstructionControl; public sealed record Gen5DppControl( uint Control, @@ -239,6 +246,10 @@ public sealed record Gen5DppControl( uint BankMask, uint RowMask) : Gen5InstructionControl; +public sealed record Gen5Dpp8Control( + uint LaneSelectors, + bool FetchInactive) : Gen5InstructionControl; + public sealed record Gen5ScalarMemoryControl( uint DestinationCount, int ImmediateOffsetBytes, @@ -257,11 +268,27 @@ public sealed record Gen5ImageBinding( IReadOnlyList SamplerDescriptor, uint? MipLevel); +// Data arrays may be rented from ArrayPool (oversized): always slice with +// DataLength, never Data.Length. Ownership transfers to the presenter, which +// returns pooled arrays after uploading them into host-visible buffers. public sealed record Gen5GlobalMemoryBinding( uint ScalarAddress, ulong BaseAddress, IReadOnlyList InstructionPcs, - byte[] Data); + byte[] Data, + int DataLength, + bool DataPooled) +{ + public bool Writable { get; set; } + + // Writable describes shader access and is also used to decide whether a + // compute dispatch has observable work. A statically reachable resource + // can nevertheless be unbound for the current scalar path; the evaluator + // supplies zero-filled storage for Vulkan in that case. Such synthetic + // storage must remain shader-writable, but must never be copied to the + // descriptor's unmapped guest address. + public bool WriteBackToGuest { get; set; } = true; +} public sealed record Gen5VertexInputBinding( uint Pc, @@ -272,12 +299,13 @@ public sealed record Gen5VertexInputBinding( ulong BaseAddress, uint Stride, uint OffsetBytes, - byte[] Data); + byte[] Data, + int DataLength, + bool DataPooled); public sealed record Gen5ShaderEvaluation( IReadOnlyList InitialScalarRegisters, IReadOnlyList ScalarRegisters, - IReadOnlyDictionary> ScalarRegistersByPc, IReadOnlyList ImageBindings, IReadOnlyList GlobalMemoryBindings, Gen5ComputeSystemRegisters? ComputeSystemRegisters = null, @@ -297,8 +325,58 @@ public sealed record Gen5ShaderProgram( ulong Address, IReadOnlyList Instructions) { + private const int ScalarRegisterCount = 256; + private IReadOnlySet? _runtimeScalarRegisters; + public IEnumerable ImageResources => Instructions .Select(instruction => instruction.Control) .OfType(); + + /// + /// The set of scalar registers the program reads or writes as runtime + /// values. It depends only on the (cached) decoded program, so it is + /// computed once and shared read-only across every draw that uses this + /// shader — the evaluator previously rebuilt this HashSet by scanning + /// every instruction on every draw, one of the largest per-draw + /// allocation and CPU sources. + /// + public IReadOnlySet RuntimeScalarRegisters => + _runtimeScalarRegisters ??= ComputeRuntimeScalarRegisters(); + + private IReadOnlySet ComputeRuntimeScalarRegisters() + { + var registers = new HashSet(); + foreach (var instruction in Instructions) + { + foreach (var operand in instruction.Sources) + { + if (operand.Kind == Gen5OperandKind.ScalarRegister && + operand.Value < ScalarRegisterCount) + { + registers.Add(operand.Value); + } + } + + foreach (var operand in instruction.Destinations) + { + if (operand.Kind == Gen5OperandKind.ScalarRegister && + operand.Value < ScalarRegisterCount) + { + registers.Add(operand.Value); + } + } + + if (instruction.Control is Gen5ScalarMemoryControl + { + DynamicOffsetRegister: { } offsetRegister, + } && + offsetRegister < ScalarRegisterCount) + { + registers.Add(offsetRegister); + } + } + + return registers; + } } diff --git a/src/SharpEmu.ShaderCompiler/Gen5ShaderMetadataReader.cs b/src/SharpEmu.ShaderCompiler/Gen5ShaderMetadataReader.cs index 9880f66..0cdbacc 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5ShaderMetadataReader.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5ShaderMetadataReader.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; +using System.Buffers.Binary; namespace SharpEmu.ShaderCompiler; @@ -17,9 +18,9 @@ public static class Gen5ShaderMetadataReader out Gen5ShaderMetadata metadata) { metadata = default!; - if (!ctx.TryReadUInt64(shaderHeaderAddress + ShaderUserDataOffset, out var userDataAddress) || + if (!TryReadUInt64(ctx, shaderHeaderAddress + ShaderUserDataOffset, out var userDataAddress) || userDataAddress == 0 || - !ctx.TryReadUInt64(userDataAddress, out var directResourceOffsetsAddress)) + !TryReadUInt64(ctx, userDataAddress, out var directResourceOffsetsAddress)) { return false; } @@ -27,7 +28,8 @@ public static class Gen5ShaderMetadataReader var resourceOffsets = new ulong[ResourceClassCount]; for (var resourceClass = 0; resourceClass < ResourceClassCount; resourceClass++) { - if (!ctx.TryReadUInt64( + if (!TryReadUInt64( + ctx, userDataAddress + 0x08 + (ulong)(resourceClass * sizeof(ulong)), out resourceOffsets[resourceClass])) { @@ -35,9 +37,9 @@ public static class Gen5ShaderMetadataReader } } - if (!ctx.TryReadUInt16(userDataAddress + 0x28, out var extendedUserDataSize) || - !ctx.TryReadUInt16(userDataAddress + 0x2A, out var shaderResourceTableSize) || - !ctx.TryReadUInt16(userDataAddress + 0x2C, out var directResourceCount) || + if (!TryReadUInt16(ctx, userDataAddress + 0x28, out var extendedUserDataSize) || + !TryReadUInt16(ctx, userDataAddress + 0x2A, out var shaderResourceTableSize) || + !TryReadUInt16(ctx, userDataAddress + 0x2C, out var directResourceCount) || directResourceCount > MaxMetadataEntries) { return false; @@ -46,7 +48,8 @@ public static class Gen5ShaderMetadataReader var resourceCounts = new ushort[ResourceClassCount]; for (var resourceClass = 0; resourceClass < ResourceClassCount; resourceClass++) { - if (!ctx.TryReadUInt16( + if (!TryReadUInt16( + ctx, userDataAddress + 0x2E + (ulong)(resourceClass * sizeof(ushort)), out resourceCounts[resourceClass]) || resourceCounts[resourceClass] > MaxMetadataEntries) @@ -65,7 +68,7 @@ public static class Gen5ShaderMetadataReader for (uint type = 0; type < directResourceCount; type++) { - if (!ctx.TryReadUInt16(directResourceOffsetsAddress + type * sizeof(ushort), out var offset)) + if (!TryReadUInt16(ctx, directResourceOffsetsAddress + type * sizeof(ushort), out var offset)) { return false; } @@ -93,7 +96,8 @@ public static class Gen5ShaderMetadataReader for (uint slot = 0; slot < count; slot++) { - if (!ctx.TryReadUInt16( + if (!TryReadUInt16( + ctx, resourceOffsets[resourceClass] + slot * sizeof(ushort), out var sharp)) { @@ -121,4 +125,30 @@ public static class Gen5ShaderMetadataReader resources); return true; } + + private static bool TryReadUInt16(CpuContext ctx, ulong address, out ushort value) + { + Span bytes = stackalloc byte[sizeof(ushort)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt16LittleEndian(bytes); + return true; + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) + { + Span bytes = stackalloc byte[sizeof(ulong)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(bytes); + return true; + } } diff --git a/src/SharpEmu.ShaderCompiler/Gen5ShaderScalarEvaluator.cs b/src/SharpEmu.ShaderCompiler/Gen5ShaderScalarEvaluator.cs index 520fa3a..0dc2d25 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5ShaderScalarEvaluator.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5ShaderScalarEvaluator.cs @@ -2,12 +2,52 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; +using System.Buffers; +using System.Buffers.Binary; +using System.Diagnostics; using System.Numerics; namespace SharpEmu.ShaderCompiler; public static class Gen5ShaderScalarEvaluator { + // When a scalar POINTER load can't be resolved statically (its descriptor + // register read back garbage — e.g. 0 or 0xFFFFFFFF, a per-draw descriptor + // setup race), abort-and-drop-the-draw loses the whole pass. Demon's Souls' + // deferred-lighting / composite pixel shaders hit this intermittently, so + // the passes that would produce the composite's feeder targets get dropped + // and the frame stays black. Degrading instead (feed 0, keep translating, + // like the buffer-load path already does) lets the pass render with the + // unresolved resource missing rather than not at all. STRICT reverts. + private static readonly bool _strictScalarLoad = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_STRICT_SCALAR_LOAD"), + "1", + StringComparison.Ordinal); + + // A stale buffer descriptor should not discard an otherwise valid shader + // pass. Treat it as an all-zero buffer by default; callers that need + // strict diagnostics can restore the old failure behaviour explicitly. + private static readonly bool _strictBufferLoad = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_STRICT_BUFFER_LOAD"), + "1", + StringComparison.Ordinal); + private static readonly object _scalarFallbackTraceGate = new(); + private static readonly HashSet<(ulong Shader, uint Pc)> _tracedScalarFallbacks = []; + + // Uniform forward branches select material/resource bodies that remain + // statically present in the translated shader. Discover the skipped body's + // descriptors by default; SHARPEMU_CFG_RESOURCE_DISCOVERY=0 is a diagnostic + // opt-out. Conditional branches are deliberately not forked because their + // fall-through is already scanned and forking vector-mask conditions grows + // exponentially without adding descriptor coverage. + private static readonly bool _cfgResourceDiscovery = + !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_CFG_RESOURCE_DISCOVERY"), + "0", + StringComparison.Ordinal); + /// /// Optional fallback for global-memory reads that ctx.Memory cannot satisfy (the /// emulator installs the HLE-tracked libc heap reader here at module load). Kept as @@ -17,17 +57,16 @@ public static class Gen5ShaderScalarEvaluator public delegate bool Gen5FallbackMemoryReader(ulong baseAddress, Span destination); + /// + /// Pool used for large draw-time guest-memory snapshots. The HLE host installs + /// its bounded transfer pool; standalone compiler tools use the shared pool. + /// + public static ArrayPool GlobalMemoryPool { get; set; } = ArrayPool.Shared; + private const int ScalarRegisterCount = 256; private const int ImageDescriptorDwords = 8; private const int SamplerDescriptorDwords = 4; private const int MaxGlobalMemoryBindingBytes = 16 * 1024 * 1024; - private static readonly int DefaultGlobalMemoryBindingBytes = - int.TryParse( - Environment.GetEnvironmentVariable("SHARPEMU_GLOBAL_BINDING_BYTES"), - out var configured) && configured >= sizeof(uint) - ? Math.Min(configured, MaxGlobalMemoryBindingBytes) - : 1 * 1024 * 1024; - public static long GlobalMemoryReadCount; public static long GlobalMemoryReadBytes; public static long GlobalMemoryReadCacheHits; @@ -35,12 +74,13 @@ public static class Gen5ShaderScalarEvaluator public static long GlobalMemoryReadLibcBytes; public static long GlobalMemoryReadReuses; - private const long CrossFrameReadCacheMaxBytes = 1024L * 1024 * 1024; - private static readonly object _crossFrameReadGate = new(); - private static readonly Dictionary<(ulong BaseAddress, int SizeBytes), byte[]> _crossFrameReadCache = new(); - private static long _crossFrameReadCacheBytes; private const ulong RdnaWaveMask = 0xFFFF_FFFFUL; + static Gen5ShaderScalarEvaluator() + { + RunScalarLoadSelfChecks(); + } + private readonly record struct BufferDescriptor( ulong BaseAddress, uint Stride, @@ -49,6 +89,31 @@ public static class Gen5ShaderScalarEvaluator uint NumberFormat, uint DataFormat); + private readonly record struct ScalarPathState( + uint StartPc, + uint[] ScalarRegisters, + ulong ExecMask, + bool ScalarConditionCode, + bool Supplemental); + + private readonly record struct ScalarPathKey(uint StartPc, ulong StateHash); + + private static ulong ComputeScalarStateHash( + IReadOnlyList registers, + ulong execMask, + bool scalarConditionCode) + { + const ulong prime = 1099511628211UL; + var hash = 14695981039346656037UL; + foreach (var value in registers) + { + hash = (hash ^ value) * prime; + } + + hash = (hash ^ execMask) * prime; + return (hash ^ (scalarConditionCode ? 1UL : 0UL)) * prime; + } + public static bool TryResolveImageBindings( CpuContext ctx, Gen5ShaderState state, @@ -71,7 +136,7 @@ public static class Gen5ShaderScalarEvaluator out Gen5ShaderEvaluation evaluation, out string error, bool resolveVertexInputs = false, - uint? vertexRecordLimit = null) + uint? requiredVertexRecordCount = null) { evaluation = default!; error = string.Empty; @@ -99,40 +164,114 @@ public static class Gen5ShaderScalarEvaluator var globalMemoryBindings = new List(); var globalMemoryByAddress = new Dictionary<(uint ScalarAddress, ulong BaseAddress), Gen5GlobalMemoryBinding>(); var vertexInputBindings = new List(); - var runtimeScalarRegisters = CollectRuntimeScalarRegisters(state.Program); - var scalarRegisterSnapshots = new Dictionary>(); - var scalarConditionCode = false; - uint? skipUntilPc = null; + // Shared, cached, read-only: computed once per decoded program. The + // set already includes every instruction's destination registers, so + // the per-load additions the loop used to make are redundant. + var runtimeScalarRegisters = state.Program.RuntimeScalarRegisters; + var resolvedImageByPc = new Dictionary(); + var finalScalarRegisters = (uint[])scalarRegisters.Clone(); + var pendingPaths = new Stack(); + var visitedPaths = new HashSet(); - foreach (var instruction in state.Program.Instructions) + void QueuePath( + uint pc, + uint[] registers, + ulong pathExecMask, + bool pathScc, + bool supplemental) { - if (skipUntilPc.HasValue) + var key = new ScalarPathKey( + pc, + ComputeScalarStateHash(registers, pathExecMask, pathScc)); + if (visitedPaths.Add(key)) { - if (instruction.Pc < skipUntilPc.Value) + pendingPaths.Push(new ScalarPathState( + pc, + registers, + pathExecMask, + pathScc, + supplemental)); + } + } + + if (state.Program.Instructions.Count != 0) + { + QueuePath( + state.Program.Instructions[0].Pc, + (uint[])scalarRegisters.Clone(), + execMask, + pathScc: false, + supplemental: false); + } + + while (pendingPaths.Count != 0) + { + var path = pendingPaths.Pop(); + scalarRegisters = path.ScalarRegisters; + execMask = path.ExecMask; + var scalarConditionCode = path.ScalarConditionCode; + uint? skipUntilPc = path.StartPc; + + foreach (var instruction in state.Program.Instructions) + { + if (skipUntilPc.HasValue) { - continue; + if (instruction.Pc < skipUntilPc.Value) + { + continue; + } + + skipUntilPc = null; } - skipUntilPc = null; - } + if (instruction.Opcode == "SEndpgm") + { + break; + } - scalarRegisterSnapshots[instruction.Pc] = (uint[])scalarRegisters.Clone(); + if (instruction.Opcode == "SBranch" && + TryGetSoppBranchTargetPc(instruction, out var targetPc)) + { + if (targetPc > instruction.Pc) + { + // The regular scalar evaluation follows the uniform + // branch. Evaluate its skipped fall-through region once + // as supplemental resource discovery: large shaders use + // forward S_BRANCH to select one material/resource body, + // and SPIR-V still needs descriptors for every body that + // remains in the statically translated CFG. Do not fork + // SC_BRANCH targets here; their fall-through regions are + // already visited linearly and forking every vector-mask + // condition causes exponential state growth. + if (_cfgResourceDiscovery) + { + var fallthroughPc = instruction.Pc + + (uint)(instruction.Words.Count * sizeof(uint)); + QueuePath( + fallthroughPc, + (uint[])scalarRegisters.Clone(), + execMask, + scalarConditionCode, + supplemental: true); + } - if (instruction.Opcode == "SEndpgm") - { - break; - } + skipUntilPc = targetPc; + continue; + } - if (instruction.Opcode == "SBranch" && - TryGetSoppBranchTargetPc(instruction, out var targetPc) && - targetPc > instruction.Pc) - { - skipUntilPc = targetPc; - continue; - } + // One linear pass over a supplemental body is sufficient + // to discover its static memory/image instructions. Do not + // iterate backward branches: loop-varying descriptors need + // runtime descriptor indexing and cannot be represented by + // cloning scalar states for an unbounded number of trips. + if (path.Supplemental) + { + break; + } + } - if (instruction.Encoding == Gen5ShaderEncoding.Sopc) - { + if (instruction.Encoding == Gen5ShaderEncoding.Sopc) + { if (!TryExecuteScalarCompare( instruction, scalarRegisters, @@ -145,7 +284,7 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Encoding == Gen5ShaderEncoding.Sopk && + if (instruction.Encoding == Gen5ShaderEncoding.Sopk && instruction.Opcode.StartsWith("SCmpk", StringComparison.Ordinal)) { if (!TryExecuteScalarCompareK( @@ -160,7 +299,7 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Encoding is + if (instruction.Encoding is Gen5ShaderEncoding.Sop1 or Gen5ShaderEncoding.Sop2 or Gen5ShaderEncoding.Sopk) @@ -184,25 +323,29 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Control is Gen5ScalarMemoryControl scalarMemory) - { - foreach (var destination in instruction.Destinations) + if (instruction.Control is Gen5ScalarMemoryControl scalarMemory) { - if (destination.Kind == Gen5OperandKind.ScalarRegister && destination.Value < ScalarRegisterCount) - { - runtimeScalarRegisters.Add(destination.Value); - } - } - - if (!TryExecuteScalarLoad(ctx, state, instruction, scalarMemory, scalarRegisters, globalMemoryBindings, globalMemoryByAddress, runtimeScalarRegisters, out error)) + // Destinations are already in the cached runtimeScalarRegisters + // set (it scans every instruction's destinations), so no + // per-load mutation is needed here. + var recordBinding = + !path.Supplemental || + !HasGlobalMemoryBindingForPc(globalMemoryBindings, instruction.Pc); + if (!TryExecuteScalarLoad(ctx, state, instruction, scalarMemory, scalarRegisters, globalMemoryBindings, globalMemoryByAddress, runtimeScalarRegisters, recordBinding, out error)) { return false; } continue; } - if (instruction.Control is Gen5GlobalMemoryControl globalMemory) - { + if (instruction.Control is Gen5GlobalMemoryControl globalMemory) + { + if (path.Supplemental && + HasGlobalMemoryBindingForPc(globalMemoryBindings, instruction.Pc)) + { + continue; + } + if (globalMemory.ScalarAddress >= ScalarRegisterCount - 1) { error = @@ -221,17 +364,24 @@ public static class Gen5ShaderScalarEvaluator } var key = (globalMemory.ScalarAddress, baseAddress); + var writable = instruction.Opcode.StartsWith( + "GlobalStore", + StringComparison.Ordinal) || + instruction.Opcode.StartsWith( + "GlobalAtomic", + StringComparison.Ordinal); if (globalMemoryByAddress.TryGetValue(key, out var existingBinding)) { - if (existingBinding.InstructionPcs is List instructionPcs) + if (existingBinding.InstructionPcs is List instructionPcs && + !instructionPcs.Contains(instruction.Pc)) { instructionPcs.Add(instruction.Pc); } + existingBinding.Writable |= writable; } else { - byte[] data; - if (!TryReadGlobalMemory(ctx, baseAddress, out data)) + if (!TryReadGlobalMemory(ctx, baseAddress, out var data, out var dataLength)) { error = $"global-memory-read-failed pc=0x{instruction.Pc:X} " + @@ -243,7 +393,10 @@ public static class Gen5ShaderScalarEvaluator globalMemory.ScalarAddress, baseAddress, new List { instruction.Pc }, - data); + data, + dataLength, + DataPooled: true); + binding.Writable = writable; globalMemoryByAddress.Add(key, binding); globalMemoryBindings.Add(binding); } @@ -251,8 +404,15 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Control is Gen5BufferMemoryControl bufferMemory) - { + if (instruction.Control is Gen5BufferMemoryControl bufferMemory) + { + if (path.Supplemental && + HasGlobalMemoryBindingForPc(globalMemoryBindings, instruction.Pc)) + { + continue; + } + + var writable = IsBufferMemoryWrite(instruction.Opcode); if (bufferMemory.ScalarResource >= ScalarRegisterCount - 3) { error = @@ -275,52 +435,89 @@ public static class Gen5ShaderScalarEvaluator if (bufferDescriptor.BaseAddress == 0) { - error = $"buffer-address-null pc=0x{instruction.Pc:X}"; - return false; + // A descriptor in a sibling block can be null for this + // invocation even though the GPU branch never executes the + // memory instruction. Vulkan still requires a descriptor + // for the statically present block, so bind a bounded zero + // buffer to this exact PC. It is never reused for another + // resource register or instruction. + var nullKey = (bufferMemory.ScalarResource, 0UL); + if (globalMemoryByAddress.TryGetValue(nullKey, out var nullBinding)) + { + nullBinding.Writable |= writable; + if (nullBinding.InstructionPcs is List nullInstructionPcs && + !nullInstructionPcs.Contains(instruction.Pc)) + { + nullInstructionPcs.Add(instruction.Pc); + } + } + else + { + var binding = new Gen5GlobalMemoryBinding( + bufferMemory.ScalarResource, + 0, + new List { instruction.Pc }, + new byte[sizeof(uint)], + sizeof(uint), + DataPooled: false) + { + Writable = writable, + }; + globalMemoryByAddress.Add(nullKey, binding); + globalMemoryBindings.Add(binding); + } + + continue; } if (resolveVertexInputs && IsVertexFetchCandidate(instruction, bufferMemory, bufferDescriptor)) { - var vertexReadSize = bufferDescriptor.SizeBytes; - if (vertexRecordLimit is { } recordLimit && - instruction.Sources.Count > 2 && - TryEvaluateScalarOperand( + if (instruction.Sources.Count <= 2 || + !TryEvaluateScalarOperand( instruction.Sources[2], scalarRegisters, out var scalarOffset)) { - var bindingOffset = unchecked((uint)bufferMemory.OffsetBytes + scalarOffset); - var requiredBytes = - (ulong)bindingOffset + - (ulong)(Math.Max(recordLimit, 1u) - 1u) * bufferDescriptor.Stride + - (ulong)bufferMemory.DwordCount * sizeof(uint); - vertexReadSize = Math.Min( - bufferDescriptor.SizeBytes, - Math.Max(requiredBytes, sizeof(uint))); + error = + $"vertex-input-offset-unresolved pc=0x{instruction.Pc:X} " + + $"s{bufferMemory.ScalarResource}"; + return false; } - if (!TryReadGlobalMemory( - ctx, - bufferDescriptor.BaseAddress, - vertexReadSize, - out var vertexData)) + var bindingOffset = unchecked( + (uint)bufferMemory.OffsetBytes + scalarOffset); + var vertexReadBytes = bufferDescriptor.SizeBytes; + if (requiredVertexRecordCount is > 0) { - error = - $"vertex-buffer-read-failed pc=0x{instruction.Pc:X} " + - $"address=0x{bufferDescriptor.BaseAddress:X16} " + - $"bytes={bufferDescriptor.SizeBytes} " + - $"stride={bufferDescriptor.Stride} records={bufferDescriptor.NumRecords}"; - return false; + // Resource descriptors commonly span an entire UE vertex + // arena (several MiB), while one draw references only a + // few hundred records. Preserve the indexed draw's exact + // reachable byte range instead of snapshotting the full + // arena for every attribute of every draw. + var lastRecordOffset = + (ulong)(requiredVertexRecordCount.Value - 1) * + bufferDescriptor.Stride; + var elementBytes = + (ulong)Math.Max(bufferMemory.DwordCount, 1u) * sizeof(uint); + var recordSpan = Math.Max( + (ulong)bufferDescriptor.Stride, + SaturatingAdd(bindingOffset, elementBytes)); + var requiredBytes = SaturatingAdd( + lastRecordOffset, + recordSpan); + vertexReadBytes = Math.Min(vertexReadBytes, requiredBytes); } if (!TryCreateVertexInputBinding( instruction, bufferMemory, bufferDescriptor, - vertexData, + checked((int)Math.Min( + vertexReadBytes, + (ulong)MaxGlobalMemoryBindingBytes)), (uint)vertexInputBindings.Count, - scalarRegisters, + scalarOffset, out var vertexInputBinding)) { error = @@ -336,37 +533,62 @@ public static class Gen5ShaderScalarEvaluator var key = (bufferMemory.ScalarResource, bufferDescriptor.BaseAddress); if (globalMemoryByAddress.TryGetValue(key, out var existingBinding)) { - if (existingBinding.InstructionPcs is List instructionPcs) + existingBinding.Writable |= writable; + if (existingBinding.InstructionPcs is List instructionPcs && + !instructionPcs.Contains(instruction.Pc)) { instructionPcs.Add(instruction.Pc); } } else { + var dataPooled = true; if (!TryReadGlobalMemory( ctx, bufferDescriptor.BaseAddress, bufferDescriptor.SizeBytes, - out var data)) + out var data, + out var dataLength)) { var descriptorWords = string.Join( ':', Enumerable.Range(0, 4).Select(index => $"{scalarRegisters[bufferMemory.ScalarResource + (uint)index]:X8}")); - error = - $"buffer-memory-read-failed pc=0x{instruction.Pc:X} " + - $"address=0x{bufferDescriptor.BaseAddress:X16} " + - $"bytes={bufferDescriptor.SizeBytes} " + - $"stride={bufferDescriptor.Stride} records={bufferDescriptor.NumRecords} " + - $"s{bufferMemory.ScalarResource}=[{descriptorWords}]"; - return false; + if (_strictBufferLoad) + { + error = + $"buffer-memory-read-failed pc=0x{instruction.Pc:X} " + + $"address=0x{bufferDescriptor.BaseAddress:X16} " + + $"bytes={bufferDescriptor.SizeBytes} " + + $"stride={bufferDescriptor.Stride} records={bufferDescriptor.NumRecords} " + + $"s{bufferMemory.ScalarResource}=[{descriptorWords}]"; + return false; + } + + dataLength = checked((int)Math.Min( + bufferDescriptor.SizeBytes, + (ulong)MaxGlobalMemoryBindingBytes)); + data = new byte[Math.Max(dataLength, sizeof(uint))]; + dataLength = data.Length; + dataPooled = false; + Console.Error.WriteLine( + $"[LOADER][WARN] AGC buffer read unavailable; using zero buffer " + + $"pc=0x{instruction.Pc:X} address=0x{bufferDescriptor.BaseAddress:X16} " + + $"bytes={bufferDescriptor.SizeBytes} guest_writeback=disabled " + + $"s{bufferMemory.ScalarResource}=[{descriptorWords}]"); } var binding = new Gen5GlobalMemoryBinding( bufferMemory.ScalarResource, bufferDescriptor.BaseAddress, new List { instruction.Pc }, - data); + data, + dataLength, + DataPooled: dataPooled) + { + Writable = writable, + WriteBackToGuest = dataPooled, + }; globalMemoryByAddress.Add(key, binding); globalMemoryBindings.Add(binding); } @@ -374,7 +596,12 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Control is not Gen5ImageControl image) + if (instruction.Control is not Gen5ImageControl image) + { + continue; + } + + if (path.Supplemental && resolvedImageByPc.ContainsKey(instruction.Pc)) { continue; } @@ -401,26 +628,69 @@ public static class Gen5ShaderScalarEvaluator return false; } - resolved.Add(new Gen5ImageBinding( - instruction.Pc, - instruction.Opcode, - image, - resourceDescriptor, - samplerDescriptor, - instruction.Opcode is "ImageLoadMip" or "ImageStoreMip" && - TryResolveVectorConstantBefore( - state.Program, + var imageBinding = new Gen5ImageBinding( instruction.Pc, - image.GetAddressRegister(2), - out var mipLevel) - ? mipLevel - : null)); + instruction.Opcode, + image, + resourceDescriptor, + samplerDescriptor, + instruction.Opcode is "ImageLoadMip" or "ImageStoreMip" && + TryResolveImageMipLevel( + state.Program, + instruction.Pc, + image, + out var mipLevel) + ? mipLevel + : null); + if (resolvedImageByPc.TryGetValue(instruction.Pc, out var existingIndex)) + { + var existing = resolved[existingIndex]; + var existingNull = existing.ResourceDescriptor.All(static word => word == 0); + var candidateNull = resourceDescriptor.All(static word => word == 0); + if (existingNull && !candidateNull) + { + resolved[existingIndex] = imageBinding; + } + else if (!candidateNull && + (!existing.ResourceDescriptor.SequenceEqual(resourceDescriptor) || + !existing.SamplerDescriptor.SequenceEqual(samplerDescriptor))) + { + error = + $"dynamic-image-descriptor pc=0x{instruction.Pc:X} " + + $"s{image.ScalarResource}/s{image.ScalarSampler}"; + return false; + } + } + else + { + resolvedImageByPc.Add(instruction.Pc, resolved.Count); + resolved.Add(imageBinding); + } + } + + if (!path.Supplemental) + { + finalScalarRegisters = (uint[])scalarRegisters.Clone(); + } + } + + if (vertexInputBindings.Count != 0) + { + if (!TryCaptureVertexInputData( + ctx, + vertexInputBindings, + out var capturedVertexInputs, + out error)) + { + return false; + } + + vertexInputBindings = capturedVertexInputs; } evaluation = new Gen5ShaderEvaluation( initialScalarRegisters, - scalarRegisters, - scalarRegisterSnapshots, + finalScalarRegisters, resolved, globalMemoryBindings, state.ComputeSystemRegisters, @@ -429,24 +699,32 @@ public static class Gen5ShaderScalarEvaluator return true; } + private static bool IsBufferMemoryWrite(string opcode) => + opcode.StartsWith("BufferStore", StringComparison.Ordinal) || + opcode.StartsWith("TBufferStore", StringComparison.Ordinal) || + opcode.StartsWith("BufferAtomic", StringComparison.Ordinal) || + opcode.StartsWith("TBufferAtomic", StringComparison.Ordinal); + + private static bool HasGlobalMemoryBindingForPc( + IReadOnlyList bindings, + uint pc) => + bindings.Any(binding => binding.InstructionPcs.Contains(pc)); + private static bool TryCreateVertexInputBinding( Gen5ShaderInstruction instruction, Gen5BufferMemoryControl control, BufferDescriptor descriptor, - byte[] data, + int desiredDataLength, uint location, - uint[] scalarRegisters, + uint scalarOffset, out Gen5VertexInputBinding binding) { binding = default!; - if (!IsVertexFetchCandidate(instruction, control, descriptor) || - instruction.Sources.Count <= 2 || - !TryEvaluateScalarOperand(instruction.Sources[2], scalarRegisters, out var scalarOffset)) + if (!IsVertexFetchCandidate(instruction, control, descriptor)) { return false; } - var bindingData = data; var bindingStride = descriptor.Stride; var bindingOffset = unchecked((uint)control.OffsetBytes + scalarOffset); var bindingDataFormat = descriptor.DataFormat; @@ -460,10 +738,138 @@ public static class Gen5ShaderScalarEvaluator descriptor.BaseAddress, bindingStride, bindingOffset, - bindingData); + Data: [], + DataLength: desiredDataLength, + DataPooled: false); return true; } + private static bool TryCaptureVertexInputData( + CpuContext ctx, + IReadOnlyList pending, + out List captured, + out string error) + { + captured = new List(pending.Count); + error = string.Empty; + var ordered = pending + .OrderBy(static binding => binding.Stride) + .ThenBy(static binding => binding.BaseAddress) + .ToArray(); + + for (var first = 0; first < ordered.Length;) + { + var stride = ordered[first].Stride; + var start = ordered[first].BaseAddress; + var end = SaturatingAdd(start, (ulong)ordered[first].DataLength); + var last = first + 1; + while (last < ordered.Length && ordered[last].Stride == stride) + { + var candidate = ordered[last]; + // Attribute descriptors for an interleaved stream commonly + // point a few bytes into the same record. Merge overlapping + // spans (and one-record adjacency) so all attributes share one + // captured array and one host vertex buffer. + if (candidate.BaseAddress > SaturatingAdd(end, stride)) + { + break; + } + + end = Math.Max( + end, + SaturatingAdd(candidate.BaseAddress, (ulong)candidate.DataLength)); + last++; + } + + var byteCount = end > start + ? Math.Min(end - start, (ulong)MaxGlobalMemoryBindingBytes) + : 0; + if (byteCount == 0 || + !TryReadGlobalMemory(ctx, start, byteCount, out var data, out var dataLength)) + { + foreach (var binding in captured) + { + if (binding.DataPooled) + { + GlobalMemoryPool.Return(binding.Data); + } + } + + error = + $"vertex-buffer-read-failed address=0x{start:X16} " + + $"bytes={byteCount} stride={stride}"; + captured.Clear(); + return false; + } + + for (var index = first; index < last; index++) + { + var binding = ordered[index]; + var delta = binding.BaseAddress - start; + captured.Add(binding with + { + BaseAddress = start, + OffsetBytes = checked((uint)(delta + binding.OffsetBytes)), + Data = data, + DataLength = dataLength, + DataPooled = index == first, + }); + } + + first = last; + } + + captured.Sort(static (left, right) => left.Location.CompareTo(right.Location)); + TraceTitleVertexInputs(captured); + return true; + } + + private static void TraceTitleVertexInputs(IReadOnlyList bindings) + { + if (!string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_VERTEX_RAW"), + "1", + StringComparison.Ordinal) || + bindings.Count != 3 || + !bindings.Any(static binding => + binding.Pc == 0xF8 && binding.Stride == 16 && + binding.OffsetBytes == 12 && binding.DataFormat == 10) || + !bindings.Any(static binding => + binding.Pc == 0x204 && binding.Stride == 16 && + binding.OffsetBytes == 0 && binding.DataFormat == 12) || + !bindings.Any(static binding => + binding.Pc == 0x280 && binding.Stride == 16 && + binding.OffsetBytes == 8 && binding.DataFormat == 5)) + { + return; + } + + var shared = bindings[0]; + var records = new List(); + foreach (var vertexIndex in new uint[] { 0, 1, 2, 3, 131, 4095 }) + { + var recordOffset = (ulong)vertexIndex * shared.Stride; + if (recordOffset + shared.Stride > (ulong)shared.DataLength) + { + continue; + } + + records.Add( + $"{vertexIndex}:" + + Convert.ToHexString( + shared.Data, + checked((int)recordOffset), + checked((int)shared.Stride))); + } + + Console.Error.WriteLine( + $"[VERTEX-RAW] title base=0x{shared.BaseAddress:X16} " + + $"length={shared.DataLength} records={string.Join(',', records)}"); + } + + private static ulong SaturatingAdd(ulong left, ulong right) => + ulong.MaxValue - left < right ? ulong.MaxValue : left + right; + private static bool IsVertexFetchCandidate( Gen5ShaderInstruction instruction, Gen5BufferMemoryControl control, @@ -476,33 +882,6 @@ public static class Gen5ShaderScalarEvaluator (instruction.Opcode.StartsWith("BufferLoadFormat", StringComparison.Ordinal) || instruction.Opcode.StartsWith("TBufferLoadFormat", StringComparison.Ordinal)); - private static HashSet CollectRuntimeScalarRegisters(Gen5ShaderProgram program) - { - var registers = new HashSet(); - foreach (var instruction in program.Instructions) - { - foreach (var operand in instruction.Sources.Concat(instruction.Destinations)) - { - if (operand.Kind == Gen5OperandKind.ScalarRegister && - operand.Value < ScalarRegisterCount) - { - registers.Add(operand.Value); - } - } - - if (instruction.Control is Gen5ScalarMemoryControl - { - DynamicOffsetRegister: { } offsetRegister, - } && - offsetRegister < ScalarRegisterCount) - { - registers.Add(offsetRegister); - } - } - - return registers; - } - private static bool TryGetSoppBranchTargetPc( Gen5ShaderInstruction instruction, out uint targetPc) @@ -561,6 +940,26 @@ public static class Gen5ShaderScalarEvaluator return false; } + private static bool TryResolveImageMipLevel( + Gen5ShaderProgram program, + uint pc, + Gen5ImageControl image, + out uint mipLevel) + { + // For the currently translated 2D image path, mip follows x/y. A16 + // packs x/y into the first VGPR and places mip in the low half of the + // next; ordinary addresses use the third VGPR. + var register = image.GetAddressRegister(image.A16 ? 1 : 2); + if (!TryResolveVectorConstantBefore(program, pc, register, out var raw)) + { + mipLevel = 0; + return false; + } + + mipLevel = image.A16 ? raw & 0xFFFF : raw; + return true; + } + private static bool TryResolveConstantOperand(Gen5Operand operand, out uint value) { if (operand.Kind == Gen5OperandKind.LiteralConstant) @@ -578,118 +977,110 @@ public static class Gen5ShaderScalarEvaluator return false; } - [ThreadStatic] - private static Dictionary<(ulong BaseAddress, int SizeBytes), byte[]>? _globalMemoryReadCache; - + // Both readers rent from ArrayPool: these run per bound buffer per draw, + // and fresh multi-megabyte allocations here kept the background GC busy + // full-time. The rented array (possibly oversized) is handed to the + // presenter, which returns it to the pool after the host-buffer upload. public static void BeginGlobalMemoryReadScope() { - _globalMemoryReadCache = new Dictionary<(ulong, int), byte[]>(); } public static void EndGlobalMemoryReadScope() { - _globalMemoryReadCache = null; } - private static bool TryReadGlobalMemory( CpuContext ctx, ulong baseAddress, - out byte[] data) + out byte[] data, + out int dataLength) { - return TryReadGlobalMemory( - ctx, - baseAddress, - (ulong)DefaultGlobalMemoryBindingBytes, - out data); + var rented = GlobalMemoryPool.Rent(MaxGlobalMemoryBindingBytes); + for (var size = MaxGlobalMemoryBindingBytes; size >= 4096; size >>= 1) + { + if (ctx.Memory.TryRead(baseAddress, rented.AsSpan(0, size))) + { + Interlocked.Increment(ref GlobalMemoryReadCount); + Interlocked.Add(ref GlobalMemoryReadBytes, size); + Interlocked.Add(ref GlobalMemoryReadPvmBytes, size); + data = rented; + dataLength = size; + return true; + } + } + + GlobalMemoryPool.Return(rented); + data = []; + dataLength = 0; + return false; } private static bool TryReadGlobalMemory( CpuContext ctx, ulong baseAddress, ulong sizeBytes, - out byte[] data) + out byte[] data, + out int dataLength) { + data = []; + dataLength = 0; if (sizeBytes == 0) { - data = []; return false; } var cappedSize = Math.Min(sizeBytes, MaxGlobalMemoryBindingBytes); if (cappedSize > int.MaxValue) { - data = []; return false; } - var cache = _globalMemoryReadCache; - var cacheKey = (baseAddress, (int)cappedSize); - if (cache is not null && cache.TryGetValue(cacheKey, out var cached)) + var rented = GlobalMemoryPool.Rent( + Math.Max((int)cappedSize, sizeof(uint))); + if (cappedSize < sizeof(uint)) { - Interlocked.Increment(ref GlobalMemoryReadCacheHits); - data = cached; - return true; - } - - byte[]? previous; - lock (_crossFrameReadGate) - { - _crossFrameReadCache.TryGetValue(cacheKey, out previous); - } - - if (previous is not null && ctx.Memory.TryCompare(baseAddress, previous)) - { - Interlocked.Increment(ref GlobalMemoryReadReuses); - if (cache is not null) + rented.AsSpan(0, sizeof(uint)).Clear(); + var exact = rented.AsSpan(0, (int)cappedSize); + var readFromPvm = ctx.Memory.TryRead(baseAddress, exact); + if (readFromPvm || FallbackMemoryReader?.Invoke(baseAddress, exact) == true) { - cache[cacheKey] = previous; + Interlocked.Increment(ref GlobalMemoryReadCount); + Interlocked.Add(ref GlobalMemoryReadBytes, sizeof(uint)); + if (readFromPvm) + { + Interlocked.Add(ref GlobalMemoryReadPvmBytes, sizeof(uint)); + } + else + { + Interlocked.Add(ref GlobalMemoryReadLibcBytes, sizeof(uint)); + } + data = rented; + dataLength = sizeof(uint); + return true; } - data = previous; - return true; + GlobalMemoryPool.Return(rented); + return false; } var candidateSize = (int)cappedSize; while (candidateSize >= sizeof(uint)) { - data = GC.AllocateUninitializedArray(candidateSize); - var readFromPvm = ctx.Memory.TryRead(baseAddress, data); - if (readFromPvm || - FallbackMemoryReader?.Invoke(baseAddress, data) == true) + var span = rented.AsSpan(0, candidateSize); + var readFromPvm = ctx.Memory.TryRead(baseAddress, span); + if (readFromPvm || FallbackMemoryReader?.Invoke(baseAddress, span) == true) { Interlocked.Increment(ref GlobalMemoryReadCount); - Interlocked.Add(ref GlobalMemoryReadBytes, data.Length); + Interlocked.Add(ref GlobalMemoryReadBytes, candidateSize); if (readFromPvm) { - Interlocked.Add(ref GlobalMemoryReadPvmBytes, data.Length); + Interlocked.Add(ref GlobalMemoryReadPvmBytes, candidateSize); } else { - Interlocked.Add(ref GlobalMemoryReadLibcBytes, data.Length); + Interlocked.Add(ref GlobalMemoryReadLibcBytes, candidateSize); } - - if (cache is not null) - { - cache[cacheKey] = data; - } - - lock (_crossFrameReadGate) - { - if (_crossFrameReadCache.TryGetValue(cacheKey, out var replaced)) - { - _crossFrameReadCacheBytes -= replaced.Length; - } - - if (_crossFrameReadCacheBytes + data.Length > CrossFrameReadCacheMaxBytes) - { - _crossFrameReadCache.Clear(); - _crossFrameReadCacheBytes = 0; - } - - _crossFrameReadCache[cacheKey] = data; - _crossFrameReadCacheBytes += data.Length; - } - + data = rented; + dataLength = candidateSize; return true; } @@ -701,7 +1092,7 @@ public static class Gen5ShaderScalarEvaluator candidateSize = Math.Max(candidateSize / 2, sizeof(uint)); } - data = []; + GlobalMemoryPool.Return(rented); return false; } @@ -775,6 +1166,13 @@ public static class Gen5ShaderScalarEvaluator value = ~value; scalarConditionCode = value != 0; } + else if (instruction.Opcode == "SWqmB64") + { + var quadAny = (value | (value >> 1) | (value >> 2) | (value >> 3)) & + 0x1111_1111_1111_1111UL; + value = quadAny * 0xFUL; + scalarConditionCode = value != 0; + } WriteScalarPair(registers, destination.Value, value, ref execMask); return true; @@ -849,6 +1247,27 @@ public static class Gen5ShaderScalarEvaluator return true; } + if (instruction.Opcode == "SBfmB64") + { + if (instruction.Sources.Count < 2 || + destination.Value >= ScalarRegisterCount - 1 || + !TryEvaluateScalarOperand(instruction.Sources[0], registers, out var widthSource) || + !TryEvaluateScalarOperand(instruction.Sources[1], registers, out var offsetSource)) + { + error = $"scalar-source64 pc=0x{instruction.Pc:X} op={instruction.Opcode}"; + return false; + } + + var width = (int)widthSource & 63; + var offset = (int)offsetSource & 63; + var value = width == 0 + ? 0UL + : (ulong.MaxValue >> (64 - width)) << offset; + WriteScalarPair(registers, destination.Value, value, ref execMask); + scalarConditionCode = value != 0; + return true; + } + if (instruction.Opcode is "SCselectB64" or "SAndB64" or @@ -894,7 +1313,12 @@ public static class Gen5ShaderScalarEvaluator } if (instruction.Sources.Count == 0 || - !TryEvaluateScalarOperand(instruction.Sources[0], registers, out var left)) + !TryEvaluateScalarOperand( + instruction.Sources[0], + registers, + execMask, + scalarConditionCode, + out var left)) { var source = instruction.Sources.Count == 0 ? "" @@ -933,7 +1357,12 @@ public static class Gen5ShaderScalarEvaluator } if (instruction.Sources.Count < 2 || - !TryEvaluateScalarOperand(instruction.Sources[1], registers, out var right)) + !TryEvaluateScalarOperand( + instruction.Sources[1], + registers, + execMask, + scalarConditionCode, + out var right)) { var source = instruction.Sources.Count < 2 ? "" @@ -1134,6 +1563,44 @@ public static class Gen5ShaderScalarEvaluator out string error) { error = string.Empty; + if (instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal)) + { + if (instruction.Destinations.Count != 1 || + instruction.Destinations[0] is not + { + Kind: Gen5OperandKind.ScalarRegister, + Value: < ScalarRegisterCount, + } destination32 || + instruction.Sources.Count == 0 || + !TryEvaluateScalarOperand(instruction.Sources[0], registers, out var source32)) + { + error = $"scalar-source32 pc=0x{instruction.Pc:X} op={instruction.Opcode}"; + return false; + } + + var oldExec32 = (uint)execMask; + var newExec32 = instruction.Opcode switch + { + "SAndSaveexecB32" => oldExec32 & source32, + "SOrSaveexecB32" => oldExec32 | source32, + "SXorSaveexecB32" => oldExec32 ^ source32, + "SAndn1SaveexecB32" => ~source32 & oldExec32, + "SAndn2SaveexecB32" => source32 & ~oldExec32, + "SOrn1SaveexecB32" => ~source32 | oldExec32, + "SOrn2SaveexecB32" => source32 | ~oldExec32, + "SNandSaveexecB32" => ~(source32 & oldExec32), + "SNorSaveexecB32" => ~(source32 | oldExec32), + "SXnorSaveexecB32" => ~(oldExec32 ^ source32), + _ => 0u, + }; + registers[destination32.Value] = oldExec32; + execMask = newExec32; + registers[126] = newExec32; + registers[127] = 0; + scalarConditionCode = newExec32 != 0; + return true; + } + if (instruction.Opcode is not ( "SAndSaveexecB64" or "SOrSaveexecB64" or @@ -1370,7 +1837,8 @@ public static class Gen5ShaderScalarEvaluator uint[] scalarRegisters, List globalMemoryBindings, Dictionary<(uint ScalarAddress, ulong BaseAddress), Gen5GlobalMemoryBinding> globalMemoryByAddress, - HashSet runtimeScalarRegisters, + IReadOnlySet runtimeScalarRegisters, + bool recordBinding, out string error) { error = string.Empty; @@ -1417,30 +1885,64 @@ public static class Gen5ShaderScalarEvaluator scalarBase.Value + 3 < ScalarRegisterCount && scalarRegisters[scalarBase.Value + 2] == 0 && scalarRegisters[scalarBase.Value + 3] == 0)); + var scalarPointerUnbound = ShouldTreatScalarPointerAsUnbound( + isBufferLoad, + address, + _strictScalarLoad); + if (scalarPointerUnbound) + { + TraceScalarPointerFallback( + state, + instruction, + scalarBase.Value, + scalarRegisters, + control, + baseAddress, + dynamicOffset); + } var bufferSize = ulong.MaxValue; - if (isBufferLoad) + if (recordBinding && isBufferLoad) { bufferSize = hasBufferDescriptor ? bufferDescriptor.SizeBytes : ulong.MaxValue; var key = (scalarBase.Value, bufferDescriptor.BaseAddress); if (globalMemoryByAddress.TryGetValue(key, out var existingBinding)) { - if (existingBinding.InstructionPcs is List instructionPcs) instructionPcs.Add(instruction.Pc); + if (existingBinding.InstructionPcs is List instructionPcs && + !instructionPcs.Contains(instruction.Pc)) + { + instructionPcs.Add(instruction.Pc); + } } else { - TryReadGlobalMemory(ctx, bufferDescriptor.BaseAddress, bufferDescriptor.SizeBytes, out var data); - var binding = new Gen5GlobalMemoryBinding(scalarBase.Value, bufferDescriptor.BaseAddress, new List { instruction.Pc }, data); + var pooled = TryReadGlobalMemory( + ctx, + bufferDescriptor.BaseAddress, + bufferDescriptor.SizeBytes, + out var data, + out var dataLength); + var binding = new Gen5GlobalMemoryBinding( + scalarBase.Value, + bufferDescriptor.BaseAddress, + new List { instruction.Pc }, + data, + dataLength, + DataPooled: pooled) + { + WriteBackToGuest = pooled, + }; globalMemoryByAddress.Add(key, binding); globalMemoryBindings.Add(binding); } } - else if (baseAddress != 0) + else if (recordBinding && baseAddress != 0) { var key = (scalarBase.Value, baseAddress); if (globalMemoryByAddress.TryGetValue(key, out var existingBinding)) { - if (existingBinding.InstructionPcs is List instructionPcs) + if (existingBinding.InstructionPcs is List instructionPcs && + !instructionPcs.Contains(instruction.Pc)) { instructionPcs.Add(instruction.Pc); } @@ -1456,26 +1958,28 @@ public static class Gen5ShaderScalarEvaluator requiredBytes = Math.Min( (requiredBytes + 4095UL) & ~4095UL, MaxGlobalMemoryBindingBytes); - if (!TryReadGlobalMemory( - ctx, - baseAddress, - requiredBytes, - out var data)) - { - data = []; - } - + var pooled = TryReadGlobalMemory( + ctx, + baseAddress, + requiredBytes, + out var data, + out var dataLength); var binding = new Gen5GlobalMemoryBinding( scalarBase.Value, baseAddress, new List { instruction.Pc }, - data); + data, + dataLength, + DataPooled: pooled) + { + WriteBackToGuest = pooled, + }; globalMemoryByAddress.Add(key, binding); globalMemoryBindings.Add(binding); } } - if (!bufferUnbound && address == 0) + if (!bufferUnbound && !scalarPointerUnbound && address == 0) { error = FormatScalarLoadError( "invalid-load-address", @@ -1509,6 +2013,7 @@ public static class Gen5ShaderScalarEvaluator var componentOffset = unchecked(byteOffset + (ulong)(index * sizeof(uint))); if (bufferUnbound || + scalarPointerUnbound || isBufferLoad && (componentOffset >= bufferSize || bufferSize - componentOffset < sizeof(uint))) @@ -1517,18 +2022,12 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (!ctx.TryReadUInt32( + if (!TryReadUInt32( + ctx, address + (ulong)(index * sizeof(uint)), - out var value) && - !TryReadUserDataScalarLoad( - state, - instruction, - control, - byteOffset, - index, - out value)) + out var value)) { - if (isBufferLoad) + if (isBufferLoad || !_strictScalarLoad) { scalarRegisters[destination.Value] = 0; continue; @@ -1552,6 +2051,87 @@ public static class Gen5ShaderScalarEvaluator return true; } + private static bool ShouldTreatScalarPointerAsUnbound( + bool isBufferLoad, + ulong address, + bool strictScalarLoad) => + !isBufferLoad && address == 0 && !strictScalarLoad; + + private static void TraceScalarPointerFallback( + Gen5ShaderState state, + Gen5ShaderInstruction instruction, + uint scalarBase, + IReadOnlyList scalarRegisters, + Gen5ScalarMemoryControl control, + ulong baseAddress, + ulong dynamicOffset) + { + lock (_scalarFallbackTraceGate) + { + if (!_tracedScalarFallbacks.Add((state.Program.Address, instruction.Pc))) + { + return; + } + } + + var definitions = state.Program.Instructions + .Where(candidate => + candidate.Pc < instruction.Pc && + candidate.Destinations.Any(destination => + destination.Kind == Gen5OperandKind.ScalarRegister && + destination.Value is var register && + register >= scalarBase && register <= scalarBase + 1)) + .TakeLast(8) + .Select(candidate => + $"0x{candidate.Pc:X}:{candidate.Opcode}[" + + string.Join(',', candidate.Words.Select(word => $"{word:X8}")) + "]") + .ToArray(); + var userData = string.Join( + ',', + state.UserData.Take(32).Select((value, index) => $"s{index}=0x{value:X8}")); + var high = scalarBase + 1 < scalarRegisters.Count + ? scalarRegisters[(int)scalarBase + 1] + : 0; + Console.Error.WriteLine( + $"[LOADER][WARN] agc.scalar_pointer_fallback " + + $"shader=0x{state.Program.Address:X16} pc=0x{instruction.Pc:X} " + + $"op={instruction.Opcode} base=s{scalarBase}" + + $"[0x{scalarRegisters[(int)scalarBase]:X8}:0x{high:X8}] " + + $"base_addr=0x{baseAddress:X16} imm={control.ImmediateOffsetBytes} " + + $"dynamic={dynamicOffset} definitions=[{string.Join(';', definitions)}] " + + $"user_data=[{userData}] metadata=" + + $"{(state.Metadata is null ? "missing" : $"srt={state.Metadata.ShaderResourceTableSizeDwords},eud={state.Metadata.ExtendedUserDataSizeDwords}")}"); + } + + [Conditional("DEBUG")] + private static void RunScalarLoadSelfChecks() + { + Debug.Assert( + ShouldTreatScalarPointerAsUnbound( + isBufferLoad: false, + address: 0, + strictScalarLoad: false), + "A null non-strict scalar pointer must read as zero instead of dropping the shader pass."); + Debug.Assert( + !ShouldTreatScalarPointerAsUnbound( + isBufferLoad: false, + address: 0, + strictScalarLoad: true), + "Strict scalar-load diagnostics must continue rejecting null pointers."); + Debug.Assert( + !ShouldTreatScalarPointerAsUnbound( + isBufferLoad: true, + address: 0, + strictScalarLoad: false), + "Buffer descriptor null handling must remain on the buffer-unbound path."); + Debug.Assert( + !ShouldTreatScalarPointerAsUnbound( + isBufferLoad: false, + address: 0x1000, + strictScalarLoad: false), + "A valid scalar pointer must not be treated as an unbound resource."); + } + private static bool TryDecodeBufferDescriptor( IReadOnlyList scalarRegisters, uint scalarBase, @@ -1592,8 +2172,14 @@ public static class Gen5ShaderScalarEvaluator var baseAddress = word0 | ((ulong)(word1 & 0xFFFFu) << 32); var stride = (word1 >> 16) & 0x3FFFu; var unifiedFormat = (word3 >> 12) & 0x7Fu; - var (dataFormat, numberFormat) = - DecodeGfx10BufferFormat(unifiedFormat); + if (!Gfx10UnifiedFormat.TryDecode( + unifiedFormat, + out var dataFormat, + out var numberFormat)) + { + return false; + } + var sizeBytes = stride == 0 ? word2 : (ulong)stride * word2; @@ -1601,111 +2187,6 @@ public static class Gen5ShaderScalarEvaluator return true; } - private static (uint DataFormat, uint NumberFormat) - DecodeGfx10BufferFormat(uint format) => - format switch - { - 0 => (0, 0), - >= 1 and <= 6 => (1, format - 1), - >= 7 and <= 13 => (2, DecodeUnifiedNumber(format - 7, 7)), - >= 14 and <= 19 => (3, format - 14), - >= 20 and <= 22 => (4, DecodeIntegerOrFloatNumber(format - 20)), - >= 23 and <= 29 => (5, DecodeUnifiedNumber(format - 23, 7)), - >= 30 and <= 36 => (6, DecodeUnifiedNumber(format - 30, 7)), - >= 37 and <= 43 => (7, DecodeUnifiedNumber(format - 37, 7)), - >= 44 and <= 49 => (8, format - 44), - >= 50 and <= 55 => (9, format - 50), - >= 56 and <= 61 => (10, format - 56), - >= 62 and <= 64 => (11, DecodeIntegerOrFloatNumber(format - 62)), - >= 65 and <= 71 => (12, DecodeUnifiedNumber(format - 65, 7)), - >= 72 and <= 74 => (13, DecodeIntegerOrFloatNumber(format - 72)), - >= 75 and <= 77 => (14, DecodeIntegerOrFloatNumber(format - 75)), - 128 => (1, 9), - 129 => (3, 9), - 130 => (10, 9), - 132 => (34, 7), - 133 => (16, 0), - 134 => (17, 0), - 135 => (18, 0), - 136 => (19, 0), - 140 => (4, 7), - _ => (0, 0), - }; - - private static uint DecodeUnifiedNumber(uint offset, uint formatCount) => - offset == formatCount - 1 ? 7u : offset; - - private static uint DecodeIntegerOrFloatNumber(uint offset) => - offset switch - { - 0 => 4, - 1 => 5, - _ => 7, - }; - - private static bool TryReadUserDataScalarLoad( - Gen5ShaderState state, - Gen5ShaderInstruction instruction, - Gen5ScalarMemoryControl control, - ulong byteOffset, - int componentIndex, - out uint value) - { - value = 0; - if (!instruction.Opcode.StartsWith("SLoadDword", StringComparison.Ordinal) || - state.Metadata is not { } metadata || - (byteOffset & 3) != 0) - { - return false; - } - - var baseDwordOffset = byteOffset >> 2; - if (baseDwordOffset > int.MaxValue) - { - return false; - } - - var dwordOffset = (long)baseDwordOffset + componentIndex; - if (dwordOffset < 0 || - dwordOffset >= state.UserData.Count || - !IsShaderUserDataResourceOffset(metadata, (uint)dwordOffset)) - { - return false; - } - - value = state.UserData[(int)dwordOffset]; - return true; - } - - private static bool IsShaderUserDataResourceOffset( - Gen5ShaderMetadata metadata, - uint dwordOffset) - { - if (dwordOffset < metadata.ShaderResourceTableSizeDwords) - { - return true; - } - - foreach (var resource in metadata.Resources) - { - var dwordCount = resource.Kind switch - { - Gen5ShaderResourceKind.ReadOnlyTexture or - Gen5ShaderResourceKind.ReadWriteTexture => 8u, - Gen5ShaderResourceKind.Sampler or - Gen5ShaderResourceKind.ConstantBuffer => 4u, - _ => 1u, - }; - if (dwordOffset >= resource.OffsetDwords && - dwordOffset < resource.OffsetDwords + dwordCount) - { - return true; - } - } - - return metadata.DirectResources.Values.Any(offset => offset == dwordOffset); - } - private static string FormatScalarLoadError( string reason, Gen5ShaderInstruction instruction, @@ -1764,6 +2245,34 @@ public static class Gen5ShaderScalarEvaluator return false; } + private static bool TryEvaluateScalarOperand( + Gen5Operand operand, + uint[] scalarRegisters, + ulong execMask, + bool scalarConditionCode, + out uint value) + { + if (operand.Kind == Gen5OperandKind.EncodedConstant) + { + switch (operand.Value) + { + // RDNA scalar-source special registers. These encodings share + // the inline-constant field but are evaluated from wave state. + case 251: // VCCZ + value = (scalarRegisters[106] | scalarRegisters[107]) == 0 ? 1u : 0u; + return true; + case 252: // EXECZ + value = execMask == 0 ? 1u : 0u; + return true; + case 253: // SCC + value = scalarConditionCode ? 1u : 0u; + return true; + } + } + + return TryEvaluateScalarOperand(operand, scalarRegisters, out value); + } + private static bool TryDecodeInlineConstant(uint encoded, out uint value) { if (encoded == 125) @@ -1838,4 +2347,16 @@ public static class Gen5ShaderScalarEvaluator opcode.StartsWith("ImageSample", StringComparison.Ordinal) || opcode.StartsWith("ImageGather", StringComparison.Ordinal); + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } } diff --git a/src/SharpEmu.ShaderCompiler/Gen5ShaderTranslator.cs b/src/SharpEmu.ShaderCompiler/Gen5ShaderTranslator.cs index 3df3d91..688bc84 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5ShaderTranslator.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5ShaderTranslator.cs @@ -3,6 +3,7 @@ using SharpEmu.HLE; using System.Buffers.Binary; +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Text; @@ -10,9 +11,83 @@ namespace SharpEmu.ShaderCompiler; public static class Gen5ShaderTranslator { + private static int _dppVectorsValidated; + /// + /// Bitmask (256 bits) of scalar registers whose values the program can + /// observe: scalar source operands (widened for 64-bit pairs), the + /// descriptor/sampler/address ranges named by instruction controls, and + /// the implicit state registers. Deterministic per instruction stream, so + /// the SPIR-V that loads exactly these registers from the per-draw + /// initial-state buffer is byte-stable across draws. + /// + public static ulong[] ComputeConsumedScalarMask(Gen5ShaderProgram program) + { + var mask = new ulong[4]; + AddConsumedScalar(mask, 106, 2); + AddConsumedScalar(mask, 124, 2); + AddConsumedScalar(mask, 126, 2); + foreach (var instruction in program.Instructions) + { + foreach (var source in instruction.Sources) + { + if (source.Kind == Gen5OperandKind.ScalarRegister) + { + AddConsumedScalar(mask, source.Value, 2); + } + } + + // Scalar memory bases can be a 4-dword buffer descriptor. + if (instruction.Encoding is Gen5ShaderEncoding.Smem or Gen5ShaderEncoding.Smrd && + instruction.Sources.Count > 0 && + instruction.Sources[0].Kind == Gen5OperandKind.ScalarRegister) + { + AddConsumedScalar(mask, instruction.Sources[0].Value, 4); + } + + switch (instruction.Control) + { + case Gen5ImageControl image: + AddConsumedScalar(mask, image.ScalarResource, 8); + AddConsumedScalar(mask, image.ScalarSampler, 4); + break; + case Gen5ScalarMemoryControl { DynamicOffsetRegister: { } offsetRegister }: + AddConsumedScalar(mask, offsetRegister, 2); + break; + case Gen5GlobalMemoryControl global: + AddConsumedScalar(mask, global.ScalarAddress, 2); + break; + case Gen5BufferMemoryControl buffer: + AddConsumedScalar(mask, buffer.ScalarResource, 4); + break; + } + } + + return mask; + } + + private static void AddConsumedScalar(ulong[] mask, uint register, uint count) + { + for (uint index = 0; index < count; index++) + { + var target = register + index; + if (target < 256) + { + mask[target >> 6] |= 1UL << (int)(target & 63); + } + } + } + + public static bool IsScalarConsumed(ulong[] mask, uint register) => + register < 256 && (mask[register >> 6] & (1UL << (int)(register & 63))) != 0; + private const int MaxInstructions = 4096; - private const int MinimumUserDataDwords = 16; - private const int MaximumUserDataDwords = 256; + private const uint PsUserDataRegister = 0x0C; + private const uint VsUserDataRegister = 0x4C; + private const uint GsUserDataRegister = 0x8C; + private const uint EsUserDataRegister = 0xCC; + private const uint ComputeUserDataRegister = 0x240; + private const uint ComputePgmRsrc2Register = 0x213; + private const int MaximumHardwareUserSgprs = 64; private static readonly ConditionalWeakTable _decodeCaches = new(); private sealed class ShaderDecodeCache @@ -124,6 +199,7 @@ public static class Gen5ShaderTranslator Gen5ComputeSystemRegisters? computeSystemRegisters = null, uint userDataScalarRegisterBase = 0) { + ValidateUserSgprCountDecoding(); state = default!; error = string.Empty; var cache = _decodeCaches.GetValue(ctx.Memory, static _ => new ShaderDecodeCache()); @@ -172,7 +248,20 @@ public static class Gen5ShaderTranslator } } - var userData = new uint[GetUserDataDwordCount(metadata)]; + if (!TryGetUserSgprCount( + shaderRegisters, + userDataBaseRegister, + out var userSgprCount, + out var rsrc2Register, + out _)) + { + error = + $"missing-user-sgpr-count ud_reg=0x{userDataBaseRegister:X} " + + $"rsrc2_reg=0x{rsrc2Register:X}"; + return false; + } + + var userData = new uint[userSgprCount]; for (uint index = 0; index < userData.Length; index++) { shaderRegisters.TryGetValue(userDataBaseRegister + index, out userData[index]); @@ -187,35 +276,72 @@ public static class Gen5ShaderTranslator return true; } - private static int GetUserDataDwordCount(Gen5ShaderMetadata? metadata) + private static bool TryGetUserSgprCount( + IReadOnlyDictionary shaderRegisters, + uint userDataBaseRegister, + out int count, + out uint rsrc2Register, + out uint rsrc2) { - var count = MinimumUserDataDwords; - if (metadata is null) + rsrc2Register = userDataBaseRegister == ComputeUserDataRegister + ? ComputePgmRsrc2Register + : userDataBaseRegister - 1; + if (!shaderRegisters.TryGetValue(rsrc2Register, out rsrc2)) { + count = 0; + return false; + } + + count = checked((int)((rsrc2 >> 1) & 0x1Fu)); + // GFX10 PS/VS/GS expose a sixth USER_SGPR bit. ES and compute do not. + // AGC's logical user-data layout (including its SRT pointer and back + // user data) describes memory reached through these SGPRs; it does not + // increase the hardware register window. + var hasUserSgprMsb = userDataBaseRegister is + PsUserDataRegister or VsUserDataRegister or GsUserDataRegister; + if (hasUserSgprMsb && + (rsrc2 & (1u << 27)) != 0) + { + count |= 0x20; + } + + if (userDataBaseRegister is not (PsUserDataRegister or + VsUserDataRegister or + GsUserDataRegister or + EsUserDataRegister or + ComputeUserDataRegister) || + count > MaximumHardwareUserSgprs) + { + count = 0; + return false; + } + + return true; + } + + [Conditional("DEBUG")] + private static void ValidateUserSgprCountDecoding() + { + static int Decode(uint baseRegister, uint rsrc2) + { + var registers = new Dictionary + { + [baseRegister == ComputeUserDataRegister + ? ComputePgmRsrc2Register + : baseRegister - 1] = rsrc2, + }; + var decoded = + TryGetUserSgprCount(registers, baseRegister, out var count, out _, out _); + Debug.Assert(decoded); return count; } - count = Math.Max(count, checked((int)Math.Min(metadata.ExtendedUserDataSizeDwords, MaximumUserDataDwords))); - count = Math.Max(count, checked((int)Math.Min(metadata.ShaderResourceTableSizeDwords, MaximumUserDataDwords))); - - foreach (var offset in metadata.DirectResources.Values) - { - count = Math.Max(count, checked((int)Math.Min(offset + 1u, MaximumUserDataDwords))); - } - - foreach (var resource in metadata.Resources) - { - var size = resource.Kind switch - { - Gen5ShaderResourceKind.ReadOnlyTexture or Gen5ShaderResourceKind.ReadWriteTexture => 8u, - Gen5ShaderResourceKind.Sampler => 4u, - Gen5ShaderResourceKind.ConstantBuffer => 4u, - _ => 1u, - }; - count = Math.Max(count, checked((int)Math.Min(resource.OffsetDwords + size, MaximumUserDataDwords))); - } - - return count; + Debug.Assert(Decode(PsUserDataRegister, 2u << 1) == 2); + Debug.Assert(Decode(PsUserDataRegister, (3u << 1) | (1u << 27)) == 35); + Debug.Assert(Decode(VsUserDataRegister, (1u << 1) | (1u << 27)) == 33); + Debug.Assert(Decode(GsUserDataRegister, (4u << 1) | (1u << 27)) == 36); + Debug.Assert(Decode(EsUserDataRegister, (7u << 1) | (1u << 27)) == 7); + Debug.Assert(Decode(ComputeUserDataRegister, (11u << 1) | (1u << 27)) == 11); } public static string DescribeState(Gen5ShaderState state) @@ -229,7 +355,8 @@ public static class Gen5ShaderTranslator if (state.Metadata is not { } metadata) { return - $"ud_base=s{state.UserDataScalarRegisterBase} ud[{userData}]" + + $"ud_base=s{state.UserDataScalarRegisterBase} hw_ud={state.UserData.Count} " + + $"ud[{userData}]" + $"{systemRegisters} metadata=missing"; } @@ -242,7 +369,8 @@ public static class Gen5ShaderTranslator $"{resource.Kind}[{resource.Slot}]@{resource.OffsetDwords}" + (resource.SizeFlag ? "+" : string.Empty))); return - $"ud_base=s{state.UserDataScalarRegisterBase} ud[{userData}]" + + $"ud_base=s{state.UserDataScalarRegisterBase} hw_ud={state.UserData.Count} " + + $"ud[{userData}]" + $"{systemRegisters} metadata[eud={metadata.ExtendedUserDataSizeDwords}," + $"srt={metadata.ShaderResourceTableSizeDwords},direct={direct},resources={resources}]"; } @@ -283,6 +411,7 @@ public static class Gen5ShaderTranslator out Gen5ShaderProgram program, out string error) { + ValidateDppControlVectors(); program = new Gen5ShaderProgram(address, []); error = string.Empty; if (address == 0) @@ -295,7 +424,7 @@ public static class Gen5ShaderTranslator var instructionCount = 0; for (uint pc = 0; instructionCount < MaxInstructions;) { - if (!ctx.TryReadUInt32(address + pc, out var word)) + if (!TryReadUInt32(ctx, address + pc, out var word)) { error = $"read-failed pc=0x{pc:X}"; return false; @@ -318,7 +447,7 @@ public static class Gen5ShaderTranslator words[0] = word; for (uint wordIndex = 1; wordIndex < sizeDwords; wordIndex++) { - if (!ctx.TryReadUInt32(address + pc + wordIndex * sizeof(uint), out words[wordIndex])) + if (!TryReadUInt32(ctx, address + pc + wordIndex * sizeof(uint), out words[wordIndex])) { error = $"read-failed pc=0x{pc + wordIndex * sizeof(uint):X}"; return false; @@ -340,6 +469,52 @@ public static class Gen5ShaderTranslator return false; } + [Conditional("DEBUG")] + private static void ValidateDppControlVectors() + { + if (System.Threading.Interlocked.Exchange(ref _dppVectorsValidated, 1) != 0) + { + return; + } + + static (uint Lane, bool InRange) Resolve(uint control, uint lane) + { + var rowBase = lane & ~15u; + var rowLane = lane & 15u; + return control switch + { + >= 0x101 and <= 0x10F => ( + rowBase + ((rowLane + (control & 15)) & 15), + rowLane + (control & 15) < 16), + >= 0x111 and <= 0x11F => ( + rowBase + ((rowLane - (control & 15)) & 15), + rowLane >= (control & 15)), + >= 0x121 and <= 0x12F => ( + rowBase + ((rowLane - (control & 15)) & 15), + true), + 0x140 => (rowBase + 15 - rowLane, true), + 0x141 => ((lane & ~7u) + 7 - (lane & 7), true), + >= 0x150 and <= 0x15F => (rowBase + (control & 15), true), + >= 0x160 and <= 0x16F => (rowBase + (rowLane ^ (control & 15)), true), + _ => (lane, false), + }; + } + + Debug.Assert(Resolve(0x101, 0) == (1u, true)); + Debug.Assert(Resolve(0x101, 15) == (0u, false)); + Debug.Assert(Resolve(0x112, 1) == (15u, false)); + Debug.Assert(Resolve(0x123, 0) == (13u, true)); + Debug.Assert(Resolve(0x140, 18) == (29u, true)); + Debug.Assert(Resolve(0x141, 9) == (14u, true)); + Debug.Assert(Resolve(0x153, 20) == (19u, true)); + Debug.Assert(Resolve(0x163, 22) == (21u, true)); + const uint dpp8 = + (7u << 0) | (6u << 3) | (5u << 6) | (4u << 9) | + (3u << 12) | (2u << 15) | (1u << 18) | (0u << 21); + Debug.Assert(((dpp8 >> (0 * 3)) & 7) == 7); + Debug.Assert(((dpp8 >> (7 * 3)) & 7) == 0); + } + private static bool TryDecodeInstruction( CpuContext ctx, ulong baseAddress, @@ -398,7 +573,7 @@ public static class Gen5ShaderTranslator case 0x34: case 0x35: encoding = Gen5ShaderEncoding.Vop3; - if (!ctx.TryReadUInt32(baseAddress + pc + sizeof(uint), out var vop3Extra)) + if (!TryReadUInt32(ctx, baseAddress + pc + sizeof(uint), out var vop3Extra)) { error = $"vop3-extra-read-failed pc=0x{pc:X}"; return false; @@ -419,7 +594,7 @@ public static class Gen5ShaderTranslator return DecodeFlat(word, out name, out sizeDwords, out error); case 0x38: encoding = Gen5ShaderEncoding.Mubuf; - if (!ctx.TryReadUInt32(baseAddress + pc + sizeof(uint), out var mubufExtra)) + if (!TryReadUInt32(ctx, baseAddress + pc + sizeof(uint), out var mubufExtra)) { error = $"mubuf-extra-read-failed pc=0x{pc:X}"; return false; @@ -428,7 +603,7 @@ public static class Gen5ShaderTranslator return DecodeMubuf(word, mubufExtra, out name, out sizeDwords, out error); case 0x3A: encoding = Gen5ShaderEncoding.Mtbuf; - if (!ctx.TryReadUInt32(baseAddress + pc + sizeof(uint), out var mtbufExtra)) + if (!TryReadUInt32(ctx, baseAddress + pc + sizeof(uint), out var mtbufExtra)) { error = $"mtbuf-extra-read-failed pc=0x{pc:X}"; return false; @@ -455,6 +630,26 @@ public static class Gen5ShaderTranslator } } + // Kept beside the production decoder so offline compatibility tools use + // precisely the same opcode tables and instruction-width rules as runtime + // shader translation. + internal static bool TryDecodeInstructionForPreflight( + CpuContext ctx, + uint pc, + uint word, + out string name, + out uint sizeDwords, + out string error) => + TryDecodeInstruction( + ctx, + 0, + pc, + word, + out _, + out name, + out sizeDwords, + out error); + private static bool DecodeSop(uint word, out string name, out uint sizeDwords, out string error) { var opcode = (word >> 23) & 0x7F; @@ -498,6 +693,16 @@ public static class Gen5ShaderTranslator 0x2B => "SXnorSaveexecB64", 0x37 => "SAndn1SaveexecB64", 0x38 => "SOrn1SaveexecB64", + 0x3C => "SAndSaveexecB32", + 0x3D => "SOrSaveexecB32", + 0x3E => "SXorSaveexecB32", + 0x3F => "SAndn2SaveexecB32", + 0x40 => "SOrn2SaveexecB32", + 0x41 => "SNandSaveexecB32", + 0x42 => "SNorSaveexecB32", + 0x43 => "SXnorSaveexecB32", + 0x44 => "SAndn1SaveexecB32", + 0x45 => "SOrn1SaveexecB32", _ => string.Empty, }; @@ -662,7 +867,7 @@ public static class Gen5ShaderTranslator { var opcode = (word >> 9) & 0xFF; var src0 = word & 0x1FF; - sizeDwords = src0 is 0xF9 or 0xFA or 0xFF ? 2u : 1u; + sizeDwords = src0 is 0xE9 or 0xEA or 0xF9 or 0xFA or 0xFF ? 2u : 1u; error = string.Empty; name = opcode switch { @@ -721,7 +926,8 @@ public static class Gen5ShaderTranslator } var src0 = word & 0x1FF; - sizeDwords = opcode is 0x20 or 0x21 or 0x2C or 0x2D || src0 is 0xF9 or 0xFA or 0xFF ? 2u : 1u; + sizeDwords = opcode is 0x20 or 0x21 or 0x2C or 0x2D || + src0 is 0xE9 or 0xEA or 0xF9 or 0xFA or 0xFF ? 2u : 1u; error = string.Empty; name = opcode switch { @@ -748,6 +954,7 @@ public static class Gen5ShaderTranslator 0x1B => "VAndB32", 0x1C => "VOrB32", 0x1D => "VXorB32", + 0x1E => "VXnorB32", 0x1F => "VMacF32", 0x20 => "VMadMkF32", 0x21 => "VMadAkF32", @@ -761,8 +968,8 @@ public static class Gen5ShaderTranslator 0x29 => "VSubbU32", 0x2A => "VSubbrevU32", 0x2B => "VFmacF32", - 0x2C => "VFmamkF32", - 0x2D => "VFmaakF32", + 0x2C => "VFmaMkF32", + 0x2D => "VFmaAkF32", 0x2F => "VCvtPkrtzF16F32", 0x30 => "VCvtPkU16U32", 0x31 => "VCvtPkI16I32", @@ -776,7 +983,7 @@ public static class Gen5ShaderTranslator { var opcode = (word >> 17) & 0xFF; var src0 = word & 0x1FF; - sizeDwords = src0 is 0xF9 or 0xFA or 0xFF ? 2u : 1u; + sizeDwords = src0 is 0xE9 or 0xEA or 0xF9 or 0xFA or 0xFF ? 2u : 1u; error = string.Empty; name = opcode switch { @@ -868,9 +1075,11 @@ public static class Gen5ShaderTranslator name = isVop3B ? opcode switch { + 0x128 => "VAddCoCiU32", 0x30F => "VAddCoU32", 0x310 => "VSubCoU32", 0x319 => "VSubrevCoU32", + 0x176 => "VMadU64U32", _ => $"Vop3bRaw{opcode:X3}", } : opcode switch @@ -914,6 +1123,12 @@ public static class Gen5ShaderTranslator 0x360 => "VReadlaneB32", 0x361 => "VWritelaneB32", 0x362 => "VLdexpF32", + 0x363 => "VBfmB32", + 0x364 => "VBcntU32B32", + 0x365 => "VMbcntLoU32B32", + 0x366 => "VMbcntHiU32B32", + 0x368 => "VCvtPknormI16F32", + 0x369 => "VCvtPknormU16F32", 0x373 => "VMadU32U16", 0x346 => "VLshlAddU32", 0x347 => "VAddLshlU32", @@ -930,7 +1145,7 @@ public static class Gen5ShaderTranslator } private static bool IsVop3BOpcode(uint opcode) => - opcode is 0x16D or 0x16E or 0x176 or 0x177 or 0x30F or 0x310 or 0x319; + opcode is 0x128 or 0x16D or 0x16E or 0x176 or 0x177 or 0x30F or 0x310 or 0x319; private static bool DecodeRaw2( uint word, @@ -956,6 +1171,7 @@ public static class Gen5ShaderTranslator error = string.Empty; name = opcode switch { + 0x00 => "DsAddU32", 0x0D => "DsWriteB32", 0x0E => "DsWrite2B32", 0x0F => "DsWrite2St64B32", @@ -964,6 +1180,10 @@ public static class Gen5ShaderTranslator 0x37 => "DsRead2B32", 0x38 => "DsRead2St64B32", 0x4D => "DsWriteB64", + 0xDE => "DsWriteB96", + 0xDF => "DsWriteB128", + 0xFE => "DsReadB96", + 0xFF => "DsReadB128", _ => string.Empty, }; @@ -1026,15 +1246,34 @@ public static class Gen5ShaderTranslator 0x01 => "BufferLoadFormatXy", 0x02 => "BufferLoadFormatXyz", 0x03 => "BufferLoadFormatXyzw", + 0x04 => "BufferStoreFormatX", + 0x05 => "BufferStoreFormatXy", + 0x06 => "BufferStoreFormatXyz", + 0x07 => "BufferStoreFormatXyzw", + 0x08 => "BufferLoadUbyte", + 0x09 => "BufferLoadSbyte", + 0x0A => "BufferLoadUshort", + 0x0B => "BufferLoadSshort", 0x0C => "BufferLoadDword", 0x0D => "BufferLoadDwordx2", 0x0E => "BufferLoadDwordx4", 0x0F => "BufferLoadDwordx3", + 0x18 => "BufferStoreByte", + 0x19 => "BufferStoreByteD16Hi", + 0x1A => "BufferStoreShort", + 0x1B => "BufferStoreShortD16Hi", 0x1C => "BufferStoreDword", 0x1D => "BufferStoreDwordx2", 0x1E => "BufferStoreDwordx4", 0x1F => "BufferStoreDwordx3", + 0x20 => "BufferLoadUbyteD16", + 0x21 => "BufferLoadUbyteD16Hi", + 0x22 => "BufferLoadSbyteD16", + 0x23 => "BufferLoadSbyteD16Hi", + 0x24 => "BufferLoadShortD16", + 0x25 => "BufferLoadShortD16Hi", 0x32 => "BufferAtomicAdd", + 0x38 => "BufferAtomicUMax", _ => $"MubufRaw{opcode:X2}", }; sizeDwords = (extra >> 24) == 0xFF ? 3u : 2u; @@ -1055,10 +1294,30 @@ public static class Gen5ShaderTranslator name = segment == 0x2 ? opcode switch { + 0x08 => "GlobalLoadUbyte", + 0x09 => "GlobalLoadSbyte", + 0x0A => "GlobalLoadUshort", + 0x0B => "GlobalLoadSshort", 0x0C => "GlobalLoadDword", 0x0D => "GlobalLoadDwordx2", 0x0E => "GlobalLoadDwordx4", 0x0F => "GlobalLoadDwordx3", + 0x18 => "GlobalStoreByte", + 0x19 => "GlobalStoreByteD16Hi", + 0x1A => "GlobalStoreShort", + 0x1B => "GlobalStoreShortD16Hi", + 0x1C => "GlobalStoreDword", + 0x1D => "GlobalStoreDwordx2", + 0x1E => "GlobalStoreDwordx4", + 0x1F => "GlobalStoreDwordx3", + 0x20 => "GlobalLoadUbyteD16", + 0x21 => "GlobalLoadUbyteD16Hi", + 0x22 => "GlobalLoadSbyteD16", + 0x23 => "GlobalLoadSbyteD16Hi", + 0x24 => "GlobalLoadShortD16", + 0x25 => "GlobalLoadShortD16Hi", + 0x32 => "GlobalAtomicAdd", + 0x38 => "GlobalAtomicUMax", _ => string.Empty, } : string.Empty; @@ -1132,6 +1391,7 @@ public static class Gen5ShaderTranslator 0x10 => "ImageAtomicCmpswap", 0x1C => "ImageAtomicDec", 0x20 => "ImageSample", + 0x22 => "ImageSampleD", 0x24 => "ImageSampleL", 0x25 => "ImageSampleB", 0x27 => "ImageSampleLz", @@ -1143,6 +1403,7 @@ public static class Gen5ShaderTranslator 0x47 => "ImageGather4Lz", 0x48 => "ImageGather4C", 0x4E => "ImageGather4CBCl", + 0x57 => "ImageGather4LzO", 0x5F => "ImageGather4CLzO", _ => string.Empty, }; @@ -1223,6 +1484,7 @@ public static class Gen5ShaderTranslator name.StartsWith("Image", StringComparison.Ordinal); public static bool IsStorageImageOperation(string name) => + name.StartsWith("ImageLoad", StringComparison.Ordinal) || name.StartsWith("ImageStore", StringComparison.Ordinal) || name.StartsWith("ImageAtomic", StringComparison.Ordinal); @@ -1239,7 +1501,11 @@ public static class Gen5ShaderTranslator var isDpp = encoding is Gen5ShaderEncoding.Vop1 or Gen5ShaderEncoding.Vop2 or Gen5ShaderEncoding.Vopc && (word & 0x1FF) == 0xFA; - var literal = !isSdwa && !isDpp && words.Length > MinimumEncodingDwords(encoding) + var isDpp8 = + encoding is Gen5ShaderEncoding.Vop1 or Gen5ShaderEncoding.Vop2 or Gen5ShaderEncoding.Vopc && + (word & 0x1FF) is 0xE9 or 0xEA; + var literal = !isSdwa && !isDpp && !isDpp8 && + words.Length > MinimumEncodingDwords(encoding) ? words[^1] : (uint?)null; IReadOnlyList sources = []; @@ -1314,20 +1580,36 @@ public static class Gen5ShaderTranslator var scalarOffset = (extra >> 25) & 0x7F; var offset = SignExtend(extra & 0x1FFFFF, 21); var count = ScalarLoadDwordCount(opcode); + var scalarOffsetOperand = Gen5Operand.Source(scalarOffset); + var dynamicOffsetRegister = scalarOffsetOperand.Kind == + Gen5OperandKind.ScalarRegister + ? scalarOffsetOperand.Value + : (uint?)null; sources = [ Gen5Operand.Scalar(scalarBase), - Gen5Operand.Scalar(scalarOffset), + scalarOffsetOperand, ]; destinations = Enumerable .Range((int)scalarDestination, checked((int)count)) .Select(index => Gen5Operand.Scalar((uint)index)) .ToArray(); - control = new Gen5ScalarMemoryControl(count, offset, scalarOffset); + control = new Gen5ScalarMemoryControl( + count, + offset, + dynamicOffsetRegister); break; } case Gen5ShaderEncoding.Vop1: - if (isDpp) + if (isDpp8) + { + var extra = words[1]; + sources = [Gen5Operand.Vector(extra & 0xFF)]; + control = new Gen5Dpp8Control( + extra >> 8, + (word & 0x1FF) == 0xEA); + } + else if (isDpp) { var extra = words[1]; sources = [Gen5Operand.Vector(extra & 0xFF)]; @@ -1339,24 +1621,36 @@ public static class Gen5ShaderTranslator var source0 = (extra & 0xFF) + ((((extra >> 23) & 1) == 0) ? 256u : 0u); sources = [Gen5Operand.Source(source0)]; - control = new Gen5SdwaControl( - (extra >> 8) & 0x7, - (extra >> 16) & 0x7, - 6, - (extra >> 21) & 1, - (extra >> 20) & 1, - (extra >> 14) & 0x3, - ((extra >> 13) & 1) != 0); + control = CreateSdwaControl(extra, isCompare: false, hasSource1: false); } else { sources = [Gen5Operand.Source(word & 0x1FF, literal)]; } - destinations = [Gen5Operand.Vector((word >> 17) & 0xFF)]; + // V_READFIRSTLANE_B32 is encoded as VOP1, but its destination + // field names an SGPR rather than a VGPR. Treating it like an + // ordinary vector destination leaves every invocation with a + // different value and corrupts scalar addresses derived from + // lane data. + destinations = opcode == "VReadfirstlaneB32" + ? [Gen5Operand.Scalar((word >> 17) & 0x7F)] + : [Gen5Operand.Vector((word >> 17) & 0xFF)]; break; case Gen5ShaderEncoding.Vop2: - if (isDpp) + if (isDpp8) + { + var extra = words[1]; + sources = + [ + Gen5Operand.Vector(extra & 0xFF), + Gen5Operand.Vector((word >> 9) & 0xFF), + ]; + control = new Gen5Dpp8Control( + extra >> 8, + (word & 0x1FF) == 0xEA); + } + else if (isDpp) { var extra = words[1]; sources = @@ -1378,14 +1672,7 @@ public static class Gen5ShaderTranslator Gen5Operand.Source(source0), Gen5Operand.Source(source1), ]; - control = new Gen5SdwaControl( - (extra >> 8) & 0x7, - (extra >> 16) & 0x7, - (extra >> 24) & 0x7, - ((extra >> 21) & 1) | (((extra >> 29) & 1) << 1), - ((extra >> 20) & 1) | (((extra >> 28) & 1) << 1), - (extra >> 14) & 0x3, - ((extra >> 13) & 1) != 0); + control = CreateSdwaControl(extra, isCompare: false, hasSource1: true); } else { @@ -1394,7 +1681,7 @@ public static class Gen5ShaderTranslator Gen5Operand.Source(word & 0x1FF, literal), Gen5Operand.Vector((word >> 9) & 0xFF), ]; - if (opcode is "VMadMkF32" or "VFmamkF32" && literal.HasValue) + if ((opcode is "VMadMkF32" or "VFmaMkF32") && literal.HasValue) { sources = [ @@ -1403,7 +1690,7 @@ public static class Gen5ShaderTranslator sources[1], ]; } - else if (opcode is "VMadAkF32" or "VFmaakF32" && literal.HasValue) + else if ((opcode is "VMadAkF32" or "VFmaAkF32") && literal.HasValue) { sources = [ @@ -1416,7 +1703,19 @@ public static class Gen5ShaderTranslator destinations = [Gen5Operand.Vector((word >> 17) & 0xFF)]; break; case Gen5ShaderEncoding.Vopc: - if (isDpp) + if (isDpp8) + { + var extra = words[1]; + sources = + [ + Gen5Operand.Vector(extra & 0xFF), + Gen5Operand.Vector((word >> 9) & 0xFF), + ]; + control = new Gen5Dpp8Control( + extra >> 8, + (word & 0x1FF) == 0xEA); + } + else if (isDpp) { var extra = words[1]; sources = @@ -1439,14 +1738,13 @@ public static class Gen5ShaderTranslator Gen5Operand.Source(source0), Gen5Operand.Source(source1), ]; - control = new Gen5SdwaControl( - (extra >> 8) & 0x7, - (extra >> 16) & 0x7, - (extra >> 24) & 0x7, - ((extra >> 21) & 1) | (((extra >> 29) & 1) << 1), - ((extra >> 20) & 1) | (((extra >> 28) & 1) << 1), - (extra >> 14) & 0x3, - ((extra >> 13) & 1) != 0); + var sdwa = CreateSdwaControl(extra, isCompare: true, hasSource1: true); + control = sdwa; + if (sdwa.ScalarDestination is { } scalarDestination && + scalarDestination != 106) + { + destinations = [Gen5Operand.Scalar(scalarDestination)]; + } } else { @@ -1479,6 +1777,7 @@ public static class Gen5ShaderTranslator (extra >> 29) & 0x7, (extra >> 27) & 0x3, ((word >> 15) & 1) != 0, + isVop3B ? 0 : (word >> 11) & 0xF, isVop3B ? (word >> 8) & 0x7F : null); break; } @@ -1495,6 +1794,10 @@ public static class Gen5ShaderTranslator ((word >> 17) & 1) != 0); sources = opcode switch { + "DsAddU32" => [ + Gen5Operand.Vector(vectorAddress), + Gen5Operand.Vector(vectorData0), + ], "DsWriteB32" => [ Gen5Operand.Vector(vectorAddress), Gen5Operand.Vector(vectorData0), @@ -1504,6 +1807,19 @@ public static class Gen5ShaderTranslator Gen5Operand.Vector(vectorData0), Gen5Operand.Vector(vectorData0 + 1), ], + "DsWriteB96" => [ + Gen5Operand.Vector(vectorAddress), + Gen5Operand.Vector(vectorData0), + Gen5Operand.Vector(vectorData0 + 1), + Gen5Operand.Vector(vectorData0 + 2), + ], + "DsWriteB128" => [ + Gen5Operand.Vector(vectorAddress), + Gen5Operand.Vector(vectorData0), + Gen5Operand.Vector(vectorData0 + 1), + Gen5Operand.Vector(vectorData0 + 2), + Gen5Operand.Vector(vectorData0 + 3), + ], "DsWrite2B32" or "DsWrite2St64B32" => [ Gen5Operand.Vector(vectorAddress), Gen5Operand.Vector(vectorData0), @@ -1521,6 +1837,17 @@ public static class Gen5ShaderTranslator Gen5Operand.Vector(vectorDestination), Gen5Operand.Vector(vectorDestination + 1), ], + "DsReadB96" => [ + Gen5Operand.Vector(vectorDestination), + Gen5Operand.Vector(vectorDestination + 1), + Gen5Operand.Vector(vectorDestination + 2), + ], + "DsReadB128" => [ + Gen5Operand.Vector(vectorDestination), + Gen5Operand.Vector(vectorDestination + 1), + Gen5Operand.Vector(vectorDestination + 2), + Gen5Operand.Vector(vectorDestination + 3), + ], _ => [], }; break; @@ -1540,10 +1867,30 @@ public static class Gen5ShaderTranslator var scalarAddress = (extra >> 16) & 0x7F; var dwordCount = opcode switch { + "GlobalLoadUbyte" or + "GlobalLoadSbyte" or + "GlobalLoadUshort" or + "GlobalLoadSshort" or + "GlobalLoadUbyteD16" or + "GlobalLoadUbyteD16Hi" or + "GlobalLoadSbyteD16" or + "GlobalLoadSbyteD16Hi" or + "GlobalLoadShortD16" or + "GlobalLoadShortD16Hi" or + "GlobalStoreByte" or + "GlobalStoreByteD16Hi" or + "GlobalStoreShort" or + "GlobalStoreShortD16Hi" or + "GlobalStoreDword" or + "GlobalAtomicAdd" or + "GlobalAtomicUMax" => 1u, "GlobalLoadDword" => 1u, "GlobalLoadDwordx2" => 2u, "GlobalLoadDwordx3" => 3u, "GlobalLoadDwordx4" => 4u, + "GlobalStoreDwordx2" => 2u, + "GlobalStoreDwordx3" => 3u, + "GlobalStoreDwordx4" => 4u, _ => 0u, }; sources = @@ -1551,10 +1898,12 @@ public static class Gen5ShaderTranslator Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(scalarAddress), ]; - destinations = Enumerable - .Range((int)vectorData, checked((int)dwordCount)) - .Select(index => Gen5Operand.Vector((uint)index)) - .ToArray(); + destinations = opcode.StartsWith("GlobalLoad", StringComparison.Ordinal) + ? Enumerable + .Range((int)vectorData, checked((int)dwordCount)) + .Select(index => Gen5Operand.Vector((uint)index)) + .ToArray() + : []; control = new Gen5GlobalMemoryControl( dwordCount, vectorAddress, @@ -1578,6 +1927,24 @@ public static class Gen5ShaderTranslator "BufferLoadFormatXy" => 2u, "BufferLoadFormatXyz" => 3u, "BufferLoadFormatXyzw" => 4u, + "BufferStoreFormatX" => 1u, + "BufferStoreFormatXy" => 2u, + "BufferStoreFormatXyz" => 3u, + "BufferStoreFormatXyzw" => 4u, + "BufferLoadUbyte" or + "BufferLoadSbyte" or + "BufferLoadUshort" or + "BufferLoadSshort" or + "BufferStoreByte" or + "BufferStoreByteD16Hi" or + "BufferStoreShort" or + "BufferStoreShortD16Hi" or + "BufferLoadUbyteD16" or + "BufferLoadUbyteD16Hi" or + "BufferLoadSbyteD16" or + "BufferLoadSbyteD16Hi" or + "BufferLoadShortD16" or + "BufferLoadShortD16Hi" => 1u, "BufferLoadDword" => 1u, "BufferLoadDwordx2" => 2u, "BufferLoadDwordx3" => 3u, @@ -1587,6 +1954,7 @@ public static class Gen5ShaderTranslator "BufferStoreDwordx3" => 3u, "BufferStoreDwordx4" => 4u, "BufferAtomicAdd" => 1u, + "BufferAtomicUMax" => 1u, _ => 0u, }; sources = @@ -1690,7 +2058,9 @@ public static class Gen5ShaderTranslator dimension, dimension is 4 or 5 or 7, ((word >> 13) & 1) != 0, - ((word >> 25) & 1) != 0); + ((word >> 25) & 1) != 0, + ((extra >> 30) & 1) != 0, + ((extra >> 31) & 1) != 0); break; } case Gen5ShaderEncoding.Exp: @@ -1726,6 +2096,30 @@ public static class Gen5ShaderTranslator (word >> 24) & 0xF, (word >> 28) & 0xF); + private static Gen5SdwaControl CreateSdwaControl( + uint word, + bool isCompare, + bool hasSource1) + { + var scalarDestination = isCompare + ? ((word >> 15) & 1) != 0 + ? (word >> 8) & 0x7Fu + : 106u + : (uint?)null; + return new Gen5SdwaControl( + isCompare ? 6u : (word >> 8) & 0x7u, + isCompare ? 0u : (word >> 11) & 0x3u, + (word >> 16) & 0x7u, + hasSource1 ? (word >> 24) & 0x7u : 6u, + ((word >> 19) & 1) != 0, + hasSource1 && ((word >> 27) & 1) != 0, + ((word >> 21) & 1) | (hasSource1 ? ((word >> 29) & 1) << 1 : 0), + ((word >> 20) & 1) | (hasSource1 ? ((word >> 28) & 1) << 1 : 0), + isCompare ? 0u : (word >> 14) & 0x3u, + !isCompare && ((word >> 13) & 1) != 0, + scalarDestination); + } + private static int MinimumEncodingDwords(Gen5ShaderEncoding encoding) => encoding switch { Gen5ShaderEncoding.Vop3 or @@ -1762,6 +2156,19 @@ public static class Gen5ShaderTranslator counts[key] = count + 1; } + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } + private readonly record struct ShaderDecodeInfo( int InstructionCount, Dictionary Counts, @@ -1824,6 +2231,7 @@ public static class Gen5ShaderTranslator $"va={addressRegisters},vd=v{image.VectorData}," + $"sr=s{image.ScalarResource},ss=s{image.ScalarSampler}," + $"dim={image.Dimension},da={(image.IsArray ? 1 : 0)}," + + $"a16={(image.A16 ? 1 : 0)},d16={(image.D16 ? 1 : 0)}," + $"glc={(image.Glc ? 1 : 0)}," + $"slc={(image.Slc ? 1 : 0)}"; } diff --git a/src/SharpEmu.ShaderCompiler/Gfx10UnifiedFormat.cs b/src/SharpEmu.ShaderCompiler/Gfx10UnifiedFormat.cs new file mode 100644 index 0000000..146ff0d --- /dev/null +++ b/src/SharpEmu.ShaderCompiler/Gfx10UnifiedFormat.cs @@ -0,0 +1,138 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler; + +/// +/// Converts the RDNA2 unified FORMAT field used by GFX10 buffer and image +/// descriptors into the data-format/number-format pair used by the rest of +/// SharpEmu's AGC pipeline. +/// +public static class Gfx10UnifiedFormat +{ + public static bool TryDecode( + uint unifiedFormat, + out uint dataFormat, + out uint numberFormat) + { + // RDNA2 ISA table 47 is intentionally sparse. In particular, the + // integer/normalized spellings that some assemblers expose for + // 10_11_11 and 11_11_10 are reserved by the hardware data-format + // table; so are 10_10_10_2 USCALED/SSCALED. Keep this an exact table + // instead of deriving ranges that accidentally make reserved encodings + // look valid. + (dataFormat, numberFormat) = unifiedFormat switch + { + 0 => (0u, 0u), + 1 => (1u, 0u), + 2 => (1u, 1u), + 3 => (1u, 2u), + 4 => (1u, 3u), + 5 => (1u, 4u), + 6 => (1u, 5u), + 7 => (2u, 0u), + 8 => (2u, 1u), + 9 => (2u, 2u), + 10 => (2u, 3u), + 11 => (2u, 4u), + 12 => (2u, 5u), + 13 => (2u, 7u), + 14 => (3u, 0u), + 15 => (3u, 1u), + 16 => (3u, 2u), + 17 => (3u, 3u), + 18 => (3u, 4u), + 19 => (3u, 5u), + 20 => (4u, 4u), + 21 => (4u, 5u), + 22 => (4u, 7u), + 23 => (5u, 0u), + 24 => (5u, 1u), + 25 => (5u, 2u), + 26 => (5u, 3u), + 27 => (5u, 4u), + 28 => (5u, 5u), + 29 => (5u, 7u), + 36 => (6u, 7u), + 43 => (7u, 7u), + 44 => (8u, 0u), + 45 => (8u, 1u), + 48 => (8u, 4u), + 49 => (8u, 5u), + 50 => (9u, 0u), + 51 => (9u, 1u), + 52 => (9u, 2u), + 53 => (9u, 3u), + 54 => (9u, 4u), + 55 => (9u, 5u), + 56 => (10u, 0u), + 57 => (10u, 1u), + 58 => (10u, 2u), + 59 => (10u, 3u), + 60 => (10u, 4u), + 61 => (10u, 5u), + 62 => (11u, 4u), + 63 => (11u, 5u), + 64 => (11u, 7u), + 65 => (12u, 0u), + 66 => (12u, 1u), + 67 => (12u, 2u), + 68 => (12u, 3u), + 69 => (12u, 4u), + 70 => (12u, 5u), + 71 => (12u, 7u), + 72 => (13u, 4u), + 73 => (13u, 5u), + 74 => (13u, 7u), + 75 => (14u, 4u), + 76 => (14u, 5u), + 77 => (14u, 7u), + 128 => (1u, 9u), + 129 => (3u, 9u), + 130 => (10u, 9u), + // Image-only encodings without a legacy DATA_FORMAT equivalent + // retain the unified identifier for exact downstream dispatch. + 131 => (131u, 7u), + 132 => (34u, 7u), + 133 => (16u, 0u), + 134 => (17u, 0u), + 135 => (18u, 0u), + 136 => (19u, 0u), + 137 => (137u, 0u), + 138 => (138u, 0u), + 139 => (139u, 0u), + 140 => (4u, 7u), + 141 => (20u, 0u), + 142 => (20u, 4u), + 143 => (21u, 0u), + 144 => (21u, 4u), + 145 => (22u, 4u), + 146 => (22u, 7u), + 147 => (32u, 0u), + 148 => (32u, 1u), + 149 => (32u, 4u), + 150 => (32u, 9u), + 151 => (33u, 0u), + 152 => (33u, 1u), + 153 => (33u, 4u), + 154 => (33u, 9u), + 169 => (169u, 0u), + 170 => (170u, 9u), + 171 => (171u, 0u), + 172 => (172u, 9u), + 173 => (173u, 0u), + 174 => (174u, 9u), + 175 => (175u, 0u), + 176 => (176u, 1u), + 177 => (177u, 0u), + 178 => (178u, 1u), + 179 => (179u, 7u), + 180 => (180u, 7u), + 181 => (181u, 0u), + 182 => (182u, 9u), + _ => (0u, 0u), + }; + + return unifiedFormat == 0 || dataFormat != 0; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Tls/GuestTlsTemplateTests.cs b/tests/SharpEmu.Libs.Tests/Tls/GuestTlsTemplateTests.cs new file mode 100644 index 0000000..db00bf1 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Tls/GuestTlsTemplateTests.cs @@ -0,0 +1,32 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using Xunit; + +namespace SharpEmu.Libs.Tests.Tls; + +public sealed class GuestTlsTemplateTests +{ + [Fact] + public void StartupReservationAcceptsTlsSpansLargerThanOneHostPage() + { + try + { + GuestTlsTemplate.Reset(); + + var staticOffset = GuestTlsTemplate.RegisterModule( + moduleId: 1, + initImage: new byte[0x20], + memorySize: 0x1870, + alignment: 0x10); + + Assert.Equal(0x1870UL, staticOffset); + Assert.True(staticOffset <= GuestTlsTemplate.StartupStaticTlsReservation); + } + finally + { + GuestTlsTemplate.Reset(); + } + } +} diff --git a/tools/SharpEmu.Tools.ShaderDump/Program.cs b/tools/SharpEmu.Tools.ShaderDump/Program.cs index 5da8bcf..47612f1 100644 --- a/tools/SharpEmu.Tools.ShaderDump/Program.cs +++ b/tools/SharpEmu.Tools.ShaderDump/Program.cs @@ -192,14 +192,22 @@ foreach (var (name, expectTranslate, words) in testPrograms) // the hand-assembled buffer_store words, and the 64-byte backing store // must cover every hand-assembled store offset. var globalBindings = storePcs.Count > 0 - ? new[] { new Gen5GlobalMemoryBinding(8u, 0UL, storePcs, new byte[64]) } + ? new[] + { + new Gen5GlobalMemoryBinding( + 8u, + 0UL, + storePcs, + new byte[64], + 64, + false), + } : Array.Empty(); var state = new Gen5ShaderState(program, new uint[16], Metadata: null); var evaluation = new Gen5ShaderEvaluation( new uint[256], new uint[256], - new Dictionary>(), Array.Empty(), globalBindings);