diff --git a/Source/Core/Common/MemArena.h b/Source/Core/Common/MemArena.h index dd07f703989..1a3093cc4d4 100644 --- a/Source/Core/Common/MemArena.h +++ b/Source/Core/Common/MemArena.h @@ -102,10 +102,22 @@ public: /// from. /// @param size Size of the region to map. /// @param base Address within the memory region from ReserveMemoryRegion() where to map it. + /// @param writeable Whether the region should be both readable and writeable, or just readable. /// /// @return The address we actually ended up mapping, which should be the given 'base'. /// - void* MapInMemoryRegion(s64 offset, size_t size, void* base); + void* MapInMemoryRegion(s64 offset, size_t size, void* base, bool writeable); + + /// + /// Changes whether a section mapped by MapInMemoryRegion is writeable. + /// + /// @param view The address returned by MapInMemoryRegion. + /// @param size The size passed to MapInMemoryRegion. + /// @param writeable Whether the region should be both readable and writeable, or just readable. + /// + /// @return Whether the operation succeeded. + /// + bool ChangeMappingProtection(void* view, size_t size, bool writeable); /// /// Unmap a memory region previously mapped with MapInMemoryRegion(). @@ -115,6 +127,11 @@ public: /// void UnmapFromMemoryRegion(void* view, size_t size); + /// + /// Return the system's page size or required page alignment, whichever is larger. + /// + size_t GetPageSize() const; + private: #ifdef _WIN32 WindowsMemoryRegion* EnsureSplitRegionForMapping(void* address, size_t size); diff --git a/Source/Core/Common/MemArenaAndroid.cpp b/Source/Core/Common/MemArenaAndroid.cpp index bcba64e1e1f..81c27b7677c 100644 --- a/Source/Core/Common/MemArenaAndroid.cpp +++ b/Source/Core/Common/MemArenaAndroid.cpp @@ -123,9 +123,13 @@ void MemArena::ReleaseMemoryRegion() } } -void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) +void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base, bool writeable) { - void* retval = mmap(base, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, m_shm_fd, offset); + int prot = PROT_READ; + if (writeable) + prot |= PROT_WRITE; + + void* retval = mmap(base, size, prot, MAP_SHARED | MAP_FIXED, m_shm_fd, offset); if (retval == MAP_FAILED) { NOTICE_LOG_FMT(MEMMAP, "mmap failed"); @@ -137,6 +141,18 @@ void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) } } +bool MemArena::ChangeMappingProtection(void* view, size_t size, bool writeable) +{ + int prot = PROT_READ; + if (writeable) + prot |= PROT_WRITE; + + int retval = mprotect(view, size, prot); + if (retval != 0) + NOTICE_LOG_FMT(MEMMAP, "mprotect failed"); + return retval == 0; +} + void MemArena::UnmapFromMemoryRegion(void* view, size_t size) { void* retval = mmap(view, size, PROT_NONE, MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, -1, 0); @@ -144,6 +160,11 @@ void MemArena::UnmapFromMemoryRegion(void* view, size_t size) NOTICE_LOG_FMT(MEMMAP, "mmap failed"); } +size_t MemArena::GetPageSize() const +{ + return sysconf(_SC_PAGESIZE); +} + LazyMemoryRegion::LazyMemoryRegion() = default; LazyMemoryRegion::~LazyMemoryRegion() diff --git a/Source/Core/Common/MemArenaDarwin.cpp b/Source/Core/Common/MemArenaDarwin.cpp index c528e327990..9414535be58 100644 --- a/Source/Core/Common/MemArenaDarwin.cpp +++ b/Source/Core/Common/MemArenaDarwin.cpp @@ -3,6 +3,8 @@ #include "Common/MemArena.h" +#include + #include "Common/Assert.h" #include "Common/Logging/Log.h" @@ -121,7 +123,7 @@ void MemArena::ReleaseMemoryRegion() m_region_size = 0; } -void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) +void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base, bool writeable) { if (m_shm_address == 0) { @@ -130,11 +132,13 @@ void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) } vm_address_t address = reinterpret_cast(base); - constexpr vm_prot_t prot = VM_PROT_READ | VM_PROT_WRITE; + vm_prot_t prot = VM_PROT_READ; + if (writeable) + prot |= VM_PROT_WRITE; kern_return_t retval = vm_map(mach_task_self(), &address, size, 0, VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, m_shm_entry, - offset, false, prot, prot, VM_INHERIT_DEFAULT); + offset, false, prot, VM_PROT_READ | VM_PROT_WRITE, VM_INHERIT_DEFAULT); if (retval != KERN_SUCCESS) { ERROR_LOG_FMT(MEMMAP, "MapInMemoryRegion failed: vm_map returned {0:#x}", retval); @@ -144,6 +148,20 @@ void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) return reinterpret_cast(address); } +bool MemArena::ChangeMappingProtection(void* view, size_t size, bool writeable) +{ + vm_address_t address = reinterpret_cast(view); + vm_prot_t prot = VM_PROT_READ; + if (writeable) + prot |= VM_PROT_WRITE; + + kern_return_t retval = vm_protect(mach_task_self(), address, size, false, prot); + if (retval != KERN_SUCCESS) + ERROR_LOG_FMT(MEMMAP, "ChangeMappingProtection failed: vm_protect returned {0:#x}", retval); + + return retval == KERN_SUCCESS; +} + void MemArena::UnmapFromMemoryRegion(void* view, size_t size) { vm_address_t address = reinterpret_cast(view); @@ -163,6 +181,11 @@ void MemArena::UnmapFromMemoryRegion(void* view, size_t size) } } +size_t MemArena::GetPageSize() const +{ + return getpagesize(); +} + LazyMemoryRegion::LazyMemoryRegion() = default; LazyMemoryRegion::~LazyMemoryRegion() diff --git a/Source/Core/Common/MemArenaUnix.cpp b/Source/Core/Common/MemArenaUnix.cpp index c25d2357f9c..fd4af4e74b3 100644 --- a/Source/Core/Common/MemArenaUnix.cpp +++ b/Source/Core/Common/MemArenaUnix.cpp @@ -87,9 +87,13 @@ void MemArena::ReleaseMemoryRegion() } } -void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) +void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base, bool writeable) { - void* retval = mmap(base, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, m_shm_fd, offset); + int prot = PROT_READ; + if (writeable) + prot |= PROT_WRITE; + + void* retval = mmap(base, size, prot, MAP_SHARED | MAP_FIXED, m_shm_fd, offset); if (retval == MAP_FAILED) { NOTICE_LOG_FMT(MEMMAP, "mmap failed"); @@ -101,6 +105,18 @@ void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) } } +bool MemArena::ChangeMappingProtection(void* view, size_t size, bool writeable) +{ + int prot = PROT_READ; + if (writeable) + prot |= PROT_WRITE; + + int retval = mprotect(view, size, prot); + if (retval != 0) + NOTICE_LOG_FMT(MEMMAP, "mprotect failed"); + return retval == 0; +} + void MemArena::UnmapFromMemoryRegion(void* view, size_t size) { void* retval = mmap(view, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); @@ -108,6 +124,11 @@ void MemArena::UnmapFromMemoryRegion(void* view, size_t size) NOTICE_LOG_FMT(MEMMAP, "mmap failed"); } +size_t MemArena::GetPageSize() const +{ + return sysconf(_SC_PAGESIZE); +} + LazyMemoryRegion::LazyMemoryRegion() = default; LazyMemoryRegion::~LazyMemoryRegion() diff --git a/Source/Core/Common/MemArenaWin.cpp b/Source/Core/Common/MemArenaWin.cpp index 79079797753..adbfab3d82c 100644 --- a/Source/Core/Common/MemArenaWin.cpp +++ b/Source/Core/Common/MemArenaWin.cpp @@ -318,8 +318,10 @@ WindowsMemoryRegion* MemArena::EnsureSplitRegionForMapping(void* start_address, } } -void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) +void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base, bool writeable) { + void* result; + if (m_memory_functions.m_api_ms_win_core_memory_l1_1_6_handle.IsOpen()) { WindowsMemoryRegion* const region = EnsureSplitRegionForMapping(base, size); @@ -329,10 +331,10 @@ void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) return nullptr; } - void* rv = static_cast(m_memory_functions.m_address_MapViewOfFile3)( + result = static_cast(m_memory_functions.m_address_MapViewOfFile3)( m_memory_handle, nullptr, base, offset, size, MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0); - if (rv) + if (result) { region->m_is_mapped = true; } @@ -342,11 +344,37 @@ void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base) // revert the split, if any JoinRegionsAfterUnmap(base, size); + + return nullptr; } - return rv; + } + else + { + result = + MapViewOfFileEx(m_memory_handle, FILE_MAP_ALL_ACCESS, 0, (DWORD)((u64)offset), size, base); + + if (!result) + return nullptr; } - return MapViewOfFileEx(m_memory_handle, FILE_MAP_ALL_ACCESS, 0, (DWORD)((u64)offset), size, base); + if (!writeable) + { + // If we want to use PAGE_READONLY for now while still being able to switch to PAGE_READWRITE + // later, we have to call MapViewOfFile with PAGE_READWRITE and then switch to PAGE_READONLY. + ChangeMappingProtection(base, size, writeable); + } + + return result; +} + +bool MemArena::ChangeMappingProtection(void* view, size_t size, bool writeable) +{ + DWORD old_protect; + const int retval = + VirtualProtect(view, size, writeable ? PAGE_READWRITE : PAGE_READONLY, &old_protect); + if (retval == 0) + PanicAlertFmt("VirtualProtect failed: {}", GetLastErrorString()); + return retval != 0; } bool MemArena::JoinRegionsAfterUnmap(void* start_address, size_t size) @@ -438,6 +466,21 @@ void MemArena::UnmapFromMemoryRegion(void* view, size_t size) UnmapViewOfFile(view); } +size_t MemArena::GetPageSize() const +{ + SYSTEM_INFO si; + GetSystemInfo(&si); + + if (!m_memory_functions.m_address_MapViewOfFile3) + { + // In this case, we can only map pages that are 64K aligned. + // See https://devblogs.microsoft.com/oldnewthing/20031008-00/?p=42223 + return std::max(si.dwPageSize, 64 * 1024); + } + + return si.dwPageSize; +} + LazyMemoryRegion::LazyMemoryRegion() { InitWindowsMemoryFunctions(&m_memory_functions); diff --git a/Source/Core/Common/Swap.h b/Source/Core/Common/Swap.h index 965d1aabc2b..bf9b7d472b1 100644 --- a/Source/Core/Common/Swap.h +++ b/Source/Core/Common/Swap.h @@ -127,6 +127,22 @@ inline u64 swap64(const u8* data) return swap64(value); } +inline void WriteSwap16(u8* data, u16 value) +{ + value = swap16(value); + std::memcpy(data, &value, sizeof(u16)); +} +inline void WriteSwap32(u8* data, u32 value) +{ + value = swap32(value); + std::memcpy(data, &value, sizeof(u32)); +} +inline void WriteSwap64(u8* data, u64 value) +{ + value = swap64(value); + std::memcpy(data, &value, sizeof(u64)); +} + template void swap(u8*); diff --git a/Source/Core/Core/HW/HW.cpp b/Source/Core/Core/HW/HW.cpp index 72b8233a886..58b4beac036 100644 --- a/Source/Core/Core/HW/HW.cpp +++ b/Source/Core/Core/HW/HW.cpp @@ -57,6 +57,8 @@ void Init(Core::System& system, const Sram* override_sram) system.GetWiiIPC().Init(); IOS::HLE::Init(system); // Depends on Memory } + + system.GetMemory().InitMMIO(system); } void Shutdown(Core::System& system) diff --git a/Source/Core/Core/HW/Memmap.cpp b/Source/Core/Core/HW/Memmap.cpp index 6556199a0a7..c7ec1debb05 100644 --- a/Source/Core/Core/HW/Memmap.cpp +++ b/Source/Core/Core/HW/Memmap.cpp @@ -10,8 +10,11 @@ #include #include +#include #include +#include #include +#include #include #include @@ -41,33 +44,44 @@ namespace Memory { -MemoryManager::MemoryManager(Core::System& system) : m_system(system) +MemoryManager::MemoryManager(Core::System& system) + : m_page_size(static_cast(m_arena.GetPageSize())), + m_guest_pages_per_host_page(m_page_size / PowerPC::HW_PAGE_SIZE), + m_host_page_type(GetHostPageTypeForPageSize(m_page_size)), m_system(system) { } MemoryManager::~MemoryManager() = default; -void MemoryManager::InitMMIO(bool is_wii) +MemoryManager::HostPageType MemoryManager::GetHostPageTypeForPageSize(u32 page_size) +{ + if (!std::has_single_bit(page_size)) + return HostPageType::Unsupported; + + return page_size > PowerPC::HW_PAGE_SIZE ? HostPageType::LargePages : HostPageType::SmallPages; +} + +void MemoryManager::InitMMIO(Core::System& system) { m_mmio_mapping = std::make_unique(); - m_system.GetCommandProcessor().RegisterMMIO(m_mmio_mapping.get(), 0x0C000000); - m_system.GetPixelEngine().RegisterMMIO(m_mmio_mapping.get(), 0x0C001000); - m_system.GetVideoInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C002000); - m_system.GetProcessorInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C003000); - m_system.GetMemoryInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C004000); - m_system.GetDSP().RegisterMMIO(m_mmio_mapping.get(), 0x0C005000); - m_system.GetDVDInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C006000, false); - m_system.GetSerialInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C006400); - m_system.GetExpansionInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C006800); - m_system.GetAudioInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C006C00); - if (is_wii) + system.GetCommandProcessor().RegisterMMIO(m_mmio_mapping.get(), 0x0C000000); + system.GetPixelEngine().RegisterMMIO(m_mmio_mapping.get(), 0x0C001000); + system.GetVideoInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C002000); + system.GetProcessorInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C003000); + system.GetMemoryInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C004000); + system.GetDSP().RegisterMMIO(m_mmio_mapping.get(), 0x0C005000); + system.GetDVDInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C006000, false); + system.GetSerialInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C006400); + system.GetExpansionInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C006800); + system.GetAudioInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0C006C00); + if (system.IsWii()) { - m_system.GetWiiIPC().RegisterMMIO(m_mmio_mapping.get(), 0x0D000000); - m_system.GetDVDInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0D006000, true); - m_system.GetSerialInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0D006400); - m_system.GetExpansionInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0D006800); - m_system.GetAudioInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0D006C00); + system.GetWiiIPC().RegisterMMIO(m_mmio_mapping.get(), 0x0D000000); + system.GetDVDInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0D006000, true); + system.GetSerialInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0D006400); + system.GetExpansionInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0D006800); + system.GetAudioInterface().RegisterMMIO(m_mmio_mapping.get(), 0x0D006C00); } } @@ -151,8 +165,6 @@ void MemoryManager::Init() m_physical_page_mappings_base = reinterpret_cast(m_physical_page_mappings.data()); m_logical_page_mappings_base = reinterpret_cast(m_logical_page_mappings.data()); - InitMMIO(wii); - Clear(); INFO_LOG_FMT(MEMMAP, "Memory system initialized. RAM at {}", fmt::ptr(m_ram)); @@ -216,8 +228,8 @@ bool MemoryManager::InitFastmemArena() if (!region.active) continue; - u8* base = m_physical_base + region.physical_address; - u8* view = (u8*)m_arena.MapInMemoryRegion(region.shm_position, region.size, base); + void* base = m_physical_base + region.physical_address; + void* view = m_arena.MapInMemoryRegion(region.shm_position, region.size, base, true); if (base != view) { @@ -233,13 +245,15 @@ bool MemoryManager::InitFastmemArena() return true; } -void MemoryManager::UpdateLogicalMemory(const PowerPC::BatTable& dbat_table) +void MemoryManager::UpdateDBATMappings(const PowerPC::BatTable& dbat_table) { - for (auto& entry : m_logical_mapped_entries) + for (const auto& [logical_address, entry] : m_dbat_mapped_entries) { m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size); } - m_logical_mapped_entries.clear(); + m_dbat_mapped_entries.clear(); + + RemoveAllPageTableMappings(); m_logical_page_mappings.fill(nullptr); @@ -285,16 +299,16 @@ void MemoryManager::UpdateLogicalMemory(const PowerPC::BatTable& dbat_table) u8* base = m_logical_base + logical_address + intersection_start - translated_address; u32 mapped_size = intersection_end - intersection_start; - void* mapped_pointer = m_arena.MapInMemoryRegion(position, mapped_size, base); + void* mapped_pointer = m_arena.MapInMemoryRegion(position, mapped_size, base, true); if (!mapped_pointer) { - PanicAlertFmt( - "Memory::UpdateLogicalMemory(): Failed to map memory region at 0x{:08X} " - "(size 0x{:08X}) into logical fastmem region at 0x{:08X}.", - intersection_start, mapped_size, logical_address); - exit(0); + PanicAlertFmt("Memory::UpdateDBATMappings(): Failed to map memory region at 0x{:08X} " + "(size 0x{:08X}) into logical fastmem region at 0x{:08X}.", + intersection_start, mapped_size, logical_address); + continue; } - m_logical_mapped_entries.push_back({mapped_pointer, mapped_size}); + m_dbat_mapped_entries.emplace(logical_address, + LogicalMemoryView{mapped_pointer, mapped_size}); } m_logical_page_mappings[i] = @@ -305,6 +319,184 @@ void MemoryManager::UpdateLogicalMemory(const PowerPC::BatTable& dbat_table) } } +void MemoryManager::AddPageTableMapping(u32 logical_address, u32 translated_address, bool writeable) +{ + if (!m_is_fastmem_arena_initialized) + return; + + switch (m_host_page_type) + { + case HostPageType::SmallPages: + return AddHostPageTableMapping(logical_address, translated_address, writeable, + PowerPC::HW_PAGE_SIZE); + case HostPageType::LargePages: + return TryAddLargePageTableMapping(logical_address, translated_address, writeable); + default: + return; + } +} + +void MemoryManager::TryAddLargePageTableMapping(u32 logical_address, u32 translated_address, + bool writeable) +{ + const bool add_readable = + TryAddLargePageTableMapping(logical_address, translated_address, m_large_readable_pages); + + const bool add_writeable = + writeable && + TryAddLargePageTableMapping(logical_address, translated_address, m_large_writeable_pages); + + if (add_readable || add_writeable) + { + AddHostPageTableMapping(logical_address & ~(m_page_size - 1), + translated_address & ~(m_page_size - 1), add_writeable, m_page_size); + } +} + +bool MemoryManager::TryAddLargePageTableMapping(u32 logical_address, u32 translated_address, + std::map>& map) +{ + std::vector& entries = map[logical_address & ~(m_page_size - 1)]; + + if (entries.empty()) + entries = std::vector(m_guest_pages_per_host_page, INVALID_MAPPING); + + entries[(logical_address & (m_page_size - 1)) / PowerPC::HW_PAGE_SIZE] = translated_address; + + return CanCreateHostMappingForGuestPages(entries); +} + +bool MemoryManager::CanCreateHostMappingForGuestPages(const std::vector& entries) const +{ + const u32 translated_address = entries[0]; + if ((translated_address & (m_page_size - 1)) != 0) + return false; + + for (size_t i = 1; i < m_guest_pages_per_host_page; ++i) + { + if (entries[i] != translated_address + i * PowerPC::HW_PAGE_SIZE) + return false; + } + + return true; +} + +void MemoryManager::AddHostPageTableMapping(u32 logical_address, u32 translated_address, + bool writeable, u32 logical_size) +{ + for (const auto& physical_region : m_physical_regions) + { + if (!physical_region.active) + continue; + + const u32 mapping_address = physical_region.physical_address; + const u32 mapping_end = mapping_address + physical_region.size; + const u32 intersection_start = std::max(mapping_address, translated_address); + const u32 intersection_end = std::min(mapping_end, translated_address + logical_size); + if (intersection_start >= intersection_end) + continue; + + // Found an overlapping region; map it. + const u32 position = physical_region.shm_position + intersection_start - mapping_address; + u8* const base = m_logical_base + logical_address + intersection_start - translated_address; + const u32 mapped_size = intersection_end - intersection_start; + + const auto it = m_page_table_mapped_entries.find(logical_address); + if (it != m_page_table_mapped_entries.end()) + { + // Update the protection of an existing mapping. + if (it->second.mapped_pointer == base && it->second.mapped_size == mapped_size) + { + if (!m_arena.ChangeMappingProtection(base, mapped_size, writeable)) + { + PanicAlertFmt("Memory::AddPageTableMapping(): Failed to change protection for memory " + "region at 0x{:08X} (size 0x{:08X}, logical fastmem region at 0x{:08X}).", + intersection_start, mapped_size, logical_address); + } + } + } + else + { + // Create a new mapping. + void* const mapped_pointer = + m_arena.MapInMemoryRegion(position, mapped_size, base, writeable); + if (!mapped_pointer) + { + PanicAlertFmt("Memory::AddPageTableMapping(): Failed to map memory region at 0x{:08X} " + "(size 0x{:08X}) into logical fastmem region at 0x{:08X}.", + intersection_start, mapped_size, logical_address); + continue; + } + m_page_table_mapped_entries.emplace(logical_address, + LogicalMemoryView{mapped_pointer, mapped_size}); + } + } +} + +void MemoryManager::RemovePageTableMappings(const std::set& mappings) +{ + switch (m_host_page_type) + { + case HostPageType::SmallPages: + return RemoveHostPageTableMappings(mappings); + case HostPageType::LargePages: + for (u32 logical_address : mappings) + RemoveLargePageTableMapping(logical_address); + return; + default: + return; + } +} + +void MemoryManager::RemoveLargePageTableMapping(u32 logical_address) +{ + RemoveLargePageTableMapping(logical_address, m_large_readable_pages); + RemoveLargePageTableMapping(logical_address, m_large_writeable_pages); + + const u32 aligned_logical_address = logical_address & ~(m_page_size - 1); + const auto it = m_page_table_mapped_entries.find(aligned_logical_address); + if (it != m_page_table_mapped_entries.end()) + { + const LogicalMemoryView& entry = it->second; + m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size); + + m_page_table_mapped_entries.erase(it); + } +} + +void MemoryManager::RemoveLargePageTableMapping(u32 logical_address, + std::map>& map) +{ + const auto it = map.find(logical_address & ~(m_page_size - 1)); + if (it != map.end()) + it->second[(logical_address & (m_page_size - 1)) / PowerPC::HW_PAGE_SIZE] = INVALID_MAPPING; +} + +void MemoryManager::RemoveHostPageTableMappings(const std::set& mappings) +{ + if (mappings.empty()) + return; + + std::erase_if(m_page_table_mapped_entries, [this, &mappings](const auto& pair) { + const auto& [logical_address, entry] = pair; + const bool remove = mappings.contains(logical_address); + if (remove) + m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size); + return remove; + }); +} + +void MemoryManager::RemoveAllPageTableMappings() +{ + for (const auto& [logical_address, entry] : m_page_table_mapped_entries) + { + m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size); + } + m_page_table_mapped_entries.clear(); + m_large_readable_pages.clear(); + m_large_writeable_pages.clear(); +} + void MemoryManager::DoState(PointerWrap& p) { const u32 current_ram_size = GetRamSize(); @@ -386,14 +578,23 @@ void MemoryManager::ShutdownFastmemArena() m_arena.UnmapFromMemoryRegion(base, region.size); } - for (auto& entry : m_logical_mapped_entries) + for (const auto& [logical_address, entry] : m_dbat_mapped_entries) { m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size); } - m_logical_mapped_entries.clear(); + m_dbat_mapped_entries.clear(); + + for (const auto& [logical_address, entry] : m_page_table_mapped_entries) + { + m_arena.UnmapFromMemoryRegion(entry.mapped_pointer, entry.mapped_size); + } + m_page_table_mapped_entries.clear(); m_arena.ReleaseMemoryRegion(); + m_large_readable_pages.clear(); + m_large_writeable_pages.clear(); + m_fastmem_arena = nullptr; m_fastmem_arena_size = 0; m_physical_base = nullptr; diff --git a/Source/Core/Core/HW/Memmap.h b/Source/Core/Core/HW/Memmap.h index 596abfa3d69..ce45d730346 100644 --- a/Source/Core/Core/HW/Memmap.h +++ b/Source/Core/Core/HW/Memmap.h @@ -4,7 +4,9 @@ #pragma once #include +#include #include +#include #include #include #include @@ -83,6 +85,8 @@ public: u8* GetPhysicalPageMappingsBase() const { return m_physical_page_mappings_base; } u8* GetLogicalPageMappingsBase() const { return m_logical_page_mappings_base; } + u32 GetHostPageSize() const { return m_page_size; } + // FIXME: these should not return their address, but AddressSpace wants that u8*& GetRAM() { return m_ram; } u8*& GetEXRAM() { return m_exram; } @@ -93,13 +97,17 @@ public: // Init and Shutdown bool IsInitialized() const { return m_is_initialized; } + void InitMMIO(Core::System& system); void Init(); void Shutdown(); bool InitFastmemArena(); void ShutdownFastmemArena(); void DoState(PointerWrap& p); - void UpdateLogicalMemory(const PowerPC::BatTable& dbat_table); + void UpdateDBATMappings(const PowerPC::BatTable& dbat_table); + void AddPageTableMapping(u32 logical_address, u32 translated_address, bool writeable); + void RemovePageTableMappings(const std::set& mappings); + void RemoveAllPageTableMappings(); void Clear(); @@ -157,6 +165,16 @@ public: } private: + enum class HostPageType + { + // 4K or smaller + SmallPages, + // 8K or larger + LargePages, + // Required APIs aren't available, or the page size isn't a power of 2 + Unsupported, + }; + // Base is a pointer to the base of the memory map. Yes, some MMU tricks // are used to set up a full GC or Wii memory map in process memory. // In 64-bit, this might point to "high memory" (above the 32-bit limit), @@ -208,6 +226,10 @@ private: // The MemArena class Common::MemArena m_arena; + const u32 m_page_size; + const u32 m_guest_pages_per_host_page; + const HostPageType m_host_page_type; + // Dolphin allocates memory to represent four regions: // - 32MB RAM (actually 24MB on hardware), available on GameCube and Wii // - 64MB "EXRAM", RAM only available on Wii @@ -248,13 +270,31 @@ private: // TODO: Do we want to handle the mirrors of the GC RAM? std::array m_physical_regions{}; - std::vector m_logical_mapped_entries; + // The key is the logical address + std::map m_dbat_mapped_entries; + std::map m_page_table_mapped_entries; std::array m_physical_page_mappings{}; std::array m_logical_page_mappings{}; + // If the host page size is larger than the guest page size, these two maps are used + // to keep track of which guest pages can be combined and mapped as one host page. + static constexpr u32 INVALID_MAPPING = 0xFFFFFFFF; + std::map> m_large_readable_pages; + std::map> m_large_writeable_pages; + Core::System& m_system; - void InitMMIO(bool is_wii); + static HostPageType GetHostPageTypeForPageSize(u32 page_size); + + void TryAddLargePageTableMapping(u32 logical_address, u32 translated_address, bool writeable); + bool TryAddLargePageTableMapping(u32 logical_address, u32 translated_address, + std::map>& map); + bool CanCreateHostMappingForGuestPages(const std::vector& entries) const; + void AddHostPageTableMapping(u32 logical_address, u32 translated_address, bool writeable, + u32 logical_size); + void RemoveLargePageTableMapping(u32 logical_address); + void RemoveLargePageTableMapping(u32 logical_address, std::map>& map); + void RemoveHostPageTableMappings(const std::set& mappings); }; } // namespace Memory diff --git a/Source/Core/Core/PowerPC/GDBStub.cpp b/Source/Core/Core/PowerPC/GDBStub.cpp index ecf3ff09da0..088a74c9263 100644 --- a/Source/Core/Core/PowerPC/GDBStub.cpp +++ b/Source/Core/Core/PowerPC/GDBStub.cpp @@ -31,6 +31,7 @@ typedef SSIZE_T ssize_t; #include "Core/Host.h" #include "Core/PowerPC/BreakPoints.h" #include "Core/PowerPC/Gekko.h" +#include "Core/PowerPC/MMU.h" #include "Core/PowerPC/PPCCache.h" #include "Core/PowerPC/PowerPC.h" #include "Core/System.h" @@ -643,6 +644,7 @@ static void WriteRegister() else if (id >= 71 && id < 87) { ppc_state.sr[id - 71] = re32hex(bufptr); + system.GetMMU().SRUpdated(); } else if (id >= 88 && id < 104) { diff --git a/Source/Core/Core/PowerPC/Interpreter/Interpreter_SystemRegisters.cpp b/Source/Core/Core/PowerPC/Interpreter/Interpreter_SystemRegisters.cpp index e632ab403d2..8f30be84f4f 100644 --- a/Source/Core/Core/PowerPC/Interpreter/Interpreter_SystemRegisters.cpp +++ b/Source/Core/Core/PowerPC/Interpreter/Interpreter_SystemRegisters.cpp @@ -203,7 +203,8 @@ void Interpreter::mtsr(Interpreter& interpreter, UGeckoInstruction inst) const u32 index = inst.SR; const u32 value = ppc_state.gpr[inst.RS]; - ppc_state.SetSR(index, value); + ppc_state.sr[index] = value; + interpreter.m_system.GetMMU().SRUpdated(); } void Interpreter::mtsrin(Interpreter& interpreter, UGeckoInstruction inst) @@ -217,7 +218,8 @@ void Interpreter::mtsrin(Interpreter& interpreter, UGeckoInstruction inst) const u32 index = (ppc_state.gpr[inst.RB] >> 28) & 0xF; const u32 value = ppc_state.gpr[inst.RS]; - ppc_state.SetSR(index, value); + ppc_state.sr[index] = value; + interpreter.m_system.GetMMU().SRUpdated(); } void Interpreter::mftb(Interpreter& interpreter, UGeckoInstruction inst) diff --git a/Source/Core/Core/PowerPC/Jit64/Jit.cpp b/Source/Core/Core/PowerPC/Jit64/Jit.cpp index 0a7341501a6..78bda8451b9 100644 --- a/Source/Core/Core/PowerPC/Jit64/Jit.cpp +++ b/Source/Core/Core/PowerPC/Jit64/Jit.cpp @@ -513,6 +513,8 @@ void Jit64::MSRUpdated(const OpArg& msr, X64Reg scratch_reg) { ASSERT(!msr.IsSimpleReg(scratch_reg)); + constexpr u32 dr_bit = 1 << UReg_MSR{}.DR.StartBit(); + // Update mem_ptr auto& memory = m_system.GetMemory(); if (msr.IsImm()) @@ -524,7 +526,7 @@ void Jit64::MSRUpdated(const OpArg& msr, X64Reg scratch_reg) { MOV(64, R(RMEM), ImmPtr(memory.GetLogicalBase())); MOV(64, R(scratch_reg), ImmPtr(memory.GetPhysicalBase())); - TEST(32, msr, Imm32(1 << (31 - 27))); + TEST(32, msr, Imm32(dr_bit)); CMOVcc(64, RMEM, R(scratch_reg), CC_Z); } MOV(64, PPCSTATE(mem_ptr), R(RMEM)); @@ -548,6 +550,25 @@ void Jit64::MSRUpdated(const OpArg& msr, X64Reg scratch_reg) OR(32, R(scratch_reg), Imm32(other_feature_flags)); MOV(32, PPCSTATE(feature_flags), R(scratch_reg)); } + + // Call PageTableUpdatedFromJit if needed + if (!msr.IsImm() || UReg_MSR(msr.Imm32()).DR) + { + gpr.Flush(); + fpr.Flush(); + FixupBranch dr_unset; + if (!msr.IsImm()) + { + TEST(32, msr, Imm32(dr_bit)); + dr_unset = J_CC(CC_Z); + } + CMP(8, PPCSTATE(pagetable_update_pending), Imm8(0)); + FixupBranch update_not_pending = J_CC(CC_E); + ABI_CallFunctionP(&PowerPC::MMU::PageTableUpdatedFromJit, &m_system.GetMMU()); + SetJumpTarget(update_not_pending); + if (!msr.IsImm()) + SetJumpTarget(dr_unset); + } } void Jit64::WriteExit(u32 destination, bool bl, u32 after) diff --git a/Source/Core/Core/PowerPC/Jit64/Jit_SystemRegisters.cpp b/Source/Core/Core/PowerPC/Jit64/Jit_SystemRegisters.cpp index 0fdeb14131d..70eb2e40279 100644 --- a/Source/Core/Core/PowerPC/Jit64/Jit_SystemRegisters.cpp +++ b/Source/Core/Core/PowerPC/Jit64/Jit_SystemRegisters.cpp @@ -436,11 +436,14 @@ void Jit64::mtmsr(UGeckoInstruction inst) FALLBACK_IF(jo.fp_exceptions); { - RCOpArg Rs = gpr.BindOrImm(inst.RS, RCMode::Read); - RegCache::Realize(Rs); - MOV(32, PPCSTATE(msr), Rs); - - MSRUpdated(Rs, RSCRATCH2); + OpArg Rs_op_arg; + { + RCOpArg Rs = gpr.BindOrImm(inst.RS, RCMode::Read); + RegCache::Realize(Rs); + MOV(32, PPCSTATE(msr), Rs); + Rs_op_arg = Rs; + } + MSRUpdated(Rs_op_arg, RSCRATCH2); } gpr.Flush(); diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp index e5a19758d15..c04c45783d2 100644 --- a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp @@ -452,10 +452,27 @@ void JitArm64::MSRUpdated(u32 msr) MOVI2R(WA, feature_flags); STR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(feature_flags)); } + + // Call PageTableUpdatedFromJit if needed + if (UReg_MSR(msr).DR) + { + gpr.Flush(FlushMode::All, ARM64Reg::INVALID_REG); + fpr.Flush(FlushMode::All, ARM64Reg::INVALID_REG); + + auto WA = gpr.GetScopedReg(); + + static_assert(PPCSTATE_OFF(pagetable_update_pending) < 0x1000); + LDRB(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(pagetable_update_pending)); + FixupBranch update_not_pending = CBZ(WA); + ABI_CallFunction(&PowerPC::MMU::PageTableUpdatedFromJit, &m_system.GetMMU()); + SetJumpTarget(update_not_pending); + } } void JitArm64::MSRUpdated(ARM64Reg msr) { + constexpr LogicalImm dr_bit(1ULL << UReg_MSR{}.DR.StartBit(), GPRSize::B32); + auto WA = gpr.GetScopedReg(); ARM64Reg XA = EncodeRegTo64(WA); @@ -463,7 +480,7 @@ void JitArm64::MSRUpdated(ARM64Reg msr) auto& memory = m_system.GetMemory(); MOVP2R(MEM_REG, jo.fastmem ? memory.GetLogicalBase() : memory.GetLogicalPageMappingsBase()); MOVP2R(XA, jo.fastmem ? memory.GetPhysicalBase() : memory.GetPhysicalPageMappingsBase()); - TST(msr, LogicalImm(1 << (31 - 27), GPRSize::B32)); + TST(msr, dr_bit); CSEL(MEM_REG, MEM_REG, XA, CCFlags::CC_NEQ); STR(IndexType::Unsigned, MEM_REG, PPC_REG, PPCSTATE_OFF(mem_ptr)); @@ -477,6 +494,18 @@ void JitArm64::MSRUpdated(ARM64Reg msr) if (other_feature_flags != 0) ORR(WA, WA, LogicalImm(other_feature_flags, GPRSize::B32)); STR(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(feature_flags)); + + // Call PageTableUpdatedFromJit if needed + MOV(WA, msr); + gpr.Flush(FlushMode::All, ARM64Reg::INVALID_REG); + fpr.Flush(FlushMode::All, ARM64Reg::INVALID_REG); + FixupBranch dr_unset = TBZ(WA, dr_bit); + static_assert(PPCSTATE_OFF(pagetable_update_pending) < 0x1000); + LDRB(IndexType::Unsigned, WA, PPC_REG, PPCSTATE_OFF(pagetable_update_pending)); + FixupBranch update_not_pending = CBZ(WA); + ABI_CallFunction(&PowerPC::MMU::PageTableUpdatedFromJit, &m_system.GetMMU()); + SetJumpTarget(update_not_pending); + SetJumpTarget(dr_unset); } void JitArm64::WriteExit(u32 destination, bool LK, u32 exit_address_after_return, diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.h b/Source/Core/Core/PowerPC/JitArm64/Jit.h index 89de1e5d6e2..7a46eb58fec 100644 --- a/Source/Core/Core/PowerPC/JitArm64/Jit.h +++ b/Source/Core/Core/PowerPC/JitArm64/Jit.h @@ -122,9 +122,7 @@ public: void mcrf(UGeckoInstruction inst); void mcrxr(UGeckoInstruction inst); void mfsr(UGeckoInstruction inst); - void mtsr(UGeckoInstruction inst); void mfsrin(UGeckoInstruction inst); - void mtsrin(UGeckoInstruction inst); void twx(UGeckoInstruction inst); void mfspr(UGeckoInstruction inst); void mftb(UGeckoInstruction inst); diff --git a/Source/Core/Core/PowerPC/JitArm64/JitArm64_RegCache.h b/Source/Core/Core/PowerPC/JitArm64/JitArm64_RegCache.h index f801829b2c9..cde4fbe6cb2 100644 --- a/Source/Core/Core/PowerPC/JitArm64/JitArm64_RegCache.h +++ b/Source/Core/Core/PowerPC/JitArm64/JitArm64_RegCache.h @@ -33,19 +33,19 @@ constexpr Arm64Gen::ARM64Reg DISPATCHER_PC = Arm64Gen::ARM64Reg::W26; PowerPC::PowerPCState, elem); \ _Pragma("GCC diagnostic pop") \ }()) +#else +#define PPCSTATE_OFF(elem) (offsetof(PowerPC::PowerPCState, elem)) +#endif #define PPCSTATE_OFF_ARRAY(elem, i) \ (PPCSTATE_OFF(elem[0]) + sizeof(PowerPC::PowerPCState::elem[0]) * (i)) -#else -#define PPCSTATE_OFF(elem) (offsetof(PowerPC::PowerPCState, elem)) -#define PPCSTATE_OFF_ARRAY(elem, i) \ - (offsetof(PowerPC::PowerPCState, elem[0]) + sizeof(PowerPC::PowerPCState::elem[0]) * (i)) -#endif +#define PPCSTATE_OFF_STD_ARRAY(elem, i) \ + (PPCSTATE_OFF(elem) + sizeof(PowerPC::PowerPCState::elem[0]) * (i)) #define PPCSTATE_OFF_GPR(i) PPCSTATE_OFF_ARRAY(gpr, i) #define PPCSTATE_OFF_CR(i) PPCSTATE_OFF_ARRAY(cr.fields, i) -#define PPCSTATE_OFF_SR(i) PPCSTATE_OFF_ARRAY(sr, i) +#define PPCSTATE_OFF_SR(i) PPCSTATE_OFF_STD_ARRAY(sr, i) #define PPCSTATE_OFF_SPR(i) PPCSTATE_OFF_ARRAY(spr, i) static_assert(std::is_same_v); diff --git a/Source/Core/Core/PowerPC/JitArm64/JitArm64_SystemRegisters.cpp b/Source/Core/Core/PowerPC/JitArm64/JitArm64_SystemRegisters.cpp index 54592951e48..21a0003887a 100644 --- a/Source/Core/Core/PowerPC/JitArm64/JitArm64_SystemRegisters.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/JitArm64_SystemRegisters.cpp @@ -291,14 +291,6 @@ void JitArm64::mfsr(UGeckoInstruction inst) LDR(IndexType::Unsigned, gpr.R(inst.RD), PPC_REG, PPCSTATE_OFF_SR(inst.SR)); } -void JitArm64::mtsr(UGeckoInstruction inst) -{ - INSTRUCTION_START - JITDISABLE(bJITSystemRegistersOff); - - STR(IndexType::Unsigned, gpr.R(inst.RS), PPC_REG, PPCSTATE_OFF_SR(inst.SR)); -} - void JitArm64::mfsrin(UGeckoInstruction inst) { INSTRUCTION_START @@ -317,24 +309,6 @@ void JitArm64::mfsrin(UGeckoInstruction inst) LDR(RD, addr, ArithOption(EncodeRegTo64(index), true)); } -void JitArm64::mtsrin(UGeckoInstruction inst) -{ - INSTRUCTION_START - JITDISABLE(bJITSystemRegistersOff); - - u32 b = inst.RB, d = inst.RD; - gpr.BindToRegister(d, d == b); - - ARM64Reg RB = gpr.R(b); - ARM64Reg RD = gpr.R(d); - auto index = gpr.GetScopedReg(); - auto addr = gpr.GetScopedReg(); - - UBFM(index, RB, 28, 31); - ADDI2R(EncodeRegTo64(addr), PPC_REG, PPCSTATE_OFF_SR(0), EncodeRegTo64(addr)); - STR(RD, EncodeRegTo64(addr), ArithOption(EncodeRegTo64(index), true)); -} - void JitArm64::twx(UGeckoInstruction inst) { INSTRUCTION_START diff --git a/Source/Core/Core/PowerPC/JitArm64/JitArm64_Tables.cpp b/Source/Core/Core/PowerPC/JitArm64/JitArm64_Tables.cpp index 17b2030e25e..dc72083d39e 100644 --- a/Source/Core/Core/PowerPC/JitArm64/JitArm64_Tables.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/JitArm64_Tables.cpp @@ -266,18 +266,18 @@ constexpr std::array s_table31{{ {759, &JitArm64::stfXX}, // stfdux {983, &JitArm64::stfXX}, // stfiwx - {19, &JitArm64::mfcr}, // mfcr - {83, &JitArm64::mfmsr}, // mfmsr - {144, &JitArm64::mtcrf}, // mtcrf - {146, &JitArm64::mtmsr}, // mtmsr - {210, &JitArm64::mtsr}, // mtsr - {242, &JitArm64::mtsrin}, // mtsrin - {339, &JitArm64::mfspr}, // mfspr - {467, &JitArm64::mtspr}, // mtspr - {371, &JitArm64::mftb}, // mftb - {512, &JitArm64::mcrxr}, // mcrxr - {595, &JitArm64::mfsr}, // mfsr - {659, &JitArm64::mfsrin}, // mfsrin + {19, &JitArm64::mfcr}, // mfcr + {83, &JitArm64::mfmsr}, // mfmsr + {144, &JitArm64::mtcrf}, // mtcrf + {146, &JitArm64::mtmsr}, // mtmsr + {210, &JitArm64::FallBackToInterpreter}, // mtsr + {242, &JitArm64::FallBackToInterpreter}, // mtsrin + {339, &JitArm64::mfspr}, // mfspr + {467, &JitArm64::mtspr}, // mtspr + {371, &JitArm64::mftb}, // mftb + {512, &JitArm64::mcrxr}, // mcrxr + {595, &JitArm64::mfsr}, // mfsr + {659, &JitArm64::mfsrin}, // mfsrin {4, &JitArm64::twx}, // tw {598, &JitArm64::DoNothing}, // sync diff --git a/Source/Core/Core/PowerPC/JitCommon/JitBase.cpp b/Source/Core/Core/PowerPC/JitCommon/JitBase.cpp index a8709c4f6e9..d0ecd0a2b45 100644 --- a/Source/Core/Core/PowerPC/JitCommon/JitBase.cpp +++ b/Source/Core/Core/PowerPC/JitCommon/JitBase.cpp @@ -19,6 +19,7 @@ #include "Core/CoreTiming.h" #include "Core/HW/CPU.h" #include "Core/MemTools.h" +#include "Core/PowerPC/MMU.h" #include "Core/PowerPC/PPCAnalyst.h" #include "Core/PowerPC/PowerPC.h" #include "Core/System.h" @@ -120,6 +121,8 @@ bool JitBase::DoesConfigNeedRefresh() const void JitBase::RefreshConfig() { + const bool wanted_page_table_mappings = WantsPageTableMappings(); + for (const auto& [member, config_info] : JIT_SETTINGS) this->*member = Config::Get(*config_info); @@ -141,6 +144,18 @@ void JitBase::RefreshConfig() jo.memcheck = m_system.IsMMUMode() || m_system.IsPauseOnPanicMode() || any_watchpoints; jo.fp_exceptions = m_enable_float_exceptions; jo.div_by_zero_exceptions = m_enable_div_by_zero_exceptions; + + if (!wanted_page_table_mappings && WantsPageTableMappings()) + { + // Mustn't call this if we're still initializing + if (Core::IsRunning(m_system)) + m_system.GetMMU().PageTableUpdated(); + } +} + +bool JitBase::WantsPageTableMappings() const +{ + return jo.fastmem; } void JitBase::InitFastmemArena() diff --git a/Source/Core/Core/PowerPC/JitCommon/JitBase.h b/Source/Core/Core/PowerPC/JitCommon/JitBase.h index 468874a9836..db615ce7e10 100644 --- a/Source/Core/Core/PowerPC/JitCommon/JitBase.h +++ b/Source/Core/Core/PowerPC/JitCommon/JitBase.h @@ -217,6 +217,8 @@ public: virtual const CommonAsmRoutinesBase* GetAsmRoutines() = 0; + virtual bool WantsPageTableMappings() const; + virtual bool HandleFault(uintptr_t access_address, SContext* ctx) = 0; bool HandleStackFault(); diff --git a/Source/Core/Core/PowerPC/JitInterface.cpp b/Source/Core/Core/PowerPC/JitInterface.cpp index fec6e5992ae..62b65b6c2db 100644 --- a/Source/Core/Core/PowerPC/JitInterface.cpp +++ b/Source/Core/Core/PowerPC/JitInterface.cpp @@ -77,6 +77,16 @@ CPUCoreBase* JitInterface::GetCore() const return m_jit.get(); } +#ifndef _ARCH_32 +bool JitInterface::WantsPageTableMappings() const +{ + if (!m_jit) + return false; + + return m_jit->WantsPageTableMappings(); +} +#endif + void JitInterface::UpdateMembase() { if (!m_jit) diff --git a/Source/Core/Core/PowerPC/JitInterface.h b/Source/Core/Core/PowerPC/JitInterface.h index a382fb1ba4a..65b6060ba29 100644 --- a/Source/Core/Core/PowerPC/JitInterface.h +++ b/Source/Core/Core/PowerPC/JitInterface.h @@ -45,6 +45,12 @@ public: CPUCoreBase* InitJitCore(PowerPC::CPUCore core); CPUCoreBase* GetCore() const; +#ifdef _ARCH_32 + constexpr bool WantsPageTableMappings() const { return false; } +#else + bool WantsPageTableMappings() const; +#endif + void UpdateMembase(); void JitBlockLogDump(const Core::CPUThreadGuard& guard, std::FILE* file) const; void WipeBlockProfilingData(const Core::CPUThreadGuard& guard); diff --git a/Source/Core/Core/PowerPC/MMU.cpp b/Source/Core/Core/PowerPC/MMU.cpp index cde4e7b03ea..07f8993be14 100644 --- a/Source/Core/Core/PowerPC/MMU.cpp +++ b/Source/Core/Core/PowerPC/MMU.cpp @@ -25,14 +25,22 @@ #include "Core/PowerPC/MMU.h" +#include #include #include #include +#include #include +#include + +#ifdef _M_X86_64 +#include +#endif #include "Common/Align.h" #include "Common/Assert.h" #include "Common/BitUtils.h" +#include "Common/ChunkFile.h" #include "Common/CommonTypes.h" #include "Common/Logging/Log.h" @@ -58,6 +66,43 @@ MMU::MMU(Core::System& system, Memory::MemoryManager& memory, PowerPC::PowerPCMa MMU::~MMU() = default; +void MMU::Reset() +{ + ClearPageTable(); +} + +void MMU::DoState(PointerWrap& p, bool sr_changed) +{ + // Instead of storing m_page_table in savestates, we *could* refetch it from memory + // here in DoState, but this could lead to us getting a more up-to-date set of page mappings + // than we had when the savestate was created, which could be a problem for TAS determinism. + if (p.IsReadMode()) + { + if (!m_system.GetJitInterface().WantsPageTableMappings()) + { + // Clear page table mappings if we have any. + p.Do(m_page_table); + ClearPageTable(); + } + else if (sr_changed) + { + // Non-incremental update of page table mappings. + p.Do(m_page_table); + SRUpdated(); + } + else + { + // Incremental update of page table mappings. + p.Do(m_temp_page_table); + PageTableUpdated(m_temp_page_table); + } + } + else + { + p.Do(m_page_table); + } +} + // Overloaded byteswap functions, for use within the templated functions below. [[maybe_unused]] static u8 bswap(u8 val) { @@ -1216,7 +1261,17 @@ void MMU::SDRUpdated() WARN_LOG_FMT(POWERPC, "Invalid HTABORG: htaborg=0x{:08x} htabmask=0x{:08x}", htaborg, htabmask); m_ppc_state.pagetable_base = htaborg << 16; - m_ppc_state.pagetable_hashmask = ((htabmask << 10) | 0x3ff); + m_ppc_state.pagetable_mask = (htabmask << 16) | 0xffc0; + + PageTableUpdated(); +} + +void MMU::SRUpdated() +{ + // Our incremental handling of page table updates can't handle SR changing, so throw away all + // existing mappings and then reparse the whole page table. + m_memory.RemoveAllPageTableMappings(); + ReloadPageTable(); } enum class TLBLookupResult @@ -1306,6 +1361,397 @@ void MMU::InvalidateTLBEntry(u32 address) m_ppc_state.tlb[PowerPC::DATA_TLB_INDEX][entry_index].Invalidate(); m_ppc_state.tlb[PowerPC::INST_TLB_INDEX][entry_index].Invalidate(); + + if (m_ppc_state.msr.DR) + PageTableUpdated(); + else + m_ppc_state.pagetable_update_pending = true; +} + +void MMU::ClearPageTable() +{ + // If we've skipped processing any update to the page table, we need to remove all host mappings, + // because we don't know which of them are still valid. + m_memory.RemoveAllPageTableMappings(); + + // Because we removed host mappings, incremental updates won't work correctly. + // Start over from scratch. + m_page_mappings.clear(); + m_page_table.clear(); +} + +void MMU::ReloadPageTable() +{ + m_page_mappings.clear(); + + m_temp_page_table.clear(); + std::swap(m_page_table, m_temp_page_table); + + if (m_system.GetJitInterface().WantsPageTableMappings()) + PageTableUpdated(m_temp_page_table); + else + m_memory.RemoveAllPageTableMappings(); +} + +void MMU::PageTableUpdated() +{ + m_ppc_state.pagetable_update_pending = false; + + if (!m_system.GetJitInterface().WantsPageTableMappings()) + { + // If the JIT has no use for page table mappings, setting them up would be a waste of time. + ClearPageTable(); + return; + } + + const u32 page_table_base = m_ppc_state.pagetable_base; + const u32 page_table_end = + Common::AlignUp(page_table_base | m_ppc_state.pagetable_mask, PAGE_TABLE_MIN_SIZE); + const u32 page_table_size = page_table_end - page_table_base; + + u8* page_table_view = m_system.GetMemory().GetPointerForRange(page_table_base, page_table_size); + if (!page_table_view) + { + WARN_LOG_FMT(POWERPC, "Failed to read page table at {:#010x}-{:#010x}", page_table_base, + page_table_end); + ClearPageTable(); + return; + } + + PageTableUpdated(std::span(page_table_view, page_table_size)); +} + +void MMU::PageTableUpdated(std::span page_table) +{ + // PowerPC's priority order for PTEs that have the same logical adress is as follows: + // + // * Primary PTEs (H=0) take priority over secondary PTEs (H=1). + // * If two PTEs have equal H values, they must be in the same PTEG due to how the hash + // incorporates the logical address and H. The PTE located first in the PTEG takes priority. + + if (page_table.size() % PAGE_TABLE_MIN_SIZE != 0) + { + // Should only happen if a maliciously crafted savestate was loaded + PanicAlertFmt("Impossible page table size {}", page_table.size()); + ClearPageTable(); + return; + } + + m_removed_mappings.clear(); + m_added_readonly_mappings.clear(); + m_added_readwrite_mappings.clear(); + + if (m_page_table.size() != page_table.size()) + { + m_memory.RemoveAllPageTableMappings(); + m_page_mappings.clear(); + m_page_table.resize(0); + m_page_table.resize(page_table.size(), 0); + } + + u8* old_page_table = m_page_table.data(); + const u8* new_page_table = page_table.data(); + + constexpr auto compare_64_bytes = [](const u8* a, const u8* b) -> bool { +#ifdef _M_X86_64 + // MSVC (x64) doesn't want to optimize the memcmp call. This has a large performance impact + // in GameCube games that use page tables, so let's use our own vectorized version instead. + const __m128i a1 = _mm_load_si128(reinterpret_cast(a)); + const __m128i b1 = _mm_load_si128(reinterpret_cast(b)); + const __m128i cmp1 = _mm_cmpeq_epi8(a1, b1); + const __m128i a2 = _mm_load_si128(reinterpret_cast(a + 0x10)); + const __m128i b2 = _mm_load_si128(reinterpret_cast(b + 0x10)); + const __m128i cmp2 = _mm_cmpeq_epi8(a2, b2); + const __m128i cmp12 = _mm_and_si128(cmp1, cmp2); + const __m128i a3 = _mm_load_si128(reinterpret_cast(a + 0x20)); + const __m128i b3 = _mm_load_si128(reinterpret_cast(b + 0x20)); + const __m128i cmp3 = _mm_cmpeq_epi8(a3, b3); + const __m128i a4 = _mm_load_si128(reinterpret_cast(a + 0x30)); + const __m128i b4 = _mm_load_si128(reinterpret_cast(b + 0x30)); + const __m128i cmp4 = _mm_cmpeq_epi8(a4, b4); + const __m128i cmp34 = _mm_and_si128(cmp3, cmp4); + const __m128i cmp1234 = _mm_and_si128(cmp12, cmp34); + return _mm_movemask_epi8(cmp1234) == 0xFFFF; +#else + return std::memcmp(std::assume_aligned<64>(a), std::assume_aligned<64>(b), 64) == 0; +#endif + }; + + const auto get_page_index = [this](UPTE_Lo pte1, u32 hash) -> std::optional { + u32 page_index_from_hash = hash ^ pte1.VSID; + if (pte1.H) + page_index_from_hash = ~page_index_from_hash; + + // Due to hash masking, the upper bits of page_index_from_hash might not match the actual + // page index. But these bits fully overlap with the API (abbreviated page index), so we can + // overwrite these bits with the API from pte1 and thereby get the correct page index. + // + // In other words: logical_address.API must be written to after logical_address.page_index! + EffectiveAddress logical_address; + logical_address.offset = 0; + logical_address.page_index = page_index_from_hash; + logical_address.API = pte1.API; + + // If the hash mask is large enough that one or more bits specified in pte1.API can also be + // obtained from page_index_from_hash, check that those bits match. + const u32 api_mask = ((m_ppc_state.pagetable_mask & ~m_ppc_state.pagetable_base) >> 16) & 0x3f; + if ((pte1.API & api_mask) != ((page_index_from_hash >> 10) & api_mask)) + return std::nullopt; + + return logical_address; + }; + + const auto fixup_shadowed_mappings = [this, old_page_table, new_page_table]( + UPTE_Lo pte1, u32 page_table_offset, bool* run_pass_2) { + DEBUG_ASSERT(pte1.V == 1); + + bool switched_to_secondary = false; + + while (true) + { + const u32 big_endian_pte1 = Common::swap32(pte1.Hex); + const u32 pteg_start = Common::AlignDown(page_table_offset, 64); + const u32 pteg_end = pteg_start + 64; + for (u32 i = page_table_offset; i < pteg_end; i += 8) + { + if (std::memcmp(new_page_table + i, &big_endian_pte1, sizeof(big_endian_pte1)) == 0) + { + // We've found a PTE that has V set and has the same logical address as the passed-in PTE. + // The found PTE was previously skipped over because the passed-in PTE had priority, but + // the passed-in PTE is being changed, so now we need to re-check the found PTE. This will + // happen naturally later in the loop that's calling this function, but only if the 8-byte + // memcmp reports that the PTE has changed. Therefore, if the PTE currently compares + // equal, change an unused bit in the PTE. + if (std::memcmp(old_page_table + i, new_page_table + i, 8) == 0) + { + UPTE_Hi pte2(Common::swap32(old_page_table + i + 4)); + pte2.reserved_1 = pte2.reserved_1 ^ 1; + const u32 big_endian_pte2 = Common::swap32(pte2.Hex); + std::memcpy(old_page_table + i + 4, &big_endian_pte2, sizeof(big_endian_pte2)); + + if (switched_to_secondary) + *run_pass_2 = true; + } + // This PTE has priority over any later PTEs we might find, so no need to keep scanning. + return; + } + } + + if (pte1.H == 1) + { + // We've scanned the secondary PTEG. Nothing left to do. + return; + } + else + { + // We've scanned the primary PTEG. Now let's scan the secondary PTEG. + pte1.H = 1; + page_table_offset = + ((~pteg_start & m_ppc_state.pagetable_mask) | m_ppc_state.pagetable_base) - + m_ppc_state.pagetable_base; + switched_to_secondary = true; + } + } + }; + + const auto try_add_mapping = [this, &get_page_index](UPTE_Lo pte1, UPTE_Hi pte2, + u32 page_table_offset) { + std::optional logical_address = get_page_index(pte1, page_table_offset / 64); + if (!logical_address) + return; + + for (u32 i = 0; i < std::size(m_ppc_state.sr); ++i) + { + const auto sr = UReg_SR{m_ppc_state.sr[i]}; + if (sr.VSID != pte1.VSID || sr.T != 0) + continue; + + logical_address->SR = i; + + bool host_mapping = true; + + const bool wi = (pte2.WIMG & 0b1100) != 0; + if (wi) + { + // There are quirks related to uncached memory that can't be correctly emulated by fast + // accesses, so we don't map uncached memory. (However, no software at all is known to + // trigger these quirks through page address translation, only through block address + // translation.) + host_mapping = false; + } + else if (m_dbat_table[logical_address->Hex >> PowerPC::BAT_INDEX_SHIFT] & + PowerPC::BAT_MAPPED_BIT) + { + // Block address translation takes priority over page address translation. + host_mapping = false; + } + else if (m_power_pc.GetMemChecks().OverlapsMemcheck(logical_address->Hex, + PowerPC::HW_PAGE_SIZE)) + { + // Fast accesses don't support memchecks, so force slow accesses by removing fastmem + // mappings for all overlapping virtual pages. + host_mapping = false; + } + + const u32 priority = (page_table_offset % 64 / 8) | (pte1.H << 3); + const PageMapping page_mapping(pte2.RPN, host_mapping, priority); + + const auto it = m_page_mappings.find(logical_address->Hex); + if (it == m_page_mappings.end()) [[likely]] + { + // There's no existing mapping for this logical address. Add a new mapping. + m_page_mappings.emplace(logical_address->Hex, page_mapping); + } + else + { + if (it->second.priority < priority) + { + // An existing mapping has priority. + continue; + } + + // The new mapping has priority over an existing mapping. Replace the existing mapping. + if (it->second.host_mapping) + m_removed_mappings.emplace(it->first); + it->second.Hex = page_mapping.Hex; + } + + // If the R bit isn't set yet, the actual host mapping will be created once + // TranslatePageAddress sets the R bit. + if (host_mapping && pte2.R) + { + const u32 physical_address = pte2.RPN << 12; + (pte2.C ? m_added_readwrite_mappings : m_added_readonly_mappings) + .emplace(logical_address->Hex, physical_address); + } + } + }; + + bool run_pass_2 = false; + + // Pass 1: Remove old mappings and add new primary (H=0) mappings. + for (u32 i = 0; i < page_table.size(); i += PAGE_TABLE_MIN_SIZE) + { + if ((i & m_ppc_state.pagetable_mask) != i || (i & m_ppc_state.pagetable_base) != 0) + continue; + + for (u32 j = 0; j < PAGE_TABLE_MIN_SIZE; j += 64) + { + if (compare_64_bytes(old_page_table + i + j, new_page_table + i + j)) [[likely]] + continue; + + for (u32 k = 0; k < 64; k += 8) + { + if (std::memcmp(old_page_table + i + j + k, new_page_table + i + j + k, 8) == 0) [[likely]] + continue; + + // Remove old mappings. + UPTE_Lo old_pte1(Common::swap32(old_page_table + i + j + k)); + if (old_pte1.V) + { + const u32 priority = (k / 8) | (old_pte1.H << 3); + std::optional logical_address = get_page_index(old_pte1, (i + j) / 64); + if (!logical_address) + continue; + + for (u32 l = 0; l < std::size(m_ppc_state.sr); ++l) + { + const auto sr = UReg_SR{m_ppc_state.sr[l]}; + if (sr.VSID != old_pte1.VSID || sr.T != 0) + continue; + + logical_address->SR = l; + + const auto it = m_page_mappings.find(logical_address->Hex); + if (it != m_page_mappings.end() && priority == it->second.priority) + { + if (it->second.host_mapping) + m_removed_mappings.emplace(logical_address->Hex); + m_page_mappings.erase(it); + + // It's unlikely but possible that this was shadowing another PTE that's using the + // same logical address but has a lower priority. If this happens, we must make sure + // that we don't skip over that other PTE because of the 8-byte memcmp. + fixup_shadowed_mappings(old_pte1, i + j + k, &run_pass_2); + } + } + } + + // Add new primary (H=0) mappings. + UPTE_Lo new_pte1(Common::swap32(new_page_table + i + j + k)); + UPTE_Hi new_pte2(Common::swap32(new_page_table + i + j + k + 4)); + if (new_pte1.V) + { + if (new_pte1.H) + { + run_pass_2 = true; + continue; + } + + try_add_mapping(new_pte1, new_pte2, i + j + k); + } + + // Update our copy of the page table. + std::memcpy(old_page_table + i + j + k, new_page_table + i + j + k, 8); + } + } + } + + // Pass 2: Add new secondary (H=1) mappings. This is a separate pass because before we can process + // whether a mapping should be added, we first need to check all PTEs that have equal or higher + // priority to see if their mappings should be removed. For adding primary mappings, this ordering + // comes naturally from doing a linear scan of the page table from start to finish. But for adding + // secondary mappings, the primary PTEG that has priority over a given secondary PTEG is in the + // other half of the page table, so we need more than one pass through the page table. But most of + // the time, there are no secondary mappings, letting us skip the second pass. + if (run_pass_2) [[unlikely]] + { + for (u32 i = 0; i < page_table.size(); i += PAGE_TABLE_MIN_SIZE) + { + if ((i & m_ppc_state.pagetable_mask) != i || (i & m_ppc_state.pagetable_base) != 0) + continue; + + for (u32 j = 0; j < PAGE_TABLE_MIN_SIZE; j += 64) + { + if (compare_64_bytes(old_page_table + i + j, new_page_table + i + j)) [[likely]] + continue; + + for (u32 k = 0; k < 64; k += 8) + { + if (std::memcmp(old_page_table + i + j + k, new_page_table + i + j + k, 8) == 0) + [[likely]] + { + continue; + } + + UPTE_Lo new_pte1(Common::swap32(new_page_table + i + j + k)); + UPTE_Hi new_pte2(Common::swap32(new_page_table + i + j + k + 4)); + + // We don't need to check new_pte1.V and new_pte1.H. If the memcmp above returned nonzero, + // pass 1 must have skipped running memcpy, which only happens if V and H are both set. + DEBUG_ASSERT(new_pte1.V == 1); + DEBUG_ASSERT(new_pte1.H == 1); + try_add_mapping(new_pte1, new_pte2, i + j + k); + + std::memcpy(old_page_table + i + j + k, new_page_table + i + j + k, 8); + } + } + } + } + + if (!m_removed_mappings.empty()) + m_memory.RemovePageTableMappings(m_removed_mappings); + + for (const auto& [logical_address, physical_address] : m_added_readonly_mappings) + m_memory.AddPageTableMapping(logical_address, physical_address, false); + + for (const auto& [logical_address, physical_address] : m_added_readwrite_mappings) + m_memory.AddPageTableMapping(logical_address, physical_address, true); +} + +void MMU::PageTableUpdatedFromJit(MMU* mmu) +{ + mmu->PageTableUpdated(); } // Page Address Translation @@ -1359,7 +1805,7 @@ MMU::TranslateAddressResult MMU::TranslatePageAddress(const EffectiveAddress add pte1.H = 1; } - u32 pteg_addr = ((hash & m_ppc_state.pagetable_hashmask) << 6) | m_ppc_state.pagetable_base; + u32 pteg_addr = ((hash << 6) & m_ppc_state.pagetable_mask) | m_ppc_state.pagetable_base; for (int i = 0; i < 8; i++, pteg_addr += 8) { @@ -1370,6 +1816,7 @@ MMU::TranslateAddressResult MMU::TranslatePageAddress(const EffectiveAddress add if (pte1.Hex == pteg) { UPTE_Hi pte2(ReadFromHardware(pteg_addr + 4)); + const UPTE_Hi old_pte2 = pte2; // set the access bits switch (flag) @@ -1389,9 +1836,29 @@ MMU::TranslateAddressResult MMU::TranslatePageAddress(const EffectiveAddress add break; } - if (!IsNoExceptionFlag(flag)) + if (!IsNoExceptionFlag(flag) && pte2.Hex != old_pte2.Hex) { m_memory.Write_U32(pte2.Hex, pteg_addr + 4); + + const u32 page_logical_address = address.Hex & ~HW_PAGE_MASK; + const auto it = m_page_mappings.find(page_logical_address); + if (it != m_page_mappings.end()) + { + const u32 priority = (pteg_addr % 64 / 8) | (pte1.H << 3); + if (it->second.Hex == PageMapping(pte2.RPN, true, priority).Hex) + { + const u32 swapped_pte1 = Common::swap32(reinterpret_cast(&pte1)); + std::memcpy(m_page_table.data() + pteg_addr - m_ppc_state.pagetable_base, + &swapped_pte1, sizeof(swapped_pte1)); + + const u32 swapped_pte2 = Common::swap32(reinterpret_cast(&pte2)); + std::memcpy(m_page_table.data() + pteg_addr + 4 - m_ppc_state.pagetable_base, + &swapped_pte2, sizeof(swapped_pte2)); + + const u32 page_translated_address = pte2.RPN << 12; + m_memory.AddPageTableMapping(page_logical_address, page_translated_address, pte2.C); + } + } } // We already updated the TLB entry if this was caused by a C bit. @@ -1533,7 +2000,13 @@ void MMU::DBATUpdated() } #ifndef _ARCH_32 - m_memory.UpdateLogicalMemory(m_dbat_table); + m_memory.UpdateDBATMappings(m_dbat_table); + + // Calling UpdateDBATMappings removes all fastmem page table mappings, so we have to recreate + // them. We need to go through them anyway because there may have been a change in which DBATs + // or memchecks are shadowing which page table mappings. + if (!m_page_table.empty()) + ReloadPageTable(); #endif // IsOptimizable*Address and dcbz depends on the BAT mapping, so we need a flush here. diff --git a/Source/Core/Core/PowerPC/MMU.h b/Source/Core/Core/PowerPC/MMU.h index 018657cd5f4..14196df06dd 100644 --- a/Source/Core/Core/PowerPC/MMU.h +++ b/Source/Core/Core/PowerPC/MMU.h @@ -5,13 +5,19 @@ #include #include +#include #include +#include +#include #include +#include #include "Common/BitField.h" #include "Common/CommonTypes.h" #include "Common/TypeUtils.h" +class PointerWrap; + namespace Core { class CPUThreadGuard; @@ -73,6 +79,8 @@ constexpr size_t HW_PAGE_MASK = HW_PAGE_SIZE - 1; constexpr u32 HW_PAGE_INDEX_SHIFT = 12; constexpr u32 HW_PAGE_INDEX_MASK = 0x3f; +constexpr u32 PAGE_TABLE_MIN_SIZE = 0x10000; + // Return value of MMU::TryReadInstruction(). struct TryReadInstResult { @@ -117,6 +125,9 @@ public: MMU& operator=(MMU&& other) = delete; ~MMU(); + void Reset(); + void DoState(PointerWrap& p, bool sr_changed); + // Routines for debugger UI, cheats, etc. to access emulated memory from the // perspective of the CPU. Not for use by core emulation routines. // Use "Host" prefix. @@ -237,7 +248,10 @@ public: // TLB functions void SDRUpdated(); + void SRUpdated(); void InvalidateTLBEntry(u32 address); + void PageTableUpdated(); + static void PageTableUpdatedFromJit(MMU* mmu); void DBATUpdated(); void IBATUpdated(); @@ -290,6 +304,26 @@ private: explicit EffectiveAddress(u32 address) : Hex{address} {} }; + union PageMapping + { + // A small priority number wins over a larger priority number. + BitField<0, 11, u32> priority; + // Whether we're allowed to create a host mapping for this mapping. + BitField<11, 1, u32> host_mapping; + // The physical address of the page. + BitField<12, 20, u32> RPN; + + u32 Hex = 0; + + PageMapping() = default; + PageMapping(u32 RPN_, bool host_mapping_, u32 priority_) + { + RPN = RPN_; + host_mapping = host_mapping_; + priority = priority_; + } + }; + template TranslateAddressResult TranslateAddress(u32 address); @@ -301,6 +335,10 @@ private: void Memcheck(u32 address, u64 var, bool write, size_t size); + void ClearPageTable(); + void ReloadPageTable(); + void PageTableUpdated(std::span page_table); + void UpdateBATs(BatTable& bat_table, u32 base_spr); void UpdateFakeMMUBat(BatTable& bat_table, u32 start_addr); @@ -317,6 +355,20 @@ private: PowerPC::PowerPCManager& m_power_pc; PowerPC::PowerPCState& m_ppc_state; + // STATE_TO_SAVE + std::vector m_page_table; + // END STATE_TO_SAVE + + // This keeps track of all valid page table mappings in m_page_table. + // The key is the logical address. + std::map m_page_mappings; + + // These are kept around just for their memory allocations. They are always cleared before use. + std::vector m_temp_page_table; + std::set m_removed_mappings; + std::map m_added_readonly_mappings; + std::map m_added_readwrite_mappings; + BatTable m_ibat_table; BatTable m_dbat_table; }; diff --git a/Source/Core/Core/PowerPC/PowerPC.cpp b/Source/Core/Core/PowerPC/PowerPC.cpp index 87a047670e4..e5ce8ffcdb0 100644 --- a/Source/Core/Core/PowerPC/PowerPC.cpp +++ b/Source/Core/Core/PowerPC/PowerPC.cpp @@ -4,6 +4,7 @@ #include "Core/PowerPC/PowerPC.h" #include +#include #include #include @@ -78,6 +79,8 @@ void PowerPCManager::DoState(PointerWrap& p) // *((u64 *)&TL(m_ppc_state)) = SystemTimers::GetFakeTimeBase(); //works since we are little // endian and TL comes first :) + const std::array old_sr = m_ppc_state.sr; + p.DoArray(m_ppc_state.gpr); p.Do(m_ppc_state.pc); p.Do(m_ppc_state.npc); @@ -94,7 +97,8 @@ void PowerPCManager::DoState(PointerWrap& p) p.DoArray(m_ppc_state.spr); p.DoArray(m_ppc_state.tlb); p.Do(m_ppc_state.pagetable_base); - p.Do(m_ppc_state.pagetable_hashmask); + p.Do(m_ppc_state.pagetable_mask); + p.Do(m_ppc_state.pagetable_update_pending); p.Do(m_ppc_state.reserve); p.Do(m_ppc_state.reserve_address); @@ -103,8 +107,11 @@ void PowerPCManager::DoState(PointerWrap& p) m_ppc_state.iCache.DoState(memory, p); m_ppc_state.dCache.DoState(memory, p); + auto& mmu = m_system.GetMMU(); if (p.IsReadMode()) { + mmu.DoState(p, old_sr != m_ppc_state.sr); + if (!m_ppc_state.m_enable_dcache) { INFO_LOG_FMT(POWERPC, "Flushing data cache"); @@ -114,10 +121,13 @@ void PowerPCManager::DoState(PointerWrap& p) RoundingModeUpdated(m_ppc_state); RecalculateAllFeatureFlags(m_ppc_state); - auto& mmu = m_system.GetMMU(); mmu.IBATUpdated(); mmu.DBATUpdated(); } + else + { + mmu.DoState(p, false); + } // SystemTimers::DecrementerSet(); // SystemTimers::TimeBaseSet(); @@ -274,12 +284,14 @@ void PowerPCManager::Init(CPUCore cpu_core) void PowerPCManager::Reset() { m_ppc_state.pagetable_base = 0; - m_ppc_state.pagetable_hashmask = 0; + m_ppc_state.pagetable_mask = 0; + m_ppc_state.pagetable_update_pending = false; m_ppc_state.tlb = {}; ResetRegisters(); m_ppc_state.iCache.Reset(m_system.GetJitInterface()); m_ppc_state.dCache.Reset(); + m_system.GetMMU().Reset(); } void PowerPCManager::ScheduleInvalidateCacheThreadSafe(u32 address) @@ -667,13 +679,10 @@ void PowerPCManager::MSRUpdated() m_ppc_state.feature_flags = static_cast( (m_ppc_state.feature_flags & FEATURE_FLAG_PERFMON) | ((m_ppc_state.msr.Hex >> 4) & 0x3)); - m_system.GetJitInterface().UpdateMembase(); -} + if (m_ppc_state.msr.DR && m_ppc_state.pagetable_update_pending) + m_system.GetMMU().PageTableUpdated(); -void PowerPCState::SetSR(u32 index, u32 value) -{ - DEBUG_LOG_FMT(POWERPC, "{:08x}: MMU: Segment register {} set to {:08x}", pc, index, value); - sr[index] = value; + m_system.GetJitInterface().UpdateMembase(); } // FPSCR update functions diff --git a/Source/Core/Core/PowerPC/PowerPC.h b/Source/Core/Core/PowerPC/PowerPC.h index ea972590cb3..8d84ed1a303 100644 --- a/Source/Core/Core/PowerPC/PowerPC.h +++ b/Source/Core/Core/PowerPC/PowerPC.h @@ -122,6 +122,9 @@ struct PowerPCState u32 pc = 0; // program counter u32 npc = 0; + // Storage for the stack pointer of the BLR optimization. + u8* stored_stack_pointer = nullptr; + // gather pipe pointer for JIT access u8* gather_pipe_ptr = nullptr; u8* gather_pipe_base_ptr = nullptr; @@ -157,6 +160,14 @@ struct PowerPCState // lscbx u16 xer_stringctrl = 0; + // Reservation monitor for lwarx and its friend stwcxd. These two don't really need to be + // this early in the struct, but due to how the padding works out, they fit nicely here. + u32 reserve_address; + bool reserve; + + bool pagetable_update_pending = false; + bool m_enable_dcache = false; + #ifdef _M_X86_64 // This member exists only for the purpose of an assertion that its offset <= 0x100. std::tuple<> above_fits_in_first_0x100; @@ -164,37 +175,28 @@ struct PowerPCState alignas(16) PairedSingle ps[32]; #endif - u32 sr[16]{}; // Segment registers. + std::array sr{}; // Segment registers. // special purpose registers - controls quantizers, DMA, and lots of other misc extensions. // also for power management, but we don't care about that. // JitArm64 needs 64-bit alignment for SPR_TL. alignas(8) u32 spr[1024]{}; - // Storage for the stack pointer of the BLR optimization. - u8* stored_stack_pointer = nullptr; u8* mem_ptr = nullptr; + u32 pagetable_base = 0; + u32 pagetable_mask = 0; + std::array, NUM_TLBS> tlb; - u32 pagetable_base = 0; - u32 pagetable_hashmask = 0; - InstructionCache iCache; - bool m_enable_dcache = false; Cache dCache; - // Reservation monitor for lwarx and its friend stwcxd. - bool reserve; - u32 reserve_address; - void UpdateCR1() { cr.SetField(1, (fpscr.FX << 3) | (fpscr.FEX << 2) | (fpscr.VX << 1) | fpscr.OX); } - void SetSR(u32 index, u32 value); - void SetCarry(u32 ca) { xer_ca = ca; } u32 GetCarry() const { return xer_ca; } diff --git a/Source/Core/Core/State.cpp b/Source/Core/Core/State.cpp index d9f37b3f487..fd71ba9df28 100644 --- a/Source/Core/Core/State.cpp +++ b/Source/Core/Core/State.cpp @@ -95,7 +95,7 @@ struct CompressAndDumpStateArgs static Common::WorkQueueThreadSP s_compress_and_dump_thread; // Don't forget to increase this after doing changes on the savestate system -constexpr u32 STATE_VERSION = 175; // Last changed in PR 13751 +constexpr u32 STATE_VERSION = 176; // Last changed in PR 13768 // Increase this if the StateExtendedHeader definition changes constexpr u32 EXTENDED_HEADER_VERSION = 1; // Last changed in PR 12217 diff --git a/Source/Core/DolphinQt/Debugger/RegisterWidget.cpp b/Source/Core/DolphinQt/Debugger/RegisterWidget.cpp index 8bb9d882916..80cd55f6745 100644 --- a/Source/Core/DolphinQt/Debugger/RegisterWidget.cpp +++ b/Source/Core/DolphinQt/Debugger/RegisterWidget.cpp @@ -15,6 +15,7 @@ #include "Core/Core.h" #include "Core/Debugger/CodeTrace.h" #include "Core/HW/ProcessorInterface.h" +#include "Core/PowerPC/MMU.h" #include "Core/PowerPC/PowerPC.h" #include "Core/System.h" #include "DolphinQt/Host.h" @@ -405,7 +406,10 @@ void RegisterWidget::PopulateTable() AddRegister( i, 7, RegisterType::sr, "SR" + std::to_string(i), [this, i] { return m_system.GetPPCState().sr[i]; }, - [this, i](u64 value) { m_system.GetPPCState().sr[i] = value; }); + [this, i](u64 value) { + m_system.GetPPCState().sr[i] = value; + m_system.GetMMU().SRUpdated(); + }); } // Special registers @@ -490,7 +494,7 @@ void RegisterWidget::PopulateTable() 31, 5, RegisterType::pt_hashmask, "Hash Mask", [this] { const auto& ppc_state = m_system.GetPPCState(); - return (ppc_state.pagetable_hashmask << 6) | ppc_state.pagetable_base; + return ppc_state.pagetable_mask | ppc_state.pagetable_base; }, nullptr); diff --git a/Source/UnitTests/Core/CMakeLists.txt b/Source/UnitTests/Core/CMakeLists.txt index 8725995729f..30aeae44775 100644 --- a/Source/UnitTests/Core/CMakeLists.txt +++ b/Source/UnitTests/Core/CMakeLists.txt @@ -21,6 +21,7 @@ add_dolphin_test(SkylandersTest IOS/USB/SkylandersTest.cpp) if(_M_X86_64) add_dolphin_test(PowerPCTest PowerPC/DivUtilsTest.cpp + PowerPC/PageTableHostMappingTest.cpp PowerPC/Jit64Common/ConvertDoubleToSingle.cpp PowerPC/Jit64Common/Fres.cpp PowerPC/Jit64Common/Frsqrte.cpp @@ -28,6 +29,7 @@ if(_M_X86_64) elseif(_M_ARM_64) add_dolphin_test(PowerPCTest PowerPC/DivUtilsTest.cpp + PowerPC/PageTableHostMappingTest.cpp PowerPC/JitArm64/ConvertSingleDouble.cpp PowerPC/JitArm64/FPRF.cpp PowerPC/JitArm64/Fres.cpp @@ -37,9 +39,11 @@ elseif(_M_ARM_64) else() add_dolphin_test(PowerPCTest PowerPC/DivUtilsTest.cpp + PowerPC/PageTableHostMappingTest.cpp ) endif() target_sources(PowerPCTest PRIVATE PowerPC/TestValues.h + StubJit.h ) diff --git a/Source/UnitTests/Core/PageFaultTest.cpp b/Source/UnitTests/Core/PageFaultTest.cpp index f7805427612..8c29d31d640 100644 --- a/Source/UnitTests/Core/PageFaultTest.cpp +++ b/Source/UnitTests/Core/PageFaultTest.cpp @@ -7,10 +7,11 @@ #include "Common/ScopeGuard.h" #include "Core/Core.h" #include "Core/MemTools.h" -#include "Core/PowerPC/JitCommon/JitBase.h" #include "Core/PowerPC/JitInterface.h" #include "Core/System.h" +#include "StubJit.h" + // include order is important #include // NOLINT @@ -23,26 +24,11 @@ enum #endif }; -class PageFaultFakeJit : public JitBase +class PageFaultFakeJit : public StubJit { public: - explicit PageFaultFakeJit(Core::System& system) : JitBase(system) {} + explicit PageFaultFakeJit(Core::System& system) : StubJit(system) {} - // CPUCoreBase methods - void Init() override {} - void Shutdown() override {} - void ClearCache() override {} - void Run() override {} - void SingleStep() override {} - const char* GetName() const override { return nullptr; } - // JitBase methods - JitBaseBlockCache* GetBlockCache() override { return nullptr; } - void Jit(u32 em_address) override {} - void EraseSingleBlock(const JitBlock&) override {} - std::vector GetMemoryStats() const override { return {}; } - std::size_t DisassembleNearCode(const JitBlock&, std::ostream&) const override { return 0; } - std::size_t DisassembleFarCode(const JitBlock&, std::ostream&) const override { return 0; } - const CommonAsmRoutinesBase* GetAsmRoutines() override { return nullptr; } bool HandleFault(uintptr_t access_address, SContext* ctx) override { m_pre_unprotect_time = std::chrono::high_resolution_clock::now(); diff --git a/Source/UnitTests/Core/PowerPC/PageTableHostMappingTest.cpp b/Source/UnitTests/Core/PowerPC/PageTableHostMappingTest.cpp new file mode 100644 index 00000000000..fcb0071054c --- /dev/null +++ b/Source/UnitTests/Core/PowerPC/PageTableHostMappingTest.cpp @@ -0,0 +1,883 @@ +// Copyright 2026 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include +#include + +#include + +#include "Common/Align.h" +#include "Common/CommonTypes.h" +#include "Common/Swap.h" +#include "Core/Core.h" +#include "Core/MemTools.h" +#include "Core/PowerPC/BreakPoints.h" +#include "Core/PowerPC/Gekko.h" +#include "Core/PowerPC/JitInterface.h" +#include "Core/PowerPC/MMU.h" +#include "Core/PowerPC/PowerPC.h" +#include "Core/System.h" + +#include "../StubJit.h" + +#include + +// All guest addresses used in this unit test are arbitrary, aside from alignment requirements +static constexpr u32 ALIGNED_PAGE_TABLE_BASE = 0x00020000; +static constexpr u32 ALIGNED_PAGE_TABLE_MASK_SMALL = 0x0000ffff; +static constexpr u32 ALIGNED_PAGE_TABLE_MASK_LARGE = 0x0001ffff; + +static constexpr u32 MISALIGNED_PAGE_TABLE_BASE = 0x00050000; +static constexpr u32 MISALIGNED_PAGE_TABLE_BASE_ALIGNED = 0x00040000; +static constexpr u32 MISALIGNED_PAGE_TABLE_MASK = 0x0003ffff; + +static constexpr u32 HOLE_MASK_PAGE_TABLE_BASE = 0x00080000; +static constexpr u32 HOLE_MASK_PAGE_TABLE_MASK = 0x0002ffff; +static constexpr u32 HOLE_MASK_PAGE_TABLE_MASK_WITHOUT_HOLE = 0x0003ffff; + +static constexpr u32 MISALIGNED_HOLE_MASK_PAGE_TABLE_BASE = 0x000e0000; +static constexpr u32 MISALIGNED_HOLE_MASK_PAGE_TABLE_BASE_ALIGNED = 0x000d0000; +static constexpr u32 MISALIGNED_HOLE_MASK_PAGE_TABLE_MASK = 0x0002ffff; +static constexpr u32 MISALIGNED_HOLE_MASK_PAGE_TABLE_MASK_WITHOUT_HOLE = 0x0003ffff; + +static constexpr u32 TEMPORARY_MEMORY = 0x00000000; +static u32 s_current_temporary_memory = TEMPORARY_MEMORY; + +// This is the max that the unit test can handle, not the max that Core can handle +static constexpr u32 MAX_HOST_PAGE_SIZE = 64 * 1024; +static u32 s_minimum_mapping_size = 0; + +static volatile const void* volatile s_detection_address = nullptr; +static volatile size_t s_detection_count = 0; +static u32 s_counter = 0; +static std::set s_temporary_mappings; + +class PageFaultDetector : public StubJit +{ +public: + explicit PageFaultDetector(Core::System& system) : StubJit(system), m_block_cache(*this) {} + + bool HandleFault(uintptr_t access_address, SContext* ctx) override + { + if (access_address != reinterpret_cast(s_detection_address)) + { + std::string logical_address; + auto& memory = Core::System::GetInstance().GetMemory(); + auto logical_base = reinterpret_cast(memory.GetLogicalBase()); + if (access_address >= logical_base && access_address < logical_base + 0x1'0000'0000) + logical_address = fmt::format(" (PPC {:#010x})", access_address - logical_base); + + ADD_FAILURE() << fmt::format("Unexpected segfault at {:#x}{}", access_address, + logical_address); + + return false; + } + + s_detection_address = nullptr; + s_detection_count = s_detection_count + 1; + + // After we return from the signal handler, the memory access will happen again. + // Let it succeed this time so the signal handler won't get called over and over. + auto& memory = Core::System::GetInstance().GetMemory(); + const uintptr_t logical_base = reinterpret_cast(memory.GetLogicalBase()); + const u32 logical_address = static_cast(access_address - logical_base); + const u32 mask = s_minimum_mapping_size - 1; + for (u32 i = logical_address & mask; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + { + const u32 current_logical_address = (logical_address & ~mask) + i; + memory.AddPageTableMapping(current_logical_address, s_current_temporary_memory + i, true); + s_temporary_mappings.emplace(current_logical_address); + } + return true; + } + + bool WantsPageTableMappings() const override { return true; } + + // PowerPC::MMU::DBATUpdated wants to clear blocks in the block cache, + // so this can't just return nullptr + JitBaseBlockCache* GetBlockCache() override { return &m_block_cache; } + +private: + StubBlockCache m_block_cache; +}; + +// This is used as a performance optimization. If several page table updates are performed while +// DR is disabled, MMU.cpp will only have to rescan the page table one time once DR is enabled again +// instead of after each page table update. +class DisableDR final +{ +public: + DisableDR() + { + auto& system = Core::System::GetInstance(); + system.GetPPCState().msr.DR = 0; + system.GetPowerPC().MSRUpdated(); + } + + ~DisableDR() + { + auto& system = Core::System::GetInstance(); + system.GetPPCState().msr.DR = 1; + system.GetPowerPC().MSRUpdated(); + } +}; + +class PageTableHostMappingTest : public ::testing::Test +{ +public: + static void SetUpTestSuite() + { + if (!EMM::IsExceptionHandlerSupported()) + GTEST_SKIP() << "Skipping PageTableHostMappingTest because exception handler is unsupported."; + + auto& system = Core::System::GetInstance(); + auto& memory = system.GetMemory(); + const u32 host_page_size = memory.GetHostPageSize(); + s_minimum_mapping_size = std::max(host_page_size, PowerPC::HW_PAGE_SIZE); + + if (!std::has_single_bit(host_page_size) || host_page_size > MAX_HOST_PAGE_SIZE) + { + GTEST_SKIP() << fmt::format( + "Skipping PageTableHostMappingTest because page size {} is unsupported.", host_page_size); + } + + memory.Init(); + if (!memory.InitFastmemArena()) + { + memory.Shutdown(); + GTEST_SKIP() << "Skipping PageTableHostMappingTest because InitFastmemArena failed."; + } + + Core::DeclareAsCPUThread(); + EMM::InstallExceptionHandler(); + system.GetJitInterface().SetJit(std::make_unique(system)); + + // Make sure BATs and SRs are cleared + auto& power_pc = system.GetPowerPC(); + power_pc.Reset(); + + // Set up an SR + SetSR(1, 123); + + // Specify a page table + SetSDR(ALIGNED_PAGE_TABLE_BASE, ALIGNED_PAGE_TABLE_MASK_SMALL); + + // Enable address translation + system.GetPPCState().msr.DR = 1; + system.GetPPCState().msr.IR = 1; + power_pc.MSRUpdated(); + } + + static void TearDownTestSuite() + { + auto& system = Core::System::GetInstance(); + + system.GetJitInterface().SetJit(nullptr); + EMM::UninstallExceptionHandler(); + Core::UndeclareAsCPUThread(); + system.GetMemory().Shutdown(); + } + + static void SetSR(size_t index, u32 vsid) + { + ASSERT_FALSE(index == 4 || index == 7) + << fmt::format("sr{} has conflicts with fake VMEM mapping", index); + + UReg_SR sr{}; + sr.VSID = vsid; + + auto& system = Core::System::GetInstance(); + system.GetPPCState().sr[index] = sr.Hex; + system.GetMMU().SRUpdated(); + } + + static void SetSDR(u32 page_table_base, u32 page_table_mask) + { + UReg_SDR1 sdr; + sdr.htabmask = page_table_mask >> 16; + sdr.reserved = 0; + sdr.htaborg = page_table_base >> 16; + + auto& system = Core::System::GetInstance(); + system.GetPPCState().spr[SPR_SDR] = sdr.Hex; + system.GetMMU().SDRUpdated(); + } + + static void SetBAT(u32 spr, UReg_BAT_Up batu, UReg_BAT_Lo batl) + { + auto& system = Core::System::GetInstance(); + auto& ppc_state = system.GetPPCState(); + auto& mmu = system.GetMMU(); + + ppc_state.spr[spr + 1] = batl.Hex; + ppc_state.spr[spr] = batu.Hex; + + if ((spr >= SPR_IBAT0U && spr <= SPR_IBAT3L) || (spr >= SPR_IBAT4U && spr <= SPR_IBAT7L)) + mmu.IBATUpdated(); + if ((spr >= SPR_DBAT0U && spr <= SPR_DBAT3L) || (spr >= SPR_DBAT4U && spr <= SPR_DBAT7L)) + mmu.DBATUpdated(); + } + + static void SetBAT(u32 spr, u32 logical_address, u32 physical_address, u32 size) + { + UReg_BAT_Up batu{}; + batu.VP = 1; + batu.VS = 1; + batu.BL = (size - 1) >> PowerPC::BAT_INDEX_SHIFT; + batu.BEPI = logical_address >> PowerPC::BAT_INDEX_SHIFT; + + UReg_BAT_Lo batl{}; + batl.PP = 2; + batl.WIMG = 0; + batl.BRPN = physical_address >> PowerPC::BAT_INDEX_SHIFT; + + SetBAT(spr, batu, batl); + } + + static void ExpectMapped(u32 logical_address, u32 physical_address) + { + SCOPED_TRACE( + fmt::format("ExpectMapped({:#010x}, {:#010x})", logical_address, physical_address)); + + auto& memory = Core::System::GetInstance().GetMemory(); + u8* physical_base = memory.GetPhysicalBase(); + u8* logical_base = memory.GetLogicalBase(); + + auto* physical_ptr = reinterpret_cast(physical_base + physical_address); + auto* logical_ptr = reinterpret_cast(logical_base + logical_address); + + *physical_ptr = ++s_counter; + EXPECT_EQ(*logical_ptr, s_counter) + << "Page was mapped to a different physical page than expected"; + + *logical_ptr = ++s_counter; + EXPECT_EQ(*physical_ptr, s_counter) + << "Page was mapped to a different physical page than expected"; + } + +#ifdef _MSC_VER +#define ASAN_DISABLE __declspec(no_sanitize_address) +#else +#define ASAN_DISABLE +#endif + + static void ASAN_DISABLE ExpectReadOnlyMapped(u32 logical_address, u32 physical_address) + { + SCOPED_TRACE( + fmt::format("ExpectReadOnlyMapped({:#010x}, {:#010x})", logical_address, physical_address)); + + auto& memory = Core::System::GetInstance().GetMemory(); + u8* physical_base = memory.GetPhysicalBase(); + u8* logical_base = memory.GetLogicalBase(); + + auto* physical_ptr = reinterpret_cast(physical_base + physical_address); + auto* logical_ptr = reinterpret_cast(logical_base + logical_address); + + *physical_ptr = ++s_counter; + EXPECT_EQ(*logical_ptr, s_counter) + << "Page was mapped to a different physical page than expected"; + + s_detection_address = logical_ptr; + s_detection_count = 0; + + // This line should fault + *logical_ptr = ++s_counter; + + memory.RemovePageTableMappings(s_temporary_mappings); + s_temporary_mappings.clear(); + EXPECT_EQ(s_detection_count, u32(1)) << "Page was mapped as writeable, against expectations"; + } + + static void ASAN_DISABLE ExpectNotMapped(u32 logical_address) + { + SCOPED_TRACE(fmt::format("ExpectNotMapped({:#010x})", logical_address)); + + auto& memory = Core::System::GetInstance().GetMemory(); + u8* logical_base = memory.GetLogicalBase(); + + auto* logical_ptr = reinterpret_cast(logical_base + logical_address); + s_detection_address = logical_ptr; + s_detection_count = 0; + + // This line should fault + *logical_ptr; + + memory.RemovePageTableMappings(s_temporary_mappings); + s_temporary_mappings.clear(); + EXPECT_EQ(s_detection_count, u32(1)) << "Page was mapped, against expectations"; + } + + static void ExpectMappedOnlyIf4KHostPages(u32 logical_address, u32 physical_address) + { + if (s_minimum_mapping_size > PowerPC::HW_PAGE_SIZE) + ExpectNotMapped(logical_address); + else + ExpectMapped(logical_address, physical_address); + } + + static std::pair GetPTE(u32 logical_address, u32 index) + { + auto& system = Core::System::GetInstance(); + auto& ppc_state = system.GetPPCState(); + + const UReg_SR sr(system.GetPPCState().sr[logical_address >> 28]); + u32 hash = sr.VSID ^ (logical_address >> 12); + if ((index & 0x8) != 0) + hash = ~hash; + + const u32 pteg_addr = ((hash << 6) & ppc_state.pagetable_mask) | ppc_state.pagetable_base; + const u32 pte_addr = (index & 0x7) * 8 + pteg_addr; + + const u8* physical_base = system.GetMemory().GetPhysicalBase(); + const UPTE_Lo pte1(Common::swap32(physical_base + pte_addr)); + const UPTE_Hi pte2(Common::swap32(physical_base + pte_addr + 4)); + + return {pte1, pte2}; + } + + static void SetPTE(UPTE_Lo pte1, UPTE_Hi pte2, u32 logical_address, u32 index) + { + auto& system = Core::System::GetInstance(); + auto& ppc_state = system.GetPPCState(); + + pte1.H = (index & 0x8) != 0; + + u32 hash = pte1.VSID ^ (logical_address >> 12); + if (pte1.H) + hash = ~hash; + + const u32 pteg_addr = ((hash << 6) & ppc_state.pagetable_mask) | ppc_state.pagetable_base; + const u32 pte_addr = (index & 0x7) * 8 + pteg_addr; + + u8* physical_base = system.GetMemory().GetPhysicalBase(); + Common::WriteSwap32(physical_base + pte_addr, pte1.Hex); + Common::WriteSwap32(physical_base + pte_addr + 4, pte2.Hex); + + system.GetMMU().InvalidateTLBEntry(logical_address); + } + + static std::pair CreateMapping(u32 logical_address, u32 physical_address) + { + auto& ppc_state = Core::System::GetInstance().GetPPCState(); + + UPTE_Lo pte1{}; + pte1.API = logical_address >> 22; + pte1.VSID = UReg_SR{ppc_state.sr[logical_address >> 28]}.VSID; + pte1.V = 1; // Mapping is valid + + UPTE_Hi pte2{}; + pte2.C = 1; // Page has been written to (MMU.cpp won't map as writeable without this) + pte2.R = 1; // Page has been read from (MMU.cpp won't map at all without this) + pte2.RPN = physical_address >> 12; + + return {pte1, pte2}; + } + + static void AddMapping(u32 logical_address, u32 physical_address, u32 index) + { + auto [pte1, pte2] = CreateMapping(logical_address, physical_address); + SetPTE(pte1, pte2, logical_address, index); + } + + static void AddHostSizedMapping(u32 logical_address, u32 physical_address, u32 index) + { + DisableDR disable_dr; + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + AddMapping(logical_address + i, physical_address + i, index); + } + + static void RemoveMapping(u32 logical_address, u32 physical_address, u32 index) + { + auto [pte1, pte2] = CreateMapping(logical_address, physical_address); + pte1.V = 0; // Mapping is invalid + SetPTE(pte1, pte2, logical_address, index); + } + + static void RemoveHostSizedMapping(u32 logical_address, u32 physical_address, u32 index) + { + DisableDR disable_dr; + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + RemoveMapping(logical_address + i, physical_address + i, index); + } +}; + +TEST_F(PageTableHostMappingTest, Basic) +{ + s_current_temporary_memory = 0x00100000; + + // Create a basic mapping + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + { + ExpectNotMapped(0x10100000 + i); + AddMapping(0x10100000 + i, 0x00100000 + i, 0); + } + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + ExpectMapped(0x10100000 + i, 0x00100000 + i); + + // Create another mapping pointing to the same physical address + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + { + ExpectNotMapped(0x10100000 + s_minimum_mapping_size + i); + AddMapping(0x10100000 + s_minimum_mapping_size + i, 0x00100000 + i, 0); + } + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + { + ExpectMapped(0x10100000 + i, 0x00100000 + i); + ExpectMapped(0x10100000 + s_minimum_mapping_size + i, 0x00100000 + i); + } + + // Remove the first page + RemoveMapping(0x10100000, 0x00100000, 0); + ExpectNotMapped(0x10100000); + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + ExpectMapped(0x10100000 + s_minimum_mapping_size + i, 0x00100000 + i); + + s_current_temporary_memory = TEMPORARY_MEMORY; +} + +TEST_F(PageTableHostMappingTest, LargeHostPageMismatchedAddresses) +{ + { + DisableDR disable_dr; + AddMapping(0x10110000, 0x00111000, 0); + for (u32 i = PowerPC::HW_PAGE_SIZE; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + AddMapping(0x10110000 + i, 0x00110000 + i, 0); + } + + ExpectMappedOnlyIf4KHostPages(0x10110000, 0x00111000); +} + +TEST_F(PageTableHostMappingTest, LargeHostPageMisalignedAddresses) +{ + { + DisableDR disable_dr; + for (u32 i = 0; i < s_minimum_mapping_size * 2; i += PowerPC::HW_PAGE_SIZE) + AddMapping(0x10120000 + i, 0x00121000 + i, 0); + } + + ExpectMappedOnlyIf4KHostPages(0x10120000, 0x00121000); + ExpectMappedOnlyIf4KHostPages(0x10120000 + s_minimum_mapping_size, + 0x00121000 + s_minimum_mapping_size); +} + +TEST_F(PageTableHostMappingTest, ChangeSR) +{ + { + DisableDR disable_dr; + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + { + auto [pte1, pte2] = CreateMapping(0x20130000 + i, 0x00130000 + i); + pte1.VSID = 0xabc; + SetPTE(pte1, pte2, 0x20130000 + i, 0); + } + } + ExpectNotMapped(0x20130000); + + SetSR(2, 0xabc); + ExpectMapped(0x20130000, 0x00130000); + ExpectNotMapped(0x30130000); + + SetSR(3, 0xabc); + ExpectMapped(0x20130000, 0x00130000); + ExpectMapped(0x30130000, 0x00130000); + ExpectNotMapped(0x00130000); + ExpectNotMapped(0x10130000); +} + +// DBAT takes priority over page table mappings. +TEST_F(PageTableHostMappingTest, DBATPriority) +{ + SetSR(5, 5); + + AddHostSizedMapping(0x50140000, 0x00150000, 0); + ExpectMapped(0x50140000, 0x00150000); + + SetBAT(SPR_DBAT0U, 0x50000000, 0x00000000, 0x01000000); + ExpectMapped(0x50140000, 0x00140000); +} + +// Host-side page table mappings are for data only, so IBAT has no effect on them. +TEST_F(PageTableHostMappingTest, IBATPriority) +{ + SetSR(6, 6); + + AddHostSizedMapping(0x60160000, 0x00170000, 0); + ExpectMapped(0x60160000, 0x00170000); + + SetBAT(SPR_IBAT0U, 0x60000000, 0x00000000, 0x01000000); + ExpectMapped(0x60160000, 0x00170000); +} + +TEST_F(PageTableHostMappingTest, Priority) +{ + // Secondary PTEs for 0x10180000 + + AddHostSizedMapping(0x10180000, 0x00180000, 10); + ExpectMapped(0x10180000, 0x00180000); + + AddHostSizedMapping(0x10180000, 0x00190000, 12); + ExpectMapped(0x10180000, 0x00180000); + + AddHostSizedMapping(0x10180000, 0x001a0000, 8); + ExpectMapped(0x10180000, 0x001a0000); + + RemoveHostSizedMapping(0x10180000, 0x00180000, 10); + ExpectMapped(0x10180000, 0x001a0000); + + RemoveHostSizedMapping(0x10180000, 0x001a0000, 8); + ExpectMapped(0x10180000, 0x00190000); + + // Primary PTEs for 0x10180000 + + AddHostSizedMapping(0x10180000, 0x00180000, 2); + ExpectMapped(0x10180000, 0x00180000); + + AddHostSizedMapping(0x10180000, 0x001a0000, 4); + ExpectMapped(0x10180000, 0x00180000); + + AddHostSizedMapping(0x10180000, 0x001b0000, 0); + ExpectMapped(0x10180000, 0x001b0000); + + RemoveHostSizedMapping(0x10180000, 0x00180000, 2); + ExpectMapped(0x10180000, 0x001b0000); + + RemoveHostSizedMapping(0x10180000, 0x001b0000, 0); + ExpectMapped(0x10180000, 0x001a0000); + + // Return to secondary PTE for 0x10180000 + + RemoveHostSizedMapping(0x10180000, 0x001a0000, 4); + ExpectMapped(0x10180000, 0x00190000); + + // Secondary PTEs for 0x11180000 + + AddHostSizedMapping(0x11180000, 0x01180000, 11); + ExpectMapped(0x11180000, 0x01180000); + + AddHostSizedMapping(0x11180000, 0x01190000, 13); + ExpectMapped(0x11180000, 0x01180000); + + AddHostSizedMapping(0x11180000, 0x011a0000, 9); + ExpectMapped(0x11180000, 0x011a0000); + + RemoveHostSizedMapping(0x11180000, 0x01180000, 11); + ExpectMapped(0x11180000, 0x011a0000); + + RemoveHostSizedMapping(0x11180000, 0x011a0000, 9); + ExpectMapped(0x11180000, 0x01190000); + + // Primary PTEs for 0x11180000 + + AddHostSizedMapping(0x11180000, 0x01180000, 3); + ExpectMapped(0x11180000, 0x01180000); + + AddHostSizedMapping(0x11180000, 0x011a0000, 5); + ExpectMapped(0x11180000, 0x01180000); + + AddHostSizedMapping(0x11180000, 0x011b0000, 1); + ExpectMapped(0x11180000, 0x011b0000); + + RemoveHostSizedMapping(0x11180000, 0x01180000, 3); + ExpectMapped(0x11180000, 0x011b0000); + + RemoveHostSizedMapping(0x11180000, 0x011b0000, 1); + ExpectMapped(0x11180000, 0x011a0000); + + // Return to secondary PTE for 0x11180000 + + RemoveHostSizedMapping(0x11180000, 0x011a0000, 5); + ExpectMapped(0x11180000, 0x01190000); + + // Check that 0x10180000 is still working properly + + ExpectMapped(0x10180000, 0x00190000); + + AddHostSizedMapping(0x10180000, 0x00180000, 0); + ExpectMapped(0x10180000, 0x00180000); + + // Check that 0x11180000 is still working properly + + ExpectMapped(0x11180000, 0x01190000); + + AddHostSizedMapping(0x11180000, 0x01180000, 1); + ExpectMapped(0x11180000, 0x01180000); +} + +TEST_F(PageTableHostMappingTest, ChangeAddress) +{ + // Initial mapping + AddHostSizedMapping(0x101c0000, 0x001c0000, 0); + ExpectMapped(0x101c0000, 0x001c0000); + + // Change physical address + AddHostSizedMapping(0x101c0000, 0x001d0000, 0); + ExpectMapped(0x101c0000, 0x001d0000); + + // Change logical address + AddHostSizedMapping(0x111c0000, 0x001d0000, 0); + ExpectMapped(0x111c0000, 0x001d0000); + ExpectNotMapped(0x101c0000); + + // Change both logical address and physical address + AddHostSizedMapping(0x101c0000, 0x011d0000, 0); + ExpectMapped(0x101c0000, 0x011d0000); + ExpectNotMapped(0x111c0000); +} + +TEST_F(PageTableHostMappingTest, InvalidPhysicalAddress) +{ + AddHostSizedMapping(0x101d0000, 0x0ff00000, 0); + ExpectNotMapped(0x101d0000); +} + +TEST_F(PageTableHostMappingTest, WIMG) +{ + for (u32 i = 0; i < 16; ++i) + { + { + DisableDR disable_dr; + for (u32 j = 0; j < s_minimum_mapping_size; j += PowerPC::HW_PAGE_SIZE) + { + auto [pte1, pte2] = CreateMapping(0x101e0000 + j, 0x001e0000 + j); + pte2.WIMG = i; + SetPTE(pte1, pte2, 0x101e0000 + j, 0); + } + } + + if ((i & 0b1100) != 0) + ExpectNotMapped(0x101e0000); + else + ExpectMapped(0x101e0000, 0x001e0000); + } +} + +TEST_F(PageTableHostMappingTest, RC) +{ + auto& mmu = Core::System::GetInstance().GetMMU(); + + const auto set_up_mapping = [] { + DisableDR disable_dr; + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + { + auto [pte1, pte2] = CreateMapping(0x101f0000 + i, 0x001f0000 + i); + pte2.R = 0; + pte2.C = 0; + SetPTE(pte1, pte2, 0x101f0000 + i, 0); + } + }; + + const auto expect_rc = [](u32 r, u32 c) { + auto [pte1, pte2] = GetPTE(0x101f0000, 0); + EXPECT_TRUE(pte1.V); + EXPECT_EQ(pte2.R, r); + EXPECT_EQ(pte2.C, c); + }; + + // Start with R=0, C=0 + set_up_mapping(); + ExpectNotMapped(0x101f0000); + expect_rc(0, 0); + + // Automatically set R=1, C=0 + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + mmu.Read(0x101f0000 + i); + ExpectReadOnlyMapped(0x101f0000, 0x001f0000); + expect_rc(1, 0); + + // Automatically set R=1, C=1 + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + mmu.Write(0x12345678, 0x101f0000 + i); + ExpectMapped(0x101f0000, 0x001f0000); + expect_rc(1, 1); + + // Start over with R=0, C=0 + set_up_mapping(); + ExpectNotMapped(0x101f0000); + expect_rc(0, 0); + + // Automatically set R=1, C=1 + for (u32 i = 0; i < s_minimum_mapping_size; i += PowerPC::HW_PAGE_SIZE) + mmu.Write(0x12345678, 0x101f0000 + i); + ExpectMapped(0x101f0000, 0x001f0000); + expect_rc(1, 1); +} + +TEST_F(PageTableHostMappingTest, ResizePageTable) +{ + AddHostSizedMapping(0x10200000, 0x00200000, 0); + AddHostSizedMapping(0x10600000, 0x00210000, 1); + ExpectMapped(0x10200000, 0x00200000); + ExpectMapped(0x10600000, 0x00210000); + + SetSDR(ALIGNED_PAGE_TABLE_BASE, ALIGNED_PAGE_TABLE_MASK_LARGE); + ExpectMapped(0x10200000, 0x00200000); + ExpectNotMapped(0x10600000); + + AddHostSizedMapping(0x10600000, 0x00220000, 1); + ExpectMapped(0x10200000, 0x00200000); + ExpectMapped(0x10600000, 0x00220000); + + AddHostSizedMapping(0x10610000, 0x00200000, 1); + ExpectMapped(0x10610000, 0x00200000); + + SetSDR(ALIGNED_PAGE_TABLE_BASE, ALIGNED_PAGE_TABLE_MASK_SMALL); + ExpectMapped(0x10200000, 0x00200000); + ExpectMapped(0x10600000, 0x00210000); + ExpectNotMapped(0x10610000); +} + +// The PEM says that all bits that are one in the page table mask must be zero in the page table +// address. What it doesn't tell you is that if this isn't obeyed, the Gekko will do a logical OR of +// the page table base and the page table offset, producing behavior that might not be intuitive. +TEST_F(PageTableHostMappingTest, MisalignedPageTable) +{ + SetSDR(MISALIGNED_PAGE_TABLE_BASE + 0x10000, PowerPC::PAGE_TABLE_MIN_SIZE - 1); + + AddHostSizedMapping(0x10a30000, 0x00230000, 4); + ExpectMapped(0x10a30000, 0x00230000); + + SetSDR(MISALIGNED_PAGE_TABLE_BASE, MISALIGNED_PAGE_TABLE_MASK); + + ExpectNotMapped(0x10a30000); + + AddHostSizedMapping(0x10230000, 0x00240000, 0); + AddHostSizedMapping(0x10630000, 0x00250000, 1); + AddHostSizedMapping(0x10a30000, 0x00260000, 2); + AddHostSizedMapping(0x10e30000, 0x00270000, 3); + + ExpectMapped(0x10230000, 0x00240000); + ExpectMapped(0x10630000, 0x00250000); + ExpectMapped(0x10a30000, 0x00260000); + ExpectMapped(0x10e30000, 0x00270000); + + // Exercise the code for falling back to a secondary PTE after removing a primary PTE. + AddHostSizedMapping(0x10a30000, 0x00270000, 10); + AddHostSizedMapping(0x10e30000, 0x00260000, 11); + RemoveHostSizedMapping(0x10a30000, 0x00260000, 2); + RemoveHostSizedMapping(0x10e30000, 0x00250000, 3); + + ExpectMapped(0x10230000, 0x00240000); + ExpectMapped(0x10630000, 0x00250000); + ExpectMapped(0x10a30000, 0x00270000); + ExpectMapped(0x10e30000, 0x00260000); + + SetSDR(MISALIGNED_PAGE_TABLE_BASE_ALIGNED, MISALIGNED_PAGE_TABLE_MASK); + + ExpectNotMapped(0x10230000); + ExpectMapped(0x10630000, 0x00250000); + ExpectMapped(0x10a30000, 0x00230000); + ExpectNotMapped(0x10e30000); + + SetSDR(ALIGNED_PAGE_TABLE_BASE, ALIGNED_PAGE_TABLE_MASK_SMALL); +} + +// Putting a zero in the middle of the page table mask's ones results in similar behavior +// to the scenario described above. +TEST_F(PageTableHostMappingTest, HoleInMask) +{ + SetSDR(HOLE_MASK_PAGE_TABLE_BASE + 0x10000, PowerPC::PAGE_TABLE_MIN_SIZE - 1); + + AddHostSizedMapping(0x10680000, 0x00280000, 4); + ExpectMapped(0x10680000, 0x00280000); + + SetSDR(HOLE_MASK_PAGE_TABLE_BASE, HOLE_MASK_PAGE_TABLE_MASK); + + ExpectNotMapped(0x10680000); + + AddHostSizedMapping(0x10280000, 0x00290000, 0); + AddHostSizedMapping(0x10680000, 0x002a0000, 1); + AddHostSizedMapping(0x10a80000, 0x002b0000, 2); + AddHostSizedMapping(0x10e80000, 0x002c0000, 3); + + ExpectMapped(0x10280000, 0x00290000); + ExpectMapped(0x10680000, 0x002a0000); + ExpectMapped(0x10a80000, 0x002b0000); + ExpectMapped(0x10e80000, 0x002c0000); + + // Exercise the code for falling back to a secondary PTE after removing a primary PTE. + AddHostSizedMapping(0x10a80000, 0x002c0000, 10); + AddHostSizedMapping(0x10e80000, 0x002b0000, 11); + RemoveHostSizedMapping(0x10a80000, 0x002b0000, 2); + RemoveHostSizedMapping(0x10e80000, 0x002c0000, 3); + + ExpectMapped(0x10280000, 0x00290000); + ExpectMapped(0x10680000, 0x002a0000); + ExpectMapped(0x10a80000, 0x002c0000); + ExpectMapped(0x10e80000, 0x002b0000); + + SetSDR(HOLE_MASK_PAGE_TABLE_BASE, HOLE_MASK_PAGE_TABLE_MASK_WITHOUT_HOLE); + + ExpectMapped(0x10280000, 0x00290000); + ExpectMapped(0x10680000, 0x00280000); + ExpectNotMapped(0x10a80000); + ExpectMapped(0x10e80000, 0x002b0000); + + SetSDR(ALIGNED_PAGE_TABLE_BASE, ALIGNED_PAGE_TABLE_MASK_SMALL); +} + +// If we combine the two scenarios above, both making the base misaligned and putting a hole in the +// mask, we get the same result as if we just make the base misaligned. +TEST_F(PageTableHostMappingTest, HoleInMaskMisalignedPageTable) +{ + SetSDR(MISALIGNED_PAGE_TABLE_BASE + 0x10000, PowerPC::PAGE_TABLE_MIN_SIZE - 1); + + AddHostSizedMapping(0x10ad0000, 0x002d0000, 4); + ExpectMapped(0x10ad0000, 0x002d0000); + + SetSDR(MISALIGNED_PAGE_TABLE_BASE, MISALIGNED_PAGE_TABLE_MASK); + + ExpectNotMapped(0x10ad0000); + + AddHostSizedMapping(0x102d0000, 0x002e0000, 0); + AddHostSizedMapping(0x106d0000, 0x002f0000, 1); + AddHostSizedMapping(0x10ad0000, 0x00300000, 2); + AddHostSizedMapping(0x10ed0000, 0x00310000, 3); + + ExpectMapped(0x102d0000, 0x002e0000); + ExpectMapped(0x106d0000, 0x002f0000); + ExpectMapped(0x10ad0000, 0x00300000); + ExpectMapped(0x10ed0000, 0x00310000); + + // Exercise the code for falling back to a secondary PTE after removing a primary PTE. + AddHostSizedMapping(0x10ad0000, 0x00310000, 10); + AddHostSizedMapping(0x10ed0000, 0x00300000, 11); + RemoveHostSizedMapping(0x10ad0000, 0x00300000, 2); + RemoveHostSizedMapping(0x10ed0000, 0x00310000, 3); + + ExpectMapped(0x102d0000, 0x002e0000); + ExpectMapped(0x106d0000, 0x002f0000); + ExpectMapped(0x10ad0000, 0x00310000); + ExpectMapped(0x10ed0000, 0x00300000); + + SetSDR(MISALIGNED_PAGE_TABLE_BASE_ALIGNED, MISALIGNED_PAGE_TABLE_MASK); + + ExpectNotMapped(0x102d0000); + ExpectMapped(0x106d0000, 0x002f0000); + ExpectMapped(0x10ad0000, 0x002d0000); + ExpectNotMapped(0x10ed0000); + + SetSDR(ALIGNED_PAGE_TABLE_BASE, ALIGNED_PAGE_TABLE_MASK_SMALL); +} + +TEST_F(PageTableHostMappingTest, MemChecks) +{ + AddHostSizedMapping(0x10320000, 0x00330000, 0); + AddHostSizedMapping(0x10330000, 0x00320000, 0); + ExpectMapped(0x10320000, 0x00330000); + ExpectMapped(0x10330000, 0x00320000); + + auto& memchecks = Core::System::GetInstance().GetPowerPC().GetMemChecks(); + TMemCheck memcheck; + memcheck.start_address = 0x10320000; + memcheck.end_address = 0x10320001; + memchecks.Add(std::move(memcheck)); + + ExpectNotMapped(0x10320000); + ExpectMapped(0x10330000, 0x00320000); + + memchecks.Remove(0x10320000); + + ExpectMapped(0x10320000, 0x00330000); + ExpectMapped(0x10330000, 0x00320000); +} diff --git a/Source/UnitTests/Core/StubJit.h b/Source/UnitTests/Core/StubJit.h new file mode 100644 index 00000000000..50a1e6efb54 --- /dev/null +++ b/Source/UnitTests/Core/StubJit.h @@ -0,0 +1,38 @@ +// Copyright 2026 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include "Core/PowerPC/JitCommon/JitBase.h" +#include "Core/PowerPC/JitCommon/JitCache.h" + +class StubJit : public JitBase +{ +public: + explicit StubJit(Core::System& system) : JitBase(system) {} + + // CPUCoreBase methods + void Init() override {} + void Shutdown() override {} + void ClearCache() override {} + void Run() override {} + void SingleStep() override {} + const char* GetName() const override { return nullptr; } + // JitBase methods + JitBaseBlockCache* GetBlockCache() override { return nullptr; } + void Jit(u32) override {} + void EraseSingleBlock(const JitBlock&) override {} + std::vector GetMemoryStats() const override { return {}; } + std::size_t DisassembleNearCode(const JitBlock&, std::ostream&) const override { return 0; } + std::size_t DisassembleFarCode(const JitBlock&, std::ostream&) const override { return 0; } + const CommonAsmRoutinesBase* GetAsmRoutines() override { return nullptr; } + bool HandleFault(uintptr_t, SContext*) override { return false; } +}; + +class StubBlockCache : public JitBaseBlockCache +{ +public: + explicit StubBlockCache(JitBase& jit) : JitBaseBlockCache(jit) {} + + void WriteLinkBlock(const JitBlock::LinkData&, const JitBlock*) override {} +}; diff --git a/Source/UnitTests/UnitTests.vcxproj b/Source/UnitTests/UnitTests.vcxproj index 5ffba903d93..6b476dad940 100644 --- a/Source/UnitTests/UnitTests.vcxproj +++ b/Source/UnitTests/UnitTests.vcxproj @@ -32,6 +32,7 @@ + @@ -74,6 +75,7 @@ +