dahjah/shadPS4
1
0
Fork 0
mirror of https://github.com/shadps4-emu/shadPS4.git synced 2026-02-02 20:50:09 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity
98fd0689ac
shadPS4/src/core/memory.cpp
Stephen Miller f6ae5544fd
Some checks failed
Build and Release / reuse (push) Has been cancelled
Details
Build and Release / clang-format (push) Has been cancelled
Details
Build and Release / get-info (push) Has been cancelled
Details
Build and Release / windows-sdl (push) Has been cancelled
Details
Build and Release / macos-sdl (push) Has been cancelled
Details
Build and Release / linux-sdl (push) Has been cancelled
Details
Build and Release / linux-sdl-gcc (push) Has been cancelled
Details
Build and Release / pre-release (push) Has been cancelled
Details
Kernel.Vmm: Protect Fixes (#3822) 
* Some mprotect fixes

The biggest thing here is preventing mprotect on memory that isn't mapped in address space. This would cause exceptions before, but succeeds on real hardware.
I've also included a couple other minor fixes, mostly based around some tests I recently performed.

Note: All changes to memory pools in this PR are assumed. I have not yet tested memory pools with any of this logic, but I do at least want to prevent mprotect on pool reserved memory to avoid crashes.

* Update memory.cpp

* clang
2025-11-22 10:32:53 +02:00

1352 lines
52 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: Copyright 2025 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "common/alignment.h"
#include "common/assert.h"
#include "common/config.h"
#include "common/debug.h"
#include "core/file_sys/fs.h"
#include "core/libraries/kernel/memory.h"
#include "core/libraries/kernel/orbis_error.h"
#include "core/libraries/kernel/process.h"
#include "core/memory.h"
#include "video_core/renderer_vulkan/vk_rasterizer.h"
namespace Core {
MemoryManager::MemoryManager() {
LOG_INFO(Kernel_Vmm, "Virtual memory space initialized with regions:");
// Construct vma_map using the regions reserved by the address space
auto regions = impl.GetUsableRegions();
u64 total_usable_space = 0;
for (auto region : regions) {
vma_map.emplace(region.lower(),
VirtualMemoryArea{region.lower(), region.upper() - region.lower()});
LOG_INFO(Kernel_Vmm, "{:#x} - {:#x}", region.lower(), region.upper());
}
}
MemoryManager::~MemoryManager() = default;
void MemoryManager::SetupMemoryRegions(u64 flexible_size, bool use_extended_mem1,
bool use_extended_mem2) {
const bool is_neo = ::Libraries::Kernel::sceKernelIsNeoMode();
auto total_size = is_neo ? ORBIS_KERNEL_TOTAL_MEM_PRO : ORBIS_KERNEL_TOTAL_MEM;
if (Config::isDevKitConsole()) {
total_size = is_neo ? ORBIS_KERNEL_TOTAL_MEM_DEV_PRO : ORBIS_KERNEL_TOTAL_MEM_DEV;
}
s32 extra_dmem = Config::getExtraDmemInMbytes();
if (Config::getExtraDmemInMbytes() != 0) {
LOG_WARNING(Kernel_Vmm,
"extraDmemInMbytes is {} MB! Old Direct Size: {:#x} -> New Direct Size: {:#x}",
extra_dmem, total_size, total_size + extra_dmem * 1_MB);
total_size += extra_dmem * 1_MB;
}
if (!use_extended_mem1 && is_neo) {
total_size -= 256_MB;
}
if (!use_extended_mem2 && !is_neo) {
total_size -= 128_MB;
}
total_flexible_size = flexible_size - ORBIS_FLEXIBLE_MEMORY_BASE;
total_direct_size = total_size - flexible_size;
// Insert an area that covers the direct memory physical address block.
// Note that this should never be called after direct memory allocations have been made.
dmem_map.clear();
dmem_map.emplace(0, DirectMemoryArea{0, total_direct_size});
// Insert an area that covers the flexible memory physical address block.
// Note that this should never be called after flexible memory allocations have been made.
const auto remaining_physical_space = total_size - total_direct_size;
fmem_map.clear();
fmem_map.emplace(total_direct_size,
FlexibleMemoryArea{total_direct_size, remaining_physical_space});
LOG_INFO(Kernel_Vmm, "Configured memory regions: flexible size = {:#x}, direct size = {:#x}",
total_flexible_size, total_direct_size);
}
u64 MemoryManager::ClampRangeSize(VAddr virtual_addr, u64 size) {
static constexpr u64 MinSizeToClamp = 3_GB;
// Dont bother with clamping if the size is small so we dont pay a map lookup on every buffer.
if (size < MinSizeToClamp) {
return size;
}
ASSERT_MSG(IsValidMapping(virtual_addr), "Attempted to access invalid address {:#x}",
virtual_addr);
// Clamp size to the remaining size of the current VMA.
auto vma = FindVMA(virtual_addr);
u64 clamped_size = vma->second.base + vma->second.size - virtual_addr;
++vma;
// Keep adding to the size while there is contigious virtual address space.
while (vma != vma_map.end() && vma->second.IsMapped() && clamped_size < size) {
clamped_size += vma->second.size;
++vma;
}
clamped_size = std::min(clamped_size, size);
if (size != clamped_size) {
LOG_WARNING(Kernel_Vmm, "Clamped requested buffer range addr={:#x}, size={:#x} to {:#x}",
virtual_addr, size, clamped_size);
}
return clamped_size;
}
void MemoryManager::SetPrtArea(u32 id, VAddr address, u64 size) {
PrtArea& area = prt_areas[id];
if (area.mapped) {
rasterizer->UnmapMemory(area.start, area.end - area.start);
}
area.start = address;
area.end = address + size;
area.mapped = true;
// Pretend the entire PRT area is mapped to avoid GPU tracking errors.
// The caches will use CopySparseMemory to fetch data which avoids unmapped areas.
rasterizer->MapMemory(address, size);
}
void MemoryManager::CopySparseMemory(VAddr virtual_addr, u8* dest, u64 size) {
ASSERT_MSG(IsValidMapping(virtual_addr), "Attempted to access invalid address {:#x}",
virtual_addr);
auto vma = FindVMA(virtual_addr);
while (size) {
u64 copy_size = std::min<u64>(vma->second.size - (virtual_addr - vma->first), size);
if (vma->second.IsMapped()) {
std::memcpy(dest, std::bit_cast<const u8*>(virtual_addr), copy_size);
} else {
std::memset(dest, 0, copy_size);
}
size -= copy_size;
virtual_addr += copy_size;
dest += copy_size;
++vma;
}
}
bool MemoryManager::TryWriteBacking(void* address, const void* data, u32 num_bytes) {
const VAddr virtual_addr = std::bit_cast<VAddr>(address);
ASSERT_MSG(IsValidMapping(virtual_addr, num_bytes), "Attempted to access invalid address {:#x}",
virtual_addr);
const auto& vma = FindVMA(virtual_addr)->second;
if (!HasPhysicalBacking(vma)) {
return false;
}
u8* backing = impl.BackingBase() + vma.phys_base + (virtual_addr - vma.base);
memcpy(backing, data, num_bytes);
return true;
}
PAddr MemoryManager::PoolExpand(PAddr search_start, PAddr search_end, u64 size, u64 alignment) {
std::scoped_lock lk{mutex};
alignment = alignment > 0 ? alignment : 64_KB;
auto dmem_area = FindDmemArea(search_start);
auto mapping_start = search_start > dmem_area->second.base
? Common::AlignUp(search_start, alignment)
: Common::AlignUp(dmem_area->second.base, alignment);
auto mapping_end = mapping_start + size;
// Find the first free, large enough dmem area in the range.
while (dmem_area->second.dma_type != DMAType::Free ||
dmem_area->second.GetEnd() < mapping_end) {
// The current dmem_area isn't suitable, move to the next one.
dmem_area++;
if (dmem_area == dmem_map.end()) {
break;
}
// Update local variables based on the new dmem_area
mapping_start = Common::AlignUp(dmem_area->second.base, alignment);
mapping_end = mapping_start + size;
}
if (dmem_area == dmem_map.end()) {
// There are no suitable mappings in this range
LOG_ERROR(Kernel_Vmm, "Unable to find free direct memory area: size = {:#x}", size);
return -1;
}
// Add the allocated region to the list and commit its pages.
auto& area = CarveDmemArea(mapping_start, size)->second;
area.dma_type = DMAType::Pooled;
area.memory_type = 3;
// Track how much dmem was allocated for pools.
pool_budget += size;
return mapping_start;
}
PAddr MemoryManager::Allocate(PAddr search_start, PAddr search_end, u64 size, u64 alignment,
s32 memory_type) {
std::scoped_lock lk{mutex};
alignment = alignment > 0 ? alignment : 16_KB;
auto dmem_area = FindDmemArea(search_start);
auto mapping_start = search_start > dmem_area->second.base
? Common::AlignUp(search_start, alignment)
: Common::AlignUp(dmem_area->second.base, alignment);
auto mapping_end = mapping_start + size;
// Find the first free, large enough dmem area in the range.
while (dmem_area->second.dma_type != DMAType::Free ||
dmem_area->second.GetEnd() < mapping_end) {
// The current dmem_area isn't suitable, move to the next one.
dmem_area++;
if (dmem_area == dmem_map.end()) {
break;
}
// Update local variables based on the new dmem_area
mapping_start = Common::AlignUp(dmem_area->second.base, alignment);
mapping_end = mapping_start + size;
}
if (dmem_area == dmem_map.end()) {
// There are no suitable mappings in this range
LOG_ERROR(Kernel_Vmm, "Unable to find free direct memory area: size = {:#x}", size);
return -1;
}
// Add the allocated region to the list and commit its pages.
auto& area = CarveDmemArea(mapping_start, size)->second;
area.memory_type = memory_type;
area.dma_type = DMAType::Allocated;
MergeAdjacent(dmem_map, dmem_area);
return mapping_start;
}
void MemoryManager::Free(PAddr phys_addr, u64 size) {
std::scoped_lock lk{mutex};
// Release any dmem mappings that reference this physical block.
std::vector<std::pair<VAddr, u64>> remove_list;
for (const auto& [addr, mapping] : vma_map) {
if (mapping.type != VMAType::Direct) {
continue;
}
if (mapping.phys_base <= phys_addr && phys_addr < mapping.phys_base + mapping.size) {
const auto vma_start_offset = phys_addr - mapping.phys_base;
const auto addr_in_vma = mapping.base + vma_start_offset;
const auto size_in_vma =
mapping.size - vma_start_offset > size ? size : mapping.size - vma_start_offset;
LOG_INFO(Kernel_Vmm, "Unmaping direct mapping {:#x} with size {:#x}", addr_in_vma,
size_in_vma);
// Unmaping might erase from vma_map. We can't do it here.
remove_list.emplace_back(addr_in_vma, size_in_vma);
}
}
for (const auto& [addr, size] : remove_list) {
UnmapMemoryImpl(addr, size);
}
// Unmap all dmem areas within this area.
auto phys_addr_to_search = phys_addr;
auto remaining_size = size;
auto dmem_area = FindDmemArea(phys_addr);
while (dmem_area != dmem_map.end() && remaining_size > 0) {
// Carve a free dmem area in place of this one.
const auto start_phys_addr =
phys_addr > dmem_area->second.base ? phys_addr : dmem_area->second.base;
const auto offset_in_dma = start_phys_addr - dmem_area->second.base;
const auto size_in_dma = dmem_area->second.size - offset_in_dma > remaining_size
? remaining_size
: dmem_area->second.size - offset_in_dma;
const auto dmem_handle = CarveDmemArea(start_phys_addr, size_in_dma);
auto& new_dmem_area = dmem_handle->second;
new_dmem_area.dma_type = DMAType::Free;
new_dmem_area.memory_type = 0;
// Merge the new dmem_area with dmem_map
MergeAdjacent(dmem_map, dmem_handle);
// Get the next relevant dmem area.
phys_addr_to_search = phys_addr + size_in_dma;
remaining_size -= size_in_dma;
dmem_area = FindDmemArea(phys_addr_to_search);
}
}
s32 MemoryManager::PoolCommit(VAddr virtual_addr, u64 size, MemoryProt prot, s32 mtype) {
ASSERT_MSG(IsValidMapping(virtual_addr, size), "Attempted to access invalid address {:#x}",
virtual_addr);
std::scoped_lock lk{mutex};
// Input addresses to PoolCommit are treated as fixed, and have a constant alignment.
const u64 alignment = 64_KB;
VAddr mapped_addr = Common::AlignUp(virtual_addr, alignment);
auto& vma = FindVMA(mapped_addr)->second;
if (vma.type != VMAType::PoolReserved) {
// If we're attempting to commit non-pooled memory, return EINVAL
LOG_ERROR(Kernel_Vmm, "Attempting to commit non-pooled memory at {:#x}", mapped_addr);
return ORBIS_KERNEL_ERROR_EINVAL;
}
if (!vma.Contains(mapped_addr, size)) {
// If there's not enough space to commit, return EINVAL
LOG_ERROR(Kernel_Vmm,
"Pooled region {:#x} to {:#x} is not large enough to commit from {:#x} to {:#x}",
vma.base, vma.base + vma.size, mapped_addr, mapped_addr + size);
return ORBIS_KERNEL_ERROR_EINVAL;
}
if (pool_budget <= size) {
// If there isn't enough pooled memory to perform the mapping, return ENOMEM
LOG_ERROR(Kernel_Vmm, "Not enough pooled memory to perform mapping");
return ORBIS_KERNEL_ERROR_ENOMEM;
} else {
// Track how much pooled memory this commit will take
pool_budget -= size;
}
if (True(prot & MemoryProt::CpuWrite)) {
// On PS4, read is appended to write mappings.
prot |= MemoryProt::CpuRead;
}
// Carve out the new VMA representing this mapping
const auto new_vma_handle = CarveVMA(mapped_addr, size);
auto& new_vma = new_vma_handle->second;
new_vma.disallow_merge = false;
new_vma.prot = prot;
new_vma.name = "anon";
new_vma.type = Core::VMAType::Pooled;
new_vma.is_exec = false;
// Find a suitable physical address
auto handle = dmem_map.begin();
while (handle != dmem_map.end() &&
(handle->second.dma_type != Core::DMAType::Pooled || handle->second.size < size)) {
handle++;
}
ASSERT_MSG(handle != dmem_map.end() && handle->second.dma_type == Core::DMAType::Pooled,
"No suitable physical memory areas to map");
// Use the start of this area as the physical backing for this mapping.
const auto new_dmem_handle = CarveDmemArea(handle->second.base, size);
auto& new_dmem_area = new_dmem_handle->second;
new_dmem_area.dma_type = DMAType::Committed;
new_dmem_area.memory_type = mtype;
new_vma.phys_base = new_dmem_area.base;
MergeAdjacent(dmem_map, new_dmem_handle);
// Perform the mapping
void* out_addr = impl.Map(mapped_addr, size, alignment, new_vma.phys_base, false);
TRACK_ALLOC(out_addr, size, "VMEM");
if (IsValidGpuMapping(mapped_addr, size)) {
rasterizer->MapMemory(mapped_addr, size);
}
return ORBIS_OK;
}
s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, MemoryProt prot,
MemoryMapFlags flags, VMAType type, std::string_view name,
bool validate_dmem, PAddr phys_addr, u64 alignment) {
// Certain games perform flexible mappings on loop to determine
// the available flexible memory size. Questionable but we need to handle this.
if (type == VMAType::Flexible && flexible_usage + size > total_flexible_size) {
LOG_ERROR(Kernel_Vmm,
"Out of flexible memory, available flexible memory = {:#x}"
" requested size = {:#x}",
total_flexible_size - flexible_usage, size);
return ORBIS_KERNEL_ERROR_EINVAL;
}
std::scoped_lock lk{mutex};
// Validate the requested physical address range
if (phys_addr != -1) {
if (total_direct_size < phys_addr + size) {
LOG_ERROR(Kernel_Vmm, "Unable to map {:#x} bytes at physical address {:#x}", size,
phys_addr);
return ORBIS_KERNEL_ERROR_ENOMEM;
}
// Validate direct memory areas involved in this call.
auto dmem_area = FindDmemArea(phys_addr);
while (dmem_area != dmem_map.end() && dmem_area->second.base < phys_addr + size) {
// If any requested dmem area is not allocated, return an error.
if (dmem_area->second.dma_type != DMAType::Allocated &&
dmem_area->second.dma_type != DMAType::Mapped) {
LOG_ERROR(Kernel_Vmm, "Unable to map {:#x} bytes at physical address {:#x}", size,
phys_addr);
return ORBIS_KERNEL_ERROR_ENOMEM;
}
// If we need to perform extra validation, then check for Mapped dmem areas too.
if (validate_dmem && dmem_area->second.dma_type == DMAType::Mapped) {
LOG_ERROR(Kernel_Vmm, "Unable to map {:#x} bytes at physical address {:#x}", size,
phys_addr);
return ORBIS_KERNEL_ERROR_EBUSY;
}
dmem_area++;
}
// If the prior loop succeeds, we need to loop through again and carve out mapped dmas.
// This needs to be a separate loop to avoid modifying dmem map during failed calls.
auto phys_addr_to_search = phys_addr;
auto remaining_size = size;
dmem_area = FindDmemArea(phys_addr);
while (dmem_area != dmem_map.end() && remaining_size > 0) {
// Carve a new dmem area in place of this one with the appropriate type.
// Ensure the carved area only covers the current dmem area.
const auto start_phys_addr =
phys_addr > dmem_area->second.base ? phys_addr : dmem_area->second.base;
const auto offset_in_dma = start_phys_addr - dmem_area->second.base;
const auto size_in_dma = dmem_area->second.size - offset_in_dma > remaining_size
? remaining_size
: dmem_area->second.size - offset_in_dma;
const auto dmem_handle = CarveDmemArea(start_phys_addr, size_in_dma);
auto& new_dmem_area = dmem_handle->second;
new_dmem_area.dma_type = DMAType::Mapped;
// Merge the new dmem_area with dmem_map
MergeAdjacent(dmem_map, dmem_handle);
// Get the next relevant dmem area.
phys_addr_to_search = phys_addr + size_in_dma;
remaining_size -= size_in_dma;
dmem_area = FindDmemArea(phys_addr_to_search);
}
}
// Limit the minimum address to SystemManagedVirtualBase to prevent hardware-specific issues.
VAddr mapped_addr = (virtual_addr == 0) ? impl.SystemManagedVirtualBase() : virtual_addr;
// Fixed mapping means the virtual address must exactly match the provided one.
// On a PS4, the Fixed flag is ignored if address 0 is provided.
if (True(flags & MemoryMapFlags::Fixed) && virtual_addr != 0) {
ASSERT_MSG(IsValidMapping(mapped_addr, size), "Attempted to access invalid address {:#x}",
mapped_addr);
auto vma = FindVMA(mapped_addr)->second;
// There's a possible edge case where we're mapping to a partially reserved range.
// To account for this, unmap any reserved areas within this mapping range first.
auto unmap_addr = mapped_addr;
auto unmap_size = size;
// If flag NoOverwrite is provided, don't overwrite mapped VMAs.
// When it isn't provided, VMAs can be overwritten regardless of if they're mapped.
while ((False(flags & MemoryMapFlags::NoOverwrite) || vma.IsFree()) &&
unmap_addr < mapped_addr + size) {
auto unmapped = UnmapBytesFromEntry(unmap_addr, vma, unmap_size);
unmap_addr += unmapped;
unmap_size -= unmapped;
vma = FindVMA(unmap_addr)->second;
}
vma = FindVMA(mapped_addr)->second;
auto remaining_size = vma.base + vma.size - mapped_addr;
if (!vma.IsFree() || remaining_size < size) {
LOG_ERROR(Kernel_Vmm, "Unable to map {:#x} bytes at address {:#x}", size, mapped_addr);
return ORBIS_KERNEL_ERROR_ENOMEM;
}
} else {
// When MemoryMapFlags::Fixed is not specified, and mapped_addr is 0,
// search from address 0x200000000 instead.
alignment = alignment > 0 ? alignment : 16_KB;
mapped_addr = virtual_addr == 0 ? 0x200000000 : mapped_addr;
mapped_addr = SearchFree(mapped_addr, size, alignment);
if (mapped_addr == -1) {
// No suitable memory areas to map to
return ORBIS_KERNEL_ERROR_ENOMEM;
}
}
// Create a memory area representing this mapping.
const auto new_vma_handle = CarveVMA(mapped_addr, size);
auto& new_vma = new_vma_handle->second;
// If type is Flexible, we need to track how much flexible memory is used here.
// We also need to determine a reasonable physical base to perform this mapping at.
if (type == VMAType::Flexible) {
flexible_usage += size;
// Find a suitable physical address
auto handle = fmem_map.begin();
while (handle != fmem_map.end() &&
(!handle->second.is_free || handle->second.size < size)) {
handle++;
}
// Some games will end up fragmenting the flexible address space.
ASSERT_MSG(handle != fmem_map.end() && handle->second.is_free,
"No suitable physical memory areas to map");
// We'll use the start of this area as the physical backing for this mapping.
const auto new_fmem_handle = CarveFmemArea(handle->second.base, size);
auto& new_fmem_area = new_fmem_handle->second;
new_fmem_area.is_free = false;
phys_addr = new_fmem_area.base;
MergeAdjacent(fmem_map, new_fmem_handle);
}
if (True(prot & MemoryProt::CpuWrite)) {
// On PS4, read is appended to write mappings.
prot |= MemoryProt::CpuRead;
}
const bool is_exec = True(prot & MemoryProt::CpuExec);
new_vma.disallow_merge = True(flags & MemoryMapFlags::NoCoalesce);
new_vma.prot = prot;
new_vma.name = name;
new_vma.type = type;
new_vma.phys_base = phys_addr == -1 ? 0 : phys_addr;
new_vma.is_exec = is_exec;
if (type == VMAType::Reserved) {
// Technically this should be done for direct and flexible mappings too,
// But some Windows-specific limitations make that hard to accomplish.
MergeAdjacent(vma_map, new_vma_handle);
}
if (type == VMAType::Reserved || type == VMAType::PoolReserved) {
// For Reserved/PoolReserved mappings, we don't perform any address space allocations.
// Just set out_addr to mapped_addr instead.
*out_addr = std::bit_cast<void*>(mapped_addr);
} else {
// If this is not a reservation, then map to GPU and address space
if (IsValidGpuMapping(mapped_addr, size)) {
rasterizer->MapMemory(mapped_addr, size);
}
*out_addr = impl.Map(mapped_addr, size, alignment, phys_addr, is_exec);
TRACK_ALLOC(*out_addr, size, "VMEM");
}
return ORBIS_OK;
}
s32 MemoryManager::MapFile(void** out_addr, VAddr virtual_addr, u64 size, MemoryProt prot,
MemoryMapFlags flags, s32 fd, s64 phys_addr) {
VAddr mapped_addr = (virtual_addr == 0) ? impl.SystemManagedVirtualBase() : virtual_addr;
ASSERT_MSG(IsValidMapping(mapped_addr, size), "Attempted to access invalid address {:#x}",
mapped_addr);
std::scoped_lock lk{mutex};
// Find first free area to map the file.
if (False(flags & MemoryMapFlags::Fixed)) {
mapped_addr = SearchFree(mapped_addr, size, 1);
if (mapped_addr == -1) {
// No suitable memory areas to map to
return ORBIS_KERNEL_ERROR_ENOMEM;
}
}
if (True(flags & MemoryMapFlags::Fixed)) {
const auto& vma = FindVMA(mapped_addr)->second;
const u64 remaining_size = vma.base + vma.size - virtual_addr;
ASSERT_MSG(!vma.IsMapped() && remaining_size >= size,
"Memory region {:#x} to {:#x} isn't free enough to map region {:#x} to {:#x}",
vma.base, vma.base + vma.size, virtual_addr, virtual_addr + size);
}
// Get the file to map
auto* h = Common::Singleton<Core::FileSys::HandleTable>::Instance();
auto file = h->GetFile(fd);
if (file == nullptr) {
LOG_WARNING(Kernel_Vmm, "Invalid file for mmap, fd {}", fd);
return ORBIS_KERNEL_ERROR_EBADF;
}
if (file->type != Core::FileSys::FileType::Regular) {
LOG_WARNING(Kernel_Vmm, "Unsupported file type for mmap, fd {}", fd);
return ORBIS_KERNEL_ERROR_EBADF;
}
if (True(prot & MemoryProt::CpuWrite)) {
// On PS4, read is appended to write mappings.
prot |= MemoryProt::CpuRead;
}
const auto handle = file->f.GetFileMapping();
if (False(file->f.GetAccessMode() & Common::FS::FileAccessMode::Write) ||
False(file->f.GetAccessMode() & Common::FS::FileAccessMode::Append)) {
// If the file does not have write access, ensure prot does not contain write permissions.
// On real hardware, these mappings succeed, but the memory cannot be written to.
prot &= ~MemoryProt::CpuWrite;
}
impl.MapFile(mapped_addr, size, phys_addr, std::bit_cast<u32>(prot), handle);
if (prot >= MemoryProt::GpuRead) {
// On real hardware, GPU file mmaps cause a full system crash due to an internal error.
ASSERT_MSG(false, "Files cannot be mapped to GPU memory");
}
if (True(prot & MemoryProt::CpuExec)) {
// On real hardware, execute permissions are silently removed.
prot &= ~MemoryProt::CpuExec;
}
// Add virtual memory area
auto& new_vma = CarveVMA(mapped_addr, size)->second;
new_vma.disallow_merge = True(flags & MemoryMapFlags::NoCoalesce);
new_vma.prot = prot;
new_vma.name = "File";
new_vma.fd = fd;
new_vma.type = VMAType::File;
*out_addr = std::bit_cast<void*>(mapped_addr);
return ORBIS_OK;
}
s32 MemoryManager::PoolDecommit(VAddr virtual_addr, u64 size) {
ASSERT_MSG(IsValidMapping(virtual_addr, size), "Attempted to access invalid address {:#x}",
virtual_addr);
std::scoped_lock lk{mutex};
const auto it = FindVMA(virtual_addr);
const auto& vma_base = it->second;
ASSERT_MSG(vma_base.Contains(virtual_addr, size),
"Existing mapping does not contain requested unmap range");
const auto vma_base_addr = vma_base.base;
const auto vma_base_size = vma_base.size;
const auto phys_base = vma_base.phys_base;
const bool is_exec = vma_base.is_exec;
const auto start_in_vma = virtual_addr - vma_base_addr;
const auto type = vma_base.type;
if (type != VMAType::PoolReserved && type != VMAType::Pooled) {
LOG_ERROR(Kernel_Vmm, "Attempting to decommit non-pooled memory!");
return ORBIS_KERNEL_ERROR_EINVAL;
}
if (type == VMAType::Pooled) {
// We always map PoolCommitted memory to GPU, so unmap when decomitting.
if (IsValidGpuMapping(virtual_addr, size)) {
rasterizer->UnmapMemory(virtual_addr, size);
}
// Track how much pooled memory is decommitted
pool_budget += size;
// Re-pool the direct memory used by this mapping
const auto unmap_phys_base = phys_base + start_in_vma;
const auto new_dmem_handle = CarveDmemArea(unmap_phys_base, size);
auto& new_dmem_area = new_dmem_handle->second;
new_dmem_area.dma_type = DMAType::Pooled;
// Coalesce with nearby direct memory areas.
MergeAdjacent(dmem_map, new_dmem_handle);
}
// Mark region as pool reserved and attempt to coalesce it with neighbours.
const auto new_it = CarveVMA(virtual_addr, size);
auto& vma = new_it->second;
vma.type = VMAType::PoolReserved;
vma.prot = MemoryProt::NoAccess;
vma.phys_base = 0;
vma.disallow_merge = false;
vma.name = "anon";
MergeAdjacent(vma_map, new_it);
if (type != VMAType::PoolReserved) {
// Unmap the memory region.
impl.Unmap(vma_base_addr, vma_base_size, start_in_vma, start_in_vma + size, phys_base,
is_exec, true, false);
TRACK_FREE(virtual_addr, "VMEM");
}
return ORBIS_OK;
}
s32 MemoryManager::UnmapMemory(VAddr virtual_addr, u64 size) {
std::scoped_lock lk{mutex};
if (size == 0) {
return ORBIS_OK;
}
virtual_addr = Common::AlignDown(virtual_addr, 16_KB);
size = Common::AlignUp(size, 16_KB);
ASSERT_MSG(IsValidMapping(virtual_addr, size), "Attempted to access invalid address {:#x}",
virtual_addr);
return UnmapMemoryImpl(virtual_addr, size);
}
u64 MemoryManager::UnmapBytesFromEntry(VAddr virtual_addr, VirtualMemoryArea vma_base, u64 size) {
const auto vma_base_addr = vma_base.base;
const auto vma_base_size = vma_base.size;
const auto type = vma_base.type;
const auto phys_base = vma_base.phys_base;
const bool is_exec = vma_base.is_exec;
const auto start_in_vma = virtual_addr - vma_base_addr;
const auto adjusted_size =
vma_base_size - start_in_vma < size ? vma_base_size - start_in_vma : size;
const bool has_backing = HasPhysicalBacking(vma_base) || type == VMAType::File;
const auto prot = vma_base.prot;
const bool readonly_file = prot == MemoryProt::CpuRead && type == VMAType::File;
if (type == VMAType::Free) {
return adjusted_size;
}
if (type == VMAType::Direct) {
// Unmap all direct memory areas covered by this unmap.
auto phys_addr = phys_base + start_in_vma;
auto remaining_size = adjusted_size;
DMemHandle dmem_handle = FindDmemArea(phys_addr);
while (dmem_handle != dmem_map.end() && remaining_size > 0) {
const auto start_in_dma = phys_addr - dmem_handle->second.base;
const auto size_in_dma = dmem_handle->second.size - start_in_dma > remaining_size
? remaining_size
: dmem_handle->second.size - start_in_dma;
dmem_handle = CarveDmemArea(phys_addr, size_in_dma);
auto& dmem_area = dmem_handle->second;
dmem_area.dma_type = DMAType::Allocated;
remaining_size -= dmem_area.size;
phys_addr += dmem_area.size;
// Check if we can coalesce any dmem areas.
MergeAdjacent(dmem_map, dmem_handle);
dmem_handle = FindDmemArea(phys_addr);
}
}
if (type == VMAType::Flexible) {
flexible_usage -= adjusted_size;
// Now that there is a physical backing used for flexible memory,
// manually erase the contents before unmapping to prevent possible issues.
const auto unmap_hardware_address = impl.BackingBase() + phys_base + start_in_vma;
std::memset(unmap_hardware_address, 0, adjusted_size);
// Address space unmap needs the physical_base from the start of the vma,
// so calculate the phys_base to unmap from here.
const auto unmap_phys_base = phys_base + start_in_vma;
const auto new_fmem_handle = CarveFmemArea(unmap_phys_base, adjusted_size);
auto& new_fmem_area = new_fmem_handle->second;
new_fmem_area.is_free = true;
MergeAdjacent(fmem_map, new_fmem_handle);
}
// Mark region as free and attempt to coalesce it with neighbours.
const auto new_it = CarveVMA(virtual_addr, adjusted_size);
auto& vma = new_it->second;
vma.type = VMAType::Free;
vma.prot = MemoryProt::NoAccess;
vma.phys_base = 0;
vma.disallow_merge = false;
vma.name = "";
MergeAdjacent(vma_map, new_it);
if (type != VMAType::Reserved && type != VMAType::PoolReserved) {
// If this mapping has GPU access, unmap from GPU.
if (IsValidGpuMapping(virtual_addr, size)) {
rasterizer->UnmapMemory(virtual_addr, size);
}
// Unmap the memory region.
impl.Unmap(vma_base_addr, vma_base_size, start_in_vma, start_in_vma + adjusted_size,
phys_base, is_exec, has_backing, readonly_file);
TRACK_FREE(virtual_addr, "VMEM");
}
return adjusted_size;
}
s32 MemoryManager::UnmapMemoryImpl(VAddr virtual_addr, u64 size) {
u64 unmapped_bytes = 0;
do {
auto it = FindVMA(virtual_addr + unmapped_bytes);
auto& vma_base = it->second;
auto unmapped =
UnmapBytesFromEntry(virtual_addr + unmapped_bytes, vma_base, size - unmapped_bytes);
ASSERT_MSG(unmapped > 0, "Failed to unmap memory, progress is impossible");
unmapped_bytes += unmapped;
} while (unmapped_bytes < size);
return ORBIS_OK;
}
s32 MemoryManager::QueryProtection(VAddr addr, void** start, void** end, u32* prot) {
ASSERT_MSG(IsValidMapping(addr), "Attempted to access invalid address {:#x}", addr);
std::scoped_lock lk{mutex};
const auto it = FindVMA(addr);
const auto& vma = it->second;
if (vma.IsFree()) {
LOG_ERROR(Kernel_Vmm, "Address {:#x} is not mapped", addr);
return ORBIS_KERNEL_ERROR_EACCES;
}
if (start != nullptr) {
*start = reinterpret_cast<void*>(vma.base);
}
if (end != nullptr) {
*end = reinterpret_cast<void*>(vma.base + vma.size);
}
if (prot != nullptr) {
*prot = static_cast<u32>(vma.prot);
}
return ORBIS_OK;
}
s64 MemoryManager::ProtectBytes(VAddr addr, VirtualMemoryArea& vma_base, u64 size,
MemoryProt prot) {
const auto start_in_vma = addr - vma_base.base;
const auto adjusted_size = std::min<u64>(vma_base.size - start_in_vma, size);
if (vma_base.type == VMAType::Free || vma_base.type == VMAType::PoolReserved) {
// On PS4, protecting freed memory does nothing.
return adjusted_size;
}
if (True(prot & MemoryProt::CpuWrite)) {
// On PS4, read is appended to write mappings.
prot |= MemoryProt::CpuRead;
}
// Set permissions
Core::MemoryPermission perms{};
if (True(prot & MemoryProt::CpuRead)) {
perms |= Core::MemoryPermission::Read;
}
if (True(prot & MemoryProt::CpuReadWrite)) {
perms |= Core::MemoryPermission::ReadWrite;
}
if (True(prot & MemoryProt::CpuExec)) {
perms |= Core::MemoryPermission::Execute;
}
if (True(prot & MemoryProt::GpuRead)) {
perms |= Core::MemoryPermission::Read;
}
if (True(prot & MemoryProt::GpuWrite)) {
perms |= Core::MemoryPermission::Write;
}
if (True(prot & MemoryProt::GpuReadWrite)) {
perms |= Core::MemoryPermission::ReadWrite;
}
if (vma_base.type == VMAType::Direct || vma_base.type == VMAType::Pooled ||
vma_base.type == VMAType::File) {
// On PS4, execute permissions are hidden from direct memory and file mappings.
// Tests show that execute permissions still apply, so handle this after reading perms.
prot &= ~MemoryProt::CpuExec;
}
// Change protection
vma_base.prot = prot;
if (vma_base.type == VMAType::Reserved) {
// On PS4, protections change vma_map, but don't apply.
// Return early to avoid protecting memory that isn't mapped in address space.
return adjusted_size;
}
impl.Protect(addr, size, perms);
return adjusted_size;
}
s32 MemoryManager::Protect(VAddr addr, u64 size, MemoryProt prot) {
std::scoped_lock lk{mutex};
// If size is zero, then there's nothing to protect
if (size == 0) {
return ORBIS_OK;
}
// Ensure the range to modify is valid
ASSERT_MSG(IsValidMapping(addr, size), "Attempted to access invalid address {:#x}", addr);
// Appropriately restrict flags.
constexpr static MemoryProt flag_mask =
MemoryProt::CpuReadWrite | MemoryProt::CpuExec | MemoryProt::GpuReadWrite;
MemoryProt valid_flags = prot & flag_mask;
// Protect all VMAs between addr and addr + size.
s64 protected_bytes = 0;
while (protected_bytes < size) {
auto it = FindVMA(addr + protected_bytes);
auto& vma_base = it->second;
if (vma_base.base > addr + protected_bytes) {
// Account for potential gaps in memory map.
protected_bytes += vma_base.base - (addr + protected_bytes);
}
auto result = ProtectBytes(addr + protected_bytes, vma_base, size - protected_bytes, prot);
if (result < 0) {
// ProtectBytes returned an error, return it
return result;
}
protected_bytes += result;
}
return ORBIS_OK;
}
s32 MemoryManager::VirtualQuery(VAddr addr, s32 flags,
::Libraries::Kernel::OrbisVirtualQueryInfo* info) {
std::scoped_lock lk{mutex};
// FindVMA on addresses before the vma_map return garbage data.
auto query_addr =
addr < impl.SystemManagedVirtualBase() ? impl.SystemManagedVirtualBase() : addr;
if (addr < query_addr && flags == 0) {
LOG_WARNING(Kernel_Vmm, "VirtualQuery on free memory region");
return ORBIS_KERNEL_ERROR_EACCES;
}
auto it = FindVMA(query_addr);
while (it != vma_map.end() && it->second.type == VMAType::Free && flags == 1) {
++it;
}
if (it == vma_map.end() || it->second.type == VMAType::Free) {
LOG_WARNING(Kernel_Vmm, "VirtualQuery on free memory region");
return ORBIS_KERNEL_ERROR_EACCES;
}
const auto& vma = it->second;
info->start = vma.base;
info->end = vma.base + vma.size;
info->offset = 0;
info->protection = static_cast<s32>(vma.prot);
info->is_flexible = vma.type == VMAType::Flexible ? 1 : 0;
info->is_direct = vma.type == VMAType::Direct ? 1 : 0;
info->is_stack = vma.type == VMAType::Stack ? 1 : 0;
info->is_pooled = vma.type == VMAType::PoolReserved || vma.type == VMAType::Pooled ? 1 : 0;
info->is_committed = vma.IsMapped() ? 1 : 0;
if (vma.type == VMAType::Direct || vma.type == VMAType::Pooled) {
// Offset is only assigned for direct and pooled mappings.
info->offset = vma.phys_base;
}
if (vma.type == VMAType::Reserved || vma.type == VMAType::PoolReserved) {
// Protection is hidden from reserved mappings.
info->protection = 0;
}
strncpy(info->name, vma.name.data(), ::Libraries::Kernel::ORBIS_KERNEL_MAXIMUM_NAME_LENGTH);
if (vma.type == VMAType::Direct) {
const auto dmem_it = FindDmemArea(vma.phys_base);
ASSERT_MSG(vma.phys_base <= dmem_it->second.GetEnd(), "vma.phys_base is not in dmem_map!");
info->memory_type = dmem_it->second.memory_type;
} else {
info->memory_type = ::Libraries::Kernel::ORBIS_KERNEL_WB_ONION;
}
return ORBIS_OK;
}
s32 MemoryManager::DirectMemoryQuery(PAddr addr, bool find_next,
::Libraries::Kernel::OrbisQueryInfo* out_info) {
std::scoped_lock lk{mutex};
if (addr >= total_direct_size) {
LOG_WARNING(Kernel_Vmm, "Unable to find allocated direct memory region to query!");
return ORBIS_KERNEL_ERROR_EACCES;
}
auto dmem_area = FindDmemArea(addr);
while (dmem_area != dmem_map.end() && dmem_area->second.dma_type == DMAType::Free &&
find_next) {
dmem_area++;
}
if (dmem_area == dmem_map.end() || dmem_area->second.dma_type == DMAType::Free) {
LOG_WARNING(Kernel_Vmm, "Unable to find allocated direct memory region to query!");
return ORBIS_KERNEL_ERROR_EACCES;
}
out_info->start = dmem_area->second.base;
out_info->memoryType = dmem_area->second.memory_type;
// Loop through all sequential mapped or allocated dmem areas
// to determine the hardware accurate end.
while (dmem_area != dmem_map.end() && dmem_area->second.memory_type == out_info->memoryType &&
(dmem_area->second.dma_type == DMAType::Mapped ||
dmem_area->second.dma_type == DMAType::Allocated)) {
out_info->end = dmem_area->second.GetEnd();
dmem_area++;
}
return ORBIS_OK;
}
s32 MemoryManager::DirectQueryAvailable(PAddr search_start, PAddr search_end, u64 alignment,
PAddr* phys_addr_out, u64* size_out) {
std::scoped_lock lk{mutex};
auto dmem_area = FindDmemArea(search_start);
PAddr paddr{};
u64 max_size{};
while (dmem_area != dmem_map.end()) {
if (dmem_area->second.dma_type != DMAType::Free) {
dmem_area++;
continue;
}
auto aligned_base = alignment > 0 ? Common::AlignUp(dmem_area->second.base, alignment)
: dmem_area->second.base;
const auto alignment_size = aligned_base - dmem_area->second.base;
auto remaining_size =
dmem_area->second.size >= alignment_size ? dmem_area->second.size - alignment_size : 0;
if (dmem_area->second.base < search_start) {
// We need to trim remaining_size to ignore addresses before search_start
remaining_size = remaining_size > (search_start - dmem_area->second.base)
? remaining_size - (search_start - dmem_area->second.base)
: 0;
aligned_base = alignment > 0 ? Common::AlignUp(search_start, alignment) : search_start;
}
if (dmem_area->second.GetEnd() > search_end) {
// We need to trim remaining_size to ignore addresses beyond search_end
remaining_size = remaining_size > (dmem_area->second.GetEnd() - search_end)
? remaining_size - (dmem_area->second.GetEnd() - search_end)
: 0;
}
if (remaining_size > max_size) {
paddr = aligned_base;
max_size = remaining_size;
}
dmem_area++;
}
*phys_addr_out = paddr;
*size_out = max_size;
return ORBIS_OK;
}
s32 MemoryManager::SetDirectMemoryType(VAddr addr, u64 size, s32 memory_type) {
std::scoped_lock lk{mutex};
ASSERT_MSG(IsValidMapping(addr, size), "Attempted to access invalid address {:#x}", addr);
// Search through all VMAs covered by the provided range.
// We aren't modifying these VMAs, so it's safe to iterate through them.
auto remaining_size = size;
auto vma_handle = FindVMA(addr);
while (vma_handle != vma_map.end() && vma_handle->second.base < addr + size) {
// Direct and Pooled mappings are the only ones with a memory type.
if (vma_handle->second.type == VMAType::Direct ||
vma_handle->second.type == VMAType::Pooled) {
// Calculate position in vma
const auto start_in_vma = addr - vma_handle->second.base;
const auto size_in_vma = vma_handle->second.size - start_in_vma;
auto phys_addr = vma_handle->second.phys_base + start_in_vma;
auto size_to_modify = remaining_size > size_in_vma ? size_in_vma : remaining_size;
// Loop through remaining dmem areas until the physical addresses represented
// are all adjusted.
DMemHandle dmem_handle = FindDmemArea(phys_addr);
while (dmem_handle != dmem_map.end() && size_in_vma >= size_to_modify &&
size_to_modify > 0) {
const auto start_in_dma = phys_addr - dmem_handle->second.base;
const auto size_in_dma = dmem_handle->second.size - start_in_dma > size_to_modify
? size_to_modify
: dmem_handle->second.size - start_in_dma;
dmem_handle = CarveDmemArea(phys_addr, size_in_dma);
auto& dmem_area = dmem_handle->second;
dmem_area.memory_type = memory_type;
size_to_modify -= dmem_area.size;
phys_addr += dmem_area.size;
// Check if we can coalesce any dmem areas now that the types are different.
MergeAdjacent(dmem_map, dmem_handle);
dmem_handle = FindDmemArea(phys_addr);
}
}
remaining_size -= vma_handle->second.size;
vma_handle++;
}
return ORBIS_OK;
}
void MemoryManager::NameVirtualRange(VAddr virtual_addr, u64 size, std::string_view name) {
std::scoped_lock lk{mutex};
// Sizes are aligned up to the nearest 16_KB
auto aligned_size = Common::AlignUp(size, 16_KB);
// Addresses are aligned down to the nearest 16_KB
auto aligned_addr = Common::AlignDown(virtual_addr, 16_KB);
ASSERT_MSG(IsValidMapping(aligned_addr, aligned_size),
"Attempted to access invalid address {:#x}", aligned_addr);
auto it = FindVMA(aligned_addr);
s64 remaining_size = aligned_size;
auto current_addr = aligned_addr;
while (remaining_size > 0) {
// Nothing needs to be done to free VMAs
if (!it->second.IsFree()) {
if (remaining_size < it->second.size) {
// We should split VMAs here, but this could cause trouble for Windows.
// Instead log a warning and name the whole VMA.
LOG_WARNING(Kernel_Vmm, "Trying to partially name a range");
}
auto& vma = it->second;
vma.name = name;
}
remaining_size -= it->second.size;
current_addr += it->second.size;
it = FindVMA(current_addr);
}
}
s32 MemoryManager::GetDirectMemoryType(PAddr addr, s32* directMemoryTypeOut,
void** directMemoryStartOut, void** directMemoryEndOut) {
if (addr >= total_direct_size) {
LOG_ERROR(Kernel_Vmm, "Unable to find allocated direct memory region to check type!");
return ORBIS_KERNEL_ERROR_ENOENT;
}
const auto& dmem_area = FindDmemArea(addr)->second;
if (dmem_area.dma_type == DMAType::Free) {
LOG_ERROR(Kernel_Vmm, "Unable to find allocated direct memory region to check type!");
return ORBIS_KERNEL_ERROR_ENOENT;
}
*directMemoryStartOut = reinterpret_cast<void*>(dmem_area.base);
*directMemoryEndOut = reinterpret_cast<void*>(dmem_area.GetEnd());
*directMemoryTypeOut = dmem_area.memory_type;
return ORBIS_OK;
}
s32 MemoryManager::IsStack(VAddr addr, void** start, void** end) {
ASSERT_MSG(IsValidMapping(addr), "Attempted to access invalid address {:#x}", addr);
const auto& vma = FindVMA(addr)->second;
if (vma.IsFree()) {
return ORBIS_KERNEL_ERROR_EACCES;
}
u64 stack_start = 0;
u64 stack_end = 0;
if (vma.type == VMAType::Stack) {
stack_start = vma.base;
stack_end = vma.base + vma.size;
}
if (start != nullptr) {
*start = reinterpret_cast<void*>(stack_start);
}
if (end != nullptr) {
*end = reinterpret_cast<void*>(stack_end);
}
return ORBIS_OK;
}
s32 MemoryManager::GetMemoryPoolStats(::Libraries::Kernel::OrbisKernelMemoryPoolBlockStats* stats) {
std::scoped_lock lk{mutex};
// Run through dmem_map, determine how much physical memory is currently committed
constexpr u64 block_size = 64_KB;
u64 committed_size = 0;
auto dma_handle = dmem_map.begin();
while (dma_handle != dmem_map.end()) {
if (dma_handle->second.dma_type == DMAType::Committed) {
committed_size += dma_handle->second.size;
}
dma_handle++;
}
stats->allocated_flushed_blocks = committed_size / block_size;
stats->available_flushed_blocks = committed_size / block_size;
// TODO: Determine how "cached blocks" work
stats->allocated_cached_blocks = 0;
stats->available_cached_blocks = 0;
return ORBIS_OK;
}
void MemoryManager::InvalidateMemory(const VAddr addr, const u64 size) const {
if (rasterizer) {
rasterizer->InvalidateMemory(addr, size);
}
}
VAddr MemoryManager::SearchFree(VAddr virtual_addr, u64 size, u32 alignment) {
// Calculate the minimum and maximum addresses present in our address space.
auto min_search_address = impl.SystemManagedVirtualBase();
auto max_search_address = impl.UserVirtualBase() + impl.UserVirtualSize();
// If the requested address is below the mapped range, start search from the lowest address
if (virtual_addr < min_search_address) {
virtual_addr = min_search_address;
}
// If the requested address is beyond the maximum our code can handle, throw an assert
ASSERT_MSG(IsValidMapping(virtual_addr), "Input address {:#x} is out of bounds", virtual_addr);
// Align up the virtual_addr first.
virtual_addr = Common::AlignUp(virtual_addr, alignment);
auto it = FindVMA(virtual_addr);
// If the VMA is free and contains the requested mapping we are done.
if (it->second.IsFree() && it->second.Contains(virtual_addr, size)) {
return virtual_addr;
}
// If we didn't hit the return above, then we know the current VMA isn't suitable
it++;
// Search for the first free VMA that fits our mapping.
while (it != vma_map.end()) {
if (!it->second.IsFree()) {
it++;
continue;
}
const auto& vma = it->second;
virtual_addr = Common::AlignUp(vma.base, alignment);
// Sometimes the alignment itself might be larger than the VMA.
if (virtual_addr > vma.base + vma.size) {
it++;
continue;
}
// Make sure the address is within our defined bounds
if (virtual_addr >= max_search_address) {
// There are no free mappings within our safely usable address space.
break;
}
// If there's enough space in the VMA, return the address.
const u64 remaining_size = vma.base + vma.size - virtual_addr;
if (remaining_size >= size) {
return virtual_addr;
}
it++;
}
// Couldn't find a suitable VMA, return an error.
LOG_ERROR(Kernel_Vmm, "Couldn't find a free mapping for address {:#x}, size {:#x}",
virtual_addr, size);
return -1;
}
MemoryManager::VMAHandle MemoryManager::CarveVMA(VAddr virtual_addr, u64 size) {
auto vma_handle = FindVMA(virtual_addr);
const VirtualMemoryArea& vma = vma_handle->second;
ASSERT_MSG(vma.base <= virtual_addr, "Adding a mapping to already mapped region");
const VAddr start_in_vma = virtual_addr - vma.base;
const VAddr end_in_vma = start_in_vma + size;
if (start_in_vma == 0 && size == vma.size) {
// if requsting the whole VMA, return it
return vma_handle;
}
ASSERT_MSG(end_in_vma <= vma.size, "Mapping cannot fit inside free region");
if (end_in_vma != vma.size) {
// Split VMA at the end of the allocated region
Split(vma_handle, end_in_vma);
}
if (start_in_vma != 0) {
// Split VMA at the start of the allocated region
vma_handle = Split(vma_handle, start_in_vma);
}
return vma_handle;
}
MemoryManager::DMemHandle MemoryManager::CarveDmemArea(PAddr addr, u64 size) {
auto dmem_handle = FindDmemArea(addr);
ASSERT_MSG(addr <= dmem_handle->second.GetEnd(), "Physical address not in dmem_map");
const DirectMemoryArea& area = dmem_handle->second;
ASSERT_MSG(area.base <= addr, "Adding an allocation to already allocated region");
const PAddr start_in_area = addr - area.base;
const PAddr end_in_vma = start_in_area + size;
ASSERT_MSG(end_in_vma <= area.size, "Mapping cannot fit inside free region: size = {:#x}",
size);
if (end_in_vma != area.size) {
// Split VMA at the end of the allocated region
Split(dmem_handle, end_in_vma);
}
if (start_in_area != 0) {
// Split VMA at the start of the allocated region
dmem_handle = Split(dmem_handle, start_in_area);
}
return dmem_handle;
}
MemoryManager::FMemHandle MemoryManager::CarveFmemArea(PAddr addr, u64 size) {
auto fmem_handle = FindFmemArea(addr);
ASSERT_MSG(addr <= fmem_handle->second.GetEnd(), "Physical address not in fmem_map");
const FlexibleMemoryArea& area = fmem_handle->second;
ASSERT_MSG(area.base <= addr, "Adding an allocation to already allocated region");
const PAddr start_in_area = addr - area.base;
const PAddr end_in_vma = start_in_area + size;
ASSERT_MSG(end_in_vma <= area.size, "Mapping cannot fit inside free region: size = {:#x}",
size);
if (end_in_vma != area.size) {
// Split VMA at the end of the allocated region
Split(fmem_handle, end_in_vma);
}
if (start_in_area != 0) {
// Split VMA at the start of the allocated region
fmem_handle = Split(fmem_handle, start_in_area);
}
return fmem_handle;
}
MemoryManager::VMAHandle MemoryManager::Split(VMAHandle vma_handle, u64 offset_in_vma) {
auto& old_vma = vma_handle->second;
ASSERT(offset_in_vma < old_vma.size && offset_in_vma > 0);
auto new_vma = old_vma;
old_vma.size = offset_in_vma;
new_vma.base += offset_in_vma;
new_vma.size -= offset_in_vma;
if (HasPhysicalBacking(new_vma)) {
new_vma.phys_base += offset_in_vma;
}
return vma_map.emplace_hint(std::next(vma_handle), new_vma.base, new_vma);
}
MemoryManager::DMemHandle MemoryManager::Split(DMemHandle dmem_handle, u64 offset_in_area) {
auto& old_area = dmem_handle->second;
ASSERT(offset_in_area < old_area.size && offset_in_area > 0);
auto new_area = old_area;
old_area.size = offset_in_area;
new_area.memory_type = old_area.memory_type;
new_area.base += offset_in_area;
new_area.size -= offset_in_area;
return dmem_map.emplace_hint(std::next(dmem_handle), new_area.base, new_area);
}
MemoryManager::FMemHandle MemoryManager::Split(FMemHandle fmem_handle, u64 offset_in_area) {
auto& old_area = fmem_handle->second;
ASSERT(offset_in_area < old_area.size && offset_in_area > 0);
auto new_area = old_area;
old_area.size = offset_in_area;
new_area.base += offset_in_area;
new_area.size -= offset_in_area;
return fmem_map.emplace_hint(std::next(fmem_handle), new_area.base, new_area);
}
} // namespace Core
Reference in New Issue View Git Blame Copy Permalink