Remain stuff

.gitmodules

@@ -117,3 +117,6 @@
 	path = externals/sdl3_mixer
 	url = https://github.com/libsdl-org/SDL_mixer
 	shallow = true
+[submodule "externals/miniz"]
+	path = externals/miniz
+	url = https://github.com/richgel999/miniz

CMakeLists.txt

@@ -512,7 +512,7 @@ set(PAD_LIB src/core/libraries/pad/pad.cpp
             src/core/libraries/pad/pad_errors.h
 )
 
-set(SYSTEM_GESTURE_LIB 
+set(SYSTEM_GESTURE_LIB
             src/core/libraries/system_gesture/system_gesture.cpp
             src/core/libraries/system_gesture/system_gesture.h
 )
@@ -693,7 +693,6 @@ set(COMMON src/common/logging/backend.cpp
            src/common/lru_cache.h
            src/common/error.cpp
            src/common/error.h
-           src/common/scope_exit.h
            src/common/fixed_value.h
            src/common/func_traits.h
            src/common/native_clock.cpp
@@ -707,6 +706,8 @@ set(COMMON src/common/logging/backend.cpp
            src/common/rdtsc.h
            src/common/recursive_lock.cpp
            src/common/recursive_lock.h
+           src/common/scope_exit.h
+           src/common/serdes.h
            src/common/sha1.h
            src/common/shared_first_mutex.h
            src/common/signal_context.h
@@ -988,8 +989,6 @@ set(VIDEO_CORE src/video_core/amdgpu/cb_db_extent.h
                src/video_core/renderer_vulkan/vk_pipeline_common.h
                src/video_core/renderer_vulkan/vk_pipeline_serialization.cpp
                src/video_core/renderer_vulkan/vk_pipeline_serialization.h
-               src/video_core/renderer_vulkan/vk_pipeline_storage.cpp
-               src/video_core/renderer_vulkan/vk_pipeline_storage.h
                src/video_core/renderer_vulkan/vk_platform.cpp
                src/video_core/renderer_vulkan/vk_platform.h
                src/video_core/renderer_vulkan/vk_presenter.cpp
@@ -1027,6 +1026,8 @@ set(VIDEO_CORE src/video_core/amdgpu/cb_db_extent.h
                src/video_core/texture_cache/tile_manager.cpp
                src/video_core/texture_cache/tile_manager.h
                src/video_core/texture_cache/types.h
+               src/video_core/cache_storage.cpp
+               src/video_core/cache_storage.h
                src/video_core/page_manager.cpp
                src/video_core/page_manager.h
                src/video_core/multi_level_page_table.h
@@ -1081,7 +1082,8 @@ add_executable(shadps4
 create_target_directory_groups(shadps4)
 
 target_link_libraries(shadps4 PRIVATE magic_enum::magic_enum fmt::fmt toml11::toml11 tsl::robin_map xbyak::xbyak Tracy::TracyClient RenderDoc::API FFmpeg::ffmpeg Dear_ImGui gcn half::half ZLIB::ZLIB PNG::PNG)
-target_link_libraries(shadps4 PRIVATE Boost::headers GPUOpen::VulkanMemoryAllocator LibAtrac9 sirit Vulkan::Headers xxHash::xxhash Zydis::Zydis glslang::glslang SDL3::SDL3 SDL3_mixer::SDL3_mixer pugixml::pugixml stb::headers libusb::usb lfreist-hwinfo::hwinfo nlohmann_json::nlohmann_json)
+target_link_libraries(shadps4 PRIVATE Boost::headers GPUOpen::VulkanMemoryAllocator LibAtrac9 sirit Vulkan::Headers xxHash::xxhash Zydis::Zydis glslang::glslang SDL3::SDL3 SDL3_mixer::SDL3_mixer pugixml::pugixml)
+target_link_libraries(shadps4 PRIVATE stb::headers libusb::usb lfreist-hwinfo::hwinfo nlohmann_json::nlohmann_json miniz)
 
 target_compile_definitions(shadps4 PRIVATE IMGUI_USER_CONFIG="imgui/imgui_config.h")
 target_compile_definitions(Dear_ImGui PRIVATE IMGUI_USER_CONFIG="${PROJECT_SOURCE_DIR}/src/imgui/imgui_config.h")

externals/CMakeLists.txt

@@ -261,3 +261,6 @@ endif()
 #nlohmann json 
 set(JSON_BuildTests OFF CACHE INTERNAL "")
 add_subdirectory(json)
+
+# miniz
+add_subdirectory(miniz)

externals/miniz

@@ -0,0 +1 @@
+Subproject commit 174573d60290f447c13a2b1b3405de2b96e27d6c

src/common/config.cpp

@@ -192,6 +192,7 @@ static ConfigEntry<bool> vkHostMarkers(false);
 static ConfigEntry<bool> vkGuestMarkers(false);
 static ConfigEntry<bool> rdocEnable(false);
 static ConfigEntry<bool> pipelineCacheEnable(false);
+static ConfigEntry<bool> pipelineCacheArchive(false);
 
 // Debug
 static ConfigEntry<bool> isDebugDump(false);
@@ -457,6 +458,10 @@ bool isPipelineCacheEnabled() {
     return pipelineCacheEnable.get();
 }
 
+bool isPipelineCacheArchived() {
+    return pipelineCacheArchive.get();
+}
+
 bool fpsColor() {
     return isFpsColor.get();
 }
@@ -612,6 +617,10 @@ void setPipelineCacheEnabled(bool enable, bool is_game_specific) {
     pipelineCacheEnable.set(enable, is_game_specific);
 }
 
+void setPipelineCacheArchived(bool enable, bool is_game_specific) {
+    pipelineCacheArchive.set(enable, is_game_specific);
+}
+
 void setVblankFreq(u32 value, bool is_game_specific) {
     vblankFrequency.set(value, is_game_specific);
 }
@@ -949,6 +958,7 @@ void load(const std::filesystem::path& path, bool is_game_specific) {
         vkGuestMarkers.setFromToml(vk, "guestMarkers", is_game_specific);
         rdocEnable.setFromToml(vk, "rdocEnable", is_game_specific);
         pipelineCacheEnable.setFromToml(vk, "pipelineCacheEnable", is_game_specific);
+        pipelineCacheArchive.setFromToml(vk, "pipelineCacheArchive", is_game_specific);
     }
 
     string current_version = {};
@@ -1118,6 +1128,7 @@ void save(const std::filesystem::path& path, bool is_game_specific) {
     vkGuestMarkers.setTomlValue(data, "Vulkan", "guestMarkers", is_game_specific);
     rdocEnable.setTomlValue(data, "Vulkan", "rdocEnable", is_game_specific);
     pipelineCacheEnable.setTomlValue(data, "Vulkan", "pipelineCacheEnable", is_game_specific);
+    pipelineCacheArchive.setTomlValue(data, "Vulkan", "pipelineCacheArchive", is_game_specific);
 
     isDebugDump.setTomlValue(data, "Debug", "DebugDump", is_game_specific);
     isShaderDebug.setTomlValue(data, "Debug", "CollectShader", is_game_specific);
@@ -1249,6 +1260,7 @@ void setDefaultValues(bool is_game_specific) {
     vkGuestMarkers.set(false, is_game_specific);
     rdocEnable.set(false, is_game_specific);
     pipelineCacheEnable.set(false, is_game_specific);
+    pipelineCacheArchive.set(false, is_game_specific);
 
     // GS - Debug
     isDebugDump.set(false, is_game_specific);

src/common/config.h

@@ -95,8 +95,10 @@ bool getEnableDiscordRPC();
 void setEnableDiscordRPC(bool enable);
 bool isRdocEnabled();
 bool isPipelineCacheEnabled();
+bool isPipelineCacheArchived();
 void setRdocEnabled(bool enable, bool is_game_specific = false);
 void setPipelineCacheEnabled(bool enable, bool is_game_specific = false);
+void setPipelineCacheArchived(bool enable, bool is_game_specific = false);
 std::string getLogType();
 void setLogType(const std::string& type, bool is_game_specific = false);
 std::string getLogFilter();

src/common/serdes.h

@@ -0,0 +1,140 @@
+// SPDX-FileCopyrightText: Copyright 2025 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "common/assert.h"
+#include "common/types.h"
+
+#include <cstddef>
+
+namespace Serialization {
+
+template <typename T>
+concept Container = requires(T t) {
+    typename T::iterator;
+    { t.begin() } -> std::same_as<typename T::iterator>;
+    { t.end() } -> std::same_as<typename T::iterator>;
+    { t.size() } -> std::convertible_to<std::size_t>;
+};
+
+struct Archive {
+    void Alloc(size_t size) {
+        container.resize(size);
+    }
+
+    void Grow(size_t size) {
+        container.resize(container.size() + size);
+    }
+
+    void Merge(const Archive& ar) {
+        container.insert(container.end(), ar.container.cbegin(), ar.container.cend());
+        offset = container.size();
+    }
+
+    [[nodiscard]] size_t SizeBytes() const {
+        return container.size();
+    }
+
+    u8* CurrPtr() {
+        return container.data() + offset;
+    }
+
+    void Advance(size_t size) {
+        ASSERT(offset + size <= container.size());
+        offset += size;
+    }
+
+    std::vector<u8>&& TakeOff() {
+        offset = 0;
+        return std::move(container);
+    }
+
+    [[nodiscard]] bool IsEoS() const {
+        return offset >= container.size();
+    }
+
+    Archive() = default;
+    explicit Archive(std::vector<u8>&& v) : container{v} {}
+
+private:
+    u32 offset{};
+    std::vector<u8> container{};
+
+    friend struct Writer;
+    friend struct Reader;
+};
+
+struct Writer {
+    template <typename T>
+    void Write(const T* ptr, size_t size) {
+        if (ar.offset + size >= ar.container.size()) {
+            ar.Grow(size);
+        }
+        std::memcpy(ar.CurrPtr(), reinterpret_cast<const void*>(ptr), size);
+        ar.Advance(size);
+    }
+
+    template <typename T>
+        requires(!Container<T>)
+    void Write(const T& value) {
+        const auto size = sizeof(value);
+        Write(&value, size);
+    }
+
+    void Write(const auto& v) {
+        Write(v.size());
+        for (const auto& elem : v) {
+            Write(elem);
+        }
+    }
+
+    void Write(const std::string& s) {
+        Write(s.size());
+        Write(s.c_str(), s.size());
+    }
+
+    Writer() = delete;
+    explicit Writer(Archive& ar_) : ar{ar_} {}
+
+    Archive& ar;
+};
+
+struct Reader {
+    template <typename T>
+    void Read(T* ptr, size_t size) {
+        ASSERT(ar.offset + size <= ar.container.size());
+        std::memcpy(reinterpret_cast<void*>(ptr), ar.CurrPtr(), size);
+        ar.Advance(size);
+    }
+
+    template <typename T>
+        requires(!Container<T>)
+    void Read(T& value) {
+        const auto size = sizeof(value);
+        Read(&value, size);
+    }
+
+    void Read(auto& v) {
+        size_t num_elements{};
+        Read(num_elements);
+        for (int i = 0; i < num_elements; ++i) {
+            v.emplace_back();
+            Read(v.back());
+        }
+    }
+
+    void Read(std::string& s) {
+        size_t length{};
+        Read(length);
+        s.resize(length);
+        Read(s.data(), length);
+    }
+
+    Reader() = delete;
+    explicit Reader(Archive& ar_) : ar{ar_} {}
+
+    Archive& ar;
+};
+
+} // namespace Serialization

src/emulator.cpp

@@ -42,6 +42,7 @@
 #include "core/linker.h"
 #include "core/memory.h"
 #include "emulator.h"
+#include "video_core/cache_storage.h"
 #include "video_core/renderdoc.h"
 
 #ifdef _WIN32
@@ -387,6 +388,7 @@ void Emulator::Run(std::filesystem::path file, std::vector<std::string> args,
     }
 
     UpdatePlayTime(id);
+    Storage::DataBase::Instance().Close();
 
     std::quick_exit(0);
 }

src/shader_recompiler/frontend/fetch_shader.h

@@ -66,7 +66,6 @@ struct FetchShaderData {
 
     void Serialize(Serialization::Archive& ar) const;
     bool Deserialize(Serialization::Archive& buffer);
-    u64 Hash() const;
 };
 
 const u32* GetFetchShaderCode(const Info& info, u32 sgpr_base);

src/shader_recompiler/profile.h

@@ -8,6 +8,10 @@
 namespace Shader {
 
 struct Profile {
+    u64 max_ubo_size{};
+    u32 max_viewport_width{};
+    u32 max_viewport_height{};
+    u32 max_shared_memory_size{};
     u32 supported_spirv{0x00010000};
     u32 subgroup_size{};
     bool support_int8{};
@@ -37,10 +41,7 @@ struct Profile {
     bool needs_lds_barriers{};
     bool needs_buffer_offsets{};
     bool needs_unorm_fixup{};
-    u64 max_ubo_size{};
-    u32 max_viewport_width{};
-    u32 max_viewport_height{};
-    u32 max_shared_memory_size{};
+    bool _pad0{};
 };
 
 } // namespace Shader

src/shader_recompiler/specialization.h

@@ -79,8 +79,8 @@ struct SamplerSpecialization {
 struct StageSpecialization {
     static constexpr size_t MaxStageResources = 128;
 
-    const Shader::Info* info;
-    RuntimeInfo runtime_info;
+    const Info* info{};
+    RuntimeInfo runtime_info{};
     std::bitset<MaxStageResources> bitset{};
     std::optional<Gcn::FetchShaderData> fetch_shader_data{};
     boost::container::small_vector<VsAttribSpecialization, 32> vs_attribs;
@@ -90,6 +90,7 @@ struct StageSpecialization {
     boost::container::small_vector<SamplerSpecialization, 16> samplers;
     Backend::Bindings start{};
 
+    StageSpecialization() = default;
     StageSpecialization(const Info& info_, RuntimeInfo runtime_info_, const Profile& profile_,
                         Backend::Bindings start_)
         : info{&info_}, runtime_info{runtime_info_}, start{start_} {
@@ -158,7 +159,7 @@ struct StageSpecialization {
         // Initialize runtime_info fields that rely on analysis in tessellation passes
         if (info->l_stage == LogicalStage::TessellationControl ||
             info->l_stage == LogicalStage::TessellationEval) {
-            Shader::TessellationDataConstantBuffer tess_constants;
+            TessellationDataConstantBuffer tess_constants{};
             info->ReadTessConstantBuffer(tess_constants);
             if (info->l_stage == LogicalStage::TessellationControl) {
                 runtime_info.hs_info.InitFromTessConstants(tess_constants);
@@ -192,21 +193,43 @@ struct StageSpecialization {
         }
     }
 
+    [[nodiscard]] bool Valid() const {
+        return info != nullptr;
+    }
+
     bool operator==(const StageSpecialization& other) const {
-        if (start != other.start) {
+        if (!Valid()) {
             return false;
         }
+
+        if (vs_attribs != other.vs_attribs) {
+            return false;
+        }
+
         if (runtime_info != other.runtime_info) {
             return false;
         }
+
         if (fetch_shader_data != other.fetch_shader_data) {
             return false;
         }
-        for (u32 i = 0; i < vs_attribs.size(); i++) {
-            if (vs_attribs[i] != other.vs_attribs[i]) {
-                return false;
-            }
+
+        if (fmasks != other.fmasks) {
+            return false;
         }
+
+        // For VS which only generates geometry and doesn't have any inputs, its start
+        // bindings still may change as they depend on previously processed FS. The check below
+        // handles this case and prevents generation of redundant permutations. This is also safe
+        // for other types of shaders with no bindings.
+        if (bitset.none() && other.bitset.none()) {
+            return true;
+        }
+
+        if (start != other.start) {
+            return false;
+        }
+
         u32 binding{};
         for (u32 i = 0; i < buffers.size(); i++) {
             if (other.bitset[binding++] && buffers[i] != other.buffers[i]) {
@@ -218,11 +241,7 @@ struct StageSpecialization {
                 return false;
             }
         }
-        for (u32 i = 0; i < fmasks.size(); i++) {
-            if (other.bitset[binding++] && fmasks[i] != other.fmasks[i]) {
-                return false;
-            }
-        }
+
         for (u32 i = 0; i < samplers.size(); i++) {
             if (samplers[i] != other.samplers[i]) {
                 return false;
@@ -231,7 +250,8 @@ struct StageSpecialization {
         return true;
     }
 
-    u64 Hash() const;
+    void Serialize(Serialization::Archive& ar) const;
+    bool Deserialize(Serialization::Archive& ar);
 };
 
 } // namespace Shader

src/video_core/cache_storage.cpp

@@ -0,0 +1,265 @@
+// SPDX-FileCopyrightText: Copyright 2025 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "common/config.h"
+#include "common/elf_info.h"
+#include "common/io_file.h"
+#include "common/polyfill_thread.h"
+#include "common/thread.h"
+
+#include "video_core/cache_storage.h"
+#include "video_core/renderer_vulkan/vk_instance.h"
+#include "video_core/renderer_vulkan/vk_pipeline_cache.h"
+
+#include <miniz.h>
+
+#include <condition_variable>
+#include <future>
+#include <mutex>
+#include <queue>
+
+namespace {
+
+std::mutex submit_mutex{};
+u32 num_requests{};
+std::condition_variable_any request_cv{};
+std::queue<std::future<void>> req_queue{};
+std::mutex m_request{};
+
+mz_zip_archive zip_ar{};
+bool ar_is_read_only{true};
+
+} // namespace
+
+namespace Storage {
+
+void ProcessIO(const std::stop_token& stoken) {
+    Common::SetCurrentThreadName("shadPS4:PipelineCacheIO");
+
+    while (!stoken.stop_requested()) {
+        {
+            std::unique_lock lk{submit_mutex};
+            Common::CondvarWait(request_cv, lk, stoken, [&] { return num_requests; });
+        }
+
+        if (stoken.stop_requested()) {
+            break;
+        }
+
+        while (num_requests) {
+            std::future<void> request{};
+            {
+                std::scoped_lock lock{m_request};
+                if (req_queue.empty()) {
+                    continue;
+                }
+                request = std::move(req_queue.front());
+                req_queue.pop();
+            }
+
+            if (request.valid()) {
+                request.wait();
+            }
+
+            --num_requests;
+        }
+    }
+}
+
+constexpr std::string GetBlobFileExtension(BlobType type) {
+    switch (type) {
+    case BlobType::ShaderMeta: {
+        return "meta";
+    }
+    case BlobType::ShaderBinary: {
+        return "spv";
+    }
+    case BlobType::PipelineKey: {
+        return "key";
+    }
+    case BlobType::ShaderProfile: {
+        return "bin";
+    }
+    default:
+        UNREACHABLE();
+    }
+}
+
+void DataBase::Open() {
+    if (opened) {
+        return;
+    }
+
+    const auto& game_info = Common::ElfInfo::Instance();
+
+    using namespace Common::FS;
+    if (Config::isPipelineCacheArchived()) {
+        mz_zip_zero_struct(&zip_ar);
+
+        cache_path = GetUserPath(PathType::CacheDir) /
+                     std::filesystem::path{game_info.GameSerial()}.replace_extension(".zip");
+
+        if (!mz_zip_reader_init_file(&zip_ar, cache_path.string().c_str(),
+                                     MZ_ZIP_FLAG_READ_ALLOW_WRITING) ||
+            !mz_zip_validate_archive(&zip_ar, 0)) {
+            LOG_INFO(Render, "Cache archive {} is not found or archive is corrupted",
+                     cache_path.string().c_str());
+            mz_zip_reader_end(&zip_ar);
+            mz_zip_writer_init_file(&zip_ar, cache_path.string().c_str(), 0);
+        }
+    } else {
+        cache_path = GetUserPath(PathType::CacheDir) / game_info.GameSerial();
+        if (!std::filesystem::exists(cache_path)) {
+            std::filesystem::create_directories(cache_path);
+        }
+    }
+
+    io_worker = std::jthread{ProcessIO};
+    opened = true;
+}
+
+void DataBase::Close() {
+    if (!IsOpened()) {
+        return;
+    }
+
+    io_worker.request_stop();
+    io_worker.join();
+
+    if (Config::isPipelineCacheArchived()) {
+        mz_zip_writer_finalize_archive(&zip_ar);
+        mz_zip_writer_end(&zip_ar);
+    }
+
+    LOG_INFO(Render, "Cache dumped");
+}
+
+template <typename T>
+bool WriteVector(BlobType type, std::filesystem::path&& path_, std::vector<T>&& v) {
+    auto request = std::async(
+        Config::isPipelineCacheArchived() ? std::launch::deferred : std::launch::async,
+        [=](std::filesystem::path&& path) {
+            path.replace_extension(GetBlobFileExtension(type));
+            if (Config::isPipelineCacheArchived()) {
+                ASSERT_MSG(!ar_is_read_only,
+                           "The archive is read-only. Did you forget to call `FinishPreload`?");
+                if (!mz_zip_writer_add_mem(&zip_ar, path.string().c_str(), v.data(),
+                                           v.size() * sizeof(T), MZ_BEST_COMPRESSION)) {
+                    LOG_ERROR(Render, "Failed to add {} to the archive", path.string().c_str());
+                }
+            } else {
+                using namespace Common::FS;
+                const auto file = IOFile{path, FileAccessMode::Create};
+                file.Write(v);
+            }
+        },
+        path_);
+
+    {
+        std::scoped_lock lock{m_request};
+        req_queue.emplace(std::move(request));
+    }
+
+    std::scoped_lock lk{submit_mutex};
+    ++num_requests;
+    request_cv.notify_one();
+    return true;
+}
+
+template <typename T>
+void LoadVector(BlobType type, std::filesystem::path& path, std::vector<T>& v) {
+    using namespace Common::FS;
+    path.replace_extension(GetBlobFileExtension(type));
+    if (Config::isPipelineCacheArchived()) {
+        int index{-1};
+        index = mz_zip_reader_locate_file(&zip_ar, path.string().c_str(), nullptr, 0);
+        if (index < 0) {
+            LOG_WARNING(Render, "File {} is not found in the archive", path.string().c_str());
+            return;
+        }
+        mz_zip_archive_file_stat stat{};
+        mz_zip_reader_file_stat(&zip_ar, index, &stat);
+        v.resize(stat.m_uncomp_size / sizeof(T));
+        mz_zip_reader_extract_to_mem(&zip_ar, index, v.data(), stat.m_uncomp_size, 0);
+    } else {
+        const auto file = IOFile{path, FileAccessMode::Read};
+        v.resize(file.GetSize() / sizeof(T));
+        file.Read(v);
+    }
+}
+
+bool DataBase::Save(BlobType type, const std::string& name, std::vector<u8>&& data) {
+    if (!opened) {
+        return false;
+    }
+
+    auto path = Config::isPipelineCacheArchived() ? std::filesystem::path{name} : cache_path / name;
+    return WriteVector(type, std::move(path), std::move(data));
+}
+
+bool DataBase::Save(BlobType type, const std::string& name, std::vector<u32>&& data) {
+    if (!opened) {
+        return false;
+    }
+
+    auto path = Config::isPipelineCacheArchived() ? std::filesystem::path{name} : cache_path / name;
+    return WriteVector(type, std::move(path), std::move(data));
+}
+
+void DataBase::Load(BlobType type, const std::string& name, std::vector<u8>& data) {
+    if (!opened) {
+        return;
+    }
+
+    auto path = Config::isPipelineCacheArchived() ? std::filesystem::path{name} : cache_path / name;
+    return LoadVector(type, path, data);
+}
+
+void DataBase::Load(BlobType type, const std::string& name, std::vector<u32>& data) {
+    if (!opened) {
+        return;
+    }
+
+    auto path = Config::isPipelineCacheArchived() ? std::filesystem::path{name} : cache_path / name;
+    return LoadVector(type, path, data);
+}
+
+void DataBase::ForEachBlob(BlobType type, const std::function<void(std::vector<u8>&& data)>& func) {
+    const auto& ext = GetBlobFileExtension(type);
+    if (Config::isPipelineCacheArchived()) {
+        const auto num_files = mz_zip_reader_get_num_files(&zip_ar);
+        for (int index = 0; index < num_files; ++index) {
+            std::array<char, MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE> file_name{};
+            file_name.fill(0);
+            mz_zip_reader_get_filename(&zip_ar, index, file_name.data(), file_name.size());
+            if (std::string{file_name.data()}.ends_with(ext)) {
+                mz_zip_archive_file_stat stat{};
+                mz_zip_reader_file_stat(&zip_ar, index, &stat);
+                std::vector<u8> data(stat.m_uncomp_size);
+                mz_zip_reader_extract_to_mem(&zip_ar, index, data.data(), data.size(), 0);
+                func(std::move(data));
+            }
+        }
+    } else {
+        for (const auto& file_name : std::filesystem::directory_iterator{cache_path}) {
+            if (file_name.path().extension().string().ends_with(ext)) {
+                using namespace Common::FS;
+                const auto& file = IOFile{file_name, FileAccessMode::Read};
+                if (file.IsOpen()) {
+                    std::vector<u8> data(file.GetSize());
+                    file.Read(data);
+                    func(std::move(data));
+                }
+            }
+        }
+    }
+}
+
+void DataBase::FinishPreload() {
+    if (Config::isPipelineCacheArchived()) {
+        mz_zip_writer_init_from_reader(&zip_ar, cache_path.string().c_str());
+        ar_is_read_only = false;
+    }
+}
+
+} // namespace Storage

src/video_core/cache_storage.h

@@ -1,20 +1,23 @@
 // SPDX-FileCopyrightText: Copyright 2025 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#pragma once
+
 #include "common/path_util.h"
 #include "common/singleton.h"
 #include "common/types.h"
 
 #include <functional>
+#include <thread>
 #include <vector>
 
-namespace Vulkan {
 namespace Storage {
 
 enum class BlobType : u32 {
     ShaderMeta,
     ShaderBinary,
     PipelineKey,
+    ShaderProfile,
 };
 
 class DataBase {
@@ -28,6 +31,7 @@ public:
     [[nodiscard]] bool IsOpened() const {
         return opened;
     }
+    void FinishPreload();
 
     bool Save(BlobType type, const std::string& name, std::vector<u8>&& data);
     bool Save(BlobType type, const std::string& name, std::vector<u32>&& data);
@@ -38,9 +42,9 @@ public:
     void ForEachBlob(BlobType type, const std::function<void(std::vector<u8>&& data)>& func);
 
 private:
-    std::filesystem::path cache_dir{};
+    std::jthread io_worker{};
+    std::filesystem::path cache_path{};
     bool opened{};
 };
 
 } // namespace Storage
-} // namespace Vulkan

src/video_core/renderer_vulkan/vk_compute_pipeline.cpp

@@ -30,7 +30,11 @@ ComputePipeline::ComputePipeline(const Instance& instance, Scheduler& scheduler,
     u32 binding{};
     boost::container::small_vector<vk::DescriptorSetLayoutBinding, 32> bindings;
     for (const auto& buffer : info->buffers) {
-        const auto sharp = preloading ? AmdGpu::Buffer{} : buffer.GetSharp(*info); // Comment
+        // During deserialization, we don't have access to the UD to fetch sharp data. To address
+        // this properly we need to track shaprs or portion of them in `sdata`, but since we're
+        // interested only in "is storage" flag (which is not even effective atm), we can take a
+        // shortcut there.
+        const auto sharp = preloading ? AmdGpu::Buffer{} : buffer.GetSharp(*info);
         bindings.push_back({
             .binding = binding++,
             .descriptorType = buffer.IsStorage(sharp) ? vk::DescriptorType::eStorageBuffer

src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp

@@ -452,7 +452,9 @@ void GraphicsPipeline::BuildDescSetLayout(bool preloading) {
         }
         const auto stage_bit = LogicalStageToStageBit[u32(stage->l_stage)];
         for (const auto& buffer : stage->buffers) {
-            const auto sharp = preloading ? AmdGpu::Buffer{} : buffer.GetSharp(*stage); // Comment
+            const auto sharp =
+                preloading ? AmdGpu::Buffer{}
+                           : buffer.GetSharp(*stage); // See for the comment in compute PL creation
             bindings.push_back({
                 .binding = binding++,
                 .descriptorType = buffer.IsStorage(sharp) ? vk::DescriptorType::eStorageBuffer

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -13,11 +13,10 @@
 #include "shader_recompiler/recompiler.h"
 #include "shader_recompiler/runtime_info.h"
 #include "video_core/amdgpu/liverpool.h"
+#include "video_core/cache_storage.h"
 #include "video_core/renderer_vulkan/liverpool_to_vk.h"
 #include "video_core/renderer_vulkan/vk_instance.h"
-#include "video_core/renderer_vulkan/vk_pipeline_cache.h"
 #include "video_core/renderer_vulkan/vk_pipeline_serialization.h"
-#include "video_core/renderer_vulkan/vk_pipeline_storage.h"
 #include "video_core/renderer_vulkan/vk_scheduler.h"
 #include "video_core/renderer_vulkan/vk_shader_util.h"
 
@@ -225,6 +224,13 @@ PipelineCache::PipelineCache(const Instance& instance_, Scheduler& scheduler_,
       desc_heap{instance, scheduler.GetMasterSemaphore(), DescriptorHeapSizes} {
     const auto& vk12_props = instance.GetVk12Properties();
     profile = Shader::Profile{
+        // When binding a UBO, we calculate its size considering the offset in the larger buffer
+        // cache underlying resource. In some cases, it may produce sizes exceeding the system
+        // maximum allowed UBO range, so we need to reduce the threshold to prevent issues.
+        .max_ubo_size = instance.UniformMaxSize() - instance.UniformMinAlignment(),
+        .max_viewport_width = instance.GetMaxViewportWidth(),
+        .max_viewport_height = instance.GetMaxViewportHeight(),
+        .max_shared_memory_size = instance.MaxComputeSharedMemorySize(),
         .supported_spirv = SpirvVersion1_6,
         .subgroup_size = instance.SubgroupSize(),
         .support_int8 = instance.IsShaderInt8Supported(),
@@ -260,13 +266,6 @@ PipelineCache::PipelineCache(const Instance& instance_, Scheduler& scheduler_,
                               instance.GetDriverID() == vk::DriverId::eMoltenvk,
         .needs_buffer_offsets = instance.StorageMinAlignment() > 4,
         .needs_unorm_fixup = instance.GetDriverID() == vk::DriverId::eMoltenvk,
-        // When binding a UBO, we calculate its size considering the offset in the larger buffer
-        // cache underlying resource. In some cases, it may produce sizes exceeding the system
-        // maximum allowed UBO range, so we need to reduce the threshold to prevent issues.
-        .max_ubo_size = instance.UniformMaxSize() - instance.UniformMinAlignment(),
-        .max_viewport_width = instance.GetMaxViewportWidth(),
-        .max_viewport_height = instance.GetMaxViewportHeight(),
-        .max_shared_memory_size = instance.MaxComputeSharedMemorySize(),
     };
 
     WarmUp();
@@ -294,6 +293,7 @@ const GraphicsPipeline* PipelineCache::GetGraphicsPipeline() {
             runtime_infos, fetch_shader, modules, sdata, false);
 
         RegisterPipelineData(graphics_key, pipeline_hash, sdata);
+        ++num_new_pipelines;
 
         if (Config::collectShadersForDebug()) {
             for (auto stage = 0; stage < MaxShaderStages; ++stage) {
@@ -322,6 +322,7 @@ const ComputePipeline* PipelineCache::GetComputePipeline() {
                                                        *pipeline_cache, compute_key, *infos[0],
                                                        modules[0], sdata, false);
         RegisterPipelineData(compute_key, sdata);
+        ++num_new_pipelines;
 
         if (Config::collectShadersForDebug()) {
             auto& m = modules[0];
@@ -571,13 +572,13 @@ PipelineCache::Result PipelineCache::GetProgram(Stage stage, LogicalStage l_stag
         auto& program = it_pgm.value();
         auto start = binding;
         const auto module = CompileModule(program->info, runtime_info, params.code, 0, binding);
-        const auto spec = Shader::StageSpecialization(program->info, runtime_info, profile, start);
-        const auto spec_hash = spec.Hash();
+        auto spec = Shader::StageSpecialization(program->info, runtime_info, profile, start);
         const auto perm_hash = HashCombine(params.hash, 0);
 
-        program->AddPermut(module, spec_hash);
-        RegisterShaderMeta(program->info, spec.fetch_shader_data, perm_hash, spec_hash, 0);
-        return std::make_tuple(&program->info, module, spec.fetch_shader_data, perm_hash);
+        RegisterShaderMeta(program->info, spec.fetch_shader_data, spec, perm_hash, 0);
+        program->AddPermut(module, std::move(spec));
+        return std::make_tuple(&program->info, module, program->modules[0].spec.fetch_shader_data,
+                               perm_hash);
     }
 
     auto& program = it_pgm.value();
@@ -585,27 +586,28 @@ PipelineCache::Result PipelineCache::GetProgram(Stage stage, LogicalStage l_stag
     info.pgm_base = params.Base(); // Needs to be actualized for inline cbuffer address fixup
     info.user_data = params.user_data;
     info.RefreshFlatBuf();
-    const auto spec = Shader::StageSpecialization(info, runtime_info, profile, binding);
-    const auto spec_hash = spec.Hash();
+    auto spec = Shader::StageSpecialization(info, runtime_info, profile, binding);
 
     size_t perm_idx = program->modules.size();
     u64 perm_hash = HashCombine(params.hash, perm_idx);
 
     vk::ShaderModule module{};
 
-    const auto it = std::ranges::find(program->modules, spec_hash, &Program::Module::spec_hash);
+    const auto it = std::ranges::find(program->modules, spec, &Program::Module::spec);
     if (it == program->modules.end()) {
         auto new_info = Shader::Info(stage, l_stage, params);
         module = CompileModule(new_info, runtime_info, params.code, perm_idx, binding);
-        program->AddPermut(module, spec_hash);
-        RegisterShaderMeta(info, spec.fetch_shader_data, perm_hash, spec_hash, perm_idx);
+
+        RegisterShaderMeta(info, spec.fetch_shader_data, spec, perm_hash, perm_idx);
+        program->AddPermut(module, std::move(spec));
     } else {
         info.AddBindings(binding);
         module = it->module;
         perm_idx = std::distance(program->modules.begin(), it);
         perm_hash = HashCombine(params.hash, perm_idx);
     }
-    return std::make_tuple(&program->info, module, spec.fetch_shader_data, perm_hash);
+    return std::make_tuple(&program->info, module,
+                           program->modules[perm_idx].spec.fetch_shader_data, perm_hash);
 }
 
 std::optional<vk::ShaderModule> PipelineCache::ReplaceShader(vk::ShaderModule module,
@@ -636,4 +638,47 @@ std::optional<vk::ShaderModule> PipelineCache::ReplaceShader(vk::ShaderModule mo
     return new_module;
 }
 
+std::string PipelineCache::GetShaderName(Shader::Stage stage, u64 hash,
+                                         std::optional<size_t> perm) {
+    if (perm) {
+        return fmt::format("{}_{:#018x}_{}", stage, hash, *perm);
+    }
+    return fmt::format("{}_{:#018x}", stage, hash);
+}
+
+void PipelineCache::DumpShader(std::span<const u32> code, u64 hash, Shader::Stage stage,
+                               size_t perm_idx, std::string_view ext) {
+    if (!Config::dumpShaders()) {
+        return;
+    }
+
+    using namespace Common::FS;
+    const auto dump_dir = GetUserPath(PathType::ShaderDir) / "dumps";
+    if (!std::filesystem::exists(dump_dir)) {
+        std::filesystem::create_directories(dump_dir);
+    }
+    const auto filename = fmt::format("{}.{}", GetShaderName(stage, hash, perm_idx), ext);
+    const auto file = IOFile{dump_dir / filename, FileAccessMode::Create};
+    file.WriteSpan(code);
+}
+
+std::optional<std::vector<u32>> PipelineCache::GetShaderPatch(u64 hash, Shader::Stage stage,
+                                                              size_t perm_idx,
+                                                              std::string_view ext) {
+
+    using namespace Common::FS;
+    const auto patch_dir = GetUserPath(PathType::ShaderDir) / "patch";
+    if (!std::filesystem::exists(patch_dir)) {
+        std::filesystem::create_directories(patch_dir);
+    }
+    const auto filename = fmt::format("{}.{}", GetShaderName(stage, hash, perm_idx), ext);
+    const auto filepath = patch_dir / filename;
+    if (!std::filesystem::exists(filepath)) {
+        return {};
+    }
+    const auto file = IOFile{patch_dir / filename, FileAccessMode::Read};
+    std::vector<u32> code(file.GetSize() / sizeof(u32));
+    file.Read(code);
+    return code;
+}
 } // namespace Vulkan

src/video_core/renderer_vulkan/vk_pipeline_cache.h

@@ -40,24 +40,26 @@ class ShaderCache;
 struct Program {
     struct Module {
         vk::ShaderModule module;
-        u64 spec_hash;
+        Shader::StageSpecialization spec;
     };
-    using ModuleList = boost::container::small_vector<Module, 8>;
+    static constexpr size_t MaxPermutations = 8;
+    using ModuleList = boost::container::small_vector<Module, MaxPermutations>;
 
     Shader::Info info;
-    ModuleList modules;
+    ModuleList modules{};
 
     Program() = default;
     Program(Shader::Stage stage, Shader::LogicalStage l_stage, Shader::ShaderParams params)
         : info{stage, l_stage, params} {}
 
-    void AddPermut(vk::ShaderModule module, u64 spec_hash) {
-        modules.emplace_back(module, spec_hash);
+    void AddPermut(vk::ShaderModule module, Shader::StageSpecialization&& spec) {
+        modules.emplace_back(module, std::move(spec));
     }
 
-    void InsertPermut(vk::ShaderModule module, u64 spec_hash, size_t perm_idx) {
-        modules.resize(std::max(modules.size(), perm_idx + 1));
-        modules[perm_idx] = {module, spec_hash};
+    void InsertPermut(vk::ShaderModule module, Shader::StageSpecialization&& spec,
+                      size_t perm_idx) {
+        modules.resize(std::max(modules.size(), perm_idx + 1)); // <-- beware of realloc
+        modules[perm_idx] = {module, std::move(spec)};
     }
 };
 
@@ -107,6 +109,10 @@ private:
                                    Shader::Backend::Bindings& binding);
     const Shader::RuntimeInfo& BuildRuntimeInfo(Shader::Stage stage, Shader::LogicalStage l_stage);
 
+    [[nodiscard]] bool IsPipelineCacheDirty() const {
+        return num_new_pipelines > 0;
+    }
+
 private:
     const Instance& instance;
     Scheduler& scheduler;
@@ -125,6 +131,7 @@ private:
     std::optional<Shader::Gcn::FetchShaderData> fetch_shader{};
     GraphicsPipelineKey graphics_key{};
     ComputePipelineKey compute_key{};
+    u32 num_new_pipelines{}; // new pipelines added to the cache since the game start
 
     // Only if Config::collectShadersForDebug()
     tsl::robin_map<vk::ShaderModule,

src/video_core/renderer_vulkan/vk_pipeline_serialization.cpp

@@ -2,124 +2,19 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include "common/config.h"
-#include "common/hash.h"
+#include "common/serdes.h"
 #include "shader_recompiler/frontend/fetch_shader.h"
 #include "shader_recompiler/info.h"
+#include "video_core/cache_storage.h"
 #include "video_core/renderer_vulkan/vk_instance.h"
 #include "video_core/renderer_vulkan/vk_pipeline_cache.h"
-#include "video_core/renderer_vulkan/vk_pipeline_storage.h"
 #include "video_core/renderer_vulkan/vk_shader_util.h"
 
 namespace Serialization {
-
 /* You should increment versions below once corresponding serialization scheme is changed. */
-static constexpr u32 ShaderBinaryVersion = 0u;
-static constexpr u32 ShaderMetaVersion = 0u;
-static constexpr u32 PipelineKeyVersion = 0u;
-
-struct Archive {
-    void Alloc(size_t size) {
-        container.resize(size);
-    }
-
-    void Grow(size_t size) {
-        container.resize(container.size() + size);
-    }
-
-    void Merge(const Archive& ar) {
-        container.insert(container.end(), ar.container.cbegin(), ar.container.cend());
-        offset = container.size();
-    }
-
-    [[nodiscard]] size_t SizeBytes() const {
-        return container.size();
-    }
-
-    u8* CurrPtr() {
-        return container.data() + offset;
-    }
-
-    void Advance(size_t size) {
-        ASSERT(offset + size <= container.size());
-        offset += size;
-    }
-
-    std::vector<u8>&& TakeOff() {
-        offset = 0;
-        return std::move(container);
-    }
-
-    [[nodiscard]] bool IsEoS() const {
-        return offset >= container.size();
-    }
-
-    Archive() = default;
-    explicit Archive(std::vector<u8>&& v) : container{v} {}
-
-    u32 offset{};
-    std::vector<u8> container{};
-};
-
-struct Writer {
-    template <typename T>
-    void Write(const T* ptr, size_t size) {
-        if ((ar.offset + size) >= ar.container.size()) {
-            ar.Grow(size);
-        }
-        std::memcpy(ar.CurrPtr(), reinterpret_cast<const void*>(ptr), size);
-        ar.Advance(size);
-    }
-
-    template <typename T>
-    void Write(const T& value) {
-        const auto size = sizeof(value);
-        Write(&value, size);
-    }
-
-    template <typename T>
-    void Write(const std::vector<T>& v) {
-        Write(v.size());
-        for (const auto& elem : v) {
-            Write(elem);
-        }
-    }
-
-    Writer() = delete;
-    explicit Writer(Archive& ar_) : ar{ar_} {}
-
-    Archive& ar;
-};
-
-struct Reader {
-    template <typename T>
-    void Read(T* ptr, size_t size) {
-        ASSERT(ar.offset + size <= ar.container.size());
-        std::memcpy(reinterpret_cast<void*>(ptr), ar.CurrPtr(), size);
-        ar.Advance(size);
-    }
-
-    template <typename T>
-    void Read(T& value) {
-        const auto size = sizeof(T);
-        Read(&value, size);
-    }
-
-    template <typename T>
-    void Read(std::vector<T>& v) {
-        size_t num_elements{};
-        Read(num_elements);
-        v.resize(num_elements);
-        for (auto& elem : v) {
-            Read(elem);
-        }
-    }
-
-    Reader() = delete;
-    explicit Reader(Archive& ar_) : ar{ar_} {}
-
-    Archive& ar;
-};
-
+static constexpr u32 ShaderBinaryVersion = 1u;
+static constexpr u32 ShaderMetaVersion = 1u;
+static constexpr u32 PipelineKeyVersion = 1u;
 } // namespace Serialization
 
 namespace Vulkan {
@@ -164,7 +59,8 @@ void RegisterPipelineData(const GraphicsPipelineKey& key, u64 hash,
 
 void RegisterShaderMeta(const Shader::Info& info,
                         const std::optional<Shader::Gcn::FetchShaderData>& fetch_shader_data,
-                        u64 perm_hash, u64 spec_hash, size_t perm_idx) {
+                        const Shader::StageSpecialization& spec, size_t perm_hash,
+                        size_t perm_idx) {
     if (!Storage::DataBase::Instance().IsOpened()) {
         return;
     }
@@ -173,20 +69,14 @@ void RegisterShaderMeta(const Shader::Info& info,
     Serialization::Writer meta{ar};
 
     meta.Write(Serialization::ShaderMetaVersion);
+    meta.Write(Serialization::ShaderBinaryVersion);
 
     meta.Write(perm_hash);
     meta.Write(perm_idx);
-    meta.Write(spec_hash);
 
+    spec.Serialize(ar);
     info.Serialize(ar);
 
-    if (fetch_shader_data) {
-        meta.Write(sizeof(*fetch_shader_data));
-        fetch_shader_data->Serialize(ar);
-    } else {
-        meta.Write(size_t{0});
-    }
-
     Storage::DataBase::Instance().Save(Storage::BlobType::ShaderMeta,
                                        fmt::format("{:#018x}", perm_hash), ar.TakeOff());
 }
@@ -203,25 +93,29 @@ void RegisterShaderBinary(std::vector<u32>&& spv, u64 pgm_hash, size_t perm_idx)
 
 bool LoadShaderMeta(Serialization::Archive& ar, Shader::Info& info,
                     std::optional<Shader::Gcn::FetchShaderData>& fetch_shader_data,
-                    size_t& spec_hash, size_t& perm_idx) {
+                    Shader::StageSpecialization& spec, size_t& perm_idx) {
     Serialization::Reader meta{ar};
 
+    u32 meta_version{};
+    meta.Read(meta_version);
+    if (meta_version != Serialization::ShaderMetaVersion) {
+        return false;
+    }
+
+    u32 binary_version{};
+    meta.Read(binary_version);
+    if (binary_version != Serialization::ShaderBinaryVersion) {
+        return false;
+    }
+
     u64 perm_hash_ar{};
     meta.Read(perm_hash_ar);
     meta.Read(perm_idx);
-    meta.Read(spec_hash);
 
+    spec.Deserialize(ar);
     info.Deserialize(ar);
 
-    u64 fetch_data_size{};
-    meta.Read(fetch_data_size);
-
-    if (fetch_data_size) {
-        Shader::Gcn::FetchShaderData fetch_data;
-        fetch_data.Deserialize(ar);
-        fetch_shader_data = fetch_data;
-    }
-
+    fetch_shader_data = spec.fetch_shader_data;
     return true;
 }
 
@@ -260,13 +154,6 @@ bool PipelineCache::LoadComputePipeline(Serialization::Archive& ar) {
     }
 
     Serialization::Archive meta_ar{std::move(meta_blob)};
-    Serialization::Reader meta{meta_ar};
-
-    u32 meta_version{};
-    meta.Read(meta_version);
-    if (meta_version != Serialization::ShaderMetaVersion) {
-        return false;
-    }
 
     if (!LoadPipelineStage(meta_ar, 0)) {
         return false;
@@ -279,7 +166,7 @@ bool PipelineCache::LoadComputePipeline(Serialization::Archive& ar) {
         std::make_unique<ComputePipeline>(instance, scheduler, desc_heap, profile, *pipeline_cache,
                                           compute_key, *infos[0], modules[0], sdata, true);
 
-    infos.fill(0);
+    infos.fill(nullptr);
     modules.fill(nullptr);
 
     return true;
@@ -301,9 +188,9 @@ bool GraphicsPipelineKey::Deserialize(Serialization::Archive& ar) {
 void GraphicsPipeline::SerializationSupport::Serialize(Serialization::Archive& ar) const {
     Serialization::Writer sdata{ar};
 
-    sdata.Write(vertex_attributes);
-    sdata.Write(vertex_bindings);
-    sdata.Write(divisors);
+    sdata.Write(&vertex_attributes, sizeof(vertex_attributes));
+    sdata.Write(&vertex_bindings, sizeof(vertex_bindings));
+    sdata.Write(&divisors, sizeof(divisors));
     sdata.Write(multisampling);
     sdata.Write(tcs);
     sdata.Write(tes);
@@ -312,11 +199,10 @@ void GraphicsPipeline::SerializationSupport::Serialize(Serialization::Archive& a
 bool GraphicsPipeline::SerializationSupport::Deserialize(Serialization::Archive& ar) {
     Serialization::Reader sdata{ar};
 
-    sdata.Read(vertex_attributes);
-    sdata.Read(vertex_bindings);
-    sdata.Read(divisors);
+    sdata.Read(&vertex_attributes, sizeof(vertex_attributes));
+    sdata.Read(&vertex_bindings, sizeof(vertex_bindings));
+    sdata.Read(&divisors, sizeof(divisors));
     sdata.Read(multisampling);
-
     sdata.Read(tcs);
     sdata.Read(tes);
     return true;
@@ -342,13 +228,6 @@ bool PipelineCache::LoadGraphicsPipeline(Serialization::Archive& ar) {
         }
 
         Serialization::Archive meta_ar{std::move(meta_blob)};
-        Serialization::Reader meta{meta_ar};
-
-        u32 meta_version{};
-        meta.Read(meta_version);
-        if (meta_version != Serialization::ShaderMetaVersion) {
-            return false;
-        }
 
         if (!LoadPipelineStage(meta_ar, stage_idx)) {
             return false;
@@ -362,7 +241,7 @@ bool PipelineCache::LoadGraphicsPipeline(Serialization::Archive& ar) {
         instance, scheduler, desc_heap, profile, graphics_key, *pipeline_cache, infos,
         runtime_infos, fetch_shader, modules, sdata, true);
 
-    infos.fill(0);
+    infos.fill(nullptr);
     modules.fill(nullptr);
     fetch_shader.reset();
 
@@ -370,13 +249,11 @@ bool PipelineCache::LoadGraphicsPipeline(Serialization::Archive& ar) {
 }
 
 bool PipelineCache::LoadPipelineStage(Serialization::Archive& ar, size_t stage) {
-    Shader::Backend::Bindings binding{}; // not needed?
-    size_t spec_hash{};
-    size_t perm_idx;
-
     auto program = std::make_unique<Program>();
-
-    if (!LoadShaderMeta(ar, program->info, fetch_shader, spec_hash, perm_idx)) {
+    Shader::StageSpecialization spec{};
+    spec.info = &program->info;
+    size_t perm_idx{};
+    if (!LoadShaderMeta(ar, program->info, fetch_shader, spec, perm_idx)) {
         return false;
     }
 
@@ -398,18 +275,18 @@ bool PipelineCache::LoadPipelineStage(Serialization::Archive& ar, size_t stage)
         module = CompileSPV(spv, instance.GetDevice());
         it_pgm.value() = std::move(program);
     } else {
-        const auto& it =
-            std::ranges::find(it_pgm.value()->modules, spec_hash, &Program::Module::spec_hash);
+        const auto& it = std::ranges::find(it_pgm.value()->modules, spec, &Program::Module::spec);
         if (it != it_pgm.value()->modules.end()) {
             // If the permutation is already preloaded, make sure it has the same permutation index
             const auto idx = std::distance(it_pgm.value()->modules.begin(), it);
-            ASSERT(perm_idx == idx);
+            ASSERT_MSG(perm_idx == idx, "Permutation {} is already inserted at {}! ({}_{:x})",
+                       perm_idx, idx, program->info.stage, program->info.pgm_hash);
             module = it->module;
         } else {
             module = CompileSPV(spv, instance.GetDevice());
         }
     }
-    it_pgm.value()->InsertPermut(module, spec_hash, perm_idx);
+    it_pgm.value()->InsertPermut(module, std::move(spec), perm_idx);
 
     infos[stage] = &it_pgm.value()->info;
     modules[stage] = module;
@@ -424,6 +301,24 @@ void PipelineCache::WarmUp() {
 
     Storage::DataBase::Instance().Open();
 
+    // Check if cache is compatible
+    std::vector<u8> profile_data{};
+    Storage::DataBase::Instance().Load(Storage::BlobType::ShaderProfile, "profile", profile_data);
+    if (profile_data.empty()) {
+        Storage::DataBase::Instance().FinishPreload();
+
+        profile_data.resize(sizeof(profile));
+        std::memcpy(profile_data.data(), &profile, sizeof(profile));
+        Storage::DataBase::Instance().Save(Storage::BlobType::ShaderProfile, "profile",
+                                           std::move(profile_data));
+        return;
+    }
+    if (std::memcmp(profile_data.data(), &profile, sizeof(profile)) != 0) {
+        LOG_WARNING(Render,
+                    "Pipeline cache isn't compatible with current system. Ignoring the cache");
+        return;
+    }
+
     u32 num_pipelines{};
     u32 num_total_pipelines{};
 
@@ -460,6 +355,8 @@ void PipelineCache::WarmUp() {
         LOG_WARNING(Render, "{} stale pipelines were found. Consider re-generating the cache",
                     num_total_pipelines - num_pipelines);
     }
+
+    Storage::DataBase::Instance().FinishPreload();
 }
 
 void PipelineCache::Sync() {
@@ -508,56 +405,7 @@ bool Gcn::FetchShaderData::Deserialize(Serialization::Archive& ar) {
     return true;
 }
 
-u64 Shader::Gcn::FetchShaderData::Hash() const {
-    XXH64_state_t* const state = XXH64_createState();
-    XXH64_reset(state, 0);
-    XXH64_update(state, &size, sizeof(size));
-    XXH64_update(state, &vertex_offset_sgpr, sizeof(vertex_offset_sgpr));
-    XXH64_update(state, &instance_offset_sgpr, sizeof(instance_offset_sgpr));
-    for (const auto& attrib : attributes) {
-        XXH64_update(state, &attrib, sizeof(attrib));
-    }
-
-    const u64 hash = XXH64_digest(state);
-    XXH64_freeState(state);
-
-    return hash;
-}
-
-u64 StageSpecialization::Hash() const {
-    XXH64_state_t* const state = XXH64_createState();
-    XXH64_reset(state, 0);
-    XXH64_update(state, &start, sizeof(start));
-    XXH64_update(state, &runtime_info,
-                 sizeof(runtime_info)); // maybe broken because of union + span in GS
-
-    for (const auto& attrib : vs_attribs) {
-        XXH64_update(state, &attrib, sizeof(attrib));
-    }
-    for (const auto& buffer : buffers) {
-        XXH64_update(state, &buffer, sizeof(buffer));
-    }
-    for (const auto& image : images) {
-        XXH64_update(state, &image, sizeof(image));
-    }
-    for (const auto& sampler : samplers) {
-        XXH64_update(state, &sampler, sizeof(sampler));
-    }
-    for (const auto& fmask : fmasks) {
-        XXH64_update(state, &fmask, sizeof(fmask));
-    }
-
-    u64 hash = XXH64_digest(state);
-    XXH64_freeState(state);
-
-    if (fetch_shader_data) {
-        hash = HashCombine(hash, fetch_shader_data->Hash());
-    }
-
-    return hash;
-}
-
-void Shader::PersistentSrtInfo::Serialize(Serialization::Archive& ar) const {
+void PersistentSrtInfo::Serialize(Serialization::Archive& ar) const {
     Serialization::Writer srt{ar};
 
     srt.Write(this, sizeof(*this));
@@ -566,7 +414,7 @@ void Shader::PersistentSrtInfo::Serialize(Serialization::Archive& ar) const {
     }
 }
 
-bool Shader::PersistentSrtInfo::Deserialize(Serialization::Archive& ar) {
+bool PersistentSrtInfo::Deserialize(Serialization::Archive& ar) {
     Serialization::Reader srt{ar};
 
     srt.Read(this, sizeof(*this));
@@ -579,4 +427,54 @@ bool Shader::PersistentSrtInfo::Deserialize(Serialization::Archive& ar) {
     return true;
 }
 
+void StageSpecialization::Serialize(Serialization::Archive& ar) const {
+    Serialization::Writer spec{ar};
+
+    spec.Write(start);
+    spec.Write(runtime_info);
+
+    spec.Write(bitset.to_string());
+
+    if (fetch_shader_data) {
+        spec.Write(sizeof(*fetch_shader_data));
+        fetch_shader_data->Serialize(ar);
+    } else {
+        spec.Write(size_t{0});
+    }
+
+    spec.Write(vs_attribs);
+    spec.Write(buffers);
+    spec.Write(images);
+    spec.Write(fmasks);
+    spec.Write(samplers);
+}
+
+bool StageSpecialization::Deserialize(Serialization::Archive& ar) {
+    Serialization::Reader spec{ar};
+
+    spec.Read(start);
+    spec.Read(runtime_info);
+
+    std::string bits{};
+    spec.Read(bits);
+    bitset = std::bitset<MaxStageResources>(bits);
+
+    u64 fetch_data_size{};
+    spec.Read(fetch_data_size);
+
+    if (fetch_data_size) {
+        Gcn::FetchShaderData fetch_data;
+        fetch_data.Deserialize(ar);
+        fetch_shader_data = fetch_data;
+    }
+
+    spec.Read(vs_attribs);
+    spec.Read(buffers);
+    spec.Read(images);
+    spec.Read(fmasks);
+    spec.Read(samplers);
+
+    return true;
+}
+
 } // namespace Shader

src/video_core/renderer_vulkan/vk_pipeline_serialization.h

@@ -1,9 +1,10 @@
 // SPDX-FileCopyrightText: Copyright 2025 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include <optional>
+#pragma once
 
 #include "shader_recompiler/frontend/fetch_shader.h"
+#include "video_core/renderer_vulkan/vk_pipeline_cache.h"
 #include "video_core/renderer_vulkan/vk_shader_util.h"
 
 namespace Vulkan {
@@ -14,7 +15,7 @@ void RegisterPipelineData(const GraphicsPipelineKey& key, u64 hash,
                           GraphicsPipeline::SerializationSupport& sdata);
 void RegisterShaderMeta(const Shader::Info& info,
                         const std::optional<Shader::Gcn::FetchShaderData>& fetch_shader_data,
-                        u64 perm_hash, u64 spec_hash, size_t perm_idx);
+                        const Shader::StageSpecialization& spec, size_t perm_hash, size_t perm_idx);
 void RegisterShaderBinary(std::vector<u32>&& spv, u64 pgm_hash, size_t perm_idx);
 
 } // namespace Vulkan

src/video_core/renderer_vulkan/vk_pipeline_storage.cpp

@@ -1,239 +0,0 @@
-// SPDX-FileCopyrightText: Copyright 2025 shadPS4 Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#include "common/config.h"
-#include "common/elf_info.h"
-#include "common/hash.h"
-#include "common/io_file.h"
-#include "common/polyfill_thread.h"
-#include "common/thread.h"
-
-#include "video_core/renderer_vulkan/vk_instance.h"
-#include "video_core/renderer_vulkan/vk_pipeline_cache.h"
-#include "video_core/renderer_vulkan/vk_pipeline_serialization.h"
-#include "video_core/renderer_vulkan/vk_pipeline_storage.h"
-#include "video_core/renderer_vulkan/vk_shader_util.h"
-
-#include <condition_variable>
-#include <future>
-#include <mutex>
-#include <thread>
-#include <queue>
-
-namespace Vulkan {
-namespace Storage {
-
-std::mutex submit_mutex{};
-u32 num_requests{};
-std::condition_variable_any request_cv{};
-std::queue<std::future<void>> req_queue{};
-std::mutex m_request{};
-std::jthread io_worker{};
-
-void ProcessIO(std::stop_token stoken) {
-    Common::SetCurrentThreadName("shadPS4:PipelineCacheIO");
-
-    while (!stoken.stop_requested()) {
-        {
-            std::unique_lock lk{submit_mutex};
-            Common::CondvarWait(request_cv, lk, stoken, [&] { return num_requests; });
-        }
-
-        if (stoken.stop_requested()) {
-            break;
-        }
-
-        while (num_requests) {
-            std::future<void> request{};
-            {
-                std::scoped_lock lock{m_request};
-                if (req_queue.empty()) {
-                    continue;
-                }
-                request = std::move(req_queue.front());
-                req_queue.pop();
-            }
-
-            if (request.valid()) {
-                request.wait();
-            }
-
-            --num_requests;
-        }
-    }
-}
-
-constexpr std::string GetBlobFileExtension(BlobType type) {
-    switch (type) {
-    case BlobType::ShaderMeta: {
-        return "meta";
-        break;
-    }
-    case BlobType::ShaderBinary: {
-        return "spv";
-        break;
-    }
-    case BlobType::PipelineKey: {
-        return "key";
-    }
-    default:
-        UNREACHABLE();
-    }
-}
-
-void DataBase::Open() {
-    if (opened) {
-        return;
-    }
-
-    const auto& game_info = Common::ElfInfo::Instance();
-
-    using namespace Common::FS;
-    cache_dir = GetUserPath(PathType::CacheDir) / game_info.GameSerial();
-    if (!std::filesystem::exists(cache_dir)) {
-        std::filesystem::create_directories(cache_dir);
-    }
-
-    io_worker = std::jthread{ProcessIO};
-    opened = true;
-}
-
-void DataBase::Close() {
-    if (!IsOpened()) {
-        return;
-    }
-
-    io_worker.request_stop();
-    io_worker.join();
-}
-
-template <typename T>
-bool WriteVector(BlobType type, std::filesystem::path&& path, std::vector<T>&& v) {
-    auto request = std::async(
-        std::launch::async,
-        [=](std::filesystem::path&& path) {
-            using namespace Common::FS;
-            path.replace_extension(GetBlobFileExtension(type));
-            const auto file = IOFile{path, FileAccessMode::Create};
-            file.Write(v);
-        },
-        path);
-
-    {
-        std::scoped_lock lock{m_request};
-        req_queue.emplace(std::move(request));
-    }
-
-    std::scoped_lock lk{submit_mutex};
-    ++num_requests;
-    request_cv.notify_one();
-    return true;
-}
-
-template <typename T>
-void LoadVector(BlobType type, std::filesystem::path& path, std::vector<T>& v) {
-    using namespace Common::FS;
-    path.replace_extension(GetBlobFileExtension(type));
-    const auto file = IOFile{path, FileAccessMode::Read};
-    v.resize(file.GetSize() / sizeof(T));
-    file.Read(v);
-}
-
-bool DataBase::Save(BlobType type, const std::string& name, std::vector<u8>&& data) {
-    if (!opened) {
-        return false;
-    }
-
-    auto path = cache_dir / name;
-    return WriteVector(type, std::move(path), std::move(data));
-}
-
-bool DataBase::Save(BlobType type, const std::string& name, std::vector<u32>&& data) {
-    if (!opened) {
-        return false;
-    }
-
-    auto path = cache_dir / name;
-    return WriteVector(type, std::move(path), std::move(data));
-}
-
-void DataBase::Load(BlobType type, const std::string& name, std::vector<u8>& data) {
-    if (!opened) {
-        return;
-    }
-
-    auto path = cache_dir / name;
-    return LoadVector(type, path, data);
-}
-
-void DataBase::Load(BlobType type, const std::string& name, std::vector<u32>& data) {
-    if (!opened) {
-        return;
-    }
-
-    auto path = cache_dir / name;
-    return LoadVector(type, path, data);
-}
-
-void DataBase::ForEachBlob(BlobType type, const std::function<void(std::vector<u8>&& data)>& func) {
-    const auto& ext = GetBlobFileExtension(type);
-    for (const auto& file_name : std::filesystem::directory_iterator{cache_dir}) {
-        if (file_name.path().extension().string().ends_with(ext)) {
-            using namespace Common::FS;
-            const auto& file = IOFile{file_name, FileAccessMode::Read};
-            if (file.IsOpen()) {
-                std::vector<u8> data(file.GetSize());
-                file.Read(data);
-                func(std::move(data));
-            }
-        }
-    }
-}
-
-} // namespace Storage
-
-std::string PipelineCache::GetShaderName(Shader::Stage stage, u64 hash,
-                                         std::optional<size_t> perm) {
-    if (perm) {
-        return fmt::format("{}_{:#018x}_{}", stage, hash, *perm);
-    }
-    return fmt::format("{}_{:#018x}", stage, hash);
-}
-
-void PipelineCache::DumpShader(std::span<const u32> code, u64 hash, Shader::Stage stage,
-                               size_t perm_idx, std::string_view ext) {
-    if (!Config::dumpShaders()) {
-        return;
-    }
-
-    using namespace Common::FS;
-    const auto dump_dir = GetUserPath(PathType::ShaderDir) / "dumps";
-    if (!std::filesystem::exists(dump_dir)) {
-        std::filesystem::create_directories(dump_dir);
-    }
-    const auto filename = fmt::format("{}.{}", GetShaderName(stage, hash, perm_idx), ext);
-    const auto file = IOFile{dump_dir / filename, FileAccessMode::Create};
-    file.WriteSpan(code);
-}
-
-std::optional<std::vector<u32>> PipelineCache::GetShaderPatch(u64 hash, Shader::Stage stage,
-                                                              size_t perm_idx,
-                                                              std::string_view ext) {
-
-    using namespace Common::FS;
-    const auto patch_dir = GetUserPath(PathType::ShaderDir) / "patch";
-    if (!std::filesystem::exists(patch_dir)) {
-        std::filesystem::create_directories(patch_dir);
-    }
-    const auto filename = fmt::format("{}.{}", GetShaderName(stage, hash, perm_idx), ext);
-    const auto filepath = patch_dir / filename;
-    if (!std::filesystem::exists(filepath)) {
-        return {};
-    }
-    const auto file = IOFile{patch_dir / filename, FileAccessMode::Read};
-    std::vector<u32> code(file.GetSize() / sizeof(u32));
-    file.Read(code);
-    return code;
-}
-
-} // namespace Vulkan