From 9f015a4d58fa0a88105f9e01b3a395f4b5ade700 Mon Sep 17 00:00:00 2001 From: iwubcode Date: Sat, 1 Mar 2025 22:16:09 -0600 Subject: [PATCH 1/9] VideoCommon: add session id to custom asset, this id is only relevant for a particular game session but is slightly faster as a numeric value for lookups --- .../Core/VideoCommon/Assets/CustomAsset.cpp | 9 +++++-- Source/Core/VideoCommon/Assets/CustomAsset.h | 6 ++++- .../VideoCommon/Assets/CustomAssetLoader.h | 27 ++++++++++--------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Source/Core/VideoCommon/Assets/CustomAsset.cpp b/Source/Core/VideoCommon/Assets/CustomAsset.cpp index 1591f93f91..96205b084c 100644 --- a/Source/Core/VideoCommon/Assets/CustomAsset.cpp +++ b/Source/Core/VideoCommon/Assets/CustomAsset.cpp @@ -6,8 +6,8 @@ namespace VideoCommon { CustomAsset::CustomAsset(std::shared_ptr library, - const CustomAssetLibrary::AssetID& asset_id) - : m_owning_library(std::move(library)), m_asset_id(asset_id) + const CustomAssetLibrary::AssetID& asset_id, u64 session_id) + : m_owning_library(std::move(library)), m_asset_id(asset_id), m_session_id(session_id) { } @@ -34,6 +34,11 @@ const CustomAssetLibrary::TimeType& CustomAsset::GetLastLoadedTime() const return m_last_loaded_time; } +std::size_t CustomAsset::GetSessionId() const +{ + return m_session_id; +} + const CustomAssetLibrary::AssetID& CustomAsset::GetAssetId() const { return m_asset_id; diff --git a/Source/Core/VideoCommon/Assets/CustomAsset.h b/Source/Core/VideoCommon/Assets/CustomAsset.h index ea4a182932..85f0471ff7 100644 --- a/Source/Core/VideoCommon/Assets/CustomAsset.h +++ b/Source/Core/VideoCommon/Assets/CustomAsset.h @@ -18,7 +18,7 @@ class CustomAsset { public: CustomAsset(std::shared_ptr library, - const CustomAssetLibrary::AssetID& asset_id); + const CustomAssetLibrary::AssetID& asset_id, u64 session_id); virtual ~CustomAsset() = default; CustomAsset(const CustomAsset&) = delete; CustomAsset(CustomAsset&&) = delete; @@ -39,6 +39,9 @@ public: // Returns an id that uniquely identifies this asset const CustomAssetLibrary::AssetID& GetAssetId() const; + // Returns an id that is unique to this session + std::size_t GetSessionId() const; + // A rough estimate of how much space this asset // will take in memroy std::size_t GetByteSizeInMemory() const; @@ -49,6 +52,7 @@ protected: private: virtual CustomAssetLibrary::LoadInfo LoadImpl(const CustomAssetLibrary::AssetID& asset_id) = 0; CustomAssetLibrary::AssetID m_asset_id; + std::size_t m_session_id; mutable std::mutex m_info_lock; std::size_t m_bytes_loaded = 0; diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLoader.h b/Source/Core/VideoCommon/Assets/CustomAssetLoader.h index 90d4f81a0e..d38a2e1db3 100644 --- a/Source/Core/VideoCommon/Assets/CustomAssetLoader.h +++ b/Source/Core/VideoCommon/Assets/CustomAssetLoader.h @@ -67,19 +67,20 @@ private: if (shared) return shared; } - std::shared_ptr ptr(new AssetType(std::move(library), asset_id), [&](AssetType* a) { - { - std::lock_guard lk(m_asset_load_lock); - m_total_bytes_loaded -= a->GetByteSizeInMemory(); - m_assets_to_monitor.erase(a->GetAssetId()); - if (m_max_memory_available >= m_total_bytes_loaded && m_memory_exceeded) - { - INFO_LOG_FMT(VIDEO, "Asset memory went below limit, new assets can begin loading."); - m_memory_exceeded = false; - } - } - delete a; - }); + std::shared_ptr ptr( + new AssetType(std::move(library), asset_id, asset_map.size()), [&](AssetType* a) { + { + std::lock_guard lk(m_asset_load_lock); + m_total_bytes_loaded -= a->GetByteSizeInMemory(); + m_assets_to_monitor.erase(a->GetAssetId()); + if (m_max_memory_available >= m_total_bytes_loaded && m_memory_exceeded) + { + INFO_LOG_FMT(VIDEO, "Asset memory went below limit, new assets can begin loading."); + m_memory_exceeded = false; + } + } + delete a; + }); it->second = ptr; m_asset_load_thread.Push(it->second); return ptr; From d56952257128b25ec420f6ebd6577e109f33352c Mon Sep 17 00:00:00 2001 From: iwubcode Date: Sat, 1 Mar 2025 21:51:21 -0600 Subject: [PATCH 2/9] VideoCommon: add resource manager and new asset loader; the resource manager uses a least recently used cache to determine which assets get priority for loading. Additionally, if the system is low on memory, assets will be purged with the less requested assets being the first to go. The loader is multithreaded now and loads assets as quickly as possible as long as memory is available --- Source/Core/DolphinLib.props | 4 + .../VideoCommon/Assets/CustomAssetLoader2.cpp | 175 +++++++++++++++ .../VideoCommon/Assets/CustomAssetLoader2.h | 65 ++++++ .../Assets/CustomResourceManager.cpp | 202 ++++++++++++++++++ .../Assets/CustomResourceManager.h | 182 ++++++++++++++++ Source/Core/VideoCommon/CMakeLists.txt | 4 + 6 files changed, 632 insertions(+) create mode 100644 Source/Core/VideoCommon/Assets/CustomAssetLoader2.cpp create mode 100644 Source/Core/VideoCommon/Assets/CustomAssetLoader2.h create mode 100644 Source/Core/VideoCommon/Assets/CustomResourceManager.cpp create mode 100644 Source/Core/VideoCommon/Assets/CustomResourceManager.h diff --git a/Source/Core/DolphinLib.props b/Source/Core/DolphinLib.props index 4c7764202d..8a50657c9b 100644 --- a/Source/Core/DolphinLib.props +++ b/Source/Core/DolphinLib.props @@ -660,6 +660,8 @@ + + @@ -1307,6 +1309,8 @@ + + diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLoader2.cpp b/Source/Core/VideoCommon/Assets/CustomAssetLoader2.cpp new file mode 100644 index 0000000000..17fc2992e4 --- /dev/null +++ b/Source/Core/VideoCommon/Assets/CustomAssetLoader2.cpp @@ -0,0 +1,175 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "VideoCommon/Assets/CustomAssetLoader2.h" + +#include "Common/Logging/Log.h" +#include "Common/Thread.h" + +namespace VideoCommon +{ +void CustomAssetLoader2::Initialize() +{ + ResizeWorkerThreads(2); +} + +void CustomAssetLoader2 ::Shutdown() +{ + Reset(false); +} + +void CustomAssetLoader2::Reset(bool restart_worker_threads) +{ + const std::size_t worker_thread_count = m_worker_threads.size(); + StopWorkerThreads(); + + { + std::lock_guard guard(m_pending_work_lock); + m_pending_assets.clear(); + m_max_memory_allowed = 0; + m_current_asset_memory = 0; + } + + { + std::lock_guard guard(m_completed_work_lock); + m_completed_asset_session_ids.clear(); + m_completed_asset_memory = 0; + } + + if (restart_worker_threads) + { + StartWorkerThreads(static_cast(worker_thread_count)); + } +} + +bool CustomAssetLoader2::StartWorkerThreads(u32 num_worker_threads) +{ + if (num_worker_threads == 0) + return true; + + for (u32 i = 0; i < num_worker_threads; i++) + { + m_worker_thread_start_result.store(false); + + void* thread_param = nullptr; + std::thread thr(&CustomAssetLoader2::WorkerThreadEntryPoint, this, thread_param); + m_init_event.Wait(); + + if (!m_worker_thread_start_result.load()) + { + WARN_LOG_FMT(VIDEO, "Failed to start asset load worker thread."); + thr.join(); + break; + } + + m_worker_threads.push_back(std::move(thr)); + } + + return HasWorkerThreads(); +} + +bool CustomAssetLoader2::ResizeWorkerThreads(u32 num_worker_threads) +{ + if (m_worker_threads.size() == num_worker_threads) + return true; + + StopWorkerThreads(); + return StartWorkerThreads(num_worker_threads); +} + +bool CustomAssetLoader2::HasWorkerThreads() const +{ + return !m_worker_threads.empty(); +} + +void CustomAssetLoader2::StopWorkerThreads() +{ + if (!HasWorkerThreads()) + return; + + // Signal worker threads to stop, and wake all of them. + { + std::lock_guard guard(m_pending_work_lock); + m_exit_flag.Set(); + m_worker_thread_wake.notify_all(); + } + + // Wait for worker threads to exit. + for (std::thread& thr : m_worker_threads) + thr.join(); + m_worker_threads.clear(); + m_exit_flag.Clear(); +} + +void CustomAssetLoader2::WorkerThreadEntryPoint(void* param) +{ + Common::SetCurrentThreadName("Asset Loader Worker"); + + m_worker_thread_start_result.store(true); + m_init_event.Set(); + + WorkerThreadRun(); +} + +void CustomAssetLoader2::WorkerThreadRun() +{ + std::unique_lock pending_lock(m_pending_work_lock); + while (!m_exit_flag.IsSet()) + { + m_worker_thread_wake.wait(pending_lock); + + while (!m_pending_assets.empty() && !m_exit_flag.IsSet()) + { + auto pending_iter = m_pending_assets.begin(); + const auto item = *pending_iter; + m_pending_assets.erase(pending_iter); + + if ((m_current_asset_memory + m_completed_asset_memory) > m_max_memory_allowed) + break; + + pending_lock.unlock(); + if (item->Load()) + { + std::lock_guard completed_guard(m_completed_work_lock); + m_completed_asset_memory += item->GetByteSizeInMemory(); + m_completed_asset_session_ids.push_back(item->GetSessionId()); + } + + pending_lock.lock(); + } + } +} + +std::vector +CustomAssetLoader2::LoadAssets(const std::list& pending_assets, + u64 current_loaded_memory, u64 max_memory_allowed) +{ + u64 total_memory = current_loaded_memory; + std::vector completed_asset_session_ids; + { + std::lock_guard guard(m_completed_work_lock); + m_completed_asset_session_ids.swap(completed_asset_session_ids); + total_memory += m_completed_asset_memory; + m_completed_asset_memory = 0; + } + + if (pending_assets.empty()) + return completed_asset_session_ids; + + if (total_memory > max_memory_allowed) + return completed_asset_session_ids; + + // There's new assets to process, notify worker threads + { + std::lock_guard guard(m_pending_work_lock); + m_pending_assets = pending_assets; + m_current_asset_memory = total_memory; + m_max_memory_allowed = max_memory_allowed; + if (m_current_asset_memory < m_max_memory_allowed) + m_worker_thread_wake.notify_all(); + } + + return completed_asset_session_ids; +} + +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLoader2.h b/Source/Core/VideoCommon/Assets/CustomAssetLoader2.h new file mode 100644 index 0000000000..18b05fef2c --- /dev/null +++ b/Source/Core/VideoCommon/Assets/CustomAssetLoader2.h @@ -0,0 +1,65 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "Common/Event.h" +#include "Common/Flag.h" +#include "VideoCommon/Assets/CustomAsset.h" + +namespace VideoCommon +{ +class CustomAssetLoader2 +{ +public: + CustomAssetLoader2() = default; + ~CustomAssetLoader2() = default; + CustomAssetLoader2(const CustomAssetLoader2&) = delete; + CustomAssetLoader2(CustomAssetLoader2&&) = delete; + CustomAssetLoader2& operator=(const CustomAssetLoader2&) = delete; + CustomAssetLoader2& operator=(CustomAssetLoader2&&) = delete; + + void Initialize(); + void Shutdown(); + + // Returns a vector of asset session ids that were loaded in the last frame + std::vector LoadAssets(const std::list& pending_assets, + u64 current_loaded_memory, u64 max_memory_allowed); + + void Reset(bool restart_worker_threads = true); + +private: + bool StartWorkerThreads(u32 num_worker_threads); + bool ResizeWorkerThreads(u32 num_worker_threads); + bool HasWorkerThreads() const; + void StopWorkerThreads(); + + void WorkerThreadEntryPoint(void* param); + void WorkerThreadRun(); + + Common::Flag m_exit_flag; + Common::Event m_init_event; + + std::vector m_worker_threads; + std::atomic_bool m_worker_thread_start_result{false}; + + std::list m_pending_assets; + std::atomic m_current_asset_memory = 0; + u64 m_max_memory_allowed = 0; + std::mutex m_pending_work_lock; + + std::condition_variable m_worker_thread_wake; + + std::vector m_completed_asset_session_ids; + std::atomic m_completed_asset_memory = 0; + std::mutex m_completed_work_lock; +}; +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomResourceManager.cpp b/Source/Core/VideoCommon/Assets/CustomResourceManager.cpp new file mode 100644 index 0000000000..10a9742f87 --- /dev/null +++ b/Source/Core/VideoCommon/Assets/CustomResourceManager.cpp @@ -0,0 +1,202 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "VideoCommon/Assets/CustomResourceManager.h" + +#include + +#include "Common/MathUtil.h" +#include "Common/MemoryUtil.h" +#include "Common/VariantUtil.h" + +#include "VideoCommon/AbstractGfx.h" +#include "VideoCommon/Assets/CustomAsset.h" +#include "VideoCommon/Assets/TextureAsset.h" +#include "VideoCommon/BPMemory.h" +#include "VideoCommon/VideoConfig.h" +#include "VideoCommon/VideoEvents.h" + +namespace VideoCommon +{ +void CustomResourceManager::Initialize() +{ + m_asset_loader.Initialize(); + + const size_t sys_mem = Common::MemPhysical(); + const size_t recommended_min_mem = 2 * size_t(1024 * 1024 * 1024); + // keep 2GB memory for system stability if system RAM is 4GB+ - use half of memory in other cases + m_max_ram_available = + (sys_mem / 2 < recommended_min_mem) ? (sys_mem / 2) : (sys_mem - recommended_min_mem); + + m_xfb_event = AfterFrameEvent::Register([this](Core::System&) { XFBTriggered(""); }, + "CustomResourceManager"); +} + +void CustomResourceManager::Shutdown() +{ + Reset(); + + m_asset_loader.Shutdown(); +} + +void CustomResourceManager::Reset() +{ + m_asset_loader.Reset(true); + + m_loaded_assets = {}; + m_pending_assets = {}; + m_session_id_to_asset_data.clear(); + m_asset_id_to_session_id.clear(); + m_ram_used = 0; +} + +void CustomResourceManager::ReloadAsset(const CustomAssetLibrary::AssetID& asset_id) +{ + std::lock_guard guard(m_reload_mutex); + m_assets_to_reload.insert(asset_id); +} + +CustomTextureData* CustomResourceManager::GetTextureDataFromAsset( + const CustomAssetLibrary::AssetID& asset_id, + std::shared_ptr library) +{ + const auto [it, inserted] = + m_texture_data_asset_cache.try_emplace(asset_id, InternalTextureDataResource{}); + if (it->second.asset_data && + it->second.asset_data->load_type == AssetData::LoadType::LoadFinalyzed) + { + m_loaded_assets.put(it->second.asset->GetSessionId(), it->second.asset); + return &it->second.texture_data->m_texture; + } + + LoadTextureDataAsset(asset_id, std::move(library), &it->second); + + return nullptr; +} + +void CustomResourceManager::LoadTextureDataAsset( + const CustomAssetLibrary::AssetID& asset_id, + std::shared_ptr library, + InternalTextureDataResource* internal_texture_data) +{ + if (!internal_texture_data->asset) + { + internal_texture_data->asset = + CreateAsset(asset_id, AssetData::AssetType::TextureData, library); + internal_texture_data->asset_data = + &m_session_id_to_asset_data[internal_texture_data->asset->GetSessionId()]; + } + + auto texture_data = internal_texture_data->asset->GetData(); + if (!texture_data || + internal_texture_data->asset_data->load_type == AssetData::LoadType::PendingReload) + { + // Tell the system we are still interested in loading this asset + const auto session_id = internal_texture_data->asset->GetSessionId(); + m_pending_assets.put(session_id, m_session_id_to_asset_data[session_id].asset.get()); + } + else if (internal_texture_data->asset_data->load_type == AssetData::LoadType::LoadFinished) + { + internal_texture_data->texture_data = std::move(texture_data); + internal_texture_data->asset_data->load_type = AssetData::LoadType::LoadFinalyzed; + } +} + +void CustomResourceManager::XFBTriggered(std::string_view) +{ + std::set session_ids_reloaded_this_frame; + + // Look for any assets requested to be reloaded + { + decltype(m_assets_to_reload) assets_to_reload; + + if (m_reload_mutex.try_lock()) + { + std::swap(assets_to_reload, m_assets_to_reload); + m_reload_mutex.unlock(); + } + + for (const auto& asset_id : assets_to_reload) + { + if (const auto it = m_asset_id_to_session_id.find(asset_id); + it != m_asset_id_to_session_id.end()) + { + const auto session_id = it->second; + session_ids_reloaded_this_frame.insert(session_id); + AssetData& asset_data = m_session_id_to_asset_data[session_id]; + asset_data.load_type = AssetData::LoadType::PendingReload; + asset_data.has_errors = false; + for (const auto owner_session_id : asset_data.asset_owners) + { + AssetData& owner_asset_data = m_session_id_to_asset_data[owner_session_id]; + if (owner_asset_data.load_type == AssetData::LoadType::LoadFinalyzed) + { + owner_asset_data.load_type = AssetData::LoadType::DependenciesChanged; + } + } + m_pending_assets.put(it->second, asset_data.asset.get()); + } + } + } + + if (m_ram_used > m_max_ram_available) + { + const u64 threshold_ram = 0.8f * m_max_ram_available; + u64 ram_used = m_ram_used; + + // Clear out least recently used resources until + // we get safely in our threshold + while (ram_used > threshold_ram && m_loaded_assets.size() > 0) + { + const auto asset = m_loaded_assets.pop(); + ram_used -= asset->GetByteSizeInMemory(); + + AssetData& asset_data = m_session_id_to_asset_data[asset->GetSessionId()]; + + if (asset_data.type == AssetData::AssetType::TextureData) + { + m_texture_data_asset_cache.erase(asset->GetAssetId()); + } + asset_data.asset.reset(); + asset_data.load_type = AssetData::LoadType::Unloaded; + } + + // Recalculate to ensure accuracy + m_ram_used = 0; + for (const auto asset : m_loaded_assets.elements()) + { + m_ram_used += asset->GetByteSizeInMemory(); + } + } + + if (m_pending_assets.empty()) + return; + + const auto asset_session_ids_loaded = + m_asset_loader.LoadAssets(m_pending_assets.elements(), m_ram_used, m_max_ram_available); + for (const std::size_t session_id : asset_session_ids_loaded) + { + // While unlikely, if we loaded an asset in the previous frame but it was reloaded + // this frame, we should ignore this load and wait on the reload + if (session_ids_reloaded_this_frame.count(session_id) > 0) [[unlikely]] + continue; + + m_pending_assets.erase(session_id); + + AssetData& asset_data = m_session_id_to_asset_data[session_id]; + m_loaded_assets.put(session_id, asset_data.asset.get()); + asset_data.load_type = AssetData::LoadType::LoadFinished; + m_ram_used += asset_data.asset->GetByteSizeInMemory(); + + for (const auto owner_session_id : asset_data.asset_owners) + { + AssetData& owner_asset_data = m_session_id_to_asset_data[owner_session_id]; + if (owner_asset_data.load_type == AssetData::LoadType::LoadFinalyzed) + { + owner_asset_data.load_type = AssetData::LoadType::DependenciesChanged; + } + } + } +} + +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomResourceManager.h b/Source/Core/VideoCommon/Assets/CustomResourceManager.h new file mode 100644 index 0000000000..b54b53400d --- /dev/null +++ b/Source/Core/VideoCommon/Assets/CustomResourceManager.h @@ -0,0 +1,182 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Common/CommonTypes.h" +#include "Common/HookableEvent.h" + +#include "VideoCommon/Assets/CustomAsset.h" +#include "VideoCommon/Assets/CustomAssetLibrary.h" +#include "VideoCommon/Assets/CustomAssetLoader2.h" +#include "VideoCommon/Assets/CustomTextureData.h" + +namespace VideoCommon +{ +class GameTextureAsset; + +class CustomResourceManager +{ +public: + void Initialize(); + void Shutdown(); + + void Reset(); + + // Requests that an asset that exists be reloaded + void ReloadAsset(const CustomAssetLibrary::AssetID& asset_id); + + void XFBTriggered(std::string_view texture_hash); + + CustomTextureData* + GetTextureDataFromAsset(const CustomAssetLibrary::AssetID& asset_id, + std::shared_ptr library); + +private: + struct AssetData + { + std::unique_ptr asset; + CustomAssetLibrary::TimeType load_request_time = {}; + std::set asset_owners; + + enum class AssetType + { + TextureData + }; + AssetType type; + + enum class LoadType + { + PendingReload, + LoadFinished, + LoadFinalyzed, + DependenciesChanged, + Unloaded + }; + LoadType load_type = LoadType::PendingReload; + bool has_errors = false; + }; + + struct InternalTextureDataResource + { + AssetData* asset_data = nullptr; + VideoCommon::GameTextureAsset* asset = nullptr; + std::shared_ptr texture_data; + }; + + void LoadTextureDataAsset(const CustomAssetLibrary::AssetID& asset_id, + std::shared_ptr library, + InternalTextureDataResource* internal_texture_data); + + template + T* CreateAsset(const CustomAssetLibrary::AssetID& asset_id, AssetData::AssetType asset_type, + std::shared_ptr library) + { + const auto [it, added] = + m_asset_id_to_session_id.try_emplace(asset_id, m_session_id_to_asset_data.size()); + + if (added) + { + AssetData asset_data; + asset_data.asset = std::make_unique(library, asset_id, it->second); + asset_data.type = asset_type; + asset_data.has_errors = false; + asset_data.load_type = AssetData::LoadType::PendingReload; + asset_data.load_request_time = {}; + + m_session_id_to_asset_data.insert_or_assign(it->second, std::move(asset_data)); + + // Synchronize the priority cache session id + m_pending_assets.prepare(); + m_loaded_assets.prepare(); + } + auto& asset_data_from_session = m_session_id_to_asset_data[it->second]; + + // Asset got unloaded, rebuild it with the same metadata + if (!asset_data_from_session.asset) + { + asset_data_from_session.asset = std::make_unique(library, asset_id, it->second); + asset_data_from_session.has_errors = false; + asset_data_from_session.load_type = AssetData::LoadType::PendingReload; + } + + return static_cast(asset_data_from_session.asset.get()); + } + + class LeastRecentlyUsedCache + { + public: + const std::list& elements() const { return m_asset_cache; } + + void put(u64 asset_session_id, CustomAsset* asset) + { + erase(asset_session_id); + m_asset_cache.push_front(asset); + m_iterator_lookup[m_asset_cache.front()->GetSessionId()] = m_asset_cache.begin(); + } + + CustomAsset* pop() + { + if (m_asset_cache.empty()) [[unlikely]] + return nullptr; + const auto ret = m_asset_cache.back(); + if (ret != nullptr) + { + m_iterator_lookup[ret->GetSessionId()].reset(); + } + m_asset_cache.pop_back(); + return ret; + } + + void prepare() { m_iterator_lookup.push_back(std::nullopt); } + + void erase(u64 asset_session_id) + { + if (const auto iter = m_iterator_lookup[asset_session_id]) + { + m_asset_cache.erase(*iter); + m_iterator_lookup[asset_session_id].reset(); + } + } + + bool empty() const { return m_asset_cache.empty(); } + + std::size_t size() const { return m_asset_cache.size(); } + + private: + std::list m_asset_cache; + + // Note: this vector is expected to be kept in sync with + // the total amount of (unique) assets ever seen + std::vector> m_iterator_lookup; + }; + + LeastRecentlyUsedCache m_loaded_assets; + LeastRecentlyUsedCache m_pending_assets; + + std::map m_session_id_to_asset_data; + std::map m_asset_id_to_session_id; + + u64 m_ram_used = 0; + u64 m_max_ram_available = 0; + + std::map m_texture_data_asset_cache; + + std::mutex m_reload_mutex; + std::set m_assets_to_reload; + + CustomAssetLoader2 m_asset_loader; + + Common::EventHook m_xfb_event; +}; + +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/CMakeLists.txt b/Source/Core/VideoCommon/CMakeLists.txt index 6361e639c4..78dc115db7 100644 --- a/Source/Core/VideoCommon/CMakeLists.txt +++ b/Source/Core/VideoCommon/CMakeLists.txt @@ -14,6 +14,10 @@ add_library(videocommon Assets/CustomAssetLibrary.h Assets/CustomAssetLoader.cpp Assets/CustomAssetLoader.h + Assets/CustomAssetLoader2.cpp + Assets/CustomAssetLoader2.h + Assets/CustomResourceManager.cpp + Assets/CustomResourceManager.h Assets/CustomTextureData.cpp Assets/CustomTextureData.h Assets/DirectFilesystemAssetLibrary.cpp From 320d5469fd15477f9b35b141832aa08438c31d7f Mon Sep 17 00:00:00 2001 From: iwubcode Date: Sat, 1 Mar 2025 21:53:34 -0600 Subject: [PATCH 3/9] Core: add CustomResourceManager to System --- Source/Core/Core/System.cpp | 7 +++++++ Source/Core/Core/System.h | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Source/Core/Core/System.cpp b/Source/Core/Core/System.cpp index 695d5861fc..94a503ab32 100644 --- a/Source/Core/Core/System.cpp +++ b/Source/Core/Core/System.cpp @@ -33,6 +33,7 @@ #include "IOS/USB/Emulated/Infinity.h" #include "IOS/USB/Emulated/Skylanders/Skylander.h" #include "VideoCommon/Assets/CustomAssetLoader.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/CommandProcessor.h" #include "VideoCommon/Fifo.h" #include "VideoCommon/GeometryShaderManager.h" @@ -95,6 +96,7 @@ struct System::Impl Interpreter m_interpreter; JitInterface m_jit_interface; VideoCommon::CustomAssetLoader m_custom_asset_loader; + VideoCommon::CustomResourceManager m_custom_resource_manager; FifoPlayer m_fifo_player; FifoRecorder m_fifo_recorder; Movie::MovieManager m_movie; @@ -332,4 +334,9 @@ VideoCommon::CustomAssetLoader& System::GetCustomAssetLoader() const { return m_impl->m_custom_asset_loader; } + +VideoCommon::CustomResourceManager& System::GetCustomResourceManager() const +{ + return m_impl->m_custom_resource_manager; +} } // namespace Core diff --git a/Source/Core/Core/System.h b/Source/Core/Core/System.h index 0ab86e658d..7b53ead5f5 100644 --- a/Source/Core/Core/System.h +++ b/Source/Core/Core/System.h @@ -108,7 +108,8 @@ class SystemTimersManager; namespace VideoCommon { class CustomAssetLoader; -} +class CustomResourceManager; +} // namespace VideoCommon namespace VideoInterface { class VideoInterfaceManager; @@ -196,6 +197,7 @@ public: XFStateManager& GetXFStateManager() const; VideoInterface::VideoInterfaceManager& GetVideoInterface() const; VideoCommon::CustomAssetLoader& GetCustomAssetLoader() const; + VideoCommon::CustomResourceManager& GetCustomResourceManager() const; private: System(); From 5acc9f105ee1509a49fbb00079dbf52cb65d8086 Mon Sep 17 00:00:00 2001 From: iwubcode Date: Sat, 1 Mar 2025 21:54:36 -0600 Subject: [PATCH 4/9] VideoCommon: initialize and shutdown the CustomResourceManager when the video thread initializes and shuts down --- Source/Core/VideoCommon/VideoBackendBase.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Source/Core/VideoCommon/VideoBackendBase.cpp b/Source/Core/VideoCommon/VideoBackendBase.cpp index 73035e0e1b..a4cc35fa09 100644 --- a/Source/Core/VideoCommon/VideoBackendBase.cpp +++ b/Source/Core/VideoCommon/VideoBackendBase.cpp @@ -41,6 +41,7 @@ #endif #include "VideoCommon/AbstractGfx.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/AsyncRequests.h" #include "VideoCommon/BPStructs.h" #include "VideoCommon/BoundingBox.h" @@ -341,12 +342,16 @@ bool VideoBackendBase::InitializeShared(std::unique_ptr gfx, } g_shader_cache->InitializeShaderCache(); + system.GetCustomResourceManager().Initialize(); return true; } void VideoBackendBase::ShutdownShared() { + auto& system = Core::System::GetInstance(); + system.GetCustomResourceManager().Shutdown(); + g_frame_dumper.reset(); g_presenter.reset(); @@ -369,7 +374,6 @@ void VideoBackendBase::ShutdownShared() m_initialized = false; - auto& system = Core::System::GetInstance(); VertexLoaderManager::Clear(); system.GetFifo().Shutdown(); } From bbb08e2f175bc4ee1b8dea571df545187c2acd65 Mon Sep 17 00:00:00 2001 From: iwubcode Date: Sat, 1 Mar 2025 21:55:07 -0600 Subject: [PATCH 5/9] VideoCommon: use CustomResourceManager in the texture cache and hook up to our hires textures --- Source/Core/VideoCommon/HiresTextures.cpp | 33 +++--- Source/Core/VideoCommon/HiresTextures.h | 8 +- Source/Core/VideoCommon/TextureCacheBase.cpp | 115 ++++--------------- Source/Core/VideoCommon/TextureCacheBase.h | 14 +-- 4 files changed, 53 insertions(+), 117 deletions(-) diff --git a/Source/Core/VideoCommon/HiresTextures.cpp b/Source/Core/VideoCommon/HiresTextures.cpp index b64c521e08..06f354e73a 100644 --- a/Source/Core/VideoCommon/HiresTextures.cpp +++ b/Source/Core/VideoCommon/HiresTextures.cpp @@ -25,7 +25,7 @@ #include "Core/ConfigManager.h" #include "Core/System.h" #include "VideoCommon/Assets/CustomAsset.h" -#include "VideoCommon/Assets/CustomAssetLoader.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/Assets/DirectFilesystemAssetLibrary.h" #include "VideoCommon/OnScreenDisplay.h" #include "VideoCommon/VideoConfig.h" @@ -95,8 +95,6 @@ void HiresTexture::Update() GetTextureDirectoriesWithGameId(File::GetUserPath(D_HIRESTEXTURES_IDX), game_id); const std::vector extensions{".png", ".dds"}; - auto& system = Core::System::GetInstance(); - for (const auto& texture_directory : texture_directories) { const auto texture_paths = @@ -130,10 +128,10 @@ void HiresTexture::Update() if (g_ActiveConfig.bCacheHiresTextures) { - auto hires_texture = std::make_shared( - has_arbitrary_mipmaps, - system.GetCustomAssetLoader().LoadGameTexture(filename, s_file_library)); - s_hires_texture_cache.try_emplace(filename, std::move(hires_texture)); + auto hires_texture = + std::make_shared(has_arbitrary_mipmaps, std::move(filename)); + static_cast(hires_texture->LoadTexture()); + s_hires_texture_cache.try_emplace(hires_texture->GetId(), hires_texture); } } } @@ -167,7 +165,7 @@ void HiresTexture::Clear() std::shared_ptr HiresTexture::Search(const TextureInfo& texture_info) { - const auto [base_filename, has_arb_mipmaps] = GetNameArbPair(texture_info); + auto [base_filename, has_arb_mipmaps] = GetNameArbPair(texture_info); if (base_filename == "") return nullptr; @@ -177,24 +175,27 @@ std::shared_ptr HiresTexture::Search(const TextureInfo& texture_in } else { - auto& system = Core::System::GetInstance(); - auto hires_texture = std::make_shared( - has_arb_mipmaps, - system.GetCustomAssetLoader().LoadGameTexture(base_filename, s_file_library)); + auto hires_texture = std::make_shared(has_arb_mipmaps, std::move(base_filename)); if (g_ActiveConfig.bCacheHiresTextures) { - s_hires_texture_cache.try_emplace(base_filename, hires_texture); + s_hires_texture_cache.try_emplace(hires_texture->GetId(), hires_texture); } return hires_texture; } } -HiresTexture::HiresTexture(bool has_arbitrary_mipmaps, - std::shared_ptr asset) - : m_has_arbitrary_mipmaps(has_arbitrary_mipmaps), m_game_texture(std::move(asset)) +HiresTexture::HiresTexture(bool has_arbitrary_mipmaps, std::string id) + : m_has_arbitrary_mipmaps(has_arbitrary_mipmaps), m_id(std::move(id)) { } +VideoCommon::CustomTextureData* HiresTexture::LoadTexture() const +{ + auto& system = Core::System::GetInstance(); + auto& custom_resource_manager = system.GetCustomResourceManager(); + return custom_resource_manager.GetTextureDataFromAsset(m_id, s_file_library); +} + std::set GetTextureDirectoriesWithGameId(const std::string& root_directory, const std::string& game_id) { diff --git a/Source/Core/VideoCommon/HiresTextures.h b/Source/Core/VideoCommon/HiresTextures.h index 5312285dd0..ceb5b3ce2b 100644 --- a/Source/Core/VideoCommon/HiresTextures.h +++ b/Source/Core/VideoCommon/HiresTextures.h @@ -10,7 +10,6 @@ #include "Common/CommonTypes.h" #include "VideoCommon/Assets/CustomTextureData.h" -#include "VideoCommon/Assets/TextureAsset.h" #include "VideoCommon/TextureConfig.h" #include "VideoCommon/TextureInfo.h" @@ -27,12 +26,13 @@ public: static void Shutdown(); static std::shared_ptr Search(const TextureInfo& texture_info); - HiresTexture(bool has_arbitrary_mipmaps, std::shared_ptr asset); + HiresTexture(bool has_arbitrary_mipmaps, std::string id); bool HasArbitraryMipmaps() const { return m_has_arbitrary_mipmaps; } - const std::shared_ptr& GetAsset() const { return m_game_texture; } + VideoCommon::CustomTextureData* LoadTexture() const; + const std::string& GetId() const { return m_id; } private: bool m_has_arbitrary_mipmaps = false; - std::shared_ptr m_game_texture; + std::string m_id; }; diff --git a/Source/Core/VideoCommon/TextureCacheBase.cpp b/Source/Core/VideoCommon/TextureCacheBase.cpp index b370481c04..3e01c0226e 100644 --- a/Source/Core/VideoCommon/TextureCacheBase.cpp +++ b/Source/Core/VideoCommon/TextureCacheBase.cpp @@ -37,13 +37,13 @@ #include "VideoCommon/AbstractFramebuffer.h" #include "VideoCommon/AbstractGfx.h" #include "VideoCommon/AbstractStagingTexture.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/Assets/CustomTextureData.h" #include "VideoCommon/BPMemory.h" #include "VideoCommon/FramebufferManager.h" #include "VideoCommon/GraphicsModSystem/Runtime/FBInfo.h" #include "VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h" #include "VideoCommon/GraphicsModSystem/Runtime/GraphicsModManager.h" -#include "VideoCommon/HiresTextures.h" #include "VideoCommon/OpcodeDecoding.h" #include "VideoCommon/PixelShaderManager.h" #include "VideoCommon/Present.h" @@ -262,25 +262,10 @@ void TextureCacheBase::SetBackupConfig(const VideoConfig& config) bool TextureCacheBase::DidLinkedAssetsChange(const TCacheEntry& entry) { - for (const auto& cached_asset : entry.linked_game_texture_assets) - { - if (cached_asset.m_asset) - { - if (cached_asset.m_asset->GetLastLoadedTime() > cached_asset.m_cached_write_time) - return true; - } - } + if (!entry.hires_texture) + return false; - for (const auto& cached_asset : entry.linked_asset_dependencies) - { - if (cached_asset.m_asset) - { - if (cached_asset.m_asset->GetLastLoadedTime() > cached_asset.m_cached_write_time) - return true; - } - } - - return false; + return entry.last_custom_texture_data != entry.hires_texture->LoadTexture(); } RcTcacheEntry TextureCacheBase::ApplyPaletteToEntry(RcTcacheEntry& entry, const u8* palette, @@ -1566,80 +1551,42 @@ RcTcacheEntry TextureCacheBase::GetTexture(const int textureCacheSafetyColorSamp InvalidateTexture(oldest_entry); } - std::vector> cached_game_assets; - std::vector> data_for_assets; + std::shared_ptr hires_texture; bool has_arbitrary_mipmaps = false; bool skip_texture_dump = false; - std::shared_ptr hires_texture; + VideoCommon::CustomTextureData* custom_texture_data = nullptr; if (g_ActiveConfig.bHiresTextures) { hires_texture = HiresTexture::Search(texture_info); if (hires_texture) { - auto asset = hires_texture->GetAsset(); - const auto loaded_time = asset->GetLastLoadedTime(); - cached_game_assets.push_back( - VideoCommon::CachedAsset{std::move(asset), loaded_time}); has_arbitrary_mipmaps = hires_texture->HasArbitraryMipmaps(); + custom_texture_data = hires_texture->LoadTexture(); skip_texture_dump = true; } } - std::vector> additional_dependencies; - std::string texture_name = ""; if (g_ActiveConfig.bGraphicMods) { u32 height = texture_info.GetRawHeight(); u32 width = texture_info.GetRawWidth(); - if (hires_texture) - { - auto asset = hires_texture->GetAsset(); - if (asset) - { - auto data = asset->GetData(); - if (data) - { - if (!data->m_texture.m_slices.empty()) - { - if (!data->m_texture.m_slices[0].m_levels.empty()) - { - height = data->m_texture.m_slices[0].m_levels[0].height; - width = data->m_texture.m_slices[0].m_levels[0].width; - } - } - } - } - } texture_name = texture_info.CalculateTextureName().GetFullName(); - GraphicsModActionData::TextureCreate texture_create{ - texture_name, width, height, &cached_game_assets, &additional_dependencies}; + GraphicsModActionData::TextureCreate texture_create{texture_name, width, height, nullptr, + nullptr}; for (const auto& action : g_graphics_mod_manager->GetTextureCreateActions(texture_name)) { action->OnTextureCreate(&texture_create); } } - data_for_assets.reserve(cached_game_assets.size()); - for (auto& cached_asset : cached_game_assets) - { - auto data = cached_asset.m_asset->GetData(); - if (data) - { - if (cached_asset.m_asset->Validate(texture_info.GetRawWidth(), texture_info.GetRawHeight())) - { - data_for_assets.push_back(data); - } - } - } - auto entry = CreateTextureEntry(TextureCreationInfo{base_hash, full_hash, bytes_per_block, palette_size}, - texture_info, textureCacheSafetyColorSampleSize, - std::move(data_for_assets), has_arbitrary_mipmaps, skip_texture_dump); - entry->linked_game_texture_assets = std::move(cached_game_assets); - entry->linked_asset_dependencies = std::move(additional_dependencies); + texture_info, textureCacheSafetyColorSampleSize, custom_texture_data, + has_arbitrary_mipmaps, skip_texture_dump); + entry->hires_texture = std::move(hires_texture); + entry->last_custom_texture_data = custom_texture_data; entry->texture_info_name = std::move(texture_name); return entry; } @@ -1649,8 +1596,7 @@ RcTcacheEntry TextureCacheBase::GetTexture(const int textureCacheSafetyColorSamp // expected because each texture is loaded into a texture array RcTcacheEntry TextureCacheBase::CreateTextureEntry( const TextureCreationInfo& creation_info, const TextureInfo& texture_info, - const int safety_color_sample_size, - std::vector> assets_data, + const int safety_color_sample_size, VideoCommon::CustomTextureData* custom_texture_data, const bool custom_arbitrary_mipmaps, bool skip_texture_dump) { #ifdef __APPLE__ @@ -1660,33 +1606,22 @@ RcTcacheEntry TextureCacheBase::CreateTextureEntry( #endif RcTcacheEntry entry; - if (!assets_data.empty()) + if (custom_texture_data) { - const auto calculate_max_levels = [&]() { - const auto max_element = std::ranges::max_element( - assets_data, {}, [](const auto& v) { return v->m_texture.m_slices[0].m_levels.size(); }); - return (*max_element)->m_texture.m_slices[0].m_levels.size(); - }; - const u32 texLevels = no_mips ? 1 : (u32)calculate_max_levels(); - const auto& first_level = assets_data[0]->m_texture.m_slices[0].m_levels[0]; - const TextureConfig config(first_level.width, first_level.height, texLevels, - static_cast(assets_data.size()), 1, first_level.format, 0, - AbstractTextureType::Texture_2DArray); + const u32 texLevels = no_mips ? 1 : (u32)custom_texture_data->m_slices[0].m_levels.size(); + const auto& first_level = custom_texture_data->m_slices[0].m_levels[0]; + const TextureConfig config(first_level.width, first_level.height, texLevels, 1, 1, + first_level.format, 0, AbstractTextureType::Texture_2DArray); entry = AllocateCacheEntry(config); if (!entry) [[unlikely]] return entry; - for (u32 data_index = 0; data_index < static_cast(assets_data.size()); data_index++) + const auto& slice = custom_texture_data->m_slices[0]; + for (u32 level_index = 0; + level_index < std::min(texLevels, static_cast(slice.m_levels.size())); ++level_index) { - const auto& asset = assets_data[data_index]; - const auto& slice = asset->m_texture.m_slices[0]; - for (u32 level_index = 0; - level_index < std::min(texLevels, static_cast(slice.m_levels.size())); - ++level_index) - { - const auto& level = slice.m_levels[level_index]; - entry->texture->Load(level_index, level.width, level.height, level.row_length, - level.data.data(), level.data.size(), data_index); - } + const auto& level = slice.m_levels[level_index]; + entry->texture->Load(level_index, level.width, level.height, level.row_length, + level.data.data(), level.data.size()); } entry->has_arbitrary_mips = custom_arbitrary_mipmaps; diff --git a/Source/Core/VideoCommon/TextureCacheBase.h b/Source/Core/VideoCommon/TextureCacheBase.h index 532feaca7d..802ae7c2ee 100644 --- a/Source/Core/VideoCommon/TextureCacheBase.h +++ b/Source/Core/VideoCommon/TextureCacheBase.h @@ -24,6 +24,7 @@ #include "VideoCommon/AbstractTexture.h" #include "VideoCommon/Assets/CustomAsset.h" #include "VideoCommon/BPMemory.h" +#include "VideoCommon/HiresTextures.h" #include "VideoCommon/TextureConfig.h" #include "VideoCommon/TextureDecoder.h" #include "VideoCommon/TextureInfo.h" @@ -167,8 +168,8 @@ struct TCacheEntry std::string texture_info_name = ""; - std::vector> linked_game_texture_assets; - std::vector> linked_asset_dependencies; + VideoCommon::CustomTextureData* last_custom_texture_data; + std::shared_ptr hires_texture; explicit TCacheEntry(std::unique_ptr tex, std::unique_ptr fb); @@ -351,11 +352,10 @@ private: void SetBackupConfig(const VideoConfig& config); - RcTcacheEntry - CreateTextureEntry(const TextureCreationInfo& creation_info, const TextureInfo& texture_info, - int safety_color_sample_size, - std::vector> assets_data, - bool custom_arbitrary_mipmaps, bool skip_texture_dump); + RcTcacheEntry CreateTextureEntry(const TextureCreationInfo& creation_info, + const TextureInfo& texture_info, int safety_color_sample_size, + VideoCommon::CustomTextureData* custom_texture_data, + bool custom_arbitrary_mipmaps, bool skip_texture_dump); RcTcacheEntry GetXFBFromCache(u32 address, u32 width, u32 height, u32 stride); From cda1502b49aa4ba370324fd5130c558fe232f06e Mon Sep 17 00:00:00 2001 From: iwubcode Date: Thu, 4 Jul 2024 17:34:14 -0500 Subject: [PATCH 6/9] Externals: add watcher, a library used to watch a filesystem location for changes --- .gitmodules | 3 +++ CMakeLists.txt | 2 ++ Externals/watcher/CMakeLists.txt | 4 ++++ Externals/watcher/watcher | 1 + Source/VSProps/Base.Dolphin.props | 1 + 5 files changed, 11 insertions(+) create mode 100644 Externals/watcher/CMakeLists.txt create mode 160000 Externals/watcher/watcher diff --git a/.gitmodules b/.gitmodules index e6e29cddda..1d2e76750d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -84,6 +84,9 @@ [submodule "Externals/Vulkan-Headers"] path = Externals/Vulkan-Headers url = https://github.com/KhronosGroup/Vulkan-Headers.git +[submodule "Externals/watcher/watcher"] + path = Externals/watcher/watcher + url = https://github.com/e-dant/watcher.git [submodule "Externals/SFML/SFML"] path = Externals/SFML/SFML url = https://github.com/SFML/SFML.git diff --git a/CMakeLists.txt b/CMakeLists.txt index af18192df9..149ae7f6ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -773,6 +773,8 @@ if (USE_RETRO_ACHIEVEMENTS) add_subdirectory(Externals/rcheevos) endif() +add_subdirectory(Externals/watcher) + ######################################## # Pre-build events: Define configuration variables and write SCM info header # diff --git a/Externals/watcher/CMakeLists.txt b/Externals/watcher/CMakeLists.txt new file mode 100644 index 0000000000..097f16d647 --- /dev/null +++ b/Externals/watcher/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(watcher INTERFACE IMPORTED GLOBAL) +set_target_properties(watcher PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/watcher/include +) diff --git a/Externals/watcher/watcher b/Externals/watcher/watcher new file mode 160000 index 0000000000..0d6b9b409c --- /dev/null +++ b/Externals/watcher/watcher @@ -0,0 +1 @@ +Subproject commit 0d6b9b409ccaed6313437ea3dc8b2fc078f3d25b diff --git a/Source/VSProps/Base.Dolphin.props b/Source/VSProps/Base.Dolphin.props index 3410b9fef6..0e0235dfb2 100644 --- a/Source/VSProps/Base.Dolphin.props +++ b/Source/VSProps/Base.Dolphin.props @@ -16,6 +16,7 @@ $(ExternalsDir)rangeset\include;%(AdditionalIncludeDirectories) $(ExternalsDir)Vulkan-Headers\include;%(AdditionalIncludeDirectories) $(ExternalsDir)VulkanMemoryAllocator\include;%(AdditionalIncludeDirectories) + $(ExternalsDir)watcher\watcher\include;%(AdditionalIncludeDirectories) $(ExternalsDir)WIL\include;%(AdditionalIncludeDirectories) WIL_SUPPRESS_EXCEPTIONS;%(PreprocessorDefinitions) From 6546662edf7f107d8d042513a9a1d22796a979ec Mon Sep 17 00:00:00 2001 From: iwubcode Date: Sun, 2 Mar 2025 23:57:13 -0600 Subject: [PATCH 7/9] Common: Add class 'FilesystemWatcher' that is used to watch paths and receive callbacks about filesystem level events for anything under that path --- Source/Core/Common/CMakeLists.txt | 3 ++ Source/Core/Common/FilesystemWatcher.cpp | 62 ++++++++++++++++++++++++ Source/Core/Common/FilesystemWatcher.h | 47 ++++++++++++++++++ Source/Core/DolphinLib.props | 2 + 4 files changed, 114 insertions(+) create mode 100644 Source/Core/Common/FilesystemWatcher.cpp create mode 100644 Source/Core/Common/FilesystemWatcher.h diff --git a/Source/Core/Common/CMakeLists.txt b/Source/Core/Common/CMakeLists.txt index c368cde1d6..2687f8b520 100644 --- a/Source/Core/Common/CMakeLists.txt +++ b/Source/Core/Common/CMakeLists.txt @@ -63,6 +63,8 @@ add_library(common FatFsUtil.h FileSearch.cpp FileSearch.h + FilesystemWatcher.cpp + FilesystemWatcher.h FileUtil.cpp FileUtil.h FixedSizeQueue.h @@ -181,6 +183,7 @@ PRIVATE FatFs Iconv::Iconv spng::spng + watcher ${VTUNE_LIBRARIES} ) diff --git a/Source/Core/Common/FilesystemWatcher.cpp b/Source/Core/Common/FilesystemWatcher.cpp new file mode 100644 index 0000000000..335a1ba7f5 --- /dev/null +++ b/Source/Core/Common/FilesystemWatcher.cpp @@ -0,0 +1,62 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "Common/FilesystemWatcher.h" + +#include + +#include "Common/Logging/Log.h" +#include "Common/StringUtil.h" + +namespace Common +{ +FilesystemWatcher::FilesystemWatcher() = default; +FilesystemWatcher::~FilesystemWatcher() = default; + +void FilesystemWatcher::Watch(const std::string& path) +{ + const auto [iter, inserted] = m_watched_paths.try_emplace(path, nullptr); + if (inserted) + { + iter->second = std::make_unique(path, [this](wtr::event e) { + if (e.path_type == wtr::event::path_type::watcher) + { + return; + } + + if (e.effect_type == wtr::event::effect_type::create) + { + const auto path = WithUnifiedPathSeparators(PathToString(e.path_name)); + PathAdded(path); + } + else if (e.effect_type == wtr::event::effect_type::modify) + { + const auto path = WithUnifiedPathSeparators(PathToString(e.path_name)); + PathModified(path); + } + else if (e.effect_type == wtr::event::effect_type::rename) + { + if (!e.associated) + { + WARN_LOG_FMT(COMMON, "Rename on path seen without association!"); + return; + } + + const auto old_path = WithUnifiedPathSeparators(PathToString(e.path_name)); + const auto new_path = WithUnifiedPathSeparators(PathToString(e.associated->path_name)); + PathRenamed(old_path, new_path); + } + else if (e.effect_type == wtr::event::effect_type::destroy) + { + const auto path = WithUnifiedPathSeparators(PathToString(e.path_name)); + PathDeleted(path); + } + }); + } +} + +void FilesystemWatcher::Unwatch(const std::string& path) +{ + m_watched_paths.erase(path); +} +} // namespace Common diff --git a/Source/Core/Common/FilesystemWatcher.h b/Source/Core/Common/FilesystemWatcher.h new file mode 100644 index 0000000000..ad1c822e2b --- /dev/null +++ b/Source/Core/Common/FilesystemWatcher.h @@ -0,0 +1,47 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include + +namespace wtr +{ +inline namespace watcher +{ +class watch; +} +} // namespace wtr + +namespace Common +{ +// A class that can watch a path and receive callbacks +// when files or directories underneath that path receive events +class FilesystemWatcher +{ +public: + FilesystemWatcher(); + virtual ~FilesystemWatcher(); + + void Watch(const std::string& path); + void Unwatch(const std::string& path); + +private: + // A new file or folder was added to one of the watched paths + virtual void PathAdded(std::string_view path) {} + + // A file or folder was modified in one of the watched paths + virtual void PathModified(std::string_view path) {} + + // A file or folder was renamed in one of the watched paths + virtual void PathRenamed(std::string_view old_path, std::string_view new_path) {} + + // A file or folder was deleted in one of the watched paths + virtual void PathDeleted(std::string_view path) {} + + std::map> m_watched_paths; +}; +} // namespace Common diff --git a/Source/Core/DolphinLib.props b/Source/Core/DolphinLib.props index 8a50657c9b..627b484262 100644 --- a/Source/Core/DolphinLib.props +++ b/Source/Core/DolphinLib.props @@ -55,6 +55,7 @@ + @@ -804,6 +805,7 @@ + From a9f5e70aa17af7038088ba743a3848e339fe1887 Mon Sep 17 00:00:00 2001 From: iwubcode Date: Thu, 4 Jul 2024 17:39:21 -0500 Subject: [PATCH 8/9] VideoCommon: watch texture pack folder for texture reloads (from dynamic input textures) --- Source/Core/DolphinLib.props | 1 + .../Assets/DirectFilesystemAssetLibrary.cpp | 42 +++++++++++++++++-- .../Assets/DirectFilesystemAssetLibrary.h | 11 +++-- .../Assets/WatchableFilesystemAssetLibrary.h | 14 +++++++ Source/Core/VideoCommon/CMakeLists.txt | 1 + Source/Core/VideoCommon/HiresTextures.cpp | 3 ++ 6 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h diff --git a/Source/Core/DolphinLib.props b/Source/Core/DolphinLib.props index 627b484262..03f0bd7a6a 100644 --- a/Source/Core/DolphinLib.props +++ b/Source/Core/DolphinLib.props @@ -669,6 +669,7 @@ + diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp index 90d77d8ec9..ccc3c3e4f7 100644 --- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp +++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp @@ -13,6 +13,8 @@ #include "Common/JsonUtil.h" #include "Common/Logging/Log.h" #include "Common/StringUtil.h" +#include "Core/System.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/Assets/MaterialAsset.h" #include "VideoCommon/Assets/MeshAsset.h" #include "VideoCommon/Assets/ShaderAsset.h" @@ -53,7 +55,7 @@ std::size_t GetAssetSize(const CustomTextureData& data) CustomAssetLibrary::TimeType DirectFilesystemAssetLibrary::GetLastAssetWriteTime(const AssetID& asset_id) const { - std::lock_guard lk(m_lock); + std::lock_guard lk(m_asset_map_lock); if (auto iter = m_assetid_to_asset_map_path.find(asset_id); iter != m_assetid_to_asset_map_path.end()) { @@ -436,8 +438,40 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const Ass void DirectFilesystemAssetLibrary::SetAssetIDMapData(const AssetID& asset_id, AssetMap asset_path_map) { - std::lock_guard lk(m_lock); - m_assetid_to_asset_map_path[asset_id] = std::move(asset_path_map); + AssetMap previous_asset_map; + { + std::lock_guard lk(m_asset_map_lock); + previous_asset_map = m_assetid_to_asset_map_path[asset_id]; + } + + { + std::lock_guard lk(m_path_map_lock); + for (const auto& [name, path] : previous_asset_map) + { + m_path_to_asset_id.erase(PathToString(path)); + } + + for (const auto& [name, path] : asset_path_map) + { + m_path_to_asset_id[PathToString(path)] = asset_id; + } + } + + { + std::lock_guard lk(m_asset_map_lock); + m_assetid_to_asset_map_path[asset_id] = std::move(asset_path_map); + } +} + +void DirectFilesystemAssetLibrary::PathModified(std::string_view path) +{ + std::lock_guard lk(m_path_map_lock); + if (const auto iter = m_path_to_asset_id.find(path); iter != m_path_to_asset_id.end()) + { + auto& system = Core::System::GetInstance(); + auto& resource_manager = system.GetCustomResourceManager(); + resource_manager.ReloadAsset(iter->second); + } } bool DirectFilesystemAssetLibrary::LoadMips(const std::filesystem::path& asset_path, @@ -495,7 +529,7 @@ bool DirectFilesystemAssetLibrary::LoadMips(const std::filesystem::path& asset_p DirectFilesystemAssetLibrary::AssetMap DirectFilesystemAssetLibrary::GetAssetMapForID(const AssetID& asset_id) const { - std::lock_guard lk(m_lock); + std::lock_guard lk(m_asset_map_lock); if (auto iter = m_assetid_to_asset_map_path.find(asset_id); iter != m_assetid_to_asset_map_path.end()) { diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h index c4d99baf82..c63ca77ad3 100644 --- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h +++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h @@ -8,14 +8,14 @@ #include #include -#include "VideoCommon/Assets/CustomAssetLibrary.h" #include "VideoCommon/Assets/CustomTextureData.h" +#include "VideoCommon/Assets/WatchableFilesystemAssetLibrary.h" namespace VideoCommon { // This class implements 'CustomAssetLibrary' and loads any assets // directly from the filesystem -class DirectFilesystemAssetLibrary final : public CustomAssetLibrary +class DirectFilesystemAssetLibrary final : public WatchableFilesystemAssetLibrary { public: using AssetMap = std::map; @@ -34,13 +34,18 @@ public: void SetAssetIDMapData(const AssetID& asset_id, AssetMap asset_path_map); private: + void PathModified(std::string_view path) override; + // Loads additional mip levels into the texture structure until _mip texture is not found bool LoadMips(const std::filesystem::path& asset_path, CustomTextureData::ArraySlice* data); // Gets the asset map given an asset id AssetMap GetAssetMapForID(const AssetID& asset_id) const; - mutable std::mutex m_lock; + mutable std::mutex m_asset_map_lock; std::map> m_assetid_to_asset_map_path; + + mutable std::mutex m_path_map_lock; + std::map> m_path_to_asset_id; }; } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h b/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h new file mode 100644 index 0000000000..196d311397 --- /dev/null +++ b/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h @@ -0,0 +1,14 @@ +// Copyright 2024 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include "Common/FilesystemWatcher.h" +#include "VideoCommon/Assets/CustomAssetLibrary.h" + +namespace VideoCommon +{ +class WatchableFilesystemAssetLibrary : public CustomAssetLibrary, public Common::FilesystemWatcher +{ +}; +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/CMakeLists.txt b/Source/Core/VideoCommon/CMakeLists.txt index 78dc115db7..3ed6882bf8 100644 --- a/Source/Core/VideoCommon/CMakeLists.txt +++ b/Source/Core/VideoCommon/CMakeLists.txt @@ -30,6 +30,7 @@ add_library(videocommon Assets/ShaderAsset.h Assets/TextureAsset.cpp Assets/TextureAsset.h + Assets/WatchableFilesystemAssetLibrary.h AsyncRequests.cpp AsyncRequests.h AsyncShaderCompiler.cpp diff --git a/Source/Core/VideoCommon/HiresTextures.cpp b/Source/Core/VideoCommon/HiresTextures.cpp index 06f354e73a..9864e28544 100644 --- a/Source/Core/VideoCommon/HiresTextures.cpp +++ b/Source/Core/VideoCommon/HiresTextures.cpp @@ -97,6 +97,9 @@ void HiresTexture::Update() for (const auto& texture_directory : texture_directories) { + // Watch this directory for any texture reloads + s_file_library->Watch(texture_directory); + const auto texture_paths = Common::DoFileSearch({texture_directory}, extensions, /*recursive*/ true); From d0d0fa18d9e42a87db99780fe691025a96986b66 Mon Sep 17 00:00:00 2001 From: iwubcode Date: Thu, 4 Jul 2024 17:41:49 -0500 Subject: [PATCH 9/9] VideoCommon: move AssetMap to a types header file, so it can be pulled in without the DirectFilesystemAssetLibrary dependencies, the header will be expanded later --- Source/Core/DolphinLib.props | 1 + .../Assets/DirectFilesystemAssetLibrary.cpp | 6 +++--- .../Assets/DirectFilesystemAssetLibrary.h | 9 ++++----- Source/Core/VideoCommon/Assets/Types.h | 13 +++++++++++++ Source/Core/VideoCommon/CMakeLists.txt | 1 + .../GraphicsModSystem/Config/GraphicsModAsset.h | 5 +++-- 6 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 Source/Core/VideoCommon/Assets/Types.h diff --git a/Source/Core/DolphinLib.props b/Source/Core/DolphinLib.props index 03f0bd7a6a..832af9d10d 100644 --- a/Source/Core/DolphinLib.props +++ b/Source/Core/DolphinLib.props @@ -669,6 +669,7 @@ + diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp index ccc3c3e4f7..68ac16a88a 100644 --- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp +++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp @@ -436,9 +436,9 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const Ass } void DirectFilesystemAssetLibrary::SetAssetIDMapData(const AssetID& asset_id, - AssetMap asset_path_map) + VideoCommon::Assets::AssetMap asset_path_map) { - AssetMap previous_asset_map; + VideoCommon::Assets::AssetMap previous_asset_map; { std::lock_guard lk(m_asset_map_lock); previous_asset_map = m_assetid_to_asset_map_path[asset_id]; @@ -526,7 +526,7 @@ bool DirectFilesystemAssetLibrary::LoadMips(const std::filesystem::path& asset_p return true; } -DirectFilesystemAssetLibrary::AssetMap +VideoCommon::Assets::AssetMap DirectFilesystemAssetLibrary::GetAssetMapForID(const AssetID& asset_id) const { std::lock_guard lk(m_asset_map_lock); diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h index c63ca77ad3..cbc72a0e9a 100644 --- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h +++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h @@ -9,6 +9,7 @@ #include #include "VideoCommon/Assets/CustomTextureData.h" +#include "VideoCommon/Assets/Types.h" #include "VideoCommon/Assets/WatchableFilesystemAssetLibrary.h" namespace VideoCommon @@ -18,8 +19,6 @@ namespace VideoCommon class DirectFilesystemAssetLibrary final : public WatchableFilesystemAssetLibrary { public: - using AssetMap = std::map; - LoadInfo LoadTexture(const AssetID& asset_id, TextureData* data) override; LoadInfo LoadPixelShader(const AssetID& asset_id, PixelShaderData* data) override; LoadInfo LoadMaterial(const AssetID& asset_id, MaterialData* data) override; @@ -31,7 +30,7 @@ public: // Assigns the asset id to a map of files, how this map is read is dependent on the data // For instance, a raw texture would expect the map to have a single entry and load that // file as the asset. But a model file data might have its data spread across multiple files - void SetAssetIDMapData(const AssetID& asset_id, AssetMap asset_path_map); + void SetAssetIDMapData(const AssetID& asset_id, Assets::AssetMap asset_path_map); private: void PathModified(std::string_view path) override; @@ -40,10 +39,10 @@ private: bool LoadMips(const std::filesystem::path& asset_path, CustomTextureData::ArraySlice* data); // Gets the asset map given an asset id - AssetMap GetAssetMapForID(const AssetID& asset_id) const; + Assets::AssetMap GetAssetMapForID(const AssetID& asset_id) const; mutable std::mutex m_asset_map_lock; - std::map> m_assetid_to_asset_map_path; + std::map m_assetid_to_asset_map_path; mutable std::mutex m_path_map_lock; std::map> m_path_to_asset_id; diff --git a/Source/Core/VideoCommon/Assets/Types.h b/Source/Core/VideoCommon/Assets/Types.h new file mode 100644 index 0000000000..c10d45587c --- /dev/null +++ b/Source/Core/VideoCommon/Assets/Types.h @@ -0,0 +1,13 @@ +// Copyright 2024 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include + +namespace VideoCommon::Assets +{ +using AssetMap = std::map; +} diff --git a/Source/Core/VideoCommon/CMakeLists.txt b/Source/Core/VideoCommon/CMakeLists.txt index 3ed6882bf8..bf7e3c70b3 100644 --- a/Source/Core/VideoCommon/CMakeLists.txt +++ b/Source/Core/VideoCommon/CMakeLists.txt @@ -30,6 +30,7 @@ add_library(videocommon Assets/ShaderAsset.h Assets/TextureAsset.cpp Assets/TextureAsset.h + Assets/Types.h Assets/WatchableFilesystemAssetLibrary.h AsyncRequests.cpp AsyncRequests.h diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h b/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h index 53d8d71892..d623a00e2e 100644 --- a/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h +++ b/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h @@ -7,12 +7,13 @@ #include -#include "VideoCommon/Assets/DirectFilesystemAssetLibrary.h" +#include "VideoCommon/Assets/CustomAssetLibrary.h" +#include "VideoCommon/Assets/Types.h" struct GraphicsModAssetConfig { VideoCommon::CustomAssetLibrary::AssetID m_asset_id; - VideoCommon::DirectFilesystemAssetLibrary::AssetMap m_map; + VideoCommon::Assets::AssetMap m_map; void SerializeToConfig(picojson::object& json_obj) const; bool DeserializeFromConfig(const picojson::object& obj);