From 9fb639a00e1a809a9ea9b5e68777ec46f534761c Mon Sep 17 00:00:00 2001 From: Mikaël Capelle Date: Sun, 24 May 2020 13:27:40 +0200 Subject: Update ModInfo to use a single file tree for all content-related operations. --- src/modinfo.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'src/modinfo.cpp') diff --git a/src/modinfo.cpp b/src/modinfo.cpp index 379fa071..0dd113c5 100644 --- a/src/modinfo.cpp +++ b/src/modinfo.cpp @@ -306,7 +306,7 @@ void ModInfo::updateIndices() ModInfo::ModInfo(PluginContainer *pluginContainer) - : m_Valid(false), m_PrimaryCategory(-1) + : m_PrimaryCategory(-1) { } @@ -528,11 +528,6 @@ bool ModInfo::categorySet(int categoryID) const return false; } -void ModInfo::testValid() -{ - m_Valid = doTestValid(); -} - QUrl ModInfo::parseCustomURL() const { if (!hasCustomURL() || getCustomURL().isEmpty()) { -- cgit v1.3.1 From a284c64dc109797514b4664d37e6b6dc3d5c36c7 Mon Sep 17 00:00:00 2001 From: Mikaël Capelle Date: Sun, 24 May 2020 13:28:00 +0200 Subject: Update file tree in parallel when loading from disk. --- src/modinfo.cpp | 8 ++++++++ src/modinfo.h | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) (limited to 'src/modinfo.cpp') diff --git a/src/modinfo.cpp b/src/modinfo.cpp index 0dd113c5..5a90d74b 100644 --- a/src/modinfo.cpp +++ b/src/modinfo.cpp @@ -29,6 +29,7 @@ along with Mod Organizer. If not, see . #include "modinfodialog.h" #include "overwriteinfodialog.h" #include "versioninfo.h" +#include "envfs.h" #include #include @@ -45,6 +46,7 @@ using namespace MOBase; using namespace MOShared; +env::ThreadPool ModInfo::s_Threads(10); std::vector ModInfo::s_Collection; std::map ModInfo::s_ModsByName; std::map, std::vector> ModInfo::s_ModsByModID; @@ -287,6 +289,12 @@ void ModInfo::updateFromDisc(const QString &modDirectory, std::sort(s_Collection.begin(), s_Collection.end(), ModInfo::ByName); updateIndices(); + + for (auto& p : s_Collection) { + auto &t = s_Threads.request(); + t.ptr = p; + t.wakeup(); + } } diff --git a/src/modinfo.h b/src/modinfo.h index b3e75bc4..d3d13d9c 100644 --- a/src/modinfo.h +++ b/src/modinfo.h @@ -22,6 +22,7 @@ along with Mod Organizer. If not, see . #include "imodinterface.h" #include "versioninfo.h" +#include "envfs.h" class PluginContainer; class QDir; @@ -839,6 +840,36 @@ protected: private: + struct ModThread + { + ModInfo::Ptr ptr; + + std::condition_variable cv; + std::mutex mutex; + bool ready = false; + + void wakeup() + { + { + std::scoped_lock lock(mutex); + ready = true; + } + + cv.notify_one(); + } + + void run() + { + std::unique_lock lock(mutex); + cv.wait(lock, [&] { return ready; }); + + ptr->isValid(); + ready = false; + } + }; + + static env::ThreadPool s_Threads; + static QMutex s_Mutex; static std::map, std::vector > s_ModsByModID; static int s_NextID; -- cgit v1.3.1 From 73248d9e4036ad6b9494f678efbc97e919ebfbcf Mon Sep 17 00:00:00 2001 From: Mikaël Capelle Date: Sun, 24 May 2020 13:36:16 +0200 Subject: Use the refresh_thread_count setting for ModInfo::updateFromDisc. --- src/modinfo.cpp | 14 +++++++++----- src/modinfo.h | 1 + src/organizercore.cpp | 6 ++++-- 3 files changed, 14 insertions(+), 7 deletions(-) (limited to 'src/modinfo.cpp') diff --git a/src/modinfo.cpp b/src/modinfo.cpp index 5a90d74b..81b45a49 100644 --- a/src/modinfo.cpp +++ b/src/modinfo.cpp @@ -46,7 +46,7 @@ using namespace MOBase; using namespace MOShared; -env::ThreadPool ModInfo::s_Threads(10); +env::ThreadPool ModInfo::s_Threads; std::vector ModInfo::s_Collection; std::map ModInfo::s_ModsByName; std::map, std::vector> ModInfo::s_ModsByModID; @@ -251,6 +251,7 @@ void ModInfo::updateFromDisc(const QString &modDirectory, DirectoryEntry **directoryStructure, PluginContainer *pluginContainer, bool displayForeign, + std::size_t refreshThreadCount, MOBase::IPluginGame const *game) { TimeThis tt("ModInfo::updateFromDisc()"); @@ -287,14 +288,17 @@ void ModInfo::updateFromDisc(const QString &modDirectory, createFromOverwrite(pluginContainer, game, directoryStructure); std::sort(s_Collection.begin(), s_Collection.end(), ModInfo::ByName); - - updateIndices(); - + + // This force loading a part of the FileTree: + s_Threads.setMax(refreshThreadCount); for (auto& p : s_Collection) { - auto &t = s_Threads.request(); + auto& t = s_Threads.request(); t.ptr = p; t.wakeup(); } + + updateIndices(); + } diff --git a/src/modinfo.h b/src/modinfo.h index d3d13d9c..081f3660 100644 --- a/src/modinfo.h +++ b/src/modinfo.h @@ -143,6 +143,7 @@ public: MOShared::DirectoryEntry **directoryStructure, PluginContainer *pluginContainer, bool displayForeign, + std::size_t refreshThreadCount, MOBase::IPluginGame const *game); static void clear() { s_Collection.clear(); s_ModsByName.clear(); s_ModsByModID.clear(); } diff --git a/src/organizercore.cpp b/src/organizercore.cpp index 74163d8d..6e4b3fbf 100644 --- a/src/organizercore.cpp +++ b/src/organizercore.cpp @@ -208,7 +208,8 @@ void OrganizerCore::updateExecutablesList() void OrganizerCore::updateModInfoFromDisc() { ModInfo::updateFromDisc( m_Settings.paths().mods(), &m_DirectoryStructure, - m_PluginContainer, m_Settings.interface().displayForeign(), managedGame()); + m_PluginContainer, m_Settings.interface().displayForeign(), + m_Settings.refreshThreadCount(), managedGame()); } void OrganizerCore::setUserInterface(IUserInterface* ui) @@ -1097,7 +1098,8 @@ void OrganizerCore::refreshModList(bool saveChanges) ModInfo::updateFromDisc( m_Settings.paths().mods(), &m_DirectoryStructure, - m_PluginContainer, m_Settings.interface().displayForeign(), managedGame()); + m_PluginContainer, m_Settings.interface().displayForeign(), + m_Settings.refreshThreadCount(), managedGame()); m_CurrentProfile->refreshModStatus(); -- cgit v1.3.1 From ff4fbbdc5fa5ada01bd5bc3d8f8004279d7cc6b8 Mon Sep 17 00:00:00 2001 From: Mikaël Capelle Date: Sun, 24 May 2020 14:39:39 +0200 Subject: Switch from ThreadPool to a simpler thread map for containers. --- src/modinfo.cpp | 11 ++--------- src/modinfo.h | 31 ------------------------------ src/thread_utils.h | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 40 deletions(-) create mode 100644 src/thread_utils.h (limited to 'src/modinfo.cpp') diff --git a/src/modinfo.cpp b/src/modinfo.cpp index 81b45a49..eb0f0b4f 100644 --- a/src/modinfo.cpp +++ b/src/modinfo.cpp @@ -29,7 +29,7 @@ along with Mod Organizer. If not, see . #include "modinfodialog.h" #include "overwriteinfodialog.h" #include "versioninfo.h" -#include "envfs.h" +#include "thread_utils.h" #include #include @@ -46,7 +46,6 @@ using namespace MOBase; using namespace MOShared; -env::ThreadPool ModInfo::s_Threads; std::vector ModInfo::s_Collection; std::map ModInfo::s_ModsByName; std::map, std::vector> ModInfo::s_ModsByModID; @@ -289,13 +288,7 @@ void ModInfo::updateFromDisc(const QString &modDirectory, std::sort(s_Collection.begin(), s_Collection.end(), ModInfo::ByName); - // This force loading a part of the FileTree: - s_Threads.setMax(refreshThreadCount); - for (auto& p : s_Collection) { - auto& t = s_Threads.request(); - t.ptr = p; - t.wakeup(); - } + parallelMap(std::begin(s_Collection), std::end(s_Collection), &ModInfo::isValid, refreshThreadCount); updateIndices(); diff --git a/src/modinfo.h b/src/modinfo.h index 081f3660..37a2d24d 100644 --- a/src/modinfo.h +++ b/src/modinfo.h @@ -22,7 +22,6 @@ along with Mod Organizer. If not, see . #include "imodinterface.h" #include "versioninfo.h" -#include "envfs.h" class PluginContainer; class QDir; @@ -841,36 +840,6 @@ protected: private: - struct ModThread - { - ModInfo::Ptr ptr; - - std::condition_variable cv; - std::mutex mutex; - bool ready = false; - - void wakeup() - { - { - std::scoped_lock lock(mutex); - ready = true; - } - - cv.notify_one(); - } - - void run() - { - std::unique_lock lock(mutex); - cv.wait(lock, [&] { return ready; }); - - ptr->isValid(); - ready = false; - } - }; - - static env::ThreadPool s_Threads; - static QMutex s_Mutex; static std::map, std::vector > s_ModsByModID; static int s_NextID; diff --git a/src/thread_utils.h b/src/thread_utils.h new file mode 100644 index 00000000..fe096a36 --- /dev/null +++ b/src/thread_utils.h @@ -0,0 +1,55 @@ +#ifndef MO2_THREAD_UTILS_H +#define MO2_THREAD_UTILS_H + +#include +#include + +namespace MOShared { + +/** + * @brief Apply the given callable to each element between the two given iterators + * in a parallel way. + * + * The callable should be independent, or properly synchronized, and the source of + * the range should not change during this call. + * + * @param start Beginning of the range. + * @param end End of the range. + * @param callable Callable to apply to every element of the range. See std::invoke + * requirements. Must be copiable. + * @param nThreads Number of threads to use. + * + */ +template +void parallelMap(It begin, It end, Callable callable, std::size_t nThreads) { + std::vector threads(nThreads); + + std::mutex m; + for (auto &thread: threads) { + thread = std::thread([&m, &begin, end, callable]() { + while (true) { + decltype(begin) it; + { + std::scoped_lock lock(m); + if (begin == end) { + break; + } + it = begin++; + } + if (it != end) { + std::invoke(callable, *it); + } + } + }); + } + + + // Join everything: + for (auto& t : threads) { + t.join(); + } +} + +} + +#endif -- cgit v1.3.1 From 4bcb92fc47181c62af62a55377b62dcfa02252ae Mon Sep 17 00:00:00 2001 From: Mikaël Capelle Date: Sun, 24 May 2020 14:49:51 +0200 Subject: Use a more generic 'prefetch' method instead of isValid(). --- src/modinfo.cpp | 2 +- src/modinfo.h | 12 ++++++++++++ src/modinfowithconflictinfo.cpp | 5 +++++ src/modinfowithconflictinfo.h | 9 +++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) (limited to 'src/modinfo.cpp') diff --git a/src/modinfo.cpp b/src/modinfo.cpp index eb0f0b4f..15fa71da 100644 --- a/src/modinfo.cpp +++ b/src/modinfo.cpp @@ -288,7 +288,7 @@ void ModInfo::updateFromDisc(const QString &modDirectory, std::sort(s_Collection.begin(), s_Collection.end(), ModInfo::ByName); - parallelMap(std::begin(s_Collection), std::end(s_Collection), &ModInfo::isValid, refreshThreadCount); + parallelMap(std::begin(s_Collection), std::end(s_Collection), &ModInfo::prefetch, refreshThreadCount); updateIndices(); diff --git a/src/modinfo.h b/src/modinfo.h index 37a2d24d..29e6124d 100644 --- a/src/modinfo.h +++ b/src/modinfo.h @@ -815,8 +815,20 @@ signals: protected: + /** + * + */ ModInfo(PluginContainer *pluginContainer); + /** + * @brief Prefetch content for this mod. + * + * This method can be used to prefetch content from the mod, e.g., for isValid() + * or getContents(). This method will only be called when first creating the mod + * using multiple threads for all the mods. + */ + virtual void prefetch() = 0; + static void updateIndices(); static bool ByName(const ModInfo::Ptr &LHS, const ModInfo::Ptr &RHS); diff --git a/src/modinfowithconflictinfo.cpp b/src/modinfowithconflictinfo.cpp index 45ecffb7..d2202c3d 100644 --- a/src/modinfowithconflictinfo.cpp +++ b/src/modinfowithconflictinfo.cpp @@ -309,6 +309,11 @@ std::shared_ptr ModInfoWithConflictInfo::contentFileTree() cons return m_FileTree; } +void ModInfoWithConflictInfo::prefetch() { + // Populating the tree to 1-depth: + contentFileTree()->size(); +} + bool ModInfoWithConflictInfo::doTestValid() const { auto mdc = m_GamePlugin->feature(); diff --git a/src/modinfowithconflictinfo.h b/src/modinfowithconflictinfo.h index 5bc1d585..68675397 100644 --- a/src/modinfowithconflictinfo.h +++ b/src/modinfowithconflictinfo.h @@ -108,6 +108,15 @@ private: protected: + /** + * @brief Prefetch content for this mod. + * + * This method can be used to prefetch content from the mod, e.g., for isValid() + * or getContents(). This method will only be called when first creating the mod + * using multiple threads for all the mods. + */ + virtual void prefetch() override; + // Current game plugin running in MO2: MOBase::IPluginGame const * const m_GamePlugin; -- cgit v1.3.1