From 3423b0a59337cf4cf99a24a1421ea33c4c641a22 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 10 Feb 2020 14:53:04 -0500 Subject: threaded refresher --- src/directoryrefresher.cpp | 188 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 143 insertions(+), 45 deletions(-) (limited to 'src/directoryrefresher.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index b7dd51ba..5bff9d37 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -24,6 +24,7 @@ along with Mod Organizer. If not, see . #include "report.h" #include "modinfo.h" #include "settings.h" +#include "envfs.h" #include #include @@ -36,8 +37,8 @@ using namespace MOBase; using namespace MOShared; -DirectoryRefresher::DirectoryRefresher() - : m_DirectoryStructure(nullptr) +DirectoryRefresher::DirectoryRefresher(std::size_t threadCount) + : m_DirectoryStructure(nullptr), m_threadCount(threadCount) { } @@ -116,59 +117,160 @@ void DirectoryRefresher::addModBSAToStructure(DirectoryEntry *directoryStructure } } -void DirectoryRefresher::addModFilesToStructure(DirectoryEntry *directoryStructure, const QString &modName, - int priority, const QString &directory, const QStringList &stealFiles) +void DirectoryRefresher::stealModFilesIntoStructure( + DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles) { std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); - if (stealFiles.length() > 0) { - // instead of adding all the files of the target directory, we just change the root of the specified - // files to this mod - FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); - for (const QString &filename : stealFiles) { - if (filename.isEmpty()) { - log::warn("Trying to find file with no name"); - continue; - } - QFileInfo fileInfo(filename); - FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); - if (file.get() != nullptr) { - if (file->getOrigin() == 0) { - // replace data as the origin on this bsa - file->removeOrigin(0); - } - origin.addFile(file->getIndex()); - file->addOrigin(origin.getID(), file->getFileTime(), L"", -1); - } else { - QString warnStr = fileInfo.absolutePath(); - if (warnStr.isEmpty()) - warnStr = filename; - log::warn("file not found: {}", warnStr); + // instead of adding all the files of the target directory, we just change the root of the specified + // files to this mod + FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); + for (const QString &filename : stealFiles) { + if (filename.isEmpty()) { + log::warn("Trying to find file with no name"); + continue; + } + QFileInfo fileInfo(filename); + FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); + if (file.get() != nullptr) { + if (file->getOrigin() == 0) { + // replace data as the origin on this bsa + file->removeOrigin(0); } + origin.addFile(file->getIndex()); + file->addOrigin(origin.getID(), file->getFileTime(), L"", -1); + } else { + QString warnStr = fileInfo.absolutePath(); + if (warnStr.isEmpty()) + warnStr = filename; + log::warn("file not found: {}", warnStr); } + } +} + +void DirectoryRefresher::addModFilesToStructure( + DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles) +{ + TimeThis tt("addModFilesToStructure()"); + + std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + + if (stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, modName, priority, directory, stealFiles); } else { directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); } } void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure - , const QString &modName - , int priority - , const QString &directory - , const QStringList &stealFiles - , const QStringList &archives) + , const QString &modName + , int priority + , const QString &directory + , const QStringList &stealFiles + , const QStringList &archives) { - addModFilesToStructure(directoryStructure, modName, priority, directory, stealFiles); + TimeThis tt("addModToStructure()"); + + if (stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, modName, priority, directory, stealFiles); + } else { + std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); + } if (Settings::instance().archiveParsing()) { addModBSAToStructure(directoryStructure, modName, priority, directory, archives); } } +struct ModThread +{ + std::wstring path; + env::Directory* dir = nullptr; + std::condition_variable cv; + std::mutex mutex; + bool ready = false; + + void wakeup() + { + ready = true; + cv.notify_one(); + } + + void run() + { + std::unique_lock lock(mutex); + cv.wait(lock, [&]{ return ready; }); + + *dir = env::getFilesAndDirs(path); + + ready = false; + } +}; + +void DirectoryRefresher::addMultipleModsFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, + const std::vector& entries, bool emitProgress) +{ + TimeThis tt(QString("add %1 mods").arg(entries.size())); + + env::ThreadPool threads(m_threadCount); + std::vector dirs(entries.size()); + + for (std::size_t i=0; i(i + 1); + + try { + if (e.stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); + } else { + auto& mt = threads.request(); + + mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); + mt.dir = &dirs[i]; + + mt.wakeup(); + } + } catch (const std::exception& ex) { + emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); + } + + if (emitProgress) { + emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + } + } + + threads.join(); + + for (std::size_t i=0; i(i + 1); + + directoryStructure->addFromList( + entries[i].modName.toStdWString(), + entries[i].absolutePath.toStdWString(), + dirs[i], + prio); + + if (Settings::instance().archiveParsing()) { + addModBSAToStructure( + directoryStructure, + entries[i].modName, + prio, + entries[i].absolutePath, + entries[i].archives); + } + } +} + void DirectoryRefresher::refresh() { SetThisThreadName("DirectoryRefresher"); - TimeThis tt("DirectoryRefresher::refresh()"); QMutexLocker locker(&m_RefreshLock); @@ -178,20 +280,16 @@ void DirectoryRefresher::refresh() IPluginGame *game = qApp->property("managed_game").value(); - std::wstring dataDirectory = QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + std::wstring dataDirectory = + QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0); - std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs){return lhs.priority < rhs.priority;}); - auto iter = m_Mods.begin(); + std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { + return lhs.priority < rhs.priority; + }); - for (int i = 1; iter != m_Mods.end(); ++iter, ++i) { - try { - addModToStructure(m_DirectoryStructure, iter->modName, i, iter->absolutePath, iter->stealFiles, iter->archives); - } catch (const std::exception &e) { - emit error(tr("failed to read mod (%1): %2").arg(iter->modName, e.what())); - } - emit progress((i * 100) / static_cast(m_Mods.size()) + 1); - } + addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); m_DirectoryStructure->getFileRegister()->sortOrigins(); -- cgit v1.3.1