From f9f31c57dcea9e3173d25e0c85e028891ad57fca Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sun, 9 Feb 2020 02:29:06 -0500 Subject: added support for FilterWidget regexes --- src/settings.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'src/settings.cpp') diff --git a/src/settings.cpp b/src/settings.cpp index 4431e7d9..761cd669 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -1992,6 +1992,24 @@ void InterfaceSettings::setDoubleClicksOpenPreviews(bool b) set(m_Settings, "Settings", "double_click_previews", b); } +FilterWidget::Options InterfaceSettings::filterOptions() const +{ + FilterWidget::Options o; + + o.useRegex = get(m_Settings, "Settings", "filter_regex", false); + o.regexCaseSensitive = get(m_Settings, "Settings", "regex_case_sensitive", false); + o.regexExtended = get(m_Settings, "Settings", "regex_extended", false); + + return o; +} + +void InterfaceSettings::setFilterOptions(const FilterWidget::Options& o) +{ + set(m_Settings, "Settings", "filter_regex", o.useRegex); + set(m_Settings, "Settings", "regex_case_sensitive", o.regexCaseSensitive); + set(m_Settings, "Settings", "regex_extended", o.regexExtended); +} + DiagnosticsSettings::DiagnosticsSettings(QSettings& settings) : m_Settings(settings) -- cgit v1.3.1 From 3423b0a59337cf4cf99a24a1421ea33c4c641a22 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 10 Feb 2020 14:53:04 -0500 Subject: threaded refresher --- src/directoryrefresher.cpp | 188 ++++++++++++++++++++++++++++++++---------- src/directoryrefresher.h | 42 ++++++---- src/envfs.cpp | 130 +++++++++++++++++------------ src/envfs.h | 88 ++++++++++++++++++++ src/organizercore.cpp | 23 +++--- src/settings.cpp | 10 +++ src/settings.h | 5 ++ src/shared/directoryentry.cpp | 27 ++++++ src/shared/directoryentry.h | 8 ++ 9 files changed, 395 insertions(+), 126 deletions(-) (limited to 'src/settings.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index b7dd51ba..5bff9d37 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -24,6 +24,7 @@ along with Mod Organizer. If not, see . #include "report.h" #include "modinfo.h" #include "settings.h" +#include "envfs.h" #include #include @@ -36,8 +37,8 @@ using namespace MOBase; using namespace MOShared; -DirectoryRefresher::DirectoryRefresher() - : m_DirectoryStructure(nullptr) +DirectoryRefresher::DirectoryRefresher(std::size_t threadCount) + : m_DirectoryStructure(nullptr), m_threadCount(threadCount) { } @@ -116,59 +117,160 @@ void DirectoryRefresher::addModBSAToStructure(DirectoryEntry *directoryStructure } } -void DirectoryRefresher::addModFilesToStructure(DirectoryEntry *directoryStructure, const QString &modName, - int priority, const QString &directory, const QStringList &stealFiles) +void DirectoryRefresher::stealModFilesIntoStructure( + DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles) { std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); - if (stealFiles.length() > 0) { - // instead of adding all the files of the target directory, we just change the root of the specified - // files to this mod - FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); - for (const QString &filename : stealFiles) { - if (filename.isEmpty()) { - log::warn("Trying to find file with no name"); - continue; - } - QFileInfo fileInfo(filename); - FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); - if (file.get() != nullptr) { - if (file->getOrigin() == 0) { - // replace data as the origin on this bsa - file->removeOrigin(0); - } - origin.addFile(file->getIndex()); - file->addOrigin(origin.getID(), file->getFileTime(), L"", -1); - } else { - QString warnStr = fileInfo.absolutePath(); - if (warnStr.isEmpty()) - warnStr = filename; - log::warn("file not found: {}", warnStr); + // instead of adding all the files of the target directory, we just change the root of the specified + // files to this mod + FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); + for (const QString &filename : stealFiles) { + if (filename.isEmpty()) { + log::warn("Trying to find file with no name"); + continue; + } + QFileInfo fileInfo(filename); + FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); + if (file.get() != nullptr) { + if (file->getOrigin() == 0) { + // replace data as the origin on this bsa + file->removeOrigin(0); } + origin.addFile(file->getIndex()); + file->addOrigin(origin.getID(), file->getFileTime(), L"", -1); + } else { + QString warnStr = fileInfo.absolutePath(); + if (warnStr.isEmpty()) + warnStr = filename; + log::warn("file not found: {}", warnStr); } + } +} + +void DirectoryRefresher::addModFilesToStructure( + DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles) +{ + TimeThis tt("addModFilesToStructure()"); + + std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + + if (stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, modName, priority, directory, stealFiles); } else { directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); } } void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure - , const QString &modName - , int priority - , const QString &directory - , const QStringList &stealFiles - , const QStringList &archives) + , const QString &modName + , int priority + , const QString &directory + , const QStringList &stealFiles + , const QStringList &archives) { - addModFilesToStructure(directoryStructure, modName, priority, directory, stealFiles); + TimeThis tt("addModToStructure()"); + + if (stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, modName, priority, directory, stealFiles); + } else { + std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); + } if (Settings::instance().archiveParsing()) { addModBSAToStructure(directoryStructure, modName, priority, directory, archives); } } +struct ModThread +{ + std::wstring path; + env::Directory* dir = nullptr; + std::condition_variable cv; + std::mutex mutex; + bool ready = false; + + void wakeup() + { + ready = true; + cv.notify_one(); + } + + void run() + { + std::unique_lock lock(mutex); + cv.wait(lock, [&]{ return ready; }); + + *dir = env::getFilesAndDirs(path); + + ready = false; + } +}; + +void DirectoryRefresher::addMultipleModsFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, + const std::vector& entries, bool emitProgress) +{ + TimeThis tt(QString("add %1 mods").arg(entries.size())); + + env::ThreadPool threads(m_threadCount); + std::vector dirs(entries.size()); + + for (std::size_t i=0; i(i + 1); + + try { + if (e.stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); + } else { + auto& mt = threads.request(); + + mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); + mt.dir = &dirs[i]; + + mt.wakeup(); + } + } catch (const std::exception& ex) { + emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); + } + + if (emitProgress) { + emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + } + } + + threads.join(); + + for (std::size_t i=0; i(i + 1); + + directoryStructure->addFromList( + entries[i].modName.toStdWString(), + entries[i].absolutePath.toStdWString(), + dirs[i], + prio); + + if (Settings::instance().archiveParsing()) { + addModBSAToStructure( + directoryStructure, + entries[i].modName, + prio, + entries[i].absolutePath, + entries[i].archives); + } + } +} + void DirectoryRefresher::refresh() { SetThisThreadName("DirectoryRefresher"); - TimeThis tt("DirectoryRefresher::refresh()"); QMutexLocker locker(&m_RefreshLock); @@ -178,20 +280,16 @@ void DirectoryRefresher::refresh() IPluginGame *game = qApp->property("managed_game").value(); - std::wstring dataDirectory = QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + std::wstring dataDirectory = + QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0); - std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs){return lhs.priority < rhs.priority;}); - auto iter = m_Mods.begin(); + std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { + return lhs.priority < rhs.priority; + }); - for (int i = 1; iter != m_Mods.end(); ++iter, ++i) { - try { - addModToStructure(m_DirectoryStructure, iter->modName, i, iter->absolutePath, iter->stealFiles, iter->archives); - } catch (const std::exception &e) { - emit error(tr("failed to read mod (%1): %2").arg(iter->modName, e.what())); - } - emit progress((i * 100) / static_cast(m_Mods.size()) + 1); - } + addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); m_DirectoryStructure->getFileRegister()->sortOrigins(); diff --git a/src/directoryrefresher.h b/src/directoryrefresher.h index a4fc5dbc..c531ba39 100644 --- a/src/directoryrefresher.h +++ b/src/directoryrefresher.h @@ -39,12 +39,23 @@ class DirectoryRefresher : public QObject Q_OBJECT public: + struct EntryInfo { + EntryInfo(const QString &modName, const QString &absolutePath, + const QStringList &stealFiles, const QStringList &archives, int priority) + : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles) + , archives(archives), priority(priority) {} + QString modName; + QString absolutePath; + QStringList stealFiles; + QStringList archives; + int priority; + }; /** * @brief constructor * **/ - DirectoryRefresher(); + DirectoryRefresher(std::size_t threadCount); ~DirectoryRefresher(); @@ -53,7 +64,7 @@ public: * * returns a pointer to the updated directory structure. DirectoryRefresher * deletes its own pointer and the caller takes custody of the pointer - * + * * @return updated directory structure **/ MOShared::DirectoryEntry *getDirectoryStructure(); @@ -107,7 +118,13 @@ public: * @param directory * @param stealFiles */ - void addModFilesToStructure(MOShared::DirectoryEntry *directoryStructure, const QString &modName, int priority, const QString &directory, const QStringList &stealFiles); + void addModFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles); + + void addMultipleModsFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, + const std::vector& entries, bool emitProgress=false); public slots: @@ -123,26 +140,15 @@ signals: void refreshed(); private: - - struct EntryInfo { - EntryInfo(const QString &modName, const QString &absolutePath, - const QStringList &stealFiles, const QStringList &archives, int priority) - : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles) - , archives(archives), priority(priority) {} - QString modName; - QString absolutePath; - QStringList stealFiles; - QStringList archives; - int priority; - }; - -private: - std::vector m_Mods; std::set m_EnabledArchives; MOShared::DirectoryEntry *m_DirectoryStructure; QMutex m_RefreshLock; + std::size_t m_threadCount; + void stealModFilesIntoStructure( + MOShared::DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles); }; #endif // DIRECTORYREFRESHER_H diff --git a/src/envfs.cpp b/src/envfs.cpp index 81a0d3ef..2716737f 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -150,44 +150,62 @@ QString toString(POBJECT_ATTRIBUTES poa) } -constexpr std::size_t AllocSize = 1024 * 1024; -std::vector> g_buffers; - - -struct HandleCloserThread +class HandleCloserThread { - std::vector handles; - std::thread thread; - std::atomic busy; - +public: HandleCloserThread() - : busy(false) + : m_ready(false) { + m_handles.reserve(50'000); } - ~HandleCloserThread() + void add(HANDLE h) { - if (thread.joinable()) { - thread.join(); - } + m_handles.push_back(h); + } + + void wakeup() + { + m_ready = true; + m_cv.notify_one(); + } + + void run() + { + std::unique_lock lock(m_mutex); + m_cv.wait(lock, [&]{ return m_ready; }); + + closeHandles(); } +private: + std::vector m_handles; + std::condition_variable m_cv; + std::mutex m_mutex; + bool m_ready; + void closeHandles() { - for (auto& h : handles) { + for (auto& h : m_handles) { NtClose(h); } - handles.clear(); - busy = false; + m_handles.clear(); + m_ready = false; } }; -std::array g_handleCloserThreads; +constexpr std::size_t AllocSize = 1024 * 1024; +static ThreadPool g_handleClosers; +void setHandleCloserThreadCount(std::size_t n) +{ + g_handleClosers.setMax(n); +} void forEachEntryImpl( - void* cx, HandleCloserThread& hc, POBJECT_ATTRIBUTES poa, std::size_t depth, + void* cx, HandleCloserThread& hc, std::vector>& buffers, + POBJECT_ATTRIBUTES poa, std::size_t depth, DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) { IO_STATUS_BLOCK iosb; @@ -207,14 +225,14 @@ void forEachEntryImpl( return; } - hc.handles.push_back(oa.RootDirectory); + hc.add(oa.RootDirectory); unsigned char* buffer; - if (depth >= g_buffers.size()) { - g_buffers.emplace_back(std::make_unique(AllocSize)); - buffer = g_buffers.back().get(); + if (depth >= buffers.size()) { + buffers.emplace_back(std::make_unique(AllocSize)); + buffer = buffers.back().get(); } else { - buffer = g_buffers[depth].get(); + buffer = buffers[depth].get(); } union @@ -268,7 +286,7 @@ void forEachEntryImpl( if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { dirStartF(cx, toStringView(&oa)); - forEachEntryImpl(cx, hc, &oa, depth+1, dirStartF, dirEndF, fileF); + forEachEntryImpl(cx, hc, buffers, &oa, depth+1, dirStartF, dirEndF, fileF); dirEndF(cx, toStringView(&oa)); } else { //log::debug("{}{}", std::wstring((depth + 1) * 2, L' '), toString(&oa)); @@ -293,35 +311,13 @@ void forEachEntryImpl( } } -std::size_t findHandleCloserThread() -{ - for (;;) { - for (std::size_t i=0; i> buffers; if (!NtOpenFile) { HMODULE m = ::LoadLibraryW(L"ntdll.dll"); @@ -342,9 +338,41 @@ void forEachEntry( oa.Length = sizeof(oa); oa.ObjectName = &ObjectName; - forEachEntryImpl(cx, hc, &oa, 0, dirStartF, dirEndF, fileF); + forEachEntryImpl(cx, hc, buffers, &oa, 0, dirStartF, dirEndF, fileF); + hc.wakeup(); +} + +Directory getFilesAndDirs(const std::wstring& path) +{ + struct Context + { + std::stack current; + }; + + Directory root; + + Context cx; + cx.current.push(&root); + + env::forEachEntry(path, &cx, + [](void* pcx, std::wstring_view path) { + Context* cx = (Context*)pcx; + cx->current.top()->dirs.push_back({std::wstring(path.begin(), path.end())}); + cx->current.push(&cx->current.top()->dirs.back()); + }, + + [](void* pcx, std::wstring_view path) { + Context* cx = (Context*)pcx; + cx->current.pop(); + }, + + [](void* pcx, std::wstring_view path, FILETIME ft) { + Context* cx = (Context*)pcx; + cx->current.top()->files.push_back({std::wstring(path.begin(), path.end()), ft}); + } + ); - hc.thread = std::thread([hci]{ g_handleCloserThreads[hci].closeHandles(); }); + return root; } } // namespace diff --git a/src/envfs.h b/src/envfs.h index aeaa6796..1fc53bcf 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -1,17 +1,105 @@ #ifndef ENV_ENVFS_H #define ENV_ENVFS_H +#include + namespace env { +struct File +{ + std::wstring name; + FILETIME ft; +}; + +struct Directory +{ + std::wstring name; + std::list dirs; + std::list files; +}; + + +template +class ThreadPool +{ +public: + ThreadPool(std::size_t max=1) + : m_threads(max) + { + } + + ~ThreadPool() + { + join(); + } + + void setMax(std::size_t n) + { + m_threads.resize(n); + } + + void join() + { + for (auto& ti : m_threads) { + if (ti.thread.joinable()) { + ti.thread.join(); + } + } + } + + T& request() + { + if (m_threads.empty()) { + std::terminate(); + } + + for (;;) { + for (auto& ti : m_threads) { + bool expected = false; + + if (ti.busy.compare_exchange_strong(expected, true)) { + if (ti.thread.joinable()) { + ti.thread.join(); + } + + ti.thread = std::thread([&]{ + ti.o.run(); + ti.busy = false; + }); + + return ti.o; + } + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + +private: + struct ThreadInfo + { + std::thread thread; + std::atomic busy; + T o; + }; + + std::list m_threads; +}; + + using DirStartF = void (void*, std::wstring_view); using DirEndF = void (void*, std::wstring_view); using FileF = void (void*, std::wstring_view, FILETIME); +void setHandleCloserThreadCount(std::size_t n); + void forEachEntry( const std::wstring& path, void* cx, DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF); +Directory getFilesAndDirs(const std::wstring& path); + } // namespace #endif // ENV_ENVFS_H diff --git a/src/organizercore.cpp b/src/organizercore.cpp index 8124be1d..86abeb35 100644 --- a/src/organizercore.cpp +++ b/src/organizercore.cpp @@ -88,24 +88,19 @@ QStringList toStringList(InputIterator current, InputIterator end) OrganizerCore::OrganizerCore(Settings &settings) : m_UserInterface(nullptr) , m_PluginContainer(nullptr) - , m_GameName() , m_CurrentProfile(nullptr) , m_Settings(settings) , m_Updater(NexusInterface::instance(m_PluginContainer)) - , m_AboutToRun() - , m_FinishedRun() - , m_ModInstalled() , m_ModList(m_PluginContainer, this) , m_PluginList(this) - , m_DirectoryRefresher() + , m_DirectoryRefresher(settings.refreshThreadCount()) , m_DirectoryStructure(new DirectoryEntry(L"data", nullptr, 0)) , m_DownloadManager(NexusInterface::instance(m_PluginContainer), this) - , m_InstallationManager() - , m_RefresherThread() , m_DirectoryUpdate(false) , m_ArchivesInit(false) , m_PluginListsWriter(std::bind(&OrganizerCore::savePluginList, this)) { + env::setHandleCloserThreadCount(settings.refreshThreadCount()); m_DownloadManager.setOutputDirectory(m_Settings.paths().downloads(), false); NexusInterface::instance(m_PluginContainer)->setCacheDirectory( @@ -1245,13 +1240,17 @@ void OrganizerCore::updateModInDirectoryStructure(unsigned int index, void OrganizerCore::updateModsInDirectoryStructure(QMap modInfo) { + std::vector entries; + for (auto idx : modInfo.keys()) { - // add files of the bsa to the directory structure - m_DirectoryRefresher.addModFilesToStructure( - m_DirectoryStructure, modInfo[idx]->name(), - m_CurrentProfile->getModPriority(idx), modInfo[idx]->absolutePath(), - modInfo[idx]->stealFiles()); + entries.push_back({ + modInfo[idx]->name(), modInfo[idx]->absolutePath(), + modInfo[idx]->stealFiles(), {}, m_CurrentProfile->getModPriority(idx)}); } + + m_DirectoryRefresher.addMultipleModsFilesToStructure( + m_DirectoryStructure, entries); + DirectoryRefresher::cleanStructure(m_DirectoryStructure); // need to refresh plugin list now so we can activate esps refreshESPList(true); diff --git a/src/settings.cpp b/src/settings.cpp index 761cd669..f0496fe8 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -209,6 +209,16 @@ void Settings::setUseSplash(bool b) set(m_Settings, "Settings", "use_splash", b); } +std::size_t Settings::refreshThreadCount() const +{ + return get(m_Settings, "Settings", "refresh_thread_count", 10); +} + +void Settings::setRefreshThreadCount(std::size_t n) const +{ + return set(m_Settings, "Settings", "refresh_thread_count", n); +} + std::optional Settings::version() const { if (auto v=getOptional(m_Settings, "General", "version")) { diff --git a/src/settings.h b/src/settings.h index 3d37bdd8..b2cb6be5 100644 --- a/src/settings.h +++ b/src/settings.h @@ -731,6 +731,11 @@ public: bool useSplash() const; void setUseSplash(bool b); + // number of threads to use when refreshing + // + std::size_t refreshThreadCount() const; + void setRefreshThreadCount(std::size_t n) const; + GameSettings& game(); const GameSettings& game() const; diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index d171be38..b7329833 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -691,6 +691,33 @@ void DirectoryEntry::addFromOrigin( m_Populated = true; } +void DirectoryEntry::addFromList( + const std::wstring &originName, const std::wstring &directory, + env::Directory& root, int priority) +{ + FilesOrigin &origin = createOrigin(originName, directory, priority); + addDir(origin, root); +} + +void DirectoryEntry::addDir(FilesOrigin& origin, env::Directory& d) +{ + for (auto& sd : d.dirs) { + auto* sdirEntry = getSubDirectory(sd.name, true, origin.getID()); + sdirEntry->addDir(origin, sd); + } + + for (auto& f : d.files) { + insert(f.name, origin, f.ft, L"", -1); + } + + std::sort( + m_SubDirectories.begin(), + m_SubDirectories.end(), + &DirCompareByName); + + m_Populated = true; +} + void DirectoryEntry::addFromBSA( const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order) diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 58e70ffa..772899d7 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -33,7 +33,9 @@ along with Mod Organizer. If not, see . #include #include #endif + #include "util.h" +#include "envfs.h" namespace MOShared { struct DirectoryEntryFileKey; } @@ -358,6 +360,10 @@ public: const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order); + void addFromList( + const std::wstring &originName, const std::wstring &directory, + env::Directory& root, int priority); + void propagateOrigin(int origin); const std::wstring &getName() const @@ -511,6 +517,8 @@ private: FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, const std::wstring &archiveName, int order); + void addDir(FilesOrigin& origin, env::Directory& d); + DirectoryEntry* getSubDirectory( std::wstring_view name, bool create, int originID = -1); -- cgit v1.3.1