From 29244f3328cdf878ac557acbbabb19a5b7190b2d Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Thu, 13 Feb 2020 20:13:37 -0500 Subject: thread-safe directory entry --- src/directoryrefresher.cpp | 113 ++++++++++++++++++++++++++---------------- src/envfs.cpp | 70 ++++++++++++++++++++++---- src/envfs.h | 10 +++- src/shared/directoryentry.cpp | 87 ++++++++++++++++++++++++-------- src/shared/directoryentry.h | 8 +++ 5 files changed, 212 insertions(+), 76 deletions(-) (limited to 'src') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index 5bff9d37..bce3f65a 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -25,6 +25,7 @@ along with Mod Organizer. If not, see . #include "modinfo.h" #include "settings.h" #include "envfs.h" +#include "modinfodialogfwd.h" #include #include @@ -125,7 +126,10 @@ void DirectoryRefresher::stealModFilesIntoStructure( // instead of adding all the files of the target directory, we just change the root of the specified // files to this mod - FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); + DirectoryStats dummy; + FilesOrigin &origin = directoryStructure->createOrigin( + ToWString(modName), directoryW, priority, dummy); + for (const QString &filename : stealFiles) { if (filename.isEmpty()) { log::warn("Trying to find file with no name"); @@ -189,8 +193,12 @@ void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure struct ModThread { + DirectoryEntry* ds = nullptr; + std::wstring modName; std::wstring path; + int prio = -1; env::Directory* dir = nullptr; + std::condition_variable cv; std::mutex mutex; bool ready = false; @@ -206,7 +214,16 @@ struct ModThread std::unique_lock lock(mutex); cv.wait(lock, [&]{ return ready; }); - *dir = env::getFilesAndDirs(path); + ds->addFromOrigin(modName, path, prio); + + /*if (Settings::instance().archiveParsing()) { + addModBSAToStructure( + directoryStructure, + entries[i].modName, + prio, + entries[i].absolutePath, + entries[i].archives); + }*/ ready = false; } @@ -216,65 +233,75 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( MOShared::DirectoryEntry *directoryStructure, const std::vector& entries, bool emitProgress) { - TimeThis tt(QString("add %1 mods").arg(entries.size())); - - env::ThreadPool threads(m_threadCount); std::vector dirs(entries.size()); - for (std::size_t i=0; i(i + 1); + { + TimeThis tt("walk dirs"); - try { - if (e.stealFiles.length() > 0) { - stealModFilesIntoStructure( - directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); - } else { - auto& mt = threads.request(); + env::ThreadPool threads(m_threadCount); - mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); - mt.dir = &dirs[i]; + for (std::size_t i=0; i(i + 1); - mt.wakeup(); + try + { + if (e.stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); + } else { + auto& mt = threads.request(); + + mt.ds = directoryStructure; + mt.modName = entries[i].modName.toStdWString(); + mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); + mt.prio = prio; + mt.dir = &dirs[i]; + + mt.wakeup(); + } + } catch (const std::exception& ex) { + emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); } - } catch (const std::exception& ex) { - emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); - } - if (emitProgress) { - emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + if (emitProgress) { + emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + } } - } - - threads.join(); - - for (std::size_t i=0; i(i + 1); - - directoryStructure->addFromList( - entries[i].modName.toStdWString(), - entries[i].absolutePath.toStdWString(), - dirs[i], - prio); - if (Settings::instance().archiveParsing()) { - addModBSAToStructure( - directoryStructure, - entries[i].modName, - prio, - entries[i].absolutePath, - entries[i].archives); - } + threads.join(); } + + //std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){ + // return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0); + //}); + + //static int run = 1; + // + //std::ofstream out("c:\\tmp\\data.csv", std::ios::app); + + //out << fmt::format("what,run,{}", DirectoryStats::csvHeader()); + // + //for (std::size_t i=0; icurrent.top()->dirs.push_back({ - std::wstring(path.begin(), path.end()), - MOShared::ToLowerCopy(path) - }); - + cx->current.top()->dirs.push_back(Directory(path)); cx->current.push(&cx->current.top()->dirs.back()); }, @@ -385,18 +381,70 @@ Directory getFilesAndDirs(const std::wstring& path) cx->current.pop(); }, - [](void* pcx, std::wstring_view path, FILETIME ft) { + [](void* pcx, std::wstring_view path, FILETIME ft) { Context* cx = (Context*)pcx; - cx->current.top()->files.push_back({ - std::wstring(path.begin(), path.end()), - MOShared::ToLowerCopy(path), - ft - }); + cx->current.top()->files.push_back(File(path, ft)); } ); return root; } +File::File(std::wstring_view n, FILETIME ft) : + name(n.begin(), n.end()), + lcname(MOShared::ToLowerCopy(name)), + lastModified(ft) +{ +} + +Directory::Directory() +{ +} + +Directory::Directory(std::wstring_view n) + : name(n.begin(), n.end()), lcname(MOShared::ToLowerCopy(name)) +{ +} + + +void getFilesAndDirsWithFindImpl(const std::wstring& path, Directory& d) +{ + const std::wstring searchString = path + L"\\*"; + + WIN32_FIND_DATAW findData; + + HANDLE searchHandle = ::FindFirstFileExW( + searchString.c_str(), FindExInfoBasic, &findData, FindExSearchNameMatch, + nullptr, FIND_FIRST_EX_LARGE_FETCH); + + if (searchHandle != INVALID_HANDLE_VALUE) { + BOOL result = true; + + while (result) { + if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + if ((wcscmp(findData.cFileName, L".") != 0) && + (wcscmp(findData.cFileName, L"..") != 0)) { + const std::wstring newPath = path + L"\\" + findData.cFileName; + d.dirs.push_back(Directory(findData.cFileName)); + getFilesAndDirsWithFindImpl(newPath, d.dirs.back()); + } + } else { + d.files.push_back(File(findData.cFileName, findData.ftLastWriteTime)); + } + + result = ::FindNextFileW(searchHandle, &findData); + } + } + + ::FindClose(searchHandle); +} + +Directory getFilesAndDirsWithFind(const std::wstring& path) +{ + Directory d; + getFilesAndDirsWithFindImpl(path, d); + return d; +} + } // namespace diff --git a/src/envfs.h b/src/envfs.h index 6e0da6d0..001f2d09 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -11,6 +11,8 @@ struct File std::wstring name; std::wstring lcname; FILETIME lastModified; + + File(std::wstring_view name, FILETIME ft); }; struct Directory @@ -18,8 +20,11 @@ struct Directory std::wstring name; std::wstring lcname; - std::list dirs; - std::list files; + std::vector dirs; + std::vector files; + + Directory(); + Directory(std::wstring_view name); }; @@ -111,6 +116,7 @@ void forEachEntry( DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF); Directory getFilesAndDirs(const std::wstring& path); +Directory getFilesAndDirsWithFind(const std::wstring& path); } // namespace diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index a90f6f08..e1464595 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -214,11 +214,14 @@ public: { int newID = createID(); - m_Origins[newID] = FilesOrigin(newID, originName, directory, priority, fileRegister, originConnection); - m_OriginsNameMap[originName] = newID; - m_OriginsPriorityMap[priority] = newID; + auto itor = m_Origins.insert({newID, FilesOrigin( + newID, originName, directory, priority, + fileRegister, originConnection)}).first; - return m_Origins[newID]; + m_OriginsNameMap.insert({originName, newID}); + m_OriginsPriorityMap.insert({priority, newID}); + + return itor->second; } bool exists(const std::wstring &name) @@ -597,9 +600,13 @@ std::vector FilesOrigin::getFiles() const { std::vector result; - for (FileEntry::Index fileIdx : m_Files) { - if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { - result.push_back(p); + { + std::scoped_lock lock(m_Mutex); + + for (FileEntry::Index fileIdx : m_Files) { + if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { + result.push_back(p); + } } } @@ -621,9 +628,16 @@ void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) { if (!enabled) { ++stats.originsNeededEnabled; - std::set copy = m_Files; + + std::set copy; + + { + std::scoped_lock lock(m_Mutex); + copy = m_Files; + m_Files.clear(); + } + m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter); - m_Files.clear(); } m_Disabled = !enabled; @@ -631,6 +645,8 @@ void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) void FilesOrigin::removeFile(FileEntry::Index index) { + std::scoped_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { @@ -640,6 +656,8 @@ void FilesOrigin::removeFile(FileEntry::Index index) bool FilesOrigin::containsArchive(std::wstring archiveName) { + std::scoped_lock lock(m_Mutex); + for (FileEntry::Index fileIdx : m_Files) { if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { if (p->isFromArchive(archiveName)) { @@ -665,6 +683,7 @@ FileRegister::~FileRegister() bool FileRegister::indexValid(FileEntry::Index index) const { + std::scoped_lock lock(m_Mutex); return (m_Files.find(index) != m_Files.end()); } @@ -675,6 +694,8 @@ FileEntry::Ptr FileRegister::createFile( FileEntry::Ptr p; stats.addFileToRegisterTimes += elapsed([&]{ + std::scoped_lock lock(m_Mutex); + auto r = m_Files.insert_or_assign( index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); @@ -692,6 +713,8 @@ FileEntry::Ptr FileRegister::createFile( FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const { + std::scoped_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { @@ -703,6 +726,8 @@ FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const bool FileRegister::removeFile(FileEntry::Index index) { + std::scoped_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { @@ -717,12 +742,15 @@ bool FileRegister::removeFile(FileEntry::Index index) void FileRegister::removeOrigin(FileEntry::Index index, int originID) { + std::unique_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { if (iter->second->removeOrigin(originID)) { - unregisterFile(iter->second); m_Files.erase(iter); + lock.unlock(); + unregisterFile(iter->second); } } else { log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index); @@ -734,17 +762,21 @@ void FileRegister::removeOriginMulti( { std::vector removedFiles; - for (auto iter = indices.begin(); iter != indices.end(); ) { - auto pos = m_Files.find(*iter); + { + std::scoped_lock lock(m_Mutex); + + for (auto iter = indices.begin(); iter != indices.end(); ) { + auto pos = m_Files.find(*iter); - if (pos != m_Files.end() - && (pos->second->lastAccessed() < notAfter) - && pos->second->removeOrigin(originID)) { - removedFiles.push_back(pos->second); - m_Files.erase(pos); - ++iter; - } else { - indices.erase(iter++); + if (pos != m_Files.end() + && (pos->second->lastAccessed() < notAfter) + && pos->second->removeOrigin(originID)) { + removedFiles.push_back(pos->second); + m_Files.erase(pos); + ++iter; + } else { + indices.erase(iter++); + } } } @@ -776,6 +808,8 @@ void FileRegister::removeOriginMulti( void FileRegister::sortOrigins() { + std::scoped_lock lock(m_Mutex); + for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) { iter->second->sortOrigins(); } @@ -1228,6 +1262,8 @@ FileEntry::Ptr DirectoryEntry::insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, std::wstring_view archive, int order) { + std::scoped_lock lock(m_FilesMutex); + std::wstring fileNameLower = ToLowerCopy(fileName); auto iter = m_Files.find(fileNameLower); @@ -1255,6 +1291,8 @@ FileEntry::Ptr DirectoryEntry::insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats) { + std::scoped_lock lock(m_FilesMutex); + FilesMap::iterator itor; stats.filesLookupTimes += elapsed([&]{ @@ -1307,7 +1345,12 @@ void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOf [](void* pcx, std::wstring_view path) { Context* cx = (Context*)pcx; auto* current= cx->current.top(); - std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); + + { + std::scoped_lock lock(current->m_SubDirMutex); + std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); + } + cx->current.pop(); }, @@ -1389,6 +1432,8 @@ void DirectoryEntry::addFiles( DirectoryEntry *DirectoryEntry::getSubDirectory( std::wstring_view name, bool create, int originID) { + std::scoped_lock lock(m_SubDirMutex); + std::wstring nameLc = ToLowerCopy(name); auto itor = m_SubDirectoriesLookup.find(nameLc); diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index c8ec8a4e..71e8b32c 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -26,6 +26,8 @@ along with Mod Organizer. If not, see . #include #include #include +#include + #define WIN32_MEAN_AND_LEAN #include #include @@ -274,6 +276,7 @@ public: void addFile(FileEntry::Index index) { + std::scoped_lock lock(m_Mutex); m_Files.insert(index); } @@ -290,6 +293,7 @@ private: int m_Priority; boost::weak_ptr m_FileRegister; boost::weak_ptr m_OriginConnection; + mutable std::mutex m_Mutex; FilesOrigin( int ID, const std::wstring &name, const std::wstring &path, int priority, @@ -313,6 +317,7 @@ public: size_t size() const { + std::scoped_lock lock(m_Mutex); return m_Files.size(); } @@ -323,6 +328,7 @@ public: void sortOrigins(); private: + mutable std::mutex m_Mutex; std::map m_Files; boost::shared_ptr m_OriginConnection; @@ -545,6 +551,8 @@ private: std::set m_Origins; bool m_Populated; bool m_TopLevel; + std::mutex m_SubDirMutex; + std::mutex m_FilesMutex; DirectoryEntry(const DirectoryEntry &reference); -- cgit v1.3.1