From 3a65665d36022c50637fc53d5c9c1ee022c5b3b5 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 10 Feb 2020 10:51:23 -0500 Subject: wstring_view when possible first implementation of NtQueryDirectoryFile --- src/shared/directoryentry.cpp | 76 ++++++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 22 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 6e44cc91..28201a11 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -20,6 +20,8 @@ along with Mod Organizer. If not, see . #include "directoryentry.h" #include "windows_error.h" #include "error_report.h" +#include "envfs.h" +#include #include #include #include @@ -172,15 +174,15 @@ FileEntry::FileEntry() : { } -FileEntry::FileEntry(Index index, const std::wstring &name, DirectoryEntry *parent) : - m_Index(index), m_Name(name), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), +FileEntry::FileEntry(Index index, std::wstring_view name, DirectoryEntry *parent) : + m_Index(index), m_Name(name.begin(), name.end()), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), m_LastAccessed(time(nullptr)) { } void FileEntry::addOrigin( - int origin, FILETIME fileTime, const std::wstring &archive, int order) + int origin, FILETIME fileTime, std::wstring_view archive, int order) { m_LastAccessed = time(nullptr); if (m_Parent != nullptr) { @@ -192,7 +194,7 @@ void FileEntry::addOrigin( // alternatives m_Origin = origin; m_FileTime = fileTime; - m_Archive = std::pair(archive, order); + m_Archive = std::pair(std::wstring(archive.begin(), archive.end()), order); } else if ( (m_Parent != nullptr) && ( @@ -213,7 +215,7 @@ void FileEntry::addOrigin( m_Origin = origin; m_FileTime = fileTime; - m_Archive = std::pair(archive, order); + m_Archive = std::pair(std::wstring(archive.begin(), archive.end()), order); } else { // This mod is just an alternative @@ -232,14 +234,14 @@ void FileEntry::addOrigin( if ((m_Parent != nullptr) && (m_Parent->getOriginByID(iter->first).getPriority() < m_Parent->getOriginByID(origin).getPriority())) { - m_Alternatives.insert(iter, {origin, {archive, order}}); + m_Alternatives.insert(iter, {origin, {std::wstring(archive.begin(), archive.end()), order}}); found = true; break; } } if (!found) { - m_Alternatives.push_back({origin, {archive, order}}); + m_Alternatives.push_back({origin, {std::wstring(archive.begin(), archive.end()), order}}); } } } @@ -509,12 +511,12 @@ bool FileRegister::indexValid(FileEntry::Index index) const return (m_Files.find(index) != m_Files.end()); } -FileEntry::Ptr FileRegister::createFile(const std::wstring &name, DirectoryEntry *parent) +FileEntry::Ptr FileRegister::createFile(std::wstring_view name, DirectoryEntry *parent) { FileEntry::Index index = generateIndex(); auto r = m_Files.insert_or_assign( - index, FileEntry::Ptr(new FileEntry(index, name, parent))); + index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); return r.first->second; } @@ -637,20 +639,20 @@ void FileRegister::unregisterFile(FileEntry::Ptr file) DirectoryEntry::DirectoryEntry( - const std::wstring &name, DirectoryEntry *parent, int originID) : + std::wstring name, DirectoryEntry *parent, int originID) : m_OriginConnection(new OriginConnection), - m_Name(name), m_Parent(parent), m_Populated(false), m_TopLevel(true) + m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(true) { m_FileRegister.reset(new FileRegister(m_OriginConnection)); m_Origins.insert(originID); } DirectoryEntry::DirectoryEntry( - const std::wstring &name, DirectoryEntry *parent, int originID, + std::wstring name, DirectoryEntry *parent, int originID, boost::shared_ptr fileRegister, boost::shared_ptr originConnection) : m_FileRegister(fileRegister), m_OriginConnection(originConnection), - m_Name(name), m_Parent(parent), m_Populated(false), m_TopLevel(false) + m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(false) { m_Origins.insert(originID); } @@ -1025,8 +1027,8 @@ void DirectoryEntry::removeFiles(const std::set &indices) } FileEntry::Ptr DirectoryEntry::insert( - const std::wstring &fileName, FilesOrigin &origin, FILETIME fileTime, - const std::wstring &archive, int order) + std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, + std::wstring_view archive, int order) { std::wstring fileNameLower = ToLowerCopy(fileName); @@ -1050,6 +1052,35 @@ FileEntry::Ptr DirectoryEntry::insert( void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOffset) { + struct Context + { + FilesOrigin& origin; + std::stack current; + }; + + Context cx = {origin}; + cx.current.push(this); + + env::forEachEntry(buffer, &cx, + [](void* pcx, std::wstring_view path) { + Context* cx = (Context*)pcx; + cx->current.push(cx->current.top()->getSubDirectory(path, true, cx->origin.getID())); + }, + + [](void* pcx, std::wstring_view path) { + Context* cx = (Context*)pcx; + auto* current= cx->current.top(); + std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); + cx->current.pop(); + }, + + [](void* pcx, std::wstring_view path, FILETIME ft) { + Context* cx = (Context*)pcx; + cx->current.top()->insert(path, cx->origin, ft, L"", -1); + } + ); + + /* WIN32_FIND_DATAW findData; _snwprintf_s(buffer + bufferOffset, MAXPATH_UNICODE - bufferOffset, _TRUNCATE, L"\\*"); @@ -1087,7 +1118,7 @@ void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOf } std::sort(m_SubDirectories.begin(), m_SubDirectories.end(), &DirCompareByName); - ::FindClose(searchHandle); + ::FindClose(searchHandle);*/ } void DirectoryEntry::addFiles( @@ -1119,17 +1150,18 @@ void DirectoryEntry::addFiles( } DirectoryEntry *DirectoryEntry::getSubDirectory( - const std::wstring &name, bool create, int originID) + std::wstring_view name, bool create, int originID) { - for (DirectoryEntry *entry : m_SubDirectories) { - if (CaseInsensitiveEqual(entry->getName(), name)) { - return entry; - } + auto itor = m_SubDirectoriesLookup.find(ToLowerCopy(name)); + + if (itor != m_SubDirectoriesLookup.end()) { + return itor->second; } if (create) { auto* entry = new DirectoryEntry( - name, this, originID, m_FileRegister, m_OriginConnection); + std::wstring(name.begin(), name.end()), this, originID, + m_FileRegister, m_OriginConnection); addDirectoryToList(entry); -- cgit v1.3.1 From c1cf76033782ff8dc4ccc465b60a2f9f1edb69b8 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 10 Feb 2020 11:05:21 -0500 Subject: don't lowercase twice --- src/shared/directoryentry.cpp | 10 ++++++---- src/shared/directoryentry.h | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 28201a11..d171be38 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -1152,7 +1152,8 @@ void DirectoryEntry::addFiles( DirectoryEntry *DirectoryEntry::getSubDirectory( std::wstring_view name, bool create, int originID) { - auto itor = m_SubDirectoriesLookup.find(ToLowerCopy(name)); + std::wstring nameLc = ToLowerCopy(name); + auto itor = m_SubDirectoriesLookup.find(nameLc); if (itor != m_SubDirectoriesLookup.end()) { return itor->second; @@ -1163,7 +1164,8 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( std::wstring(name.begin(), name.end()), this, originID, m_FileRegister, m_OriginConnection); - addDirectoryToList(entry); + addDirectoryToList(entry, std::move(nameLc)); + // nameLc is moved from this point return entry; } else { @@ -1211,10 +1213,10 @@ void DirectoryEntry::removeDirRecursive() m_SubDirectoriesLookup.clear(); } -void DirectoryEntry::addDirectoryToList(DirectoryEntry* e) +void DirectoryEntry::addDirectoryToList(DirectoryEntry* e, std::wstring nameLc) { m_SubDirectories.push_back(e); - m_SubDirectoriesLookup.emplace(ToLowerCopy(e->getName()), e); + m_SubDirectoriesLookup.emplace(std::move(nameLc), e); } void DirectoryEntry::removeDirectoryFromList(SubDirectories::iterator itor) diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 1a8a4247..58e70ffa 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -519,7 +519,7 @@ private: void removeDirRecursive(); - void addDirectoryToList(DirectoryEntry* e); + void addDirectoryToList(DirectoryEntry* e, std::wstring nameLc); void removeDirectoryFromList(SubDirectories::iterator itor); void addFileToList(std::wstring fileNameLower, FileEntry::Index index); -- cgit v1.3.1 From 3423b0a59337cf4cf99a24a1421ea33c4c641a22 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 10 Feb 2020 14:53:04 -0500 Subject: threaded refresher --- src/directoryrefresher.cpp | 188 ++++++++++++++++++++++++++++++++---------- src/directoryrefresher.h | 42 ++++++---- src/envfs.cpp | 130 +++++++++++++++++------------ src/envfs.h | 88 ++++++++++++++++++++ src/organizercore.cpp | 23 +++--- src/settings.cpp | 10 +++ src/settings.h | 5 ++ src/shared/directoryentry.cpp | 27 ++++++ src/shared/directoryentry.h | 8 ++ 9 files changed, 395 insertions(+), 126 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index b7dd51ba..5bff9d37 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -24,6 +24,7 @@ along with Mod Organizer. If not, see . #include "report.h" #include "modinfo.h" #include "settings.h" +#include "envfs.h" #include #include @@ -36,8 +37,8 @@ using namespace MOBase; using namespace MOShared; -DirectoryRefresher::DirectoryRefresher() - : m_DirectoryStructure(nullptr) +DirectoryRefresher::DirectoryRefresher(std::size_t threadCount) + : m_DirectoryStructure(nullptr), m_threadCount(threadCount) { } @@ -116,59 +117,160 @@ void DirectoryRefresher::addModBSAToStructure(DirectoryEntry *directoryStructure } } -void DirectoryRefresher::addModFilesToStructure(DirectoryEntry *directoryStructure, const QString &modName, - int priority, const QString &directory, const QStringList &stealFiles) +void DirectoryRefresher::stealModFilesIntoStructure( + DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles) { std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); - if (stealFiles.length() > 0) { - // instead of adding all the files of the target directory, we just change the root of the specified - // files to this mod - FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); - for (const QString &filename : stealFiles) { - if (filename.isEmpty()) { - log::warn("Trying to find file with no name"); - continue; - } - QFileInfo fileInfo(filename); - FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); - if (file.get() != nullptr) { - if (file->getOrigin() == 0) { - // replace data as the origin on this bsa - file->removeOrigin(0); - } - origin.addFile(file->getIndex()); - file->addOrigin(origin.getID(), file->getFileTime(), L"", -1); - } else { - QString warnStr = fileInfo.absolutePath(); - if (warnStr.isEmpty()) - warnStr = filename; - log::warn("file not found: {}", warnStr); + // instead of adding all the files of the target directory, we just change the root of the specified + // files to this mod + FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); + for (const QString &filename : stealFiles) { + if (filename.isEmpty()) { + log::warn("Trying to find file with no name"); + continue; + } + QFileInfo fileInfo(filename); + FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); + if (file.get() != nullptr) { + if (file->getOrigin() == 0) { + // replace data as the origin on this bsa + file->removeOrigin(0); } + origin.addFile(file->getIndex()); + file->addOrigin(origin.getID(), file->getFileTime(), L"", -1); + } else { + QString warnStr = fileInfo.absolutePath(); + if (warnStr.isEmpty()) + warnStr = filename; + log::warn("file not found: {}", warnStr); } + } +} + +void DirectoryRefresher::addModFilesToStructure( + DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles) +{ + TimeThis tt("addModFilesToStructure()"); + + std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + + if (stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, modName, priority, directory, stealFiles); } else { directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); } } void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure - , const QString &modName - , int priority - , const QString &directory - , const QStringList &stealFiles - , const QStringList &archives) + , const QString &modName + , int priority + , const QString &directory + , const QStringList &stealFiles + , const QStringList &archives) { - addModFilesToStructure(directoryStructure, modName, priority, directory, stealFiles); + TimeThis tt("addModToStructure()"); + + if (stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, modName, priority, directory, stealFiles); + } else { + std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); + } if (Settings::instance().archiveParsing()) { addModBSAToStructure(directoryStructure, modName, priority, directory, archives); } } +struct ModThread +{ + std::wstring path; + env::Directory* dir = nullptr; + std::condition_variable cv; + std::mutex mutex; + bool ready = false; + + void wakeup() + { + ready = true; + cv.notify_one(); + } + + void run() + { + std::unique_lock lock(mutex); + cv.wait(lock, [&]{ return ready; }); + + *dir = env::getFilesAndDirs(path); + + ready = false; + } +}; + +void DirectoryRefresher::addMultipleModsFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, + const std::vector& entries, bool emitProgress) +{ + TimeThis tt(QString("add %1 mods").arg(entries.size())); + + env::ThreadPool threads(m_threadCount); + std::vector dirs(entries.size()); + + for (std::size_t i=0; i(i + 1); + + try { + if (e.stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); + } else { + auto& mt = threads.request(); + + mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); + mt.dir = &dirs[i]; + + mt.wakeup(); + } + } catch (const std::exception& ex) { + emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); + } + + if (emitProgress) { + emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + } + } + + threads.join(); + + for (std::size_t i=0; i(i + 1); + + directoryStructure->addFromList( + entries[i].modName.toStdWString(), + entries[i].absolutePath.toStdWString(), + dirs[i], + prio); + + if (Settings::instance().archiveParsing()) { + addModBSAToStructure( + directoryStructure, + entries[i].modName, + prio, + entries[i].absolutePath, + entries[i].archives); + } + } +} + void DirectoryRefresher::refresh() { SetThisThreadName("DirectoryRefresher"); - TimeThis tt("DirectoryRefresher::refresh()"); QMutexLocker locker(&m_RefreshLock); @@ -178,20 +280,16 @@ void DirectoryRefresher::refresh() IPluginGame *game = qApp->property("managed_game").value(); - std::wstring dataDirectory = QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + std::wstring dataDirectory = + QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0); - std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs){return lhs.priority < rhs.priority;}); - auto iter = m_Mods.begin(); + std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { + return lhs.priority < rhs.priority; + }); - for (int i = 1; iter != m_Mods.end(); ++iter, ++i) { - try { - addModToStructure(m_DirectoryStructure, iter->modName, i, iter->absolutePath, iter->stealFiles, iter->archives); - } catch (const std::exception &e) { - emit error(tr("failed to read mod (%1): %2").arg(iter->modName, e.what())); - } - emit progress((i * 100) / static_cast(m_Mods.size()) + 1); - } + addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); m_DirectoryStructure->getFileRegister()->sortOrigins(); diff --git a/src/directoryrefresher.h b/src/directoryrefresher.h index a4fc5dbc..c531ba39 100644 --- a/src/directoryrefresher.h +++ b/src/directoryrefresher.h @@ -39,12 +39,23 @@ class DirectoryRefresher : public QObject Q_OBJECT public: + struct EntryInfo { + EntryInfo(const QString &modName, const QString &absolutePath, + const QStringList &stealFiles, const QStringList &archives, int priority) + : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles) + , archives(archives), priority(priority) {} + QString modName; + QString absolutePath; + QStringList stealFiles; + QStringList archives; + int priority; + }; /** * @brief constructor * **/ - DirectoryRefresher(); + DirectoryRefresher(std::size_t threadCount); ~DirectoryRefresher(); @@ -53,7 +64,7 @@ public: * * returns a pointer to the updated directory structure. DirectoryRefresher * deletes its own pointer and the caller takes custody of the pointer - * + * * @return updated directory structure **/ MOShared::DirectoryEntry *getDirectoryStructure(); @@ -107,7 +118,13 @@ public: * @param directory * @param stealFiles */ - void addModFilesToStructure(MOShared::DirectoryEntry *directoryStructure, const QString &modName, int priority, const QString &directory, const QStringList &stealFiles); + void addModFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles); + + void addMultipleModsFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, + const std::vector& entries, bool emitProgress=false); public slots: @@ -123,26 +140,15 @@ signals: void refreshed(); private: - - struct EntryInfo { - EntryInfo(const QString &modName, const QString &absolutePath, - const QStringList &stealFiles, const QStringList &archives, int priority) - : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles) - , archives(archives), priority(priority) {} - QString modName; - QString absolutePath; - QStringList stealFiles; - QStringList archives; - int priority; - }; - -private: - std::vector m_Mods; std::set m_EnabledArchives; MOShared::DirectoryEntry *m_DirectoryStructure; QMutex m_RefreshLock; + std::size_t m_threadCount; + void stealModFilesIntoStructure( + MOShared::DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles); }; #endif // DIRECTORYREFRESHER_H diff --git a/src/envfs.cpp b/src/envfs.cpp index 81a0d3ef..2716737f 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -150,44 +150,62 @@ QString toString(POBJECT_ATTRIBUTES poa) } -constexpr std::size_t AllocSize = 1024 * 1024; -std::vector> g_buffers; - - -struct HandleCloserThread +class HandleCloserThread { - std::vector handles; - std::thread thread; - std::atomic busy; - +public: HandleCloserThread() - : busy(false) + : m_ready(false) { + m_handles.reserve(50'000); } - ~HandleCloserThread() + void add(HANDLE h) { - if (thread.joinable()) { - thread.join(); - } + m_handles.push_back(h); + } + + void wakeup() + { + m_ready = true; + m_cv.notify_one(); + } + + void run() + { + std::unique_lock lock(m_mutex); + m_cv.wait(lock, [&]{ return m_ready; }); + + closeHandles(); } +private: + std::vector m_handles; + std::condition_variable m_cv; + std::mutex m_mutex; + bool m_ready; + void closeHandles() { - for (auto& h : handles) { + for (auto& h : m_handles) { NtClose(h); } - handles.clear(); - busy = false; + m_handles.clear(); + m_ready = false; } }; -std::array g_handleCloserThreads; +constexpr std::size_t AllocSize = 1024 * 1024; +static ThreadPool g_handleClosers; +void setHandleCloserThreadCount(std::size_t n) +{ + g_handleClosers.setMax(n); +} void forEachEntryImpl( - void* cx, HandleCloserThread& hc, POBJECT_ATTRIBUTES poa, std::size_t depth, + void* cx, HandleCloserThread& hc, std::vector>& buffers, + POBJECT_ATTRIBUTES poa, std::size_t depth, DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) { IO_STATUS_BLOCK iosb; @@ -207,14 +225,14 @@ void forEachEntryImpl( return; } - hc.handles.push_back(oa.RootDirectory); + hc.add(oa.RootDirectory); unsigned char* buffer; - if (depth >= g_buffers.size()) { - g_buffers.emplace_back(std::make_unique(AllocSize)); - buffer = g_buffers.back().get(); + if (depth >= buffers.size()) { + buffers.emplace_back(std::make_unique(AllocSize)); + buffer = buffers.back().get(); } else { - buffer = g_buffers[depth].get(); + buffer = buffers[depth].get(); } union @@ -268,7 +286,7 @@ void forEachEntryImpl( if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { dirStartF(cx, toStringView(&oa)); - forEachEntryImpl(cx, hc, &oa, depth+1, dirStartF, dirEndF, fileF); + forEachEntryImpl(cx, hc, buffers, &oa, depth+1, dirStartF, dirEndF, fileF); dirEndF(cx, toStringView(&oa)); } else { //log::debug("{}{}", std::wstring((depth + 1) * 2, L' '), toString(&oa)); @@ -293,35 +311,13 @@ void forEachEntryImpl( } } -std::size_t findHandleCloserThread() -{ - for (;;) { - for (std::size_t i=0; i> buffers; if (!NtOpenFile) { HMODULE m = ::LoadLibraryW(L"ntdll.dll"); @@ -342,9 +338,41 @@ void forEachEntry( oa.Length = sizeof(oa); oa.ObjectName = &ObjectName; - forEachEntryImpl(cx, hc, &oa, 0, dirStartF, dirEndF, fileF); + forEachEntryImpl(cx, hc, buffers, &oa, 0, dirStartF, dirEndF, fileF); + hc.wakeup(); +} + +Directory getFilesAndDirs(const std::wstring& path) +{ + struct Context + { + std::stack current; + }; + + Directory root; + + Context cx; + cx.current.push(&root); + + env::forEachEntry(path, &cx, + [](void* pcx, std::wstring_view path) { + Context* cx = (Context*)pcx; + cx->current.top()->dirs.push_back({std::wstring(path.begin(), path.end())}); + cx->current.push(&cx->current.top()->dirs.back()); + }, + + [](void* pcx, std::wstring_view path) { + Context* cx = (Context*)pcx; + cx->current.pop(); + }, + + [](void* pcx, std::wstring_view path, FILETIME ft) { + Context* cx = (Context*)pcx; + cx->current.top()->files.push_back({std::wstring(path.begin(), path.end()), ft}); + } + ); - hc.thread = std::thread([hci]{ g_handleCloserThreads[hci].closeHandles(); }); + return root; } } // namespace diff --git a/src/envfs.h b/src/envfs.h index aeaa6796..1fc53bcf 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -1,17 +1,105 @@ #ifndef ENV_ENVFS_H #define ENV_ENVFS_H +#include + namespace env { +struct File +{ + std::wstring name; + FILETIME ft; +}; + +struct Directory +{ + std::wstring name; + std::list dirs; + std::list files; +}; + + +template +class ThreadPool +{ +public: + ThreadPool(std::size_t max=1) + : m_threads(max) + { + } + + ~ThreadPool() + { + join(); + } + + void setMax(std::size_t n) + { + m_threads.resize(n); + } + + void join() + { + for (auto& ti : m_threads) { + if (ti.thread.joinable()) { + ti.thread.join(); + } + } + } + + T& request() + { + if (m_threads.empty()) { + std::terminate(); + } + + for (;;) { + for (auto& ti : m_threads) { + bool expected = false; + + if (ti.busy.compare_exchange_strong(expected, true)) { + if (ti.thread.joinable()) { + ti.thread.join(); + } + + ti.thread = std::thread([&]{ + ti.o.run(); + ti.busy = false; + }); + + return ti.o; + } + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + +private: + struct ThreadInfo + { + std::thread thread; + std::atomic busy; + T o; + }; + + std::list m_threads; +}; + + using DirStartF = void (void*, std::wstring_view); using DirEndF = void (void*, std::wstring_view); using FileF = void (void*, std::wstring_view, FILETIME); +void setHandleCloserThreadCount(std::size_t n); + void forEachEntry( const std::wstring& path, void* cx, DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF); +Directory getFilesAndDirs(const std::wstring& path); + } // namespace #endif // ENV_ENVFS_H diff --git a/src/organizercore.cpp b/src/organizercore.cpp index 8124be1d..86abeb35 100644 --- a/src/organizercore.cpp +++ b/src/organizercore.cpp @@ -88,24 +88,19 @@ QStringList toStringList(InputIterator current, InputIterator end) OrganizerCore::OrganizerCore(Settings &settings) : m_UserInterface(nullptr) , m_PluginContainer(nullptr) - , m_GameName() , m_CurrentProfile(nullptr) , m_Settings(settings) , m_Updater(NexusInterface::instance(m_PluginContainer)) - , m_AboutToRun() - , m_FinishedRun() - , m_ModInstalled() , m_ModList(m_PluginContainer, this) , m_PluginList(this) - , m_DirectoryRefresher() + , m_DirectoryRefresher(settings.refreshThreadCount()) , m_DirectoryStructure(new DirectoryEntry(L"data", nullptr, 0)) , m_DownloadManager(NexusInterface::instance(m_PluginContainer), this) - , m_InstallationManager() - , m_RefresherThread() , m_DirectoryUpdate(false) , m_ArchivesInit(false) , m_PluginListsWriter(std::bind(&OrganizerCore::savePluginList, this)) { + env::setHandleCloserThreadCount(settings.refreshThreadCount()); m_DownloadManager.setOutputDirectory(m_Settings.paths().downloads(), false); NexusInterface::instance(m_PluginContainer)->setCacheDirectory( @@ -1245,13 +1240,17 @@ void OrganizerCore::updateModInDirectoryStructure(unsigned int index, void OrganizerCore::updateModsInDirectoryStructure(QMap modInfo) { + std::vector entries; + for (auto idx : modInfo.keys()) { - // add files of the bsa to the directory structure - m_DirectoryRefresher.addModFilesToStructure( - m_DirectoryStructure, modInfo[idx]->name(), - m_CurrentProfile->getModPriority(idx), modInfo[idx]->absolutePath(), - modInfo[idx]->stealFiles()); + entries.push_back({ + modInfo[idx]->name(), modInfo[idx]->absolutePath(), + modInfo[idx]->stealFiles(), {}, m_CurrentProfile->getModPriority(idx)}); } + + m_DirectoryRefresher.addMultipleModsFilesToStructure( + m_DirectoryStructure, entries); + DirectoryRefresher::cleanStructure(m_DirectoryStructure); // need to refresh plugin list now so we can activate esps refreshESPList(true); diff --git a/src/settings.cpp b/src/settings.cpp index 761cd669..f0496fe8 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -209,6 +209,16 @@ void Settings::setUseSplash(bool b) set(m_Settings, "Settings", "use_splash", b); } +std::size_t Settings::refreshThreadCount() const +{ + return get(m_Settings, "Settings", "refresh_thread_count", 10); +} + +void Settings::setRefreshThreadCount(std::size_t n) const +{ + return set(m_Settings, "Settings", "refresh_thread_count", n); +} + std::optional Settings::version() const { if (auto v=getOptional(m_Settings, "General", "version")) { diff --git a/src/settings.h b/src/settings.h index 3d37bdd8..b2cb6be5 100644 --- a/src/settings.h +++ b/src/settings.h @@ -731,6 +731,11 @@ public: bool useSplash() const; void setUseSplash(bool b); + // number of threads to use when refreshing + // + std::size_t refreshThreadCount() const; + void setRefreshThreadCount(std::size_t n) const; + GameSettings& game(); const GameSettings& game() const; diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index d171be38..b7329833 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -691,6 +691,33 @@ void DirectoryEntry::addFromOrigin( m_Populated = true; } +void DirectoryEntry::addFromList( + const std::wstring &originName, const std::wstring &directory, + env::Directory& root, int priority) +{ + FilesOrigin &origin = createOrigin(originName, directory, priority); + addDir(origin, root); +} + +void DirectoryEntry::addDir(FilesOrigin& origin, env::Directory& d) +{ + for (auto& sd : d.dirs) { + auto* sdirEntry = getSubDirectory(sd.name, true, origin.getID()); + sdirEntry->addDir(origin, sd); + } + + for (auto& f : d.files) { + insert(f.name, origin, f.ft, L"", -1); + } + + std::sort( + m_SubDirectories.begin(), + m_SubDirectories.end(), + &DirCompareByName); + + m_Populated = true; +} + void DirectoryEntry::addFromBSA( const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order) diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 58e70ffa..772899d7 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -33,7 +33,9 @@ along with Mod Organizer. If not, see . #include #include #endif + #include "util.h" +#include "envfs.h" namespace MOShared { struct DirectoryEntryFileKey; } @@ -358,6 +360,10 @@ public: const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order); + void addFromList( + const std::wstring &originName, const std::wstring &directory, + env::Directory& root, int priority); + void propagateOrigin(int origin); const std::wstring &getName() const @@ -511,6 +517,8 @@ private: FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, const std::wstring &archiveName, int order); + void addDir(FilesOrigin& origin, env::Directory& d); + DirectoryEntry* getSubDirectory( std::wstring_view name, bool create, int originID = -1); -- cgit v1.3.1 From 7860e927a771d6a07d9b1905071bd389a0e64001 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 10 Feb 2020 15:10:27 -0500 Subject: removed dead insertFile() avoided a few string copies lowercase filename in thread --- src/envfs.cpp | 8 +++++++- src/envfs.h | 3 ++- src/shared/directoryentry.cpp | 45 ++++++++++++++++++++++++------------------- src/shared/directoryentry.h | 10 ++++++---- 4 files changed, 40 insertions(+), 26 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/envfs.cpp b/src/envfs.cpp index 2716737f..80749046 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -1,4 +1,5 @@ #include "envfs.h" +#include "util.h" #include #include @@ -368,7 +369,12 @@ Directory getFilesAndDirs(const std::wstring& path) [](void* pcx, std::wstring_view path, FILETIME ft) { Context* cx = (Context*)pcx; - cx->current.top()->files.push_back({std::wstring(path.begin(), path.end()), ft}); + + cx->current.top()->files.push_back({ + std::wstring(path.begin(), path.end()), + MOShared::ToLowerCopy(path), + ft + }); } ); diff --git a/src/envfs.h b/src/envfs.h index 1fc53bcf..6cc98775 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -9,7 +9,8 @@ namespace env struct File { std::wstring name; - FILETIME ft; + std::wstring lcname; + FILETIME lastModified; }; struct Directory diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index b7329833..5df2c1ba 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -174,8 +174,8 @@ FileEntry::FileEntry() : { } -FileEntry::FileEntry(Index index, std::wstring_view name, DirectoryEntry *parent) : - m_Index(index), m_Name(name.begin(), name.end()), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), +FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : + m_Index(index), m_Name(std::move(name)), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), m_LastAccessed(time(nullptr)) { @@ -511,7 +511,7 @@ bool FileRegister::indexValid(FileEntry::Index index) const return (m_Files.find(index) != m_Files.end()); } -FileEntry::Ptr FileRegister::createFile(std::wstring_view name, DirectoryEntry *parent) +FileEntry::Ptr FileRegister::createFile(std::wstring name, DirectoryEntry *parent) { FileEntry::Index index = generateIndex(); @@ -707,7 +707,7 @@ void DirectoryEntry::addDir(FilesOrigin& origin, env::Directory& d) } for (auto& f : d.files) { - insert(f.name, origin, f.ft, L"", -1); + insert(f, origin, L"", -1); } std::sort( @@ -944,20 +944,6 @@ const FileEntry::Ptr DirectoryEntry::searchFile( return FileEntry::Ptr(); } -void DirectoryEntry::insertFile( - const std::wstring &filePath, FilesOrigin &origin, FILETIME fileTime) -{ - size_t pos = filePath.find_first_of(L"\\/"); - - if (pos == std::string::npos) { - this->insert(filePath, origin, fileTime, std::wstring(), -1); - } else { - std::wstring dirName = filePath.substr(0, pos); - std::wstring rest = filePath.substr(pos + 1); - getSubDirectoryRecursive(dirName, true, origin.getID())->insertFile(rest, origin, fileTime); - } -} - void DirectoryEntry::removeFile(FileEntry::Index index) { removeFileFromList(index); @@ -1065,9 +1051,8 @@ FileEntry::Ptr DirectoryEntry::insert( if (iter != m_Files.end()) { file = m_FileRegister->getFile(iter->second); } else { - file = m_FileRegister->createFile(fileName, this); + file = m_FileRegister->createFile(std::wstring(fileName.begin(), fileName.end()), this); addFileToList(std::move(fileNameLower), file->getIndex()); - // fileNameLower has moved from this point } @@ -1077,6 +1062,26 @@ FileEntry::Ptr DirectoryEntry::insert( return file; } +FileEntry::Ptr DirectoryEntry::insert( + env::File& file, FilesOrigin &origin, std::wstring_view archive, int order) +{ + auto iter = m_Files.find(file.lcname); + FileEntry::Ptr fe; + + if (iter != m_Files.end()) { + fe = m_FileRegister->getFile(iter->second); + } else { + fe = m_FileRegister->createFile(std::move(file.name), this); + addFileToList(std::move(file.lcname), fe->getIndex()); + // both file.name and file.lcname have been moved from this point + } + + fe->addOrigin(origin.getID(), file.lastModified, archive, order); + origin.addFile(fe->getIndex()); + + return fe; +} + void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOffset) { struct Context diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 772899d7..f8c72d77 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -80,7 +80,7 @@ public: AlternativesVector; FileEntry(); - FileEntry(Index index, std::wstring_view name, DirectoryEntry *parent); + FileEntry(Index index, std::wstring name, DirectoryEntry *parent); Index getIndex() const { @@ -263,7 +263,7 @@ public: bool indexValid(FileEntry::Index index) const; - FileEntry::Ptr createFile(std::wstring_view name, DirectoryEntry *parent); + FileEntry::Ptr createFile(std::wstring name, DirectoryEntry *parent); FileEntry::Ptr getFile(FileEntry::Index index) const; size_t size() const @@ -459,8 +459,6 @@ public: const FileEntry::Ptr searchFile( const std::wstring &path, const DirectoryEntry **directory=nullptr) const; - void insertFile(const std::wstring &filePath, FilesOrigin &origin, FILETIME fileTime); - void removeFile(FileEntry::Index index); // remove the specified file from the tree. This can be a path leading to a @@ -510,6 +508,10 @@ private: std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, std::wstring_view archive, int order); + FileEntry::Ptr insert( + env::File& file, FilesOrigin &origin, + std::wstring_view archive, int order); + void addFiles( FilesOrigin &origin, wchar_t *buffer, int bufferOffset); -- cgit v1.3.1 From d082a91ad617808a6dd89ad9fdb9479524795520 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Wed, 12 Feb 2020 16:14:18 -0500 Subject: faster getSubDirectory() for env::Directory, added lcname instrumentation --- src/envfs.cpp | 32 ++++- src/envfs.h | 11 ++ src/shared/directoryentry.cpp | 313 +++++++++++++++++++++++++++++++++++++----- src/shared/directoryentry.h | 59 +++++++- 4 files changed, 366 insertions(+), 49 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/envfs.cpp b/src/envfs.cpp index 80749046..d7245f66 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -1,4 +1,5 @@ #include "envfs.h" +#include "env.h" #include "util.h" #include #include @@ -160,6 +161,11 @@ public: m_handles.reserve(50'000); } + void shrink() + { + m_handles.shrink_to_fit(); + } + void add(HANDLE h) { m_handles.push_back(h); @@ -204,6 +210,15 @@ void setHandleCloserThreadCount(std::size_t n) g_handleClosers.setMax(n); } +void shrinkFs() +{ + g_handleClosers.join(); + + g_handleClosers.forEach([](auto&& t) { + t.shrink(); + }); +} + void forEachEntryImpl( void* cx, HandleCloserThread& hc, std::vector>& buffers, POBJECT_ATTRIBUTES poa, std::size_t depth, @@ -290,7 +305,6 @@ void forEachEntryImpl( forEachEntryImpl(cx, hc, buffers, &oa, depth+1, dirStartF, dirEndF, fileF); dirEndF(cx, toStringView(&oa)); } else { - //log::debug("{}{}", std::wstring((depth + 1) * 2, L' '), toString(&oa)); FILETIME ft; ft.dwLowDateTime = DirInfo->LastWriteTime.LowPart; ft.dwHighDateTime = DirInfo->LastWriteTime.HighPart; @@ -321,11 +335,10 @@ void forEachEntry( std::vector> buffers; if (!NtOpenFile) { - HMODULE m = ::LoadLibraryW(L"ntdll.dll"); - NtOpenFile = (NtOpenFile_type)::GetProcAddress(m, "NtOpenFile"); - NtQueryDirectoryFile = (NtQueryDirectoryFile_type)::GetProcAddress(m, "NtQueryDirectoryFile"); - NtClose = (NtClose_type)::GetProcAddress(m, "NtClose"); - ::FreeLibrary(m); + LibraryPtr m(::LoadLibraryW(L"ntdll.dll")); + NtOpenFile = (NtOpenFile_type)::GetProcAddress(m.get(), "NtOpenFile"); + NtQueryDirectoryFile = (NtQueryDirectoryFile_type)::GetProcAddress(m.get(), "NtQueryDirectoryFile"); + NtClose = (NtClose_type)::GetProcAddress(m.get(), "NtClose"); } const std::wstring ntpath = std::wstring(L"\\??\\") + path; @@ -358,7 +371,12 @@ Directory getFilesAndDirs(const std::wstring& path) env::forEachEntry(path, &cx, [](void* pcx, std::wstring_view path) { Context* cx = (Context*)pcx; - cx->current.top()->dirs.push_back({std::wstring(path.begin(), path.end())}); + + cx->current.top()->dirs.push_back({ + std::wstring(path.begin(), path.end()), + MOShared::ToLowerCopy(path) + }); + cx->current.push(&cx->current.top()->dirs.back()); }, diff --git a/src/envfs.h b/src/envfs.h index 6cc98775..6e0da6d0 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -16,6 +16,8 @@ struct File struct Directory { std::wstring name; + std::wstring lcname; + std::list dirs; std::list files; }; @@ -77,6 +79,14 @@ public: } } + template + void forEach(F&& f) + { + for (auto& ti : m_threads) { + f(ti.o); + } + } + private: struct ThreadInfo { @@ -94,6 +104,7 @@ using DirEndF = void (void*, std::wstring_view); using FileF = void (void*, std::wstring_view, FILETIME); void setHandleCloserThreadCount(std::size_t n); +void shrinkFs(); void forEachEntry( const std::wstring& path, void* cx, diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 5df2c1ba..a90f6f08 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -40,6 +40,30 @@ namespace MOShared using namespace MOBase; static const int MAXPATH_UNICODE = 32767; + +static std::atomic FileEntryCount(0); +static std::atomic FilesOriginCount(0); +static std::atomic FileRegisterCount(0); +static std::atomic DirectoryEntryCount(0); +static std::atomic OriginConnectionCount(0); + +template +std::chrono::nanoseconds elapsed(F&& f) +{ + const auto start = std::chrono::high_resolution_clock::now(); + f(); + const auto end = std::chrono::high_resolution_clock::now(); + return (end - start); +} + +void logcounts(std::string w) +{ + log::debug( + "{}: FileEntry={} FilesOrigin={} FileRegister={} DirectoryEntry={} OriginConnection={}", + w, FileEntryCount, FilesOriginCount, FileRegisterCount, DirectoryEntryCount, OriginConnectionCount); +} + + static std::wstring tail(const std::wstring &source, const size_t count) { if (count >= source.length()) { @@ -71,6 +95,101 @@ static bool DirCompareByName(const DirectoryEntry *lhs, const DirectoryEntry *rh } +DirectoryStats::DirectoryStats() +{ + std::memset(this, 0, sizeof(DirectoryStats)); +} + +DirectoryStats& DirectoryStats::operator+=(const DirectoryStats& o) +{ + dirTimes += o.dirTimes; + fileTimes += o.fileTimes; + sortTimes += o.sortTimes; + + subdirLookupTimes += o.subdirLookupTimes; + addDirectoryTimes += o.addDirectoryTimes; + + filesLookupTimes += o.filesLookupTimes; + addFileTimes += o.addFileTimes; + addOriginToFileTimes += o.addOriginToFileTimes; + addFileToOriginTimes += o.addFileToOriginTimes; + addFileToRegisterTimes += o.addFileToRegisterTimes; + + originExists += o.originExists; + originCreate += o.originCreate; + originsNeededEnabled += o.originsNeededEnabled; + + subdirExists += o.subdirExists; + subdirCreate += o.subdirCreate; + + fileExists += o.fileExists; + fileCreate += o.fileCreate; + filesInsertedInRegister += o.filesInsertedInRegister; + filesAssignedInRegister += o.filesAssignedInRegister; + + return *this; +} + +std::string DirectoryStats::csvHeader() +{ + QStringList sl = { + "dirTimes", + "fileTimes", + "sortTimes", + "subdirLookupTimes", + "addDirectoryTimes", + "filesLookupTimes", + "addFileTimes", + "addOriginToFileTimes", + "addFileToOriginTimes", + "addFileToRegisterTimes", + "originExists", + "originCreate", + "originsNeededEnabled", + "subdirExists", + "subdirCreate", + "fileExists", + "fileCreate", + "filesInsertedInRegister", + "filesAssignedInRegister"}; + + return sl.join(",").toStdString(); +} + +std::string DirectoryStats::toCsv() const +{ + QStringList oss; + + oss + << QString::number(dirTimes.count()) + << QString::number(fileTimes.count()) + << QString::number(sortTimes.count()) + + << QString::number(subdirLookupTimes.count()) + << QString::number(addDirectoryTimes.count()) + + << QString::number(filesLookupTimes.count()) + << QString::number(addFileTimes.count()) + << QString::number(addOriginToFileTimes.count()) + << QString::number(addFileToOriginTimes.count()) + << QString::number(addFileToRegisterTimes.count()) + + << QString::number(originExists) + << QString::number(originCreate) + << QString::number(originsNeededEnabled) + + << QString::number(subdirExists) + << QString::number(subdirCreate) + + << QString::number(fileExists) + << QString::number(fileCreate) + << QString::number(filesInsertedInRegister) + << QString::number(filesAssignedInRegister); + + return oss.join(",").toStdString(); +} + + class OriginConnection { public: @@ -80,6 +199,12 @@ public: OriginConnection() : m_NextID(0) { + ++OriginConnectionCount; + } + + ~OriginConnection() + { + --OriginConnectionCount; } FilesOrigin& createOrigin( @@ -172,6 +297,7 @@ FileEntry::FileEntry() : m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), m_LastAccessed(time(nullptr)) { + ++FileEntryCount; } FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : @@ -179,6 +305,16 @@ FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), m_LastAccessed(time(nullptr)) { + ++FileEntryCount; +} + +FileEntry::~FileEntry() +{ + while (!m_Alternatives.empty()) { + m_Alternatives.pop_back(); + } + + --FileEntryCount; } void FileEntry::addOrigin( @@ -407,6 +543,7 @@ bool FileEntry::recurseParents(std::wstring &path, const DirectoryEntry *parent) FilesOrigin::FilesOrigin() : m_ID(0), m_Disabled(false), m_Name(), m_Path(), m_Priority(0) { + ++FilesOriginCount; } FilesOrigin::FilesOrigin(const FilesOrigin &reference) @@ -418,6 +555,7 @@ FilesOrigin::FilesOrigin(const FilesOrigin &reference) , m_FileRegister(reference.m_FileRegister) , m_OriginConnection(reference.m_OriginConnection) { + ++FilesOriginCount; } FilesOrigin::FilesOrigin( @@ -428,6 +566,12 @@ FilesOrigin::FilesOrigin( m_Priority(priority), m_FileRegister(fileRegister), m_OriginConnection(originConnection) { + ++FilesOriginCount; +} + +FilesOrigin::~FilesOrigin() +{ + --FilesOriginCount; } void FilesOrigin::setPriority(int priority) @@ -468,8 +612,15 @@ FileEntry::Ptr FilesOrigin::findFile(FileEntry::Index index) const } void FilesOrigin::enable(bool enabled, time_t notAfter) +{ + DirectoryStats dummy; + enable(enabled, dummy, notAfter); +} + +void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) { if (!enabled) { + ++stats.originsNeededEnabled; std::set copy = m_Files; m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter); m_Files.clear(); @@ -504,6 +655,12 @@ bool FilesOrigin::containsArchive(std::wstring archiveName) FileRegister::FileRegister(boost::shared_ptr originConnection) : m_OriginConnection(originConnection) { + ++FileRegisterCount; +} + +FileRegister::~FileRegister() +{ + --FileRegisterCount; } bool FileRegister::indexValid(FileEntry::Index index) const @@ -511,14 +668,26 @@ bool FileRegister::indexValid(FileEntry::Index index) const return (m_Files.find(index) != m_Files.end()); } -FileEntry::Ptr FileRegister::createFile(std::wstring name, DirectoryEntry *parent) +FileEntry::Ptr FileRegister::createFile( + std::wstring name, DirectoryEntry *parent, DirectoryStats& stats) { FileEntry::Index index = generateIndex(); + FileEntry::Ptr p; - auto r = m_Files.insert_or_assign( - index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); + stats.addFileToRegisterTimes += elapsed([&]{ + auto r = m_Files.insert_or_assign( + index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); - return r.first->second; + if (r.second) { + ++stats.filesInsertedInRegister; + } else { + ++stats.filesAssignedInRegister; + } + + p = r.first->second; + }); + + return p; } FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const @@ -643,6 +812,7 @@ DirectoryEntry::DirectoryEntry( m_OriginConnection(new OriginConnection), m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(true) { + ++DirectoryEntryCount; m_FileRegister.reset(new FileRegister(m_OriginConnection)); m_Origins.insert(originID); } @@ -654,23 +824,24 @@ DirectoryEntry::DirectoryEntry( m_FileRegister(fileRegister), m_OriginConnection(originConnection), m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(false) { + ++DirectoryEntryCount; m_Origins.insert(originID); } DirectoryEntry::~DirectoryEntry() { + --DirectoryEntryCount; clear(); } void DirectoryEntry::clear() { - m_Files.clear(); - m_FilesLookup.clear(); - - for (DirectoryEntry *entry : m_SubDirectories) { - delete entry; + for (auto itor=m_SubDirectories.rbegin(); itor!=m_SubDirectories.rend(); ++itor) { + delete *itor; } + m_Files.clear(); + m_FilesLookup.clear(); m_SubDirectories.clear(); m_SubDirectoriesLookup.clear(); } @@ -678,7 +849,8 @@ void DirectoryEntry::clear() void DirectoryEntry::addFromOrigin( const std::wstring &originName, const std::wstring &directory, int priority) { - FilesOrigin &origin = createOrigin(originName, directory, priority); + DirectoryStats dummy; + FilesOrigin &origin = createOrigin(originName, directory, priority, dummy); if (directory.length() != 0) { boost::scoped_array buffer(new wchar_t[MAXPATH_UNICODE + 1]); @@ -693,27 +865,36 @@ void DirectoryEntry::addFromOrigin( void DirectoryEntry::addFromList( const std::wstring &originName, const std::wstring &directory, - env::Directory& root, int priority) + env::Directory& root, int priority, DirectoryStats& stats) { - FilesOrigin &origin = createOrigin(originName, directory, priority); - addDir(origin, root); + stats = {}; + + FilesOrigin &origin = createOrigin(originName, directory, priority, stats); + addDir(origin, root, stats); } -void DirectoryEntry::addDir(FilesOrigin& origin, env::Directory& d) +void DirectoryEntry::addDir( + FilesOrigin& origin, env::Directory& d, DirectoryStats& stats) { - for (auto& sd : d.dirs) { - auto* sdirEntry = getSubDirectory(sd.name, true, origin.getID()); - sdirEntry->addDir(origin, sd); - } + stats.dirTimes += elapsed([&]{ + for (auto& sd : d.dirs) { + auto* sdirEntry = getSubDirectory(sd, true, stats, origin.getID()); + sdirEntry->addDir(origin, sd, stats); + } + }); - for (auto& f : d.files) { - insert(f, origin, L"", -1); - } + stats.fileTimes += elapsed([&]{ + for (auto& f : d.files) { + insert(f, origin, L"", -1, stats); + } + }); - std::sort( - m_SubDirectories.begin(), - m_SubDirectories.end(), - &DirCompareByName); + stats.sortTimes += elapsed([&]{ + std::sort( + m_SubDirectories.begin(), + m_SubDirectories.end(), + &DirCompareByName); + }); m_Populated = true; } @@ -722,7 +903,8 @@ void DirectoryEntry::addFromBSA( const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order) { - FilesOrigin &origin = createOrigin(originName, directory, priority); + DirectoryStats dummy; + FilesOrigin &origin = createOrigin(originName, directory, priority, dummy); WIN32_FILE_ATTRIBUTE_DATA fileData; if (::GetFileAttributesExW(fileName.c_str(), GetFileExInfoStandard, &fileData) == 0) { @@ -1022,13 +1204,16 @@ bool DirectoryEntry::hasContentsFromOrigin(int originID) const } FilesOrigin &DirectoryEntry::createOrigin( - const std::wstring &originName, const std::wstring &directory, int priority) + const std::wstring &originName, const std::wstring &directory, int priority, + DirectoryStats& stats) { if (m_OriginConnection->exists(originName)) { + ++stats.originExists; FilesOrigin &origin = m_OriginConnection->getByName(originName); - origin.enable(true); + origin.enable(true, stats); return origin; } else { + ++stats.originCreate; return m_OriginConnection->createOrigin( originName, directory, priority, m_FileRegister, m_OriginConnection); } @@ -1051,7 +1236,11 @@ FileEntry::Ptr DirectoryEntry::insert( if (iter != m_Files.end()) { file = m_FileRegister->getFile(iter->second); } else { - file = m_FileRegister->createFile(std::wstring(fileName.begin(), fileName.end()), this); + DirectoryStats dummy; + + file = m_FileRegister->createFile( + std::wstring(fileName.begin(), fileName.end()), this, dummy); + addFileToList(std::move(fileNameLower), file->getIndex()); // fileNameLower has moved from this point } @@ -1063,21 +1252,37 @@ FileEntry::Ptr DirectoryEntry::insert( } FileEntry::Ptr DirectoryEntry::insert( - env::File& file, FilesOrigin &origin, std::wstring_view archive, int order) + env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, + DirectoryStats& stats) { - auto iter = m_Files.find(file.lcname); + FilesMap::iterator itor; + + stats.filesLookupTimes += elapsed([&]{ + itor = m_Files.find(file.lcname); + }); + FileEntry::Ptr fe; - if (iter != m_Files.end()) { - fe = m_FileRegister->getFile(iter->second); + if (itor != m_Files.end()) { + ++stats.fileExists; + fe = m_FileRegister->getFile(itor->second); } else { - fe = m_FileRegister->createFile(std::move(file.name), this); - addFileToList(std::move(file.lcname), fe->getIndex()); + fe = m_FileRegister->createFile(std::move(file.name), this, stats); + + stats.addFileTimes += elapsed([&]{ + addFileToList(std::move(file.lcname), fe->getIndex()); + }); + // both file.name and file.lcname have been moved from this point } - fe->addOrigin(origin.getID(), file.lastModified, archive, order); - origin.addFile(fe->getIndex()); + stats.addOriginToFileTimes += elapsed([&]{ + fe->addOrigin(origin.getID(), file.lastModified, archive, order); + }); + + stats.addFileToOriginTimes += elapsed([&]{ + origin.addFile(fe->getIndex()); + }); return fe; } @@ -1205,6 +1410,40 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( } } +DirectoryEntry *DirectoryEntry::getSubDirectory( + env::Directory& dir, bool create, DirectoryStats& stats, int originID) +{ + SubDirectoriesLookup::iterator itor; + + stats.subdirLookupTimes += elapsed([&] { + itor = m_SubDirectoriesLookup.find(dir.lcname); + }); + + if (itor != m_SubDirectoriesLookup.end()) { + ++stats.subdirExists; + return itor->second; + } + + if (create) { + ++stats.subdirCreate; + + auto* entry = new DirectoryEntry( + std::move(dir.name), this, originID, + m_FileRegister, m_OriginConnection); + // dir.name is moved from this point + + stats.addDirectoryTimes += elapsed([&]{ + addDirectoryToList(entry, std::move(dir.lcname)); + }); + + // dir.lcname is moved from this point + + return entry; + } else { + return nullptr; + } +} + DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive( const std::wstring &path, bool create, int originID) { diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index f8c72d77..c8ec8a4e 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -60,6 +60,44 @@ class OriginConnection; class FileRegister; +struct DirectoryStats +{ + std::string mod; + + std::chrono::nanoseconds dirTimes; + std::chrono::nanoseconds fileTimes; + std::chrono::nanoseconds sortTimes; + + std::chrono::nanoseconds subdirLookupTimes; + std::chrono::nanoseconds addDirectoryTimes; + + std::chrono::nanoseconds filesLookupTimes; + std::chrono::nanoseconds addFileTimes; + std::chrono::nanoseconds addOriginToFileTimes; + std::chrono::nanoseconds addFileToOriginTimes; + std::chrono::nanoseconds addFileToRegisterTimes; + + int64_t originExists; + int64_t originCreate; + int64_t originsNeededEnabled; + + int64_t subdirExists; + int64_t subdirCreate; + + int64_t fileExists; + int64_t fileCreate; + int64_t filesInsertedInRegister; + int64_t filesAssignedInRegister; + + DirectoryStats(); + + DirectoryStats& operator+=(const DirectoryStats& o); + + static std::string csvHeader(); + std::string toCsv() const; +}; + + class FileEntry { public: @@ -81,6 +119,7 @@ public: FileEntry(); FileEntry(Index index, std::wstring name, DirectoryEntry *parent); + ~FileEntry(); Index getIndex() const { @@ -195,6 +234,7 @@ class FilesOrigin public: FilesOrigin(); FilesOrigin(const FilesOrigin &reference); + ~FilesOrigin(); // sets priority for this origin, but it will overwrite the existing mapping // for this priority, the previous origin will no longer be referenced @@ -224,7 +264,9 @@ public: std::vector getFiles() const; FileEntry::Ptr findFile(FileEntry::Index index) const; + void enable(bool enabled, DirectoryStats& stats, time_t notAfter = LONG_MAX); void enable(bool enabled, time_t notAfter = LONG_MAX); + bool isDisabled() const { return m_Disabled; @@ -260,10 +302,13 @@ class FileRegister { public: FileRegister(boost::shared_ptr originConnection); + ~FileRegister(); bool indexValid(FileEntry::Index index) const; - FileEntry::Ptr createFile(std::wstring name, DirectoryEntry *parent); + FileEntry::Ptr createFile( + std::wstring name, DirectoryEntry *parent, DirectoryStats& stats); + FileEntry::Ptr getFile(FileEntry::Index index) const; size_t size() const @@ -362,7 +407,7 @@ public: void addFromList( const std::wstring &originName, const std::wstring &directory, - env::Directory& root, int priority); + env::Directory& root, int priority, DirectoryStats& stats); void propagateOrigin(int origin); @@ -477,7 +522,7 @@ public: FilesOrigin &createOrigin( const std::wstring &originName, - const std::wstring &directory, int priority); + const std::wstring &directory, int priority, DirectoryStats& stats); void removeFiles(const std::set &indices); @@ -510,7 +555,7 @@ private: FileEntry::Ptr insert( env::File& file, FilesOrigin &origin, - std::wstring_view archive, int order); + std::wstring_view archive, int order, DirectoryStats& stats); void addFiles( FilesOrigin &origin, wchar_t *buffer, int bufferOffset); @@ -519,11 +564,15 @@ private: FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, const std::wstring &archiveName, int order); - void addDir(FilesOrigin& origin, env::Directory& d); + void addDir(FilesOrigin& origin, env::Directory& d, DirectoryStats& stats); DirectoryEntry* getSubDirectory( std::wstring_view name, bool create, int originID = -1); + DirectoryEntry* getSubDirectory( + env::Directory& dir, bool create, DirectoryStats& stats, + int originID = -1); + DirectoryEntry* getSubDirectoryRecursive( const std::wstring &path, bool create, int originID = -1); -- cgit v1.3.1 From 29244f3328cdf878ac557acbbabb19a5b7190b2d Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Thu, 13 Feb 2020 20:13:37 -0500 Subject: thread-safe directory entry --- src/directoryrefresher.cpp | 113 ++++++++++++++++++++++++++---------------- src/envfs.cpp | 70 ++++++++++++++++++++++---- src/envfs.h | 10 +++- src/shared/directoryentry.cpp | 87 ++++++++++++++++++++++++-------- src/shared/directoryentry.h | 8 +++ 5 files changed, 212 insertions(+), 76 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index 5bff9d37..bce3f65a 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -25,6 +25,7 @@ along with Mod Organizer. If not, see . #include "modinfo.h" #include "settings.h" #include "envfs.h" +#include "modinfodialogfwd.h" #include #include @@ -125,7 +126,10 @@ void DirectoryRefresher::stealModFilesIntoStructure( // instead of adding all the files of the target directory, we just change the root of the specified // files to this mod - FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); + DirectoryStats dummy; + FilesOrigin &origin = directoryStructure->createOrigin( + ToWString(modName), directoryW, priority, dummy); + for (const QString &filename : stealFiles) { if (filename.isEmpty()) { log::warn("Trying to find file with no name"); @@ -189,8 +193,12 @@ void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure struct ModThread { + DirectoryEntry* ds = nullptr; + std::wstring modName; std::wstring path; + int prio = -1; env::Directory* dir = nullptr; + std::condition_variable cv; std::mutex mutex; bool ready = false; @@ -206,7 +214,16 @@ struct ModThread std::unique_lock lock(mutex); cv.wait(lock, [&]{ return ready; }); - *dir = env::getFilesAndDirs(path); + ds->addFromOrigin(modName, path, prio); + + /*if (Settings::instance().archiveParsing()) { + addModBSAToStructure( + directoryStructure, + entries[i].modName, + prio, + entries[i].absolutePath, + entries[i].archives); + }*/ ready = false; } @@ -216,65 +233,75 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( MOShared::DirectoryEntry *directoryStructure, const std::vector& entries, bool emitProgress) { - TimeThis tt(QString("add %1 mods").arg(entries.size())); - - env::ThreadPool threads(m_threadCount); std::vector dirs(entries.size()); - for (std::size_t i=0; i(i + 1); + { + TimeThis tt("walk dirs"); - try { - if (e.stealFiles.length() > 0) { - stealModFilesIntoStructure( - directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); - } else { - auto& mt = threads.request(); + env::ThreadPool threads(m_threadCount); - mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); - mt.dir = &dirs[i]; + for (std::size_t i=0; i(i + 1); - mt.wakeup(); + try + { + if (e.stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); + } else { + auto& mt = threads.request(); + + mt.ds = directoryStructure; + mt.modName = entries[i].modName.toStdWString(); + mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); + mt.prio = prio; + mt.dir = &dirs[i]; + + mt.wakeup(); + } + } catch (const std::exception& ex) { + emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); } - } catch (const std::exception& ex) { - emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); - } - if (emitProgress) { - emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + if (emitProgress) { + emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + } } - } - - threads.join(); - - for (std::size_t i=0; i(i + 1); - - directoryStructure->addFromList( - entries[i].modName.toStdWString(), - entries[i].absolutePath.toStdWString(), - dirs[i], - prio); - if (Settings::instance().archiveParsing()) { - addModBSAToStructure( - directoryStructure, - entries[i].modName, - prio, - entries[i].absolutePath, - entries[i].archives); - } + threads.join(); } + + //std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){ + // return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0); + //}); + + //static int run = 1; + // + //std::ofstream out("c:\\tmp\\data.csv", std::ios::app); + + //out << fmt::format("what,run,{}", DirectoryStats::csvHeader()); + // + //for (std::size_t i=0; icurrent.top()->dirs.push_back({ - std::wstring(path.begin(), path.end()), - MOShared::ToLowerCopy(path) - }); - + cx->current.top()->dirs.push_back(Directory(path)); cx->current.push(&cx->current.top()->dirs.back()); }, @@ -385,18 +381,70 @@ Directory getFilesAndDirs(const std::wstring& path) cx->current.pop(); }, - [](void* pcx, std::wstring_view path, FILETIME ft) { + [](void* pcx, std::wstring_view path, FILETIME ft) { Context* cx = (Context*)pcx; - cx->current.top()->files.push_back({ - std::wstring(path.begin(), path.end()), - MOShared::ToLowerCopy(path), - ft - }); + cx->current.top()->files.push_back(File(path, ft)); } ); return root; } +File::File(std::wstring_view n, FILETIME ft) : + name(n.begin(), n.end()), + lcname(MOShared::ToLowerCopy(name)), + lastModified(ft) +{ +} + +Directory::Directory() +{ +} + +Directory::Directory(std::wstring_view n) + : name(n.begin(), n.end()), lcname(MOShared::ToLowerCopy(name)) +{ +} + + +void getFilesAndDirsWithFindImpl(const std::wstring& path, Directory& d) +{ + const std::wstring searchString = path + L"\\*"; + + WIN32_FIND_DATAW findData; + + HANDLE searchHandle = ::FindFirstFileExW( + searchString.c_str(), FindExInfoBasic, &findData, FindExSearchNameMatch, + nullptr, FIND_FIRST_EX_LARGE_FETCH); + + if (searchHandle != INVALID_HANDLE_VALUE) { + BOOL result = true; + + while (result) { + if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + if ((wcscmp(findData.cFileName, L".") != 0) && + (wcscmp(findData.cFileName, L"..") != 0)) { + const std::wstring newPath = path + L"\\" + findData.cFileName; + d.dirs.push_back(Directory(findData.cFileName)); + getFilesAndDirsWithFindImpl(newPath, d.dirs.back()); + } + } else { + d.files.push_back(File(findData.cFileName, findData.ftLastWriteTime)); + } + + result = ::FindNextFileW(searchHandle, &findData); + } + } + + ::FindClose(searchHandle); +} + +Directory getFilesAndDirsWithFind(const std::wstring& path) +{ + Directory d; + getFilesAndDirsWithFindImpl(path, d); + return d; +} + } // namespace diff --git a/src/envfs.h b/src/envfs.h index 6e0da6d0..001f2d09 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -11,6 +11,8 @@ struct File std::wstring name; std::wstring lcname; FILETIME lastModified; + + File(std::wstring_view name, FILETIME ft); }; struct Directory @@ -18,8 +20,11 @@ struct Directory std::wstring name; std::wstring lcname; - std::list dirs; - std::list files; + std::vector dirs; + std::vector files; + + Directory(); + Directory(std::wstring_view name); }; @@ -111,6 +116,7 @@ void forEachEntry( DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF); Directory getFilesAndDirs(const std::wstring& path); +Directory getFilesAndDirsWithFind(const std::wstring& path); } // namespace diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index a90f6f08..e1464595 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -214,11 +214,14 @@ public: { int newID = createID(); - m_Origins[newID] = FilesOrigin(newID, originName, directory, priority, fileRegister, originConnection); - m_OriginsNameMap[originName] = newID; - m_OriginsPriorityMap[priority] = newID; + auto itor = m_Origins.insert({newID, FilesOrigin( + newID, originName, directory, priority, + fileRegister, originConnection)}).first; - return m_Origins[newID]; + m_OriginsNameMap.insert({originName, newID}); + m_OriginsPriorityMap.insert({priority, newID}); + + return itor->second; } bool exists(const std::wstring &name) @@ -597,9 +600,13 @@ std::vector FilesOrigin::getFiles() const { std::vector result; - for (FileEntry::Index fileIdx : m_Files) { - if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { - result.push_back(p); + { + std::scoped_lock lock(m_Mutex); + + for (FileEntry::Index fileIdx : m_Files) { + if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { + result.push_back(p); + } } } @@ -621,9 +628,16 @@ void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) { if (!enabled) { ++stats.originsNeededEnabled; - std::set copy = m_Files; + + std::set copy; + + { + std::scoped_lock lock(m_Mutex); + copy = m_Files; + m_Files.clear(); + } + m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter); - m_Files.clear(); } m_Disabled = !enabled; @@ -631,6 +645,8 @@ void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) void FilesOrigin::removeFile(FileEntry::Index index) { + std::scoped_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { @@ -640,6 +656,8 @@ void FilesOrigin::removeFile(FileEntry::Index index) bool FilesOrigin::containsArchive(std::wstring archiveName) { + std::scoped_lock lock(m_Mutex); + for (FileEntry::Index fileIdx : m_Files) { if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { if (p->isFromArchive(archiveName)) { @@ -665,6 +683,7 @@ FileRegister::~FileRegister() bool FileRegister::indexValid(FileEntry::Index index) const { + std::scoped_lock lock(m_Mutex); return (m_Files.find(index) != m_Files.end()); } @@ -675,6 +694,8 @@ FileEntry::Ptr FileRegister::createFile( FileEntry::Ptr p; stats.addFileToRegisterTimes += elapsed([&]{ + std::scoped_lock lock(m_Mutex); + auto r = m_Files.insert_or_assign( index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); @@ -692,6 +713,8 @@ FileEntry::Ptr FileRegister::createFile( FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const { + std::scoped_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { @@ -703,6 +726,8 @@ FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const bool FileRegister::removeFile(FileEntry::Index index) { + std::scoped_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { @@ -717,12 +742,15 @@ bool FileRegister::removeFile(FileEntry::Index index) void FileRegister::removeOrigin(FileEntry::Index index, int originID) { + std::unique_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { if (iter->second->removeOrigin(originID)) { - unregisterFile(iter->second); m_Files.erase(iter); + lock.unlock(); + unregisterFile(iter->second); } } else { log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index); @@ -734,17 +762,21 @@ void FileRegister::removeOriginMulti( { std::vector removedFiles; - for (auto iter = indices.begin(); iter != indices.end(); ) { - auto pos = m_Files.find(*iter); + { + std::scoped_lock lock(m_Mutex); + + for (auto iter = indices.begin(); iter != indices.end(); ) { + auto pos = m_Files.find(*iter); - if (pos != m_Files.end() - && (pos->second->lastAccessed() < notAfter) - && pos->second->removeOrigin(originID)) { - removedFiles.push_back(pos->second); - m_Files.erase(pos); - ++iter; - } else { - indices.erase(iter++); + if (pos != m_Files.end() + && (pos->second->lastAccessed() < notAfter) + && pos->second->removeOrigin(originID)) { + removedFiles.push_back(pos->second); + m_Files.erase(pos); + ++iter; + } else { + indices.erase(iter++); + } } } @@ -776,6 +808,8 @@ void FileRegister::removeOriginMulti( void FileRegister::sortOrigins() { + std::scoped_lock lock(m_Mutex); + for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) { iter->second->sortOrigins(); } @@ -1228,6 +1262,8 @@ FileEntry::Ptr DirectoryEntry::insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, std::wstring_view archive, int order) { + std::scoped_lock lock(m_FilesMutex); + std::wstring fileNameLower = ToLowerCopy(fileName); auto iter = m_Files.find(fileNameLower); @@ -1255,6 +1291,8 @@ FileEntry::Ptr DirectoryEntry::insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats) { + std::scoped_lock lock(m_FilesMutex); + FilesMap::iterator itor; stats.filesLookupTimes += elapsed([&]{ @@ -1307,7 +1345,12 @@ void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOf [](void* pcx, std::wstring_view path) { Context* cx = (Context*)pcx; auto* current= cx->current.top(); - std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); + + { + std::scoped_lock lock(current->m_SubDirMutex); + std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); + } + cx->current.pop(); }, @@ -1389,6 +1432,8 @@ void DirectoryEntry::addFiles( DirectoryEntry *DirectoryEntry::getSubDirectory( std::wstring_view name, bool create, int originID) { + std::scoped_lock lock(m_SubDirMutex); + std::wstring nameLc = ToLowerCopy(name); auto itor = m_SubDirectoriesLookup.find(nameLc); diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index c8ec8a4e..71e8b32c 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -26,6 +26,8 @@ along with Mod Organizer. If not, see . #include #include #include +#include + #define WIN32_MEAN_AND_LEAN #include #include @@ -274,6 +276,7 @@ public: void addFile(FileEntry::Index index) { + std::scoped_lock lock(m_Mutex); m_Files.insert(index); } @@ -290,6 +293,7 @@ private: int m_Priority; boost::weak_ptr m_FileRegister; boost::weak_ptr m_OriginConnection; + mutable std::mutex m_Mutex; FilesOrigin( int ID, const std::wstring &name, const std::wstring &path, int priority, @@ -313,6 +317,7 @@ public: size_t size() const { + std::scoped_lock lock(m_Mutex); return m_Files.size(); } @@ -323,6 +328,7 @@ public: void sortOrigins(); private: + mutable std::mutex m_Mutex; std::map m_Files; boost::shared_ptr m_OriginConnection; @@ -545,6 +551,8 @@ private: std::set m_Origins; bool m_Populated; bool m_TopLevel; + std::mutex m_SubDirMutex; + std::mutex m_FilesMutex; DirectoryEntry(const DirectoryEntry &reference); -- cgit v1.3.1 From 763a5d6c08006c319ed92f4088a4d3c211f80cf6 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Thu, 13 Feb 2020 23:03:56 -0500 Subject: more instrumentation tighter mutexes, required thread-safe FileEntry --- src/directoryrefresher.cpp | 106 ++++++++++------ src/envfs.cpp | 2 + src/mainwindow.cpp | 6 +- src/shared/directoryentry.cpp | 282 +++++++++++++++++++++++------------------- src/shared/directoryentry.h | 18 +-- 5 files changed, 241 insertions(+), 173 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index bce3f65a..20d6a52b 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -160,12 +160,14 @@ void DirectoryRefresher::addModFilesToStructure( TimeThis tt("addModFilesToStructure()"); std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + DirectoryStats dummy; if (stealFiles.length() > 0) { stealModFilesIntoStructure( directoryStructure, modName, priority, directory, stealFiles); } else { - directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); + directoryStructure->addFromOrigin( + ToWString(modName), directoryW, priority, dummy); } } @@ -178,16 +180,20 @@ void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure { TimeThis tt("addModToStructure()"); + DirectoryStats dummy; + if (stealFiles.length() > 0) { stealModFilesIntoStructure( directoryStructure, modName, priority, directory, stealFiles); } else { std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); - directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); + directoryStructure->addFromOrigin( + ToWString(modName), directoryW, priority, dummy); } if (Settings::instance().archiveParsing()) { - addModBSAToStructure(directoryStructure, modName, priority, directory, archives); + addModBSAToStructure( + directoryStructure, modName, priority, directory, archives); } } @@ -198,6 +204,7 @@ struct ModThread std::wstring path; int prio = -1; env::Directory* dir = nullptr; + DirectoryStats* stats = nullptr; std::condition_variable cv; std::mutex mutex; @@ -214,7 +221,8 @@ struct ModThread std::unique_lock lock(mutex); cv.wait(lock, [&]{ return ready; }); - ds->addFromOrigin(modName, path, prio); + SetThisThreadName(QString::fromStdWString(modName + L" refresher")); + ds->addFromOrigin(modName, path, prio, *stats); /*if (Settings::instance().archiveParsing()) { addModBSAToStructure( @@ -229,11 +237,39 @@ struct ModThread } }; +void dumpStats(std::vector& stats) +{ + static int run = 0; + static const std::string file("c:\\tmp\\data.csv"); + + if (run == 0) { + std::ofstream out(file, std::ios::out|std::ios::trunc); + out << fmt::format("what,run,{}", DirectoryStats::csvHeader()) << "\n"; + } + + std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){ + return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0); + }); + + std::ofstream out(file, std::ios::app); + + DirectoryStats total; + for (const auto& s : stats) { + out << fmt::format("{},{},{}", s.mod, run, s.toCsv()) << "\n"; + total += s; + } + + out << fmt::format("total,{},{}", run, total.toCsv()) << "\n"; + + ++run; +} + void DirectoryRefresher::addMultipleModsFilesToStructure( MOShared::DirectoryEntry *directoryStructure, const std::vector& entries, bool emitProgress) { std::vector dirs(entries.size()); + std::vector stats(entries.size()); { TimeThis tt("walk dirs"); @@ -257,6 +293,9 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); mt.prio = prio; mt.dir = &dirs[i]; + mt.stats = &stats[i]; + + stats[i].mod = entries[i].modName.toStdString(); mt.wakeup(); } @@ -272,23 +311,7 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( threads.join(); } - //std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){ - // return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0); - //}); - - //static int run = 1; - // - //std::ofstream out("c:\\tmp\\data.csv", std::ios::app); - - //out << fmt::format("what,run,{}", DirectoryStats::csvHeader()); - // - //for (std::size_t i=0; iproperty("managed_game").value(); + IPluginGame *game = qApp->property("managed_game").value(); - std::wstring dataDirectory = - QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + std::wstring dataDirectory = + QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); - m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0); + { + DirectoryStats dummy; + m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0, dummy); + } - std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { - return lhs.priority < rhs.priority; - }); + std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { + return lhs.priority < rhs.priority; + }); - addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); + addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); - m_DirectoryStructure->getFileRegister()->sortOrigins(); + m_DirectoryStructure->getFileRegister()->sortOrigins(); - emit progress(100); + emit progress(100); - cleanStructure(m_DirectoryStructure); + cleanStructure(m_DirectoryStructure); - emit refreshed(); + emit refreshed(); - //logcounts("after refresh"); + //logcounts("after refresh"); + } } diff --git a/src/envfs.cpp b/src/envfs.cpp index 022ca513..5cd36957 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -179,6 +179,8 @@ public: void run() { + MOShared::SetThisThreadName("HandleCloserThread"); + std::unique_lock lock(m_mutex); m_cv.wait(lock, [&]{ return m_ready; }); diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index 3e03c52d..2145011f 100644 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -5148,7 +5148,11 @@ void MainWindow::originModified(int originID) { FilesOrigin &origin = m_OrganizerCore.directoryStructure()->getOriginByID(originID); origin.enable(false); - m_OrganizerCore.directoryStructure()->addFromOrigin(origin.getName(), origin.getPath(), origin.getPriority()); + + DirectoryStats dummy; + m_OrganizerCore.directoryStructure()->addFromOrigin( + origin.getName(), origin.getPath(), origin.getPriority(), dummy); + DirectoryRefresher::cleanStructure(m_OrganizerCore.directoryStructure()); } diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index e1464595..19500167 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -160,19 +160,23 @@ std::string DirectoryStats::toCsv() const { QStringList oss; + auto s = [](auto ns) { + return ns.count() / 1000.0 / 1000.0 / 1000.0; + }; + oss - << QString::number(dirTimes.count()) - << QString::number(fileTimes.count()) - << QString::number(sortTimes.count()) + << QString::number(s(dirTimes)) + << QString::number(s(fileTimes)) + << QString::number(s(sortTimes)) - << QString::number(subdirLookupTimes.count()) - << QString::number(addDirectoryTimes.count()) + << QString::number(s(subdirLookupTimes)) + << QString::number(s(addDirectoryTimes)) - << QString::number(filesLookupTimes.count()) - << QString::number(addFileTimes.count()) - << QString::number(addOriginToFileTimes.count()) - << QString::number(addFileToOriginTimes.count()) - << QString::number(addFileToRegisterTimes.count()) + << QString::number(s(filesLookupTimes)) + << QString::number(s(addFileTimes)) + << QString::number(s(addOriginToFileTimes)) + << QString::number(s(addFileToOriginTimes)) + << QString::number(s(addFileToRegisterTimes)) << QString::number(originExists) << QString::number(originCreate) @@ -323,6 +327,8 @@ FileEntry::~FileEntry() void FileEntry::addOrigin( int origin, FILETIME fileTime, std::wstring_view archive, int order) { + std::scoped_lock lock(m_OriginsMutex); + m_LastAccessed = time(nullptr); if (m_Parent != nullptr) { m_Parent->propagateOrigin(origin); @@ -387,6 +393,8 @@ void FileEntry::addOrigin( bool FileEntry::removeOrigin(int origin) { + std::scoped_lock lock(m_OriginsMutex); + if (m_Origin == origin) { if (!m_Alternatives.empty()) { // find alternative with the highest priority @@ -440,6 +448,8 @@ bool FileEntry::removeOrigin(int origin) void FileEntry::sortOrigins() { + std::scoped_lock lock(m_OriginsMutex); + m_Alternatives.push_back({m_Origin, m_Archive}); std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) { @@ -480,6 +490,8 @@ void FileEntry::sortOrigins() bool FileEntry::isFromArchive(std::wstring archiveName) const { + std::scoped_lock lock(m_OriginsMutex); + if (archiveName.length() == 0) { return m_Archive.first.length() != 0; } @@ -499,6 +511,8 @@ bool FileEntry::isFromArchive(std::wstring archiveName) const std::wstring FileEntry::getFullPath(int originID) const { + std::scoped_lock lock(m_OriginsMutex); + if (originID == -1) { bool ignore = false; originID = getOrigin(ignore); @@ -690,27 +704,34 @@ bool FileRegister::indexValid(FileEntry::Index index) const FileEntry::Ptr FileRegister::createFile( std::wstring name, DirectoryEntry *parent, DirectoryStats& stats) { - FileEntry::Index index = generateIndex(); + const auto index = generateIndex(); FileEntry::Ptr p; stats.addFileToRegisterTimes += elapsed([&]{ - std::scoped_lock lock(m_Mutex); + bool inserted = false; + p = FileEntry::Ptr(new FileEntry(index, std::move(name), parent)); - auto r = m_Files.insert_or_assign( - index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); + { + std::scoped_lock lock(m_Mutex); + inserted = m_Files.insert_or_assign(index, p).second; + } - if (r.second) { + if (inserted) { ++stats.filesInsertedInRegister; } else { ++stats.filesAssignedInRegister; } - - p = r.first->second; }); return p; } +FileEntry::Index FileRegister::generateIndex() +{ + static std::atomic sIndex(0); + return sIndex++; +} + FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const { std::scoped_lock lock(m_Mutex); @@ -815,12 +836,6 @@ void FileRegister::sortOrigins() } } -FileEntry::Index FileRegister::generateIndex() -{ - static std::atomic sIndex(0); - return sIndex++; -} - void FileRegister::unregisterFile(FileEntry::Ptr file) { bool ignore; @@ -881,17 +896,13 @@ void DirectoryEntry::clear() } void DirectoryEntry::addFromOrigin( - const std::wstring &originName, const std::wstring &directory, int priority) + const std::wstring &originName, const std::wstring &directory, int priority, + DirectoryStats& stats) { - DirectoryStats dummy; - FilesOrigin &origin = createOrigin(originName, directory, priority, dummy); + FilesOrigin &origin = createOrigin(originName, directory, priority, stats); - if (directory.length() != 0) { - boost::scoped_array buffer(new wchar_t[MAXPATH_UNICODE + 1]); - memset(buffer.get(), L'\0', MAXPATH_UNICODE + 1); - int offset = _snwprintf(buffer.get(), MAXPATH_UNICODE, L"%ls", directory.c_str()); - buffer.get()[offset] = L'\0'; - addFiles(origin, buffer.get(), offset); + if (!directory.empty()) { + addFiles(origin, directory, stats); } m_Populated = true; @@ -1260,58 +1271,80 @@ void DirectoryEntry::removeFiles(const std::set &indices) FileEntry::Ptr DirectoryEntry::insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, - std::wstring_view archive, int order) + std::wstring_view archive, int order, DirectoryStats& stats) { - std::scoped_lock lock(m_FilesMutex); - std::wstring fileNameLower = ToLowerCopy(fileName); + FileEntry::Ptr fe; - auto iter = m_Files.find(fileNameLower); - FileEntry::Ptr file; + FileKey key(std::move(fileNameLower)); - if (iter != m_Files.end()) { - file = m_FileRegister->getFile(iter->second); - } else { - DirectoryStats dummy; + { + std::unique_lock lock(m_FilesMutex); + + FilesLookup::iterator itor; + + stats.filesLookupTimes += elapsed([&]{ + itor = m_FilesLookup.find(key); + }); + + if (itor != m_FilesLookup.end()) { + lock.unlock(); + ++stats.fileExists; + fe = m_FileRegister->getFile(itor->second); + } else { + ++stats.fileCreate; + fe = m_FileRegister->createFile( + std::wstring(fileName.begin(), fileName.end()), this, stats); - file = m_FileRegister->createFile( - std::wstring(fileName.begin(), fileName.end()), this, dummy); + stats.addFileTimes += elapsed([&] { + addFileToList(std::move(key.value), fe->getIndex()); + }); - addFileToList(std::move(fileNameLower), file->getIndex()); - // fileNameLower has moved from this point + // fileNameLower has moved from this point + } } - file->addOrigin(origin.getID(), fileTime, archive, order); - origin.addFile(file->getIndex()); + stats.addOriginToFileTimes += elapsed([&]{ + fe->addOrigin(origin.getID(), fileTime, archive, order); + }); + + stats.addFileToOriginTimes += elapsed([&]{ + origin.addFile(fe->getIndex()); + }); - return file; + return fe; } FileEntry::Ptr DirectoryEntry::insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats) { - std::scoped_lock lock(m_FilesMutex); + FileEntry::Ptr fe; - FilesMap::iterator itor; + { + std::unique_lock lock(m_FilesMutex); - stats.filesLookupTimes += elapsed([&]{ - itor = m_Files.find(file.lcname); - }); + FilesMap::iterator itor; - FileEntry::Ptr fe; + stats.filesLookupTimes += elapsed([&]{ + itor = m_Files.find(file.lcname); + }); - if (itor != m_Files.end()) { - ++stats.fileExists; - fe = m_FileRegister->getFile(itor->second); - } else { - fe = m_FileRegister->createFile(std::move(file.name), this, stats); + if (itor != m_Files.end()) { + lock.unlock(); + ++stats.fileExists; + fe = m_FileRegister->getFile(itor->second); + } else { + ++stats.fileCreate; + fe = m_FileRegister->createFile(std::move(file.name), this, stats); + // file.name has been moved from this point - stats.addFileTimes += elapsed([&]{ - addFileToList(std::move(file.lcname), fe->getIndex()); - }); + stats.addFileTimes += elapsed([&]{ + addFileToList(std::move(file.lcname), fe->getIndex()); + }); - // both file.name and file.lcname have been moved from this point + // file.lcname has been moved from this point + } } stats.addOriginToFileTimes += elapsed([&]{ @@ -1325,91 +1358,75 @@ FileEntry::Ptr DirectoryEntry::insert( return fe; } -void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOffset) +void DirectoryEntry::addFiles( + FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats) { struct Context { FilesOrigin& origin; + DirectoryStats& stats; std::stack current; }; - Context cx = {origin}; + Context cx = {origin, stats}; cx.current.push(this); - env::forEachEntry(buffer, &cx, - [](void* pcx, std::wstring_view path) { - Context* cx = (Context*)pcx; - cx->current.push(cx->current.top()->getSubDirectory(path, true, cx->origin.getID())); - }, - - [](void* pcx, std::wstring_view path) { + env::forEachEntry(path, &cx, + [](void* pcx, std::wstring_view path) + { Context* cx = (Context*)pcx; - auto* current= cx->current.top(); + cx->stats.dirTimes += elapsed([&] { + auto* sd = cx->current.top()->getSubDirectory( + path, true, cx->stats, cx->origin.getID()); - { - std::scoped_lock lock(current->m_SubDirMutex); - std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); - } - - cx->current.pop(); + cx->current.push(sd); + }); }, - [](void* pcx, std::wstring_view path, FILETIME ft) { + [](void* pcx, std::wstring_view path) + { Context* cx = (Context*)pcx; - cx->current.top()->insert(path, cx->origin, ft, L"", -1); - } - ); - - /* - WIN32_FIND_DATAW findData; - _snwprintf_s(buffer + bufferOffset, MAXPATH_UNICODE - bufferOffset, _TRUNCATE, L"\\*"); + cx->stats.dirTimes += elapsed([&] { + auto* current= cx->current.top(); - HANDLE searchHandle = nullptr; - - if (SupportOptimizedFind()) { - searchHandle = ::FindFirstFileExW( - buffer, FindExInfoBasic, &findData, FindExSearchNameMatch, nullptr, - FIND_FIRST_EX_LARGE_FETCH); - } else { - searchHandle = ::FindFirstFileExW( - buffer, FindExInfoStandard, &findData, FindExSearchNameMatch, nullptr, 0); - } + { + std::scoped_lock lock(current->m_SubDirMutex); - if (searchHandle != INVALID_HANDLE_VALUE) { - BOOL result = true; + std::sort( + current->m_SubDirectories.begin(), + current->m_SubDirectories.end(), + &DirCompareByName); + } - while (result) { - if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - if ((wcscmp(findData.cFileName, L".") != 0) && - (wcscmp(findData.cFileName, L"..") != 0)) { - int offset = _snwprintf(buffer + bufferOffset, MAXPATH_UNICODE, L"\\%ls", findData.cFileName); + cx->current.pop(); + }); + }, - // recurse into subdirectories - DirectoryEntry* sd = getSubDirectory(findData.cFileName, true, origin.getID()); - sd->addFiles(origin, buffer, bufferOffset + offset); - } - } else { - insert(findData.cFileName, origin, findData.ftLastWriteTime, L"", -1); - } + [](void* pcx, std::wstring_view path, FILETIME ft) + { + Context* cx = (Context*)pcx; - result = ::FindNextFileW(searchHandle, &findData); + cx->stats.fileTimes += elapsed([&]{ + cx->current.top()->insert(path, cx->origin, ft, L"", -1, cx->stats); + }); } - } - - std::sort(m_SubDirectories.begin(), m_SubDirectories.end(), &DirCompareByName); - ::FindClose(searchHandle);*/ + ); } void DirectoryEntry::addFiles( FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, const std::wstring &archiveName, int order) { + DirectoryStats dummy; + // add files for (unsigned int fileIdx = 0; fileIdx < archiveFolder->getNumFiles(); ++fileIdx) { BSA::File::Ptr file = archiveFolder->getFile(fileIdx); - auto f = insert(ToWString(file->getName(), true), origin, fileTime, archiveName, order); + auto f = insert( + ToWString(file->getName(), true), origin, fileTime, + archiveName, order, dummy); if (f) { if (file->getUncompressedFileSize() > 0) { @@ -1423,31 +1440,41 @@ void DirectoryEntry::addFiles( // recurse into subdirectories for (unsigned int folderIdx = 0; folderIdx < archiveFolder->getNumSubFolders(); ++folderIdx) { BSA::Folder::Ptr folder = archiveFolder->getSubFolder(folderIdx); - DirectoryEntry *folderEntry = getSubDirectoryRecursive(ToWString(folder->getName(), true), true, origin.getID()); + DirectoryEntry *folderEntry = getSubDirectoryRecursive( + ToWString(folder->getName(), true), true, origin.getID()); folderEntry->addFiles(origin, folder, fileTime, archiveName, order); } } DirectoryEntry *DirectoryEntry::getSubDirectory( - std::wstring_view name, bool create, int originID) + std::wstring_view name, bool create, DirectoryStats& stats, int originID) { + std::wstring nameLc = ToLowerCopy(name); + std::scoped_lock lock(m_SubDirMutex); - std::wstring nameLc = ToLowerCopy(name); - auto itor = m_SubDirectoriesLookup.find(nameLc); + SubDirectoriesLookup::iterator itor; + stats.subdirLookupTimes += elapsed([&] { + itor = m_SubDirectoriesLookup.find(nameLc); + }); if (itor != m_SubDirectoriesLookup.end()) { + ++stats.subdirExists; return itor->second; } if (create) { + ++stats.subdirCreate; + auto* entry = new DirectoryEntry( std::wstring(name.begin(), name.end()), this, originID, m_FileRegister, m_OriginConnection); - addDirectoryToList(entry, std::move(nameLc)); - // nameLc is moved from this point + stats.addDirectoryTimes += elapsed([&] { + addDirectoryToList(entry, std::move(nameLc)); + // nameLc is moved from this point + }); return entry; } else { @@ -1498,16 +1525,19 @@ DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive( } const size_t pos = path.find_first_of(L"\\/"); + DirectoryStats dummy; if (pos == std::wstring::npos) { - return getSubDirectory(path, create); + return getSubDirectory(path, create, dummy); } else { - DirectoryEntry *nextChild = getSubDirectory(path.substr(0, pos), create, originID); + DirectoryEntry *nextChild = getSubDirectory( + path.substr(0, pos), create, dummy, originID); if (nextChild == nullptr) { return nullptr; } else { - return nextChild->getSubDirectoryRecursive(path.substr(pos + 1), create, originID); + return nextChild->getSubDirectoryRecursive( + path.substr(pos + 1), create, originID); } } } diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 71e8b32c..8f6afbb1 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -221,6 +221,7 @@ private: DirectoryEntry *m_Parent; mutable FILETIME m_FileTime; uint64_t m_FileSize, m_CompressedFileSize; + mutable std::mutex m_OriginsMutex; time_t m_LastAccessed; @@ -328,12 +329,14 @@ public: void sortOrigins(); private: + using FileMap = std::map; + mutable std::mutex m_Mutex; - std::map m_Files; + FileMap m_Files; boost::shared_ptr m_OriginConnection; - FileEntry::Index generateIndex(); void unregisterFile(FileEntry::Ptr file); + FileEntry::Index generateIndex(); }; @@ -354,7 +357,7 @@ struct DirectoryEntryFileKey return std::hash()(value); } - const std::wstring value; + std::wstring value; const std::size_t hash; }; @@ -405,7 +408,7 @@ public: // That origin may exist or not void addFromOrigin( const std::wstring &originName, - const std::wstring &directory, int priority); + const std::wstring &directory, int priority, DirectoryStats& stats); void addFromBSA( const std::wstring &originName, std::wstring &directory, @@ -559,14 +562,14 @@ private: FileEntry::Ptr insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, - std::wstring_view archive, int order); + std::wstring_view archive, int order, DirectoryStats& stats); FileEntry::Ptr insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats); void addFiles( - FilesOrigin &origin, wchar_t *buffer, int bufferOffset); + FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats); void addFiles( FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, @@ -575,7 +578,8 @@ private: void addDir(FilesOrigin& origin, env::Directory& d, DirectoryStats& stats); DirectoryEntry* getSubDirectory( - std::wstring_view name, bool create, int originID = -1); + std::wstring_view name, bool create, DirectoryStats& stats, + int originID = -1); DirectoryEntry* getSubDirectory( env::Directory& dir, bool create, DirectoryStats& stats, -- cgit v1.3.1 From c8fc7abade6e28507f2a1590007a098e647026a9 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Fri, 14 Feb 2020 01:28:50 -0500 Subject: thread-safe OriginConnection ThreadPool now keeps threads running keep ModThreads around so avoid reallocating buffers --- src/directoryrefresher.cpp | 37 +++++++++------- src/envfs.cpp | 24 +++++----- src/envfs.h | 96 +++++++++++++++++++++++++++++++++------ src/shared/directoryentry.cpp | 101 +++++++++++++++++++++++++++++++++--------- src/shared/directoryentry.h | 8 +++- 5 files changed, 202 insertions(+), 64 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index 20d6a52b..a59ddf9d 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -203,8 +203,8 @@ struct ModThread std::wstring modName; std::wstring path; int prio = -1; - env::Directory* dir = nullptr; DirectoryStats* stats = nullptr; + env::DirectoryWalker walker; std::condition_variable cv; std::mutex mutex; @@ -212,7 +212,11 @@ struct ModThread void wakeup() { - ready = true; + { + std::scoped_lock lock(mutex); + ready = true; + } + cv.notify_one(); } @@ -222,7 +226,7 @@ struct ModThread cv.wait(lock, [&]{ return ready; }); SetThisThreadName(QString::fromStdWString(modName + L" refresher")); - ds->addFromOrigin(modName, path, prio, *stats); + ds->addFromOrigin(walker, modName, path, prio, *stats); /*if (Settings::instance().archiveParsing()) { addModBSAToStructure( @@ -237,6 +241,9 @@ struct ModThread } }; +env::ThreadPool g_threads; + + void dumpStats(std::vector& stats) { static int run = 0; @@ -268,35 +275,33 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( MOShared::DirectoryEntry *directoryStructure, const std::vector& entries, bool emitProgress) { - std::vector dirs(entries.size()); std::vector stats(entries.size()); + g_threads.setMax(m_threadCount); + { TimeThis tt("walk dirs"); - env::ThreadPool threads(m_threadCount); - for (std::size_t i=0; i(i + 1); + stats[i].mod = entries[i].modName.toStdString(); + try { if (e.stealFiles.length() > 0) { stealModFilesIntoStructure( directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); } else { - auto& mt = threads.request(); + auto& mt = g_threads.request(); mt.ds = directoryStructure; mt.modName = entries[i].modName.toStdWString(); mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); mt.prio = prio; - mt.dir = &dirs[i]; mt.stats = &stats[i]; - stats[i].mod = entries[i].modName.toStdString(); - mt.wakeup(); } } catch (const std::exception& ex) { @@ -308,7 +313,7 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( } } - threads.join(); + g_threads.waitForAll(); } dumpStats(stats); @@ -347,12 +352,12 @@ void DirectoryRefresher::refresh() m_DirectoryStructure->getFileRegister()->sortOrigins(); - emit progress(100); - cleanStructure(m_DirectoryStructure); + } - emit refreshed(); + emit progress(100); - //logcounts("after refresh"); - } + emit refreshed(); + + //logcounts("after refresh"); } diff --git a/src/envfs.cpp b/src/envfs.cpp index 5cd36957..a67faf84 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -212,15 +212,6 @@ void setHandleCloserThreadCount(std::size_t n) g_handleClosers.setMax(n); } -void shrinkFs() -{ - g_handleClosers.join(); - - g_handleClosers.forEach([](auto&& t) { - t.shrink(); - }); -} - void forEachEntryImpl( void* cx, HandleCloserThread& hc, std::vector>& buffers, POBJECT_ATTRIBUTES poa, std::size_t depth, @@ -328,14 +319,13 @@ void forEachEntryImpl( } } -void forEachEntry( + +void DirectoryWalker::forEachEntry( const std::wstring& path, void* cx, DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) { auto& hc = g_handleClosers.request(); - std::vector> buffers; - if (!NtOpenFile) { LibraryPtr m(::LoadLibraryW(L"ntdll.dll")); NtOpenFile = (NtOpenFile_type)::GetProcAddress(m.get(), "NtOpenFile"); @@ -354,10 +344,18 @@ void forEachEntry( oa.Length = sizeof(oa); oa.ObjectName = &ObjectName; - forEachEntryImpl(cx, hc, buffers, &oa, 0, dirStartF, dirEndF, fileF); + forEachEntryImpl(cx, hc, m_buffers, &oa, 0, dirStartF, dirEndF, fileF); hc.wakeup(); } + +void forEachEntry( + const std::wstring& path, void* cx, + DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) +{ + DirectoryWalker().forEachEntry(path, cx, dirStartF, dirEndF, fileF); +} + Directory getFilesAndDirs(const std::wstring& path) { struct Context diff --git a/src/envfs.h b/src/envfs.h index 001f2d09..8790b071 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -33,13 +33,13 @@ class ThreadPool { public: ThreadPool(std::size_t max=1) - : m_threads(max) { + setMax(max); } ~ThreadPool() { - join(); + stopAndJoin(); } void setMax(std::size_t n) @@ -47,8 +47,13 @@ public: m_threads.resize(n); } - void join() + void stopAndJoin() { + for (auto& ti : m_threads) { + ti.stop = true; + ti.wakeup(); + } + for (auto& ti : m_threads) { if (ti.thread.joinable()) { ti.thread.join(); @@ -56,6 +61,26 @@ public: } } + void waitForAll() + { + for (;;) { + bool done = true; + + for (auto& ti : m_threads) { + if (ti.busy) { + done = false; + break; + } + } + + if (done) { + break; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + T& request() { if (m_threads.empty()) { @@ -67,15 +92,7 @@ public: bool expected = false; if (ti.busy.compare_exchange_strong(expected, true)) { - if (ti.thread.joinable()) { - ti.thread.join(); - } - - ti.thread = std::thread([&]{ - ti.o.run(); - ti.busy = false; - }); - + ti.wakeup(); return ti.o; } } @@ -98,6 +115,47 @@ private: std::thread thread; std::atomic busy; T o; + + std::condition_variable cv; + std::mutex mutex; + bool ready; + + std::atomic stop; + + ThreadInfo() + : busy(true), ready(false), stop(false) + { + thread = std::thread([&]{ run(); }); + } + + void wakeup() + { + { + std::scoped_lock lock(mutex); + ready = true; + } + + cv.notify_one(); + } + + void run() + { + busy = false; + + while (!stop) { + std::unique_lock lock(mutex); + cv.wait(lock, [&]{ return ready; }); + + if (stop) { + break; + } + + o.run(); + + ready = false; + busy = false; + } + } }; std::list m_threads; @@ -109,7 +167,19 @@ using DirEndF = void (void*, std::wstring_view); using FileF = void (void*, std::wstring_view, FILETIME); void setHandleCloserThreadCount(std::size_t n); -void shrinkFs(); + + +class DirectoryWalker +{ +public: + void forEachEntry( + const std::wstring& path, void* cx, + DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF); + +private: + std::vector> m_buffers; +}; + void forEachEntry( const std::wstring& path, void* cx, diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 19500167..4e64d1c1 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -211,35 +211,57 @@ public: --OriginConnectionCount; } + std::pair getOrCreate( + const std::wstring &originName, const std::wstring &directory, int priority, + const boost::shared_ptr& fileRegister, + const boost::shared_ptr& originConnection, + DirectoryStats& stats) + { + std::unique_lock lock(m_Mutex); + + auto itor = m_OriginsNameMap.find(originName); + + if (itor == m_OriginsNameMap.end()) { + FilesOrigin& origin = createOriginNoLock( + originName, directory, priority, fileRegister, originConnection); + + return {origin, true}; + } else { + FilesOrigin& origin = m_Origins[itor->second]; + lock.unlock(); + + origin.enable(true, stats); + return {origin, false}; + } + } + FilesOrigin& createOrigin( const std::wstring &originName, const std::wstring &directory, int priority, boost::shared_ptr fileRegister, boost::shared_ptr originConnection) { - int newID = createID(); - - auto itor = m_Origins.insert({newID, FilesOrigin( - newID, originName, directory, priority, - fileRegister, originConnection)}).first; - - m_OriginsNameMap.insert({originName, newID}); - m_OriginsPriorityMap.insert({priority, newID}); + std::scoped_lock lock(m_Mutex); - return itor->second; + return createOriginNoLock( + originName, directory, priority, fileRegister, originConnection); } bool exists(const std::wstring &name) { + std::scoped_lock lock(m_Mutex); return m_OriginsNameMap.find(name) != m_OriginsNameMap.end(); } FilesOrigin &getByID(Index ID) { + std::scoped_lock lock(m_Mutex); return m_Origins[ID]; } const FilesOrigin* findByID(Index ID) const { + std::scoped_lock lock(m_Mutex); + auto itor = m_Origins.find(ID); if (itor == m_Origins.end()) { @@ -251,6 +273,8 @@ public: FilesOrigin &getByName(const std::wstring &name) { + std::scoped_lock lock(m_Mutex); + std::map::iterator iter = m_OriginsNameMap.find(name); if (iter != m_OriginsNameMap.end()) { @@ -264,6 +288,8 @@ public: void changePriorityLookup(int oldPriority, int newPriority) { + std::scoped_lock lock(m_Mutex); + auto iter = m_OriginsPriorityMap.find(oldPriority); if (iter != m_OriginsPriorityMap.end()) { @@ -275,6 +301,8 @@ public: void changeNameLookup(const std::wstring &oldName, const std::wstring &newName) { + std::scoped_lock lock(m_Mutex); + auto iter = m_OriginsNameMap.find(oldName); if (iter != m_OriginsNameMap.end()) { @@ -291,11 +319,29 @@ private: std::map m_Origins; std::map m_OriginsNameMap; std::map m_OriginsPriorityMap; + mutable std::mutex m_Mutex; Index createID() { return m_NextID++; } + + FilesOrigin& createOriginNoLock( + const std::wstring &originName, const std::wstring &directory, int priority, + boost::shared_ptr fileRegister, + boost::shared_ptr originConnection) + { + int newID = createID(); + + auto itor = m_Origins.insert({newID, FilesOrigin( + newID, originName, directory, priority, + fileRegister, originConnection)}).first; + + m_OriginsNameMap.insert({originName, newID}); + m_OriginsPriorityMap.insert({priority, newID}); + + return itor->second; + } }; @@ -898,11 +944,19 @@ void DirectoryEntry::clear() void DirectoryEntry::addFromOrigin( const std::wstring &originName, const std::wstring &directory, int priority, DirectoryStats& stats) +{ + env::DirectoryWalker walker; + addFromOrigin(walker, originName, directory, priority, stats); +} + +void DirectoryEntry::addFromOrigin( + env::DirectoryWalker& walker, const std::wstring &originName, + const std::wstring &directory, int priority, DirectoryStats& stats) { FilesOrigin &origin = createOrigin(originName, directory, priority, stats); if (!directory.empty()) { - addFiles(origin, directory, stats); + addFiles(walker, origin, directory, stats); } m_Populated = true; @@ -993,7 +1047,10 @@ void DirectoryEntry::addFromBSA( void DirectoryEntry::propagateOrigin(int origin) { - m_Origins.insert(origin); + { + std::scoped_lock lock(m_OriginsMutex); + m_Origins.insert(origin); + } if (m_Parent != nullptr) { m_Parent->propagateOrigin(origin); @@ -1252,16 +1309,17 @@ FilesOrigin &DirectoryEntry::createOrigin( const std::wstring &originName, const std::wstring &directory, int priority, DirectoryStats& stats) { - if (m_OriginConnection->exists(originName)) { - ++stats.originExists; - FilesOrigin &origin = m_OriginConnection->getByName(originName); - origin.enable(true, stats); - return origin; - } else { + auto r = m_OriginConnection->getOrCreate( + originName, directory, priority, + m_FileRegister, m_OriginConnection, stats); + + if (r.second) { ++stats.originCreate; - return m_OriginConnection->createOrigin( - originName, directory, priority, m_FileRegister, m_OriginConnection); + } else { + ++stats.originExists; } + + return r.first; } void DirectoryEntry::removeFiles(const std::set &indices) @@ -1359,7 +1417,8 @@ FileEntry::Ptr DirectoryEntry::insert( } void DirectoryEntry::addFiles( - FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats) + env::DirectoryWalker& walker, FilesOrigin &origin, + const std::wstring& path, DirectoryStats& stats) { struct Context { @@ -1371,7 +1430,7 @@ void DirectoryEntry::addFiles( Context cx = {origin, stats}; cx.current.push(this); - env::forEachEntry(path, &cx, + walker.forEachEntry(path, &cx, [](void* pcx, std::wstring_view path) { Context* cx = (Context*)pcx; diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 8f6afbb1..2a70d486 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -410,6 +410,10 @@ public: const std::wstring &originName, const std::wstring &directory, int priority, DirectoryStats& stats); + void addFromOrigin( + env::DirectoryWalker& walker, const std::wstring &originName, + const std::wstring &directory, int priority, DirectoryStats& stats); + void addFromBSA( const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order); @@ -556,6 +560,7 @@ private: bool m_TopLevel; std::mutex m_SubDirMutex; std::mutex m_FilesMutex; + std::mutex m_OriginsMutex; DirectoryEntry(const DirectoryEntry &reference); @@ -569,7 +574,8 @@ private: std::wstring_view archive, int order, DirectoryStats& stats); void addFiles( - FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats); + env::DirectoryWalker& walker, FilesOrigin &origin, + const std::wstring& path, DirectoryStats& stats); void addFiles( FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, -- cgit v1.3.1 From 567fe019ac21ee78a07beae1a387ab5688e7f97b Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Fri, 14 Feb 2020 01:53:59 -0500 Subject: removed some unnecessary counters delete old structure in thread --- src/directoryrefresher.cpp | 69 ++++++++++++++++++------------------------- src/directoryrefresher.h | 2 +- src/organizercore.cpp | 68 +++++++++++++++++++++++++++++++----------- src/organizercore.h | 2 ++ src/shared/directoryentry.cpp | 48 ------------------------------ src/shared/directoryentry.h | 3 -- 6 files changed, 82 insertions(+), 110 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index a59ddf9d..cb5bfc5b 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -48,7 +48,7 @@ DirectoryRefresher::~DirectoryRefresher() delete m_DirectoryStructure; } -DirectoryEntry *DirectoryRefresher::getDirectoryStructure() +DirectoryEntry *DirectoryRefresher::stealDirectoryStructure() { QMutexLocker locker(&m_RefreshLock); DirectoryEntry *result = m_DirectoryStructure; @@ -279,58 +279,50 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( g_threads.setMax(m_threadCount); - { - TimeThis tt("walk dirs"); - - for (std::size_t i=0; i(i + 1); + for (std::size_t i=0; i(i + 1); - stats[i].mod = entries[i].modName.toStdString(); + stats[i].mod = entries[i].modName.toStdString(); - try - { - if (e.stealFiles.length() > 0) { - stealModFilesIntoStructure( - directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); - } else { - auto& mt = g_threads.request(); - - mt.ds = directoryStructure; - mt.modName = entries[i].modName.toStdWString(); - mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); - mt.prio = prio; - mt.stats = &stats[i]; - - mt.wakeup(); - } - } catch (const std::exception& ex) { - emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); - } - - if (emitProgress) { - emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + try + { + if (e.stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); + } else { + auto& mt = g_threads.request(); + + mt.ds = directoryStructure; + mt.modName = entries[i].modName.toStdWString(); + mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); + mt.prio = prio; + mt.stats = &stats[i]; + + mt.wakeup(); } + } catch (const std::exception& ex) { + emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); } - g_threads.waitForAll(); + if (emitProgress) { + emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + } } + g_threads.waitForAll(); + dumpStats(stats); } -namespace MOShared{ void logcounts(std::string w); } - void DirectoryRefresher::refresh() { SetThisThreadName("DirectoryRefresher"); + TimeThis tt("refresh"); for (int i=0; i<1; ++i) { QMutexLocker locker(&m_RefreshLock); - - //logcounts("before delete"); delete m_DirectoryStructure; - //logcounts("after delete"); m_DirectoryStructure = new DirectoryEntry(L"data", nullptr, 0); @@ -355,9 +347,6 @@ void DirectoryRefresher::refresh() cleanStructure(m_DirectoryStructure); } - emit progress(100); - + emit progress(100); emit refreshed(); - - //logcounts("after refresh"); } diff --git a/src/directoryrefresher.h b/src/directoryrefresher.h index c531ba39..18bfed4d 100644 --- a/src/directoryrefresher.h +++ b/src/directoryrefresher.h @@ -67,7 +67,7 @@ public: * * @return updated directory structure **/ - MOShared::DirectoryEntry *getDirectoryStructure(); + MOShared::DirectoryEntry* stealDirectoryStructure(); /** * @brief sets up the mods to be included in the directory structure diff --git a/src/organizercore.cpp b/src/organizercore.cpp index 86abeb35..a3202153 100644 --- a/src/organizercore.cpp +++ b/src/organizercore.cpp @@ -144,6 +144,10 @@ OrganizerCore::~OrganizerCore() m_RefresherThread.exit(); m_RefresherThread.wait(); + if (m_StructureDeleter.joinable()) { + m_StructureDeleter.join(); + } + saveCurrentProfile(); // profile has to be cleaned up before the modinfo-buffer is cleared @@ -1379,46 +1383,74 @@ std::vector OrganizerCore::enabledArchives() void OrganizerCore::refreshDirectoryStructure() { - if (!m_DirectoryUpdate) { - m_CurrentProfile->writeModlistNow(true); + if (m_DirectoryUpdate) { + log::debug("can't refresh, already in progress"); + return; + } - m_DirectoryUpdate = true; - std::vector> activeModList - = m_CurrentProfile->getActiveMods(); - auto archives = enabledArchives(); - m_DirectoryRefresher.setMods( - activeModList, std::set(archives.begin(), archives.end())); + log::debug("refreshing structure"); + m_DirectoryUpdate = true; - QTimer::singleShot(0, &m_DirectoryRefresher, SLOT(refresh())); - } + m_CurrentProfile->writeModlistNow(true); + const auto activeModList = m_CurrentProfile->getActiveMods(); + const auto archives = enabledArchives(); + + m_DirectoryRefresher.setMods( + activeModList, std::set(archives.begin(), archives.end())); + + // runs refresh() in a thread + QTimer::singleShot(0, &m_DirectoryRefresher, SLOT(refresh())); } void OrganizerCore::directory_refreshed() { - DirectoryEntry *newStructure = m_DirectoryRefresher.getDirectoryStructure(); + log::debug("structure refreshed"); + + DirectoryEntry *newStructure = m_DirectoryRefresher.stealDirectoryStructure(); Q_ASSERT(newStructure != m_DirectoryStructure); - if (newStructure != nullptr) { - std::swap(m_DirectoryStructure, newStructure); - delete newStructure; - } else { + + if (newStructure == nullptr) { // TODO: don't know why this happens, this slot seems to get called twice // with only one emit return; } + + std::swap(m_DirectoryStructure, newStructure); + + if (m_StructureDeleter.joinable()) { + m_StructureDeleter.join(); + } + + m_StructureDeleter = std::thread([=]{ + log::debug("structure deleter thread start"); + delete newStructure; + log::debug("structure deleter thread done"); + }); + m_DirectoryUpdate = false; + log::debug("clearing caches"); for (int i = 0; i < m_ModList.rowCount(); ++i) { ModInfo::Ptr modInfo = ModInfo::getByIndex(i); modInfo->clearCaches(); } - for (auto task : m_PostRefreshTasks) { - task(); + + if (!m_PostRefreshTasks.empty()) { + log::debug("running {} post refresh tasks", m_PostRefreshTasks.size()); + + for (auto task : m_PostRefreshTasks) { + task(); + } + + m_PostRefreshTasks.clear(); } - m_PostRefreshTasks.clear(); if (m_CurrentProfile != nullptr) { + log::debug("refreshing lists"); refreshLists(); } + + log::debug("refresh done"); } void OrganizerCore::profileRefresh() diff --git a/src/organizercore.h b/src/organizercore.h index a63dc959..223eb6cb 100644 --- a/src/organizercore.h +++ b/src/organizercore.h @@ -346,6 +346,8 @@ private: QThread m_RefresherThread; + std::thread m_StructureDeleter; + bool m_DirectoryUpdate; bool m_ArchivesInit; bool m_ArchiveParsing{ m_Settings.archiveParsing() }; diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 4e64d1c1..c5702026 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -40,13 +40,6 @@ namespace MOShared using namespace MOBase; static const int MAXPATH_UNICODE = 32767; - -static std::atomic FileEntryCount(0); -static std::atomic FilesOriginCount(0); -static std::atomic FileRegisterCount(0); -static std::atomic DirectoryEntryCount(0); -static std::atomic OriginConnectionCount(0); - template std::chrono::nanoseconds elapsed(F&& f) { @@ -56,13 +49,6 @@ std::chrono::nanoseconds elapsed(F&& f) return (end - start); } -void logcounts(std::string w) -{ - log::debug( - "{}: FileEntry={} FilesOrigin={} FileRegister={} DirectoryEntry={} OriginConnection={}", - w, FileEntryCount, FilesOriginCount, FileRegisterCount, DirectoryEntryCount, OriginConnectionCount); -} - static std::wstring tail(const std::wstring &source, const size_t count) { @@ -203,12 +189,6 @@ public: OriginConnection() : m_NextID(0) { - ++OriginConnectionCount; - } - - ~OriginConnection() - { - --OriginConnectionCount; } std::pair getOrCreate( @@ -350,7 +330,6 @@ FileEntry::FileEntry() : m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), m_LastAccessed(time(nullptr)) { - ++FileEntryCount; } FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : @@ -358,16 +337,6 @@ FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), m_LastAccessed(time(nullptr)) { - ++FileEntryCount; -} - -FileEntry::~FileEntry() -{ - while (!m_Alternatives.empty()) { - m_Alternatives.pop_back(); - } - - --FileEntryCount; } void FileEntry::addOrigin( @@ -606,7 +575,6 @@ bool FileEntry::recurseParents(std::wstring &path, const DirectoryEntry *parent) FilesOrigin::FilesOrigin() : m_ID(0), m_Disabled(false), m_Name(), m_Path(), m_Priority(0) { - ++FilesOriginCount; } FilesOrigin::FilesOrigin(const FilesOrigin &reference) @@ -618,7 +586,6 @@ FilesOrigin::FilesOrigin(const FilesOrigin &reference) , m_FileRegister(reference.m_FileRegister) , m_OriginConnection(reference.m_OriginConnection) { - ++FilesOriginCount; } FilesOrigin::FilesOrigin( @@ -629,12 +596,6 @@ FilesOrigin::FilesOrigin( m_Priority(priority), m_FileRegister(fileRegister), m_OriginConnection(originConnection) { - ++FilesOriginCount; -} - -FilesOrigin::~FilesOrigin() -{ - --FilesOriginCount; } void FilesOrigin::setPriority(int priority) @@ -733,12 +694,6 @@ bool FilesOrigin::containsArchive(std::wstring archiveName) FileRegister::FileRegister(boost::shared_ptr originConnection) : m_OriginConnection(originConnection) { - ++FileRegisterCount; -} - -FileRegister::~FileRegister() -{ - --FileRegisterCount; } bool FileRegister::indexValid(FileEntry::Index index) const @@ -907,7 +862,6 @@ DirectoryEntry::DirectoryEntry( m_OriginConnection(new OriginConnection), m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(true) { - ++DirectoryEntryCount; m_FileRegister.reset(new FileRegister(m_OriginConnection)); m_Origins.insert(originID); } @@ -919,13 +873,11 @@ DirectoryEntry::DirectoryEntry( m_FileRegister(fileRegister), m_OriginConnection(originConnection), m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(false) { - ++DirectoryEntryCount; m_Origins.insert(originID); } DirectoryEntry::~DirectoryEntry() { - --DirectoryEntryCount; clear(); } diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 2a70d486..94ea068d 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -121,7 +121,6 @@ public: FileEntry(); FileEntry(Index index, std::wstring name, DirectoryEntry *parent); - ~FileEntry(); Index getIndex() const { @@ -237,7 +236,6 @@ class FilesOrigin public: FilesOrigin(); FilesOrigin(const FilesOrigin &reference); - ~FilesOrigin(); // sets priority for this origin, but it will overwrite the existing mapping // for this priority, the previous origin will no longer be referenced @@ -307,7 +305,6 @@ class FileRegister { public: FileRegister(boost::shared_ptr originConnection); - ~FileRegister(); bool indexValid(FileEntry::Index index) const; -- cgit v1.3.1 From fb3fbd34211a28b27214abe070c46951057258ca Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Fri, 14 Feb 2020 03:19:21 -0500 Subject: removed lastAccessed from FileEntry, not used FileRegister keeps a vector instead of a map --- src/directoryrefresher.cpp | 13 ++- src/directoryrefresher.h | 1 + src/shared/directoryentry.cpp | 178 ++++++++++++++++++++++-------------------- src/shared/directoryentry.h | 25 +++--- 4 files changed, 118 insertions(+), 99 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index cb5bfc5b..f3285cfc 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -39,7 +39,7 @@ using namespace MOShared; DirectoryRefresher::DirectoryRefresher(std::size_t threadCount) - : m_DirectoryStructure(nullptr), m_threadCount(threadCount) + : m_DirectoryStructure(nullptr), m_threadCount(threadCount), m_lastFileCount(0) { } @@ -283,7 +283,9 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( const auto& e = entries[i]; const int prio = static_cast(i + 1); - stats[i].mod = entries[i].modName.toStdString(); + if constexpr (DirectoryStats::EnableInstrumentation) { + stats[i].mod = entries[i].modName.toStdString(); + } try { @@ -312,7 +314,9 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( g_threads.waitForAll(); - dumpStats(stats); + if constexpr (DirectoryStats::EnableInstrumentation) { + dumpStats(stats); + } } void DirectoryRefresher::refresh() @@ -325,6 +329,7 @@ void DirectoryRefresher::refresh() delete m_DirectoryStructure; m_DirectoryStructure = new DirectoryEntry(L"data", nullptr, 0); + m_DirectoryStructure->getFileRegister()->reserve(m_lastFileCount); IPluginGame *game = qApp->property("managed_game").value(); @@ -347,6 +352,8 @@ void DirectoryRefresher::refresh() cleanStructure(m_DirectoryStructure); } + m_lastFileCount = m_DirectoryStructure->getFileRegister()->highestCount(); + emit progress(100); emit refreshed(); } diff --git a/src/directoryrefresher.h b/src/directoryrefresher.h index 18bfed4d..5c829980 100644 --- a/src/directoryrefresher.h +++ b/src/directoryrefresher.h @@ -145,6 +145,7 @@ private: MOShared::DirectoryEntry *m_DirectoryStructure; QMutex m_RefreshLock; std::size_t m_threadCount; + std::size_t m_lastFileCount; void stealModFilesIntoStructure( MOShared::DirectoryEntry *directoryStructure, const QString &modName, diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index c5702026..0be5e870 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -38,17 +38,26 @@ namespace MOShared { using namespace MOBase; -static const int MAXPATH_UNICODE = 32767; +const int MAXPATH_UNICODE = 32767; template -std::chrono::nanoseconds elapsed(F&& f) +void elapsedImpl(std::chrono::nanoseconds& out, F&& f) { - const auto start = std::chrono::high_resolution_clock::now(); - f(); - const auto end = std::chrono::high_resolution_clock::now(); - return (end - start); + if constexpr (DirectoryStats::EnableInstrumentation) { + const auto start = std::chrono::high_resolution_clock::now(); + f(); + const auto end = std::chrono::high_resolution_clock::now(); + out += (end - start); + } else { + f(); + } } +// elapsed() is not optimized out when EnableInstrumentation is false even +// though it's equivalent that this macro +//#define elapsed(OUT, F) (F)(); +#define elapsed(OUT, F) elapsedImpl(OUT, F); + static std::wstring tail(const std::wstring &source, const size_t count) { @@ -327,15 +336,13 @@ private: FileEntry::FileEntry() : m_Index(UINT_MAX), m_Name(), m_Origin(-1), m_Parent(nullptr), - m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), - m_LastAccessed(time(nullptr)) + m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize) { } FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : m_Index(index), m_Name(std::move(name)), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), - m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), - m_LastAccessed(time(nullptr)) + m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize) { } @@ -344,7 +351,6 @@ void FileEntry::addOrigin( { std::scoped_lock lock(m_OriginsMutex); - m_LastAccessed = time(nullptr); if (m_Parent != nullptr) { m_Parent->propagateOrigin(origin); } @@ -639,13 +645,13 @@ FileEntry::Ptr FilesOrigin::findFile(FileEntry::Index index) const return m_FileRegister.lock()->getFile(index); } -void FilesOrigin::enable(bool enabled, time_t notAfter) +void FilesOrigin::enable(bool enabled) { DirectoryStats dummy; - enable(enabled, dummy, notAfter); + enable(enabled, dummy); } -void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) +void FilesOrigin::enable(bool enabled, DirectoryStats& stats) { if (!enabled) { ++stats.originsNeededEnabled; @@ -658,7 +664,7 @@ void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) m_Files.clear(); } - m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter); + m_FileRegister.lock()->removeOriginMulti(copy, m_ID); } m_Disabled = !enabled; @@ -692,57 +698,53 @@ bool FilesOrigin::containsArchive(std::wstring archiveName) FileRegister::FileRegister(boost::shared_ptr originConnection) - : m_OriginConnection(originConnection) + : m_OriginConnection(originConnection), m_NextIndex(0) { } bool FileRegister::indexValid(FileEntry::Index index) const { std::scoped_lock lock(m_Mutex); - return (m_Files.find(index) != m_Files.end()); + + if (index < m_Files.size()) { + return (m_Files[index].get() != nullptr); + } + + return false; } FileEntry::Ptr FileRegister::createFile( std::wstring name, DirectoryEntry *parent, DirectoryStats& stats) { const auto index = generateIndex(); - FileEntry::Ptr p; + auto p = FileEntry::Ptr(new FileEntry(index, std::move(name), parent)); - stats.addFileToRegisterTimes += elapsed([&]{ - bool inserted = false; - p = FileEntry::Ptr(new FileEntry(index, std::move(name), parent)); + { + std::scoped_lock lock(m_Mutex); - { - std::scoped_lock lock(m_Mutex); - inserted = m_Files.insert_or_assign(index, p).second; + if (index >= m_Files.size()) { + m_Files.resize(index + 1); } - if (inserted) { - ++stats.filesInsertedInRegister; - } else { - ++stats.filesAssignedInRegister; - } - }); + m_Files[index] = p; + } return p; } FileEntry::Index FileRegister::generateIndex() { - static std::atomic sIndex(0); - return sIndex++; + return m_NextIndex++; } FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const { std::scoped_lock lock(m_Mutex); - auto iter = m_Files.find(index); - - if (iter != m_Files.end()) { - return iter->second; + if (index < m_Files.size()) { + return m_Files[index]; } else { - return FileEntry::Ptr(); + return {}; } } @@ -750,37 +752,42 @@ bool FileRegister::removeFile(FileEntry::Index index) { std::scoped_lock lock(m_Mutex); - auto iter = m_Files.find(index); + if (index < m_Files.size()) { + FileEntry::Ptr p; + m_Files[index].swap(p); - if (iter != m_Files.end()) { - unregisterFile(iter->second); - m_Files.erase(index); - return true; - } else { - log::error(QObject::tr("invalid file index for remove: {}").toStdString(), index); - return false; + if (p) { + unregisterFile(p); + return true; + } } + + log::error(QObject::tr("invalid file index for remove: {}").toStdString(), index); + return false; } void FileRegister::removeOrigin(FileEntry::Index index, int originID) { std::unique_lock lock(m_Mutex); - auto iter = m_Files.find(index); + if (index < m_Files.size()) { + FileEntry::Ptr& p = m_Files[index]; - if (iter != m_Files.end()) { - if (iter->second->removeOrigin(originID)) { - m_Files.erase(iter); - lock.unlock(); - unregisterFile(iter->second); + if (p) { + if (p->removeOrigin(originID)) { + m_Files[index] = {}; + lock.unlock(); + unregisterFile(p); + return; + } } - } else { - log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index); } + + log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index); } void FileRegister::removeOriginMulti( - std::set indices, int originID, time_t notAfter) + std::set indices, int originID) { std::vector removedFiles; @@ -788,17 +795,20 @@ void FileRegister::removeOriginMulti( std::scoped_lock lock(m_Mutex); for (auto iter = indices.begin(); iter != indices.end(); ) { - auto pos = m_Files.find(*iter); - - if (pos != m_Files.end() - && (pos->second->lastAccessed() < notAfter) - && pos->second->removeOrigin(originID)) { - removedFiles.push_back(pos->second); - m_Files.erase(pos); - ++iter; - } else { - indices.erase(iter++); + const auto index = *iter; + + if (index < m_Files.size()) { + const auto& p = m_Files[index]; + + if (p && p->removeOrigin(originID)) { + removedFiles.push_back(p); + m_Files[index] = {}; + ++iter; + continue; + } } + + iter = indices.erase(iter); } } @@ -832,8 +842,8 @@ void FileRegister::sortOrigins() { std::scoped_lock lock(m_Mutex); - for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) { - iter->second->sortOrigins(); + for (auto&& p : m_Files) { + p->sortOrigins(); } } @@ -927,20 +937,20 @@ void DirectoryEntry::addFromList( void DirectoryEntry::addDir( FilesOrigin& origin, env::Directory& d, DirectoryStats& stats) { - stats.dirTimes += elapsed([&]{ + elapsed(stats.dirTimes, [&]{ for (auto& sd : d.dirs) { auto* sdirEntry = getSubDirectory(sd, true, stats, origin.getID()); sdirEntry->addDir(origin, sd, stats); } }); - stats.fileTimes += elapsed([&]{ + elapsed(stats.fileTimes, [&]{ for (auto& f : d.files) { insert(f, origin, L"", -1, stats); } }); - stats.sortTimes += elapsed([&]{ + elapsed(stats.sortTimes, [&]{ std::sort( m_SubDirectories.begin(), m_SubDirectories.end(), @@ -1293,7 +1303,7 @@ FileEntry::Ptr DirectoryEntry::insert( FilesLookup::iterator itor; - stats.filesLookupTimes += elapsed([&]{ + elapsed(stats.filesLookupTimes, [&]{ itor = m_FilesLookup.find(key); }); @@ -1306,7 +1316,7 @@ FileEntry::Ptr DirectoryEntry::insert( fe = m_FileRegister->createFile( std::wstring(fileName.begin(), fileName.end()), this, stats); - stats.addFileTimes += elapsed([&] { + elapsed(stats.addFileTimes, [&] { addFileToList(std::move(key.value), fe->getIndex()); }); @@ -1314,11 +1324,11 @@ FileEntry::Ptr DirectoryEntry::insert( } } - stats.addOriginToFileTimes += elapsed([&]{ + elapsed(stats.addOriginToFileTimes, [&]{ fe->addOrigin(origin.getID(), fileTime, archive, order); }); - stats.addFileToOriginTimes += elapsed([&]{ + elapsed(stats.addFileToOriginTimes, [&]{ origin.addFile(fe->getIndex()); }); @@ -1336,7 +1346,7 @@ FileEntry::Ptr DirectoryEntry::insert( FilesMap::iterator itor; - stats.filesLookupTimes += elapsed([&]{ + elapsed(stats.filesLookupTimes, [&]{ itor = m_Files.find(file.lcname); }); @@ -1349,7 +1359,7 @@ FileEntry::Ptr DirectoryEntry::insert( fe = m_FileRegister->createFile(std::move(file.name), this, stats); // file.name has been moved from this point - stats.addFileTimes += elapsed([&]{ + elapsed(stats.addFileTimes, [&]{ addFileToList(std::move(file.lcname), fe->getIndex()); }); @@ -1357,11 +1367,11 @@ FileEntry::Ptr DirectoryEntry::insert( } } - stats.addOriginToFileTimes += elapsed([&]{ + elapsed(stats.addOriginToFileTimes, [&]{ fe->addOrigin(origin.getID(), file.lastModified, archive, order); }); - stats.addFileToOriginTimes += elapsed([&]{ + elapsed(stats.addFileToOriginTimes, [&]{ origin.addFile(fe->getIndex()); }); @@ -1386,7 +1396,7 @@ void DirectoryEntry::addFiles( [](void* pcx, std::wstring_view path) { Context* cx = (Context*)pcx; - cx->stats.dirTimes += elapsed([&] { + elapsed(cx->stats.dirTimes, [&] { auto* sd = cx->current.top()->getSubDirectory( path, true, cx->stats, cx->origin.getID()); @@ -1398,7 +1408,7 @@ void DirectoryEntry::addFiles( { Context* cx = (Context*)pcx; - cx->stats.dirTimes += elapsed([&] { + elapsed(cx->stats.dirTimes, [&] { auto* current= cx->current.top(); { @@ -1418,7 +1428,7 @@ void DirectoryEntry::addFiles( { Context* cx = (Context*)pcx; - cx->stats.fileTimes += elapsed([&]{ + elapsed(cx->stats.fileTimes, [&]{ cx->current.top()->insert(path, cx->origin, ft, L"", -1, cx->stats); }); } @@ -1466,7 +1476,7 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( std::scoped_lock lock(m_SubDirMutex); SubDirectoriesLookup::iterator itor; - stats.subdirLookupTimes += elapsed([&] { + elapsed(stats.subdirLookupTimes, [&] { itor = m_SubDirectoriesLookup.find(nameLc); }); @@ -1482,7 +1492,7 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( std::wstring(name.begin(), name.end()), this, originID, m_FileRegister, m_OriginConnection); - stats.addDirectoryTimes += elapsed([&] { + elapsed(stats.addDirectoryTimes, [&] { addDirectoryToList(entry, std::move(nameLc)); // nameLc is moved from this point }); @@ -1498,7 +1508,7 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( { SubDirectoriesLookup::iterator itor; - stats.subdirLookupTimes += elapsed([&] { + elapsed(stats.subdirLookupTimes, [&] { itor = m_SubDirectoriesLookup.find(dir.lcname); }); @@ -1515,7 +1525,7 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( m_FileRegister, m_OriginConnection); // dir.name is moved from this point - stats.addDirectoryTimes += elapsed([&]{ + elapsed(stats.addDirectoryTimes, [&]{ addDirectoryToList(entry, std::move(dir.lcname)); }); diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 94ea068d..74d50625 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -64,6 +64,8 @@ class FileRegister; struct DirectoryStats { + static constexpr bool EnableInstrumentation = true; + std::string mod; std::chrono::nanoseconds dirTimes; @@ -127,11 +129,6 @@ public: return m_Index; } - time_t lastAccessed() const - { - return m_LastAccessed; - } - void addOrigin( int origin, FILETIME fileTime, std::wstring_view archive, int order); @@ -222,8 +219,6 @@ private: uint64_t m_FileSize, m_CompressedFileSize; mutable std::mutex m_OriginsMutex; - time_t m_LastAccessed; - bool recurseParents(std::wstring &path, const DirectoryEntry *parent) const; }; @@ -265,8 +260,8 @@ public: std::vector getFiles() const; FileEntry::Ptr findFile(FileEntry::Index index) const; - void enable(bool enabled, DirectoryStats& stats, time_t notAfter = LONG_MAX); - void enable(bool enabled, time_t notAfter = LONG_MAX); + void enable(bool enabled, DirectoryStats& stats); + void enable(bool enabled); bool isDisabled() const { @@ -313,24 +308,30 @@ public: FileEntry::Ptr getFile(FileEntry::Index index) const; - size_t size() const + size_t highestCount() const { std::scoped_lock lock(m_Mutex); return m_Files.size(); } + void reserve(std::size_t n) + { + m_Files.reserve(n); + } + bool removeFile(FileEntry::Index index); void removeOrigin(FileEntry::Index index, int originID); - void removeOriginMulti(std::set indices, int originID, time_t notAfter); + void removeOriginMulti(std::set indices, int originID); void sortOrigins(); private: - using FileMap = std::map; + using FileMap = std::vector; mutable std::mutex m_Mutex; FileMap m_Files; boost::shared_ptr m_OriginConnection; + std::atomic m_NextIndex; void unregisterFile(FileEntry::Ptr file); FileEntry::Index generateIndex(); -- cgit v1.3.1 From 75359bc9c2bd9f593c59fb156186b3491d0b7c47 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Fri, 14 Feb 2020 03:30:23 -0500 Subject: fixed crash when disabling mod --- src/shared/directoryentry.cpp | 8 +++++--- src/shared/directoryentry.h | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 0be5e870..2605d083 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -55,8 +55,8 @@ void elapsedImpl(std::chrono::nanoseconds& out, F&& f) // elapsed() is not optimized out when EnableInstrumentation is false even // though it's equivalent that this macro -//#define elapsed(OUT, F) (F)(); -#define elapsed(OUT, F) elapsedImpl(OUT, F); +#define elapsed(OUT, F) (F)(); +//#define elapsed(OUT, F) elapsedImpl(OUT, F); static std::wstring tail(const std::wstring &source, const size_t count) @@ -843,7 +843,9 @@ void FileRegister::sortOrigins() std::scoped_lock lock(m_Mutex); for (auto&& p : m_Files) { - p->sortOrigins(); + if (p) { + p->sortOrigins(); + } } } diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 74d50625..f406ced6 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -64,7 +64,7 @@ class FileRegister; struct DirectoryStats { - static constexpr bool EnableInstrumentation = true; + static constexpr bool EnableInstrumentation = false; std::string mod; -- cgit v1.3.1 From 4a74102736a20a6fa5bf9df9cc7716cbecb5b777 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Fri, 14 Feb 2020 08:01:27 -0500 Subject: moved dump() to DirectoryEntry static functions instead of lambdas --- src/filetree.cpp | 25 +------- src/shared/directoryentry.cpp | 131 +++++++++++++++++++++++++++++++----------- src/shared/directoryentry.h | 15 ++++- 3 files changed, 111 insertions(+), 60 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/filetree.cpp b/src/filetree.cpp index e5ea4e9d..45c21c7f 100644 --- a/src/filetree.cpp +++ b/src/filetree.cpp @@ -438,30 +438,7 @@ void FileTree::dumpToFile() const return; } - QFile out(file); - - if (!out.open(QIODevice::WriteOnly)) { - QMessageBox::critical( - m_tree->window(), tr("Error"), tr("Failed to open file '%1': %2") - .arg(file) - .arg(out.errorString())); - - return; - } - - try - { - dumpToFile(out, "Data", *m_core.directoryStructure()); - } - catch(DumpFailed&) - { - // try to remove it silently - if (out.exists()) { - if (!out.remove()) { - log::error("failed to remove '{}', ignoring", file); - } - } - } + m_core.directoryStructure()->dump(file.toStdWString()); } void FileTree::dumpToFile( diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 2605d083..394eda7e 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -1380,61 +1380,80 @@ FileEntry::Ptr DirectoryEntry::insert( return fe; } +struct DirectoryEntry::Context +{ + FilesOrigin& origin; + DirectoryStats& stats; + std::stack current; +}; + void DirectoryEntry::addFiles( env::DirectoryWalker& walker, FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats) { - struct Context - { - FilesOrigin& origin; - DirectoryStats& stats; - std::stack current; - }; - Context cx = {origin, stats}; cx.current.push(this); walker.forEachEntry(path, &cx, [](void* pcx, std::wstring_view path) { - Context* cx = (Context*)pcx; - elapsed(cx->stats.dirTimes, [&] { - auto* sd = cx->current.top()->getSubDirectory( - path, true, cx->stats, cx->origin.getID()); - - cx->current.push(sd); - }); + onDirectoryStart((Context*)pcx, path); }, [](void* pcx, std::wstring_view path) { - Context* cx = (Context*)pcx; + onDirectoryEnd((Context*)pcx, path); + }, - elapsed(cx->stats.dirTimes, [&] { - auto* current= cx->current.top(); + [](void* pcx, std::wstring_view path, FILETIME ft) + { + onFile((Context*)pcx, path, ft); + } + ); - { - std::scoped_lock lock(current->m_SubDirMutex); + { + std::scoped_lock lock(m_SubDirMutex); - std::sort( - current->m_SubDirectories.begin(), - current->m_SubDirectories.end(), - &DirCompareByName); - } + std::sort( + m_SubDirectories.begin(), + m_SubDirectories.end(), + &DirCompareByName); + } +} - cx->current.pop(); - }); - }, +void DirectoryEntry::onDirectoryStart(Context* cx, std::wstring_view path) +{ + elapsed(cx->stats.dirTimes, [&] { + auto* sd = cx->current.top()->getSubDirectory( + path, true, cx->stats, cx->origin.getID()); + + cx->current.push(sd); + }); +} + +void DirectoryEntry::onDirectoryEnd(Context* cx, std::wstring_view path) +{ + elapsed(cx->stats.dirTimes, [&] { + auto* current = cx->current.top(); - [](void* pcx, std::wstring_view path, FILETIME ft) { - Context* cx = (Context*)pcx; + std::scoped_lock lock(current->m_SubDirMutex); - elapsed(cx->stats.fileTimes, [&]{ - cx->current.top()->insert(path, cx->origin, ft, L"", -1, cx->stats); - }); + std::sort( + current->m_SubDirectories.begin(), + current->m_SubDirectories.end(), + &DirCompareByName); } - ); + + cx->current.pop(); + }); +} + +void DirectoryEntry::onFile(Context* cx, std::wstring_view path, FILETIME ft) +{ + elapsed(cx->stats.fileTimes, [&]{ + cx->current.top()->insert(path, cx->origin, ft, L"", -1, cx->stats); + }); } void DirectoryEntry::addFiles( @@ -1510,6 +1529,8 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( { SubDirectoriesLookup::iterator itor; + std::scoped_lock lock(m_SubDirMutex); + elapsed(stats.subdirLookupTimes, [&] { itor = m_SubDirectoriesLookup.find(dir.lcname); }); @@ -1664,4 +1685,48 @@ void DirectoryEntry::addFileToList( // fileNameLower has been moved from this point } +void DirectoryEntry::dump(const std::wstring& file) const +{ + std::FILE* f = nullptr; + auto e = _wfopen_s(&f, file.c_str(), L"wb"); + + dump(f, L"Data"); + + std::fclose(f); +} + +void DirectoryEntry::dump(std::FILE* f, const std::wstring& parentPath) const +{ + { + std::scoped_lock lock(m_FilesMutex); + + for (auto&& index : m_Files) { + const auto file = m_FileRegister->getFile(index.second); + if (!file) { + continue; + } + + if (file->isFromArchive()) { + // TODO: don't list files from archives. maybe make this an option? + continue; + } + + const auto o = m_OriginConnection->getByID(file->getOrigin()); + const auto path = parentPath + L"\\" + file->getName(); + const auto line = path + L"\t(" + o.getName() + L")\r\n"; + + const auto lineu8 = MOShared::ToString(line, true); + std::fwrite(lineu8.data(), lineu8.size(), 1, f); + } + } + + { + std::scoped_lock lock(m_SubDirMutex); + for (auto&& d : m_SubDirectories) { + const auto path = parentPath + L"\\" + d->m_Name; + d->dump(f, path); + } + } +} + } // namespace MOShared diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index f406ced6..ce50da44 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -537,6 +537,8 @@ public: void removeFiles(const std::set &indices); + void dump(const std::wstring& file) const; + private: using FilesMap = std::map; using FilesLookup = std::unordered_map; @@ -556,9 +558,9 @@ private: std::set m_Origins; bool m_Populated; bool m_TopLevel; - std::mutex m_SubDirMutex; - std::mutex m_FilesMutex; - std::mutex m_OriginsMutex; + mutable std::mutex m_SubDirMutex; + mutable std::mutex m_FilesMutex; + mutable std::mutex m_OriginsMutex; DirectoryEntry(const DirectoryEntry &reference); @@ -600,6 +602,13 @@ private: void addFileToList(std::wstring fileNameLower, FileEntry::Index index); void removeFileFromList(FileEntry::Index index); void removeFilesFromList(const std::set& indices); + + struct Context; + static void onDirectoryStart(Context* cx, std::wstring_view path); + static void onDirectoryEnd(Context* cx, std::wstring_view path); + static void onFile(Context* cx, std::wstring_view path, FILETIME ft); + + void dump(std::FILE* f, const std::wstring& parentPath) const; }; } // namespace MOShared -- cgit v1.3.1 From 88ef0530001d43be8f18fac43a280169b45db852 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sat, 15 Feb 2020 12:25:37 -0500 Subject: error checking in dump() removed a bunch of "{} saved" in the logs --- src/directoryrefresher.cpp | 37 ++++++++++++++++++------------------- src/filetree.cpp | 39 --------------------------------------- src/filetree.h | 4 ---- src/mainwindow.cpp | 4 +--- src/pluginlist.cpp | 5 +---- src/profile.cpp | 6 +----- src/shared/directoryentry.cpp | 34 +++++++++++++++++++++++++++++----- 7 files changed, 50 insertions(+), 79 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index f3285cfc..8bf349f1 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -324,35 +324,34 @@ void DirectoryRefresher::refresh() SetThisThreadName("DirectoryRefresher"); TimeThis tt("refresh"); - for (int i=0; i<1; ++i) { - QMutexLocker locker(&m_RefreshLock); - delete m_DirectoryStructure; + QMutexLocker locker(&m_RefreshLock); + delete m_DirectoryStructure; - m_DirectoryStructure = new DirectoryEntry(L"data", nullptr, 0); - m_DirectoryStructure->getFileRegister()->reserve(m_lastFileCount); + m_DirectoryStructure = new DirectoryEntry(L"data", nullptr, 0); + m_DirectoryStructure->getFileRegister()->reserve(m_lastFileCount); - IPluginGame *game = qApp->property("managed_game").value(); + IPluginGame *game = qApp->property("managed_game").value(); - std::wstring dataDirectory = - QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + std::wstring dataDirectory = + QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); - { - DirectoryStats dummy; - m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0, dummy); - } + { + DirectoryStats dummy; + m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0, dummy); + } - std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { - return lhs.priority < rhs.priority; - }); + std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { + return lhs.priority < rhs.priority; + }); - addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); + addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); - m_DirectoryStructure->getFileRegister()->sortOrigins(); + m_DirectoryStructure->getFileRegister()->sortOrigins(); - cleanStructure(m_DirectoryStructure); - } + cleanStructure(m_DirectoryStructure); m_lastFileCount = m_DirectoryStructure->getFileRegister()->highestCount(); + log::debug("refresher saw {} files", m_lastFileCount); emit progress(100); emit refreshed(); diff --git a/src/filetree.cpp b/src/filetree.cpp index 45c21c7f..f628dff3 100644 --- a/src/filetree.cpp +++ b/src/filetree.cpp @@ -441,45 +441,6 @@ void FileTree::dumpToFile() const m_core.directoryStructure()->dump(file.toStdWString()); } -void FileTree::dumpToFile( - QFile& out, const QString& parentPath, const DirectoryEntry& entry) const -{ - entry.forEachFile([&](auto&& file) { - bool isArchive = false; - const int originID = file.getOrigin(isArchive); - - if (isArchive) { - // TODO: don't list files from archives. maybe make this an option? - return true; - } - - const auto& origin = m_core.directoryStructure()->getOriginByID(originID); - const auto originName = QString::fromStdWString(origin.getName()); - - const QString path = - parentPath + "\\" + QString::fromStdWString(file.getName()); - - if (out.write(path.toUtf8() + "\t(" + originName.toUtf8() + ")\r\n") == -1) { - QMessageBox::critical( - m_tree->window(), tr("Error"), tr("Failed to write to file %1: %2") - .arg(out.fileName()) - .arg(out.errorString())); - - throw DumpFailed(); - } - - return true; - }); - - entry.forEachDirectory([&](auto&& dir) { - const auto newParentPath = - parentPath + "\\" + QString::fromStdWString(dir.getName()); - - dumpToFile(out, newParentPath, dir); - return true; - }); -} - void FileTree::onExpandedChanged(const QModelIndex& index, bool expanded) { if (auto* item=m_model->itemFromIndex(proxiedIndex(index))) { diff --git a/src/filetree.h b/src/filetree.h index c1458179..2669e53b 100644 --- a/src/filetree.h +++ b/src/filetree.h @@ -65,10 +65,6 @@ private: void toggleVisibility(bool b, FileTreeItem* item=nullptr); QModelIndex proxiedIndex(const QModelIndex& index); - - void dumpToFile( - QFile& out, const QString& parentPath, - const MOShared::DirectoryEntry& entry) const; }; #endif // MODORGANIZER_FILETREE_INCLUDED diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index 2145011f..f9f1dfe5 100644 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -4116,9 +4116,7 @@ void MainWindow::saveArchiveList() } } } - if (archiveFile.commitIfDifferent(m_ArchiveListHash)) { - log::debug("{} saved", QDir::toNativeSeparators(m_OrganizerCore.currentProfile()->getArchivesFileName())); - } + archiveFile.commitIfDifferent(m_ArchiveListHash); } else { log::warn("archive list not initialised"); } diff --git a/src/pluginlist.cpp b/src/pluginlist.cpp index 266fe35c..f348f64c 100644 --- a/src/pluginlist.cpp +++ b/src/pluginlist.cpp @@ -507,7 +507,6 @@ void PluginList::writeLockedOrder(const QString &fileName) const file->write(QString("%1|%2\r\n").arg(iter->first).arg(iter->second).toUtf8()); } file.commit(); - log::debug("{} saved", QDir::toNativeSeparators(fileName)); } @@ -531,9 +530,7 @@ void PluginList::saveTo(const QString &lockedOrderFileName deleterFile->write("\r\n"); } } - if (deleterFile.commitIfDifferent(m_LastSaveHash[deleterFileName])) { - log::debug("{} saved", QDir::toNativeSeparators(deleterFileName)); - } + deleterFile.commitIfDifferent(m_LastSaveHash[deleterFileName]); } else if (QFile::exists(deleterFileName)) { shellDelete(QStringList() << deleterFileName); } diff --git a/src/profile.cpp b/src/profile.cpp index f041a241..19c0d750 100644 --- a/src/profile.cpp +++ b/src/profile.cpp @@ -250,9 +250,7 @@ void Profile::doWriteModlist() } } - if (file.commitIfDifferent(m_LastModlistHash)) { - log::debug("{} saved", QDir::toNativeSeparators(fileName)); - } + file.commitIfDifferent(m_LastModlistHash); } catch (const std::exception &e) { reportError(tr("failed to write mod list: %1").arg(e.what())); return; @@ -292,8 +290,6 @@ void Profile::createTweakedIniFile() reportError(tr("failed to create tweaked ini: %1") .arg(QString::fromStdWString(formatSystemMessage(e)))); } - - log::debug("{} saved", QDir::toNativeSeparators(tweakedIni)); } // static diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 394eda7e..6c5ad9ae 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -1685,14 +1685,33 @@ void DirectoryEntry::addFileToList( // fileNameLower has been moved from this point } +struct DumpFailed : public std::runtime_error +{ + using runtime_error::runtime_error; +}; + void DirectoryEntry::dump(const std::wstring& file) const { - std::FILE* f = nullptr; - auto e = _wfopen_s(&f, file.c_str(), L"wb"); + try + { + std::FILE* f = nullptr; + auto e = _wfopen_s(&f, file.c_str(), L"wb"); + + if (e != 0 || !f) { + throw DumpFailed(fmt::format( + "failed to open, {} ({})", std::strerror(e), e)); + } - dump(f, L"Data"); + Guard g([&]{ std::fclose(f); }); - std::fclose(f); + dump(f, L"Data"); + } + catch(DumpFailed& e) + { + log::error( + "failed to write list to '{}': {}", + QString::fromStdWString(file).toStdString(), e.what()); + } } void DirectoryEntry::dump(std::FILE* f, const std::wstring& parentPath) const @@ -1716,7 +1735,12 @@ void DirectoryEntry::dump(std::FILE* f, const std::wstring& parentPath) const const auto line = path + L"\t(" + o.getName() + L")\r\n"; const auto lineu8 = MOShared::ToString(line, true); - std::fwrite(lineu8.data(), lineu8.size(), 1, f); + + if (std::fwrite(lineu8.data(), lineu8.size(), 1, f) != 1) { + const auto e = errno; + throw DumpFailed(fmt::format( + "failed to write, {} ({})", std::strerror(e), e)); + } } } -- cgit v1.3.1 From b1cf498924e461556c8f2fe961172685d92bb03f Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sat, 15 Feb 2020 13:24:56 -0500 Subject: split directoryentry made classes noncopyable, fixed a few unintended copies --- src/CMakeLists.txt | 21 +- src/datatab.cpp | 1 - src/directoryrefresher.cpp | 11 +- src/directoryrefresher.h | 5 +- src/filetree.cpp | 3 + src/filetreeitem.h | 6 +- src/filetreemodel.cpp | 12 +- src/filetreemodel.h | 6 +- src/loglist.h | 3 +- src/main.cpp | 1 + src/mainwindow.cpp | 14 +- src/modinfodialog.cpp | 2 + src/modinfodialogconflicts.cpp | 21 +- src/modinfodialogconflicts.h | 14 +- src/modinfodialogtab.cpp | 2 +- src/modinfowithconflictinfo.cpp | 11 +- src/modlist.cpp | 8 +- src/modlist.h | 1 - src/organizercore.cpp | 7 +- src/organizercore.h | 1 - src/pch.h | 4 + src/pluginlist.cpp | 16 +- src/pluginlist.h | 1 - src/shared/directoryentry.cpp | 745 ++-------------------------------------- src/shared/directoryentry.h | 368 ++------------------ src/shared/fileentry.cpp | 251 ++++++++++++++ src/shared/fileentry.h | 125 +++++++ src/shared/fileregister.cpp | 184 ++++++++++ src/shared/fileregister.h | 58 ++++ src/shared/fileregisterfwd.h | 89 +++++ src/shared/filesorigin.cpp | 137 ++++++++ src/shared/filesorigin.h | 87 +++++ src/shared/originconnection.cpp | 145 ++++++++ src/shared/originconnection.h | 59 ++++ src/syncoverwritedialog.cpp | 7 +- src/syncoverwritedialog.h | 11 +- 36 files changed, 1310 insertions(+), 1127 deletions(-) create mode 100644 src/shared/fileentry.cpp create mode 100644 src/shared/fileentry.h create mode 100644 src/shared/fileregister.cpp create mode 100644 src/shared/fileregister.h create mode 100644 src/shared/fileregisterfwd.h create mode 100644 src/shared/filesorigin.cpp create mode 100644 src/shared/filesorigin.h create mode 100644 src/shared/originconnection.cpp create mode 100644 src/shared/originconnection.h (limited to 'src/shared/directoryentry.cpp') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e121c572..85d8af0f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -155,6 +155,10 @@ SET(organizer_SRCS shared/windows_error.cpp shared/error_report.cpp shared/directoryentry.cpp + shared/fileentry.cpp + shared/filesorigin.cpp + shared/fileregister.cpp + shared/originconnection.cpp shared/util.cpp shared/appconfig.cpp shared/leaktrace.cpp @@ -286,6 +290,10 @@ SET(organizer_HDRS shared/windows_error.h shared/error_report.h shared/directoryentry.h + shared/fileentry.h + shared/filesorigin.h + shared/fileregister.h + shared/originconnection.h shared/util.h shared/appconfig.h shared/appconfig.inc @@ -350,8 +358,6 @@ set(browser set(core categories - shared/directoryentry - directoryrefresher installationmanager instancemanager loadmechanism @@ -470,6 +476,15 @@ set(profiles profilesdialog ) +set(register + shared/directoryentry + shared/fileentry + shared/filesorigin + shared/fileregister + shared/originconnection + directoryrefresher +) + set(settings settings settingsutilities @@ -521,7 +536,7 @@ set(widgets set(src_filters application core browser dialogs downloads env executables loot mainwindow - modinfo modinfo\\dialog modlist plugins previews profiles settings + modinfo modinfo\\dialog modlist plugins previews profiles register settings settingsdialog utilities widgets ) diff --git a/src/datatab.cpp b/src/datatab.cpp index 19bfa134..b6119d52 100644 --- a/src/datatab.cpp +++ b/src/datatab.cpp @@ -2,7 +2,6 @@ #include "ui_mainwindow.h" #include "settings.h" #include "organizercore.h" -#include "directoryentry.h" #include "messagedialog.h" #include "filetree.h" #include "filetreemodel.h" diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index 8bf349f1..3b92389a 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -18,6 +18,9 @@ along with Mod Organizer. If not, see . */ #include "directoryrefresher.h" +#include "shared/fileentry.h" +#include "shared/filesorigin.h" +#include "shared/directoryentry.h" #include "iplugingame.h" #include "utility.h" @@ -26,12 +29,16 @@ along with Mod Organizer. If not, see . #include "settings.h" #include "envfs.h" #include "modinfodialogfwd.h" +#include "util.h" + +#include #include #include #include #include -#include + +#include using namespace MOBase; @@ -136,7 +143,7 @@ void DirectoryRefresher::stealModFilesIntoStructure( continue; } QFileInfo fileInfo(filename); - FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); + FileEntryPtr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); if (file.get() != nullptr) { if (file->getOrigin() == 0) { // replace data as the origin on this bsa diff --git a/src/directoryrefresher.h b/src/directoryrefresher.h index 5c829980..2102140c 100644 --- a/src/directoryrefresher.h +++ b/src/directoryrefresher.h @@ -20,7 +20,6 @@ along with Mod Organizer. If not, see . #ifndef DIRECTORYREFRESHER_H #define DIRECTORYREFRESHER_H -#include #include #include #include @@ -59,6 +58,10 @@ public: ~DirectoryRefresher(); + // noncopyable + DirectoryRefresher(const DirectoryRefresher&) = delete; + DirectoryRefresher& operator=(const DirectoryRefresher&) = delete; + /** * @brief retrieve the updated directory structure * diff --git a/src/filetree.cpp b/src/filetree.cpp index f628dff3..cdcf2feb 100644 --- a/src/filetree.cpp +++ b/src/filetree.cpp @@ -3,6 +3,9 @@ #include "filetreeitem.h" #include "organizercore.h" #include "envshell.h" +#include "shared/fileentry.h" +#include "shared/directoryentry.h" +#include "shared/filesorigin.h" #include #include diff --git a/src/filetreeitem.h b/src/filetreeitem.h index 390d5499..2092782e 100644 --- a/src/filetreeitem.h +++ b/src/filetreeitem.h @@ -1,7 +1,7 @@ #ifndef MODORGANIZER_FILETREEITEM_INCLUDED #define MODORGANIZER_FILETREEITEM_INCLUDED -#include "directoryentry.h" +#include "shared/fileregisterfwd.h" #include class FileTreeModel; @@ -126,7 +126,7 @@ public: return m_wsLcFile; } - const MOShared::DirectoryEntry::FileKey& key() const + const MOShared::DirectoryEntryFileKey& key() const { return m_key; } @@ -288,7 +288,7 @@ private: const QString m_virtualParentPath; const std::wstring m_wsFile, m_wsLcFile; - const MOShared::DirectoryEntry::FileKey m_key; + const MOShared::DirectoryEntryFileKey m_key; const QString m_file; const bool m_isDirectory; diff --git a/src/filetreemodel.cpp b/src/filetreemodel.cpp index 2130bf89..1a5433c5 100644 --- a/src/filetreemodel.cpp +++ b/src/filetreemodel.cpp @@ -1,5 +1,9 @@ #include "filetreemodel.h" #include "organizercore.h" +#include "filesorigin.h" +#include "util.h" +#include "shared/directoryentry.h" +#include "shared/fileentry.h" #include using namespace MOBase; @@ -637,7 +641,7 @@ bool FileTreeModel::updateFiles( // removeDisappearingFiles() will add files that are in the tree and still on // the filesystem to this set; addNewFiless() will use this to figure out if // a file is new or not - std::unordered_set seen; + std::unordered_set seen; int firstFileRow = 0; @@ -647,7 +651,7 @@ bool FileTreeModel::updateFiles( void FileTreeModel::removeDisappearingFiles( FileTreeItem& parentItem, const MOShared::DirectoryEntry& parentEntry, - int& firstFileRow, std::unordered_set& seen) + int& firstFileRow, std::unordered_set& seen) { auto& children = parentItem.children(); auto itor = children.begin(); @@ -708,7 +712,7 @@ void FileTreeModel::removeDisappearingFiles( bool FileTreeModel::addNewFiles( FileTreeItem& parentItem, const MOShared::DirectoryEntry& parentEntry, const std::wstring& parentPath, const int firstFileRow, - const std::unordered_set& seen) + const std::unordered_set& seen) { // keeps track of the contiguous files that need to be added to // avoid calling beginAddRows(), etc. for each item @@ -958,7 +962,7 @@ std::wstring FileTreeModel::makeModName( { static const std::wstring Unmanaged = UnmanagedModName().toStdWString(); - const auto origin = m_core.directoryStructure()->getOriginByID(originID); + const auto& origin = m_core.directoryStructure()->getOriginByID(originID); if (origin.getID() == 0) { return Unmanaged; diff --git a/src/filetreemodel.h b/src/filetreemodel.h index 093407b0..5bfb75aa 100644 --- a/src/filetreemodel.h +++ b/src/filetreemodel.h @@ -3,7 +3,7 @@ #include "filetreeitem.h" #include "iconfetcher.h" -#include "directoryentry.h" +#include "shared/fileregisterfwd.h" #include class OrganizerCore; @@ -126,12 +126,12 @@ private: void removeDisappearingFiles( FileTreeItem& parentItem, const MOShared::DirectoryEntry& parentEntry, - int& firstFileRow, std::unordered_set& seen); + int& firstFileRow, std::unordered_set& seen); bool addNewFiles( FileTreeItem& parentItem, const MOShared::DirectoryEntry& parentEntry, const std::wstring& parentPath, int firstFileRow, - const std::unordered_set& seen); + const std::unordered_set& seen); FileTreeItem::Ptr createDirectoryItem( diff --git a/src/loglist.h b/src/loglist.h index 56b95d65..b26f1561 100644 --- a/src/loglist.h +++ b/src/loglist.h @@ -20,8 +20,9 @@ along with Mod Organizer. If not, see . #ifndef LOGBUFFER_H #define LOGBUFFER_H -#include #include +#include +#include class OrganizerCore; diff --git a/src/main.cpp b/src/main.cpp index 6e2220ae..105299a8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -48,6 +48,7 @@ along with Mod Organizer. If not, see . #include "organizercore.h" #include "env.h" #include "envmodule.h" +#include "util.h" #include #include diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index f9f1dfe5..adbac94e 100644 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -20,8 +20,6 @@ along with Mod Organizer. If not, see . #include "mainwindow.h" #include "ui_mainwindow.h" -#include "directoryentry.h" -#include "directoryrefresher.h" #include "executableinfo.h" #include "executableslist.h" #include "guessedvalue.h" @@ -89,6 +87,10 @@ along with Mod Organizer. If not, see . #include "envshortcut.h" #include "browserdialog.h" +#include "shared/directoryentry.h" +#include "shared/fileentry.h" +#include "shared/filesorigin.h" + #include #include #include @@ -1870,7 +1872,7 @@ void MainWindow::updateBSAList(const QStringList &defaultArchives, const QString std::vector> items; BSAInvalidation * invalidation = m_OrganizerCore.managedGame()->feature(); - std::vector files = m_OrganizerCore.directoryStructure()->getFiles(); + std::vector files = m_OrganizerCore.directoryStructure()->getFiles(); QStringList plugins = m_OrganizerCore.findFiles("", [](const QString &fileName) -> bool { return fileName.endsWith(".esp", Qt::CaseInsensitive) @@ -1889,7 +1891,7 @@ void MainWindow::updateBSAList(const QStringList &defaultArchives, const QString return false; }; - for (FileEntry::Ptr current : files) { + for (FileEntryPtr current : files) { QFileInfo fileInfo(ToQString(current->getName().c_str())); if (fileInfo.suffix().toLower() == "bsa" || fileInfo.suffix().toLower() == "ba2") { @@ -1942,7 +1944,7 @@ void MainWindow::updateBSAList(const QStringList &defaultArchives, const QString for (auto iter = items.begin(); iter != items.end(); ++iter) { int originID = iter->second->data(1, Qt::UserRole).toInt(); - FilesOrigin origin = m_OrganizerCore.directoryStructure()->getOriginByID(originID); + const FilesOrigin& origin = m_OrganizerCore.directoryStructure()->getOriginByID(originID); QString modName; const unsigned int modIndex = ModInfo::getIndex(ToQString(origin.getName())); @@ -2504,7 +2506,7 @@ void MainWindow::modRenamed(const QString &oldName, const QString &newName) void MainWindow::fileMoved(const QString &filePath, const QString &oldOriginName, const QString &newOriginName) { - const FileEntry::Ptr filePtr = m_OrganizerCore.directoryStructure()->findFile(ToWString(filePath)); + const FileEntryPtr filePtr = m_OrganizerCore.directoryStructure()->findFile(ToWString(filePath)); if (filePtr.get() != nullptr) { try { if (m_OrganizerCore.directoryStructure()->originExists(ToWString(newOriginName))) { diff --git a/src/modinfodialog.cpp b/src/modinfodialog.cpp index bfb8d2c7..c3239e0d 100644 --- a/src/modinfodialog.cpp +++ b/src/modinfodialog.cpp @@ -29,6 +29,8 @@ along with Mod Organizer. If not, see . #include "modinfodialogcategories.h" #include "modinfodialognexus.h" #include "modinfodialogfiletree.h" +#include "shared/directoryentry.h" +#include "shared/filesorigin.h" #include using namespace MOBase; diff --git a/src/modinfodialogconflicts.cpp b/src/modinfodialogconflicts.cpp index 63d89a1a..b61dbb7b 100644 --- a/src/modinfodialogconflicts.cpp +++ b/src/modinfodialogconflicts.cpp @@ -4,6 +4,9 @@ #include "utility.h" #include "settings.h" #include "organizercore.h" +#include "shared/directoryentry.h" +#include "shared/filesorigin.h" +#include "shared/fileentry.h" using namespace MOShared; using namespace MOBase; @@ -17,7 +20,7 @@ class ConflictItem public: ConflictItem( QString before, QString relativeName, QString after, - FileEntry::Index index, QString fileName, + FileIndex index, QString fileName, bool hasAltOrigins, QString altOrigin, bool archive) : m_before(std::move(before)), m_relativeName(std::move(relativeName)), @@ -65,7 +68,7 @@ public: return m_isArchive; } - FileEntry::Index fileIndex() const + FileIndex fileIndex() const { return m_index; } @@ -104,7 +107,7 @@ private: QString m_before; QString m_relativeName; QString m_after; - FileEntry::Index m_index; + FileIndex m_index; QString m_fileName; bool m_hasAltOrigins; QString m_altOrigin; @@ -1006,8 +1009,8 @@ bool GeneralConflictsTab::update() } ConflictItem GeneralConflictsTab::createOverwriteItem( - FileEntry::Index index, bool archive, QString fileName, QString relativeName, - const FileEntry::AlternativesVector& alternatives) + FileIndex index, bool archive, QString fileName, QString relativeName, + const MOShared::AlternativesVector& alternatives) { const auto& ds = *m_core.directoryStructure(); std::wstring altString; @@ -1028,7 +1031,7 @@ ConflictItem GeneralConflictsTab::createOverwriteItem( } ConflictItem GeneralConflictsTab::createNoConflictItem( - FileEntry::Index index, bool archive, QString fileName, QString relativeName) + FileIndex index, bool archive, QString fileName, QString relativeName) { return ConflictItem( QString(), std::move(relativeName), QString(), index, @@ -1036,7 +1039,7 @@ ConflictItem GeneralConflictsTab::createNoConflictItem( } ConflictItem GeneralConflictsTab::createOverwrittenItem( - FileEntry::Index index, int fileOrigin, bool archive, + FileIndex index, int fileOrigin, bool archive, QString fileName, QString relativeName) { const auto& ds = *m_core.directoryStructure(); @@ -1205,9 +1208,9 @@ void AdvancedConflictsTab::update() } std::optional AdvancedConflictsTab::createItem( - FileEntry::Index index, int fileOrigin, bool archive, + FileIndex index, int fileOrigin, bool archive, QString fileName, QString relativeName, - const MOShared::FileEntry::AlternativesVector& alternatives) + const MOShared::AlternativesVector& alternatives) { const auto& ds = *m_core.directoryStructure(); diff --git a/src/modinfodialogconflicts.h b/src/modinfodialogconflicts.h index c4845019..945f464d 100644 --- a/src/modinfodialogconflicts.h +++ b/src/modinfodialogconflicts.h @@ -4,7 +4,7 @@ #include "modinfodialogtab.h" #include "expanderwidget.h" #include "filterwidget.h" -#include "directoryentry.h" +#include "shared/fileregisterfwd.h" #include #include @@ -52,16 +52,16 @@ private: FilterWidget m_filterNoConflicts; ConflictItem createOverwriteItem( - MOShared::FileEntry::Index index, bool archive, + MOShared::FileIndex index, bool archive, QString fileName, QString relativeName, - const MOShared::FileEntry::AlternativesVector& alternatives); + const MOShared::AlternativesVector& alternatives); ConflictItem createNoConflictItem( - MOShared::FileEntry::Index index, bool archive, + MOShared::FileIndex index, bool archive, QString fileName, QString relativeName); ConflictItem createOverwrittenItem( - MOShared::FileEntry::Index index, int fileOrigin, bool archive, + MOShared::FileIndex index, int fileOrigin, bool archive, QString fileName, QString relativeName); void onOverwriteActivated(const QModelIndex& index); @@ -94,9 +94,9 @@ private: ConflictListModel* m_model; std::optional createItem( - MOShared::FileEntry::Index index, int fileOrigin, bool archive, + MOShared::FileIndex index, int fileOrigin, bool archive, QString fileName, QString relativeName, - const MOShared::FileEntry::AlternativesVector& alternatives); + const MOShared::AlternativesVector& alternatives); }; diff --git a/src/modinfodialogtab.cpp b/src/modinfodialogtab.cpp index 9748d059..d662e2b2 100644 --- a/src/modinfodialogtab.cpp +++ b/src/modinfodialogtab.cpp @@ -1,8 +1,8 @@ #include "modinfodialogtab.h" #include "ui_modinfodialog.h" #include "texteditor.h" -#include "directoryentry.h" #include "modinfo.h" +#include "shared/filesorigin.h" ModInfoDialogTab::ModInfoDialogTab(ModInfoDialogTabContext cx) : ui(cx.ui), m_core(cx.core), m_plugin(cx.plugin), m_parent(cx.parent), diff --git a/src/modinfowithconflictinfo.cpp b/src/modinfowithconflictinfo.cpp index 861782d9..2b4fa11c 100644 --- a/src/modinfowithconflictinfo.cpp +++ b/src/modinfowithconflictinfo.cpp @@ -1,7 +1,8 @@ #include "modinfowithconflictinfo.h" - -#include "directoryentry.h" #include "utility.h" +#include "shared/directoryentry.h" +#include "shared/filesorigin.h" +#include "shared/fileentry.h" #include using namespace MOBase; @@ -98,11 +99,11 @@ void ModInfoWithConflictInfo::doConflictCheck() const if ((*m_DirectoryStructure)->originExists(name)) { FilesOrigin &origin = (*m_DirectoryStructure)->getOriginByName(name); - std::vector files = origin.getFiles(); + std::vector files = origin.getFiles(); std::set checkedDirs; // for all files in this origin - for (FileEntry::Ptr file : files) { + for (FileEntryPtr file : files) { // skip hiidden file check if already found one if (!hasHiddenFiles) { @@ -267,7 +268,7 @@ bool ModInfoWithConflictInfo::isRedundant() const std::wstring name = ToWString(this->name()); if ((*m_DirectoryStructure)->originExists(name)) { FilesOrigin &origin = (*m_DirectoryStructure)->getOriginByName(name); - std::vector files = origin.getFiles(); + std::vector files = origin.getFiles(); bool ignore = false; for (auto iter = files.begin(); iter != files.end(); ++iter) { if ((*iter)->getOrigin(ignore) == origin.getID()) { diff --git a/src/modlist.cpp b/src/modlist.cpp index afc1b65f..e84910fd 100644 --- a/src/modlist.cpp +++ b/src/modlist.cpp @@ -27,6 +27,10 @@ along with Mod Organizer. If not, see . #include "pluginlist.h" #include "settings.h" #include "modinforegular.h" +#include "shared/directoryentry.h" +#include "shared/fileentry.h" +#include "shared/filesorigin.h" + #include #include #include @@ -872,7 +876,7 @@ void ModList::highlightMods(const QItemSelectionModel *selection, const MOShared for (QModelIndex idx : selection->selectedRows(PluginList::COL_NAME)) { QString modName = idx.data().toString(); - const MOShared::FileEntry::Ptr fileEntry = directoryEntry.findFile(modName.toStdWString()); + const MOShared::FileEntryPtr fileEntry = directoryEntry.findFile(modName.toStdWString()); if (fileEntry.get() != nullptr) { bool archive = false; std::vector>> origins; @@ -881,7 +885,7 @@ void ModList::highlightMods(const QItemSelectionModel *selection, const MOShared origins.insert(origins.end(), std::pair>(fileEntry->getOrigin(archive), fileEntry->getArchive())); } for (auto originInfo : origins) { - MOShared::FilesOrigin &origin = directoryEntry.getOriginByID(originInfo.first); + MOShared::FilesOrigin& origin = directoryEntry.getOriginByID(originInfo.first); for (unsigned int i = 0; i < ModInfo::getNumMods(); ++i) { if (ModInfo::getByIndex(i)->internalName() == QString::fromStdWString(origin.getName())) { ModInfo::getByIndex(i)->setPluginSelected(true); diff --git a/src/modlist.h b/src/modlist.h index 7452b28b..d8980dec 100644 --- a/src/modlist.h +++ b/src/modlist.h @@ -27,7 +27,6 @@ along with Mod Organizer. If not, see . #include "profile.h" #include -#include #include #include diff --git a/src/organizercore.cpp b/src/organizercore.cpp index a3202153..c9c0bee5 100644 --- a/src/organizercore.cpp +++ b/src/organizercore.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -34,6 +33,10 @@ #include "previewdialog.h" #include "env.h" #include "envmodule.h" +#include "envfs.h" +#include "shared/directoryentry.h" +#include "shared/filesorigin.h" +#include "shared/fileentry.h" #include #include @@ -810,7 +813,7 @@ QString OrganizerCore::resolvePath(const QString &fileName) const if (m_DirectoryStructure == nullptr) { return QString(); } - const FileEntry::Ptr file + const FileEntryPtr file = m_DirectoryStructure->searchFile(ToWString(fileName), nullptr); if (file.get() != nullptr) { return ToQString(file->getFullPath()); diff --git a/src/organizercore.h b/src/organizercore.h index 223eb6cb..980156d3 100644 --- a/src/organizercore.h +++ b/src/organizercore.h @@ -15,7 +15,6 @@ #include "moshortcut.h" #include "processrunner.h" #include "uilocker.h" -#include #include #include #include diff --git a/src/pch.h b/src/pch.h index 030ee634..c66550be 100644 --- a/src/pch.h +++ b/src/pch.h @@ -5,14 +5,17 @@ #include #include #include +#include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -21,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/src/pluginlist.cpp b/src/pluginlist.cpp index f348f64c..4b2eedbd 100644 --- a/src/pluginlist.cpp +++ b/src/pluginlist.cpp @@ -23,6 +23,10 @@ along with Mod Organizer. If not, see . #include "modinfo.h" #include "modlist.h" #include "viewmarkingscrollbar.h" +#include "shared/directoryentry.h" +#include "shared/filesorigin.h" +#include "shared/fileentry.h" + #include #include #include @@ -144,10 +148,10 @@ void PluginList::highlightPlugins(const QItemSelectionModel *selection, const MO if (!selectedMod.isNull() && profile.modEnabled(modIndex)) { QDir dir(selectedMod->absolutePath()); QStringList plugins = dir.entryList(QStringList() << "*.esp" << "*.esm" << "*.esl"); - MOShared::FilesOrigin origin = directoryEntry.getOriginByName(selectedMod->internalName().toStdWString()); + const MOShared::FilesOrigin& origin = directoryEntry.getOriginByName(selectedMod->internalName().toStdWString()); if (plugins.size() > 0) { for (auto plugin : plugins) { - MOShared::FileEntry::Ptr file = directoryEntry.findFile(plugin.toStdWString()); + MOShared::FileEntryPtr file = directoryEntry.findFile(plugin.toStdWString()); if (file && file->getOrigin() != origin.getID()) { const std::vector>> alternatives = file->getAlternatives(); if (std::find_if(alternatives.begin(), alternatives.end(), [&](const std::pair>& element) { return element.first == origin.getID(); }) == alternatives.end()) @@ -185,8 +189,8 @@ void PluginList::refresh(const QString &profileName QStringList availablePlugins; - std::vector files = baseDirectory.getFiles(); - for (FileEntry::Ptr current : files) { + std::vector files = baseDirectory.getFiles(); + for (FileEntryPtr current : files) { if (current.get() == nullptr) { continue; } @@ -217,7 +221,7 @@ void PluginList::refresh(const QString &profileName bool hasIni = baseDirectory.findFile(ToWString(iniPath)).get() != nullptr; std::set loadedArchives; QString candidateName; - for (FileEntry::Ptr archiveCandidate : files) { + for (FileEntryPtr archiveCandidate : files) { candidateName = ToQString(archiveCandidate->getName()); if (candidateName.startsWith(baseName, Qt::CaseInsensitive) && (candidateName.endsWith(".bsa", Qt::CaseInsensitive) || @@ -548,7 +552,7 @@ bool PluginList::saveLoadOrder(DirectoryEntry &directoryStructure) for (ESPInfo &esp : m_ESPs) { std::wstring espName = ToWString(esp.name); - const FileEntry::Ptr fileEntry = directoryStructure.findFile(espName); + const FileEntryPtr fileEntry = directoryStructure.findFile(espName); if (fileEntry.get() != nullptr) { QString fileName; bool archive = false; diff --git a/src/pluginlist.h b/src/pluginlist.h index fb6d0543..cdab18b5 100644 --- a/src/pluginlist.h +++ b/src/pluginlist.h @@ -20,7 +20,6 @@ along with Mod Organizer. If not, see . #ifndef PLUGINLIST_H #define PLUGINLIST_H -#include #include #include "profile.h" #include "loot.h" diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 6c5ad9ae..1036dfe6 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -18,21 +18,14 @@ along with Mod Organizer. If not, see . */ #include "directoryentry.h" -#include "windows_error.h" -#include "error_report.h" +#include "originconnection.h" +#include "filesorigin.h" +#include "fileentry.h" #include "envfs.h" -#include +#include "util.h" +#include "windows_error.h" #include -#include -#include -#include -#define WIN32_LEAN_AND_MEAN -#include -#include -#include -#include -#include -#include +#include namespace MOShared { @@ -58,16 +51,6 @@ void elapsedImpl(std::chrono::nanoseconds& out, F&& f) #define elapsed(OUT, F) (F)(); //#define elapsed(OUT, F) elapsedImpl(OUT, F); - -static std::wstring tail(const std::wstring &source, const size_t count) -{ - if (count >= source.length()) { - return source; - } - - return source.substr(source.length() - count); -} - static bool SupportOptimizedFind() { // large fetch and basic info for FindFirstFileEx is supported on win server 2008 r2, win 7 and newer @@ -189,686 +172,6 @@ std::string DirectoryStats::toCsv() const } -class OriginConnection -{ -public: - typedef int Index; - static const int INVALID_INDEX = INT_MIN; - - OriginConnection() - : m_NextID(0) - { - } - - std::pair getOrCreate( - const std::wstring &originName, const std::wstring &directory, int priority, - const boost::shared_ptr& fileRegister, - const boost::shared_ptr& originConnection, - DirectoryStats& stats) - { - std::unique_lock lock(m_Mutex); - - auto itor = m_OriginsNameMap.find(originName); - - if (itor == m_OriginsNameMap.end()) { - FilesOrigin& origin = createOriginNoLock( - originName, directory, priority, fileRegister, originConnection); - - return {origin, true}; - } else { - FilesOrigin& origin = m_Origins[itor->second]; - lock.unlock(); - - origin.enable(true, stats); - return {origin, false}; - } - } - - FilesOrigin& createOrigin( - const std::wstring &originName, const std::wstring &directory, int priority, - boost::shared_ptr fileRegister, - boost::shared_ptr originConnection) - { - std::scoped_lock lock(m_Mutex); - - return createOriginNoLock( - originName, directory, priority, fileRegister, originConnection); - } - - bool exists(const std::wstring &name) - { - std::scoped_lock lock(m_Mutex); - return m_OriginsNameMap.find(name) != m_OriginsNameMap.end(); - } - - FilesOrigin &getByID(Index ID) - { - std::scoped_lock lock(m_Mutex); - return m_Origins[ID]; - } - - const FilesOrigin* findByID(Index ID) const - { - std::scoped_lock lock(m_Mutex); - - auto itor = m_Origins.find(ID); - - if (itor == m_Origins.end()) { - return nullptr; - } else { - return &itor->second; - } - } - - FilesOrigin &getByName(const std::wstring &name) - { - std::scoped_lock lock(m_Mutex); - - std::map::iterator iter = m_OriginsNameMap.find(name); - - if (iter != m_OriginsNameMap.end()) { - return m_Origins[iter->second]; - } else { - std::ostringstream stream; - stream << QObject::tr("invalid origin name: ").toStdString() << ToString(name, true); - throw std::runtime_error(stream.str()); - } - } - - void changePriorityLookup(int oldPriority, int newPriority) - { - std::scoped_lock lock(m_Mutex); - - auto iter = m_OriginsPriorityMap.find(oldPriority); - - if (iter != m_OriginsPriorityMap.end()) { - Index idx = iter->second; - m_OriginsPriorityMap.erase(iter); - m_OriginsPriorityMap[newPriority] = idx; - } - } - - void changeNameLookup(const std::wstring &oldName, const std::wstring &newName) - { - std::scoped_lock lock(m_Mutex); - - auto iter = m_OriginsNameMap.find(oldName); - - if (iter != m_OriginsNameMap.end()) { - Index idx = iter->second; - m_OriginsNameMap.erase(iter); - m_OriginsNameMap[newName] = idx; - } else { - log::error(QObject::tr("failed to change name lookup from {} to {}").toStdString(), oldName, newName); - } - } - -private: - Index m_NextID; - std::map m_Origins; - std::map m_OriginsNameMap; - std::map m_OriginsPriorityMap; - mutable std::mutex m_Mutex; - - Index createID() - { - return m_NextID++; - } - - FilesOrigin& createOriginNoLock( - const std::wstring &originName, const std::wstring &directory, int priority, - boost::shared_ptr fileRegister, - boost::shared_ptr originConnection) - { - int newID = createID(); - - auto itor = m_Origins.insert({newID, FilesOrigin( - newID, originName, directory, priority, - fileRegister, originConnection)}).first; - - m_OriginsNameMap.insert({originName, newID}); - m_OriginsPriorityMap.insert({priority, newID}); - - return itor->second; - } -}; - - -FileEntry::FileEntry() : - m_Index(UINT_MAX), m_Name(), m_Origin(-1), m_Parent(nullptr), - m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize) -{ -} - -FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : - m_Index(index), m_Name(std::move(name)), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), - m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize) -{ -} - -void FileEntry::addOrigin( - int origin, FILETIME fileTime, std::wstring_view archive, int order) -{ - std::scoped_lock lock(m_OriginsMutex); - - if (m_Parent != nullptr) { - m_Parent->propagateOrigin(origin); - } - - if (m_Origin == -1) { - // If this file has no previous origin, this mod is now the origin with no - // alternatives - m_Origin = origin; - m_FileTime = fileTime; - m_Archive = std::pair(std::wstring(archive.begin(), archive.end()), order); - } - else if ( - (m_Parent != nullptr) && ( - (m_Parent->getOriginByID(origin).getPriority() > m_Parent->getOriginByID(m_Origin).getPriority()) || - (archive.size() == 0 && m_Archive.first.size() > 0 )) - ) { - // If this mod has a higher priority than the origin mod OR - // this mod has a loose file and the origin mod has an archived file, - // this mod is now the origin and the previous origin is the first alternative - - auto itor = std::find_if( - m_Alternatives.begin(), m_Alternatives.end(), - [&](auto&& i) { return i.first == m_Origin; }); - - if (itor == m_Alternatives.end()) { - m_Alternatives.push_back({m_Origin, m_Archive}); - } - - m_Origin = origin; - m_FileTime = fileTime; - m_Archive = std::pair(std::wstring(archive.begin(), archive.end()), order); - } - else { - // This mod is just an alternative - bool found = false; - - if (m_Origin == origin) { - // already an origin - return; - } - - for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) { - if (iter->first == origin) { - // already an origin - return; - } - - if ((m_Parent != nullptr) && - (m_Parent->getOriginByID(iter->first).getPriority() < m_Parent->getOriginByID(origin).getPriority())) { - m_Alternatives.insert(iter, {origin, {std::wstring(archive.begin(), archive.end()), order}}); - found = true; - break; - } - } - - if (!found) { - m_Alternatives.push_back({origin, {std::wstring(archive.begin(), archive.end()), order}}); - } - } -} - -bool FileEntry::removeOrigin(int origin) -{ - std::scoped_lock lock(m_OriginsMutex); - - if (m_Origin == origin) { - if (!m_Alternatives.empty()) { - // find alternative with the highest priority - auto currentIter = m_Alternatives.begin(); - for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) { - if (iter->first != origin) { - //Both files are not from archives. - if (!iter->second.first.size() && !currentIter->second.first.size()) { - if ((m_Parent->getOriginByID(iter->first).getPriority() > m_Parent->getOriginByID(currentIter->first).getPriority())) { - currentIter = iter; - } - } - else { - //Both files are from archives - if (iter->second.first.size() && currentIter->second.first.size()) { - if (iter->second.second > currentIter->second.second) { - currentIter = iter; - } - } - else { - //Only one of the two is an archive, so we change currentIter only if he is the archive one. - if (currentIter->second.first.size()) { - currentIter = iter; - } - } - } - } - } - - int currentID = currentIter->first; - m_Archive = currentIter->second; - m_Alternatives.erase(currentIter); - - m_Origin = currentID; - } else { - m_Origin = -1; - m_Archive = std::pair(L"", -1); - return true; - } - } else { - auto newEnd = std::remove_if( - m_Alternatives.begin(), m_Alternatives.end(), - [&](auto &i) { return i.first == origin; }); - - if (newEnd != m_Alternatives.end()) { - m_Alternatives.erase(newEnd, m_Alternatives.end()); - } - } - return false; -} - -void FileEntry::sortOrigins() -{ - std::scoped_lock lock(m_OriginsMutex); - - m_Alternatives.push_back({m_Origin, m_Archive}); - - std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) { - if (!LHS.second.first.size() && !RHS.second.first.size()) { - int l = m_Parent->getOriginByID(LHS.first).getPriority(); - if (l < 0) { - l = INT_MAX; - } - - int r = m_Parent->getOriginByID(RHS.first).getPriority(); - if (r < 0) { - r = INT_MAX; - } - - return l < r; - } - - if (LHS.second.first.size() && RHS.second.first.size()) { - int l = LHS.second.second; if (l < 0) l = INT_MAX; - int r = RHS.second.second; if (r < 0) r = INT_MAX; - - return l < r; - } - - if (RHS.second.first.size()) { - return false; - } - - return true; - }); - - if (!m_Alternatives.empty()) { - m_Origin = m_Alternatives.back().first; - m_Archive = m_Alternatives.back().second; - m_Alternatives.pop_back(); - } -} - -bool FileEntry::isFromArchive(std::wstring archiveName) const -{ - std::scoped_lock lock(m_OriginsMutex); - - if (archiveName.length() == 0) { - return m_Archive.first.length() != 0; - } - - if (m_Archive.first.compare(archiveName) == 0) { - return true; - } - - for (auto alternative : m_Alternatives) { - if (alternative.second.first.compare(archiveName) == 0) { - return true; - } - } - - return false; -} - -std::wstring FileEntry::getFullPath(int originID) const -{ - std::scoped_lock lock(m_OriginsMutex); - - if (originID == -1) { - bool ignore = false; - originID = getOrigin(ignore); - } - - // base directory for origin - const auto* o = m_Parent->findOriginByID(originID); - if (!o) { - return {}; - } - - std::wstring result = o->getPath(); - - // all intermediate directories - recurseParents(result, m_Parent); - - return result + L"\\" + m_Name; -} - -std::wstring FileEntry::getRelativePath() const -{ - std::wstring result; - - // all intermediate directories - recurseParents(result, m_Parent); - - return result + L"\\" + m_Name; -} - -bool FileEntry::recurseParents(std::wstring &path, const DirectoryEntry *parent) const -{ - if (parent == nullptr) { - return false; - } else { - // don't append the topmost parent because it is the virtual data-root - if (recurseParents(path, parent->getParent())) { - path.append(L"\\").append(parent->getName()); - } - - return true; - } -} - - -FilesOrigin::FilesOrigin() - : m_ID(0), m_Disabled(false), m_Name(), m_Path(), m_Priority(0) -{ -} - -FilesOrigin::FilesOrigin(const FilesOrigin &reference) - : m_ID(reference.m_ID) - , m_Disabled(reference.m_Disabled) - , m_Name(reference.m_Name) - , m_Path(reference.m_Path) - , m_Priority(reference.m_Priority) - , m_FileRegister(reference.m_FileRegister) - , m_OriginConnection(reference.m_OriginConnection) -{ -} - -FilesOrigin::FilesOrigin( - int ID, const std::wstring &name, const std::wstring &path, int priority, - boost::shared_ptr fileRegister, - boost::shared_ptr originConnection) : - m_ID(ID), m_Disabled(false), m_Name(name), m_Path(path), - m_Priority(priority), m_FileRegister(fileRegister), - m_OriginConnection(originConnection) -{ -} - -void FilesOrigin::setPriority(int priority) -{ - m_OriginConnection.lock()->changePriorityLookup(m_Priority, priority); - - m_Priority = priority; -} - -void FilesOrigin::setName(const std::wstring &name) -{ - m_OriginConnection.lock()->changeNameLookup(m_Name, name); - - // change path too - if (tail(m_Path, m_Name.length()) == m_Name) { - m_Path = m_Path.substr(0, m_Path.length() - m_Name.length()).append(name); - } - - m_Name = name; -} - -std::vector FilesOrigin::getFiles() const -{ - std::vector result; - - { - std::scoped_lock lock(m_Mutex); - - for (FileEntry::Index fileIdx : m_Files) { - if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { - result.push_back(p); - } - } - } - - return result; -} - -FileEntry::Ptr FilesOrigin::findFile(FileEntry::Index index) const -{ - return m_FileRegister.lock()->getFile(index); -} - -void FilesOrigin::enable(bool enabled) -{ - DirectoryStats dummy; - enable(enabled, dummy); -} - -void FilesOrigin::enable(bool enabled, DirectoryStats& stats) -{ - if (!enabled) { - ++stats.originsNeededEnabled; - - std::set copy; - - { - std::scoped_lock lock(m_Mutex); - copy = m_Files; - m_Files.clear(); - } - - m_FileRegister.lock()->removeOriginMulti(copy, m_ID); - } - - m_Disabled = !enabled; -} - -void FilesOrigin::removeFile(FileEntry::Index index) -{ - std::scoped_lock lock(m_Mutex); - - auto iter = m_Files.find(index); - - if (iter != m_Files.end()) { - m_Files.erase(iter); - } -} - -bool FilesOrigin::containsArchive(std::wstring archiveName) -{ - std::scoped_lock lock(m_Mutex); - - for (FileEntry::Index fileIdx : m_Files) { - if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { - if (p->isFromArchive(archiveName)) { - return true; - } - } - } - - return false; -} - - -FileRegister::FileRegister(boost::shared_ptr originConnection) - : m_OriginConnection(originConnection), m_NextIndex(0) -{ -} - -bool FileRegister::indexValid(FileEntry::Index index) const -{ - std::scoped_lock lock(m_Mutex); - - if (index < m_Files.size()) { - return (m_Files[index].get() != nullptr); - } - - return false; -} - -FileEntry::Ptr FileRegister::createFile( - std::wstring name, DirectoryEntry *parent, DirectoryStats& stats) -{ - const auto index = generateIndex(); - auto p = FileEntry::Ptr(new FileEntry(index, std::move(name), parent)); - - { - std::scoped_lock lock(m_Mutex); - - if (index >= m_Files.size()) { - m_Files.resize(index + 1); - } - - m_Files[index] = p; - } - - return p; -} - -FileEntry::Index FileRegister::generateIndex() -{ - return m_NextIndex++; -} - -FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const -{ - std::scoped_lock lock(m_Mutex); - - if (index < m_Files.size()) { - return m_Files[index]; - } else { - return {}; - } -} - -bool FileRegister::removeFile(FileEntry::Index index) -{ - std::scoped_lock lock(m_Mutex); - - if (index < m_Files.size()) { - FileEntry::Ptr p; - m_Files[index].swap(p); - - if (p) { - unregisterFile(p); - return true; - } - } - - log::error(QObject::tr("invalid file index for remove: {}").toStdString(), index); - return false; -} - -void FileRegister::removeOrigin(FileEntry::Index index, int originID) -{ - std::unique_lock lock(m_Mutex); - - if (index < m_Files.size()) { - FileEntry::Ptr& p = m_Files[index]; - - if (p) { - if (p->removeOrigin(originID)) { - m_Files[index] = {}; - lock.unlock(); - unregisterFile(p); - return; - } - } - } - - log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index); -} - -void FileRegister::removeOriginMulti( - std::set indices, int originID) -{ - std::vector removedFiles; - - { - std::scoped_lock lock(m_Mutex); - - for (auto iter = indices.begin(); iter != indices.end(); ) { - const auto index = *iter; - - if (index < m_Files.size()) { - const auto& p = m_Files[index]; - - if (p && p->removeOrigin(originID)) { - removedFiles.push_back(p); - m_Files[index] = {}; - ++iter; - continue; - } - } - - iter = indices.erase(iter); - } - } - - // optimization: this is only called when disabling an origin and in this case - // we don't have to remove the file from the origin - - // need to remove files from their parent directories. multiple ways to go - // about this: - // a) for each file, search its parents file-list (preferably by name) and - // remove what is found - // b) gather the parent directories, go through the file list for each once - // and remove all files that have been removed - // - // the latter should be faster when there are many files in few directories. - // since this is called only when disabling an origin that is probably - // frequently the case - - std::set parents; - for (const FileEntry::Ptr &file : removedFiles) { - if (file->getParent() != nullptr) { - parents.insert(file->getParent()); - } - } - - for (DirectoryEntry *parent : parents) { - parent->removeFiles(indices); - } -} - -void FileRegister::sortOrigins() -{ - std::scoped_lock lock(m_Mutex); - - for (auto&& p : m_Files) { - if (p) { - p->sortOrigins(); - } - } -} - -void FileRegister::unregisterFile(FileEntry::Ptr file) -{ - bool ignore; - - // unregister from origin - int originID = file->getOrigin(ignore); - m_OriginConnection->getByID(originID).removeFile(file->getIndex()); - const auto& alternatives = file->getAlternatives(); - - for (auto iter = alternatives.begin(); iter != alternatives.end(); ++iter) { - m_OriginConnection->getByID(iter->first).removeFile(file->getIndex()); - } - - // unregister from directory - if (file->getParent() != nullptr) { - file->getParent()->removeFile(file->getIndex()); - } -} - - DirectoryEntry::DirectoryEntry( std::wstring name, DirectoryEntry *parent, int originID) : m_OriginConnection(new OriginConnection), @@ -1046,7 +349,7 @@ int DirectoryEntry::anyOrigin() const bool ignore; for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) { - FileEntry::Ptr entry = m_FileRegister->getFile(iter->second); + FileEntryPtr entry = m_FileRegister->getFile(iter->second); if ((entry.get() != nullptr) && !entry->isFromArchive()) { return entry->getOrigin(ignore); } @@ -1064,9 +367,9 @@ int DirectoryEntry::anyOrigin() const return *(m_Origins.begin()); } -std::vector DirectoryEntry::getFiles() const +std::vector DirectoryEntry::getFiles() const { - std::vector result; + std::vector result; for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) { result.push_back(m_FileRegister->getFile(iter->second)); @@ -1098,7 +401,7 @@ DirectoryEntry *DirectoryEntry::findSubDirectoryRecursive(const std::wstring &pa return getSubDirectoryRecursive(path, false, -1); } -const FileEntry::Ptr DirectoryEntry::findFile( +const FileEntryPtr DirectoryEntry::findFile( const std::wstring &name, bool alreadyLowerCase) const { FilesLookup::const_iterator iter; @@ -1112,18 +415,18 @@ const FileEntry::Ptr DirectoryEntry::findFile( if (iter != m_FilesLookup.end()) { return m_FileRegister->getFile(iter->second); } else { - return FileEntry::Ptr(); + return FileEntryPtr(); } } -const FileEntry::Ptr DirectoryEntry::findFile(const FileKey& key) const +const FileEntryPtr DirectoryEntry::findFile(const FileKey& key) const { auto iter = m_FilesLookup.find(key); if (iter != m_FilesLookup.end()) { return m_FileRegister->getFile(iter->second); } else { - return FileEntry::Ptr(); + return FileEntryPtr(); } } @@ -1135,7 +438,7 @@ bool DirectoryEntry::hasFile(const std::wstring& name) const bool DirectoryEntry::containsArchive(std::wstring archiveName) { for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) { - FileEntry::Ptr entry = m_FileRegister->getFile(iter->second); + FileEntryPtr entry = m_FileRegister->getFile(iter->second); if (entry->isFromArchive(archiveName)) { return true; } @@ -1144,7 +447,7 @@ bool DirectoryEntry::containsArchive(std::wstring archiveName) return false; } -const FileEntry::Ptr DirectoryEntry::searchFile( +const FileEntryPtr DirectoryEntry::searchFile( const std::wstring &path, const DirectoryEntry **directory) const { if (directory != nullptr) { @@ -1157,7 +460,7 @@ const FileEntry::Ptr DirectoryEntry::searchFile( *directory = this; } - return FileEntry::Ptr(); + return FileEntryPtr(); } const size_t len = path.find_first_of(L"\\/"); @@ -1182,14 +485,14 @@ const FileEntry::Ptr DirectoryEntry::searchFile( if (temp != nullptr) { if (len >= path.size()) { log::error(QObject::tr("unexpected end of path").toStdString()); - return FileEntry::Ptr(); + return FileEntryPtr(); } return temp->searchFile(path.substr(len + 1), directory); } } - return FileEntry::Ptr(); + return FileEntryPtr(); } void DirectoryEntry::removeFile(FileEntry::Index index) @@ -1251,7 +554,7 @@ bool DirectoryEntry::remove(const std::wstring &fileName, int *origin) if (iter != m_Files.end()) { if (origin != nullptr) { - FileEntry::Ptr entry = m_FileRegister->getFile(iter->second); + FileEntryPtr entry = m_FileRegister->getFile(iter->second); if (entry.get() != nullptr) { bool ignore; *origin = entry->getOrigin(ignore); @@ -1291,12 +594,12 @@ void DirectoryEntry::removeFiles(const std::set &indices) removeFilesFromList(indices); } -FileEntry::Ptr DirectoryEntry::insert( +FileEntryPtr DirectoryEntry::insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, std::wstring_view archive, int order, DirectoryStats& stats) { std::wstring fileNameLower = ToLowerCopy(fileName); - FileEntry::Ptr fe; + FileEntryPtr fe; FileKey key(std::move(fileNameLower)); @@ -1337,11 +640,11 @@ FileEntry::Ptr DirectoryEntry::insert( return fe; } -FileEntry::Ptr DirectoryEntry::insert( +FileEntryPtr DirectoryEntry::insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats) { - FileEntry::Ptr fe; + FileEntryPtr fe; { std::unique_lock lock(m_FilesMutex); @@ -1730,7 +1033,7 @@ void DirectoryEntry::dump(std::FILE* f, const std::wstring& parentPath) const continue; } - const auto o = m_OriginConnection->getByID(file->getOrigin()); + const auto& o = m_OriginConnection->getByID(file->getOrigin()); const auto path = parentPath + L"\\" + file->getName(); const auto line = path + L"\t(" + o.getName() + L")\r\n"; diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index ce50da44..df946946 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -17,29 +17,18 @@ You should have received a copy of the GNU General Public License along with Mod Organizer. If not, see . */ -#ifndef DIRECTORYENTRY_H -#define DIRECTORYENTRY_H +#ifndef MO_REGISTER_DIRECTORYENTRY_INCLUDED +#define MO_REGISTER_DIRECTORYENTRY_INCLUDED - -#include -#include -#include -#include -#include -#include - -#define WIN32_MEAN_AND_LEAN -#include +#include "fileregister.h" #include -#ifndef Q_MOC_RUN -#include -#include -#endif - -#include "util.h" -#include "envfs.h" -namespace MOShared { struct DirectoryEntryFileKey; } +namespace env +{ + class DirectoryWalker; + struct Directory; + struct File; +} namespace std { @@ -57,309 +46,6 @@ namespace std namespace MOShared { -class DirectoryEntry; -class OriginConnection; -class FileRegister; - - -struct DirectoryStats -{ - static constexpr bool EnableInstrumentation = false; - - std::string mod; - - std::chrono::nanoseconds dirTimes; - std::chrono::nanoseconds fileTimes; - std::chrono::nanoseconds sortTimes; - - std::chrono::nanoseconds subdirLookupTimes; - std::chrono::nanoseconds addDirectoryTimes; - - std::chrono::nanoseconds filesLookupTimes; - std::chrono::nanoseconds addFileTimes; - std::chrono::nanoseconds addOriginToFileTimes; - std::chrono::nanoseconds addFileToOriginTimes; - std::chrono::nanoseconds addFileToRegisterTimes; - - int64_t originExists; - int64_t originCreate; - int64_t originsNeededEnabled; - - int64_t subdirExists; - int64_t subdirCreate; - - int64_t fileExists; - int64_t fileCreate; - int64_t filesInsertedInRegister; - int64_t filesAssignedInRegister; - - DirectoryStats(); - - DirectoryStats& operator+=(const DirectoryStats& o); - - static std::string csvHeader(); - std::string toCsv() const; -}; - - -class FileEntry -{ -public: - static constexpr uint64_t NoFileSize = - std::numeric_limits::max(); - - typedef unsigned int Index; - typedef boost::shared_ptr Ptr; - - // a vector of {originId, {archiveName, order}} - // - // if a file is in an archive, archiveName is the name of the bsa and order - // is the order of the associated plugin in the plugins list - // - // is a file is not in an archive, archiveName is empty and order is usually - // -1 - typedef std::vector>> - AlternativesVector; - - FileEntry(); - FileEntry(Index index, std::wstring name, DirectoryEntry *parent); - - Index getIndex() const - { - return m_Index; - } - - void addOrigin( - int origin, FILETIME fileTime, std::wstring_view archive, int order); - - // remove the specified origin from the list of origins that contain this - // file. if no origin is left, the file is effectively deleted and true is - // returned. otherwise, false is returned - bool removeOrigin(int origin); - - void sortOrigins(); - - // gets the list of alternative origins (origins with lower priority than - // the primary one). if sortOrigins has been called, it is sorted by priority - // (ascending) - const AlternativesVector &getAlternatives() const - { - return m_Alternatives; - } - - const std::wstring &getName() const - { - return m_Name; - } - - int getOrigin() const - { - return m_Origin; - } - - int getOrigin(bool &archive) const - { - archive = (m_Archive.first.length() != 0); - return m_Origin; - } - - const std::pair &getArchive() const - { - return m_Archive; - } - - bool isFromArchive(std::wstring archiveName = L"") const; - - // if originID is -1, uses the main origin; if this file doesn't exist in the - // given origin, returns an empty string - // - std::wstring getFullPath(int originID=-1) const; - - std::wstring getRelativePath() const; - - DirectoryEntry *getParent() - { - return m_Parent; - } - - void setFileTime(FILETIME fileTime) const - { - m_FileTime = fileTime; - } - - FILETIME getFileTime() const - { - return m_FileTime; - } - - void setFileSize(uint64_t size, uint64_t compressedSize) - { - m_FileSize = size; - m_CompressedFileSize = compressedSize; - } - - uint64_t getFileSize() const - { - return m_FileSize; - } - - uint64_t getCompressedFileSize() const - { - return m_CompressedFileSize; - } - -private: - Index m_Index; - std::wstring m_Name; - int m_Origin; - std::pair m_Archive; - AlternativesVector m_Alternatives; - DirectoryEntry *m_Parent; - mutable FILETIME m_FileTime; - uint64_t m_FileSize, m_CompressedFileSize; - mutable std::mutex m_OriginsMutex; - - bool recurseParents(std::wstring &path, const DirectoryEntry *parent) const; -}; - - -// represents a mod or the data directory, providing files to the tree -class FilesOrigin -{ - friend class OriginConnection; - -public: - FilesOrigin(); - FilesOrigin(const FilesOrigin &reference); - - // sets priority for this origin, but it will overwrite the existing mapping - // for this priority, the previous origin will no longer be referenced - void setPriority(int priority); - - int getPriority() const - { - return m_Priority; - } - - void setName(const std::wstring &name); - const std::wstring &getName() const - { - return m_Name; - } - - int getID() const - { - return m_ID; - } - - const std::wstring &getPath() const - { - return m_Path; - } - - std::vector getFiles() const; - FileEntry::Ptr findFile(FileEntry::Index index) const; - - void enable(bool enabled, DirectoryStats& stats); - void enable(bool enabled); - - bool isDisabled() const - { - return m_Disabled; - } - - void addFile(FileEntry::Index index) - { - std::scoped_lock lock(m_Mutex); - m_Files.insert(index); - } - - void removeFile(FileEntry::Index index); - - bool containsArchive(std::wstring archiveName); - -private: - int m_ID; - bool m_Disabled; - std::set m_Files; - std::wstring m_Name; - std::wstring m_Path; - int m_Priority; - boost::weak_ptr m_FileRegister; - boost::weak_ptr m_OriginConnection; - mutable std::mutex m_Mutex; - - FilesOrigin( - int ID, const std::wstring &name, const std::wstring &path, int priority, - boost::shared_ptr fileRegister, - boost::shared_ptr originConnection); -}; - - -class FileRegister -{ -public: - FileRegister(boost::shared_ptr originConnection); - - bool indexValid(FileEntry::Index index) const; - - FileEntry::Ptr createFile( - std::wstring name, DirectoryEntry *parent, DirectoryStats& stats); - - FileEntry::Ptr getFile(FileEntry::Index index) const; - - size_t highestCount() const - { - std::scoped_lock lock(m_Mutex); - return m_Files.size(); - } - - void reserve(std::size_t n) - { - m_Files.reserve(n); - } - - bool removeFile(FileEntry::Index index); - void removeOrigin(FileEntry::Index index, int originID); - void removeOriginMulti(std::set indices, int originID); - - void sortOrigins(); - -private: - using FileMap = std::vector; - - mutable std::mutex m_Mutex; - FileMap m_Files; - boost::shared_ptr m_OriginConnection; - std::atomic m_NextIndex; - - void unregisterFile(FileEntry::Ptr file); - FileEntry::Index generateIndex(); -}; - - -struct DirectoryEntryFileKey -{ - DirectoryEntryFileKey(std::wstring v) - : value(std::move(v)), hash(getHash(value)) - { - } - - bool operator==(const DirectoryEntryFileKey& o) const - { - return (value == o.value); - } - - static std::size_t getHash(const std::wstring& value) - { - return std::hash()(value); - } - - std::wstring value; - const std::size_t hash; -}; - - class DirectoryEntry { public: @@ -373,6 +59,10 @@ public: boost::shared_ptr fileRegister, boost::shared_ptr originConnection); + // noncopyable + DirectoryEntry(const DirectoryEntry&) = delete; + DirectoryEntry& operator=(const DirectoryEntry&) = delete; + ~DirectoryEntry(); void clear(); @@ -439,7 +129,7 @@ public: int anyOrigin() const; - std::vector getFiles() const; + std::vector getFiles() const; void getSubDirectories( std::vector::const_iterator &begin, @@ -486,7 +176,7 @@ public: } } - FileEntry::Ptr getFileByIndex(FileEntry::Index index) const + FileEntryPtr getFileByIndex(FileIndex index) const { return m_FileRegister->getFile(index); } @@ -500,8 +190,8 @@ public: * @param name name of the file * @return fileentry object for the file or nullptr if no file matches */ - const FileEntry::Ptr findFile(const std::wstring &name, bool alreadyLowerCase=false) const; - const FileEntry::Ptr findFile(const FileKey& key) const; + const FileEntryPtr findFile(const std::wstring &name, bool alreadyLowerCase=false) const; + const FileEntryPtr findFile(const FileKey& key) const; bool hasFile(const std::wstring& name) const; bool containsArchive(std::wstring archiveName); @@ -512,10 +202,10 @@ public: // if directory is not nullptr, the referenced variable will be set to the // path containing the file // - const FileEntry::Ptr searchFile( + const FileEntryPtr searchFile( const std::wstring &path, const DirectoryEntry **directory=nullptr) const; - void removeFile(FileEntry::Index index); + void removeFile(FileIndex index); // remove the specified file from the tree. This can be a path leading to a // file in a subdirectory @@ -535,13 +225,13 @@ public: const std::wstring &originName, const std::wstring &directory, int priority, DirectoryStats& stats); - void removeFiles(const std::set &indices); + void removeFiles(const std::set &indices); void dump(const std::wstring& file) const; private: - using FilesMap = std::map; - using FilesLookup = std::unordered_map; + using FilesMap = std::map; + using FilesLookup = std::unordered_map; using SubDirectories = std::vector; using SubDirectoriesLookup = std::unordered_map; @@ -563,13 +253,11 @@ private: mutable std::mutex m_OriginsMutex; - DirectoryEntry(const DirectoryEntry &reference); - - FileEntry::Ptr insert( + FileEntryPtr insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, std::wstring_view archive, int order, DirectoryStats& stats); - FileEntry::Ptr insert( + FileEntryPtr insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats); @@ -599,9 +287,9 @@ private: void addDirectoryToList(DirectoryEntry* e, std::wstring nameLc); void removeDirectoryFromList(SubDirectories::iterator itor); - void addFileToList(std::wstring fileNameLower, FileEntry::Index index); - void removeFileFromList(FileEntry::Index index); - void removeFilesFromList(const std::set& indices); + void addFileToList(std::wstring fileNameLower, FileIndex index); + void removeFileFromList(FileIndex index); + void removeFilesFromList(const std::set& indices); struct Context; static void onDirectoryStart(Context* cx, std::wstring_view path); @@ -624,4 +312,4 @@ namespace std } } -#endif // DIRECTORYENTRY_H +#endif // MO_REGISTER_DIRECTORYENTRY_INCLUDED diff --git a/src/shared/fileentry.cpp b/src/shared/fileentry.cpp new file mode 100644 index 00000000..d4e1beb4 --- /dev/null +++ b/src/shared/fileentry.cpp @@ -0,0 +1,251 @@ +#include "fileentry.h" +#include "directoryentry.h" +#include "filesorigin.h" + +namespace MOShared +{ + +FileEntry::FileEntry() : + m_Index(UINT_MAX), m_Name(), m_Origin(-1), m_Parent(nullptr), + m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize) +{ +} + +FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : + m_Index(index), m_Name(std::move(name)), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), + m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize) +{ +} + +void FileEntry::addOrigin( + int origin, FILETIME fileTime, std::wstring_view archive, int order) +{ + std::scoped_lock lock(m_OriginsMutex); + + if (m_Parent != nullptr) { + m_Parent->propagateOrigin(origin); + } + + if (m_Origin == -1) { + // If this file has no previous origin, this mod is now the origin with no + // alternatives + m_Origin = origin; + m_FileTime = fileTime; + m_Archive = std::pair(std::wstring(archive.begin(), archive.end()), order); + } + else if ( + (m_Parent != nullptr) && ( + (m_Parent->getOriginByID(origin).getPriority() > m_Parent->getOriginByID(m_Origin).getPriority()) || + (archive.size() == 0 && m_Archive.first.size() > 0 )) + ) { + // If this mod has a higher priority than the origin mod OR + // this mod has a loose file and the origin mod has an archived file, + // this mod is now the origin and the previous origin is the first alternative + + auto itor = std::find_if( + m_Alternatives.begin(), m_Alternatives.end(), + [&](auto&& i) { return i.first == m_Origin; }); + + if (itor == m_Alternatives.end()) { + m_Alternatives.push_back({m_Origin, m_Archive}); + } + + m_Origin = origin; + m_FileTime = fileTime; + m_Archive = std::pair(std::wstring(archive.begin(), archive.end()), order); + } + else { + // This mod is just an alternative + bool found = false; + + if (m_Origin == origin) { + // already an origin + return; + } + + for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) { + if (iter->first == origin) { + // already an origin + return; + } + + if ((m_Parent != nullptr) && + (m_Parent->getOriginByID(iter->first).getPriority() < m_Parent->getOriginByID(origin).getPriority())) { + m_Alternatives.insert(iter, {origin, {std::wstring(archive.begin(), archive.end()), order}}); + found = true; + break; + } + } + + if (!found) { + m_Alternatives.push_back({origin, {std::wstring(archive.begin(), archive.end()), order}}); + } + } +} + +bool FileEntry::removeOrigin(int origin) +{ + std::scoped_lock lock(m_OriginsMutex); + + if (m_Origin == origin) { + if (!m_Alternatives.empty()) { + // find alternative with the highest priority + auto currentIter = m_Alternatives.begin(); + for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) { + if (iter->first != origin) { + //Both files are not from archives. + if (!iter->second.first.size() && !currentIter->second.first.size()) { + if ((m_Parent->getOriginByID(iter->first).getPriority() > m_Parent->getOriginByID(currentIter->first).getPriority())) { + currentIter = iter; + } + } + else { + //Both files are from archives + if (iter->second.first.size() && currentIter->second.first.size()) { + if (iter->second.second > currentIter->second.second) { + currentIter = iter; + } + } + else { + //Only one of the two is an archive, so we change currentIter only if he is the archive one. + if (currentIter->second.first.size()) { + currentIter = iter; + } + } + } + } + } + + int currentID = currentIter->first; + m_Archive = currentIter->second; + m_Alternatives.erase(currentIter); + + m_Origin = currentID; + } else { + m_Origin = -1; + m_Archive = std::pair(L"", -1); + return true; + } + } else { + auto newEnd = std::remove_if( + m_Alternatives.begin(), m_Alternatives.end(), + [&](auto &i) { return i.first == origin; }); + + if (newEnd != m_Alternatives.end()) { + m_Alternatives.erase(newEnd, m_Alternatives.end()); + } + } + return false; +} + +void FileEntry::sortOrigins() +{ + std::scoped_lock lock(m_OriginsMutex); + + m_Alternatives.push_back({m_Origin, m_Archive}); + + std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) { + if (!LHS.second.first.size() && !RHS.second.first.size()) { + int l = m_Parent->getOriginByID(LHS.first).getPriority(); + if (l < 0) { + l = INT_MAX; + } + + int r = m_Parent->getOriginByID(RHS.first).getPriority(); + if (r < 0) { + r = INT_MAX; + } + + return l < r; + } + + if (LHS.second.first.size() && RHS.second.first.size()) { + int l = LHS.second.second; if (l < 0) l = INT_MAX; + int r = RHS.second.second; if (r < 0) r = INT_MAX; + + return l < r; + } + + if (RHS.second.first.size()) { + return false; + } + + return true; + }); + + if (!m_Alternatives.empty()) { + m_Origin = m_Alternatives.back().first; + m_Archive = m_Alternatives.back().second; + m_Alternatives.pop_back(); + } +} + +bool FileEntry::isFromArchive(std::wstring archiveName) const +{ + std::scoped_lock lock(m_OriginsMutex); + + if (archiveName.length() == 0) { + return m_Archive.first.length() != 0; + } + + if (m_Archive.first.compare(archiveName) == 0) { + return true; + } + + for (auto alternative : m_Alternatives) { + if (alternative.second.first.compare(archiveName) == 0) { + return true; + } + } + + return false; +} + +std::wstring FileEntry::getFullPath(int originID) const +{ + std::scoped_lock lock(m_OriginsMutex); + + if (originID == -1) { + bool ignore = false; + originID = getOrigin(ignore); + } + + // base directory for origin + const auto* o = m_Parent->findOriginByID(originID); + if (!o) { + return {}; + } + + std::wstring result = o->getPath(); + + // all intermediate directories + recurseParents(result, m_Parent); + + return result + L"\\" + m_Name; +} + +std::wstring FileEntry::getRelativePath() const +{ + std::wstring result; + + // all intermediate directories + recurseParents(result, m_Parent); + + return result + L"\\" + m_Name; +} + +bool FileEntry::recurseParents(std::wstring &path, const DirectoryEntry *parent) const +{ + if (parent == nullptr) { + return false; + } else { + // don't append the topmost parent because it is the virtual data-root + if (recurseParents(path, parent->getParent())) { + path.append(L"\\").append(parent->getName()); + } + + return true; + } +} + +} // namespace diff --git a/src/shared/fileentry.h b/src/shared/fileentry.h new file mode 100644 index 00000000..52ce89cd --- /dev/null +++ b/src/shared/fileentry.h @@ -0,0 +1,125 @@ +#ifndef MO_REGISTER_FILEENTRY_INCLUDED +#define MO_REGISTER_FILEENTRY_INCLUDED + +#include "fileregisterfwd.h" + +namespace MOShared +{ + +class FileEntry +{ +public: + static constexpr uint64_t NoFileSize = + std::numeric_limits::max(); + + typedef unsigned int Index; + typedef boost::shared_ptr Ptr; + + FileEntry(); + FileEntry(Index index, std::wstring name, DirectoryEntry *parent); + + // noncopyable + FileEntry(const FileEntry&) = delete; + FileEntry& operator=(const FileEntry&) = delete; + + Index getIndex() const + { + return m_Index; + } + + void addOrigin( + int origin, FILETIME fileTime, std::wstring_view archive, int order); + + // remove the specified origin from the list of origins that contain this + // file. if no origin is left, the file is effectively deleted and true is + // returned. otherwise, false is returned + bool removeOrigin(int origin); + + void sortOrigins(); + + // gets the list of alternative origins (origins with lower priority than + // the primary one). if sortOrigins has been called, it is sorted by priority + // (ascending) + const AlternativesVector &getAlternatives() const + { + return m_Alternatives; + } + + const std::wstring &getName() const + { + return m_Name; + } + + int getOrigin() const + { + return m_Origin; + } + + int getOrigin(bool &archive) const + { + archive = (m_Archive.first.length() != 0); + return m_Origin; + } + + const std::pair &getArchive() const + { + return m_Archive; + } + + bool isFromArchive(std::wstring archiveName = L"") const; + + // if originID is -1, uses the main origin; if this file doesn't exist in the + // given origin, returns an empty string + // + std::wstring getFullPath(int originID=-1) const; + + std::wstring getRelativePath() const; + + DirectoryEntry *getParent() + { + return m_Parent; + } + + void setFileTime(FILETIME fileTime) const + { + m_FileTime = fileTime; + } + + FILETIME getFileTime() const + { + return m_FileTime; + } + + void setFileSize(uint64_t size, uint64_t compressedSize) + { + m_FileSize = size; + m_CompressedFileSize = compressedSize; + } + + uint64_t getFileSize() const + { + return m_FileSize; + } + + uint64_t getCompressedFileSize() const + { + return m_CompressedFileSize; + } + +private: + Index m_Index; + std::wstring m_Name; + int m_Origin; + std::pair m_Archive; + AlternativesVector m_Alternatives; + DirectoryEntry *m_Parent; + mutable FILETIME m_FileTime; + uint64_t m_FileSize, m_CompressedFileSize; + mutable std::mutex m_OriginsMutex; + + bool recurseParents(std::wstring &path, const DirectoryEntry *parent) const; +}; + +} // namespace + +#endif // MO_REGISTER_FILEENTRY_INCLUDED diff --git a/src/shared/fileregister.cpp b/src/shared/fileregister.cpp new file mode 100644 index 00000000..4add0455 --- /dev/null +++ b/src/shared/fileregister.cpp @@ -0,0 +1,184 @@ +#include "fileregister.h" +#include "fileentry.h" +#include "directoryentry.h" +#include "originconnection.h" +#include "filesorigin.h" +#include + +namespace MOShared +{ + +using namespace MOBase; + +FileRegister::FileRegister(boost::shared_ptr originConnection) + : m_OriginConnection(originConnection), m_NextIndex(0) +{ +} + +bool FileRegister::indexValid(FileIndex index) const +{ + std::scoped_lock lock(m_Mutex); + + if (index < m_Files.size()) { + return (m_Files[index].get() != nullptr); + } + + return false; +} + +FileEntryPtr FileRegister::createFile( + std::wstring name, DirectoryEntry *parent, DirectoryStats& stats) +{ + const auto index = generateIndex(); + auto p = FileEntryPtr(new FileEntry(index, std::move(name), parent)); + + { + std::scoped_lock lock(m_Mutex); + + if (index >= m_Files.size()) { + m_Files.resize(index + 1); + } + + m_Files[index] = p; + } + + return p; +} + +FileIndex FileRegister::generateIndex() +{ + return m_NextIndex++; +} + +FileEntryPtr FileRegister::getFile(FileIndex index) const +{ + std::scoped_lock lock(m_Mutex); + + if (index < m_Files.size()) { + return m_Files[index]; + } else { + return {}; + } +} + +bool FileRegister::removeFile(FileIndex index) +{ + std::scoped_lock lock(m_Mutex); + + if (index < m_Files.size()) { + FileEntryPtr p; + m_Files[index].swap(p); + + if (p) { + unregisterFile(p); + return true; + } + } + + log::error(QObject::tr("invalid file index for remove: {}").toStdString(), index); + return false; +} + +void FileRegister::removeOrigin(FileIndex index, int originID) +{ + std::unique_lock lock(m_Mutex); + + if (index < m_Files.size()) { + FileEntryPtr& p = m_Files[index]; + + if (p) { + if (p->removeOrigin(originID)) { + m_Files[index] = {}; + lock.unlock(); + unregisterFile(p); + return; + } + } + } + + log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index); +} + +void FileRegister::removeOriginMulti( + std::set indices, int originID) +{ + std::vector removedFiles; + + { + std::scoped_lock lock(m_Mutex); + + for (auto iter = indices.begin(); iter != indices.end(); ) { + const auto index = *iter; + + if (index < m_Files.size()) { + const auto& p = m_Files[index]; + + if (p && p->removeOrigin(originID)) { + removedFiles.push_back(p); + m_Files[index] = {}; + ++iter; + continue; + } + } + + iter = indices.erase(iter); + } + } + + // optimization: this is only called when disabling an origin and in this case + // we don't have to remove the file from the origin + + // need to remove files from their parent directories. multiple ways to go + // about this: + // a) for each file, search its parents file-list (preferably by name) and + // remove what is found + // b) gather the parent directories, go through the file list for each once + // and remove all files that have been removed + // + // the latter should be faster when there are many files in few directories. + // since this is called only when disabling an origin that is probably + // frequently the case + + std::set parents; + for (const FileEntryPtr &file : removedFiles) { + if (file->getParent() != nullptr) { + parents.insert(file->getParent()); + } + } + + for (DirectoryEntry *parent : parents) { + parent->removeFiles(indices); + } +} + +void FileRegister::sortOrigins() +{ + std::scoped_lock lock(m_Mutex); + + for (auto&& p : m_Files) { + if (p) { + p->sortOrigins(); + } + } +} + +void FileRegister::unregisterFile(FileEntryPtr file) +{ + bool ignore; + + // unregister from origin + int originID = file->getOrigin(ignore); + m_OriginConnection->getByID(originID).removeFile(file->getIndex()); + const auto& alternatives = file->getAlternatives(); + + for (auto iter = alternatives.begin(); iter != alternatives.end(); ++iter) { + m_OriginConnection->getByID(iter->first).removeFile(file->getIndex()); + } + + // unregister from directory + if (file->getParent() != nullptr) { + file->getParent()->removeFile(file->getIndex()); + } +} + +} // namespace diff --git a/src/shared/fileregister.h b/src/shared/fileregister.h new file mode 100644 index 00000000..d5e360c1 --- /dev/null +++ b/src/shared/fileregister.h @@ -0,0 +1,58 @@ +#ifndef MO_REGISTER_FILESREGISTER_INCLUDED +#define MO_REGISTER_FILESREGISTER_INCLUDED + +#include "fileregisterfwd.h" +#include +#include + +namespace MOShared +{ + +class FileRegister +{ +public: + FileRegister(boost::shared_ptr originConnection); + + // noncopyable + FileRegister(const FileRegister&) = delete; + FileRegister& operator=(const FileRegister&) = delete; + + bool indexValid(FileIndex index) const; + + FileEntryPtr createFile( + std::wstring name, DirectoryEntry *parent, DirectoryStats& stats); + + FileEntryPtr getFile(FileIndex index) const; + + size_t highestCount() const + { + std::scoped_lock lock(m_Mutex); + return m_Files.size(); + } + + void reserve(std::size_t n) + { + m_Files.reserve(n); + } + + bool removeFile(FileIndex index); + void removeOrigin(FileIndex index, int originID); + void removeOriginMulti(std::set indices, int originID); + + void sortOrigins(); + +private: + using FileMap = std::vector; + + mutable std::mutex m_Mutex; + FileMap m_Files; + boost::shared_ptr m_OriginConnection; + std::atomic m_NextIndex; + + void unregisterFile(FileEntryPtr file); + FileIndex generateIndex(); +}; + +} // namespace + +#endif // MO_REGISTER_FILESREGISTER_INCLUDED diff --git a/src/shared/fileregisterfwd.h b/src/shared/fileregisterfwd.h new file mode 100644 index 00000000..b8d33378 --- /dev/null +++ b/src/shared/fileregisterfwd.h @@ -0,0 +1,89 @@ +#ifndef MO_REGISTER_FILEREGISTERFWD_INCLUDED +#define MO_REGISTER_FILEREGISTERFWD_INCLUDED + +namespace MOShared +{ + +struct DirectoryEntryFileKey +{ + DirectoryEntryFileKey(std::wstring v) + : value(std::move(v)), hash(getHash(value)) + { + } + + bool operator==(const DirectoryEntryFileKey& o) const + { + return (value == o.value); + } + + static std::size_t getHash(const std::wstring& value) + { + return std::hash()(value); + } + + std::wstring value; + const std::size_t hash; +}; + + +class DirectoryEntry; +class OriginConnection; +class FileRegister; +class FilesOrigin; +class FileEntry; +struct DirectoryStats; + +using FileEntryPtr = boost::shared_ptr; +using FileIndex = unsigned int; + +// a vector of {originId, {archiveName, order}} +// +// if a file is in an archive, archiveName is the name of the bsa and order +// is the order of the associated plugin in the plugins list +// +// is a file is not in an archive, archiveName is empty and order is usually +// -1 +using AlternativesVector = std::vector>>; + +struct DirectoryStats +{ + static constexpr bool EnableInstrumentation = false; + + std::string mod; + + std::chrono::nanoseconds dirTimes; + std::chrono::nanoseconds fileTimes; + std::chrono::nanoseconds sortTimes; + + std::chrono::nanoseconds subdirLookupTimes; + std::chrono::nanoseconds addDirectoryTimes; + + std::chrono::nanoseconds filesLookupTimes; + std::chrono::nanoseconds addFileTimes; + std::chrono::nanoseconds addOriginToFileTimes; + std::chrono::nanoseconds addFileToOriginTimes; + std::chrono::nanoseconds addFileToRegisterTimes; + + int64_t originExists; + int64_t originCreate; + int64_t originsNeededEnabled; + + int64_t subdirExists; + int64_t subdirCreate; + + int64_t fileExists; + int64_t fileCreate; + int64_t filesInsertedInRegister; + int64_t filesAssignedInRegister; + + DirectoryStats(); + + DirectoryStats& operator+=(const DirectoryStats& o); + + static std::string csvHeader(); + std::string toCsv() const; +}; + +} // namespace + +#endif // MO_REGISTER_FILEREGISTERFWD_INCLUDED diff --git a/src/shared/filesorigin.cpp b/src/shared/filesorigin.cpp new file mode 100644 index 00000000..47ae4b94 --- /dev/null +++ b/src/shared/filesorigin.cpp @@ -0,0 +1,137 @@ +#include "filesorigin.h" +#include "originconnection.h" +#include "fileregister.h" +#include "fileentry.h" + +namespace MOShared +{ + +std::wstring tail(const std::wstring &source, const size_t count) +{ + if (count >= source.length()) { + return source; + } + + return source.substr(source.length() - count); +} + + +FilesOrigin::FilesOrigin() + : m_ID(0), m_Disabled(false), m_Name(), m_Path(), m_Priority(0) +{ +} + +/*FilesOrigin::FilesOrigin(const FilesOrigin &reference) + : m_ID(reference.m_ID) + , m_Disabled(reference.m_Disabled) + , m_Name(reference.m_Name) + , m_Path(reference.m_Path) + , m_Priority(reference.m_Priority) + , m_FileRegister(reference.m_FileRegister) + , m_OriginConnection(reference.m_OriginConnection) +{ +}*/ + +FilesOrigin::FilesOrigin( + int ID, const std::wstring &name, const std::wstring &path, int priority, + boost::shared_ptr fileRegister, + boost::shared_ptr originConnection) : + m_ID(ID), m_Disabled(false), m_Name(name), m_Path(path), + m_Priority(priority), m_FileRegister(fileRegister), + m_OriginConnection(originConnection) +{ +} + +void FilesOrigin::setPriority(int priority) +{ + m_OriginConnection.lock()->changePriorityLookup(m_Priority, priority); + + m_Priority = priority; +} + +void FilesOrigin::setName(const std::wstring &name) +{ + m_OriginConnection.lock()->changeNameLookup(m_Name, name); + + // change path too + if (tail(m_Path, m_Name.length()) == m_Name) { + m_Path = m_Path.substr(0, m_Path.length() - m_Name.length()).append(name); + } + + m_Name = name; +} + +std::vector FilesOrigin::getFiles() const +{ + std::vector result; + + { + std::scoped_lock lock(m_Mutex); + + for (FileIndex fileIdx : m_Files) { + if (FileEntryPtr p = m_FileRegister.lock()->getFile(fileIdx)) { + result.push_back(p); + } + } + } + + return result; +} + +FileEntryPtr FilesOrigin::findFile(FileIndex index) const +{ + return m_FileRegister.lock()->getFile(index); +} + +void FilesOrigin::enable(bool enabled) +{ + DirectoryStats dummy; + enable(enabled, dummy); +} + +void FilesOrigin::enable(bool enabled, DirectoryStats& stats) +{ + if (!enabled) { + ++stats.originsNeededEnabled; + + std::set copy; + + { + std::scoped_lock lock(m_Mutex); + copy = m_Files; + m_Files.clear(); + } + + m_FileRegister.lock()->removeOriginMulti(copy, m_ID); + } + + m_Disabled = !enabled; +} + +void FilesOrigin::removeFile(FileIndex index) +{ + std::scoped_lock lock(m_Mutex); + + auto iter = m_Files.find(index); + + if (iter != m_Files.end()) { + m_Files.erase(iter); + } +} + +bool FilesOrigin::containsArchive(std::wstring archiveName) +{ + std::scoped_lock lock(m_Mutex); + + for (FileIndex fileIdx : m_Files) { + if (FileEntryPtr p = m_FileRegister.lock()->getFile(fileIdx)) { + if (p->isFromArchive(archiveName)) { + return true; + } + } + } + + return false; +} + +} // namespace diff --git a/src/shared/filesorigin.h b/src/shared/filesorigin.h new file mode 100644 index 00000000..4ea21f57 --- /dev/null +++ b/src/shared/filesorigin.h @@ -0,0 +1,87 @@ +#ifndef MO_REGISTER_FILESORIGIN_INCLUDED +#define MO_REGISTER_FILESORIGIN_INCLUDED + +#include "fileregisterfwd.h" + +namespace MOShared +{ + +// represents a mod or the data directory, providing files to the tree +class FilesOrigin +{ +public: + FilesOrigin(); +// FilesOrigin(const FilesOrigin &reference); + + FilesOrigin( + int ID, const std::wstring &name, const std::wstring &path, int priority, + boost::shared_ptr fileRegister, + boost::shared_ptr originConnection); + + // noncopyable + FilesOrigin(const FilesOrigin&) = delete; + FilesOrigin& operator=(const FilesOrigin&) = delete; + FilesOrigin(FilesOrigin&&) = default; + FilesOrigin& operator=(FilesOrigin&&) = default; + + // sets priority for this origin, but it will overwrite the existing mapping + // for this priority, the previous origin will no longer be referenced + void setPriority(int priority); + + int getPriority() const + { + return m_Priority; + } + + void setName(const std::wstring &name); + const std::wstring &getName() const + { + return m_Name; + } + + int getID() const + { + return m_ID; + } + + const std::wstring &getPath() const + { + return m_Path; + } + + std::vector getFiles() const; + FileEntryPtr findFile(FileIndex index) const; + + void enable(bool enabled, DirectoryStats& stats); + void enable(bool enabled); + + bool isDisabled() const + { + return m_Disabled; + } + + void addFile(FileIndex index) + { + std::scoped_lock lock(m_Mutex); + m_Files.insert(index); + } + + void removeFile(FileIndex index); + + bool containsArchive(std::wstring archiveName); + +private: + int m_ID; + bool m_Disabled; + std::set m_Files; + std::wstring m_Name; + std::wstring m_Path; + int m_Priority; + boost::weak_ptr m_FileRegister; + boost::weak_ptr m_OriginConnection; + mutable std::mutex m_Mutex; +}; + +} // namespace + +#endif // MO_REGISTER_FILESORIGIN_INCLUDED diff --git a/src/shared/originconnection.cpp b/src/shared/originconnection.cpp new file mode 100644 index 00000000..a56126cc --- /dev/null +++ b/src/shared/originconnection.cpp @@ -0,0 +1,145 @@ +#include "originconnection.h" +#include "filesorigin.h" +#include "util.h" +#include + +namespace MOShared +{ + +using namespace MOBase; + +OriginConnection::OriginConnection() + : m_NextID(0) +{ +} + +std::pair OriginConnection::getOrCreate( + const std::wstring &originName, const std::wstring &directory, int priority, + const boost::shared_ptr& fileRegister, + const boost::shared_ptr& originConnection, + DirectoryStats& stats) +{ + std::unique_lock lock(m_Mutex); + + auto itor = m_OriginsNameMap.find(originName); + + if (itor == m_OriginsNameMap.end()) { + FilesOrigin& origin = createOriginNoLock( + originName, directory, priority, fileRegister, originConnection); + + return {origin, true}; + } else { + FilesOrigin& origin = m_Origins[itor->second]; + lock.unlock(); + + origin.enable(true, stats); + return {origin, false}; + } +} + +FilesOrigin& OriginConnection::createOrigin( + const std::wstring &originName, const std::wstring &directory, int priority, + boost::shared_ptr fileRegister, + boost::shared_ptr originConnection) +{ + std::scoped_lock lock(m_Mutex); + + return createOriginNoLock( + originName, directory, priority, fileRegister, originConnection); +} + +bool OriginConnection::exists(const std::wstring &name) +{ + std::scoped_lock lock(m_Mutex); + return m_OriginsNameMap.find(name) != m_OriginsNameMap.end(); +} + +FilesOrigin& OriginConnection::getByID(Index ID) +{ + std::scoped_lock lock(m_Mutex); + return m_Origins[ID]; +} + +const FilesOrigin* OriginConnection::findByID(Index ID) const +{ + std::scoped_lock lock(m_Mutex); + + auto itor = m_Origins.find(ID); + + if (itor == m_Origins.end()) { + return nullptr; + } else { + return &itor->second; + } +} + +FilesOrigin& OriginConnection::getByName(const std::wstring &name) +{ + std::scoped_lock lock(m_Mutex); + + std::map::iterator iter = m_OriginsNameMap.find(name); + + if (iter != m_OriginsNameMap.end()) { + return m_Origins[iter->second]; + } else { + std::ostringstream stream; + stream << QObject::tr("invalid origin name: ").toStdString() << ToString(name, true); + throw std::runtime_error(stream.str()); + } +} + +void OriginConnection::changePriorityLookup(int oldPriority, int newPriority) +{ + std::scoped_lock lock(m_Mutex); + + auto iter = m_OriginsPriorityMap.find(oldPriority); + + if (iter != m_OriginsPriorityMap.end()) { + Index idx = iter->second; + m_OriginsPriorityMap.erase(iter); + m_OriginsPriorityMap[newPriority] = idx; + } +} + +void OriginConnection::changeNameLookup(const std::wstring &oldName, const std::wstring &newName) +{ + std::scoped_lock lock(m_Mutex); + + auto iter = m_OriginsNameMap.find(oldName); + + if (iter != m_OriginsNameMap.end()) { + Index idx = iter->second; + m_OriginsNameMap.erase(iter); + m_OriginsNameMap[newName] = idx; + } else { + log::error(QObject::tr("failed to change name lookup from {} to {}").toStdString(), oldName, newName); + } +} + +OriginConnection::Index OriginConnection::createID() +{ + return m_NextID++; +} + +FilesOrigin& OriginConnection::createOriginNoLock( + const std::wstring &originName, const std::wstring &directory, int priority, + boost::shared_ptr fileRegister, + boost::shared_ptr originConnection) +{ + int newID = createID(); + + auto itor = m_Origins.emplace( + std::piecewise_construct, + std::forward_as_tuple(newID), + std::forward_as_tuple( + newID, originName, directory, priority, + fileRegister, originConnection)) + .first; + + m_OriginsNameMap.insert({originName, newID}); + m_OriginsPriorityMap.insert({priority, newID}); + + return itor->second; +} + +} // namespace diff --git a/src/shared/originconnection.h b/src/shared/originconnection.h new file mode 100644 index 00000000..1fbb07ac --- /dev/null +++ b/src/shared/originconnection.h @@ -0,0 +1,59 @@ +#ifndef MO_REGISTER_ORIGINCONNECTION_INCLUDED +#define MO_REGISTER_ORIGINCONNECTION_INCLUDED + +#include "fileregisterfwd.h" + +namespace MOShared +{ + +class OriginConnection +{ +public: + typedef int Index; + static const int INVALID_INDEX = INT_MIN; + + OriginConnection(); + + // noncopyable + OriginConnection(const OriginConnection&) = delete; + OriginConnection& operator=(const OriginConnection&) = delete; + + std::pair getOrCreate( + const std::wstring &originName, const std::wstring &directory, int priority, + const boost::shared_ptr& fileRegister, + const boost::shared_ptr& originConnection, + DirectoryStats& stats); + + FilesOrigin& createOrigin( + const std::wstring &originName, const std::wstring &directory, int priority, + boost::shared_ptr fileRegister, + boost::shared_ptr originConnection); + + bool exists(const std::wstring &name); + + FilesOrigin &getByID(Index ID); + const FilesOrigin* findByID(Index ID) const; + FilesOrigin &getByName(const std::wstring &name); + + void changePriorityLookup(int oldPriority, int newPriority); + + void changeNameLookup(const std::wstring &oldName, const std::wstring &newName); + +private: + Index m_NextID; + std::map m_Origins; + std::map m_OriginsNameMap; + std::map m_OriginsPriorityMap; + mutable std::mutex m_Mutex; + + Index createID(); + + FilesOrigin& createOriginNoLock( + const std::wstring &originName, const std::wstring &directory, int priority, + boost::shared_ptr fileRegister, + boost::shared_ptr originConnection); +}; + +} // namespace + +#endif // MO_REGISTER_ORIGINCONNECTION_INCLUDED diff --git a/src/syncoverwritedialog.cpp b/src/syncoverwritedialog.cpp index b1643b2d..0afb1121 100644 --- a/src/syncoverwritedialog.cpp +++ b/src/syncoverwritedialog.cpp @@ -18,8 +18,11 @@ along with Mod Organizer. If not, see . */ #include "syncoverwritedialog.h" - +#include "shared/directoryentry.h" +#include "shared/fileentry.h" +#include "shared/filesorigin.h" #include "ui_syncoverwritedialog.h" + #include #include #include @@ -92,7 +95,7 @@ void SyncOverwriteDialog::readTree(const QString &path, DirectoryEntry *director newItem = nullptr; } } else { - const FileEntry::Ptr entry = directoryStructure->findFile(ToWString(file)); + const FileEntryPtr entry = directoryStructure->findFile(ToWString(file)); QComboBox* combo = new QComboBox(ui->syncTree); combo->addItem(tr(""), -1); if (entry.get() != nullptr) { diff --git a/src/syncoverwritedialog.h b/src/syncoverwritedialog.h index 79b30a8d..cb4c92fb 100644 --- a/src/syncoverwritedialog.h +++ b/src/syncoverwritedialog.h @@ -20,11 +20,9 @@ along with Mod Organizer. If not, see . #ifndef SYNCOVERWRITEDIALOG_H #define SYNCOVERWRITEDIALOG_H - #include "tutorabledialog.h" +#include "shared/fileregisterfwd.h" #include -#include - namespace Ui { class SyncOverwriteDialog; @@ -33,9 +31,12 @@ class SyncOverwriteDialog; class SyncOverwriteDialog : public MOBase::TutorableDialog { Q_OBJECT - + public: - explicit SyncOverwriteDialog(const QString &path, MOShared::DirectoryEntry *directoryStructure, QWidget *parent = 0); + explicit SyncOverwriteDialog( + const QString &path, MOShared::DirectoryEntry *directoryStructure, + QWidget *parent = 0); + ~SyncOverwriteDialog(); void apply(const QString &modDirectory); -- cgit v1.3.1 From 777d822f13fcd15fc5f5ac0bb45589460a1e16f1 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sat, 15 Feb 2020 13:44:34 -0500 Subject: cleanup, typedefs --- src/organizercore.cpp | 20 ++++++++++---------- src/shared/directoryentry.cpp | 23 +++++++++++------------ src/shared/directoryentry.h | 36 +++++++++++++++++------------------- src/shared/fileentry.cpp | 14 +++++++------- src/shared/fileentry.h | 21 +++++++++------------ src/shared/fileregister.cpp | 6 +++--- src/shared/fileregister.h | 4 ++-- src/shared/fileregisterfwd.h | 7 ++++++- src/shared/filesorigin.cpp | 13 +------------ src/shared/filesorigin.h | 10 ++++------ src/shared/originconnection.cpp | 14 +++++++------- src/shared/originconnection.h | 17 +++++++---------- 12 files changed, 84 insertions(+), 101 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/organizercore.cpp b/src/organizercore.cpp index c9c0bee5..2c680847 100644 --- a/src/organizercore.cpp +++ b/src/organizercore.cpp @@ -847,8 +847,8 @@ QStringList OrganizerCore::findFiles( if (!path.isEmpty()) dir = dir->findSubDirectoryRecursive(ToWString(path)); if (dir != nullptr) { - std::vector files = dir->getFiles(); - foreach (FileEntry::Ptr file, files) { + std::vector files = dir->getFiles(); + foreach (FileEntryPtr file, files) { if (filter(ToQString(file->getFullPath()))) { result.append(ToQString(file->getFullPath())); } @@ -860,7 +860,7 @@ QStringList OrganizerCore::findFiles( QStringList OrganizerCore::getFileOrigins(const QString &fileName) const { QStringList result; - const FileEntry::Ptr file = m_DirectoryStructure->searchFile(ToWString(fileName), nullptr); + const FileEntryPtr file = m_DirectoryStructure->searchFile(ToWString(fileName), nullptr); if (file.get() != nullptr) { result.append(ToQString( @@ -883,8 +883,8 @@ QList OrganizerCore::findFileInfos( if (!path.isEmpty()) dir = dir->findSubDirectoryRecursive(ToWString(path)); if (dir != nullptr) { - std::vector files = dir->getFiles(); - foreach (FileEntry::Ptr file, files) { + std::vector files = dir->getFiles(); + foreach (FileEntryPtr file, files) { IOrganizer::FileInfo info; info.filePath = ToQString(file->getFullPath()); bool fromArchive = false; @@ -966,7 +966,7 @@ bool OrganizerCore::previewFileWithAlternatives( - const FileEntry::Ptr file = directoryStructure()->searchFile(ToWString(fileName), nullptr); + const FileEntryPtr file = directoryStructure()->searchFile(ToWString(fileName), nullptr); if (file.get() == nullptr) { reportError(tr("file not found: %1").arg(qUtf8Printable(fileName))); @@ -1180,7 +1180,7 @@ void OrganizerCore::updateModsActiveState(const QList &modIndices, QDir dir(modInfo->absolutePath()); for (const QString &esm : dir.entryList(QStringList() << "*.esm", QDir::Files)) { - const FileEntry::Ptr file = m_DirectoryStructure->findFile(ToWString(esm)); + const FileEntryPtr file = m_DirectoryStructure->findFile(ToWString(esm)); if (file.get() == nullptr) { log::warn("failed to activate {}", esm); continue; @@ -1196,7 +1196,7 @@ void OrganizerCore::updateModsActiveState(const QList &modIndices, for (const QString &esl : dir.entryList(QStringList() << "*.esl", QDir::Files)) { - const FileEntry::Ptr file = m_DirectoryStructure->findFile(ToWString(esl)); + const FileEntryPtr file = m_DirectoryStructure->findFile(ToWString(esl)); if (file.get() == nullptr) { log::warn("failed to activate {}", esl); continue; @@ -1212,7 +1212,7 @@ void OrganizerCore::updateModsActiveState(const QList &modIndices, } QStringList esps = dir.entryList(QStringList() << "*.esp", QDir::Files); for (const QString &esp : esps) { - const FileEntry::Ptr file = m_DirectoryStructure->findFile(ToWString(esp)); + const FileEntryPtr file = m_DirectoryStructure->findFile(ToWString(esp)); if (file.get() == nullptr) { log::warn("failed to activate {}", esp); continue; @@ -1899,7 +1899,7 @@ std::vector OrganizerCore::fileMapping( { std::vector result; - for (FileEntry::Ptr current : directoryEntry->getFiles()) { + for (FileEntryPtr current : directoryEntry->getFiles()) { bool isArchive = false; int origin = current->getOrigin(isArchive); if (isArchive || (origin == 0)) { diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 1036dfe6..14c8323e 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -359,7 +359,7 @@ int DirectoryEntry::anyOrigin() const // we continue looking in subdirectories for (DirectoryEntry *entry : m_SubDirectories) { int res = entry->anyOrigin(); - if (res != -1){ + if (res != InvalidOriginID){ return res; } } @@ -398,7 +398,7 @@ DirectoryEntry *DirectoryEntry::findSubDirectory( DirectoryEntry *DirectoryEntry::findSubDirectoryRecursive(const std::wstring &path) { - return getSubDirectoryRecursive(path, false, -1); + return getSubDirectoryRecursive(path, false, InvalidOriginID); } const FileEntryPtr DirectoryEntry::findFile( @@ -407,9 +407,9 @@ const FileEntryPtr DirectoryEntry::findFile( FilesLookup::const_iterator iter; if (alreadyLowerCase) { - iter = m_FilesLookup.find(FileKey(name)); + iter = m_FilesLookup.find(DirectoryEntryFileKey(name)); } else { - iter = m_FilesLookup.find(FileKey(ToLowerCopy(name))); + iter = m_FilesLookup.find(DirectoryEntryFileKey(ToLowerCopy(name))); } if (iter != m_FilesLookup.end()) { @@ -419,7 +419,7 @@ const FileEntryPtr DirectoryEntry::findFile( } } -const FileEntryPtr DirectoryEntry::findFile(const FileKey& key) const +const FileEntryPtr DirectoryEntry::findFile(const DirectoryEntryFileKey& key) const { auto iter = m_FilesLookup.find(key); @@ -495,7 +495,7 @@ const FileEntryPtr DirectoryEntry::searchFile( return FileEntryPtr(); } -void DirectoryEntry::removeFile(FileEntry::Index index) +void DirectoryEntry::removeFile(FileIndex index) { removeFileFromList(index); } @@ -589,7 +589,7 @@ FilesOrigin &DirectoryEntry::createOrigin( return r.first; } -void DirectoryEntry::removeFiles(const std::set &indices) +void DirectoryEntry::removeFiles(const std::set &indices) { removeFilesFromList(indices); } @@ -601,7 +601,7 @@ FileEntryPtr DirectoryEntry::insert( std::wstring fileNameLower = ToLowerCopy(fileName); FileEntryPtr fe; - FileKey key(std::move(fileNameLower)); + DirectoryEntryFileKey key(std::move(fileNameLower)); { std::unique_lock lock(m_FilesMutex); @@ -931,7 +931,7 @@ void DirectoryEntry::removeDirectoryFromList(SubDirectories::iterator itor) m_SubDirectories.erase(itor); } -void DirectoryEntry::removeFileFromList(FileEntry::Index index) +void DirectoryEntry::removeFileFromList(FileIndex index) { auto removeFrom = [&](auto& list) { auto iter = std::find_if( @@ -961,7 +961,7 @@ void DirectoryEntry::removeFileFromList(FileEntry::Index index) removeFrom(m_Files); } -void DirectoryEntry::removeFilesFromList(const std::set& indices) +void DirectoryEntry::removeFilesFromList(const std::set& indices) { for (auto iter = m_Files.begin(); iter != m_Files.end();) { if (indices.find(iter->second) != indices.end()) { @@ -980,8 +980,7 @@ void DirectoryEntry::removeFilesFromList(const std::set& indic } } -void DirectoryEntry::addFileToList( - std::wstring fileNameLower, FileEntry::Index index) +void DirectoryEntry::addFileToList(std::wstring fileNameLower, FileIndex index) { m_FilesLookup.emplace(fileNameLower, index); m_Files.emplace(std::move(fileNameLower), index); diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index df946946..5ca7389d 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -49,22 +49,20 @@ namespace MOShared class DirectoryEntry { public: - using FileKey = DirectoryEntryFileKey; - DirectoryEntry( - std::wstring name, DirectoryEntry *parent, int originID); + std::wstring name, DirectoryEntry *parent, OriginID originID); DirectoryEntry( - std::wstring name, DirectoryEntry *parent, int originID, + std::wstring name, DirectoryEntry *parent, OriginID originID, boost::shared_ptr fileRegister, boost::shared_ptr originConnection); + ~DirectoryEntry(); + // noncopyable DirectoryEntry(const DirectoryEntry&) = delete; DirectoryEntry& operator=(const DirectoryEntry&) = delete; - ~DirectoryEntry(); - void clear(); bool isPopulated() const @@ -110,7 +108,7 @@ public: const std::wstring &originName, const std::wstring &directory, env::Directory& root, int priority, DirectoryStats& stats); - void propagateOrigin(int origin); + void propagateOrigin(OriginID origin); const std::wstring &getName() const { @@ -123,11 +121,11 @@ public: } bool originExists(const std::wstring &name) const; - FilesOrigin &getOriginByID(int ID) const; + FilesOrigin &getOriginByID(OriginID ID) const; FilesOrigin &getOriginByName(const std::wstring &name) const; - const FilesOrigin* findOriginByID(int ID) const; + const FilesOrigin* findOriginByID(OriginID ID) const; - int anyOrigin() const; + OriginID anyOrigin() const; std::vector getFiles() const; @@ -191,7 +189,7 @@ public: * @return fileentry object for the file or nullptr if no file matches */ const FileEntryPtr findFile(const std::wstring &name, bool alreadyLowerCase=false) const; - const FileEntryPtr findFile(const FileKey& key) const; + const FileEntryPtr findFile(const DirectoryEntryFileKey& key) const; bool hasFile(const std::wstring& name) const; bool containsArchive(std::wstring archiveName); @@ -209,7 +207,7 @@ public: // remove the specified file from the tree. This can be a path leading to a // file in a subdirectory - bool removeFile(const std::wstring &filePath, int *origin = nullptr); + bool removeFile(const std::wstring &filePath, OriginID *origin = nullptr); /** * @brief remove the specified directory @@ -217,9 +215,9 @@ public: */ void removeDir(const std::wstring &path); - bool remove(const std::wstring &fileName, int *origin); + bool remove(const std::wstring &fileName, OriginID *origin); - bool hasContentsFromOrigin(int originID) const; + bool hasContentsFromOrigin(OriginID originID) const; FilesOrigin &createOrigin( const std::wstring &originName, @@ -231,7 +229,7 @@ public: private: using FilesMap = std::map; - using FilesLookup = std::unordered_map; + using FilesLookup = std::unordered_map; using SubDirectories = std::vector; using SubDirectoriesLookup = std::unordered_map; @@ -245,7 +243,7 @@ private: SubDirectoriesLookup m_SubDirectoriesLookup; DirectoryEntry *m_Parent; - std::set m_Origins; + std::set m_Origins; bool m_Populated; bool m_TopLevel; mutable std::mutex m_SubDirMutex; @@ -273,14 +271,14 @@ private: DirectoryEntry* getSubDirectory( std::wstring_view name, bool create, DirectoryStats& stats, - int originID = -1); + OriginID originID = InvalidOriginID); DirectoryEntry* getSubDirectory( env::Directory& dir, bool create, DirectoryStats& stats, - int originID = -1); + OriginID originID = InvalidOriginID); DirectoryEntry* getSubDirectoryRecursive( - const std::wstring &path, bool create, int originID = -1); + const std::wstring &path, bool create, OriginID originID = InvalidOriginID); void removeDirRecursive(); diff --git a/src/shared/fileentry.cpp b/src/shared/fileentry.cpp index d4e1beb4..e57c3cc5 100644 --- a/src/shared/fileentry.cpp +++ b/src/shared/fileentry.cpp @@ -6,19 +6,19 @@ namespace MOShared { FileEntry::FileEntry() : - m_Index(UINT_MAX), m_Name(), m_Origin(-1), m_Parent(nullptr), + m_Index(InvalidFileIndex), m_Name(), m_Origin(-1), m_Parent(nullptr), m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize) { } -FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : +FileEntry::FileEntry(FileIndex index, std::wstring name, DirectoryEntry *parent) : m_Index(index), m_Name(std::move(name)), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize) { } void FileEntry::addOrigin( - int origin, FILETIME fileTime, std::wstring_view archive, int order) + OriginID origin, FILETIME fileTime, std::wstring_view archive, int order) { std::scoped_lock lock(m_OriginsMutex); @@ -83,7 +83,7 @@ void FileEntry::addOrigin( } } -bool FileEntry::removeOrigin(int origin) +bool FileEntry::removeOrigin(OriginID origin) { std::scoped_lock lock(m_OriginsMutex); @@ -116,7 +116,7 @@ bool FileEntry::removeOrigin(int origin) } } - int currentID = currentIter->first; + OriginID currentID = currentIter->first; m_Archive = currentIter->second; m_Alternatives.erase(currentIter); @@ -201,11 +201,11 @@ bool FileEntry::isFromArchive(std::wstring archiveName) const return false; } -std::wstring FileEntry::getFullPath(int originID) const +std::wstring FileEntry::getFullPath(OriginID originID) const { std::scoped_lock lock(m_OriginsMutex); - if (originID == -1) { + if (originID == InvalidOriginID) { bool ignore = false; originID = getOrigin(ignore); } diff --git a/src/shared/fileentry.h b/src/shared/fileentry.h index 52ce89cd..aceeec57 100644 --- a/src/shared/fileentry.h +++ b/src/shared/fileentry.h @@ -12,28 +12,25 @@ public: static constexpr uint64_t NoFileSize = std::numeric_limits::max(); - typedef unsigned int Index; - typedef boost::shared_ptr Ptr; - FileEntry(); - FileEntry(Index index, std::wstring name, DirectoryEntry *parent); + FileEntry(FileIndex index, std::wstring name, DirectoryEntry *parent); // noncopyable FileEntry(const FileEntry&) = delete; FileEntry& operator=(const FileEntry&) = delete; - Index getIndex() const + FileIndex getIndex() const { return m_Index; } void addOrigin( - int origin, FILETIME fileTime, std::wstring_view archive, int order); + OriginID origin, FILETIME fileTime, std::wstring_view archive, int order); // remove the specified origin from the list of origins that contain this // file. if no origin is left, the file is effectively deleted and true is // returned. otherwise, false is returned - bool removeOrigin(int origin); + bool removeOrigin(OriginID origin); void sortOrigins(); @@ -50,12 +47,12 @@ public: return m_Name; } - int getOrigin() const + OriginID getOrigin() const { return m_Origin; } - int getOrigin(bool &archive) const + OriginID getOrigin(bool &archive) const { archive = (m_Archive.first.length() != 0); return m_Origin; @@ -71,7 +68,7 @@ public: // if originID is -1, uses the main origin; if this file doesn't exist in the // given origin, returns an empty string // - std::wstring getFullPath(int originID=-1) const; + std::wstring getFullPath(OriginID originID=InvalidOriginID) const; std::wstring getRelativePath() const; @@ -107,9 +104,9 @@ public: } private: - Index m_Index; + FileIndex m_Index; std::wstring m_Name; - int m_Origin; + OriginID m_Origin; std::pair m_Archive; AlternativesVector m_Alternatives; DirectoryEntry *m_Parent; diff --git a/src/shared/fileregister.cpp b/src/shared/fileregister.cpp index 4add0455..b56599d1 100644 --- a/src/shared/fileregister.cpp +++ b/src/shared/fileregister.cpp @@ -79,7 +79,7 @@ bool FileRegister::removeFile(FileIndex index) return false; } -void FileRegister::removeOrigin(FileIndex index, int originID) +void FileRegister::removeOrigin(FileIndex index, OriginID originID) { std::unique_lock lock(m_Mutex); @@ -100,7 +100,7 @@ void FileRegister::removeOrigin(FileIndex index, int originID) } void FileRegister::removeOriginMulti( - std::set indices, int originID) + std::set indices, OriginID originID) { std::vector removedFiles; @@ -167,7 +167,7 @@ void FileRegister::unregisterFile(FileEntryPtr file) bool ignore; // unregister from origin - int originID = file->getOrigin(ignore); + OriginID originID = file->getOrigin(ignore); m_OriginConnection->getByID(originID).removeFile(file->getIndex()); const auto& alternatives = file->getAlternatives(); diff --git a/src/shared/fileregister.h b/src/shared/fileregister.h index d5e360c1..9294d20e 100644 --- a/src/shared/fileregister.h +++ b/src/shared/fileregister.h @@ -36,8 +36,8 @@ public: } bool removeFile(FileIndex index); - void removeOrigin(FileIndex index, int originID); - void removeOriginMulti(std::set indices, int originID); + void removeOrigin(FileIndex index, OriginID originID); + void removeOriginMulti(std::set indices, OriginID originID); void sortOrigins(); diff --git a/src/shared/fileregisterfwd.h b/src/shared/fileregisterfwd.h index b8d33378..6348fee1 100644 --- a/src/shared/fileregisterfwd.h +++ b/src/shared/fileregisterfwd.h @@ -35,6 +35,11 @@ struct DirectoryStats; using FileEntryPtr = boost::shared_ptr; using FileIndex = unsigned int; +using OriginID = int; + +constexpr FileIndex InvalidFileIndex = UINT_MAX; +constexpr OriginID InvalidOriginID = -1; + // a vector of {originId, {archiveName, order}} // @@ -43,7 +48,7 @@ using FileIndex = unsigned int; // // is a file is not in an archive, archiveName is empty and order is usually // -1 -using AlternativesVector = std::vector>>; +using AlternativesVector = std::vector>>; struct DirectoryStats { diff --git a/src/shared/filesorigin.cpp b/src/shared/filesorigin.cpp index 47ae4b94..7476b6c5 100644 --- a/src/shared/filesorigin.cpp +++ b/src/shared/filesorigin.cpp @@ -21,19 +21,8 @@ FilesOrigin::FilesOrigin() { } -/*FilesOrigin::FilesOrigin(const FilesOrigin &reference) - : m_ID(reference.m_ID) - , m_Disabled(reference.m_Disabled) - , m_Name(reference.m_Name) - , m_Path(reference.m_Path) - , m_Priority(reference.m_Priority) - , m_FileRegister(reference.m_FileRegister) - , m_OriginConnection(reference.m_OriginConnection) -{ -}*/ - FilesOrigin::FilesOrigin( - int ID, const std::wstring &name, const std::wstring &path, int priority, + OriginID ID, const std::wstring &name, const std::wstring &path, int priority, boost::shared_ptr fileRegister, boost::shared_ptr originConnection) : m_ID(ID), m_Disabled(false), m_Name(name), m_Path(path), diff --git a/src/shared/filesorigin.h b/src/shared/filesorigin.h index 4ea21f57..222a834f 100644 --- a/src/shared/filesorigin.h +++ b/src/shared/filesorigin.h @@ -11,18 +11,16 @@ class FilesOrigin { public: FilesOrigin(); -// FilesOrigin(const FilesOrigin &reference); FilesOrigin( - int ID, const std::wstring &name, const std::wstring &path, int priority, + OriginID ID, const std::wstring &name, const std::wstring &path, + int priority, boost::shared_ptr fileRegister, boost::shared_ptr originConnection); // noncopyable FilesOrigin(const FilesOrigin&) = delete; FilesOrigin& operator=(const FilesOrigin&) = delete; - FilesOrigin(FilesOrigin&&) = default; - FilesOrigin& operator=(FilesOrigin&&) = default; // sets priority for this origin, but it will overwrite the existing mapping // for this priority, the previous origin will no longer be referenced @@ -39,7 +37,7 @@ public: return m_Name; } - int getID() const + OriginID getID() const { return m_ID; } @@ -71,7 +69,7 @@ public: bool containsArchive(std::wstring archiveName); private: - int m_ID; + OriginID m_ID; bool m_Disabled; std::set m_Files; std::wstring m_Name; diff --git a/src/shared/originconnection.cpp b/src/shared/originconnection.cpp index a56126cc..c1c096f0 100644 --- a/src/shared/originconnection.cpp +++ b/src/shared/originconnection.cpp @@ -54,13 +54,13 @@ bool OriginConnection::exists(const std::wstring &name) return m_OriginsNameMap.find(name) != m_OriginsNameMap.end(); } -FilesOrigin& OriginConnection::getByID(Index ID) +FilesOrigin& OriginConnection::getByID(OriginID ID) { std::scoped_lock lock(m_Mutex); return m_Origins[ID]; } -const FilesOrigin* OriginConnection::findByID(Index ID) const +const FilesOrigin* OriginConnection::findByID(OriginID ID) const { std::scoped_lock lock(m_Mutex); @@ -77,7 +77,7 @@ FilesOrigin& OriginConnection::getByName(const std::wstring &name) { std::scoped_lock lock(m_Mutex); - std::map::iterator iter = m_OriginsNameMap.find(name); + auto iter = m_OriginsNameMap.find(name); if (iter != m_OriginsNameMap.end()) { return m_Origins[iter->second]; @@ -95,7 +95,7 @@ void OriginConnection::changePriorityLookup(int oldPriority, int newPriority) auto iter = m_OriginsPriorityMap.find(oldPriority); if (iter != m_OriginsPriorityMap.end()) { - Index idx = iter->second; + OriginID idx = iter->second; m_OriginsPriorityMap.erase(iter); m_OriginsPriorityMap[newPriority] = idx; } @@ -108,7 +108,7 @@ void OriginConnection::changeNameLookup(const std::wstring &oldName, const std:: auto iter = m_OriginsNameMap.find(oldName); if (iter != m_OriginsNameMap.end()) { - Index idx = iter->second; + OriginID idx = iter->second; m_OriginsNameMap.erase(iter); m_OriginsNameMap[newName] = idx; } else { @@ -116,7 +116,7 @@ void OriginConnection::changeNameLookup(const std::wstring &oldName, const std:: } } -OriginConnection::Index OriginConnection::createID() +OriginID OriginConnection::createID() { return m_NextID++; } @@ -126,7 +126,7 @@ FilesOrigin& OriginConnection::createOriginNoLock( boost::shared_ptr fileRegister, boost::shared_ptr originConnection) { - int newID = createID(); + OriginID newID = createID(); auto itor = m_Origins.emplace( std::piecewise_construct, diff --git a/src/shared/originconnection.h b/src/shared/originconnection.h index 1fbb07ac..5fecd63a 100644 --- a/src/shared/originconnection.h +++ b/src/shared/originconnection.h @@ -9,9 +9,6 @@ namespace MOShared class OriginConnection { public: - typedef int Index; - static const int INVALID_INDEX = INT_MIN; - OriginConnection(); // noncopyable @@ -31,8 +28,8 @@ public: bool exists(const std::wstring &name); - FilesOrigin &getByID(Index ID); - const FilesOrigin* findByID(Index ID) const; + FilesOrigin &getByID(OriginID ID); + const FilesOrigin* findByID(OriginID ID) const; FilesOrigin &getByName(const std::wstring &name); void changePriorityLookup(int oldPriority, int newPriority); @@ -40,13 +37,13 @@ public: void changeNameLookup(const std::wstring &oldName, const std::wstring &newName); private: - Index m_NextID; - std::map m_Origins; - std::map m_OriginsNameMap; - std::map m_OriginsPriorityMap; + std::atomic m_NextID; + std::map m_Origins; + std::map m_OriginsNameMap; + std::map m_OriginsPriorityMap; mutable std::mutex m_Mutex; - Index createID(); + OriginID createID(); FilesOrigin& createOriginNoLock( const std::wstring &originName, const std::wstring &directory, int priority, -- cgit v1.3.1 From dbb3f5117a3f977bbef34ebdab217f379429d1f2 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sat, 15 Feb 2020 14:22:42 -0500 Subject: removed directoryrefresher.h from organizercore.h --- src/directoryrefresher.cpp | 174 ++++++++++++++++++++++++++++++++---------- src/directoryrefresher.h | 8 +- src/organizercore.cpp | 19 ++--- src/organizercore.h | 41 ++++------ src/shared/directoryentry.cpp | 99 ------------------------ 5 files changed, 160 insertions(+), 181 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index 3b92389a..8a64d4a9 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -45,22 +45,141 @@ using namespace MOBase; using namespace MOShared; -DirectoryRefresher::DirectoryRefresher(std::size_t threadCount) - : m_DirectoryStructure(nullptr), m_threadCount(threadCount), m_lastFileCount(0) +DirectoryStats::DirectoryStats() +{ + std::memset(this, 0, sizeof(DirectoryStats)); +} + +DirectoryStats& DirectoryStats::operator+=(const DirectoryStats& o) +{ + dirTimes += o.dirTimes; + fileTimes += o.fileTimes; + sortTimes += o.sortTimes; + + subdirLookupTimes += o.subdirLookupTimes; + addDirectoryTimes += o.addDirectoryTimes; + + filesLookupTimes += o.filesLookupTimes; + addFileTimes += o.addFileTimes; + addOriginToFileTimes += o.addOriginToFileTimes; + addFileToOriginTimes += o.addFileToOriginTimes; + addFileToRegisterTimes += o.addFileToRegisterTimes; + + originExists += o.originExists; + originCreate += o.originCreate; + originsNeededEnabled += o.originsNeededEnabled; + + subdirExists += o.subdirExists; + subdirCreate += o.subdirCreate; + + fileExists += o.fileExists; + fileCreate += o.fileCreate; + filesInsertedInRegister += o.filesInsertedInRegister; + filesAssignedInRegister += o.filesAssignedInRegister; + + return *this; +} + +std::string DirectoryStats::csvHeader() +{ + QStringList sl = { + "dirTimes", + "fileTimes", + "sortTimes", + "subdirLookupTimes", + "addDirectoryTimes", + "filesLookupTimes", + "addFileTimes", + "addOriginToFileTimes", + "addFileToOriginTimes", + "addFileToRegisterTimes", + "originExists", + "originCreate", + "originsNeededEnabled", + "subdirExists", + "subdirCreate", + "fileExists", + "fileCreate", + "filesInsertedInRegister", + "filesAssignedInRegister"}; + + return sl.join(",").toStdString(); +} + +std::string DirectoryStats::toCsv() const { + QStringList oss; + + auto s = [](auto ns) { + return ns.count() / 1000.0 / 1000.0 / 1000.0; + }; + + oss + << QString::number(s(dirTimes)) + << QString::number(s(fileTimes)) + << QString::number(s(sortTimes)) + + << QString::number(s(subdirLookupTimes)) + << QString::number(s(addDirectoryTimes)) + + << QString::number(s(filesLookupTimes)) + << QString::number(s(addFileTimes)) + << QString::number(s(addOriginToFileTimes)) + << QString::number(s(addFileToOriginTimes)) + << QString::number(s(addFileToRegisterTimes)) + + << QString::number(originExists) + << QString::number(originCreate) + << QString::number(originsNeededEnabled) + + << QString::number(subdirExists) + << QString::number(subdirCreate) + + << QString::number(fileExists) + << QString::number(fileCreate) + << QString::number(filesInsertedInRegister) + << QString::number(filesAssignedInRegister); + + return oss.join(",").toStdString(); +} + +void dumpStats(std::vector& stats) +{ + static int run = 0; + static const std::string file("c:\\tmp\\data.csv"); + + if (run == 0) { + std::ofstream out(file, std::ios::out|std::ios::trunc); + out << fmt::format("what,run,{}", DirectoryStats::csvHeader()) << "\n"; + } + + std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){ + return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0); + }); + + std::ofstream out(file, std::ios::app); + + DirectoryStats total; + for (const auto& s : stats) { + out << fmt::format("{},{},{}", s.mod, run, s.toCsv()) << "\n"; + total += s; + } + + out << fmt::format("total,{},{}", run, total.toCsv()) << "\n"; + + ++run; } -DirectoryRefresher::~DirectoryRefresher() + +DirectoryRefresher::DirectoryRefresher(std::size_t threadCount) + : m_threadCount(threadCount), m_lastFileCount(0) { - delete m_DirectoryStructure; } DirectoryEntry *DirectoryRefresher::stealDirectoryStructure() { QMutexLocker locker(&m_RefreshLock); - DirectoryEntry *result = m_DirectoryStructure; - m_DirectoryStructure = nullptr; - return result; + return m_Root.release(); } void DirectoryRefresher::setMods(const std::vector > &mods @@ -251,32 +370,6 @@ struct ModThread env::ThreadPool g_threads; -void dumpStats(std::vector& stats) -{ - static int run = 0; - static const std::string file("c:\\tmp\\data.csv"); - - if (run == 0) { - std::ofstream out(file, std::ios::out|std::ios::trunc); - out << fmt::format("what,run,{}", DirectoryStats::csvHeader()) << "\n"; - } - - std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){ - return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0); - }); - - std::ofstream out(file, std::ios::app); - - DirectoryStats total; - for (const auto& s : stats) { - out << fmt::format("{},{},{}", s.mod, run, s.toCsv()) << "\n"; - total += s; - } - - out << fmt::format("total,{},{}", run, total.toCsv()) << "\n"; - - ++run; -} void DirectoryRefresher::addMultipleModsFilesToStructure( MOShared::DirectoryEntry *directoryStructure, @@ -332,10 +425,9 @@ void DirectoryRefresher::refresh() TimeThis tt("refresh"); QMutexLocker locker(&m_RefreshLock); - delete m_DirectoryStructure; - m_DirectoryStructure = new DirectoryEntry(L"data", nullptr, 0); - m_DirectoryStructure->getFileRegister()->reserve(m_lastFileCount); + m_Root.reset(new DirectoryEntry(L"data", nullptr, 0)); + m_Root->getFileRegister()->reserve(m_lastFileCount); IPluginGame *game = qApp->property("managed_game").value(); @@ -344,20 +436,20 @@ void DirectoryRefresher::refresh() { DirectoryStats dummy; - m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0, dummy); + m_Root->addFromOrigin(L"data", dataDirectory, 0, dummy); } std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { return lhs.priority < rhs.priority; }); - addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); + addMultipleModsFilesToStructure(m_Root.get(), m_Mods, true); - m_DirectoryStructure->getFileRegister()->sortOrigins(); + m_Root->getFileRegister()->sortOrigins(); - cleanStructure(m_DirectoryStructure); + cleanStructure(m_Root.get()); - m_lastFileCount = m_DirectoryStructure->getFileRegister()->highestCount(); + m_lastFileCount = m_Root->getFileRegister()->highestCount(); log::debug("refresher saw {} files", m_lastFileCount); emit progress(100); diff --git a/src/directoryrefresher.h b/src/directoryrefresher.h index 2102140c..e81b55dd 100644 --- a/src/directoryrefresher.h +++ b/src/directoryrefresher.h @@ -50,14 +50,8 @@ public: int priority; }; - /** - * @brief constructor - * - **/ DirectoryRefresher(std::size_t threadCount); - ~DirectoryRefresher(); - // noncopyable DirectoryRefresher(const DirectoryRefresher&) = delete; DirectoryRefresher& operator=(const DirectoryRefresher&) = delete; @@ -145,7 +139,7 @@ signals: private: std::vector m_Mods; std::set m_EnabledArchives; - MOShared::DirectoryEntry *m_DirectoryStructure; + std::unique_ptr m_Root; QMutex m_RefreshLock; std::size_t m_threadCount; std::size_t m_lastFileCount; diff --git a/src/organizercore.cpp b/src/organizercore.cpp index 2c680847..80df044a 100644 --- a/src/organizercore.cpp +++ b/src/organizercore.cpp @@ -34,6 +34,7 @@ #include "env.h" #include "envmodule.h" #include "envfs.h" +#include "directoryrefresher.h" #include "shared/directoryentry.h" #include "shared/filesorigin.h" #include "shared/fileentry.h" @@ -96,7 +97,7 @@ OrganizerCore::OrganizerCore(Settings &settings) , m_Updater(NexusInterface::instance(m_PluginContainer)) , m_ModList(m_PluginContainer, this) , m_PluginList(this) - , m_DirectoryRefresher(settings.refreshThreadCount()) + , m_DirectoryRefresher(new DirectoryRefresher(settings.refreshThreadCount())) , m_DirectoryStructure(new DirectoryEntry(L"data", nullptr, 0)) , m_DownloadManager(NexusInterface::instance(m_PluginContainer), this) , m_DirectoryUpdate(false) @@ -114,7 +115,7 @@ OrganizerCore::OrganizerCore(Settings &settings) connect(&m_DownloadManager, SIGNAL(downloadSpeed(QString, int)), this, SLOT(downloadSpeed(QString, int))); - connect(&m_DirectoryRefresher, SIGNAL(refreshed()), this, + connect(m_DirectoryRefresher.get(), SIGNAL(refreshed()), this, SLOT(directory_refreshed())); connect(&m_ModList, SIGNAL(removeOrigin(QString)), this, @@ -139,7 +140,7 @@ OrganizerCore::OrganizerCore(Settings &settings) // make directory refresher run in a separate thread m_RefresherThread.start(); - m_DirectoryRefresher.moveToThread(&m_RefresherThread); + m_DirectoryRefresher->moveToThread(&m_RefresherThread); } OrganizerCore::~OrganizerCore() @@ -1255,7 +1256,7 @@ void OrganizerCore::updateModsInDirectoryStructure(QMapstealFiles(), {}, m_CurrentProfile->getModPriority(idx)}); } - m_DirectoryRefresher.addMultipleModsFilesToStructure( + m_DirectoryRefresher->addMultipleModsFilesToStructure( m_DirectoryStructure, entries); DirectoryRefresher::cleanStructure(m_DirectoryStructure); @@ -1274,13 +1275,13 @@ void OrganizerCore::updateModsInDirectoryStructure(QMap archives = enabledArchives(); - m_DirectoryRefresher.setMods( + m_DirectoryRefresher->setMods( m_CurrentProfile->getActiveMods(), std::set(archives.begin(), archives.end())); // finally also add files from bsas to the directory structure for (auto idx : modInfo.keys()) { - m_DirectoryRefresher.addModBSAToStructure( + m_DirectoryRefresher->addModBSAToStructure( m_DirectoryStructure, modInfo[idx]->name(), m_CurrentProfile->getModPriority(idx), modInfo[idx]->absolutePath(), modInfo[idx]->archives()); @@ -1398,18 +1399,18 @@ void OrganizerCore::refreshDirectoryStructure() const auto activeModList = m_CurrentProfile->getActiveMods(); const auto archives = enabledArchives(); - m_DirectoryRefresher.setMods( + m_DirectoryRefresher->setMods( activeModList, std::set(archives.begin(), archives.end())); // runs refresh() in a thread - QTimer::singleShot(0, &m_DirectoryRefresher, SLOT(refresh())); + QTimer::singleShot(0, m_DirectoryRefresher.get(), SLOT(refresh())); } void OrganizerCore::directory_refreshed() { log::debug("structure refreshed"); - DirectoryEntry *newStructure = m_DirectoryRefresher.stealDirectoryStructure(); + DirectoryEntry *newStructure = m_DirectoryRefresher->stealDirectoryStructure(); Q_ASSERT(newStructure != m_DirectoryStructure); if (newStructure == nullptr) { diff --git a/src/organizercore.h b/src/organizercore.h index 980156d3..a4d1a799 100644 --- a/src/organizercore.h +++ b/src/organizercore.h @@ -1,13 +1,11 @@ #ifndef ORGANIZERCORE_H #define ORGANIZERCORE_H - #include "selfupdater.h" #include "settings.h" #include "modlist.h" #include "modinfo.h" #include "pluginlist.h" -#include "directoryrefresher.h" #include "installationmanager.h" #include "downloadmanager.h" #include "executableslist.h" @@ -23,17 +21,6 @@ #include "executableinfo.h" #include -class ModListSortProxy; -class PluginListSortProxy; -class Profile; -class IUserInterface; - -namespace MOBase { - template class GuessedValue; - class IModInterface; -} -namespace MOShared { class DirectoryEntry; } - #include #include #include @@ -44,21 +31,25 @@ namespace MOShared { class DirectoryEntry; } #include #include -class QNetworkReply; -class QUrl; -class QWidget; - -#include //for HANDLE, LPDWORD - -#include -#include - +class ModListSortProxy; +class PluginListSortProxy; +class Profile; +class IUserInterface; class PluginContainer; +class DirectoryRefresher; -namespace MOBase { +namespace MOBase +{ + template class GuessedValue; + class IModInterface; class IPluginGame; } +namespace MOShared +{ + class DirectoryEntry; +} + class OrganizerCore : public QObject, public MOBase::IPluginDiagnose { @@ -113,7 +104,7 @@ public: SelfUpdater *updater() { return &m_Updater; } InstallationManager *installationManager(); MOShared::DirectoryEntry *directoryStructure() { return m_DirectoryStructure; } - DirectoryRefresher *directoryRefresher() { return &m_DirectoryRefresher; } + DirectoryRefresher *directoryRefresher() { return m_DirectoryRefresher.get(); } ExecutablesList *executablesList() { return &m_ExecutablesList; } void setExecutablesList(const ExecutablesList &executablesList) { m_ExecutablesList = executablesList; @@ -337,7 +328,7 @@ private: QStringList m_DefaultArchives; QStringList m_ActiveArchives; - DirectoryRefresher m_DirectoryRefresher; + std::unique_ptr m_DirectoryRefresher; MOShared::DirectoryEntry *m_DirectoryStructure; DownloadManager m_DownloadManager; diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 14c8323e..7f140750 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -73,105 +73,6 @@ static bool DirCompareByName(const DirectoryEntry *lhs, const DirectoryEntry *rh } -DirectoryStats::DirectoryStats() -{ - std::memset(this, 0, sizeof(DirectoryStats)); -} - -DirectoryStats& DirectoryStats::operator+=(const DirectoryStats& o) -{ - dirTimes += o.dirTimes; - fileTimes += o.fileTimes; - sortTimes += o.sortTimes; - - subdirLookupTimes += o.subdirLookupTimes; - addDirectoryTimes += o.addDirectoryTimes; - - filesLookupTimes += o.filesLookupTimes; - addFileTimes += o.addFileTimes; - addOriginToFileTimes += o.addOriginToFileTimes; - addFileToOriginTimes += o.addFileToOriginTimes; - addFileToRegisterTimes += o.addFileToRegisterTimes; - - originExists += o.originExists; - originCreate += o.originCreate; - originsNeededEnabled += o.originsNeededEnabled; - - subdirExists += o.subdirExists; - subdirCreate += o.subdirCreate; - - fileExists += o.fileExists; - fileCreate += o.fileCreate; - filesInsertedInRegister += o.filesInsertedInRegister; - filesAssignedInRegister += o.filesAssignedInRegister; - - return *this; -} - -std::string DirectoryStats::csvHeader() -{ - QStringList sl = { - "dirTimes", - "fileTimes", - "sortTimes", - "subdirLookupTimes", - "addDirectoryTimes", - "filesLookupTimes", - "addFileTimes", - "addOriginToFileTimes", - "addFileToOriginTimes", - "addFileToRegisterTimes", - "originExists", - "originCreate", - "originsNeededEnabled", - "subdirExists", - "subdirCreate", - "fileExists", - "fileCreate", - "filesInsertedInRegister", - "filesAssignedInRegister"}; - - return sl.join(",").toStdString(); -} - -std::string DirectoryStats::toCsv() const -{ - QStringList oss; - - auto s = [](auto ns) { - return ns.count() / 1000.0 / 1000.0 / 1000.0; - }; - - oss - << QString::number(s(dirTimes)) - << QString::number(s(fileTimes)) - << QString::number(s(sortTimes)) - - << QString::number(s(subdirLookupTimes)) - << QString::number(s(addDirectoryTimes)) - - << QString::number(s(filesLookupTimes)) - << QString::number(s(addFileTimes)) - << QString::number(s(addOriginToFileTimes)) - << QString::number(s(addFileToOriginTimes)) - << QString::number(s(addFileToRegisterTimes)) - - << QString::number(originExists) - << QString::number(originCreate) - << QString::number(originsNeededEnabled) - - << QString::number(subdirExists) - << QString::number(subdirCreate) - - << QString::number(fileExists) - << QString::number(fileCreate) - << QString::number(filesInsertedInRegister) - << QString::number(filesAssignedInRegister); - - return oss.join(",").toStdString(); -} - - DirectoryEntry::DirectoryEntry( std::wstring name, DirectoryEntry *parent, int originID) : m_OriginConnection(new OriginConnection), -- cgit v1.3.1 From 5e569d0107aefe292b76ac0458f9420077f350a0 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sat, 15 Feb 2020 17:22:12 -0500 Subject: re-enabled archive parsing --- src/directoryrefresher.cpp | 126 ++++++++++++++++++++------------- src/mainwindow.cpp | 1 + src/shared/directoryentry.cpp | 161 ++++++++++++++++++++++++------------------ src/shared/directoryentry.h | 79 ++++++++++++--------- src/shared/util.h | 10 +++ 5 files changed, 223 insertions(+), 154 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index 8a64d4a9..5e1c05a8 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -210,38 +210,41 @@ void DirectoryRefresher::cleanStructure(DirectoryEntry *structure) } } -void DirectoryRefresher::addModBSAToStructure(DirectoryEntry *directoryStructure, const QString &modName, - int priority, const QString &directory, const QStringList &archives) +void DirectoryRefresher::addModBSAToStructure( + DirectoryEntry* root, const QString& modName, + int priority, const QString& directory, const QStringList& archives) { - std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); - IPluginGame *game = qApp->property("managed_game").value(); + const IPluginGame *game = qApp->property("managed_game").value(); GamePlugins *gamePlugins = game->feature(); QStringList loadOrder = QStringList(); gamePlugins->getLoadOrder(loadOrder); - for (const QString &archive : archives) { - QFileInfo fileInfo(archive); - if (m_EnabledArchives.find(fileInfo.fileName()) != m_EnabledArchives.end()) { - - int order = -1; + std::vector lo; + for (auto&& s : loadOrder) { + lo.push_back(s.toStdWString()); + } - for (auto plugin : loadOrder) - { - QString name = plugin.left(plugin.size() - 4); - if (fileInfo.fileName().startsWith(name + " - ", Qt::CaseInsensitive) || fileInfo.fileName().startsWith(name + ".", Qt::CaseInsensitive)) { - order = loadOrder.indexOf(plugin); - } - } + std::vector archivesW; + for (auto&& a : archives) { + archivesW.push_back(a.toStdWString()); + } - try { - IPluginGame *game = qApp->property("managed_game").value(); - directoryStructure->addFromBSA(ToWString(modName), directoryW, ToWString(QDir::toNativeSeparators(fileInfo.absoluteFilePath())), priority, order); - } catch (const std::exception &e) { - throw MyException(tr("failed to parse bsa %1: %2").arg(archive, e.what())); - } - } + std::set enabledArchives; + for (auto&& a : m_EnabledArchives) { + enabledArchives.insert(a.toStdWString()); } + + DirectoryStats dummy; + + root->addFromAllBSAs( + modName.toStdWString(), + QDir::toNativeSeparators(directory).toStdWString(), + priority, + archivesW, + enabledArchives, + lo, + dummy); } void DirectoryRefresher::stealModFilesIntoStructure( @@ -329,6 +332,8 @@ struct ModThread std::wstring modName; std::wstring path; int prio = -1; + std::vector archives; + std::set enabledArchives; DirectoryStats* stats = nullptr; env::DirectoryWalker walker; @@ -354,14 +359,21 @@ struct ModThread SetThisThreadName(QString::fromStdWString(modName + L" refresher")); ds->addFromOrigin(walker, modName, path, prio, *stats); - /*if (Settings::instance().archiveParsing()) { - addModBSAToStructure( - directoryStructure, - entries[i].modName, - prio, - entries[i].absolutePath, - entries[i].archives); - }*/ + if (Settings::instance().archiveParsing()) { + const IPluginGame *game = qApp->property("managed_game").value(); + + GamePlugins *gamePlugins = game->feature(); + QStringList loadOrder = QStringList(); + gamePlugins->getLoadOrder(loadOrder); + + std::vector lo; + for (auto&& s : loadOrder) { + lo.push_back(s.toStdWString()); + } + + ds->addFromAllBSAs( + modName, path, prio, archives, enabledArchives, lo, *stats); + } ready = false; } @@ -377,6 +389,7 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( { std::vector stats(entries.size()); + log::debug("refresher: using {} threads", m_threadCount); g_threads.setMax(m_threadCount); for (std::size_t i=0; igetFileRegister()->reserve(m_lastFileCount); + m_Root.reset(new DirectoryEntry(L"data", nullptr, 0)); + m_Root->getFileRegister()->reserve(m_lastFileCount); - IPluginGame *game = qApp->property("managed_game").value(); + IPluginGame *game = qApp->property("managed_game").value(); - std::wstring dataDirectory = - QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + std::wstring dataDirectory = + QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); - { - DirectoryStats dummy; - m_Root->addFromOrigin(L"data", dataDirectory, 0, dummy); - } + { + DirectoryStats dummy; + m_Root->addFromOrigin(L"data", dataDirectory, 0, dummy); + } - std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { - return lhs.priority < rhs.priority; - }); + std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { + return lhs.priority < rhs.priority; + }); - addMultipleModsFilesToStructure(m_Root.get(), m_Mods, true); + addMultipleModsFilesToStructure(m_Root.get(), m_Mods, true); - m_Root->getFileRegister()->sortOrigins(); + m_Root->getFileRegister()->sortOrigins(); - cleanStructure(m_Root.get()); + cleanStructure(m_Root.get()); - m_lastFileCount = m_Root->getFileRegister()->highestCount(); - log::debug("refresher saw {} files", m_lastFileCount); + m_lastFileCount = m_Root->getFileRegister()->highestCount(); + log::debug("refresher saw {} files", m_lastFileCount); + } emit progress(100); emit refreshed(); diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index adbac94e..f5749392 100644 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -87,6 +87,7 @@ along with Mod Organizer. If not, see . #include "envshortcut.h" #include "browserdialog.h" +#include "directoryrefresher.h" #include "shared/directoryentry.h" #include "shared/fileentry.h" #include "shared/filesorigin.h" diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 7f140750..84d68675 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -67,14 +67,14 @@ static bool SupportOptimizedFind() return (::VerifyVersionInfo(&versionInfo, VER_MAJORVERSION | VER_MINORVERSION, mask) == TRUE); } -static bool DirCompareByName(const DirectoryEntry *lhs, const DirectoryEntry *rhs) +static bool DirCompareByName(const DirectoryEntry* lhs, const DirectoryEntry* rhs) { return _wcsicmp(lhs->getName().c_str(), rhs->getName().c_str()) < 0; } DirectoryEntry::DirectoryEntry( - std::wstring name, DirectoryEntry *parent, int originID) : + std::wstring name, DirectoryEntry* parent, int originID) : m_OriginConnection(new OriginConnection), m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(true) { @@ -83,7 +83,7 @@ DirectoryEntry::DirectoryEntry( } DirectoryEntry::DirectoryEntry( - std::wstring name, DirectoryEntry *parent, int originID, + std::wstring name, DirectoryEntry* parent, int originID, boost::shared_ptr fileRegister, boost::shared_ptr originConnection) : m_FileRegister(fileRegister), m_OriginConnection(originConnection), @@ -166,51 +166,69 @@ void DirectoryEntry::addDir( m_Populated = true; } -void DirectoryEntry::addFromBSA( - const std::wstring &originName, std::wstring &directory, - const std::wstring &fileName, int priority, int order) +void DirectoryEntry::addFromAllBSAs( + const std::wstring& originName, const std::wstring& directory, + int priority, const std::vector& archives, + const std::set& enabledArchives, + const std::vector& loadOrder, + DirectoryStats& stats) { - DirectoryStats dummy; - FilesOrigin &origin = createOrigin(originName, directory, priority, dummy); + for (const auto& archive : archives) { + const std::filesystem::path archivePath(archive); + const auto filename = archivePath.filename().native(); - WIN32_FILE_ATTRIBUTE_DATA fileData; - if (::GetFileAttributesExW(fileName.c_str(), GetFileExInfoStandard, &fileData) == 0) { - throw windows_error(QObject::tr("failed to determine file time").toStdString()); - } + if (!enabledArchives.contains(filename)) { + continue; + } - FILETIME now; - ::GetSystemTimeAsFileTime(&now); + const auto filenameLc = ToLowerCopy(filename); - const double clfSecondsPer100ns = 100. * 1.E-9; + int order = -1; - ((ULARGE_INTEGER *)&now)->QuadPart -= ((double)5) / clfSecondsPer100ns; + for (auto plugin : loadOrder) + { + const auto pluginNameLc = + ToLowerCopy(std::filesystem::path(plugin).stem().native()); + + if (filenameLc.starts_with(pluginNameLc + L" - ") || + filenameLc.starts_with(pluginNameLc + L".")) { + auto itor = std::find(loadOrder.begin(), loadOrder.end(), plugin); + if (itor != loadOrder.end()) { + order = std::distance(loadOrder.begin(), itor); + } + } + } - size_t namePos = fileName.find_last_of(L"\\/"); - if (namePos == std::wstring::npos) { - namePos = 0; + addFromBSA( + originName, directory, archivePath.native(), + priority, order, stats); } - else { - ++namePos; +} + +void DirectoryEntry::addFromBSA( + const std::wstring& originName, const std::wstring& directory, + const std::wstring& archivePath, int priority, int order, DirectoryStats& stats) +{ + FilesOrigin& origin = createOrigin(originName, directory, priority, stats); + const auto archiveName = std::filesystem::path(archivePath).filename().native(); + + if (containsArchive(archiveName)) { + return; } - if (!containsArchive(fileName.substr(namePos)) || ::CompareFileTime(&fileData.ftLastWriteTime, &now) > 0) { - BSA::Archive archive; - BSA::EErrorCode res = archive.read(ToString(fileName, false).c_str(), false); + BSA::Archive archive; + BSA::EErrorCode res = archive.read(ToString(archivePath, false).c_str(), false); - if ((res != BSA::ERROR_NONE) && (res != BSA::ERROR_INVALIDHASHES)) { - std::ostringstream stream; + if ((res != BSA::ERROR_NONE) && (res != BSA::ERROR_INVALIDHASHES)) { + log::error("invalid bsa '{}', error {}", archivePath, res); + return; + } - stream - << QObject::tr("invalid bsa file: ").toStdString() - << ToString(fileName, false) - << " error code " << res << " - " << ::GetLastError(); + const auto ft = ToFILETIME(std::filesystem::last_write_time(archivePath)); - throw std::runtime_error(stream.str()); - } + addFiles(origin, archive.getRoot(), ft, archiveName, order, stats); - addFiles(origin, archive.getRoot(), fileData.ftLastWriteTime, fileName.substr(namePos), order); - m_Populated = true; - } + m_Populated = true; } void DirectoryEntry::propagateOrigin(int origin) @@ -258,7 +276,7 @@ int DirectoryEntry::anyOrigin() const // if we got here, no file directly within this directory is a valid indicator for a mod, thus // we continue looking in subdirectories - for (DirectoryEntry *entry : m_SubDirectories) { + for (DirectoryEntry* entry : m_SubDirectories) { int res = entry->anyOrigin(); if (res != InvalidOriginID){ return res; @@ -279,7 +297,7 @@ std::vector DirectoryEntry::getFiles() const return result; } -DirectoryEntry *DirectoryEntry::findSubDirectory( +DirectoryEntry* DirectoryEntry::findSubDirectory( const std::wstring &name, bool alreadyLowerCase) const { SubDirectoriesLookup::const_iterator itor; @@ -297,9 +315,10 @@ DirectoryEntry *DirectoryEntry::findSubDirectory( return itor->second; } -DirectoryEntry *DirectoryEntry::findSubDirectoryRecursive(const std::wstring &path) +DirectoryEntry* DirectoryEntry::findSubDirectoryRecursive(const std::wstring &path) { - return getSubDirectoryRecursive(path, false, InvalidOriginID); + DirectoryStats dummy; + return getSubDirectoryRecursive(path, false, dummy, InvalidOriginID); } const FileEntryPtr DirectoryEntry::findFile( @@ -349,7 +368,7 @@ bool DirectoryEntry::containsArchive(std::wstring archiveName) } const FileEntryPtr DirectoryEntry::searchFile( - const std::wstring &path, const DirectoryEntry **directory) const + const std::wstring &path, const DirectoryEntry** directory) const { if (directory != nullptr) { *directory = nullptr; @@ -373,7 +392,7 @@ const FileEntryPtr DirectoryEntry::searchFile( if (iter != m_Files.end()) { return m_FileRegister->getFile(iter->second); } else if (directory != nullptr) { - DirectoryEntry *temp = findSubDirectory(path); + DirectoryEntry* temp = findSubDirectory(path); if (temp != nullptr) { *directory = temp; } @@ -381,7 +400,7 @@ const FileEntryPtr DirectoryEntry::searchFile( } else { // file is in a subdirectory, recurse into the matching subdirectory std::wstring pathComponent = path.substr(0, len); - DirectoryEntry *temp = findSubDirectory(pathComponent); + DirectoryEntry* temp = findSubDirectory(pathComponent); if (temp != nullptr) { if (len >= path.size()) { @@ -401,7 +420,7 @@ void DirectoryEntry::removeFile(FileIndex index) removeFileFromList(index); } -bool DirectoryEntry::removeFile(const std::wstring &filePath, int *origin) +bool DirectoryEntry::removeFile(const std::wstring &filePath, int* origin) { size_t pos = filePath.find_first_of(L"\\/"); @@ -411,7 +430,9 @@ bool DirectoryEntry::removeFile(const std::wstring &filePath, int *origin) std::wstring dirName = filePath.substr(0, pos); std::wstring rest = filePath.substr(pos + 1); - DirectoryEntry *entry = getSubDirectoryRecursive(dirName, false); + + DirectoryStats dummy; + DirectoryEntry* entry = getSubDirectoryRecursive(dirName, false, dummy); if (entry != nullptr) { return entry->removeFile(rest, origin); @@ -426,7 +447,7 @@ void DirectoryEntry::removeDir(const std::wstring &path) if (pos == std::string::npos) { for (auto iter = m_SubDirectories.begin(); iter != m_SubDirectories.end(); ++iter) { - DirectoryEntry *entry = *iter; + DirectoryEntry* entry = *iter; if (CaseInsensitiveEqual(entry->getName(), path)) { entry->removeDirRecursive(); @@ -438,7 +459,9 @@ void DirectoryEntry::removeDir(const std::wstring &path) } else { std::wstring dirName = path.substr(0, pos); std::wstring rest = path.substr(pos + 1); - DirectoryEntry *entry = getSubDirectoryRecursive(dirName, false); + + DirectoryStats dummy; + DirectoryEntry* entry = getSubDirectoryRecursive(dirName, false, dummy); if (entry != nullptr) { entry->removeDir(rest); @@ -446,7 +469,7 @@ void DirectoryEntry::removeDir(const std::wstring &path) } } -bool DirectoryEntry::remove(const std::wstring &fileName, int *origin) +bool DirectoryEntry::remove(const std::wstring &fileName, int* origin) { const auto lcFileName = ToLowerCopy(fileName); @@ -661,18 +684,17 @@ void DirectoryEntry::onFile(Context* cx, std::wstring_view path, FILETIME ft) } void DirectoryEntry::addFiles( - FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, - const std::wstring &archiveName, int order) + FilesOrigin& origin, const BSA::Folder::Ptr archiveFolder, FILETIME fileTime, + const std::wstring& archiveName, int order, DirectoryStats& stats) { - DirectoryStats dummy; - // add files - for (unsigned int fileIdx = 0; fileIdx < archiveFolder->getNumFiles(); ++fileIdx) { - BSA::File::Ptr file = archiveFolder->getFile(fileIdx); + const auto fileCount = archiveFolder->getNumFiles(); + for (unsigned int i=0; igetFile(i); auto f = insert( ToWString(file->getName(), true), origin, fileTime, - archiveName, order, dummy); + archiveName, order, stats); if (f) { if (file->getUncompressedFileSize() > 0) { @@ -684,16 +706,18 @@ void DirectoryEntry::addFiles( } // recurse into subdirectories - for (unsigned int folderIdx = 0; folderIdx < archiveFolder->getNumSubFolders(); ++folderIdx) { - BSA::Folder::Ptr folder = archiveFolder->getSubFolder(folderIdx); - DirectoryEntry *folderEntry = getSubDirectoryRecursive( - ToWString(folder->getName(), true), true, origin.getID()); + const auto dirCount = archiveFolder->getNumSubFolders(); + for (unsigned int i=0; igetSubFolder(i); + + DirectoryEntry* folderEntry = getSubDirectoryRecursive( + ToWString(folder->getName(), true), true, stats, origin.getID()); - folderEntry->addFiles(origin, folder, fileTime, archiveName, order); + folderEntry->addFiles(origin, folder, fileTime, archiveName, order, stats); } } -DirectoryEntry *DirectoryEntry::getSubDirectory( +DirectoryEntry* DirectoryEntry::getSubDirectory( std::wstring_view name, bool create, DirectoryStats& stats, int originID) { std::wstring nameLc = ToLowerCopy(name); @@ -728,7 +752,7 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( } } -DirectoryEntry *DirectoryEntry::getSubDirectory( +DirectoryEntry* DirectoryEntry::getSubDirectory( env::Directory& dir, bool create, DirectoryStats& stats, int originID) { SubDirectoriesLookup::iterator itor; @@ -764,8 +788,8 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( } } -DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive( - const std::wstring &path, bool create, int originID) +DirectoryEntry* DirectoryEntry::getSubDirectoryRecursive( + const std::wstring& path, bool create, DirectoryStats& stats, int originID) { if (path.length() == 0) { // path ended with a backslash? @@ -773,19 +797,18 @@ DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive( } const size_t pos = path.find_first_of(L"\\/"); - DirectoryStats dummy; if (pos == std::wstring::npos) { - return getSubDirectory(path, create, dummy); + return getSubDirectory(path, create, stats); } else { - DirectoryEntry *nextChild = getSubDirectory( - path.substr(0, pos), create, dummy, originID); + DirectoryEntry* nextChild = getSubDirectory( + path.substr(0, pos), create, stats, originID); if (nextChild == nullptr) { return nullptr; } else { return nextChild->getSubDirectoryRecursive( - path.substr(pos + 1), create, originID); + path.substr(pos + 1), create, stats, originID); } } } @@ -798,7 +821,7 @@ void DirectoryEntry::removeDirRecursive() m_FilesLookup.clear(); - for (DirectoryEntry *entry : m_SubDirectories) { + for (DirectoryEntry* entry : m_SubDirectories) { entry->removeDirRecursive(); delete entry; } diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 5ca7389d..a28ceeae 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -50,10 +50,10 @@ class DirectoryEntry { public: DirectoryEntry( - std::wstring name, DirectoryEntry *parent, OriginID originID); + std::wstring name, DirectoryEntry* parent, OriginID originID); DirectoryEntry( - std::wstring name, DirectoryEntry *parent, OriginID originID, + std::wstring name, DirectoryEntry* parent, OriginID originID, boost::shared_ptr fileRegister, boost::shared_ptr originConnection); @@ -85,7 +85,7 @@ public: return !m_Files.empty(); } - const DirectoryEntry *getParent() const + const DirectoryEntry* getParent() const { return m_Parent; } @@ -93,24 +93,32 @@ public: // add files to this directory (and subdirectories) from the specified origin. // That origin may exist or not void addFromOrigin( - const std::wstring &originName, - const std::wstring &directory, int priority, DirectoryStats& stats); + const std::wstring& originName, + const std::wstring& directory, int priority, DirectoryStats& stats); void addFromOrigin( - env::DirectoryWalker& walker, const std::wstring &originName, - const std::wstring &directory, int priority, DirectoryStats& stats); + env::DirectoryWalker& walker, const std::wstring& originName, + const std::wstring& directory, int priority, DirectoryStats& stats); + + void addFromAllBSAs( + const std::wstring& originName, const std::wstring& directory, + int priority, const std::vector& archives, + const std::set& enabledArchives, + const std::vector& loadOrder, + DirectoryStats& stats); void addFromBSA( - const std::wstring &originName, std::wstring &directory, - const std::wstring &fileName, int priority, int order); + const std::wstring& originName, const std::wstring& directory, + const std::wstring& archivePath, int priority, int order, + DirectoryStats& stats); void addFromList( - const std::wstring &originName, const std::wstring &directory, + const std::wstring& originName, const std::wstring& directory, env::Directory& root, int priority, DirectoryStats& stats); void propagateOrigin(OriginID origin); - const std::wstring &getName() const + const std::wstring& getName() const { return m_Name; } @@ -120,9 +128,9 @@ public: return m_FileRegister; } - bool originExists(const std::wstring &name) const; - FilesOrigin &getOriginByID(OriginID ID) const; - FilesOrigin &getOriginByName(const std::wstring &name) const; + bool originExists(const std::wstring& name) const; + FilesOrigin& getOriginByID(OriginID ID) const; + FilesOrigin& getOriginByName(const std::wstring& name) const; const FilesOrigin* findOriginByID(OriginID ID) const; OriginID anyOrigin() const; @@ -130,8 +138,8 @@ public: std::vector getFiles() const; void getSubDirectories( - std::vector::const_iterator &begin, - std::vector::const_iterator &end) const + std::vector::const_iterator& begin, + std::vector::const_iterator& end) const { begin = m_SubDirectories.begin(); end = m_SubDirectories.end(); @@ -179,16 +187,16 @@ public: return m_FileRegister->getFile(index); } - DirectoryEntry *findSubDirectory( - const std::wstring &name, bool alreadyLowerCase=false) const; + DirectoryEntry* findSubDirectory( + const std::wstring& name, bool alreadyLowerCase=false) const; - DirectoryEntry *findSubDirectoryRecursive(const std::wstring &path); + DirectoryEntry* findSubDirectoryRecursive(const std::wstring& path); /** retrieve a file in this directory by name. * @param name name of the file * @return fileentry object for the file or nullptr if no file matches */ - const FileEntryPtr findFile(const std::wstring &name, bool alreadyLowerCase=false) const; + const FileEntryPtr findFile(const std::wstring& name, bool alreadyLowerCase=false) const; const FileEntryPtr findFile(const DirectoryEntryFileKey& key) const; bool hasFile(const std::wstring& name) const; @@ -201,29 +209,29 @@ public: // path containing the file // const FileEntryPtr searchFile( - const std::wstring &path, const DirectoryEntry **directory=nullptr) const; + const std::wstring& path, const DirectoryEntry** directory=nullptr) const; void removeFile(FileIndex index); // remove the specified file from the tree. This can be a path leading to a // file in a subdirectory - bool removeFile(const std::wstring &filePath, OriginID *origin = nullptr); + bool removeFile(const std::wstring& filePath, OriginID* origin = nullptr); /** * @brief remove the specified directory * @param path directory to remove */ - void removeDir(const std::wstring &path); + void removeDir(const std::wstring& path); - bool remove(const std::wstring &fileName, OriginID *origin); + bool remove(const std::wstring& fileName, OriginID* origin); bool hasContentsFromOrigin(OriginID originID) const; - FilesOrigin &createOrigin( - const std::wstring &originName, - const std::wstring &directory, int priority, DirectoryStats& stats); + FilesOrigin& createOrigin( + const std::wstring& originName, + const std::wstring& directory, int priority, DirectoryStats& stats); - void removeFiles(const std::set &indices); + void removeFiles(const std::set& indices); void dump(const std::wstring& file) const; @@ -242,7 +250,7 @@ private: SubDirectories m_SubDirectories; SubDirectoriesLookup m_SubDirectoriesLookup; - DirectoryEntry *m_Parent; + DirectoryEntry* m_Parent; std::set m_Origins; bool m_Populated; bool m_TopLevel; @@ -252,20 +260,20 @@ private: FileEntryPtr insert( - std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, + std::wstring_view fileName, FilesOrigin& origin, FILETIME fileTime, std::wstring_view archive, int order, DirectoryStats& stats); FileEntryPtr insert( - env::File& file, FilesOrigin &origin, + env::File& file, FilesOrigin& origin, std::wstring_view archive, int order, DirectoryStats& stats); void addFiles( - env::DirectoryWalker& walker, FilesOrigin &origin, + env::DirectoryWalker& walker, FilesOrigin& origin, const std::wstring& path, DirectoryStats& stats); void addFiles( - FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, - const std::wstring &archiveName, int order); + FilesOrigin& origin, BSA::Folder::Ptr archiveFolder, FILETIME fileTime, + const std::wstring& archiveName, int order, DirectoryStats& stats); void addDir(FilesOrigin& origin, env::Directory& d, DirectoryStats& stats); @@ -278,7 +286,8 @@ private: OriginID originID = InvalidOriginID); DirectoryEntry* getSubDirectoryRecursive( - const std::wstring &path, bool create, OriginID originID = InvalidOriginID); + const std::wstring& path, bool create, DirectoryStats& stats, + OriginID originID = InvalidOriginID); void removeDirRecursive(); diff --git a/src/shared/util.h b/src/shared/util.h index dc6e72ad..1d3cce82 100644 --- a/src/shared/util.h +++ b/src/shared/util.h @@ -21,6 +21,7 @@ along with Mod Organizer. If not, see . #define UTIL_H #include +#include #include class Executable; @@ -51,6 +52,15 @@ QString getUsvfsVersionString(); void SetThisThreadName(const QString& s); void checkDuplicateShortcuts(const QMenu& m); +inline FILETIME ToFILETIME(std::filesystem::file_time_type t) +{ + FILETIME ft; + static_assert(sizeof(t) == sizeof(ft)); + + std::memcpy(&ft, &t, sizeof(FILETIME)); + return ft; +} + } // namespace MOShared -- cgit v1.3.1 From 528fa988b54342d3d5372ecf9613b60fc7287613 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sat, 15 Feb 2020 17:26:27 -0500 Subject: error handling for last modified time --- src/shared/directoryentry.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 84d68675..639392d2 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -224,7 +224,17 @@ void DirectoryEntry::addFromBSA( return; } - const auto ft = ToFILETIME(std::filesystem::last_write_time(archivePath)); + std::error_code ec; + const auto lwt = std::filesystem::last_write_time(archivePath, ec); + FILETIME ft = {}; + + if (ec) { + log::warn( + "failed to get last modified date for '{}', {}", + archivePath, ec.message()); + } else { + ft = ToFILETIME(lwt); + } addFiles(origin, archive.getRoot(), ft, archiveName, order, stats); -- cgit v1.3.1