From 3a65665d36022c50637fc53d5c9c1ee022c5b3b5 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 10 Feb 2020 10:51:23 -0500 Subject: wstring_view when possible first implementation of NtQueryDirectoryFile --- src/envfs.cpp | 303 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 src/envfs.cpp (limited to 'src/envfs.cpp') diff --git a/src/envfs.cpp b/src/envfs.cpp new file mode 100644 index 00000000..cb2764ae --- /dev/null +++ b/src/envfs.cpp @@ -0,0 +1,303 @@ +#include "envfs.h" +#include +#include + +using namespace MOBase; + +struct NtCloser +{ + using pointer = HANDLE; + void operator()(HANDLE h) + { + } +}; + +using NtHandle = std::unique_ptr; + +typedef struct _UNICODE_STRING { + USHORT Length; + USHORT MaximumLength; + PWSTR Buffer; +} UNICODE_STRING, *PUNICODE_STRING; +typedef const UNICODE_STRING *PCUNICODE_STRING; + +typedef struct _OBJECT_ATTRIBUTES { + ULONG Length; + HANDLE RootDirectory; + PUNICODE_STRING ObjectName; + ULONG Attributes; + PVOID SecurityDescriptor; + PVOID SecurityQualityOfService; +} OBJECT_ATTRIBUTES, *POBJECT_ATTRIBUTES; + + +typedef struct _FILE_DIRECTORY_INFORMATION { + ULONG NextEntryOffset; + ULONG FileIndex; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER EndOfFile; + LARGE_INTEGER AllocationSize; + ULONG FileAttributes; + ULONG FileNameLength; + WCHAR FileName[1]; +} FILE_DIRECTORY_INFORMATION, *PFILE_DIRECTORY_INFORMATION; + + +#define FILE_SHARE_VALID_FLAGS 0x00000007 + +// copied from ntstatus.h +#define STATUS_SUCCESS ((NTSTATUS)0x00000000L) +#define STATUS_BUFFER_OVERFLOW ((NTSTATUS)0x80000005L) +#define STATUS_NO_MORE_FILES ((NTSTATUS)0x80000006L) +#define STATUS_NO_SUCH_FILE ((NTSTATUS)0xC000000FL) + +typedef struct _IO_STATUS_BLOCK IO_STATUS_BLOCK; + +typedef struct _IO_STATUS_BLOCK *PIO_STATUS_BLOCK; +// typedef VOID (NTAPI *PIO_APC_ROUTINE )(__in PVOID ApcContext, __in +// PIO_STATUS_BLOCK IoStatusBlock, __in ULONG Reserved); +typedef VOID(NTAPI *PIO_APC_ROUTINE)(PVOID ApcContext, + PIO_STATUS_BLOCK IoStatusBlock, + ULONG Reserved); + + +typedef enum _FILE_INFORMATION_CLASS { + FileDirectoryInformation = 1 +} FILE_INFORMATION_CLASS; + +typedef NTSTATUS(WINAPI *NtQueryDirectoryFile_type)( + HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BLOCK, PVOID, ULONG, + FILE_INFORMATION_CLASS, BOOLEAN, PUNICODE_STRING, BOOLEAN); + +typedef NTSTATUS(WINAPI *NtOpenFile_type)(PHANDLE, ACCESS_MASK, + POBJECT_ATTRIBUTES, PIO_STATUS_BLOCK, + ULONG, ULONG); + +NtOpenFile_type NtOpenFile = nullptr; +NtQueryDirectoryFile_type NtQueryDirectoryFile = nullptr; + + +#define FILE_DIRECTORY_FILE 0x00000001 +#define FILE_WRITE_THROUGH 0x00000002 +#define FILE_SEQUENTIAL_ONLY 0x00000004 +#define FILE_NO_INTERMEDIATE_BUFFERING 0x00000008 + +#define FILE_SYNCHRONOUS_IO_ALERT 0x00000010 +#define FILE_SYNCHRONOUS_IO_NONALERT 0x00000020 +#define FILE_NON_DIRECTORY_FILE 0x00000040 +#define FILE_CREATE_TREE_CONNECTION 0x00000080 + +#define FILE_COMPLETE_IF_OPLOCKED 0x00000100 +#define FILE_NO_EA_KNOWLEDGE 0x00000200 +#define FILE_OPEN_REMOTE_INSTANCE 0x00000400 +#define FILE_RANDOM_ACCESS 0x00000800 + +#define FILE_DELETE_ON_CLOSE 0x00001000 +#define FILE_OPEN_BY_FILE_ID 0x00002000 +#define FILE_OPEN_FOR_BACKUP_INTENT 0x00004000 +#define FILE_NO_COMPRESSION 0x00008000 + +#if (_WIN32_WINNT >= _WIN32_WINNT_WIN7) +#define FILE_OPEN_REQUIRING_OPLOCK 0x00010000 +#endif + +#define FILE_RESERVE_OPFILTER 0x00100000 +#define FILE_OPEN_REPARSE_POINT 0x00200000 +#define FILE_OPEN_NO_RECALL 0x00400000 +#define FILE_OPEN_FOR_FREE_SPACE_QUERY 0x00800000 + +#define FILE_VALID_OPTION_FLAGS 0x00ffffff +#define FILE_VALID_PIPE_OPTION_FLAGS 0x00000032 +#define FILE_VALID_MAILSLOT_OPTION_FLAGS 0x00000032 +#define FILE_VALID_SET_FLAGS 0x00000036 + +typedef struct _IO_STATUS_BLOCK { +#pragma warning(push) +#pragma warning(disable: 4201) // we'll always use the Microsoft compiler + union { + NTSTATUS Status; + PVOID Pointer; + } DUMMYUNIONNAME; +#pragma warning(pop) + + ULONG_PTR Information; +} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK; + + + +namespace env +{ + +std::wstring_view toStringView(const UNICODE_STRING* s) +{ + if (s && s->Buffer) { + return {s->Buffer, (s->Length / sizeof(wchar_t))}; + } else { + return {}; + } +} + +std::wstring_view toStringView(POBJECT_ATTRIBUTES poa) +{ + if (poa->ObjectName) { + return toStringView(poa->ObjectName); + } + + return {}; +} + +QString toString(POBJECT_ATTRIBUTES poa) +{ + const auto sv = toStringView(poa); + return QString::fromWCharArray(sv.data(), static_cast(sv.size())); +} + + +constexpr std::size_t AllocSize = 1024 * 1024; +std::vector> g_buffers; + + + +void forEachEntryImpl( + void* cx, POBJECT_ATTRIBUTES poa, std::size_t depth, + DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) +{ + IO_STATUS_BLOCK iosb; + UNICODE_STRING ObjectName; + OBJECT_ATTRIBUTES oa = { sizeof(oa), 0, &ObjectName }; + NTSTATUS status; + + //log::debug("{}{}", std::wstring(depth * 2, L' '), toString(poa)); + + status = NtOpenFile( + &oa.RootDirectory, FILE_GENERIC_READ, poa, &iosb, FILE_SHARE_VALID_FLAGS, + FILE_SYNCHRONOUS_IO_NONALERT|FILE_OPEN_REPARSE_POINT|FILE_OPEN_FOR_BACKUP_INTENT); + + if (status < 0) { + log::error( + "NtOpenFile() failed for '{}', {}", + toString(poa), formatSystemMessage(status)); + + return; + } + + NtHandle rootDirectoryHandle(oa.RootDirectory); + + //std::unique_ptr buffer(new unsigned char[AllocSize]); + + unsigned char* buffer; + + if (depth >= g_buffers.size()) { + g_buffers.emplace_back(std::make_unique(AllocSize)); + buffer = g_buffers.back().get(); + } else { + buffer = g_buffers[depth].get(); + } + + union + { + PVOID pv; + PBYTE pb; + PFILE_DIRECTORY_INFORMATION DirInfo; + }; + + for (;;) { + status = NtQueryDirectoryFile( + oa.RootDirectory, NULL, NULL, NULL, &iosb, + buffer, AllocSize, FileDirectoryInformation, FALSE, NULL, FALSE); + + if (status == STATUS_NO_MORE_FILES) { + break; + } else if (status < 0) { + log::error( + "NtQueryDirectoryFile() failed for '{}', {}", + toString(poa), formatSystemMessage(status)); + break; + } + + ULONG NextEntryOffset = 0; + + pv = buffer; + + auto isDotDir = [](auto* o) { + if (o->Length == 2 && o->Buffer[0] == '.') { + return true; + } + + if (o->Length == 4 && o->Buffer[0] == '.' && o->Buffer[1] == '.') { + return true; + } + + return false; + }; + + std::size_t count = 0; + + for (;;) { + ++count; + pb += NextEntryOffset; + + ObjectName.Buffer = DirInfo->FileName; + ObjectName.Length = (USHORT)DirInfo->FileNameLength; + + if (!isDotDir(&ObjectName)) { + ObjectName.MaximumLength = ObjectName.Length; + + if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + dirStartF(cx, toStringView(&oa)); + forEachEntryImpl(cx, &oa, depth+1, dirStartF, dirEndF, fileF); + dirEndF(cx, toStringView(&oa)); + } else { + //log::debug("{}{}", std::wstring((depth + 1) * 2, L' '), toString(&oa)); + FILETIME ft; + ft.dwLowDateTime = DirInfo->LastWriteTime.LowPart; + ft.dwHighDateTime = DirInfo->LastWriteTime.HighPart; + fileF(cx, toStringView(&oa), ft); + } + } + + NextEntryOffset = DirInfo->NextEntryOffset; + + if (NextEntryOffset == 0) { + break; + } + } + + //log::debug("{}{} processed {} files", std::wstring(depth * 2, L' '), toString(poa), count); + + if (AllocSize - iosb.Information > (ULONG)FIELD_OFFSET(FILE_DIRECTORY_INFORMATION, FileName[256])) { + // NO_MORE_FILES + break; + } + } +} + +void forEachEntry( + const std::wstring& path, void* cx, + DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) +{ + if (!NtOpenFile) { + HMODULE m = ::LoadLibraryW(L"ntdll.dll"); + NtOpenFile = (NtOpenFile_type)::GetProcAddress(m, "NtOpenFile"); + NtQueryDirectoryFile = (NtQueryDirectoryFile_type)::GetProcAddress(m, "NtQueryDirectoryFile"); + ::FreeLibrary(m); + } + + const std::wstring ntpath = std::wstring(L"\\??\\") + path; + + UNICODE_STRING ObjectName = {}; + ObjectName.Buffer = const_cast(ntpath.c_str()); + ObjectName.Length = (USHORT)ntpath.size() * sizeof(wchar_t); + ObjectName.MaximumLength = ObjectName.Length; + + OBJECT_ATTRIBUTES oa = {}; + oa.Length = sizeof(oa); + oa.ObjectName = &ObjectName; + + forEachEntryImpl(cx, &oa, 0, dirStartF, dirEndF, fileF); +} + +} // namespace -- cgit v1.3.1 From 6ee4285588019c9842e221cd627f6627cf8ac2fa Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 10 Feb 2020 12:00:15 -0500 Subject: close handles in thread --- src/envfs.cpp | 89 +++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 21 deletions(-) (limited to 'src/envfs.cpp') diff --git a/src/envfs.cpp b/src/envfs.cpp index cb2764ae..81a0d3ef 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -4,16 +4,6 @@ using namespace MOBase; -struct NtCloser -{ - using pointer = HANDLE; - void operator()(HANDLE h) - { - } -}; - -using NtHandle = std::unique_ptr; - typedef struct _UNICODE_STRING { USHORT Length; USHORT MaximumLength; @@ -76,8 +66,12 @@ typedef NTSTATUS(WINAPI *NtOpenFile_type)(PHANDLE, ACCESS_MASK, POBJECT_ATTRIBUTES, PIO_STATUS_BLOCK, ULONG, ULONG); +typedef NTSTATUS(WINAPI *NtClose_type)(HANDLE); + + NtOpenFile_type NtOpenFile = nullptr; NtQueryDirectoryFile_type NtQueryDirectoryFile = nullptr; +extern NtClose_type NtClose = nullptr; #define FILE_DIRECTORY_FILE 0x00000001 @@ -160,9 +154,40 @@ constexpr std::size_t AllocSize = 1024 * 1024; std::vector> g_buffers; +struct HandleCloserThread +{ + std::vector handles; + std::thread thread; + std::atomic busy; + + HandleCloserThread() + : busy(false) + { + } + + ~HandleCloserThread() + { + if (thread.joinable()) { + thread.join(); + } + } + + void closeHandles() + { + for (auto& h : handles) { + NtClose(h); + } + + handles.clear(); + busy = false; + } +}; + +std::array g_handleCloserThreads; + void forEachEntryImpl( - void* cx, POBJECT_ATTRIBUTES poa, std::size_t depth, + void* cx, HandleCloserThread& hc, POBJECT_ATTRIBUTES poa, std::size_t depth, DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) { IO_STATUS_BLOCK iosb; @@ -170,8 +195,6 @@ void forEachEntryImpl( OBJECT_ATTRIBUTES oa = { sizeof(oa), 0, &ObjectName }; NTSTATUS status; - //log::debug("{}{}", std::wstring(depth * 2, L' '), toString(poa)); - status = NtOpenFile( &oa.RootDirectory, FILE_GENERIC_READ, poa, &iosb, FILE_SHARE_VALID_FLAGS, FILE_SYNCHRONOUS_IO_NONALERT|FILE_OPEN_REPARSE_POINT|FILE_OPEN_FOR_BACKUP_INTENT); @@ -184,10 +207,7 @@ void forEachEntryImpl( return; } - NtHandle rootDirectoryHandle(oa.RootDirectory); - - //std::unique_ptr buffer(new unsigned char[AllocSize]); - + hc.handles.push_back(oa.RootDirectory); unsigned char* buffer; if (depth >= g_buffers.size()) { @@ -248,7 +268,7 @@ void forEachEntryImpl( if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { dirStartF(cx, toStringView(&oa)); - forEachEntryImpl(cx, &oa, depth+1, dirStartF, dirEndF, fileF); + forEachEntryImpl(cx, hc, &oa, depth+1, dirStartF, dirEndF, fileF); dirEndF(cx, toStringView(&oa)); } else { //log::debug("{}{}", std::wstring((depth + 1) * 2, L' '), toString(&oa)); @@ -266,8 +286,6 @@ void forEachEntryImpl( } } - //log::debug("{}{} processed {} files", std::wstring(depth * 2, L' '), toString(poa), count); - if (AllocSize - iosb.Information > (ULONG)FIELD_OFFSET(FILE_DIRECTORY_INFORMATION, FileName[256])) { // NO_MORE_FILES break; @@ -275,14 +293,41 @@ void forEachEntryImpl( } } +std::size_t findHandleCloserThread() +{ + for (;;) { + for (std::size_t i=0; i Date: Mon, 10 Feb 2020 14:53:04 -0500 Subject: threaded refresher --- src/directoryrefresher.cpp | 188 ++++++++++++++++++++++++++++++++---------- src/directoryrefresher.h | 42 ++++++---- src/envfs.cpp | 130 +++++++++++++++++------------ src/envfs.h | 88 ++++++++++++++++++++ src/organizercore.cpp | 23 +++--- src/settings.cpp | 10 +++ src/settings.h | 5 ++ src/shared/directoryentry.cpp | 27 ++++++ src/shared/directoryentry.h | 8 ++ 9 files changed, 395 insertions(+), 126 deletions(-) (limited to 'src/envfs.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index b7dd51ba..5bff9d37 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -24,6 +24,7 @@ along with Mod Organizer. If not, see . #include "report.h" #include "modinfo.h" #include "settings.h" +#include "envfs.h" #include #include @@ -36,8 +37,8 @@ using namespace MOBase; using namespace MOShared; -DirectoryRefresher::DirectoryRefresher() - : m_DirectoryStructure(nullptr) +DirectoryRefresher::DirectoryRefresher(std::size_t threadCount) + : m_DirectoryStructure(nullptr), m_threadCount(threadCount) { } @@ -116,59 +117,160 @@ void DirectoryRefresher::addModBSAToStructure(DirectoryEntry *directoryStructure } } -void DirectoryRefresher::addModFilesToStructure(DirectoryEntry *directoryStructure, const QString &modName, - int priority, const QString &directory, const QStringList &stealFiles) +void DirectoryRefresher::stealModFilesIntoStructure( + DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles) { std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); - if (stealFiles.length() > 0) { - // instead of adding all the files of the target directory, we just change the root of the specified - // files to this mod - FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); - for (const QString &filename : stealFiles) { - if (filename.isEmpty()) { - log::warn("Trying to find file with no name"); - continue; - } - QFileInfo fileInfo(filename); - FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); - if (file.get() != nullptr) { - if (file->getOrigin() == 0) { - // replace data as the origin on this bsa - file->removeOrigin(0); - } - origin.addFile(file->getIndex()); - file->addOrigin(origin.getID(), file->getFileTime(), L"", -1); - } else { - QString warnStr = fileInfo.absolutePath(); - if (warnStr.isEmpty()) - warnStr = filename; - log::warn("file not found: {}", warnStr); + // instead of adding all the files of the target directory, we just change the root of the specified + // files to this mod + FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); + for (const QString &filename : stealFiles) { + if (filename.isEmpty()) { + log::warn("Trying to find file with no name"); + continue; + } + QFileInfo fileInfo(filename); + FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName())); + if (file.get() != nullptr) { + if (file->getOrigin() == 0) { + // replace data as the origin on this bsa + file->removeOrigin(0); } + origin.addFile(file->getIndex()); + file->addOrigin(origin.getID(), file->getFileTime(), L"", -1); + } else { + QString warnStr = fileInfo.absolutePath(); + if (warnStr.isEmpty()) + warnStr = filename; + log::warn("file not found: {}", warnStr); } + } +} + +void DirectoryRefresher::addModFilesToStructure( + DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles) +{ + TimeThis tt("addModFilesToStructure()"); + + std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + + if (stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, modName, priority, directory, stealFiles); } else { directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); } } void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure - , const QString &modName - , int priority - , const QString &directory - , const QStringList &stealFiles - , const QStringList &archives) + , const QString &modName + , int priority + , const QString &directory + , const QStringList &stealFiles + , const QStringList &archives) { - addModFilesToStructure(directoryStructure, modName, priority, directory, stealFiles); + TimeThis tt("addModToStructure()"); + + if (stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, modName, priority, directory, stealFiles); + } else { + std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); + } if (Settings::instance().archiveParsing()) { addModBSAToStructure(directoryStructure, modName, priority, directory, archives); } } +struct ModThread +{ + std::wstring path; + env::Directory* dir = nullptr; + std::condition_variable cv; + std::mutex mutex; + bool ready = false; + + void wakeup() + { + ready = true; + cv.notify_one(); + } + + void run() + { + std::unique_lock lock(mutex); + cv.wait(lock, [&]{ return ready; }); + + *dir = env::getFilesAndDirs(path); + + ready = false; + } +}; + +void DirectoryRefresher::addMultipleModsFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, + const std::vector& entries, bool emitProgress) +{ + TimeThis tt(QString("add %1 mods").arg(entries.size())); + + env::ThreadPool threads(m_threadCount); + std::vector dirs(entries.size()); + + for (std::size_t i=0; i(i + 1); + + try { + if (e.stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); + } else { + auto& mt = threads.request(); + + mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); + mt.dir = &dirs[i]; + + mt.wakeup(); + } + } catch (const std::exception& ex) { + emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); + } + + if (emitProgress) { + emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + } + } + + threads.join(); + + for (std::size_t i=0; i(i + 1); + + directoryStructure->addFromList( + entries[i].modName.toStdWString(), + entries[i].absolutePath.toStdWString(), + dirs[i], + prio); + + if (Settings::instance().archiveParsing()) { + addModBSAToStructure( + directoryStructure, + entries[i].modName, + prio, + entries[i].absolutePath, + entries[i].archives); + } + } +} + void DirectoryRefresher::refresh() { SetThisThreadName("DirectoryRefresher"); - TimeThis tt("DirectoryRefresher::refresh()"); QMutexLocker locker(&m_RefreshLock); @@ -178,20 +280,16 @@ void DirectoryRefresher::refresh() IPluginGame *game = qApp->property("managed_game").value(); - std::wstring dataDirectory = QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + std::wstring dataDirectory = + QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0); - std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs){return lhs.priority < rhs.priority;}); - auto iter = m_Mods.begin(); + std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { + return lhs.priority < rhs.priority; + }); - for (int i = 1; iter != m_Mods.end(); ++iter, ++i) { - try { - addModToStructure(m_DirectoryStructure, iter->modName, i, iter->absolutePath, iter->stealFiles, iter->archives); - } catch (const std::exception &e) { - emit error(tr("failed to read mod (%1): %2").arg(iter->modName, e.what())); - } - emit progress((i * 100) / static_cast(m_Mods.size()) + 1); - } + addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); m_DirectoryStructure->getFileRegister()->sortOrigins(); diff --git a/src/directoryrefresher.h b/src/directoryrefresher.h index a4fc5dbc..c531ba39 100644 --- a/src/directoryrefresher.h +++ b/src/directoryrefresher.h @@ -39,12 +39,23 @@ class DirectoryRefresher : public QObject Q_OBJECT public: + struct EntryInfo { + EntryInfo(const QString &modName, const QString &absolutePath, + const QStringList &stealFiles, const QStringList &archives, int priority) + : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles) + , archives(archives), priority(priority) {} + QString modName; + QString absolutePath; + QStringList stealFiles; + QStringList archives; + int priority; + }; /** * @brief constructor * **/ - DirectoryRefresher(); + DirectoryRefresher(std::size_t threadCount); ~DirectoryRefresher(); @@ -53,7 +64,7 @@ public: * * returns a pointer to the updated directory structure. DirectoryRefresher * deletes its own pointer and the caller takes custody of the pointer - * + * * @return updated directory structure **/ MOShared::DirectoryEntry *getDirectoryStructure(); @@ -107,7 +118,13 @@ public: * @param directory * @param stealFiles */ - void addModFilesToStructure(MOShared::DirectoryEntry *directoryStructure, const QString &modName, int priority, const QString &directory, const QStringList &stealFiles); + void addModFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles); + + void addMultipleModsFilesToStructure( + MOShared::DirectoryEntry *directoryStructure, + const std::vector& entries, bool emitProgress=false); public slots: @@ -123,26 +140,15 @@ signals: void refreshed(); private: - - struct EntryInfo { - EntryInfo(const QString &modName, const QString &absolutePath, - const QStringList &stealFiles, const QStringList &archives, int priority) - : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles) - , archives(archives), priority(priority) {} - QString modName; - QString absolutePath; - QStringList stealFiles; - QStringList archives; - int priority; - }; - -private: - std::vector m_Mods; std::set m_EnabledArchives; MOShared::DirectoryEntry *m_DirectoryStructure; QMutex m_RefreshLock; + std::size_t m_threadCount; + void stealModFilesIntoStructure( + MOShared::DirectoryEntry *directoryStructure, const QString &modName, + int priority, const QString &directory, const QStringList &stealFiles); }; #endif // DIRECTORYREFRESHER_H diff --git a/src/envfs.cpp b/src/envfs.cpp index 81a0d3ef..2716737f 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -150,44 +150,62 @@ QString toString(POBJECT_ATTRIBUTES poa) } -constexpr std::size_t AllocSize = 1024 * 1024; -std::vector> g_buffers; - - -struct HandleCloserThread +class HandleCloserThread { - std::vector handles; - std::thread thread; - std::atomic busy; - +public: HandleCloserThread() - : busy(false) + : m_ready(false) { + m_handles.reserve(50'000); } - ~HandleCloserThread() + void add(HANDLE h) { - if (thread.joinable()) { - thread.join(); - } + m_handles.push_back(h); + } + + void wakeup() + { + m_ready = true; + m_cv.notify_one(); + } + + void run() + { + std::unique_lock lock(m_mutex); + m_cv.wait(lock, [&]{ return m_ready; }); + + closeHandles(); } +private: + std::vector m_handles; + std::condition_variable m_cv; + std::mutex m_mutex; + bool m_ready; + void closeHandles() { - for (auto& h : handles) { + for (auto& h : m_handles) { NtClose(h); } - handles.clear(); - busy = false; + m_handles.clear(); + m_ready = false; } }; -std::array g_handleCloserThreads; +constexpr std::size_t AllocSize = 1024 * 1024; +static ThreadPool g_handleClosers; +void setHandleCloserThreadCount(std::size_t n) +{ + g_handleClosers.setMax(n); +} void forEachEntryImpl( - void* cx, HandleCloserThread& hc, POBJECT_ATTRIBUTES poa, std::size_t depth, + void* cx, HandleCloserThread& hc, std::vector>& buffers, + POBJECT_ATTRIBUTES poa, std::size_t depth, DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) { IO_STATUS_BLOCK iosb; @@ -207,14 +225,14 @@ void forEachEntryImpl( return; } - hc.handles.push_back(oa.RootDirectory); + hc.add(oa.RootDirectory); unsigned char* buffer; - if (depth >= g_buffers.size()) { - g_buffers.emplace_back(std::make_unique(AllocSize)); - buffer = g_buffers.back().get(); + if (depth >= buffers.size()) { + buffers.emplace_back(std::make_unique(AllocSize)); + buffer = buffers.back().get(); } else { - buffer = g_buffers[depth].get(); + buffer = buffers[depth].get(); } union @@ -268,7 +286,7 @@ void forEachEntryImpl( if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { dirStartF(cx, toStringView(&oa)); - forEachEntryImpl(cx, hc, &oa, depth+1, dirStartF, dirEndF, fileF); + forEachEntryImpl(cx, hc, buffers, &oa, depth+1, dirStartF, dirEndF, fileF); dirEndF(cx, toStringView(&oa)); } else { //log::debug("{}{}", std::wstring((depth + 1) * 2, L' '), toString(&oa)); @@ -293,35 +311,13 @@ void forEachEntryImpl( } } -std::size_t findHandleCloserThread() -{ - for (;;) { - for (std::size_t i=0; i> buffers; if (!NtOpenFile) { HMODULE m = ::LoadLibraryW(L"ntdll.dll"); @@ -342,9 +338,41 @@ void forEachEntry( oa.Length = sizeof(oa); oa.ObjectName = &ObjectName; - forEachEntryImpl(cx, hc, &oa, 0, dirStartF, dirEndF, fileF); + forEachEntryImpl(cx, hc, buffers, &oa, 0, dirStartF, dirEndF, fileF); + hc.wakeup(); +} + +Directory getFilesAndDirs(const std::wstring& path) +{ + struct Context + { + std::stack current; + }; + + Directory root; + + Context cx; + cx.current.push(&root); + + env::forEachEntry(path, &cx, + [](void* pcx, std::wstring_view path) { + Context* cx = (Context*)pcx; + cx->current.top()->dirs.push_back({std::wstring(path.begin(), path.end())}); + cx->current.push(&cx->current.top()->dirs.back()); + }, + + [](void* pcx, std::wstring_view path) { + Context* cx = (Context*)pcx; + cx->current.pop(); + }, + + [](void* pcx, std::wstring_view path, FILETIME ft) { + Context* cx = (Context*)pcx; + cx->current.top()->files.push_back({std::wstring(path.begin(), path.end()), ft}); + } + ); - hc.thread = std::thread([hci]{ g_handleCloserThreads[hci].closeHandles(); }); + return root; } } // namespace diff --git a/src/envfs.h b/src/envfs.h index aeaa6796..1fc53bcf 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -1,17 +1,105 @@ #ifndef ENV_ENVFS_H #define ENV_ENVFS_H +#include + namespace env { +struct File +{ + std::wstring name; + FILETIME ft; +}; + +struct Directory +{ + std::wstring name; + std::list dirs; + std::list files; +}; + + +template +class ThreadPool +{ +public: + ThreadPool(std::size_t max=1) + : m_threads(max) + { + } + + ~ThreadPool() + { + join(); + } + + void setMax(std::size_t n) + { + m_threads.resize(n); + } + + void join() + { + for (auto& ti : m_threads) { + if (ti.thread.joinable()) { + ti.thread.join(); + } + } + } + + T& request() + { + if (m_threads.empty()) { + std::terminate(); + } + + for (;;) { + for (auto& ti : m_threads) { + bool expected = false; + + if (ti.busy.compare_exchange_strong(expected, true)) { + if (ti.thread.joinable()) { + ti.thread.join(); + } + + ti.thread = std::thread([&]{ + ti.o.run(); + ti.busy = false; + }); + + return ti.o; + } + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + +private: + struct ThreadInfo + { + std::thread thread; + std::atomic busy; + T o; + }; + + std::list m_threads; +}; + + using DirStartF = void (void*, std::wstring_view); using DirEndF = void (void*, std::wstring_view); using FileF = void (void*, std::wstring_view, FILETIME); +void setHandleCloserThreadCount(std::size_t n); + void forEachEntry( const std::wstring& path, void* cx, DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF); +Directory getFilesAndDirs(const std::wstring& path); + } // namespace #endif // ENV_ENVFS_H diff --git a/src/organizercore.cpp b/src/organizercore.cpp index 8124be1d..86abeb35 100644 --- a/src/organizercore.cpp +++ b/src/organizercore.cpp @@ -88,24 +88,19 @@ QStringList toStringList(InputIterator current, InputIterator end) OrganizerCore::OrganizerCore(Settings &settings) : m_UserInterface(nullptr) , m_PluginContainer(nullptr) - , m_GameName() , m_CurrentProfile(nullptr) , m_Settings(settings) , m_Updater(NexusInterface::instance(m_PluginContainer)) - , m_AboutToRun() - , m_FinishedRun() - , m_ModInstalled() , m_ModList(m_PluginContainer, this) , m_PluginList(this) - , m_DirectoryRefresher() + , m_DirectoryRefresher(settings.refreshThreadCount()) , m_DirectoryStructure(new DirectoryEntry(L"data", nullptr, 0)) , m_DownloadManager(NexusInterface::instance(m_PluginContainer), this) - , m_InstallationManager() - , m_RefresherThread() , m_DirectoryUpdate(false) , m_ArchivesInit(false) , m_PluginListsWriter(std::bind(&OrganizerCore::savePluginList, this)) { + env::setHandleCloserThreadCount(settings.refreshThreadCount()); m_DownloadManager.setOutputDirectory(m_Settings.paths().downloads(), false); NexusInterface::instance(m_PluginContainer)->setCacheDirectory( @@ -1245,13 +1240,17 @@ void OrganizerCore::updateModInDirectoryStructure(unsigned int index, void OrganizerCore::updateModsInDirectoryStructure(QMap modInfo) { + std::vector entries; + for (auto idx : modInfo.keys()) { - // add files of the bsa to the directory structure - m_DirectoryRefresher.addModFilesToStructure( - m_DirectoryStructure, modInfo[idx]->name(), - m_CurrentProfile->getModPriority(idx), modInfo[idx]->absolutePath(), - modInfo[idx]->stealFiles()); + entries.push_back({ + modInfo[idx]->name(), modInfo[idx]->absolutePath(), + modInfo[idx]->stealFiles(), {}, m_CurrentProfile->getModPriority(idx)}); } + + m_DirectoryRefresher.addMultipleModsFilesToStructure( + m_DirectoryStructure, entries); + DirectoryRefresher::cleanStructure(m_DirectoryStructure); // need to refresh plugin list now so we can activate esps refreshESPList(true); diff --git a/src/settings.cpp b/src/settings.cpp index 761cd669..f0496fe8 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -209,6 +209,16 @@ void Settings::setUseSplash(bool b) set(m_Settings, "Settings", "use_splash", b); } +std::size_t Settings::refreshThreadCount() const +{ + return get(m_Settings, "Settings", "refresh_thread_count", 10); +} + +void Settings::setRefreshThreadCount(std::size_t n) const +{ + return set(m_Settings, "Settings", "refresh_thread_count", n); +} + std::optional Settings::version() const { if (auto v=getOptional(m_Settings, "General", "version")) { diff --git a/src/settings.h b/src/settings.h index 3d37bdd8..b2cb6be5 100644 --- a/src/settings.h +++ b/src/settings.h @@ -731,6 +731,11 @@ public: bool useSplash() const; void setUseSplash(bool b); + // number of threads to use when refreshing + // + std::size_t refreshThreadCount() const; + void setRefreshThreadCount(std::size_t n) const; + GameSettings& game(); const GameSettings& game() const; diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index d171be38..b7329833 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -691,6 +691,33 @@ void DirectoryEntry::addFromOrigin( m_Populated = true; } +void DirectoryEntry::addFromList( + const std::wstring &originName, const std::wstring &directory, + env::Directory& root, int priority) +{ + FilesOrigin &origin = createOrigin(originName, directory, priority); + addDir(origin, root); +} + +void DirectoryEntry::addDir(FilesOrigin& origin, env::Directory& d) +{ + for (auto& sd : d.dirs) { + auto* sdirEntry = getSubDirectory(sd.name, true, origin.getID()); + sdirEntry->addDir(origin, sd); + } + + for (auto& f : d.files) { + insert(f.name, origin, f.ft, L"", -1); + } + + std::sort( + m_SubDirectories.begin(), + m_SubDirectories.end(), + &DirCompareByName); + + m_Populated = true; +} + void DirectoryEntry::addFromBSA( const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order) diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 58e70ffa..772899d7 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -33,7 +33,9 @@ along with Mod Organizer. If not, see . #include #include #endif + #include "util.h" +#include "envfs.h" namespace MOShared { struct DirectoryEntryFileKey; } @@ -358,6 +360,10 @@ public: const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order); + void addFromList( + const std::wstring &originName, const std::wstring &directory, + env::Directory& root, int priority); + void propagateOrigin(int origin); const std::wstring &getName() const @@ -511,6 +517,8 @@ private: FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, const std::wstring &archiveName, int order); + void addDir(FilesOrigin& origin, env::Directory& d); + DirectoryEntry* getSubDirectory( std::wstring_view name, bool create, int originID = -1); -- cgit v1.3.1 From 7860e927a771d6a07d9b1905071bd389a0e64001 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 10 Feb 2020 15:10:27 -0500 Subject: removed dead insertFile() avoided a few string copies lowercase filename in thread --- src/envfs.cpp | 8 +++++++- src/envfs.h | 3 ++- src/shared/directoryentry.cpp | 45 ++++++++++++++++++++++++------------------- src/shared/directoryentry.h | 10 ++++++---- 4 files changed, 40 insertions(+), 26 deletions(-) (limited to 'src/envfs.cpp') diff --git a/src/envfs.cpp b/src/envfs.cpp index 2716737f..80749046 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -1,4 +1,5 @@ #include "envfs.h" +#include "util.h" #include #include @@ -368,7 +369,12 @@ Directory getFilesAndDirs(const std::wstring& path) [](void* pcx, std::wstring_view path, FILETIME ft) { Context* cx = (Context*)pcx; - cx->current.top()->files.push_back({std::wstring(path.begin(), path.end()), ft}); + + cx->current.top()->files.push_back({ + std::wstring(path.begin(), path.end()), + MOShared::ToLowerCopy(path), + ft + }); } ); diff --git a/src/envfs.h b/src/envfs.h index 1fc53bcf..6cc98775 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -9,7 +9,8 @@ namespace env struct File { std::wstring name; - FILETIME ft; + std::wstring lcname; + FILETIME lastModified; }; struct Directory diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index b7329833..5df2c1ba 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -174,8 +174,8 @@ FileEntry::FileEntry() : { } -FileEntry::FileEntry(Index index, std::wstring_view name, DirectoryEntry *parent) : - m_Index(index), m_Name(name.begin(), name.end()), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), +FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : + m_Index(index), m_Name(std::move(name)), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent), m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), m_LastAccessed(time(nullptr)) { @@ -511,7 +511,7 @@ bool FileRegister::indexValid(FileEntry::Index index) const return (m_Files.find(index) != m_Files.end()); } -FileEntry::Ptr FileRegister::createFile(std::wstring_view name, DirectoryEntry *parent) +FileEntry::Ptr FileRegister::createFile(std::wstring name, DirectoryEntry *parent) { FileEntry::Index index = generateIndex(); @@ -707,7 +707,7 @@ void DirectoryEntry::addDir(FilesOrigin& origin, env::Directory& d) } for (auto& f : d.files) { - insert(f.name, origin, f.ft, L"", -1); + insert(f, origin, L"", -1); } std::sort( @@ -944,20 +944,6 @@ const FileEntry::Ptr DirectoryEntry::searchFile( return FileEntry::Ptr(); } -void DirectoryEntry::insertFile( - const std::wstring &filePath, FilesOrigin &origin, FILETIME fileTime) -{ - size_t pos = filePath.find_first_of(L"\\/"); - - if (pos == std::string::npos) { - this->insert(filePath, origin, fileTime, std::wstring(), -1); - } else { - std::wstring dirName = filePath.substr(0, pos); - std::wstring rest = filePath.substr(pos + 1); - getSubDirectoryRecursive(dirName, true, origin.getID())->insertFile(rest, origin, fileTime); - } -} - void DirectoryEntry::removeFile(FileEntry::Index index) { removeFileFromList(index); @@ -1065,9 +1051,8 @@ FileEntry::Ptr DirectoryEntry::insert( if (iter != m_Files.end()) { file = m_FileRegister->getFile(iter->second); } else { - file = m_FileRegister->createFile(fileName, this); + file = m_FileRegister->createFile(std::wstring(fileName.begin(), fileName.end()), this); addFileToList(std::move(fileNameLower), file->getIndex()); - // fileNameLower has moved from this point } @@ -1077,6 +1062,26 @@ FileEntry::Ptr DirectoryEntry::insert( return file; } +FileEntry::Ptr DirectoryEntry::insert( + env::File& file, FilesOrigin &origin, std::wstring_view archive, int order) +{ + auto iter = m_Files.find(file.lcname); + FileEntry::Ptr fe; + + if (iter != m_Files.end()) { + fe = m_FileRegister->getFile(iter->second); + } else { + fe = m_FileRegister->createFile(std::move(file.name), this); + addFileToList(std::move(file.lcname), fe->getIndex()); + // both file.name and file.lcname have been moved from this point + } + + fe->addOrigin(origin.getID(), file.lastModified, archive, order); + origin.addFile(fe->getIndex()); + + return fe; +} + void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOffset) { struct Context diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 772899d7..f8c72d77 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -80,7 +80,7 @@ public: AlternativesVector; FileEntry(); - FileEntry(Index index, std::wstring_view name, DirectoryEntry *parent); + FileEntry(Index index, std::wstring name, DirectoryEntry *parent); Index getIndex() const { @@ -263,7 +263,7 @@ public: bool indexValid(FileEntry::Index index) const; - FileEntry::Ptr createFile(std::wstring_view name, DirectoryEntry *parent); + FileEntry::Ptr createFile(std::wstring name, DirectoryEntry *parent); FileEntry::Ptr getFile(FileEntry::Index index) const; size_t size() const @@ -459,8 +459,6 @@ public: const FileEntry::Ptr searchFile( const std::wstring &path, const DirectoryEntry **directory=nullptr) const; - void insertFile(const std::wstring &filePath, FilesOrigin &origin, FILETIME fileTime); - void removeFile(FileEntry::Index index); // remove the specified file from the tree. This can be a path leading to a @@ -510,6 +508,10 @@ private: std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, std::wstring_view archive, int order); + FileEntry::Ptr insert( + env::File& file, FilesOrigin &origin, + std::wstring_view archive, int order); + void addFiles( FilesOrigin &origin, wchar_t *buffer, int bufferOffset); -- cgit v1.3.1 From d082a91ad617808a6dd89ad9fdb9479524795520 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Wed, 12 Feb 2020 16:14:18 -0500 Subject: faster getSubDirectory() for env::Directory, added lcname instrumentation --- src/envfs.cpp | 32 ++++- src/envfs.h | 11 ++ src/shared/directoryentry.cpp | 313 +++++++++++++++++++++++++++++++++++++----- src/shared/directoryentry.h | 59 +++++++- 4 files changed, 366 insertions(+), 49 deletions(-) (limited to 'src/envfs.cpp') diff --git a/src/envfs.cpp b/src/envfs.cpp index 80749046..d7245f66 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -1,4 +1,5 @@ #include "envfs.h" +#include "env.h" #include "util.h" #include #include @@ -160,6 +161,11 @@ public: m_handles.reserve(50'000); } + void shrink() + { + m_handles.shrink_to_fit(); + } + void add(HANDLE h) { m_handles.push_back(h); @@ -204,6 +210,15 @@ void setHandleCloserThreadCount(std::size_t n) g_handleClosers.setMax(n); } +void shrinkFs() +{ + g_handleClosers.join(); + + g_handleClosers.forEach([](auto&& t) { + t.shrink(); + }); +} + void forEachEntryImpl( void* cx, HandleCloserThread& hc, std::vector>& buffers, POBJECT_ATTRIBUTES poa, std::size_t depth, @@ -290,7 +305,6 @@ void forEachEntryImpl( forEachEntryImpl(cx, hc, buffers, &oa, depth+1, dirStartF, dirEndF, fileF); dirEndF(cx, toStringView(&oa)); } else { - //log::debug("{}{}", std::wstring((depth + 1) * 2, L' '), toString(&oa)); FILETIME ft; ft.dwLowDateTime = DirInfo->LastWriteTime.LowPart; ft.dwHighDateTime = DirInfo->LastWriteTime.HighPart; @@ -321,11 +335,10 @@ void forEachEntry( std::vector> buffers; if (!NtOpenFile) { - HMODULE m = ::LoadLibraryW(L"ntdll.dll"); - NtOpenFile = (NtOpenFile_type)::GetProcAddress(m, "NtOpenFile"); - NtQueryDirectoryFile = (NtQueryDirectoryFile_type)::GetProcAddress(m, "NtQueryDirectoryFile"); - NtClose = (NtClose_type)::GetProcAddress(m, "NtClose"); - ::FreeLibrary(m); + LibraryPtr m(::LoadLibraryW(L"ntdll.dll")); + NtOpenFile = (NtOpenFile_type)::GetProcAddress(m.get(), "NtOpenFile"); + NtQueryDirectoryFile = (NtQueryDirectoryFile_type)::GetProcAddress(m.get(), "NtQueryDirectoryFile"); + NtClose = (NtClose_type)::GetProcAddress(m.get(), "NtClose"); } const std::wstring ntpath = std::wstring(L"\\??\\") + path; @@ -358,7 +371,12 @@ Directory getFilesAndDirs(const std::wstring& path) env::forEachEntry(path, &cx, [](void* pcx, std::wstring_view path) { Context* cx = (Context*)pcx; - cx->current.top()->dirs.push_back({std::wstring(path.begin(), path.end())}); + + cx->current.top()->dirs.push_back({ + std::wstring(path.begin(), path.end()), + MOShared::ToLowerCopy(path) + }); + cx->current.push(&cx->current.top()->dirs.back()); }, diff --git a/src/envfs.h b/src/envfs.h index 6cc98775..6e0da6d0 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -16,6 +16,8 @@ struct File struct Directory { std::wstring name; + std::wstring lcname; + std::list dirs; std::list files; }; @@ -77,6 +79,14 @@ public: } } + template + void forEach(F&& f) + { + for (auto& ti : m_threads) { + f(ti.o); + } + } + private: struct ThreadInfo { @@ -94,6 +104,7 @@ using DirEndF = void (void*, std::wstring_view); using FileF = void (void*, std::wstring_view, FILETIME); void setHandleCloserThreadCount(std::size_t n); +void shrinkFs(); void forEachEntry( const std::wstring& path, void* cx, diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 5df2c1ba..a90f6f08 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -40,6 +40,30 @@ namespace MOShared using namespace MOBase; static const int MAXPATH_UNICODE = 32767; + +static std::atomic FileEntryCount(0); +static std::atomic FilesOriginCount(0); +static std::atomic FileRegisterCount(0); +static std::atomic DirectoryEntryCount(0); +static std::atomic OriginConnectionCount(0); + +template +std::chrono::nanoseconds elapsed(F&& f) +{ + const auto start = std::chrono::high_resolution_clock::now(); + f(); + const auto end = std::chrono::high_resolution_clock::now(); + return (end - start); +} + +void logcounts(std::string w) +{ + log::debug( + "{}: FileEntry={} FilesOrigin={} FileRegister={} DirectoryEntry={} OriginConnection={}", + w, FileEntryCount, FilesOriginCount, FileRegisterCount, DirectoryEntryCount, OriginConnectionCount); +} + + static std::wstring tail(const std::wstring &source, const size_t count) { if (count >= source.length()) { @@ -71,6 +95,101 @@ static bool DirCompareByName(const DirectoryEntry *lhs, const DirectoryEntry *rh } +DirectoryStats::DirectoryStats() +{ + std::memset(this, 0, sizeof(DirectoryStats)); +} + +DirectoryStats& DirectoryStats::operator+=(const DirectoryStats& o) +{ + dirTimes += o.dirTimes; + fileTimes += o.fileTimes; + sortTimes += o.sortTimes; + + subdirLookupTimes += o.subdirLookupTimes; + addDirectoryTimes += o.addDirectoryTimes; + + filesLookupTimes += o.filesLookupTimes; + addFileTimes += o.addFileTimes; + addOriginToFileTimes += o.addOriginToFileTimes; + addFileToOriginTimes += o.addFileToOriginTimes; + addFileToRegisterTimes += o.addFileToRegisterTimes; + + originExists += o.originExists; + originCreate += o.originCreate; + originsNeededEnabled += o.originsNeededEnabled; + + subdirExists += o.subdirExists; + subdirCreate += o.subdirCreate; + + fileExists += o.fileExists; + fileCreate += o.fileCreate; + filesInsertedInRegister += o.filesInsertedInRegister; + filesAssignedInRegister += o.filesAssignedInRegister; + + return *this; +} + +std::string DirectoryStats::csvHeader() +{ + QStringList sl = { + "dirTimes", + "fileTimes", + "sortTimes", + "subdirLookupTimes", + "addDirectoryTimes", + "filesLookupTimes", + "addFileTimes", + "addOriginToFileTimes", + "addFileToOriginTimes", + "addFileToRegisterTimes", + "originExists", + "originCreate", + "originsNeededEnabled", + "subdirExists", + "subdirCreate", + "fileExists", + "fileCreate", + "filesInsertedInRegister", + "filesAssignedInRegister"}; + + return sl.join(",").toStdString(); +} + +std::string DirectoryStats::toCsv() const +{ + QStringList oss; + + oss + << QString::number(dirTimes.count()) + << QString::number(fileTimes.count()) + << QString::number(sortTimes.count()) + + << QString::number(subdirLookupTimes.count()) + << QString::number(addDirectoryTimes.count()) + + << QString::number(filesLookupTimes.count()) + << QString::number(addFileTimes.count()) + << QString::number(addOriginToFileTimes.count()) + << QString::number(addFileToOriginTimes.count()) + << QString::number(addFileToRegisterTimes.count()) + + << QString::number(originExists) + << QString::number(originCreate) + << QString::number(originsNeededEnabled) + + << QString::number(subdirExists) + << QString::number(subdirCreate) + + << QString::number(fileExists) + << QString::number(fileCreate) + << QString::number(filesInsertedInRegister) + << QString::number(filesAssignedInRegister); + + return oss.join(",").toStdString(); +} + + class OriginConnection { public: @@ -80,6 +199,12 @@ public: OriginConnection() : m_NextID(0) { + ++OriginConnectionCount; + } + + ~OriginConnection() + { + --OriginConnectionCount; } FilesOrigin& createOrigin( @@ -172,6 +297,7 @@ FileEntry::FileEntry() : m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), m_LastAccessed(time(nullptr)) { + ++FileEntryCount; } FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : @@ -179,6 +305,16 @@ FileEntry::FileEntry(Index index, std::wstring name, DirectoryEntry *parent) : m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize), m_LastAccessed(time(nullptr)) { + ++FileEntryCount; +} + +FileEntry::~FileEntry() +{ + while (!m_Alternatives.empty()) { + m_Alternatives.pop_back(); + } + + --FileEntryCount; } void FileEntry::addOrigin( @@ -407,6 +543,7 @@ bool FileEntry::recurseParents(std::wstring &path, const DirectoryEntry *parent) FilesOrigin::FilesOrigin() : m_ID(0), m_Disabled(false), m_Name(), m_Path(), m_Priority(0) { + ++FilesOriginCount; } FilesOrigin::FilesOrigin(const FilesOrigin &reference) @@ -418,6 +555,7 @@ FilesOrigin::FilesOrigin(const FilesOrigin &reference) , m_FileRegister(reference.m_FileRegister) , m_OriginConnection(reference.m_OriginConnection) { + ++FilesOriginCount; } FilesOrigin::FilesOrigin( @@ -428,6 +566,12 @@ FilesOrigin::FilesOrigin( m_Priority(priority), m_FileRegister(fileRegister), m_OriginConnection(originConnection) { + ++FilesOriginCount; +} + +FilesOrigin::~FilesOrigin() +{ + --FilesOriginCount; } void FilesOrigin::setPriority(int priority) @@ -468,8 +612,15 @@ FileEntry::Ptr FilesOrigin::findFile(FileEntry::Index index) const } void FilesOrigin::enable(bool enabled, time_t notAfter) +{ + DirectoryStats dummy; + enable(enabled, dummy, notAfter); +} + +void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) { if (!enabled) { + ++stats.originsNeededEnabled; std::set copy = m_Files; m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter); m_Files.clear(); @@ -504,6 +655,12 @@ bool FilesOrigin::containsArchive(std::wstring archiveName) FileRegister::FileRegister(boost::shared_ptr originConnection) : m_OriginConnection(originConnection) { + ++FileRegisterCount; +} + +FileRegister::~FileRegister() +{ + --FileRegisterCount; } bool FileRegister::indexValid(FileEntry::Index index) const @@ -511,14 +668,26 @@ bool FileRegister::indexValid(FileEntry::Index index) const return (m_Files.find(index) != m_Files.end()); } -FileEntry::Ptr FileRegister::createFile(std::wstring name, DirectoryEntry *parent) +FileEntry::Ptr FileRegister::createFile( + std::wstring name, DirectoryEntry *parent, DirectoryStats& stats) { FileEntry::Index index = generateIndex(); + FileEntry::Ptr p; - auto r = m_Files.insert_or_assign( - index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); + stats.addFileToRegisterTimes += elapsed([&]{ + auto r = m_Files.insert_or_assign( + index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); - return r.first->second; + if (r.second) { + ++stats.filesInsertedInRegister; + } else { + ++stats.filesAssignedInRegister; + } + + p = r.first->second; + }); + + return p; } FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const @@ -643,6 +812,7 @@ DirectoryEntry::DirectoryEntry( m_OriginConnection(new OriginConnection), m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(true) { + ++DirectoryEntryCount; m_FileRegister.reset(new FileRegister(m_OriginConnection)); m_Origins.insert(originID); } @@ -654,23 +824,24 @@ DirectoryEntry::DirectoryEntry( m_FileRegister(fileRegister), m_OriginConnection(originConnection), m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(false) { + ++DirectoryEntryCount; m_Origins.insert(originID); } DirectoryEntry::~DirectoryEntry() { + --DirectoryEntryCount; clear(); } void DirectoryEntry::clear() { - m_Files.clear(); - m_FilesLookup.clear(); - - for (DirectoryEntry *entry : m_SubDirectories) { - delete entry; + for (auto itor=m_SubDirectories.rbegin(); itor!=m_SubDirectories.rend(); ++itor) { + delete *itor; } + m_Files.clear(); + m_FilesLookup.clear(); m_SubDirectories.clear(); m_SubDirectoriesLookup.clear(); } @@ -678,7 +849,8 @@ void DirectoryEntry::clear() void DirectoryEntry::addFromOrigin( const std::wstring &originName, const std::wstring &directory, int priority) { - FilesOrigin &origin = createOrigin(originName, directory, priority); + DirectoryStats dummy; + FilesOrigin &origin = createOrigin(originName, directory, priority, dummy); if (directory.length() != 0) { boost::scoped_array buffer(new wchar_t[MAXPATH_UNICODE + 1]); @@ -693,27 +865,36 @@ void DirectoryEntry::addFromOrigin( void DirectoryEntry::addFromList( const std::wstring &originName, const std::wstring &directory, - env::Directory& root, int priority) + env::Directory& root, int priority, DirectoryStats& stats) { - FilesOrigin &origin = createOrigin(originName, directory, priority); - addDir(origin, root); + stats = {}; + + FilesOrigin &origin = createOrigin(originName, directory, priority, stats); + addDir(origin, root, stats); } -void DirectoryEntry::addDir(FilesOrigin& origin, env::Directory& d) +void DirectoryEntry::addDir( + FilesOrigin& origin, env::Directory& d, DirectoryStats& stats) { - for (auto& sd : d.dirs) { - auto* sdirEntry = getSubDirectory(sd.name, true, origin.getID()); - sdirEntry->addDir(origin, sd); - } + stats.dirTimes += elapsed([&]{ + for (auto& sd : d.dirs) { + auto* sdirEntry = getSubDirectory(sd, true, stats, origin.getID()); + sdirEntry->addDir(origin, sd, stats); + } + }); - for (auto& f : d.files) { - insert(f, origin, L"", -1); - } + stats.fileTimes += elapsed([&]{ + for (auto& f : d.files) { + insert(f, origin, L"", -1, stats); + } + }); - std::sort( - m_SubDirectories.begin(), - m_SubDirectories.end(), - &DirCompareByName); + stats.sortTimes += elapsed([&]{ + std::sort( + m_SubDirectories.begin(), + m_SubDirectories.end(), + &DirCompareByName); + }); m_Populated = true; } @@ -722,7 +903,8 @@ void DirectoryEntry::addFromBSA( const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order) { - FilesOrigin &origin = createOrigin(originName, directory, priority); + DirectoryStats dummy; + FilesOrigin &origin = createOrigin(originName, directory, priority, dummy); WIN32_FILE_ATTRIBUTE_DATA fileData; if (::GetFileAttributesExW(fileName.c_str(), GetFileExInfoStandard, &fileData) == 0) { @@ -1022,13 +1204,16 @@ bool DirectoryEntry::hasContentsFromOrigin(int originID) const } FilesOrigin &DirectoryEntry::createOrigin( - const std::wstring &originName, const std::wstring &directory, int priority) + const std::wstring &originName, const std::wstring &directory, int priority, + DirectoryStats& stats) { if (m_OriginConnection->exists(originName)) { + ++stats.originExists; FilesOrigin &origin = m_OriginConnection->getByName(originName); - origin.enable(true); + origin.enable(true, stats); return origin; } else { + ++stats.originCreate; return m_OriginConnection->createOrigin( originName, directory, priority, m_FileRegister, m_OriginConnection); } @@ -1051,7 +1236,11 @@ FileEntry::Ptr DirectoryEntry::insert( if (iter != m_Files.end()) { file = m_FileRegister->getFile(iter->second); } else { - file = m_FileRegister->createFile(std::wstring(fileName.begin(), fileName.end()), this); + DirectoryStats dummy; + + file = m_FileRegister->createFile( + std::wstring(fileName.begin(), fileName.end()), this, dummy); + addFileToList(std::move(fileNameLower), file->getIndex()); // fileNameLower has moved from this point } @@ -1063,21 +1252,37 @@ FileEntry::Ptr DirectoryEntry::insert( } FileEntry::Ptr DirectoryEntry::insert( - env::File& file, FilesOrigin &origin, std::wstring_view archive, int order) + env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, + DirectoryStats& stats) { - auto iter = m_Files.find(file.lcname); + FilesMap::iterator itor; + + stats.filesLookupTimes += elapsed([&]{ + itor = m_Files.find(file.lcname); + }); + FileEntry::Ptr fe; - if (iter != m_Files.end()) { - fe = m_FileRegister->getFile(iter->second); + if (itor != m_Files.end()) { + ++stats.fileExists; + fe = m_FileRegister->getFile(itor->second); } else { - fe = m_FileRegister->createFile(std::move(file.name), this); - addFileToList(std::move(file.lcname), fe->getIndex()); + fe = m_FileRegister->createFile(std::move(file.name), this, stats); + + stats.addFileTimes += elapsed([&]{ + addFileToList(std::move(file.lcname), fe->getIndex()); + }); + // both file.name and file.lcname have been moved from this point } - fe->addOrigin(origin.getID(), file.lastModified, archive, order); - origin.addFile(fe->getIndex()); + stats.addOriginToFileTimes += elapsed([&]{ + fe->addOrigin(origin.getID(), file.lastModified, archive, order); + }); + + stats.addFileToOriginTimes += elapsed([&]{ + origin.addFile(fe->getIndex()); + }); return fe; } @@ -1205,6 +1410,40 @@ DirectoryEntry *DirectoryEntry::getSubDirectory( } } +DirectoryEntry *DirectoryEntry::getSubDirectory( + env::Directory& dir, bool create, DirectoryStats& stats, int originID) +{ + SubDirectoriesLookup::iterator itor; + + stats.subdirLookupTimes += elapsed([&] { + itor = m_SubDirectoriesLookup.find(dir.lcname); + }); + + if (itor != m_SubDirectoriesLookup.end()) { + ++stats.subdirExists; + return itor->second; + } + + if (create) { + ++stats.subdirCreate; + + auto* entry = new DirectoryEntry( + std::move(dir.name), this, originID, + m_FileRegister, m_OriginConnection); + // dir.name is moved from this point + + stats.addDirectoryTimes += elapsed([&]{ + addDirectoryToList(entry, std::move(dir.lcname)); + }); + + // dir.lcname is moved from this point + + return entry; + } else { + return nullptr; + } +} + DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive( const std::wstring &path, bool create, int originID) { diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index f8c72d77..c8ec8a4e 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -60,6 +60,44 @@ class OriginConnection; class FileRegister; +struct DirectoryStats +{ + std::string mod; + + std::chrono::nanoseconds dirTimes; + std::chrono::nanoseconds fileTimes; + std::chrono::nanoseconds sortTimes; + + std::chrono::nanoseconds subdirLookupTimes; + std::chrono::nanoseconds addDirectoryTimes; + + std::chrono::nanoseconds filesLookupTimes; + std::chrono::nanoseconds addFileTimes; + std::chrono::nanoseconds addOriginToFileTimes; + std::chrono::nanoseconds addFileToOriginTimes; + std::chrono::nanoseconds addFileToRegisterTimes; + + int64_t originExists; + int64_t originCreate; + int64_t originsNeededEnabled; + + int64_t subdirExists; + int64_t subdirCreate; + + int64_t fileExists; + int64_t fileCreate; + int64_t filesInsertedInRegister; + int64_t filesAssignedInRegister; + + DirectoryStats(); + + DirectoryStats& operator+=(const DirectoryStats& o); + + static std::string csvHeader(); + std::string toCsv() const; +}; + + class FileEntry { public: @@ -81,6 +119,7 @@ public: FileEntry(); FileEntry(Index index, std::wstring name, DirectoryEntry *parent); + ~FileEntry(); Index getIndex() const { @@ -195,6 +234,7 @@ class FilesOrigin public: FilesOrigin(); FilesOrigin(const FilesOrigin &reference); + ~FilesOrigin(); // sets priority for this origin, but it will overwrite the existing mapping // for this priority, the previous origin will no longer be referenced @@ -224,7 +264,9 @@ public: std::vector getFiles() const; FileEntry::Ptr findFile(FileEntry::Index index) const; + void enable(bool enabled, DirectoryStats& stats, time_t notAfter = LONG_MAX); void enable(bool enabled, time_t notAfter = LONG_MAX); + bool isDisabled() const { return m_Disabled; @@ -260,10 +302,13 @@ class FileRegister { public: FileRegister(boost::shared_ptr originConnection); + ~FileRegister(); bool indexValid(FileEntry::Index index) const; - FileEntry::Ptr createFile(std::wstring name, DirectoryEntry *parent); + FileEntry::Ptr createFile( + std::wstring name, DirectoryEntry *parent, DirectoryStats& stats); + FileEntry::Ptr getFile(FileEntry::Index index) const; size_t size() const @@ -362,7 +407,7 @@ public: void addFromList( const std::wstring &originName, const std::wstring &directory, - env::Directory& root, int priority); + env::Directory& root, int priority, DirectoryStats& stats); void propagateOrigin(int origin); @@ -477,7 +522,7 @@ public: FilesOrigin &createOrigin( const std::wstring &originName, - const std::wstring &directory, int priority); + const std::wstring &directory, int priority, DirectoryStats& stats); void removeFiles(const std::set &indices); @@ -510,7 +555,7 @@ private: FileEntry::Ptr insert( env::File& file, FilesOrigin &origin, - std::wstring_view archive, int order); + std::wstring_view archive, int order, DirectoryStats& stats); void addFiles( FilesOrigin &origin, wchar_t *buffer, int bufferOffset); @@ -519,11 +564,15 @@ private: FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, const std::wstring &archiveName, int order); - void addDir(FilesOrigin& origin, env::Directory& d); + void addDir(FilesOrigin& origin, env::Directory& d, DirectoryStats& stats); DirectoryEntry* getSubDirectory( std::wstring_view name, bool create, int originID = -1); + DirectoryEntry* getSubDirectory( + env::Directory& dir, bool create, DirectoryStats& stats, + int originID = -1); + DirectoryEntry* getSubDirectoryRecursive( const std::wstring &path, bool create, int originID = -1); -- cgit v1.3.1 From 29244f3328cdf878ac557acbbabb19a5b7190b2d Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Thu, 13 Feb 2020 20:13:37 -0500 Subject: thread-safe directory entry --- src/directoryrefresher.cpp | 113 ++++++++++++++++++++++++++---------------- src/envfs.cpp | 70 ++++++++++++++++++++++---- src/envfs.h | 10 +++- src/shared/directoryentry.cpp | 87 ++++++++++++++++++++++++-------- src/shared/directoryentry.h | 8 +++ 5 files changed, 212 insertions(+), 76 deletions(-) (limited to 'src/envfs.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index 5bff9d37..bce3f65a 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -25,6 +25,7 @@ along with Mod Organizer. If not, see . #include "modinfo.h" #include "settings.h" #include "envfs.h" +#include "modinfodialogfwd.h" #include #include @@ -125,7 +126,10 @@ void DirectoryRefresher::stealModFilesIntoStructure( // instead of adding all the files of the target directory, we just change the root of the specified // files to this mod - FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority); + DirectoryStats dummy; + FilesOrigin &origin = directoryStructure->createOrigin( + ToWString(modName), directoryW, priority, dummy); + for (const QString &filename : stealFiles) { if (filename.isEmpty()) { log::warn("Trying to find file with no name"); @@ -189,8 +193,12 @@ void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure struct ModThread { + DirectoryEntry* ds = nullptr; + std::wstring modName; std::wstring path; + int prio = -1; env::Directory* dir = nullptr; + std::condition_variable cv; std::mutex mutex; bool ready = false; @@ -206,7 +214,16 @@ struct ModThread std::unique_lock lock(mutex); cv.wait(lock, [&]{ return ready; }); - *dir = env::getFilesAndDirs(path); + ds->addFromOrigin(modName, path, prio); + + /*if (Settings::instance().archiveParsing()) { + addModBSAToStructure( + directoryStructure, + entries[i].modName, + prio, + entries[i].absolutePath, + entries[i].archives); + }*/ ready = false; } @@ -216,65 +233,75 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( MOShared::DirectoryEntry *directoryStructure, const std::vector& entries, bool emitProgress) { - TimeThis tt(QString("add %1 mods").arg(entries.size())); - - env::ThreadPool threads(m_threadCount); std::vector dirs(entries.size()); - for (std::size_t i=0; i(i + 1); + { + TimeThis tt("walk dirs"); - try { - if (e.stealFiles.length() > 0) { - stealModFilesIntoStructure( - directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); - } else { - auto& mt = threads.request(); + env::ThreadPool threads(m_threadCount); - mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); - mt.dir = &dirs[i]; + for (std::size_t i=0; i(i + 1); - mt.wakeup(); + try + { + if (e.stealFiles.length() > 0) { + stealModFilesIntoStructure( + directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); + } else { + auto& mt = threads.request(); + + mt.ds = directoryStructure; + mt.modName = entries[i].modName.toStdWString(); + mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); + mt.prio = prio; + mt.dir = &dirs[i]; + + mt.wakeup(); + } + } catch (const std::exception& ex) { + emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); } - } catch (const std::exception& ex) { - emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what())); - } - if (emitProgress) { - emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + if (emitProgress) { + emit progress((static_cast(i) * 100) / static_cast(entries.size()) + 1); + } } - } - - threads.join(); - - for (std::size_t i=0; i(i + 1); - - directoryStructure->addFromList( - entries[i].modName.toStdWString(), - entries[i].absolutePath.toStdWString(), - dirs[i], - prio); - if (Settings::instance().archiveParsing()) { - addModBSAToStructure( - directoryStructure, - entries[i].modName, - prio, - entries[i].absolutePath, - entries[i].archives); - } + threads.join(); } + + //std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){ + // return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0); + //}); + + //static int run = 1; + // + //std::ofstream out("c:\\tmp\\data.csv", std::ios::app); + + //out << fmt::format("what,run,{}", DirectoryStats::csvHeader()); + // + //for (std::size_t i=0; icurrent.top()->dirs.push_back({ - std::wstring(path.begin(), path.end()), - MOShared::ToLowerCopy(path) - }); - + cx->current.top()->dirs.push_back(Directory(path)); cx->current.push(&cx->current.top()->dirs.back()); }, @@ -385,18 +381,70 @@ Directory getFilesAndDirs(const std::wstring& path) cx->current.pop(); }, - [](void* pcx, std::wstring_view path, FILETIME ft) { + [](void* pcx, std::wstring_view path, FILETIME ft) { Context* cx = (Context*)pcx; - cx->current.top()->files.push_back({ - std::wstring(path.begin(), path.end()), - MOShared::ToLowerCopy(path), - ft - }); + cx->current.top()->files.push_back(File(path, ft)); } ); return root; } +File::File(std::wstring_view n, FILETIME ft) : + name(n.begin(), n.end()), + lcname(MOShared::ToLowerCopy(name)), + lastModified(ft) +{ +} + +Directory::Directory() +{ +} + +Directory::Directory(std::wstring_view n) + : name(n.begin(), n.end()), lcname(MOShared::ToLowerCopy(name)) +{ +} + + +void getFilesAndDirsWithFindImpl(const std::wstring& path, Directory& d) +{ + const std::wstring searchString = path + L"\\*"; + + WIN32_FIND_DATAW findData; + + HANDLE searchHandle = ::FindFirstFileExW( + searchString.c_str(), FindExInfoBasic, &findData, FindExSearchNameMatch, + nullptr, FIND_FIRST_EX_LARGE_FETCH); + + if (searchHandle != INVALID_HANDLE_VALUE) { + BOOL result = true; + + while (result) { + if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + if ((wcscmp(findData.cFileName, L".") != 0) && + (wcscmp(findData.cFileName, L"..") != 0)) { + const std::wstring newPath = path + L"\\" + findData.cFileName; + d.dirs.push_back(Directory(findData.cFileName)); + getFilesAndDirsWithFindImpl(newPath, d.dirs.back()); + } + } else { + d.files.push_back(File(findData.cFileName, findData.ftLastWriteTime)); + } + + result = ::FindNextFileW(searchHandle, &findData); + } + } + + ::FindClose(searchHandle); +} + +Directory getFilesAndDirsWithFind(const std::wstring& path) +{ + Directory d; + getFilesAndDirsWithFindImpl(path, d); + return d; +} + } // namespace diff --git a/src/envfs.h b/src/envfs.h index 6e0da6d0..001f2d09 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -11,6 +11,8 @@ struct File std::wstring name; std::wstring lcname; FILETIME lastModified; + + File(std::wstring_view name, FILETIME ft); }; struct Directory @@ -18,8 +20,11 @@ struct Directory std::wstring name; std::wstring lcname; - std::list dirs; - std::list files; + std::vector dirs; + std::vector files; + + Directory(); + Directory(std::wstring_view name); }; @@ -111,6 +116,7 @@ void forEachEntry( DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF); Directory getFilesAndDirs(const std::wstring& path); +Directory getFilesAndDirsWithFind(const std::wstring& path); } // namespace diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index a90f6f08..e1464595 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -214,11 +214,14 @@ public: { int newID = createID(); - m_Origins[newID] = FilesOrigin(newID, originName, directory, priority, fileRegister, originConnection); - m_OriginsNameMap[originName] = newID; - m_OriginsPriorityMap[priority] = newID; + auto itor = m_Origins.insert({newID, FilesOrigin( + newID, originName, directory, priority, + fileRegister, originConnection)}).first; - return m_Origins[newID]; + m_OriginsNameMap.insert({originName, newID}); + m_OriginsPriorityMap.insert({priority, newID}); + + return itor->second; } bool exists(const std::wstring &name) @@ -597,9 +600,13 @@ std::vector FilesOrigin::getFiles() const { std::vector result; - for (FileEntry::Index fileIdx : m_Files) { - if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { - result.push_back(p); + { + std::scoped_lock lock(m_Mutex); + + for (FileEntry::Index fileIdx : m_Files) { + if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { + result.push_back(p); + } } } @@ -621,9 +628,16 @@ void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) { if (!enabled) { ++stats.originsNeededEnabled; - std::set copy = m_Files; + + std::set copy; + + { + std::scoped_lock lock(m_Mutex); + copy = m_Files; + m_Files.clear(); + } + m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter); - m_Files.clear(); } m_Disabled = !enabled; @@ -631,6 +645,8 @@ void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter) void FilesOrigin::removeFile(FileEntry::Index index) { + std::scoped_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { @@ -640,6 +656,8 @@ void FilesOrigin::removeFile(FileEntry::Index index) bool FilesOrigin::containsArchive(std::wstring archiveName) { + std::scoped_lock lock(m_Mutex); + for (FileEntry::Index fileIdx : m_Files) { if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) { if (p->isFromArchive(archiveName)) { @@ -665,6 +683,7 @@ FileRegister::~FileRegister() bool FileRegister::indexValid(FileEntry::Index index) const { + std::scoped_lock lock(m_Mutex); return (m_Files.find(index) != m_Files.end()); } @@ -675,6 +694,8 @@ FileEntry::Ptr FileRegister::createFile( FileEntry::Ptr p; stats.addFileToRegisterTimes += elapsed([&]{ + std::scoped_lock lock(m_Mutex); + auto r = m_Files.insert_or_assign( index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); @@ -692,6 +713,8 @@ FileEntry::Ptr FileRegister::createFile( FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const { + std::scoped_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { @@ -703,6 +726,8 @@ FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const bool FileRegister::removeFile(FileEntry::Index index) { + std::scoped_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { @@ -717,12 +742,15 @@ bool FileRegister::removeFile(FileEntry::Index index) void FileRegister::removeOrigin(FileEntry::Index index, int originID) { + std::unique_lock lock(m_Mutex); + auto iter = m_Files.find(index); if (iter != m_Files.end()) { if (iter->second->removeOrigin(originID)) { - unregisterFile(iter->second); m_Files.erase(iter); + lock.unlock(); + unregisterFile(iter->second); } } else { log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index); @@ -734,17 +762,21 @@ void FileRegister::removeOriginMulti( { std::vector removedFiles; - for (auto iter = indices.begin(); iter != indices.end(); ) { - auto pos = m_Files.find(*iter); + { + std::scoped_lock lock(m_Mutex); + + for (auto iter = indices.begin(); iter != indices.end(); ) { + auto pos = m_Files.find(*iter); - if (pos != m_Files.end() - && (pos->second->lastAccessed() < notAfter) - && pos->second->removeOrigin(originID)) { - removedFiles.push_back(pos->second); - m_Files.erase(pos); - ++iter; - } else { - indices.erase(iter++); + if (pos != m_Files.end() + && (pos->second->lastAccessed() < notAfter) + && pos->second->removeOrigin(originID)) { + removedFiles.push_back(pos->second); + m_Files.erase(pos); + ++iter; + } else { + indices.erase(iter++); + } } } @@ -776,6 +808,8 @@ void FileRegister::removeOriginMulti( void FileRegister::sortOrigins() { + std::scoped_lock lock(m_Mutex); + for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) { iter->second->sortOrigins(); } @@ -1228,6 +1262,8 @@ FileEntry::Ptr DirectoryEntry::insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, std::wstring_view archive, int order) { + std::scoped_lock lock(m_FilesMutex); + std::wstring fileNameLower = ToLowerCopy(fileName); auto iter = m_Files.find(fileNameLower); @@ -1255,6 +1291,8 @@ FileEntry::Ptr DirectoryEntry::insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats) { + std::scoped_lock lock(m_FilesMutex); + FilesMap::iterator itor; stats.filesLookupTimes += elapsed([&]{ @@ -1307,7 +1345,12 @@ void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOf [](void* pcx, std::wstring_view path) { Context* cx = (Context*)pcx; auto* current= cx->current.top(); - std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); + + { + std::scoped_lock lock(current->m_SubDirMutex); + std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); + } + cx->current.pop(); }, @@ -1389,6 +1432,8 @@ void DirectoryEntry::addFiles( DirectoryEntry *DirectoryEntry::getSubDirectory( std::wstring_view name, bool create, int originID) { + std::scoped_lock lock(m_SubDirMutex); + std::wstring nameLc = ToLowerCopy(name); auto itor = m_SubDirectoriesLookup.find(nameLc); diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index c8ec8a4e..71e8b32c 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -26,6 +26,8 @@ along with Mod Organizer. If not, see . #include #include #include +#include + #define WIN32_MEAN_AND_LEAN #include #include @@ -274,6 +276,7 @@ public: void addFile(FileEntry::Index index) { + std::scoped_lock lock(m_Mutex); m_Files.insert(index); } @@ -290,6 +293,7 @@ private: int m_Priority; boost::weak_ptr m_FileRegister; boost::weak_ptr m_OriginConnection; + mutable std::mutex m_Mutex; FilesOrigin( int ID, const std::wstring &name, const std::wstring &path, int priority, @@ -313,6 +317,7 @@ public: size_t size() const { + std::scoped_lock lock(m_Mutex); return m_Files.size(); } @@ -323,6 +328,7 @@ public: void sortOrigins(); private: + mutable std::mutex m_Mutex; std::map m_Files; boost::shared_ptr m_OriginConnection; @@ -545,6 +551,8 @@ private: std::set m_Origins; bool m_Populated; bool m_TopLevel; + std::mutex m_SubDirMutex; + std::mutex m_FilesMutex; DirectoryEntry(const DirectoryEntry &reference); -- cgit v1.3.1 From 763a5d6c08006c319ed92f4088a4d3c211f80cf6 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Thu, 13 Feb 2020 23:03:56 -0500 Subject: more instrumentation tighter mutexes, required thread-safe FileEntry --- src/directoryrefresher.cpp | 106 ++++++++++------ src/envfs.cpp | 2 + src/mainwindow.cpp | 6 +- src/shared/directoryentry.cpp | 282 +++++++++++++++++++++++------------------- src/shared/directoryentry.h | 18 +-- 5 files changed, 241 insertions(+), 173 deletions(-) (limited to 'src/envfs.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index bce3f65a..20d6a52b 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -160,12 +160,14 @@ void DirectoryRefresher::addModFilesToStructure( TimeThis tt("addModFilesToStructure()"); std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); + DirectoryStats dummy; if (stealFiles.length() > 0) { stealModFilesIntoStructure( directoryStructure, modName, priority, directory, stealFiles); } else { - directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); + directoryStructure->addFromOrigin( + ToWString(modName), directoryW, priority, dummy); } } @@ -178,16 +180,20 @@ void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure { TimeThis tt("addModToStructure()"); + DirectoryStats dummy; + if (stealFiles.length() > 0) { stealModFilesIntoStructure( directoryStructure, modName, priority, directory, stealFiles); } else { std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory)); - directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority); + directoryStructure->addFromOrigin( + ToWString(modName), directoryW, priority, dummy); } if (Settings::instance().archiveParsing()) { - addModBSAToStructure(directoryStructure, modName, priority, directory, archives); + addModBSAToStructure( + directoryStructure, modName, priority, directory, archives); } } @@ -198,6 +204,7 @@ struct ModThread std::wstring path; int prio = -1; env::Directory* dir = nullptr; + DirectoryStats* stats = nullptr; std::condition_variable cv; std::mutex mutex; @@ -214,7 +221,8 @@ struct ModThread std::unique_lock lock(mutex); cv.wait(lock, [&]{ return ready; }); - ds->addFromOrigin(modName, path, prio); + SetThisThreadName(QString::fromStdWString(modName + L" refresher")); + ds->addFromOrigin(modName, path, prio, *stats); /*if (Settings::instance().archiveParsing()) { addModBSAToStructure( @@ -229,11 +237,39 @@ struct ModThread } }; +void dumpStats(std::vector& stats) +{ + static int run = 0; + static const std::string file("c:\\tmp\\data.csv"); + + if (run == 0) { + std::ofstream out(file, std::ios::out|std::ios::trunc); + out << fmt::format("what,run,{}", DirectoryStats::csvHeader()) << "\n"; + } + + std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){ + return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0); + }); + + std::ofstream out(file, std::ios::app); + + DirectoryStats total; + for (const auto& s : stats) { + out << fmt::format("{},{},{}", s.mod, run, s.toCsv()) << "\n"; + total += s; + } + + out << fmt::format("total,{},{}", run, total.toCsv()) << "\n"; + + ++run; +} + void DirectoryRefresher::addMultipleModsFilesToStructure( MOShared::DirectoryEntry *directoryStructure, const std::vector& entries, bool emitProgress) { std::vector dirs(entries.size()); + std::vector stats(entries.size()); { TimeThis tt("walk dirs"); @@ -257,6 +293,9 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); mt.prio = prio; mt.dir = &dirs[i]; + mt.stats = &stats[i]; + + stats[i].mod = entries[i].modName.toStdString(); mt.wakeup(); } @@ -272,23 +311,7 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( threads.join(); } - //std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){ - // return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0); - //}); - - //static int run = 1; - // - //std::ofstream out("c:\\tmp\\data.csv", std::ios::app); - - //out << fmt::format("what,run,{}", DirectoryStats::csvHeader()); - // - //for (std::size_t i=0; iproperty("managed_game").value(); + IPluginGame *game = qApp->property("managed_game").value(); - std::wstring dataDirectory = - QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); + std::wstring dataDirectory = + QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString(); - m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0); + { + DirectoryStats dummy; + m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0, dummy); + } - std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { - return lhs.priority < rhs.priority; - }); + std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) { + return lhs.priority < rhs.priority; + }); - addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); + addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true); - m_DirectoryStructure->getFileRegister()->sortOrigins(); + m_DirectoryStructure->getFileRegister()->sortOrigins(); - emit progress(100); + emit progress(100); - cleanStructure(m_DirectoryStructure); + cleanStructure(m_DirectoryStructure); - emit refreshed(); + emit refreshed(); - //logcounts("after refresh"); + //logcounts("after refresh"); + } } diff --git a/src/envfs.cpp b/src/envfs.cpp index 022ca513..5cd36957 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -179,6 +179,8 @@ public: void run() { + MOShared::SetThisThreadName("HandleCloserThread"); + std::unique_lock lock(m_mutex); m_cv.wait(lock, [&]{ return m_ready; }); diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index 3e03c52d..2145011f 100644 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -5148,7 +5148,11 @@ void MainWindow::originModified(int originID) { FilesOrigin &origin = m_OrganizerCore.directoryStructure()->getOriginByID(originID); origin.enable(false); - m_OrganizerCore.directoryStructure()->addFromOrigin(origin.getName(), origin.getPath(), origin.getPriority()); + + DirectoryStats dummy; + m_OrganizerCore.directoryStructure()->addFromOrigin( + origin.getName(), origin.getPath(), origin.getPriority(), dummy); + DirectoryRefresher::cleanStructure(m_OrganizerCore.directoryStructure()); } diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index e1464595..19500167 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -160,19 +160,23 @@ std::string DirectoryStats::toCsv() const { QStringList oss; + auto s = [](auto ns) { + return ns.count() / 1000.0 / 1000.0 / 1000.0; + }; + oss - << QString::number(dirTimes.count()) - << QString::number(fileTimes.count()) - << QString::number(sortTimes.count()) + << QString::number(s(dirTimes)) + << QString::number(s(fileTimes)) + << QString::number(s(sortTimes)) - << QString::number(subdirLookupTimes.count()) - << QString::number(addDirectoryTimes.count()) + << QString::number(s(subdirLookupTimes)) + << QString::number(s(addDirectoryTimes)) - << QString::number(filesLookupTimes.count()) - << QString::number(addFileTimes.count()) - << QString::number(addOriginToFileTimes.count()) - << QString::number(addFileToOriginTimes.count()) - << QString::number(addFileToRegisterTimes.count()) + << QString::number(s(filesLookupTimes)) + << QString::number(s(addFileTimes)) + << QString::number(s(addOriginToFileTimes)) + << QString::number(s(addFileToOriginTimes)) + << QString::number(s(addFileToRegisterTimes)) << QString::number(originExists) << QString::number(originCreate) @@ -323,6 +327,8 @@ FileEntry::~FileEntry() void FileEntry::addOrigin( int origin, FILETIME fileTime, std::wstring_view archive, int order) { + std::scoped_lock lock(m_OriginsMutex); + m_LastAccessed = time(nullptr); if (m_Parent != nullptr) { m_Parent->propagateOrigin(origin); @@ -387,6 +393,8 @@ void FileEntry::addOrigin( bool FileEntry::removeOrigin(int origin) { + std::scoped_lock lock(m_OriginsMutex); + if (m_Origin == origin) { if (!m_Alternatives.empty()) { // find alternative with the highest priority @@ -440,6 +448,8 @@ bool FileEntry::removeOrigin(int origin) void FileEntry::sortOrigins() { + std::scoped_lock lock(m_OriginsMutex); + m_Alternatives.push_back({m_Origin, m_Archive}); std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) { @@ -480,6 +490,8 @@ void FileEntry::sortOrigins() bool FileEntry::isFromArchive(std::wstring archiveName) const { + std::scoped_lock lock(m_OriginsMutex); + if (archiveName.length() == 0) { return m_Archive.first.length() != 0; } @@ -499,6 +511,8 @@ bool FileEntry::isFromArchive(std::wstring archiveName) const std::wstring FileEntry::getFullPath(int originID) const { + std::scoped_lock lock(m_OriginsMutex); + if (originID == -1) { bool ignore = false; originID = getOrigin(ignore); @@ -690,27 +704,34 @@ bool FileRegister::indexValid(FileEntry::Index index) const FileEntry::Ptr FileRegister::createFile( std::wstring name, DirectoryEntry *parent, DirectoryStats& stats) { - FileEntry::Index index = generateIndex(); + const auto index = generateIndex(); FileEntry::Ptr p; stats.addFileToRegisterTimes += elapsed([&]{ - std::scoped_lock lock(m_Mutex); + bool inserted = false; + p = FileEntry::Ptr(new FileEntry(index, std::move(name), parent)); - auto r = m_Files.insert_or_assign( - index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); + { + std::scoped_lock lock(m_Mutex); + inserted = m_Files.insert_or_assign(index, p).second; + } - if (r.second) { + if (inserted) { ++stats.filesInsertedInRegister; } else { ++stats.filesAssignedInRegister; } - - p = r.first->second; }); return p; } +FileEntry::Index FileRegister::generateIndex() +{ + static std::atomic sIndex(0); + return sIndex++; +} + FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const { std::scoped_lock lock(m_Mutex); @@ -815,12 +836,6 @@ void FileRegister::sortOrigins() } } -FileEntry::Index FileRegister::generateIndex() -{ - static std::atomic sIndex(0); - return sIndex++; -} - void FileRegister::unregisterFile(FileEntry::Ptr file) { bool ignore; @@ -881,17 +896,13 @@ void DirectoryEntry::clear() } void DirectoryEntry::addFromOrigin( - const std::wstring &originName, const std::wstring &directory, int priority) + const std::wstring &originName, const std::wstring &directory, int priority, + DirectoryStats& stats) { - DirectoryStats dummy; - FilesOrigin &origin = createOrigin(originName, directory, priority, dummy); + FilesOrigin &origin = createOrigin(originName, directory, priority, stats); - if (directory.length() != 0) { - boost::scoped_array buffer(new wchar_t[MAXPATH_UNICODE + 1]); - memset(buffer.get(), L'\0', MAXPATH_UNICODE + 1); - int offset = _snwprintf(buffer.get(), MAXPATH_UNICODE, L"%ls", directory.c_str()); - buffer.get()[offset] = L'\0'; - addFiles(origin, buffer.get(), offset); + if (!directory.empty()) { + addFiles(origin, directory, stats); } m_Populated = true; @@ -1260,58 +1271,80 @@ void DirectoryEntry::removeFiles(const std::set &indices) FileEntry::Ptr DirectoryEntry::insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, - std::wstring_view archive, int order) + std::wstring_view archive, int order, DirectoryStats& stats) { - std::scoped_lock lock(m_FilesMutex); - std::wstring fileNameLower = ToLowerCopy(fileName); + FileEntry::Ptr fe; - auto iter = m_Files.find(fileNameLower); - FileEntry::Ptr file; + FileKey key(std::move(fileNameLower)); - if (iter != m_Files.end()) { - file = m_FileRegister->getFile(iter->second); - } else { - DirectoryStats dummy; + { + std::unique_lock lock(m_FilesMutex); + + FilesLookup::iterator itor; + + stats.filesLookupTimes += elapsed([&]{ + itor = m_FilesLookup.find(key); + }); + + if (itor != m_FilesLookup.end()) { + lock.unlock(); + ++stats.fileExists; + fe = m_FileRegister->getFile(itor->second); + } else { + ++stats.fileCreate; + fe = m_FileRegister->createFile( + std::wstring(fileName.begin(), fileName.end()), this, stats); - file = m_FileRegister->createFile( - std::wstring(fileName.begin(), fileName.end()), this, dummy); + stats.addFileTimes += elapsed([&] { + addFileToList(std::move(key.value), fe->getIndex()); + }); - addFileToList(std::move(fileNameLower), file->getIndex()); - // fileNameLower has moved from this point + // fileNameLower has moved from this point + } } - file->addOrigin(origin.getID(), fileTime, archive, order); - origin.addFile(file->getIndex()); + stats.addOriginToFileTimes += elapsed([&]{ + fe->addOrigin(origin.getID(), fileTime, archive, order); + }); + + stats.addFileToOriginTimes += elapsed([&]{ + origin.addFile(fe->getIndex()); + }); - return file; + return fe; } FileEntry::Ptr DirectoryEntry::insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats) { - std::scoped_lock lock(m_FilesMutex); + FileEntry::Ptr fe; - FilesMap::iterator itor; + { + std::unique_lock lock(m_FilesMutex); - stats.filesLookupTimes += elapsed([&]{ - itor = m_Files.find(file.lcname); - }); + FilesMap::iterator itor; - FileEntry::Ptr fe; + stats.filesLookupTimes += elapsed([&]{ + itor = m_Files.find(file.lcname); + }); - if (itor != m_Files.end()) { - ++stats.fileExists; - fe = m_FileRegister->getFile(itor->second); - } else { - fe = m_FileRegister->createFile(std::move(file.name), this, stats); + if (itor != m_Files.end()) { + lock.unlock(); + ++stats.fileExists; + fe = m_FileRegister->getFile(itor->second); + } else { + ++stats.fileCreate; + fe = m_FileRegister->createFile(std::move(file.name), this, stats); + // file.name has been moved from this point - stats.addFileTimes += elapsed([&]{ - addFileToList(std::move(file.lcname), fe->getIndex()); - }); + stats.addFileTimes += elapsed([&]{ + addFileToList(std::move(file.lcname), fe->getIndex()); + }); - // both file.name and file.lcname have been moved from this point + // file.lcname has been moved from this point + } } stats.addOriginToFileTimes += elapsed([&]{ @@ -1325,91 +1358,75 @@ FileEntry::Ptr DirectoryEntry::insert( return fe; } -void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOffset) +void DirectoryEntry::addFiles( + FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats) { struct Context { FilesOrigin& origin; + DirectoryStats& stats; std::stack current; }; - Context cx = {origin}; + Context cx = {origin, stats}; cx.current.push(this); - env::forEachEntry(buffer, &cx, - [](void* pcx, std::wstring_view path) { - Context* cx = (Context*)pcx; - cx->current.push(cx->current.top()->getSubDirectory(path, true, cx->origin.getID())); - }, - - [](void* pcx, std::wstring_view path) { + env::forEachEntry(path, &cx, + [](void* pcx, std::wstring_view path) + { Context* cx = (Context*)pcx; - auto* current= cx->current.top(); + cx->stats.dirTimes += elapsed([&] { + auto* sd = cx->current.top()->getSubDirectory( + path, true, cx->stats, cx->origin.getID()); - { - std::scoped_lock lock(current->m_SubDirMutex); - std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); - } - - cx->current.pop(); + cx->current.push(sd); + }); }, - [](void* pcx, std::wstring_view path, FILETIME ft) { + [](void* pcx, std::wstring_view path) + { Context* cx = (Context*)pcx; - cx->current.top()->insert(path, cx->origin, ft, L"", -1); - } - ); - - /* - WIN32_FIND_DATAW findData; - _snwprintf_s(buffer + bufferOffset, MAXPATH_UNICODE - bufferOffset, _TRUNCATE, L"\\*"); + cx->stats.dirTimes += elapsed([&] { + auto* current= cx->current.top(); - HANDLE searchHandle = nullptr; - - if (SupportOptimizedFind()) { - searchHandle = ::FindFirstFileExW( - buffer, FindExInfoBasic, &findData, FindExSearchNameMatch, nullptr, - FIND_FIRST_EX_LARGE_FETCH); - } else { - searchHandle = ::FindFirstFileExW( - buffer, FindExInfoStandard, &findData, FindExSearchNameMatch, nullptr, 0); - } + { + std::scoped_lock lock(current->m_SubDirMutex); - if (searchHandle != INVALID_HANDLE_VALUE) { - BOOL result = true; + std::sort( + current->m_SubDirectories.begin(), + current->m_SubDirectories.end(), + &DirCompareByName); + } - while (result) { - if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - if ((wcscmp(findData.cFileName, L".") != 0) && - (wcscmp(findData.cFileName, L"..") != 0)) { - int offset = _snwprintf(buffer + bufferOffset, MAXPATH_UNICODE, L"\\%ls", findData.cFileName); + cx->current.pop(); + }); + }, - // recurse into subdirectories - DirectoryEntry* sd = getSubDirectory(findData.cFileName, true, origin.getID()); - sd->addFiles(origin, buffer, bufferOffset + offset); - } - } else { - insert(findData.cFileName, origin, findData.ftLastWriteTime, L"", -1); - } + [](void* pcx, std::wstring_view path, FILETIME ft) + { + Context* cx = (Context*)pcx; - result = ::FindNextFileW(searchHandle, &findData); + cx->stats.fileTimes += elapsed([&]{ + cx->current.top()->insert(path, cx->origin, ft, L"", -1, cx->stats); + }); } - } - - std::sort(m_SubDirectories.begin(), m_SubDirectories.end(), &DirCompareByName); - ::FindClose(searchHandle);*/ + ); } void DirectoryEntry::addFiles( FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, const std::wstring &archiveName, int order) { + DirectoryStats dummy; + // add files for (unsigned int fileIdx = 0; fileIdx < archiveFolder->getNumFiles(); ++fileIdx) { BSA::File::Ptr file = archiveFolder->getFile(fileIdx); - auto f = insert(ToWString(file->getName(), true), origin, fileTime, archiveName, order); + auto f = insert( + ToWString(file->getName(), true), origin, fileTime, + archiveName, order, dummy); if (f) { if (file->getUncompressedFileSize() > 0) { @@ -1423,31 +1440,41 @@ void DirectoryEntry::addFiles( // recurse into subdirectories for (unsigned int folderIdx = 0; folderIdx < archiveFolder->getNumSubFolders(); ++folderIdx) { BSA::Folder::Ptr folder = archiveFolder->getSubFolder(folderIdx); - DirectoryEntry *folderEntry = getSubDirectoryRecursive(ToWString(folder->getName(), true), true, origin.getID()); + DirectoryEntry *folderEntry = getSubDirectoryRecursive( + ToWString(folder->getName(), true), true, origin.getID()); folderEntry->addFiles(origin, folder, fileTime, archiveName, order); } } DirectoryEntry *DirectoryEntry::getSubDirectory( - std::wstring_view name, bool create, int originID) + std::wstring_view name, bool create, DirectoryStats& stats, int originID) { + std::wstring nameLc = ToLowerCopy(name); + std::scoped_lock lock(m_SubDirMutex); - std::wstring nameLc = ToLowerCopy(name); - auto itor = m_SubDirectoriesLookup.find(nameLc); + SubDirectoriesLookup::iterator itor; + stats.subdirLookupTimes += elapsed([&] { + itor = m_SubDirectoriesLookup.find(nameLc); + }); if (itor != m_SubDirectoriesLookup.end()) { + ++stats.subdirExists; return itor->second; } if (create) { + ++stats.subdirCreate; + auto* entry = new DirectoryEntry( std::wstring(name.begin(), name.end()), this, originID, m_FileRegister, m_OriginConnection); - addDirectoryToList(entry, std::move(nameLc)); - // nameLc is moved from this point + stats.addDirectoryTimes += elapsed([&] { + addDirectoryToList(entry, std::move(nameLc)); + // nameLc is moved from this point + }); return entry; } else { @@ -1498,16 +1525,19 @@ DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive( } const size_t pos = path.find_first_of(L"\\/"); + DirectoryStats dummy; if (pos == std::wstring::npos) { - return getSubDirectory(path, create); + return getSubDirectory(path, create, dummy); } else { - DirectoryEntry *nextChild = getSubDirectory(path.substr(0, pos), create, originID); + DirectoryEntry *nextChild = getSubDirectory( + path.substr(0, pos), create, dummy, originID); if (nextChild == nullptr) { return nullptr; } else { - return nextChild->getSubDirectoryRecursive(path.substr(pos + 1), create, originID); + return nextChild->getSubDirectoryRecursive( + path.substr(pos + 1), create, originID); } } } diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 71e8b32c..8f6afbb1 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -221,6 +221,7 @@ private: DirectoryEntry *m_Parent; mutable FILETIME m_FileTime; uint64_t m_FileSize, m_CompressedFileSize; + mutable std::mutex m_OriginsMutex; time_t m_LastAccessed; @@ -328,12 +329,14 @@ public: void sortOrigins(); private: + using FileMap = std::map; + mutable std::mutex m_Mutex; - std::map m_Files; + FileMap m_Files; boost::shared_ptr m_OriginConnection; - FileEntry::Index generateIndex(); void unregisterFile(FileEntry::Ptr file); + FileEntry::Index generateIndex(); }; @@ -354,7 +357,7 @@ struct DirectoryEntryFileKey return std::hash()(value); } - const std::wstring value; + std::wstring value; const std::size_t hash; }; @@ -405,7 +408,7 @@ public: // That origin may exist or not void addFromOrigin( const std::wstring &originName, - const std::wstring &directory, int priority); + const std::wstring &directory, int priority, DirectoryStats& stats); void addFromBSA( const std::wstring &originName, std::wstring &directory, @@ -559,14 +562,14 @@ private: FileEntry::Ptr insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, - std::wstring_view archive, int order); + std::wstring_view archive, int order, DirectoryStats& stats); FileEntry::Ptr insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats); void addFiles( - FilesOrigin &origin, wchar_t *buffer, int bufferOffset); + FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats); void addFiles( FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, @@ -575,7 +578,8 @@ private: void addDir(FilesOrigin& origin, env::Directory& d, DirectoryStats& stats); DirectoryEntry* getSubDirectory( - std::wstring_view name, bool create, int originID = -1); + std::wstring_view name, bool create, DirectoryStats& stats, + int originID = -1); DirectoryEntry* getSubDirectory( env::Directory& dir, bool create, DirectoryStats& stats, -- cgit v1.3.1 From c8fc7abade6e28507f2a1590007a098e647026a9 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Fri, 14 Feb 2020 01:28:50 -0500 Subject: thread-safe OriginConnection ThreadPool now keeps threads running keep ModThreads around so avoid reallocating buffers --- src/directoryrefresher.cpp | 37 +++++++++------- src/envfs.cpp | 24 +++++----- src/envfs.h | 96 +++++++++++++++++++++++++++++++++------ src/shared/directoryentry.cpp | 101 +++++++++++++++++++++++++++++++++--------- src/shared/directoryentry.h | 8 +++- 5 files changed, 202 insertions(+), 64 deletions(-) (limited to 'src/envfs.cpp') diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp index 20d6a52b..a59ddf9d 100644 --- a/src/directoryrefresher.cpp +++ b/src/directoryrefresher.cpp @@ -203,8 +203,8 @@ struct ModThread std::wstring modName; std::wstring path; int prio = -1; - env::Directory* dir = nullptr; DirectoryStats* stats = nullptr; + env::DirectoryWalker walker; std::condition_variable cv; std::mutex mutex; @@ -212,7 +212,11 @@ struct ModThread void wakeup() { - ready = true; + { + std::scoped_lock lock(mutex); + ready = true; + } + cv.notify_one(); } @@ -222,7 +226,7 @@ struct ModThread cv.wait(lock, [&]{ return ready; }); SetThisThreadName(QString::fromStdWString(modName + L" refresher")); - ds->addFromOrigin(modName, path, prio, *stats); + ds->addFromOrigin(walker, modName, path, prio, *stats); /*if (Settings::instance().archiveParsing()) { addModBSAToStructure( @@ -237,6 +241,9 @@ struct ModThread } }; +env::ThreadPool g_threads; + + void dumpStats(std::vector& stats) { static int run = 0; @@ -268,35 +275,33 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( MOShared::DirectoryEntry *directoryStructure, const std::vector& entries, bool emitProgress) { - std::vector dirs(entries.size()); std::vector stats(entries.size()); + g_threads.setMax(m_threadCount); + { TimeThis tt("walk dirs"); - env::ThreadPool threads(m_threadCount); - for (std::size_t i=0; i(i + 1); + stats[i].mod = entries[i].modName.toStdString(); + try { if (e.stealFiles.length() > 0) { stealModFilesIntoStructure( directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles); } else { - auto& mt = threads.request(); + auto& mt = g_threads.request(); mt.ds = directoryStructure; mt.modName = entries[i].modName.toStdWString(); mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString(); mt.prio = prio; - mt.dir = &dirs[i]; mt.stats = &stats[i]; - stats[i].mod = entries[i].modName.toStdString(); - mt.wakeup(); } } catch (const std::exception& ex) { @@ -308,7 +313,7 @@ void DirectoryRefresher::addMultipleModsFilesToStructure( } } - threads.join(); + g_threads.waitForAll(); } dumpStats(stats); @@ -347,12 +352,12 @@ void DirectoryRefresher::refresh() m_DirectoryStructure->getFileRegister()->sortOrigins(); - emit progress(100); - cleanStructure(m_DirectoryStructure); + } - emit refreshed(); + emit progress(100); - //logcounts("after refresh"); - } + emit refreshed(); + + //logcounts("after refresh"); } diff --git a/src/envfs.cpp b/src/envfs.cpp index 5cd36957..a67faf84 100644 --- a/src/envfs.cpp +++ b/src/envfs.cpp @@ -212,15 +212,6 @@ void setHandleCloserThreadCount(std::size_t n) g_handleClosers.setMax(n); } -void shrinkFs() -{ - g_handleClosers.join(); - - g_handleClosers.forEach([](auto&& t) { - t.shrink(); - }); -} - void forEachEntryImpl( void* cx, HandleCloserThread& hc, std::vector>& buffers, POBJECT_ATTRIBUTES poa, std::size_t depth, @@ -328,14 +319,13 @@ void forEachEntryImpl( } } -void forEachEntry( + +void DirectoryWalker::forEachEntry( const std::wstring& path, void* cx, DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) { auto& hc = g_handleClosers.request(); - std::vector> buffers; - if (!NtOpenFile) { LibraryPtr m(::LoadLibraryW(L"ntdll.dll")); NtOpenFile = (NtOpenFile_type)::GetProcAddress(m.get(), "NtOpenFile"); @@ -354,10 +344,18 @@ void forEachEntry( oa.Length = sizeof(oa); oa.ObjectName = &ObjectName; - forEachEntryImpl(cx, hc, buffers, &oa, 0, dirStartF, dirEndF, fileF); + forEachEntryImpl(cx, hc, m_buffers, &oa, 0, dirStartF, dirEndF, fileF); hc.wakeup(); } + +void forEachEntry( + const std::wstring& path, void* cx, + DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF) +{ + DirectoryWalker().forEachEntry(path, cx, dirStartF, dirEndF, fileF); +} + Directory getFilesAndDirs(const std::wstring& path) { struct Context diff --git a/src/envfs.h b/src/envfs.h index 001f2d09..8790b071 100644 --- a/src/envfs.h +++ b/src/envfs.h @@ -33,13 +33,13 @@ class ThreadPool { public: ThreadPool(std::size_t max=1) - : m_threads(max) { + setMax(max); } ~ThreadPool() { - join(); + stopAndJoin(); } void setMax(std::size_t n) @@ -47,8 +47,13 @@ public: m_threads.resize(n); } - void join() + void stopAndJoin() { + for (auto& ti : m_threads) { + ti.stop = true; + ti.wakeup(); + } + for (auto& ti : m_threads) { if (ti.thread.joinable()) { ti.thread.join(); @@ -56,6 +61,26 @@ public: } } + void waitForAll() + { + for (;;) { + bool done = true; + + for (auto& ti : m_threads) { + if (ti.busy) { + done = false; + break; + } + } + + if (done) { + break; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + T& request() { if (m_threads.empty()) { @@ -67,15 +92,7 @@ public: bool expected = false; if (ti.busy.compare_exchange_strong(expected, true)) { - if (ti.thread.joinable()) { - ti.thread.join(); - } - - ti.thread = std::thread([&]{ - ti.o.run(); - ti.busy = false; - }); - + ti.wakeup(); return ti.o; } } @@ -98,6 +115,47 @@ private: std::thread thread; std::atomic busy; T o; + + std::condition_variable cv; + std::mutex mutex; + bool ready; + + std::atomic stop; + + ThreadInfo() + : busy(true), ready(false), stop(false) + { + thread = std::thread([&]{ run(); }); + } + + void wakeup() + { + { + std::scoped_lock lock(mutex); + ready = true; + } + + cv.notify_one(); + } + + void run() + { + busy = false; + + while (!stop) { + std::unique_lock lock(mutex); + cv.wait(lock, [&]{ return ready; }); + + if (stop) { + break; + } + + o.run(); + + ready = false; + busy = false; + } + } }; std::list m_threads; @@ -109,7 +167,19 @@ using DirEndF = void (void*, std::wstring_view); using FileF = void (void*, std::wstring_view, FILETIME); void setHandleCloserThreadCount(std::size_t n); -void shrinkFs(); + + +class DirectoryWalker +{ +public: + void forEachEntry( + const std::wstring& path, void* cx, + DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF); + +private: + std::vector> m_buffers; +}; + void forEachEntry( const std::wstring& path, void* cx, diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index 19500167..4e64d1c1 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -211,35 +211,57 @@ public: --OriginConnectionCount; } + std::pair getOrCreate( + const std::wstring &originName, const std::wstring &directory, int priority, + const boost::shared_ptr& fileRegister, + const boost::shared_ptr& originConnection, + DirectoryStats& stats) + { + std::unique_lock lock(m_Mutex); + + auto itor = m_OriginsNameMap.find(originName); + + if (itor == m_OriginsNameMap.end()) { + FilesOrigin& origin = createOriginNoLock( + originName, directory, priority, fileRegister, originConnection); + + return {origin, true}; + } else { + FilesOrigin& origin = m_Origins[itor->second]; + lock.unlock(); + + origin.enable(true, stats); + return {origin, false}; + } + } + FilesOrigin& createOrigin( const std::wstring &originName, const std::wstring &directory, int priority, boost::shared_ptr fileRegister, boost::shared_ptr originConnection) { - int newID = createID(); - - auto itor = m_Origins.insert({newID, FilesOrigin( - newID, originName, directory, priority, - fileRegister, originConnection)}).first; - - m_OriginsNameMap.insert({originName, newID}); - m_OriginsPriorityMap.insert({priority, newID}); + std::scoped_lock lock(m_Mutex); - return itor->second; + return createOriginNoLock( + originName, directory, priority, fileRegister, originConnection); } bool exists(const std::wstring &name) { + std::scoped_lock lock(m_Mutex); return m_OriginsNameMap.find(name) != m_OriginsNameMap.end(); } FilesOrigin &getByID(Index ID) { + std::scoped_lock lock(m_Mutex); return m_Origins[ID]; } const FilesOrigin* findByID(Index ID) const { + std::scoped_lock lock(m_Mutex); + auto itor = m_Origins.find(ID); if (itor == m_Origins.end()) { @@ -251,6 +273,8 @@ public: FilesOrigin &getByName(const std::wstring &name) { + std::scoped_lock lock(m_Mutex); + std::map::iterator iter = m_OriginsNameMap.find(name); if (iter != m_OriginsNameMap.end()) { @@ -264,6 +288,8 @@ public: void changePriorityLookup(int oldPriority, int newPriority) { + std::scoped_lock lock(m_Mutex); + auto iter = m_OriginsPriorityMap.find(oldPriority); if (iter != m_OriginsPriorityMap.end()) { @@ -275,6 +301,8 @@ public: void changeNameLookup(const std::wstring &oldName, const std::wstring &newName) { + std::scoped_lock lock(m_Mutex); + auto iter = m_OriginsNameMap.find(oldName); if (iter != m_OriginsNameMap.end()) { @@ -291,11 +319,29 @@ private: std::map m_Origins; std::map m_OriginsNameMap; std::map m_OriginsPriorityMap; + mutable std::mutex m_Mutex; Index createID() { return m_NextID++; } + + FilesOrigin& createOriginNoLock( + const std::wstring &originName, const std::wstring &directory, int priority, + boost::shared_ptr fileRegister, + boost::shared_ptr originConnection) + { + int newID = createID(); + + auto itor = m_Origins.insert({newID, FilesOrigin( + newID, originName, directory, priority, + fileRegister, originConnection)}).first; + + m_OriginsNameMap.insert({originName, newID}); + m_OriginsPriorityMap.insert({priority, newID}); + + return itor->second; + } }; @@ -898,11 +944,19 @@ void DirectoryEntry::clear() void DirectoryEntry::addFromOrigin( const std::wstring &originName, const std::wstring &directory, int priority, DirectoryStats& stats) +{ + env::DirectoryWalker walker; + addFromOrigin(walker, originName, directory, priority, stats); +} + +void DirectoryEntry::addFromOrigin( + env::DirectoryWalker& walker, const std::wstring &originName, + const std::wstring &directory, int priority, DirectoryStats& stats) { FilesOrigin &origin = createOrigin(originName, directory, priority, stats); if (!directory.empty()) { - addFiles(origin, directory, stats); + addFiles(walker, origin, directory, stats); } m_Populated = true; @@ -993,7 +1047,10 @@ void DirectoryEntry::addFromBSA( void DirectoryEntry::propagateOrigin(int origin) { - m_Origins.insert(origin); + { + std::scoped_lock lock(m_OriginsMutex); + m_Origins.insert(origin); + } if (m_Parent != nullptr) { m_Parent->propagateOrigin(origin); @@ -1252,16 +1309,17 @@ FilesOrigin &DirectoryEntry::createOrigin( const std::wstring &originName, const std::wstring &directory, int priority, DirectoryStats& stats) { - if (m_OriginConnection->exists(originName)) { - ++stats.originExists; - FilesOrigin &origin = m_OriginConnection->getByName(originName); - origin.enable(true, stats); - return origin; - } else { + auto r = m_OriginConnection->getOrCreate( + originName, directory, priority, + m_FileRegister, m_OriginConnection, stats); + + if (r.second) { ++stats.originCreate; - return m_OriginConnection->createOrigin( - originName, directory, priority, m_FileRegister, m_OriginConnection); + } else { + ++stats.originExists; } + + return r.first; } void DirectoryEntry::removeFiles(const std::set &indices) @@ -1359,7 +1417,8 @@ FileEntry::Ptr DirectoryEntry::insert( } void DirectoryEntry::addFiles( - FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats) + env::DirectoryWalker& walker, FilesOrigin &origin, + const std::wstring& path, DirectoryStats& stats) { struct Context { @@ -1371,7 +1430,7 @@ void DirectoryEntry::addFiles( Context cx = {origin, stats}; cx.current.push(this); - env::forEachEntry(path, &cx, + walker.forEachEntry(path, &cx, [](void* pcx, std::wstring_view path) { Context* cx = (Context*)pcx; diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h index 8f6afbb1..2a70d486 100644 --- a/src/shared/directoryentry.h +++ b/src/shared/directoryentry.h @@ -410,6 +410,10 @@ public: const std::wstring &originName, const std::wstring &directory, int priority, DirectoryStats& stats); + void addFromOrigin( + env::DirectoryWalker& walker, const std::wstring &originName, + const std::wstring &directory, int priority, DirectoryStats& stats); + void addFromBSA( const std::wstring &originName, std::wstring &directory, const std::wstring &fileName, int priority, int order); @@ -556,6 +560,7 @@ private: bool m_TopLevel; std::mutex m_SubDirMutex; std::mutex m_FilesMutex; + std::mutex m_OriginsMutex; DirectoryEntry(const DirectoryEntry &reference); @@ -569,7 +574,8 @@ private: std::wstring_view archive, int order, DirectoryStats& stats); void addFiles( - FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats); + env::DirectoryWalker& walker, FilesOrigin &origin, + const std::wstring& path, DirectoryStats& stats); void addFiles( FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, -- cgit v1.3.1