From 763a5d6c08006c319ed92f4088a4d3c211f80cf6 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Thu, 13 Feb 2020 23:03:56 -0500 Subject: more instrumentation tighter mutexes, required thread-safe FileEntry --- src/shared/directoryentry.cpp | 282 +++++++++++++++++++++++------------------- 1 file changed, 156 insertions(+), 126 deletions(-) (limited to 'src/shared/directoryentry.cpp') diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp index e1464595..19500167 100644 --- a/src/shared/directoryentry.cpp +++ b/src/shared/directoryentry.cpp @@ -160,19 +160,23 @@ std::string DirectoryStats::toCsv() const { QStringList oss; + auto s = [](auto ns) { + return ns.count() / 1000.0 / 1000.0 / 1000.0; + }; + oss - << QString::number(dirTimes.count()) - << QString::number(fileTimes.count()) - << QString::number(sortTimes.count()) + << QString::number(s(dirTimes)) + << QString::number(s(fileTimes)) + << QString::number(s(sortTimes)) - << QString::number(subdirLookupTimes.count()) - << QString::number(addDirectoryTimes.count()) + << QString::number(s(subdirLookupTimes)) + << QString::number(s(addDirectoryTimes)) - << QString::number(filesLookupTimes.count()) - << QString::number(addFileTimes.count()) - << QString::number(addOriginToFileTimes.count()) - << QString::number(addFileToOriginTimes.count()) - << QString::number(addFileToRegisterTimes.count()) + << QString::number(s(filesLookupTimes)) + << QString::number(s(addFileTimes)) + << QString::number(s(addOriginToFileTimes)) + << QString::number(s(addFileToOriginTimes)) + << QString::number(s(addFileToRegisterTimes)) << QString::number(originExists) << QString::number(originCreate) @@ -323,6 +327,8 @@ FileEntry::~FileEntry() void FileEntry::addOrigin( int origin, FILETIME fileTime, std::wstring_view archive, int order) { + std::scoped_lock lock(m_OriginsMutex); + m_LastAccessed = time(nullptr); if (m_Parent != nullptr) { m_Parent->propagateOrigin(origin); @@ -387,6 +393,8 @@ void FileEntry::addOrigin( bool FileEntry::removeOrigin(int origin) { + std::scoped_lock lock(m_OriginsMutex); + if (m_Origin == origin) { if (!m_Alternatives.empty()) { // find alternative with the highest priority @@ -440,6 +448,8 @@ bool FileEntry::removeOrigin(int origin) void FileEntry::sortOrigins() { + std::scoped_lock lock(m_OriginsMutex); + m_Alternatives.push_back({m_Origin, m_Archive}); std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) { @@ -480,6 +490,8 @@ void FileEntry::sortOrigins() bool FileEntry::isFromArchive(std::wstring archiveName) const { + std::scoped_lock lock(m_OriginsMutex); + if (archiveName.length() == 0) { return m_Archive.first.length() != 0; } @@ -499,6 +511,8 @@ bool FileEntry::isFromArchive(std::wstring archiveName) const std::wstring FileEntry::getFullPath(int originID) const { + std::scoped_lock lock(m_OriginsMutex); + if (originID == -1) { bool ignore = false; originID = getOrigin(ignore); @@ -690,27 +704,34 @@ bool FileRegister::indexValid(FileEntry::Index index) const FileEntry::Ptr FileRegister::createFile( std::wstring name, DirectoryEntry *parent, DirectoryStats& stats) { - FileEntry::Index index = generateIndex(); + const auto index = generateIndex(); FileEntry::Ptr p; stats.addFileToRegisterTimes += elapsed([&]{ - std::scoped_lock lock(m_Mutex); + bool inserted = false; + p = FileEntry::Ptr(new FileEntry(index, std::move(name), parent)); - auto r = m_Files.insert_or_assign( - index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent))); + { + std::scoped_lock lock(m_Mutex); + inserted = m_Files.insert_or_assign(index, p).second; + } - if (r.second) { + if (inserted) { ++stats.filesInsertedInRegister; } else { ++stats.filesAssignedInRegister; } - - p = r.first->second; }); return p; } +FileEntry::Index FileRegister::generateIndex() +{ + static std::atomic sIndex(0); + return sIndex++; +} + FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const { std::scoped_lock lock(m_Mutex); @@ -815,12 +836,6 @@ void FileRegister::sortOrigins() } } -FileEntry::Index FileRegister::generateIndex() -{ - static std::atomic sIndex(0); - return sIndex++; -} - void FileRegister::unregisterFile(FileEntry::Ptr file) { bool ignore; @@ -881,17 +896,13 @@ void DirectoryEntry::clear() } void DirectoryEntry::addFromOrigin( - const std::wstring &originName, const std::wstring &directory, int priority) + const std::wstring &originName, const std::wstring &directory, int priority, + DirectoryStats& stats) { - DirectoryStats dummy; - FilesOrigin &origin = createOrigin(originName, directory, priority, dummy); + FilesOrigin &origin = createOrigin(originName, directory, priority, stats); - if (directory.length() != 0) { - boost::scoped_array buffer(new wchar_t[MAXPATH_UNICODE + 1]); - memset(buffer.get(), L'\0', MAXPATH_UNICODE + 1); - int offset = _snwprintf(buffer.get(), MAXPATH_UNICODE, L"%ls", directory.c_str()); - buffer.get()[offset] = L'\0'; - addFiles(origin, buffer.get(), offset); + if (!directory.empty()) { + addFiles(origin, directory, stats); } m_Populated = true; @@ -1260,58 +1271,80 @@ void DirectoryEntry::removeFiles(const std::set &indices) FileEntry::Ptr DirectoryEntry::insert( std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime, - std::wstring_view archive, int order) + std::wstring_view archive, int order, DirectoryStats& stats) { - std::scoped_lock lock(m_FilesMutex); - std::wstring fileNameLower = ToLowerCopy(fileName); + FileEntry::Ptr fe; - auto iter = m_Files.find(fileNameLower); - FileEntry::Ptr file; + FileKey key(std::move(fileNameLower)); - if (iter != m_Files.end()) { - file = m_FileRegister->getFile(iter->second); - } else { - DirectoryStats dummy; + { + std::unique_lock lock(m_FilesMutex); + + FilesLookup::iterator itor; + + stats.filesLookupTimes += elapsed([&]{ + itor = m_FilesLookup.find(key); + }); + + if (itor != m_FilesLookup.end()) { + lock.unlock(); + ++stats.fileExists; + fe = m_FileRegister->getFile(itor->second); + } else { + ++stats.fileCreate; + fe = m_FileRegister->createFile( + std::wstring(fileName.begin(), fileName.end()), this, stats); - file = m_FileRegister->createFile( - std::wstring(fileName.begin(), fileName.end()), this, dummy); + stats.addFileTimes += elapsed([&] { + addFileToList(std::move(key.value), fe->getIndex()); + }); - addFileToList(std::move(fileNameLower), file->getIndex()); - // fileNameLower has moved from this point + // fileNameLower has moved from this point + } } - file->addOrigin(origin.getID(), fileTime, archive, order); - origin.addFile(file->getIndex()); + stats.addOriginToFileTimes += elapsed([&]{ + fe->addOrigin(origin.getID(), fileTime, archive, order); + }); + + stats.addFileToOriginTimes += elapsed([&]{ + origin.addFile(fe->getIndex()); + }); - return file; + return fe; } FileEntry::Ptr DirectoryEntry::insert( env::File& file, FilesOrigin &origin, std::wstring_view archive, int order, DirectoryStats& stats) { - std::scoped_lock lock(m_FilesMutex); + FileEntry::Ptr fe; - FilesMap::iterator itor; + { + std::unique_lock lock(m_FilesMutex); - stats.filesLookupTimes += elapsed([&]{ - itor = m_Files.find(file.lcname); - }); + FilesMap::iterator itor; - FileEntry::Ptr fe; + stats.filesLookupTimes += elapsed([&]{ + itor = m_Files.find(file.lcname); + }); - if (itor != m_Files.end()) { - ++stats.fileExists; - fe = m_FileRegister->getFile(itor->second); - } else { - fe = m_FileRegister->createFile(std::move(file.name), this, stats); + if (itor != m_Files.end()) { + lock.unlock(); + ++stats.fileExists; + fe = m_FileRegister->getFile(itor->second); + } else { + ++stats.fileCreate; + fe = m_FileRegister->createFile(std::move(file.name), this, stats); + // file.name has been moved from this point - stats.addFileTimes += elapsed([&]{ - addFileToList(std::move(file.lcname), fe->getIndex()); - }); + stats.addFileTimes += elapsed([&]{ + addFileToList(std::move(file.lcname), fe->getIndex()); + }); - // both file.name and file.lcname have been moved from this point + // file.lcname has been moved from this point + } } stats.addOriginToFileTimes += elapsed([&]{ @@ -1325,91 +1358,75 @@ FileEntry::Ptr DirectoryEntry::insert( return fe; } -void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOffset) +void DirectoryEntry::addFiles( + FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats) { struct Context { FilesOrigin& origin; + DirectoryStats& stats; std::stack current; }; - Context cx = {origin}; + Context cx = {origin, stats}; cx.current.push(this); - env::forEachEntry(buffer, &cx, - [](void* pcx, std::wstring_view path) { - Context* cx = (Context*)pcx; - cx->current.push(cx->current.top()->getSubDirectory(path, true, cx->origin.getID())); - }, - - [](void* pcx, std::wstring_view path) { + env::forEachEntry(path, &cx, + [](void* pcx, std::wstring_view path) + { Context* cx = (Context*)pcx; - auto* current= cx->current.top(); + cx->stats.dirTimes += elapsed([&] { + auto* sd = cx->current.top()->getSubDirectory( + path, true, cx->stats, cx->origin.getID()); - { - std::scoped_lock lock(current->m_SubDirMutex); - std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName); - } - - cx->current.pop(); + cx->current.push(sd); + }); }, - [](void* pcx, std::wstring_view path, FILETIME ft) { + [](void* pcx, std::wstring_view path) + { Context* cx = (Context*)pcx; - cx->current.top()->insert(path, cx->origin, ft, L"", -1); - } - ); - - /* - WIN32_FIND_DATAW findData; - _snwprintf_s(buffer + bufferOffset, MAXPATH_UNICODE - bufferOffset, _TRUNCATE, L"\\*"); + cx->stats.dirTimes += elapsed([&] { + auto* current= cx->current.top(); - HANDLE searchHandle = nullptr; - - if (SupportOptimizedFind()) { - searchHandle = ::FindFirstFileExW( - buffer, FindExInfoBasic, &findData, FindExSearchNameMatch, nullptr, - FIND_FIRST_EX_LARGE_FETCH); - } else { - searchHandle = ::FindFirstFileExW( - buffer, FindExInfoStandard, &findData, FindExSearchNameMatch, nullptr, 0); - } + { + std::scoped_lock lock(current->m_SubDirMutex); - if (searchHandle != INVALID_HANDLE_VALUE) { - BOOL result = true; + std::sort( + current->m_SubDirectories.begin(), + current->m_SubDirectories.end(), + &DirCompareByName); + } - while (result) { - if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - if ((wcscmp(findData.cFileName, L".") != 0) && - (wcscmp(findData.cFileName, L"..") != 0)) { - int offset = _snwprintf(buffer + bufferOffset, MAXPATH_UNICODE, L"\\%ls", findData.cFileName); + cx->current.pop(); + }); + }, - // recurse into subdirectories - DirectoryEntry* sd = getSubDirectory(findData.cFileName, true, origin.getID()); - sd->addFiles(origin, buffer, bufferOffset + offset); - } - } else { - insert(findData.cFileName, origin, findData.ftLastWriteTime, L"", -1); - } + [](void* pcx, std::wstring_view path, FILETIME ft) + { + Context* cx = (Context*)pcx; - result = ::FindNextFileW(searchHandle, &findData); + cx->stats.fileTimes += elapsed([&]{ + cx->current.top()->insert(path, cx->origin, ft, L"", -1, cx->stats); + }); } - } - - std::sort(m_SubDirectories.begin(), m_SubDirectories.end(), &DirCompareByName); - ::FindClose(searchHandle);*/ + ); } void DirectoryEntry::addFiles( FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime, const std::wstring &archiveName, int order) { + DirectoryStats dummy; + // add files for (unsigned int fileIdx = 0; fileIdx < archiveFolder->getNumFiles(); ++fileIdx) { BSA::File::Ptr file = archiveFolder->getFile(fileIdx); - auto f = insert(ToWString(file->getName(), true), origin, fileTime, archiveName, order); + auto f = insert( + ToWString(file->getName(), true), origin, fileTime, + archiveName, order, dummy); if (f) { if (file->getUncompressedFileSize() > 0) { @@ -1423,31 +1440,41 @@ void DirectoryEntry::addFiles( // recurse into subdirectories for (unsigned int folderIdx = 0; folderIdx < archiveFolder->getNumSubFolders(); ++folderIdx) { BSA::Folder::Ptr folder = archiveFolder->getSubFolder(folderIdx); - DirectoryEntry *folderEntry = getSubDirectoryRecursive(ToWString(folder->getName(), true), true, origin.getID()); + DirectoryEntry *folderEntry = getSubDirectoryRecursive( + ToWString(folder->getName(), true), true, origin.getID()); folderEntry->addFiles(origin, folder, fileTime, archiveName, order); } } DirectoryEntry *DirectoryEntry::getSubDirectory( - std::wstring_view name, bool create, int originID) + std::wstring_view name, bool create, DirectoryStats& stats, int originID) { + std::wstring nameLc = ToLowerCopy(name); + std::scoped_lock lock(m_SubDirMutex); - std::wstring nameLc = ToLowerCopy(name); - auto itor = m_SubDirectoriesLookup.find(nameLc); + SubDirectoriesLookup::iterator itor; + stats.subdirLookupTimes += elapsed([&] { + itor = m_SubDirectoriesLookup.find(nameLc); + }); if (itor != m_SubDirectoriesLookup.end()) { + ++stats.subdirExists; return itor->second; } if (create) { + ++stats.subdirCreate; + auto* entry = new DirectoryEntry( std::wstring(name.begin(), name.end()), this, originID, m_FileRegister, m_OriginConnection); - addDirectoryToList(entry, std::move(nameLc)); - // nameLc is moved from this point + stats.addDirectoryTimes += elapsed([&] { + addDirectoryToList(entry, std::move(nameLc)); + // nameLc is moved from this point + }); return entry; } else { @@ -1498,16 +1525,19 @@ DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive( } const size_t pos = path.find_first_of(L"\\/"); + DirectoryStats dummy; if (pos == std::wstring::npos) { - return getSubDirectory(path, create); + return getSubDirectory(path, create, dummy); } else { - DirectoryEntry *nextChild = getSubDirectory(path.substr(0, pos), create, originID); + DirectoryEntry *nextChild = getSubDirectory( + path.substr(0, pos), create, dummy, originID); if (nextChild == nullptr) { return nullptr; } else { - return nextChild->getSubDirectoryRecursive(path.substr(pos + 1), create, originID); + return nextChild->getSubDirectoryRecursive( + path.substr(pos + 1), create, originID); } } } -- cgit v1.3.1