more instrumentation

tighter mutexes, required thread-safe FileEntry

5 files changed, 241 insertions, 173 deletions

diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp
index bce3f65a..20d6a52b 100644
--- a/src/directoryrefresher.cpp
+++ b/src/directoryrefresher.cpp
@@ -160,12 +160,14 @@ void DirectoryRefresher::addModFilesToStructure(
   TimeThis tt("addModFilesToStructure()");

 

   std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory));

+  DirectoryStats dummy;

 

   if (stealFiles.length() > 0) {

     stealModFilesIntoStructure(

       directoryStructure, modName, priority, directory, stealFiles);

   } else {

-    directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority);

+    directoryStructure->addFromOrigin(

+      ToWString(modName), directoryW, priority, dummy);

   }

 }

 

@@ -178,16 +180,20 @@ void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure
 {

   TimeThis tt("addModToStructure()");

 

+  DirectoryStats dummy;

+

   if (stealFiles.length() > 0) {

     stealModFilesIntoStructure(

       directoryStructure, modName, priority, directory, stealFiles);

   } else {

     std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory));

-    directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority);

+    directoryStructure->addFromOrigin(

+      ToWString(modName), directoryW, priority, dummy);

   }

 

   if (Settings::instance().archiveParsing()) {

-    addModBSAToStructure(directoryStructure, modName, priority, directory, archives);

+    addModBSAToStructure(

+      directoryStructure, modName, priority, directory, archives);

   }

 }

 

@@ -198,6 +204,7 @@ struct ModThread
   std::wstring path;

   int prio = -1;

   env::Directory* dir = nullptr;

+  DirectoryStats* stats =  nullptr;

 

   std::condition_variable cv;

   std::mutex mutex;

@@ -214,7 +221,8 @@ struct ModThread
     std::unique_lock lock(mutex);

     cv.wait(lock, [&]{ return ready; });

 

-    ds->addFromOrigin(modName, path, prio);

+    SetThisThreadName(QString::fromStdWString(modName + L" refresher"));

+    ds->addFromOrigin(modName, path, prio, *stats);

 

     /*if (Settings::instance().archiveParsing()) {

       addModBSAToStructure(

@@ -229,11 +237,39 @@ struct ModThread
   }

 };

 

+void dumpStats(std::vector<DirectoryStats>& stats)

+{

+  static int run = 0;

+  static const std::string file("c:\\tmp\\data.csv");

+

+  if (run == 0) {

+    std::ofstream out(file, std::ios::out|std::ios::trunc);

+    out << fmt::format("what,run,{}", DirectoryStats::csvHeader()) << "\n";

+  }

+

+  std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){

+    return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0);

+  });

+

+  std::ofstream out(file, std::ios::app);

+

+  DirectoryStats total;

+  for (const auto& s : stats) {

+    out << fmt::format("{},{},{}", s.mod, run, s.toCsv()) << "\n";

+    total += s;

+  }

+

+  out << fmt::format("total,{},{}", run, total.toCsv()) << "\n";

+

+  ++run;

+}

+

 void DirectoryRefresher::addMultipleModsFilesToStructure(

   MOShared::DirectoryEntry *directoryStructure,

   const std::vector<EntryInfo>& entries, bool emitProgress)

 {

   std::vector<env::Directory> dirs(entries.size());

+  std::vector<DirectoryStats> stats(entries.size());

 

   {

     TimeThis tt("walk dirs");

@@ -257,6 +293,9 @@ void DirectoryRefresher::addMultipleModsFilesToStructure(
           mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString();

           mt.prio = prio;

           mt.dir = &dirs[i];

+          mt.stats = &stats[i];

+

+          stats[i].mod = entries[i].modName.toStdString();

 

           mt.wakeup();

         }

@@ -272,23 +311,7 @@ void DirectoryRefresher::addMultipleModsFilesToStructure(
     threads.join();

   }

 

-  //std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){

-  //  return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0);

-  //});

-

-  //static int run = 1;

-  //

-  //std::ofstream out("c:\\tmp\\data.csv", std::ios::app);

-

-  //out << fmt::format("what,run,{}", DirectoryStats::csvHeader());

-  //

-  //for (std::size_t i=0; i<entries.size(); ++i) {

-  //  out << fmt::format("{},{},{}", stats[i].mod, run, stats[i].toCsv()) << "\n";

-  //}

-  //

-  //out << fmt::format("total,{},{}", run, total.toCsv()) << "\n";

-  //

-  //++run;

+  dumpStats(stats);

 }

 

 namespace MOShared{ void logcounts(std::string w); }

@@ -297,34 +320,39 @@ void DirectoryRefresher::refresh()
 {

   SetThisThreadName("DirectoryRefresher");

 

-  QMutexLocker locker(&m_RefreshLock);

+  for (int i=0; i<1; ++i) {

+    QMutexLocker locker(&m_RefreshLock);

 

-  //logcounts("before delete");

-  delete m_DirectoryStructure;

-  //logcounts("after delete");

+    //logcounts("before delete");

+    delete m_DirectoryStructure;

+    //logcounts("after delete");

 

-  m_DirectoryStructure = new DirectoryEntry(L"data", nullptr, 0);

+    m_DirectoryStructure = new DirectoryEntry(L"data", nullptr, 0);

 

-  IPluginGame *game = qApp->property("managed_game").value<IPluginGame*>();

+    IPluginGame *game = qApp->property("managed_game").value<IPluginGame*>();

 

-  std::wstring dataDirectory =

-    QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString();

+    std::wstring dataDirectory =

+      QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString();

 

-  m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0);

+    {

+      DirectoryStats dummy;

+      m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0, dummy);

+    }

 

-  std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) {

-    return lhs.priority < rhs.priority;

-  });

+    std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) {

+      return lhs.priority < rhs.priority;

+    });

 

-  addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true);

+    addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true);

 

-  m_DirectoryStructure->getFileRegister()->sortOrigins();

+    m_DirectoryStructure->getFileRegister()->sortOrigins();

 

-  emit progress(100);

+    emit progress(100);

 

-  cleanStructure(m_DirectoryStructure);

+    cleanStructure(m_DirectoryStructure);

 

-  emit refreshed();

+    emit refreshed();

 

-  //logcounts("after refresh");

+    //logcounts("after refresh");

+  }

 }

diff --git a/src/envfs.cpp b/src/envfs.cpp
index 022ca513..5cd36957 100644
--- a/src/envfs.cpp
+++ b/src/envfs.cpp
@@ -179,6 +179,8 @@ public:
 
   void run()
   {
+    MOShared::SetThisThreadName("HandleCloserThread");
+
     std::unique_lock lock(m_mutex);
     m_cv.wait(lock, [&]{ return m_ready; });
 
diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp
index 3e03c52d..2145011f 100644
--- a/src/mainwindow.cpp
+++ b/src/mainwindow.cpp
@@ -5148,7 +5148,11 @@ void MainWindow::originModified(int originID)
 {
   FilesOrigin &origin = m_OrganizerCore.directoryStructure()->getOriginByID(originID);
   origin.enable(false);
-  m_OrganizerCore.directoryStructure()->addFromOrigin(origin.getName(), origin.getPath(), origin.getPriority());
+
+  DirectoryStats dummy;
+  m_OrganizerCore.directoryStructure()->addFromOrigin(
+    origin.getName(), origin.getPath(), origin.getPriority(), dummy);
+
   DirectoryRefresher::cleanStructure(m_OrganizerCore.directoryStructure());
 }
 
diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp
index e1464595..19500167 100644
--- a/src/shared/directoryentry.cpp
+++ b/src/shared/directoryentry.cpp
@@ -160,19 +160,23 @@ std::string DirectoryStats::toCsv() const
 {

   QStringList oss;

 

+  auto s = [](auto ns) {

+    return ns.count() / 1000.0 / 1000.0 / 1000.0;

+  };

+

   oss

-    << QString::number(dirTimes.count())

-    << QString::number(fileTimes.count())

-    << QString::number(sortTimes.count())

+    << QString::number(s(dirTimes))

+    << QString::number(s(fileTimes))

+    << QString::number(s(sortTimes))

 

-    << QString::number(subdirLookupTimes.count())

-    << QString::number(addDirectoryTimes.count())

+    << QString::number(s(subdirLookupTimes))

+    << QString::number(s(addDirectoryTimes))

 

-    << QString::number(filesLookupTimes.count())

-    << QString::number(addFileTimes.count())

-    << QString::number(addOriginToFileTimes.count())

-    << QString::number(addFileToOriginTimes.count())

-    << QString::number(addFileToRegisterTimes.count())

+    << QString::number(s(filesLookupTimes))

+    << QString::number(s(addFileTimes))

+    << QString::number(s(addOriginToFileTimes))

+    << QString::number(s(addFileToOriginTimes))

+    << QString::number(s(addFileToRegisterTimes))

 

     << QString::number(originExists)

     << QString::number(originCreate)

@@ -323,6 +327,8 @@ FileEntry::~FileEntry()
 void FileEntry::addOrigin(

   int origin, FILETIME fileTime, std::wstring_view archive, int order)

 {

+  std::scoped_lock lock(m_OriginsMutex);

+

   m_LastAccessed = time(nullptr);

   if (m_Parent != nullptr) {

     m_Parent->propagateOrigin(origin);

@@ -387,6 +393,8 @@ void FileEntry::addOrigin(
 

 bool FileEntry::removeOrigin(int origin)

 {

+  std::scoped_lock lock(m_OriginsMutex);

+

   if (m_Origin == origin) {

     if (!m_Alternatives.empty()) {

       // find alternative with the highest priority

@@ -440,6 +448,8 @@ bool FileEntry::removeOrigin(int origin)
 

 void FileEntry::sortOrigins()

 {

+  std::scoped_lock lock(m_OriginsMutex);

+

   m_Alternatives.push_back({m_Origin, m_Archive});

 

   std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) {

@@ -480,6 +490,8 @@ void FileEntry::sortOrigins()
 

 bool FileEntry::isFromArchive(std::wstring archiveName) const

 {

+  std::scoped_lock lock(m_OriginsMutex);

+

   if (archiveName.length() == 0) {

     return m_Archive.first.length() != 0;

   }

@@ -499,6 +511,8 @@ bool FileEntry::isFromArchive(std::wstring archiveName) const
 

 std::wstring FileEntry::getFullPath(int originID) const

 {

+  std::scoped_lock lock(m_OriginsMutex);

+

   if (originID == -1) {

     bool ignore = false;

     originID = getOrigin(ignore);

@@ -690,27 +704,34 @@ bool FileRegister::indexValid(FileEntry::Index index) const
 FileEntry::Ptr FileRegister::createFile(

   std::wstring name, DirectoryEntry *parent, DirectoryStats& stats)

 {

-  FileEntry::Index index = generateIndex();

+  const auto index = generateIndex();

   FileEntry::Ptr p;

 

   stats.addFileToRegisterTimes += elapsed([&]{

-    std::scoped_lock lock(m_Mutex);

+    bool inserted = false;

+    p = FileEntry::Ptr(new FileEntry(index, std::move(name), parent));

 

-    auto r = m_Files.insert_or_assign(

-      index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent)));

+    {

+      std::scoped_lock lock(m_Mutex);

+      inserted = m_Files.insert_or_assign(index, p).second;

+    }

 

-    if (r.second) {

+    if (inserted) {

       ++stats.filesInsertedInRegister;

     } else {

       ++stats.filesAssignedInRegister;

     }

-

-    p = r.first->second;

   });

 

   return p;

 }

 

+FileEntry::Index FileRegister::generateIndex()

+{

+  static std::atomic<FileEntry::Index> sIndex(0);

+  return sIndex++;

+}

+

 FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const

 {

   std::scoped_lock lock(m_Mutex);

@@ -815,12 +836,6 @@ void FileRegister::sortOrigins()
   }

 }

 

-FileEntry::Index FileRegister::generateIndex()

-{

-  static std::atomic<FileEntry::Index> sIndex(0);

-  return sIndex++;

-}

-

 void FileRegister::unregisterFile(FileEntry::Ptr file)

 {

   bool ignore;

@@ -881,17 +896,13 @@ void DirectoryEntry::clear()
 }

 

 void DirectoryEntry::addFromOrigin(

-  const std::wstring &originName, const std::wstring &directory, int priority)

+  const std::wstring &originName, const std::wstring &directory, int priority,

+  DirectoryStats& stats)

 {

-  DirectoryStats dummy;

-  FilesOrigin &origin = createOrigin(originName, directory, priority, dummy);

+  FilesOrigin &origin = createOrigin(originName, directory, priority, stats);

 

-  if (directory.length() != 0) {

-    boost::scoped_array<wchar_t> buffer(new wchar_t[MAXPATH_UNICODE + 1]);

-    memset(buffer.get(), L'\0', MAXPATH_UNICODE + 1);

-    int offset = _snwprintf(buffer.get(), MAXPATH_UNICODE, L"%ls", directory.c_str());

-    buffer.get()[offset] = L'\0';

-    addFiles(origin, buffer.get(), offset);

+  if (!directory.empty()) {

+    addFiles(origin, directory, stats);

   }

 

   m_Populated = true;

@@ -1260,58 +1271,80 @@ void DirectoryEntry::removeFiles(const std::set<FileEntry::Index> &indices)
 

 FileEntry::Ptr DirectoryEntry::insert(

   std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime,

-  std::wstring_view archive, int order)

+  std::wstring_view archive, int order, DirectoryStats& stats)

 {

-  std::scoped_lock lock(m_FilesMutex);

-

   std::wstring fileNameLower = ToLowerCopy(fileName);

+  FileEntry::Ptr fe;

 

-  auto iter = m_Files.find(fileNameLower);

-  FileEntry::Ptr file;

+  FileKey key(std::move(fileNameLower));

 

-  if (iter != m_Files.end()) {

-    file = m_FileRegister->getFile(iter->second);

-  } else {

-    DirectoryStats dummy;

+  {

+    std::unique_lock lock(m_FilesMutex);

+

+    FilesLookup::iterator itor;

+

+    stats.filesLookupTimes += elapsed([&]{

+      itor = m_FilesLookup.find(key);

+    });

+

+    if (itor != m_FilesLookup.end()) {

+      lock.unlock();

+      ++stats.fileExists;

+      fe = m_FileRegister->getFile(itor->second);

+    } else {

+      ++stats.fileCreate;

+      fe = m_FileRegister->createFile(

+        std::wstring(fileName.begin(), fileName.end()), this, stats);

 

-    file = m_FileRegister->createFile(

-      std::wstring(fileName.begin(), fileName.end()), this, dummy);

+      stats.addFileTimes += elapsed([&] {

+        addFileToList(std::move(key.value), fe->getIndex());

+      });

 

-    addFileToList(std::move(fileNameLower), file->getIndex());

-    // fileNameLower has moved from this point

+      // fileNameLower has moved from this point

+    }

   }

 

-  file->addOrigin(origin.getID(), fileTime, archive, order);

-  origin.addFile(file->getIndex());

+  stats.addOriginToFileTimes += elapsed([&]{

+    fe->addOrigin(origin.getID(), fileTime, archive, order);

+  });

+

+  stats.addFileToOriginTimes += elapsed([&]{

+    origin.addFile(fe->getIndex());

+  });

 

-  return file;

+  return fe;

 }

 

 FileEntry::Ptr DirectoryEntry::insert(

   env::File& file, FilesOrigin &origin, std::wstring_view archive, int order,

   DirectoryStats& stats)

 {

-  std::scoped_lock lock(m_FilesMutex);

+  FileEntry::Ptr fe;

 

-  FilesMap::iterator itor;

+  {

+    std::unique_lock lock(m_FilesMutex);

 

-  stats.filesLookupTimes += elapsed([&]{

-    itor = m_Files.find(file.lcname);

-  });

+    FilesMap::iterator itor;

 

-  FileEntry::Ptr fe;

+    stats.filesLookupTimes += elapsed([&]{

+      itor = m_Files.find(file.lcname);

+    });

 

-  if (itor != m_Files.end()) {

-    ++stats.fileExists;

-    fe = m_FileRegister->getFile(itor->second);

-  } else {

-    fe = m_FileRegister->createFile(std::move(file.name), this, stats);

+    if (itor != m_Files.end()) {

+      lock.unlock();

+      ++stats.fileExists;

+      fe = m_FileRegister->getFile(itor->second);

+    } else {

+      ++stats.fileCreate;

+      fe = m_FileRegister->createFile(std::move(file.name), this, stats);

+      // file.name has been moved from this point

 

-    stats.addFileTimes += elapsed([&]{

-      addFileToList(std::move(file.lcname), fe->getIndex());

-    });

+      stats.addFileTimes += elapsed([&]{

+        addFileToList(std::move(file.lcname), fe->getIndex());

+      });

 

-    // both file.name and file.lcname have been moved from this point

+      // file.lcname has been moved from this point

+    }

   }

 

   stats.addOriginToFileTimes += elapsed([&]{

@@ -1325,91 +1358,75 @@ FileEntry::Ptr DirectoryEntry::insert(
   return fe;

 }

 

-void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOffset)

+void DirectoryEntry::addFiles(

+  FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats)

 {

   struct Context

   {

     FilesOrigin& origin;

+    DirectoryStats& stats;

     std::stack<DirectoryEntry*> current;

   };

 

-  Context cx = {origin};

+  Context cx = {origin, stats};

   cx.current.push(this);

 

-  env::forEachEntry(buffer, &cx,

-    [](void* pcx, std::wstring_view path) {

-      Context* cx = (Context*)pcx;

-      cx->current.push(cx->current.top()->getSubDirectory(path, true, cx->origin.getID()));

-    },

-

-    [](void* pcx, std::wstring_view path) {

+  env::forEachEntry(path, &cx,

+    [](void* pcx, std::wstring_view path)

+    {

       Context* cx = (Context*)pcx;

-      auto* current= cx->current.top();

+      cx->stats.dirTimes += elapsed([&] {

+        auto* sd = cx->current.top()->getSubDirectory(

+          path, true, cx->stats, cx->origin.getID());

 

-      {

-        std::scoped_lock lock(current->m_SubDirMutex);

-        std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName);

-      }

-

-      cx->current.pop();

+        cx->current.push(sd);

+      });

     },

 

-    [](void* pcx, std::wstring_view path, FILETIME ft) {

+    [](void* pcx, std::wstring_view path)

+    {

       Context* cx = (Context*)pcx;

-      cx->current.top()->insert(path, cx->origin, ft, L"", -1);

-    }

-  );

-

-  /*

-  WIN32_FIND_DATAW findData;

 

-  _snwprintf_s(buffer + bufferOffset, MAXPATH_UNICODE - bufferOffset, _TRUNCATE, L"\\*");

+      cx->stats.dirTimes += elapsed([&] {

+        auto* current= cx->current.top();

 

-  HANDLE searchHandle = nullptr;

-

-  if (SupportOptimizedFind()) {

-    searchHandle = ::FindFirstFileExW(

-      buffer, FindExInfoBasic, &findData, FindExSearchNameMatch, nullptr,

-      FIND_FIRST_EX_LARGE_FETCH);

-  } else {

-    searchHandle = ::FindFirstFileExW(

-      buffer, FindExInfoStandard, &findData, FindExSearchNameMatch, nullptr, 0);

-  }

+        {

+          std::scoped_lock lock(current->m_SubDirMutex);

 

-  if (searchHandle != INVALID_HANDLE_VALUE) {

-    BOOL result = true;

+          std::sort(

+            current->m_SubDirectories.begin(),

+            current->m_SubDirectories.end(),

+            &DirCompareByName);

+        }

 

-    while (result) {

-      if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {

-        if ((wcscmp(findData.cFileName, L".") != 0) &&

-            (wcscmp(findData.cFileName, L"..") != 0)) {

-          int offset = _snwprintf(buffer + bufferOffset, MAXPATH_UNICODE, L"\\%ls", findData.cFileName);

+        cx->current.pop();

+      });

+    },

 

-          // recurse into subdirectories

-          DirectoryEntry* sd = getSubDirectory(findData.cFileName, true, origin.getID());

-          sd->addFiles(origin, buffer, bufferOffset + offset);

-        }

-      } else {

-        insert(findData.cFileName, origin, findData.ftLastWriteTime, L"", -1);

-      }

+    [](void* pcx, std::wstring_view path, FILETIME ft)

+    {

+      Context* cx = (Context*)pcx;

 

-      result = ::FindNextFileW(searchHandle, &findData);

+      cx->stats.fileTimes += elapsed([&]{

+        cx->current.top()->insert(path, cx->origin, ft, L"", -1, cx->stats);

+      });

     }

-  }

-

-  std::sort(m_SubDirectories.begin(), m_SubDirectories.end(), &DirCompareByName);

-  ::FindClose(searchHandle);*/

+  );

 }

 

 void DirectoryEntry::addFiles(

   FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime,

   const std::wstring &archiveName, int order)

 {

+  DirectoryStats dummy;

+

   // add files

   for (unsigned int fileIdx = 0; fileIdx < archiveFolder->getNumFiles(); ++fileIdx) {

     BSA::File::Ptr file = archiveFolder->getFile(fileIdx);

 

-    auto f = insert(ToWString(file->getName(), true), origin, fileTime, archiveName, order);

+    auto f = insert(

+      ToWString(file->getName(), true), origin, fileTime,

+      archiveName, order, dummy);

 

     if (f) {

       if (file->getUncompressedFileSize() > 0) {

@@ -1423,31 +1440,41 @@ void DirectoryEntry::addFiles(
   // recurse into subdirectories

   for (unsigned int folderIdx = 0; folderIdx < archiveFolder->getNumSubFolders(); ++folderIdx) {

     BSA::Folder::Ptr folder = archiveFolder->getSubFolder(folderIdx);

-    DirectoryEntry *folderEntry = getSubDirectoryRecursive(ToWString(folder->getName(), true), true, origin.getID());

+    DirectoryEntry *folderEntry = getSubDirectoryRecursive(

+      ToWString(folder->getName(), true), true, origin.getID());

 

     folderEntry->addFiles(origin, folder, fileTime, archiveName, order);

   }

 }

 

 DirectoryEntry *DirectoryEntry::getSubDirectory(

-  std::wstring_view name, bool create, int originID)

+  std::wstring_view name, bool create, DirectoryStats& stats, int originID)

 {

+  std::wstring nameLc = ToLowerCopy(name);

+

   std::scoped_lock lock(m_SubDirMutex);

 

-  std::wstring nameLc = ToLowerCopy(name);

-  auto itor = m_SubDirectoriesLookup.find(nameLc);

+  SubDirectoriesLookup::iterator itor;

+  stats.subdirLookupTimes += elapsed([&] {

+    itor = m_SubDirectoriesLookup.find(nameLc);

+  });

 

   if (itor != m_SubDirectoriesLookup.end()) {

+    ++stats.subdirExists;

     return itor->second;

   }

 

   if (create) {

+    ++stats.subdirCreate;

+

     auto* entry = new DirectoryEntry(

       std::wstring(name.begin(), name.end()), this, originID,

       m_FileRegister, m_OriginConnection);

 

-    addDirectoryToList(entry, std::move(nameLc));

-    // nameLc is moved from this point

+    stats.addDirectoryTimes += elapsed([&] {

+      addDirectoryToList(entry, std::move(nameLc));

+      // nameLc is moved from this point

+    });

 

     return entry;

   } else {

@@ -1498,16 +1525,19 @@ DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive(
   }

 

   const size_t pos = path.find_first_of(L"\\/");

+  DirectoryStats dummy;

 

   if (pos == std::wstring::npos) {

-    return getSubDirectory(path, create);

+    return getSubDirectory(path, create, dummy);

   } else {

-    DirectoryEntry *nextChild = getSubDirectory(path.substr(0, pos), create, originID);

+    DirectoryEntry *nextChild = getSubDirectory(

+      path.substr(0, pos), create, dummy, originID);

 

     if (nextChild == nullptr) {

       return nullptr;

     } else {

-      return nextChild->getSubDirectoryRecursive(path.substr(pos + 1), create, originID);

+      return nextChild->getSubDirectoryRecursive(

+        path.substr(pos + 1), create, originID);

     }

   }

 }

diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h
index 71e8b32c..8f6afbb1 100644
--- a/src/shared/directoryentry.h
+++ b/src/shared/directoryentry.h
@@ -221,6 +221,7 @@ private:
   DirectoryEntry *m_Parent;

   mutable FILETIME m_FileTime;

   uint64_t m_FileSize, m_CompressedFileSize;

+  mutable std::mutex m_OriginsMutex;

 

   time_t m_LastAccessed;

 

@@ -328,12 +329,14 @@ public:
   void sortOrigins();

 

 private:

+  using FileMap = std::map<FileEntry::Index, FileEntry::Ptr>;

+

   mutable std::mutex m_Mutex;

-  std::map<FileEntry::Index, FileEntry::Ptr> m_Files;

+  FileMap m_Files;

   boost::shared_ptr<OriginConnection> m_OriginConnection;

 

-  FileEntry::Index generateIndex();

   void unregisterFile(FileEntry::Ptr file);

+  FileEntry::Index generateIndex();

 };

 

 

@@ -354,7 +357,7 @@ struct DirectoryEntryFileKey
     return std::hash<std::wstring>()(value);

   }

 

-  const std::wstring value;

+  std::wstring value;

   const std::size_t hash;

 };

 

@@ -405,7 +408,7 @@ public:
   // That origin may exist or not

   void addFromOrigin(

     const std::wstring &originName,

-    const std::wstring &directory, int priority);

+    const std::wstring &directory, int priority, DirectoryStats& stats);

 

   void addFromBSA(

     const std::wstring &originName, std::wstring &directory,

@@ -559,14 +562,14 @@ private:
 

   FileEntry::Ptr insert(

     std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime,

-    std::wstring_view archive, int order);

+    std::wstring_view archive, int order, DirectoryStats& stats);

 

   FileEntry::Ptr insert(

     env::File& file, FilesOrigin &origin,

     std::wstring_view archive, int order, DirectoryStats& stats);

 

   void addFiles(

-    FilesOrigin &origin, wchar_t *buffer, int bufferOffset);

+    FilesOrigin &origin, const std::wstring& path, DirectoryStats& stats);

 

   void addFiles(

     FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime,

@@ -575,7 +578,8 @@ private:
   void addDir(FilesOrigin& origin, env::Directory& d, DirectoryStats& stats);

 

   DirectoryEntry* getSubDirectory(

-    std::wstring_view name, bool create, int originID = -1);

+    std::wstring_view name, bool create, DirectoryStats& stats,

+    int originID = -1);

 

   DirectoryEntry* getSubDirectory(

     env::Directory& dir, bool create, DirectoryStats& stats,