thread-safe directory entry

5 files changed, 210 insertions, 74 deletions

diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp
index 5bff9d37..bce3f65a 100644
--- a/src/directoryrefresher.cpp
+++ b/src/directoryrefresher.cpp
@@ -25,6 +25,7 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "modinfo.h"

 #include "settings.h"

 #include "envfs.h"

+#include "modinfodialogfwd.h"

 

 #include <QApplication>

 #include <QDir>

@@ -125,7 +126,10 @@ void DirectoryRefresher::stealModFilesIntoStructure(
 

   // instead of adding all the files of the target directory, we just change the root of the specified

   // files to this mod

-  FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority);

+  DirectoryStats dummy;

+  FilesOrigin &origin = directoryStructure->createOrigin(

+    ToWString(modName), directoryW, priority, dummy);

+

   for (const QString &filename : stealFiles) {

     if (filename.isEmpty()) {

       log::warn("Trying to find file with no name");

@@ -189,8 +193,12 @@ void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure
 

 struct ModThread

 {

+  DirectoryEntry* ds = nullptr;

+  std::wstring modName;

   std::wstring path;

+  int prio = -1;

   env::Directory* dir = nullptr;

+

   std::condition_variable cv;

   std::mutex mutex;

   bool ready = false;

@@ -206,7 +214,16 @@ struct ModThread
     std::unique_lock lock(mutex);

     cv.wait(lock, [&]{ return ready; });

 

-    *dir = env::getFilesAndDirs(path);

+    ds->addFromOrigin(modName, path, prio);

+

+    /*if (Settings::instance().archiveParsing()) {

+      addModBSAToStructure(

+        directoryStructure,

+        entries[i].modName,

+        prio,

+        entries[i].absolutePath,

+        entries[i].archives);

+    }*/

 

     ready = false;

   }

@@ -216,65 +233,75 @@ void DirectoryRefresher::addMultipleModsFilesToStructure(
   MOShared::DirectoryEntry *directoryStructure,

   const std::vector<EntryInfo>& entries, bool emitProgress)

 {

-  TimeThis tt(QString("add %1 mods").arg(entries.size()));

-

-  env::ThreadPool<ModThread> threads(m_threadCount);

   std::vector<env::Directory> dirs(entries.size());

 

-  for (std::size_t i=0; i<entries.size(); ++i) {

-    const auto& e = entries[i];

-    const int prio = static_cast<int>(i + 1);

+  {

+    TimeThis tt("walk dirs");

 

-    try {

-      if (e.stealFiles.length() > 0) {

-        stealModFilesIntoStructure(

-          directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles);

-      } else {

-        auto& mt = threads.request();

+    env::ThreadPool<ModThread> threads(m_threadCount);

 

-        mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString();

-        mt.dir = &dirs[i];

+    for (std::size_t i=0; i<entries.size(); ++i) {

+      const auto& e = entries[i];

+      const int prio = static_cast<int>(i + 1);

 

-        mt.wakeup();

+      try

+      {

+        if (e.stealFiles.length() > 0) {

+          stealModFilesIntoStructure(

+            directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles);

+        } else {

+          auto& mt = threads.request();

+

+          mt.ds = directoryStructure;

+          mt.modName = entries[i].modName.toStdWString();

+          mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString();

+          mt.prio = prio;

+          mt.dir = &dirs[i];

+

+          mt.wakeup();

+        }

+      } catch (const std::exception& ex) {

+        emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what()));

       }

-    } catch (const std::exception& ex) {

-      emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what()));

-    }

 

-    if (emitProgress) {

-      emit progress((static_cast<int>(i) * 100) / static_cast<int>(entries.size()) + 1);

+      if (emitProgress) {

+        emit progress((static_cast<int>(i) * 100) / static_cast<int>(entries.size()) + 1);

+      }

     }

-  }

 

-  threads.join();

+    threads.join();

+  }

 

-  for (std::size_t i=0; i<entries.size(); ++i) {

-    const int prio = static_cast<int>(i + 1);

+  //std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){

+  //  return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0);

+  //});

 

-    directoryStructure->addFromList(

-      entries[i].modName.toStdWString(),

-      entries[i].absolutePath.toStdWString(),

-      dirs[i],

-      prio);

+  //static int run = 1;

+  //

+  //std::ofstream out("c:\\tmp\\data.csv", std::ios::app);

 

-    if (Settings::instance().archiveParsing()) {

-      addModBSAToStructure(

-        directoryStructure,

-        entries[i].modName,

-        prio,

-        entries[i].absolutePath,

-        entries[i].archives);

-    }

-  }

+  //out << fmt::format("what,run,{}", DirectoryStats::csvHeader());

+  //

+  //for (std::size_t i=0; i<entries.size(); ++i) {

+  //  out << fmt::format("{},{},{}", stats[i].mod, run, stats[i].toCsv()) << "\n";

+  //}

+  //

+  //out << fmt::format("total,{},{}", run, total.toCsv()) << "\n";

+  //

+  //++run;

 }

 

+namespace MOShared{ void logcounts(std::string w); }

+

 void DirectoryRefresher::refresh()

 {

   SetThisThreadName("DirectoryRefresher");

 

   QMutexLocker locker(&m_RefreshLock);

 

+  //logcounts("before delete");

   delete m_DirectoryStructure;

+  //logcounts("after delete");

 

   m_DirectoryStructure = new DirectoryEntry(L"data", nullptr, 0);

 

@@ -298,4 +325,6 @@ void DirectoryRefresher::refresh()
   cleanStructure(m_DirectoryStructure);

 

   emit refreshed();

+

+  //logcounts("after refresh");

 }

diff --git a/src/envfs.cpp b/src/envfs.cpp
index d7245f66..022ca513 100644
--- a/src/envfs.cpp
+++ b/src/envfs.cpp
@@ -372,11 +372,7 @@ Directory getFilesAndDirs(const std::wstring& path)
     [](void* pcx, std::wstring_view path) {
       Context* cx = (Context*)pcx;
 
-      cx->current.top()->dirs.push_back({
-        std::wstring(path.begin(), path.end()),
-        MOShared::ToLowerCopy(path)
-      });
-
+      cx->current.top()->dirs.push_back(Directory(path));
       cx->current.push(&cx->current.top()->dirs.back());
     },
 
@@ -385,18 +381,70 @@ Directory getFilesAndDirs(const std::wstring& path)
       cx->current.pop();
     },
 
-      [](void* pcx, std::wstring_view path, FILETIME ft) {
+    [](void* pcx, std::wstring_view path, FILETIME ft) {
       Context* cx = (Context*)pcx;
 
-      cx->current.top()->files.push_back({
-        std::wstring(path.begin(), path.end()),
-        MOShared::ToLowerCopy(path),
-        ft
-      });
+      cx->current.top()->files.push_back(File(path, ft));
     }
   );
 
   return root;
 }
 
+File::File(std::wstring_view n, FILETIME ft) :
+  name(n.begin(), n.end()),
+  lcname(MOShared::ToLowerCopy(name)),
+  lastModified(ft)
+{
+}
+
+Directory::Directory()
+{
+}
+
+Directory::Directory(std::wstring_view n)
+  : name(n.begin(), n.end()), lcname(MOShared::ToLowerCopy(name))
+{
+}
+
+
+void getFilesAndDirsWithFindImpl(const std::wstring& path, Directory& d)
+{
+  const std::wstring searchString = path + L"\\*";
+
+  WIN32_FIND_DATAW findData;
+
+  HANDLE searchHandle = ::FindFirstFileExW(
+    searchString.c_str(), FindExInfoBasic, &findData, FindExSearchNameMatch,
+    nullptr, FIND_FIRST_EX_LARGE_FETCH);
+
+  if (searchHandle != INVALID_HANDLE_VALUE) {
+    BOOL result = true;
+
+    while (result) {
+      if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+        if ((wcscmp(findData.cFileName, L".") != 0) &&
+          (wcscmp(findData.cFileName, L"..") != 0)) {
+          const std::wstring newPath = path + L"\\" + findData.cFileName;
+          d.dirs.push_back(Directory(findData.cFileName));
+          getFilesAndDirsWithFindImpl(newPath, d.dirs.back());
+        }
+      } else {
+        d.files.push_back(File(findData.cFileName, findData.ftLastWriteTime));
+      }
+
+      result = ::FindNextFileW(searchHandle, &findData);
+    }
+  }
+
+  ::FindClose(searchHandle);
+}
+
+Directory getFilesAndDirsWithFind(const std::wstring& path)
+{
+  Directory d;
+  getFilesAndDirsWithFindImpl(path, d);
+  return d;
+}
+
 } // namespace
diff --git a/src/envfs.h b/src/envfs.h
index 6e0da6d0..001f2d09 100644
--- a/src/envfs.h
+++ b/src/envfs.h
@@ -11,6 +11,8 @@ struct File
   std::wstring name;
   std::wstring lcname;
   FILETIME lastModified;
+
+  File(std::wstring_view name, FILETIME ft);
 };
 
 struct Directory
@@ -18,8 +20,11 @@ struct Directory
   std::wstring name;
   std::wstring lcname;
 
-  std::list<Directory> dirs;
-  std::list<File> files;
+  std::vector<Directory> dirs;
+  std::vector<File> files;
+
+  Directory();
+  Directory(std::wstring_view name);
 };
 
 
@@ -111,6 +116,7 @@ void forEachEntry(
   DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF);
 
 Directory getFilesAndDirs(const std::wstring& path);
+Directory getFilesAndDirsWithFind(const std::wstring& path);
 
 } // namespace
 
diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp
index a90f6f08..e1464595 100644
--- a/src/shared/directoryentry.cpp
+++ b/src/shared/directoryentry.cpp
@@ -214,11 +214,14 @@ public:
   {

     int newID = createID();

 

-    m_Origins[newID] = FilesOrigin(newID, originName, directory, priority, fileRegister, originConnection);

-    m_OriginsNameMap[originName] = newID;

-    m_OriginsPriorityMap[priority] = newID;

+    auto itor = m_Origins.insert({newID, FilesOrigin(

+      newID, originName, directory, priority,

+      fileRegister, originConnection)}).first;

 

-    return m_Origins[newID];

+    m_OriginsNameMap.insert({originName, newID});

+    m_OriginsPriorityMap.insert({priority, newID});

+

+    return itor->second;

   }

 

   bool exists(const std::wstring &name)

@@ -597,9 +600,13 @@ std::vector<FileEntry::Ptr> FilesOrigin::getFiles() const
 {

   std::vector<FileEntry::Ptr> result;

 

-  for (FileEntry::Index fileIdx : m_Files) {

-    if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) {

-      result.push_back(p);

+  {

+    std::scoped_lock lock(m_Mutex);

+

+    for (FileEntry::Index fileIdx : m_Files) {

+      if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) {

+        result.push_back(p);

+      }

     }

   }

 

@@ -621,9 +628,16 @@ void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter)
 {

   if (!enabled) {

     ++stats.originsNeededEnabled;

-    std::set<FileEntry::Index> copy = m_Files;

+

+    std::set<FileEntry::Index> copy;

+

+    {

+      std::scoped_lock lock(m_Mutex);

+      copy = m_Files;

+      m_Files.clear();

+    }

+

     m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter);

-    m_Files.clear();

   }

 

   m_Disabled = !enabled;

@@ -631,6 +645,8 @@ void FilesOrigin::enable(bool enabled, DirectoryStats& stats, time_t notAfter)
 

 void FilesOrigin::removeFile(FileEntry::Index index)

 {

+  std::scoped_lock lock(m_Mutex);

+

   auto iter = m_Files.find(index);

 

   if (iter != m_Files.end()) {

@@ -640,6 +656,8 @@ void FilesOrigin::removeFile(FileEntry::Index index)
 

 bool FilesOrigin::containsArchive(std::wstring archiveName)

 {

+  std::scoped_lock lock(m_Mutex);

+

   for (FileEntry::Index fileIdx : m_Files) {

     if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) {

       if (p->isFromArchive(archiveName)) {

@@ -665,6 +683,7 @@ FileRegister::~FileRegister()
 

 bool FileRegister::indexValid(FileEntry::Index index) const

 {

+  std::scoped_lock lock(m_Mutex);

   return (m_Files.find(index) != m_Files.end());

 }

 

@@ -675,6 +694,8 @@ FileEntry::Ptr FileRegister::createFile(
   FileEntry::Ptr p;

 

   stats.addFileToRegisterTimes += elapsed([&]{

+    std::scoped_lock lock(m_Mutex);

+

     auto r = m_Files.insert_or_assign(

       index, FileEntry::Ptr(new FileEntry(index, std::move(name), parent)));

 

@@ -692,6 +713,8 @@ FileEntry::Ptr FileRegister::createFile(
 

 FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const

 {

+  std::scoped_lock lock(m_Mutex);

+

   auto iter = m_Files.find(index);

 

   if (iter != m_Files.end()) {

@@ -703,6 +726,8 @@ FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const
 

 bool FileRegister::removeFile(FileEntry::Index index)

 {

+  std::scoped_lock lock(m_Mutex);

+

   auto iter = m_Files.find(index);

 

   if (iter != m_Files.end()) {

@@ -717,12 +742,15 @@ bool FileRegister::removeFile(FileEntry::Index index)
 

 void FileRegister::removeOrigin(FileEntry::Index index, int originID)

 {

+  std::unique_lock lock(m_Mutex);

+

   auto iter = m_Files.find(index);

 

   if (iter != m_Files.end()) {

     if (iter->second->removeOrigin(originID)) {

-      unregisterFile(iter->second);

       m_Files.erase(iter);

+      lock.unlock();

+      unregisterFile(iter->second);

     }

   } else {

     log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index);

@@ -734,17 +762,21 @@ void FileRegister::removeOriginMulti(
 {

   std::vector<FileEntry::Ptr> removedFiles;

 

-  for (auto iter = indices.begin(); iter != indices.end(); ) {

-    auto pos = m_Files.find(*iter);

+  {

+    std::scoped_lock lock(m_Mutex);

+

+    for (auto iter = indices.begin(); iter != indices.end(); ) {

+      auto pos = m_Files.find(*iter);

 

-    if (pos != m_Files.end()

-        && (pos->second->lastAccessed() < notAfter)

-        && pos->second->removeOrigin(originID)) {

-      removedFiles.push_back(pos->second);

-      m_Files.erase(pos);

-      ++iter;

-    } else {

-      indices.erase(iter++);

+      if (pos != m_Files.end()

+          && (pos->second->lastAccessed() < notAfter)

+          && pos->second->removeOrigin(originID)) {

+        removedFiles.push_back(pos->second);

+        m_Files.erase(pos);

+        ++iter;

+      } else {

+        indices.erase(iter++);

+      }

     }

   }

 

@@ -776,6 +808,8 @@ void FileRegister::removeOriginMulti(
 

 void FileRegister::sortOrigins()

 {

+  std::scoped_lock lock(m_Mutex);

+

   for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) {

     iter->second->sortOrigins();

   }

@@ -1228,6 +1262,8 @@ FileEntry::Ptr DirectoryEntry::insert(
   std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime,

   std::wstring_view archive, int order)

 {

+  std::scoped_lock lock(m_FilesMutex);

+

   std::wstring fileNameLower = ToLowerCopy(fileName);

 

   auto iter = m_Files.find(fileNameLower);

@@ -1255,6 +1291,8 @@ FileEntry::Ptr DirectoryEntry::insert(
   env::File& file, FilesOrigin &origin, std::wstring_view archive, int order,

   DirectoryStats& stats)

 {

+  std::scoped_lock lock(m_FilesMutex);

+

   FilesMap::iterator itor;

 

   stats.filesLookupTimes += elapsed([&]{

@@ -1307,7 +1345,12 @@ void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOf
     [](void* pcx, std::wstring_view path) {

       Context* cx = (Context*)pcx;

       auto* current= cx->current.top();

-      std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName);

+

+      {

+        std::scoped_lock lock(current->m_SubDirMutex);

+        std::sort(current->m_SubDirectories.begin(), current->m_SubDirectories.end(), &DirCompareByName);

+      }

+

       cx->current.pop();

     },

 

@@ -1389,6 +1432,8 @@ void DirectoryEntry::addFiles(
 DirectoryEntry *DirectoryEntry::getSubDirectory(

   std::wstring_view name, bool create, int originID)

 {

+  std::scoped_lock lock(m_SubDirMutex);

+

   std::wstring nameLc = ToLowerCopy(name);

   auto itor = m_SubDirectoriesLookup.find(nameLc);

 

diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h
index c8ec8a4e..71e8b32c 100644
--- a/src/shared/directoryentry.h
+++ b/src/shared/directoryentry.h
@@ -26,6 +26,8 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include <vector>

 #include <map>

 #include <cassert>

+#include <mutex>

+

 #define WIN32_MEAN_AND_LEAN

 #include <Windows.h>

 #include <bsatk.h>

@@ -274,6 +276,7 @@ public:
 

   void addFile(FileEntry::Index index)

   {

+    std::scoped_lock lock(m_Mutex);

     m_Files.insert(index);

   }

 

@@ -290,6 +293,7 @@ private:
   int m_Priority;

   boost::weak_ptr<FileRegister> m_FileRegister;

   boost::weak_ptr<OriginConnection> m_OriginConnection;

+  mutable std::mutex m_Mutex;

 

   FilesOrigin(

     int ID, const std::wstring &name, const std::wstring &path, int priority,

@@ -313,6 +317,7 @@ public:
 

   size_t size() const

   {

+    std::scoped_lock lock(m_Mutex);

     return m_Files.size();

   }

 

@@ -323,6 +328,7 @@ public:
   void sortOrigins();

 

 private:

+  mutable std::mutex m_Mutex;

   std::map<FileEntry::Index, FileEntry::Ptr> m_Files;

   boost::shared_ptr<OriginConnection> m_OriginConnection;

 

@@ -545,6 +551,8 @@ private:
   std::set<int> m_Origins;

   bool m_Populated;

   bool m_TopLevel;

+  std::mutex m_SubDirMutex;

+  std::mutex m_FilesMutex;

 

 

   DirectoryEntry(const DirectoryEntry &reference);