threaded refresher

9 files changed, 395 insertions, 126 deletions

diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp
index b7dd51ba..5bff9d37 100644
--- a/src/directoryrefresher.cpp
+++ b/src/directoryrefresher.cpp
@@ -24,6 +24,7 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "report.h"

 #include "modinfo.h"

 #include "settings.h"

+#include "envfs.h"

 

 #include <QApplication>

 #include <QDir>

@@ -36,8 +37,8 @@ using namespace MOBase;
 using namespace MOShared;

 

 

-DirectoryRefresher::DirectoryRefresher()

-  : m_DirectoryStructure(nullptr)

+DirectoryRefresher::DirectoryRefresher(std::size_t threadCount)

+  : m_DirectoryStructure(nullptr), m_threadCount(threadCount)

 {

 }

 

@@ -116,59 +117,160 @@ void DirectoryRefresher::addModBSAToStructure(DirectoryEntry *directoryStructure
   }

 }

 

-void DirectoryRefresher::addModFilesToStructure(DirectoryEntry *directoryStructure, const QString &modName,

-                                                int priority, const QString &directory, const QStringList &stealFiles)

+void DirectoryRefresher::stealModFilesIntoStructure(

+  DirectoryEntry *directoryStructure, const QString &modName,

+  int priority, const QString &directory, const QStringList &stealFiles)

 {

   std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory));

 

-  if (stealFiles.length() > 0) {

-    // instead of adding all the files of the target directory, we just change the root of the specified

-    // files to this mod

-    FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority);

-    for (const QString &filename : stealFiles) {

-      if (filename.isEmpty()) {

-        log::warn("Trying to find file with no name");

-        continue;

-      }

-      QFileInfo fileInfo(filename);

-      FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName()));

-      if (file.get() != nullptr) {

-        if (file->getOrigin() == 0) {

-          // replace data as the origin on this bsa

-          file->removeOrigin(0);

-        }

-        origin.addFile(file->getIndex());

-        file->addOrigin(origin.getID(), file->getFileTime(), L"", -1);

-      } else {

-        QString warnStr = fileInfo.absolutePath();

-        if (warnStr.isEmpty())

-          warnStr = filename;

-        log::warn("file not found: {}", warnStr);

+  // instead of adding all the files of the target directory, we just change the root of the specified

+  // files to this mod

+  FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority);

+  for (const QString &filename : stealFiles) {

+    if (filename.isEmpty()) {

+      log::warn("Trying to find file with no name");

+      continue;

+    }

+    QFileInfo fileInfo(filename);

+    FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName()));

+    if (file.get() != nullptr) {

+      if (file->getOrigin() == 0) {

+        // replace data as the origin on this bsa

+        file->removeOrigin(0);

       }

+      origin.addFile(file->getIndex());

+      file->addOrigin(origin.getID(), file->getFileTime(), L"", -1);

+    } else {

+      QString warnStr = fileInfo.absolutePath();

+      if (warnStr.isEmpty())

+        warnStr = filename;

+      log::warn("file not found: {}", warnStr);

     }

+  }

+}

+

+void DirectoryRefresher::addModFilesToStructure(

+  DirectoryEntry *directoryStructure, const QString &modName,

+  int priority, const QString &directory, const QStringList &stealFiles)

+{

+  TimeThis tt("addModFilesToStructure()");

+

+  std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory));

+

+  if (stealFiles.length() > 0) {

+    stealModFilesIntoStructure(

+      directoryStructure, modName, priority, directory, stealFiles);

   } else {

     directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority);

   }

 }

 

 void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure

-                                           , const QString &modName

-                                           , int priority

-                                           , const QString &directory

-                                           , const QStringList &stealFiles

-                                           , const QStringList &archives)

+  , const QString &modName

+  , int priority

+  , const QString &directory

+  , const QStringList &stealFiles

+  , const QStringList &archives)

 {

-  addModFilesToStructure(directoryStructure, modName, priority, directory, stealFiles);

+  TimeThis tt("addModToStructure()");

+

+  if (stealFiles.length() > 0) {

+    stealModFilesIntoStructure(

+      directoryStructure, modName, priority, directory, stealFiles);

+  } else {

+    std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory));

+    directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority);

+  }

 

   if (Settings::instance().archiveParsing()) {

     addModBSAToStructure(directoryStructure, modName, priority, directory, archives);

   }

 }

 

+struct ModThread

+{

+  std::wstring path;

+  env::Directory* dir = nullptr;

+  std::condition_variable cv;

+  std::mutex mutex;

+  bool ready = false;

+

+  void wakeup()

+  {

+    ready = true;

+    cv.notify_one();

+  }

+

+  void run()

+  {

+    std::unique_lock lock(mutex);

+    cv.wait(lock, [&]{ return ready; });

+

+    *dir = env::getFilesAndDirs(path);

+

+    ready = false;

+  }

+};

+

+void DirectoryRefresher::addMultipleModsFilesToStructure(

+  MOShared::DirectoryEntry *directoryStructure,

+  const std::vector<EntryInfo>& entries, bool emitProgress)

+{

+  TimeThis tt(QString("add %1 mods").arg(entries.size()));

+

+  env::ThreadPool<ModThread> threads(m_threadCount);

+  std::vector<env::Directory> dirs(entries.size());

+

+  for (std::size_t i=0; i<entries.size(); ++i) {

+    const auto& e = entries[i];

+    const int prio = static_cast<int>(i + 1);

+

+    try {

+      if (e.stealFiles.length() > 0) {

+        stealModFilesIntoStructure(

+          directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles);

+      } else {

+        auto& mt = threads.request();

+

+        mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString();

+        mt.dir = &dirs[i];

+

+        mt.wakeup();

+      }

+    } catch (const std::exception& ex) {

+      emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what()));

+    }

+

+    if (emitProgress) {

+      emit progress((static_cast<int>(i) * 100) / static_cast<int>(entries.size()) + 1);

+    }

+  }

+

+  threads.join();

+

+  for (std::size_t i=0; i<entries.size(); ++i) {

+    const int prio = static_cast<int>(i + 1);

+

+    directoryStructure->addFromList(

+      entries[i].modName.toStdWString(),

+      entries[i].absolutePath.toStdWString(),

+      dirs[i],

+      prio);

+

+    if (Settings::instance().archiveParsing()) {

+      addModBSAToStructure(

+        directoryStructure,

+        entries[i].modName,

+        prio,

+        entries[i].absolutePath,

+        entries[i].archives);

+    }

+  }

+}

+

 void DirectoryRefresher::refresh()

 {

   SetThisThreadName("DirectoryRefresher");

-  TimeThis tt("DirectoryRefresher::refresh()");

 

   QMutexLocker locker(&m_RefreshLock);

 

@@ -178,20 +280,16 @@ void DirectoryRefresher::refresh()
 

   IPluginGame *game = qApp->property("managed_game").value<IPluginGame*>();

 

-  std::wstring dataDirectory = QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString();

+  std::wstring dataDirectory =

+    QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString();

+

   m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0);

 

-  std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs){return lhs.priority < rhs.priority;});

-  auto iter = m_Mods.begin();

+  std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) {

+    return lhs.priority < rhs.priority;

+  });

 

-  for (int i = 1; iter != m_Mods.end(); ++iter, ++i) {

-    try {

-      addModToStructure(m_DirectoryStructure, iter->modName, i, iter->absolutePath, iter->stealFiles, iter->archives);

-    } catch (const std::exception &e) {

-      emit error(tr("failed to read mod (%1): %2").arg(iter->modName, e.what()));

-    }

-    emit progress((i * 100) / static_cast<int>(m_Mods.size()) + 1);

-  }

+  addMultipleModsFilesToStructure(m_DirectoryStructure, m_Mods, true);

 

   m_DirectoryStructure->getFileRegister()->sortOrigins();

 

diff --git a/src/directoryrefresher.h b/src/directoryrefresher.h
index a4fc5dbc..c531ba39 100644
--- a/src/directoryrefresher.h
+++ b/src/directoryrefresher.h
@@ -39,12 +39,23 @@ class DirectoryRefresher : public QObject
   Q_OBJECT

 

 public:

+  struct EntryInfo {

+    EntryInfo(const QString &modName, const QString &absolutePath,

+      const QStringList &stealFiles, const QStringList &archives, int priority)

+      : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles)

+      , archives(archives), priority(priority) {}

+    QString modName;

+    QString absolutePath;

+    QStringList stealFiles;

+    QStringList archives;

+    int priority;

+  };

 

   /**

    * @brief constructor

    *

    **/

-  DirectoryRefresher();

+  DirectoryRefresher(std::size_t threadCount);

 

   ~DirectoryRefresher();

 

@@ -53,7 +64,7 @@ public:
    *

    * returns a pointer to the updated directory structure. DirectoryRefresher

    * deletes its own pointer and the caller takes custody of the pointer

-   * 

+   *

    * @return updated directory structure

    **/

   MOShared::DirectoryEntry *getDirectoryStructure();

@@ -107,7 +118,13 @@ public:
    * @param directory

    * @param stealFiles

    */

-  void addModFilesToStructure(MOShared::DirectoryEntry *directoryStructure, const QString &modName, int priority, const QString &directory, const QStringList &stealFiles);

+  void addModFilesToStructure(

+    MOShared::DirectoryEntry *directoryStructure, const QString &modName,

+    int priority, const QString &directory, const QStringList &stealFiles);

+

+  void addMultipleModsFilesToStructure(

+    MOShared::DirectoryEntry *directoryStructure,

+    const std::vector<EntryInfo>& entries, bool emitProgress=false);

 

 public slots:

 

@@ -123,26 +140,15 @@ signals:
   void refreshed();

 

 private:

-

-  struct EntryInfo {

-    EntryInfo(const QString &modName, const QString &absolutePath,

-              const QStringList &stealFiles, const QStringList &archives, int priority)

-      : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles)

-      , archives(archives), priority(priority) {}

-    QString modName;

-    QString absolutePath;

-    QStringList stealFiles;

-    QStringList archives;

-    int priority;

-  };

-

-private:

-

   std::vector<EntryInfo> m_Mods;

   std::set<QString> m_EnabledArchives;

   MOShared::DirectoryEntry *m_DirectoryStructure;

   QMutex m_RefreshLock;

+  std::size_t m_threadCount;

 

+  void stealModFilesIntoStructure(

+    MOShared::DirectoryEntry *directoryStructure, const QString &modName,

+    int priority, const QString &directory, const QStringList &stealFiles);

 };

 

 #endif // DIRECTORYREFRESHER_H

diff --git a/src/envfs.cpp b/src/envfs.cpp
index 81a0d3ef..2716737f 100644
--- a/src/envfs.cpp
+++ b/src/envfs.cpp
@@ -150,44 +150,62 @@ QString toString(POBJECT_ATTRIBUTES poa)
 }
 
 
-constexpr std::size_t AllocSize = 1024 * 1024;
-std::vector<std::unique_ptr<unsigned char[]>> g_buffers;
-
-
-struct HandleCloserThread
+class HandleCloserThread
 {
-  std::vector<HANDLE> handles;
-  std::thread thread;
-  std::atomic<bool> busy;
-
+public:
   HandleCloserThread()
-    : busy(false)
+    : m_ready(false)
   {
+    m_handles.reserve(50'000);
   }
 
-  ~HandleCloserThread()
+  void add(HANDLE h)
   {
-    if (thread.joinable()) {
-      thread.join();
-    }
+    m_handles.push_back(h);
+  }
+
+  void wakeup()
+  {
+    m_ready = true;
+    m_cv.notify_one();
+  }
+
+  void run()
+  {
+    std::unique_lock lock(m_mutex);
+    m_cv.wait(lock, [&]{ return m_ready; });
+
+    closeHandles();
   }
 
+private:
+  std::vector<HANDLE> m_handles;
+  std::condition_variable m_cv;
+  std::mutex m_mutex;
+  bool m_ready;
+
   void closeHandles()
   {
-    for (auto& h : handles) {
+    for (auto& h : m_handles) {
       NtClose(h);
     }
 
-    handles.clear();
-    busy = false;
+    m_handles.clear();
+    m_ready = false;
   }
 };
 
-std::array<HandleCloserThread, 10> g_handleCloserThreads;
+constexpr std::size_t AllocSize = 1024 * 1024;
+static ThreadPool<HandleCloserThread> g_handleClosers;
 
+void setHandleCloserThreadCount(std::size_t n)
+{
+  g_handleClosers.setMax(n);
+}
 
 void forEachEntryImpl(
-  void* cx, HandleCloserThread& hc, POBJECT_ATTRIBUTES poa, std::size_t depth,
+  void* cx, HandleCloserThread& hc, std::vector<std::unique_ptr<unsigned char[]>>& buffers,
+  POBJECT_ATTRIBUTES poa, std::size_t depth,
   DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF)
 {
   IO_STATUS_BLOCK iosb;
@@ -207,14 +225,14 @@ void forEachEntryImpl(
     return;
   }
 
-  hc.handles.push_back(oa.RootDirectory);
+  hc.add(oa.RootDirectory);
   unsigned char* buffer;
 
-  if (depth >= g_buffers.size()) {
-    g_buffers.emplace_back(std::make_unique<unsigned char[]>(AllocSize));
-    buffer = g_buffers.back().get();
+  if (depth >= buffers.size()) {
+    buffers.emplace_back(std::make_unique<unsigned char[]>(AllocSize));
+    buffer = buffers.back().get();
   } else {
-    buffer = g_buffers[depth].get();
+    buffer = buffers[depth].get();
   }
 
   union
@@ -268,7 +286,7 @@ void forEachEntryImpl(
 
         if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
           dirStartF(cx, toStringView(&oa));
-          forEachEntryImpl(cx, hc, &oa, depth+1, dirStartF, dirEndF, fileF);
+          forEachEntryImpl(cx, hc, buffers, &oa, depth+1, dirStartF, dirEndF, fileF);
           dirEndF(cx, toStringView(&oa));
         } else {
           //log::debug("{}{}", std::wstring((depth + 1) * 2, L' '), toString(&oa));
@@ -293,35 +311,13 @@ void forEachEntryImpl(
   }
 }
 
-std::size_t findHandleCloserThread()
-{
-  for (;;) {
-    for (std::size_t i=0; i<g_handleCloserThreads.size(); ++i) {
-      auto& t = g_handleCloserThreads[i];
-
-      if (!t.busy) {
-        if (t.thread.joinable()) {
-          t.thread.join();
-        }
-
-        t.busy = true;
-
-        return i;
-      }
-    }
-
-    std::this_thread::sleep_for(std::chrono::milliseconds(1));
-  }
-}
-
 void forEachEntry(
   const std::wstring& path, void* cx,
   DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF)
 {
-  const std::size_t hci = findHandleCloserThread();
+  auto& hc = g_handleClosers.request();
 
-  auto& hc = g_handleCloserThreads[hci];
-  hc.handles.reserve(50'000);
+  std::vector<std::unique_ptr<unsigned char[]>> buffers;
 
   if (!NtOpenFile) {
     HMODULE m = ::LoadLibraryW(L"ntdll.dll");
@@ -342,9 +338,41 @@ void forEachEntry(
   oa.Length = sizeof(oa);
   oa.ObjectName = &ObjectName;
 
-  forEachEntryImpl(cx, hc, &oa, 0, dirStartF, dirEndF, fileF);
+  forEachEntryImpl(cx, hc, buffers, &oa, 0, dirStartF, dirEndF, fileF);
+  hc.wakeup();
+}
+
+Directory getFilesAndDirs(const std::wstring& path)
+{
+  struct Context
+  {
+    std::stack<Directory*> current;
+  };
+
+  Directory root;
+
+  Context cx;
+  cx.current.push(&root);
+
+  env::forEachEntry(path, &cx,
+    [](void* pcx, std::wstring_view path) {
+      Context* cx = (Context*)pcx;
+      cx->current.top()->dirs.push_back({std::wstring(path.begin(), path.end())});
+      cx->current.push(&cx->current.top()->dirs.back());
+    },
+
+    [](void* pcx, std::wstring_view path) {
+      Context* cx = (Context*)pcx;
+      cx->current.pop();
+    },
+
+      [](void* pcx, std::wstring_view path, FILETIME ft) {
+      Context* cx = (Context*)pcx;
+      cx->current.top()->files.push_back({std::wstring(path.begin(), path.end()), ft});
+    }
+  );
 
-  hc.thread = std::thread([hci]{ g_handleCloserThreads[hci].closeHandles(); });
+  return root;
 }
 
 } // namespace
diff --git a/src/envfs.h b/src/envfs.h
index aeaa6796..1fc53bcf 100644
--- a/src/envfs.h
+++ b/src/envfs.h
@@ -1,17 +1,105 @@
 #ifndef ENV_ENVFS_H
 #define ENV_ENVFS_H
 
+#include <thread>
+
 namespace env
 {
 
+struct File
+{
+  std::wstring name;
+  FILETIME ft;
+};
+
+struct Directory
+{
+  std::wstring name;
+  std::list<Directory> dirs;
+  std::list<File> files;
+};
+
+
+template <class T>
+class ThreadPool
+{
+public:
+  ThreadPool(std::size_t max=1)
+    : m_threads(max)
+  {
+  }
+
+  ~ThreadPool()
+  {
+    join();
+  }
+
+  void setMax(std::size_t n)
+  {
+    m_threads.resize(n);
+  }
+
+  void join()
+  {
+    for (auto& ti : m_threads) {
+      if (ti.thread.joinable()) {
+        ti.thread.join();
+      }
+    }
+  }
+
+  T& request()
+  {
+    if (m_threads.empty()) {
+      std::terminate();
+    }
+
+    for (;;) {
+      for (auto& ti : m_threads) {
+        bool expected = false;
+
+        if (ti.busy.compare_exchange_strong(expected, true)) {
+          if (ti.thread.joinable()) {
+            ti.thread.join();
+          }
+
+          ti.thread = std::thread([&]{
+            ti.o.run();
+            ti.busy = false;
+          });
+
+          return ti.o;
+        }
+      }
+
+      std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+  }
+
+private:
+  struct ThreadInfo
+  {
+    std::thread thread;
+    std::atomic<bool> busy;
+    T o;
+  };
+
+  std::list<ThreadInfo> m_threads;
+};
+
+
 using DirStartF = void (void*, std::wstring_view);
 using DirEndF = void (void*, std::wstring_view);
 using FileF = void (void*, std::wstring_view, FILETIME);
 
+void setHandleCloserThreadCount(std::size_t n);
+
 void forEachEntry(
   const std::wstring& path, void* cx,
   DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF);
 
+Directory getFilesAndDirs(const std::wstring& path);
+
 } // namespace
 
 #endif // ENV_ENVFS_H
diff --git a/src/organizercore.cpp b/src/organizercore.cpp
index 8124be1d..86abeb35 100644
--- a/src/organizercore.cpp
+++ b/src/organizercore.cpp
@@ -88,24 +88,19 @@ QStringList toStringList(InputIterator current, InputIterator end)
 OrganizerCore::OrganizerCore(Settings &settings)
   : m_UserInterface(nullptr)
   , m_PluginContainer(nullptr)
-  , m_GameName()
   , m_CurrentProfile(nullptr)
   , m_Settings(settings)
   , m_Updater(NexusInterface::instance(m_PluginContainer))
-  , m_AboutToRun()
-  , m_FinishedRun()
-  , m_ModInstalled()
   , m_ModList(m_PluginContainer, this)
   , m_PluginList(this)
-  , m_DirectoryRefresher()
+  , m_DirectoryRefresher(settings.refreshThreadCount())
   , m_DirectoryStructure(new DirectoryEntry(L"data", nullptr, 0))
   , m_DownloadManager(NexusInterface::instance(m_PluginContainer), this)
-  , m_InstallationManager()
-  , m_RefresherThread()
   , m_DirectoryUpdate(false)
   , m_ArchivesInit(false)
   , m_PluginListsWriter(std::bind(&OrganizerCore::savePluginList, this))
 {
+  env::setHandleCloserThreadCount(settings.refreshThreadCount());
   m_DownloadManager.setOutputDirectory(m_Settings.paths().downloads(), false);
 
   NexusInterface::instance(m_PluginContainer)->setCacheDirectory(
@@ -1245,13 +1240,17 @@ void OrganizerCore::updateModInDirectoryStructure(unsigned int index,
 
 void OrganizerCore::updateModsInDirectoryStructure(QMap<unsigned int, ModInfo::Ptr> modInfo)
 {
+  std::vector<DirectoryRefresher::EntryInfo> entries;
+
   for (auto idx : modInfo.keys()) {
-    // add files of the bsa to the directory structure
-    m_DirectoryRefresher.addModFilesToStructure(
-      m_DirectoryStructure, modInfo[idx]->name(),
-      m_CurrentProfile->getModPriority(idx), modInfo[idx]->absolutePath(),
-      modInfo[idx]->stealFiles());
+    entries.push_back({
+      modInfo[idx]->name(), modInfo[idx]->absolutePath(),
+      modInfo[idx]->stealFiles(), {}, m_CurrentProfile->getModPriority(idx)});
   }
+
+  m_DirectoryRefresher.addMultipleModsFilesToStructure(
+    m_DirectoryStructure, entries);
+
   DirectoryRefresher::cleanStructure(m_DirectoryStructure);
   // need to refresh plugin list now so we can activate esps
   refreshESPList(true);
diff --git a/src/settings.cpp b/src/settings.cpp
index 761cd669..f0496fe8 100644
--- a/src/settings.cpp
+++ b/src/settings.cpp
@@ -209,6 +209,16 @@ void Settings::setUseSplash(bool b)
   set(m_Settings, "Settings", "use_splash", b);
 }
 
+std::size_t Settings::refreshThreadCount() const
+{
+  return get<std::size_t>(m_Settings, "Settings", "refresh_thread_count", 10);
+}
+
+void Settings::setRefreshThreadCount(std::size_t n) const
+{
+  return set(m_Settings, "Settings", "refresh_thread_count", n);
+}
+
 std::optional<QVersionNumber> Settings::version() const
 {
   if (auto v=getOptional<QString>(m_Settings, "General", "version")) {
diff --git a/src/settings.h b/src/settings.h
index 3d37bdd8..b2cb6be5 100644
--- a/src/settings.h
+++ b/src/settings.h
@@ -731,6 +731,11 @@ public:
   bool useSplash() const;
   void setUseSplash(bool b);
 
+  // number of threads to use when refreshing
+  //
+  std::size_t refreshThreadCount() const;
+  void setRefreshThreadCount(std::size_t n) const;
+
   GameSettings& game();
   const GameSettings& game() const;
 
diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp
index d171be38..b7329833 100644
--- a/src/shared/directoryentry.cpp
+++ b/src/shared/directoryentry.cpp
@@ -691,6 +691,33 @@ void DirectoryEntry::addFromOrigin(
   m_Populated = true;

 }

 

+void DirectoryEntry::addFromList(

+  const std::wstring &originName, const std::wstring &directory,

+  env::Directory& root, int priority)

+{

+  FilesOrigin &origin = createOrigin(originName, directory, priority);

+  addDir(origin, root);

+}

+

+void DirectoryEntry::addDir(FilesOrigin& origin, env::Directory& d)

+{

+  for (auto& sd : d.dirs) {

+    auto* sdirEntry = getSubDirectory(sd.name, true, origin.getID());

+    sdirEntry->addDir(origin, sd);

+  }

+

+  for (auto& f : d.files) {

+    insert(f.name, origin, f.ft, L"", -1);

+  }

+

+  std::sort(

+    m_SubDirectories.begin(),

+    m_SubDirectories.end(),

+    &DirCompareByName);

+

+  m_Populated = true;

+}

+

 void DirectoryEntry::addFromBSA(

   const std::wstring &originName, std::wstring &directory,

   const std::wstring &fileName, int priority, int order)

diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h
index 58e70ffa..772899d7 100644
--- a/src/shared/directoryentry.h
+++ b/src/shared/directoryentry.h
@@ -33,7 +33,9 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include <boost/shared_ptr.hpp>

 #include <boost/weak_ptr.hpp>

 #endif

+

 #include "util.h"

+#include "envfs.h"

 

 namespace MOShared { struct DirectoryEntryFileKey; }

 

@@ -358,6 +360,10 @@ public:
     const std::wstring &originName, std::wstring &directory,

     const std::wstring &fileName, int priority, int order);

 

+  void addFromList(

+    const std::wstring &originName, const std::wstring &directory,

+    env::Directory& root, int priority);

+

   void propagateOrigin(int origin);

 

   const std::wstring &getName() const

@@ -511,6 +517,8 @@ private:
     FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime,

     const std::wstring &archiveName, int order);

 

+  void addDir(FilesOrigin& origin, env::Directory& d);

+

   DirectoryEntry* getSubDirectory(

     std::wstring_view name, bool create, int originID = -1);