Merge pull request #1003 from isanae/file-list-improvements

Refresh optimizations

66 files changed, 3313 insertions, 1514 deletions

diff --git a/.gitignore b/.gitignore
index 6b4e04ac..935c4260 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ edit
 /*std*.log
 /*build
 
+/src/version.aps
diff --git a/modorganizer.natvis b/modorganizer.natvis
new file mode 100644
index 00000000..fe4a7ce2
--- /dev/null
+++ b/modorganizer.natvis
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010">
+<Type Name="FileTreeItem"><DisplayString>{{
+file={m_wsFile}
+ dir={m_isDirectory}
+ origin={m_originID}
+ flags={m_flags.i}
+ loaded={m_loaded}
+ expanded={m_expanded}
+}}</DisplayString></Type>
+
+</AutoVisualizer>
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ad8f74c0..85d8af0f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -132,6 +132,7 @@ SET(organizer_SRCS
 	filerenamer.cpp
 	texteditor.cpp
 	env.cpp
+	envfs.cpp
 	envmetrics.cpp
 	envmodule.cpp
 	envsecurity.cpp
@@ -154,6 +155,10 @@ SET(organizer_SRCS
     shared/windows_error.cpp
     shared/error_report.cpp
     shared/directoryentry.cpp
+	shared/fileentry.cpp
+	shared/filesorigin.cpp
+	shared/fileregister.cpp
+	shared/originconnection.cpp
     shared/util.cpp
     shared/appconfig.cpp
     shared/leaktrace.cpp
@@ -262,6 +267,7 @@ SET(organizer_HDRS
 	filerenamer.h
 	texteditor.h
 	env.h
+	envfs.h
 	envmetrics.h
 	envmodule.h
 	envsecurity.h
@@ -284,6 +290,10 @@ SET(organizer_HDRS
     shared/windows_error.h
     shared/error_report.h
     shared/directoryentry.h
+	shared/fileentry.h
+	shared/filesorigin.h
+	shared/fileregister.h
+	shared/originconnection.h
     shared/util.h
     shared/appconfig.h
     shared/appconfig.inc
@@ -348,8 +358,6 @@ set(browser
 
 set(core
 	categories
-	shared/directoryentry
-	directoryrefresher
 	installationmanager
 	instancemanager
 	loadmechanism
@@ -392,6 +400,7 @@ set(downloads
 
 set(env
 	env
+	envfs
 	envmetrics
 	envmodule
 	envsecurity
@@ -467,6 +476,15 @@ set(profiles
 	profilesdialog
 )
 
+set(register
+	shared/directoryentry
+	shared/fileentry
+	shared/filesorigin
+	shared/fileregister
+	shared/originconnection
+	directoryrefresher
+)
+
 set(settings
 	settings
 	settingsutilities
@@ -518,7 +536,7 @@ set(widgets
 
 set(src_filters
 	application core browser dialogs downloads env executables loot mainwindow
-	modinfo modinfo\\dialog modlist plugins previews profiles settings
+	modinfo modinfo\\dialog modlist plugins previews profiles register settings
 	settingsdialog utilities widgets
 )
 
@@ -534,6 +552,8 @@ endforeach()
 file(GLOB_RECURSE rule_files ../vsbuild/CMakeFiles/*.rule)
 file(GLOB qm_files ../vsbuild/src/*.qm)
 
+set(organizer_misc ../modorganizer.natvis)
+
 source_group(cmake FILES CMakeLists.txt)
 source_group(autogen FILES ${rule_files} ${qm_files})
 source_group(resources FILES ${organizer_RCS} ${organizer_QRCS})
@@ -626,7 +646,10 @@ ELSE()
   SET(usvfs_name usvfs_x86)
 ENDIF()
 
-ADD_EXECUTABLE(ModOrganizer WIN32 ${organizer_HDRS} ${organizer_SRCS} ${organizer_UIS} ${organizer_RCS} ${organizer_QRCS} ${organizer_translations_qm})
+ADD_EXECUTABLE(
+	ModOrganizer WIN32
+	${organizer_HDRS} ${organizer_SRCS} ${organizer_UIS} ${organizer_RCS}
+	${organizer_QRCS} ${organizer_translations_qm} ${organizer_misc})
 
 TARGET_LINK_LIBRARIES(ModOrganizer
                       Qt5::Widgets Qt5::WinExtras Qt5::WebEngineWidgets Qt5::Quick
diff --git a/src/datatab.cpp b/src/datatab.cpp
index f263c2c6..b6119d52 100644
--- a/src/datatab.cpp
+++ b/src/datatab.cpp
@@ -2,7 +2,6 @@
 #include "ui_mainwindow.h"
 #include "settings.h"
 #include "organizercore.h"
-#include "directoryentry.h"
 #include "messagedialog.h"
 #include "filetree.h"
 #include "filetreemodel.h"
@@ -26,10 +25,10 @@ DataTab::DataTab(
     m_firstActivation(true)
 {
   m_filetree.reset(new FileTree(core, m_pluginContainer, ui.tree));
-  m_filter.setEdit(mwui->dataTabFilter);
-  m_filter.setList(mwui->dataTree);
   m_filter.setUseSourceSort(true);
   m_filter.setFilterColumn(FileTreeModel::FileName);
+  m_filter.setEdit(mwui->dataTabFilter);
+  m_filter.setList(mwui->dataTree);
 
   if (auto* m=m_filter.proxyModel()) {
     m->setDynamicSortFilter(false);
diff --git a/src/directoryrefresher.cpp b/src/directoryrefresher.cpp
index 87305599..e8c26c90 100644
--- a/src/directoryrefresher.cpp
+++ b/src/directoryrefresher.cpp
@@ -18,40 +18,168 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 */

 

 #include "directoryrefresher.h"

+#include "shared/fileentry.h"

+#include "shared/filesorigin.h"

+#include "shared/directoryentry.h"

 

 #include "iplugingame.h"

 #include "utility.h"

 #include "report.h"

 #include "modinfo.h"

 #include "settings.h"

+#include "envfs.h"

+#include "modinfodialogfwd.h"

+#include "util.h"

+

+#include <gameplugins.h>

 

 #include <QApplication>

 #include <QDir>

 #include <QString>

 #include <QTextCodec>

-#include <gameplugins.h>

+

+#include <fstream>

 

 

 using namespace MOBase;

 using namespace MOShared;

 

 

-DirectoryRefresher::DirectoryRefresher()

-  : m_DirectoryStructure(nullptr)

+DirectoryStats::DirectoryStats()

 {

+  std::memset(this, 0, sizeof(DirectoryStats));

+}

+

+DirectoryStats& DirectoryStats::operator+=(const DirectoryStats& o)

+{

+  dirTimes += o.dirTimes;

+  fileTimes += o.fileTimes;

+  sortTimes += o.sortTimes;

+

+  subdirLookupTimes += o.subdirLookupTimes;

+  addDirectoryTimes += o.addDirectoryTimes;

+

+  filesLookupTimes += o.filesLookupTimes;

+  addFileTimes += o.addFileTimes;

+  addOriginToFileTimes += o.addOriginToFileTimes;

+  addFileToOriginTimes += o.addFileToOriginTimes;

+  addFileToRegisterTimes += o.addFileToRegisterTimes;

+

+  originExists += o.originExists;

+  originCreate += o.originCreate;

+  originsNeededEnabled += o.originsNeededEnabled;

+

+  subdirExists += o.subdirExists;

+  subdirCreate += o.subdirCreate;

+

+  fileExists += o.fileExists;

+  fileCreate += o.fileCreate;

+  filesInsertedInRegister += o.filesInsertedInRegister;

+  filesAssignedInRegister += o.filesAssignedInRegister;

+

+  return *this;

 }

 

-DirectoryRefresher::~DirectoryRefresher()

+std::string DirectoryStats::csvHeader()

 {

-  delete m_DirectoryStructure;

+  QStringList sl = {

+    "dirTimes",

+    "fileTimes",

+    "sortTimes",

+    "subdirLookupTimes",

+    "addDirectoryTimes",

+    "filesLookupTimes",

+    "addFileTimes",

+    "addOriginToFileTimes",

+    "addFileToOriginTimes",

+    "addFileToRegisterTimes",

+    "originExists",

+    "originCreate",

+    "originsNeededEnabled",

+    "subdirExists",

+    "subdirCreate",

+    "fileExists",

+    "fileCreate",

+    "filesInsertedInRegister",

+    "filesAssignedInRegister"};

+

+  return sl.join(",").toStdString();

 }

 

-DirectoryEntry *DirectoryRefresher::getDirectoryStructure()

+std::string DirectoryStats::toCsv() const

+{

+  QStringList oss;

+

+  auto s = [](auto ns) {

+    return ns.count() / 1000.0 / 1000.0 / 1000.0;

+  };

+

+  oss

+    << QString::number(s(dirTimes))

+    << QString::number(s(fileTimes))

+    << QString::number(s(sortTimes))

+

+    << QString::number(s(subdirLookupTimes))

+    << QString::number(s(addDirectoryTimes))

+

+    << QString::number(s(filesLookupTimes))

+    << QString::number(s(addFileTimes))

+    << QString::number(s(addOriginToFileTimes))

+    << QString::number(s(addFileToOriginTimes))

+    << QString::number(s(addFileToRegisterTimes))

+

+    << QString::number(originExists)

+    << QString::number(originCreate)

+    << QString::number(originsNeededEnabled)

+

+    << QString::number(subdirExists)

+    << QString::number(subdirCreate)

+

+    << QString::number(fileExists)

+    << QString::number(fileCreate)

+    << QString::number(filesInsertedInRegister)

+    << QString::number(filesAssignedInRegister);

+

+  return oss.join(",").toStdString();

+}

+

+void dumpStats(std::vector<DirectoryStats>& stats)

+{

+  static int run = 0;

+  static const std::string file("c:\\tmp\\data.csv");

+

+  if (run == 0) {

+    std::ofstream out(file, std::ios::out|std::ios::trunc);

+    out << fmt::format("what,run,{}", DirectoryStats::csvHeader()) << "\n";

+  }

+

+  std::sort(stats.begin(), stats.end(), [](auto&& a, auto&& b){

+    return (naturalCompare(QString::fromStdString(a.mod), QString::fromStdString(b.mod)) < 0);

+    });

+

+  std::ofstream out(file, std::ios::app);

+

+  DirectoryStats total;

+  for (const auto& s : stats) {

+    out << fmt::format("{},{},{}", s.mod, run, s.toCsv()) << "\n";

+    total += s;

+  }

+

+  out << fmt::format("total,{},{}", run, total.toCsv()) << "\n";

+

+  ++run;

+}

+

+

+DirectoryRefresher::DirectoryRefresher(std::size_t threadCount)

+  : m_threadCount(threadCount), m_lastFileCount(0)

+{

+}

+

+DirectoryEntry *DirectoryRefresher::stealDirectoryStructure()

 {

   QMutexLocker locker(&m_RefreshLock);

-  DirectoryEntry *result = m_DirectoryStructure;

-  m_DirectoryStructure = nullptr;

-  return result;

+  return m_Root.release();

 }

 

 void DirectoryRefresher::setMods(const std::vector<std::tuple<QString, QString, int> > &mods

@@ -82,119 +210,293 @@ void DirectoryRefresher::cleanStructure(DirectoryEntry *structure)
   }

 }

 

-void DirectoryRefresher::addModBSAToStructure(DirectoryEntry *directoryStructure, const QString &modName,

-                                              int priority, const QString &directory, const QStringList &archives)

+void DirectoryRefresher::addModBSAToStructure(

+  DirectoryEntry* root, const QString& modName,

+  int priority, const QString& directory, const QStringList& archives)

 {

-  std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory));

-  IPluginGame *game = qApp->property("managed_game").value<IPluginGame*>();

+  const IPluginGame *game = qApp->property("managed_game").value<IPluginGame*>();

 

   GamePlugins *gamePlugins = game->feature<GamePlugins>();

   QStringList loadOrder = QStringList();

   gamePlugins->getLoadOrder(loadOrder);

 

-  for (const QString &archive : archives) {

-    QFileInfo fileInfo(archive);

-    if (m_EnabledArchives.find(fileInfo.fileName()) != m_EnabledArchives.end()) {

+  std::vector<std::wstring> lo;

+  for (auto&& s : loadOrder) {

+    lo.push_back(s.toStdWString());

+  }

 

-      int order = -1;

+  std::vector<std::wstring> archivesW;

+  for (auto&& a : archives) {

+    archivesW.push_back(a.toStdWString());

+  }

 

-      for (auto plugin : loadOrder)

-      {

-        QString name = plugin.left(plugin.size() - 4);

-        if (fileInfo.fileName().startsWith(name + " - ", Qt::CaseInsensitive) || fileInfo.fileName().startsWith(name + ".", Qt::CaseInsensitive)) {

-          order = loadOrder.indexOf(plugin);

-        }

-      }

+  std::set<std::wstring> enabledArchives;

+  for (auto&& a : m_EnabledArchives) {

+    enabledArchives.insert(a.toStdWString());

+  }

 

-      try {

-        IPluginGame *game = qApp->property("managed_game").value<IPluginGame*>();

-        directoryStructure->addFromBSA(ToWString(modName), directoryW, ToWString(QDir::toNativeSeparators(fileInfo.absoluteFilePath())), priority, order);

-      } catch (const std::exception &e) {

-        throw MyException(tr("failed to parse bsa %1: %2").arg(archive, e.what()));

+  DirectoryStats dummy;

+

+  root->addFromAllBSAs(

+    modName.toStdWString(),

+    QDir::toNativeSeparators(directory).toStdWString(),

+    priority,

+    archivesW,

+    enabledArchives,

+    lo,

+    dummy);

+}

+

+void DirectoryRefresher::stealModFilesIntoStructure(

+  DirectoryEntry *directoryStructure, const QString &modName,

+  int priority, const QString &directory, const QStringList &stealFiles)

+{

+  std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory));

+

+  // instead of adding all the files of the target directory, we just change the root of the specified

+  // files to this mod

+  DirectoryStats dummy;

+  FilesOrigin &origin = directoryStructure->createOrigin(

+    ToWString(modName), directoryW, priority, dummy);

+

+  for (const QString &filename : stealFiles) {

+    if (filename.isEmpty()) {

+      log::warn("Trying to find file with no name");

+      continue;

+    }

+    QFileInfo fileInfo(filename);

+    FileEntryPtr file = directoryStructure->findFile(ToWString(fileInfo.fileName()));

+    if (file.get() != nullptr) {

+      if (file->getOrigin() == 0) {

+        // replace data as the origin on this bsa

+        file->removeOrigin(0);

       }

+      origin.addFile(file->getIndex());

+      file->addOrigin(origin.getID(), file->getFileTime(), L"", -1);

+    } else {

+      QString warnStr = fileInfo.absolutePath();

+      if (warnStr.isEmpty())

+        warnStr = filename;

+      log::warn("file not found: {}", warnStr);

     }

   }

 }

 

-void DirectoryRefresher::addModFilesToStructure(DirectoryEntry *directoryStructure, const QString &modName,

-                                                int priority, const QString &directory, const QStringList &stealFiles)

+void DirectoryRefresher::addModFilesToStructure(

+  DirectoryEntry *directoryStructure, const QString &modName,

+  int priority, const QString &directory, const QStringList &stealFiles)

 {

+  TimeThis tt("addModFilesToStructure()");

+

   std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory));

+  DirectoryStats dummy;

 

   if (stealFiles.length() > 0) {

-    // instead of adding all the files of the target directory, we just change the root of the specified

-    // files to this mod

-    FilesOrigin &origin = directoryStructure->createOrigin(ToWString(modName), directoryW, priority);

-    for (const QString &filename : stealFiles) {

-      if (filename.isEmpty()) {

-        log::warn("Trying to find file with no name");

-        continue;

-      }

-      QFileInfo fileInfo(filename);

-      FileEntry::Ptr file = directoryStructure->findFile(ToWString(fileInfo.fileName()));

-      if (file.get() != nullptr) {

-        if (file->getOrigin() == 0) {

-          // replace data as the origin on this bsa

-          file->removeOrigin(0);

-        }

-        origin.addFile(file->getIndex());

-        file->addOrigin(origin.getID(), file->getFileTime(), L"", -1);

-      } else {

-        QString warnStr = fileInfo.absolutePath();

-        if (warnStr.isEmpty())

-          warnStr = filename;

-        log::warn("file not found: {}", warnStr);

-      }

-    }

+    stealModFilesIntoStructure(

+      directoryStructure, modName, priority, directory, stealFiles);

   } else {

-    directoryStructure->addFromOrigin(ToWString(modName), directoryW, priority);

+    directoryStructure->addFromOrigin(

+      ToWString(modName), directoryW, priority, dummy);

   }

 }

 

 void DirectoryRefresher::addModToStructure(DirectoryEntry *directoryStructure

-                                           , const QString &modName

-                                           , int priority

-                                           , const QString &directory

-                                           , const QStringList &stealFiles

-                                           , const QStringList &archives)

+  , const QString &modName

+  , int priority

+  , const QString &directory

+  , const QStringList &stealFiles

+  , const QStringList &archives)

 {

-  addModFilesToStructure(directoryStructure, modName, priority, directory, stealFiles);

-  

+  TimeThis tt("addModToStructure()");

+

+  DirectoryStats dummy;

+

+  if (stealFiles.length() > 0) {

+    stealModFilesIntoStructure(

+      directoryStructure, modName, priority, directory, stealFiles);

+  } else {

+    std::wstring directoryW = ToWString(QDir::toNativeSeparators(directory));

+    directoryStructure->addFromOrigin(

+      ToWString(modName), directoryW, priority, dummy);

+  }

+

   if (Settings::instance().archiveParsing()) {

-    addModBSAToStructure(directoryStructure, modName, priority, directory, archives);

+    addModBSAToStructure(

+      directoryStructure, modName, priority, directory, archives);

   }

 }

 

-void DirectoryRefresher::refresh()

+

+struct ModThread

 {

-  QMutexLocker locker(&m_RefreshLock);

+  DirectoryRefreshProgress* progress = nullptr;

+  DirectoryEntry* ds = nullptr;

+  std::wstring modName;

+  std::wstring path;

+  int prio = -1;

+  std::vector<std::wstring> archives;

+  std::set<std::wstring> enabledArchives;

+  DirectoryStats* stats =  nullptr;

+  env::DirectoryWalker walker;

 

-  delete m_DirectoryStructure;

+  std::condition_variable cv;

+  std::mutex mutex;

+  bool ready = false;

 

-  m_DirectoryStructure = new DirectoryEntry(L"data", nullptr, 0);

+  void wakeup()

+  {

+    {

+      std::scoped_lock lock(mutex);

+      ready = true;

+    }

+

+    cv.notify_one();

+  }

+

+  void run()

+  {

+    std::unique_lock lock(mutex);

+    cv.wait(lock, [&]{ return ready; });

 

-  IPluginGame *game = qApp->property("managed_game").value<IPluginGame*>();

+    SetThisThreadName(QString::fromStdWString(modName + L" refresher"));

+    ds->addFromOrigin(walker, modName, path, prio, *stats);

 

-  std::wstring dataDirectory = QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString();

-  m_DirectoryStructure->addFromOrigin(L"data", dataDirectory, 0);

+    if (Settings::instance().archiveParsing()) {

+      const IPluginGame *game = qApp->property("managed_game").value<IPluginGame*>();

 

-  std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs){return lhs.priority < rhs.priority;});

-  auto iter = m_Mods.begin();

+      GamePlugins *gamePlugins = game->feature<GamePlugins>();

+      QStringList loadOrder = QStringList();

+      gamePlugins->getLoadOrder(loadOrder);

+

+      std::vector<std::wstring> lo;

+      for (auto&& s : loadOrder) {

+        lo.push_back(s.toStdWString());

+      }

+

+      ds->addFromAllBSAs(

+        modName, path, prio, archives, enabledArchives, lo, *stats);

+    }

 

-  for (int i = 1; iter != m_Mods.end(); ++iter, ++i) {

-    try {

-      addModToStructure(m_DirectoryStructure, iter->modName, i, iter->absolutePath, iter->stealFiles, iter->archives);

-    } catch (const std::exception &e) {

-      emit error(tr("failed to read mod (%1): %2").arg(iter->modName, e.what()));

+    if (progress) {

+      progress->addDone();

     }

-    emit progress((i * 100) / static_cast<int>(m_Mods.size()) + 1);

+

+    SetThisThreadName(QString::fromStdWString(L"idle refresher"));

+    ready = false;

   }

+};

+

+env::ThreadPool<ModThread> g_threads;

+

+

+void DirectoryRefresher::updateProgress(const DirectoryRefreshProgress* p)

+{

+  // careful: called from multiple threads

+  emit progress(p);

+}

 

-  m_DirectoryStructure->getFileRegister()->sortOrigins();

+void DirectoryRefresher::addMultipleModsFilesToStructure(

+  MOShared::DirectoryEntry *directoryStructure,

+  const std::vector<EntryInfo>& entries, DirectoryRefreshProgress* progress)

+{

+  std::vector<DirectoryStats> stats(entries.size());

 

-  emit progress(100);

+  if (progress) {

+    progress->start(entries.size());

+  }

+

+  log::debug("refresher: using {} threads", m_threadCount);

+  g_threads.setMax(m_threadCount);

+

+  for (std::size_t i=0; i<entries.size(); ++i) {

+    const auto& e = entries[i];

+    const int prio = static_cast<int>(i + 1);

+

+    if constexpr (DirectoryStats::EnableInstrumentation) {

+      stats[i].mod = entries[i].modName.toStdString();

+    }

+

+    try

+    {

+      if (e.stealFiles.length() > 0) {

+        stealModFilesIntoStructure(

+          directoryStructure, e.modName, prio, e.absolutePath, e.stealFiles);

+

+        if (progress) {

+          progress->addDone();

+        }

+      } else {

+        auto& mt = g_threads.request();

+

+        mt.progress = progress;

+        mt.ds = directoryStructure;

+        mt.modName = e.modName.toStdWString();

+        mt.path = QDir::toNativeSeparators(e.absolutePath).toStdWString();

+        mt.prio = prio;

+

+        mt.archives.clear();

+        for (auto&& a : e.archives) {

+          mt.archives.push_back(a.toStdWString());

+        }

+

+        mt.enabledArchives.clear();

+        for (auto&& a : m_EnabledArchives) {

+          mt.enabledArchives.insert(a.toStdWString());

+        }

+

+        mt.stats = &stats[i];

+

+        mt.wakeup();

+      }

+    } catch (const std::exception& ex) {

+      emit error(tr("failed to read mod (%1): %2").arg(e.modName, ex.what()));

+    }

+  }

+

+  g_threads.waitForAll();

+

+  if constexpr (DirectoryStats::EnableInstrumentation) {

+    dumpStats(stats);

+  }

+}

+

+void DirectoryRefresher::refresh()

+{

+  SetThisThreadName("DirectoryRefresher");

+  TimeThis tt("refresh");

+  auto* p = new DirectoryRefreshProgress(this);

+

+  {

+    QMutexLocker locker(&m_RefreshLock);

+

+    m_Root.reset(new DirectoryEntry(L"data", nullptr, 0));

+

+    IPluginGame *game = qApp->property("managed_game").value<IPluginGame*>();

+

+    std::wstring dataDirectory =

+      QDir::toNativeSeparators(game->dataDirectory().absolutePath()).toStdWString();

+

+    {

+      DirectoryStats dummy;

+      m_Root->addFromOrigin(L"data", dataDirectory, 0, dummy);

+    }

+

+    std::sort(m_Mods.begin(), m_Mods.end(), [](auto lhs, auto rhs) {

+      return lhs.priority < rhs.priority;

+    });

+

+    addMultipleModsFilesToStructure(m_Root.get(), m_Mods, p);

+

+    m_Root->getFileRegister()->sortOrigins();

+

+    cleanStructure(m_Root.get());

+

+    m_lastFileCount = m_Root->getFileRegister()->highestCount();

+    log::debug("refresher saw {} files", m_lastFileCount);

+  }

 

-  cleanStructure(m_DirectoryStructure);

+  p->finish();

 

+  emit progress(p);

   emit refreshed();

 }

diff --git a/src/directoryrefresher.h b/src/directoryrefresher.h
index a4fc5dbc..bd08dad6 100644
--- a/src/directoryrefresher.h
+++ b/src/directoryrefresher.h
@@ -20,15 +20,14 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #ifndef DIRECTORYREFRESHER_H

 #define DIRECTORYREFRESHER_H

 

-#include <directoryentry.h>

+#include "fileregisterfwd.h"

+#include "profile.h"

 #include <QObject>

 #include <QMutex>

 #include <QStringList>

 #include <vector>

 #include <set>

 #include <tuple>

-#include "profile.h"

-

 

 /**

  * @brief used to asynchronously generate the virtual view of the combined data directory

@@ -39,24 +38,37 @@ class DirectoryRefresher : public QObject
   Q_OBJECT

 

 public:

+  struct EntryInfo

+  {

+    EntryInfo(const QString &modName, const QString &absolutePath,

+      const QStringList &stealFiles, const QStringList &archives, int priority)

+      : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles)

+      , archives(archives), priority(priority)

+    {

+    }

 

-  /**

-   * @brief constructor

-   *

-   **/

-  DirectoryRefresher();

+    QString modName;

+    QString absolutePath;

+    QStringList stealFiles;

+    QStringList archives;

+    int priority;

+  };

+

+  DirectoryRefresher(std::size_t threadCount);

 

-  ~DirectoryRefresher();

+  // noncopyable

+  DirectoryRefresher(const DirectoryRefresher&) = delete;

+  DirectoryRefresher& operator=(const DirectoryRefresher&) = delete;

 

   /**

    * @brief retrieve the updated directory structure

    *

    * returns a pointer to the updated directory structure. DirectoryRefresher

    * deletes its own pointer and the caller takes custody of the pointer

-   * 

+   *

    * @return updated directory structure

    **/

-  MOShared::DirectoryEntry *getDirectoryStructure();

+  MOShared::DirectoryEntry* stealDirectoryStructure();

 

   /**

    * @brief sets up the mods to be included in the directory structure

@@ -107,7 +119,16 @@ public:
    * @param directory

    * @param stealFiles

    */

-  void addModFilesToStructure(MOShared::DirectoryEntry *directoryStructure, const QString &modName, int priority, const QString &directory, const QStringList &stealFiles);

+  void addModFilesToStructure(

+    MOShared::DirectoryEntry *directoryStructure, const QString &modName,

+    int priority, const QString &directory, const QStringList &stealFiles);

+

+  void addMultipleModsFilesToStructure(

+    MOShared::DirectoryEntry *directoryStructure,

+    const std::vector<EntryInfo>& entries,

+    DirectoryRefreshProgress* progress=nullptr);

+

+  void updateProgress(const DirectoryRefreshProgress* p);

 

 public slots:

 

@@ -118,31 +139,76 @@ public slots:
 

 signals:

 

-  void progress(int progress);

+  void progress(const DirectoryRefreshProgress* p);

   void error(const QString &error);

   void refreshed();

 

 private:

-

-  struct EntryInfo {

-    EntryInfo(const QString &modName, const QString &absolutePath,

-              const QStringList &stealFiles, const QStringList &archives, int priority)

-      : modName(modName), absolutePath(absolutePath), stealFiles(stealFiles)

-      , archives(archives), priority(priority) {}

-    QString modName;

-    QString absolutePath;

-    QStringList stealFiles;

-    QStringList archives;

-    int priority;

-  };

-

-private:

-

   std::vector<EntryInfo> m_Mods;

   std::set<QString> m_EnabledArchives;

-  MOShared::DirectoryEntry *m_DirectoryStructure;

+  std::unique_ptr<MOShared::DirectoryEntry> m_Root;

   QMutex m_RefreshLock;

+  std::size_t m_threadCount;

+  std::size_t m_lastFileCount;

 

+  void stealModFilesIntoStructure(

+    MOShared::DirectoryEntry *directoryStructure, const QString &modName,

+    int priority, const QString &directory, const QStringList &stealFiles);

+};

+

+

+class DirectoryRefreshProgress : QObject

+{

+  Q_OBJECT;

+

+public:

+  DirectoryRefreshProgress(DirectoryRefresher* r) :

+    QObject(r), m_refresher(r), m_modCount(0), m_modDone(0), m_finished(false)

+  {

+  }

+

+  void start(std::size_t modCount)

+  {

+    m_modCount = modCount;

+    m_modDone = 0;

+    m_finished = false;

+  }

+

+

+  bool finished() const

+  {

+    return m_finished;

+  }

+

+  int percentDone() const

+  {

+    int percent = 100;

+

+    if (m_modCount > 0) {

+      const double d = static_cast<double>(m_modDone) / m_modCount;

+      percent = static_cast<int>(d * 100);

+    }

+

+    return percent;

+  }

+

+

+  void finish()

+  {

+    m_finished = true;

+  }

+

+  void addDone()

+  {

+    ++m_modDone;

+    m_refresher->updateProgress(this);

+  }

+

+private:

+  DirectoryRefresher* m_refresher;

+  std::size_t m_modCount;

+  std::atomic<std::size_t> m_modDone;

+  bool m_finished;

 };

 

 #endif // DIRECTORYREFRESHER_H

diff --git a/src/envfs.cpp b/src/envfs.cpp
new file mode 100644
index 00000000..a67faf84
--- /dev/null
+++ b/src/envfs.cpp
@@ -0,0 +1,450 @@
+#include "envfs.h"
+#include "env.h"
+#include "util.h"
+#include <utility.h>
+#include <log.h>
+
+using namespace MOBase;
+
+typedef struct _UNICODE_STRING {
+  USHORT Length;
+  USHORT MaximumLength;
+  PWSTR Buffer;
+} UNICODE_STRING, *PUNICODE_STRING;
+typedef const UNICODE_STRING *PCUNICODE_STRING;
+
+typedef struct _OBJECT_ATTRIBUTES {
+  ULONG Length;
+  HANDLE RootDirectory;
+  PUNICODE_STRING ObjectName;
+  ULONG Attributes;
+  PVOID SecurityDescriptor;
+  PVOID SecurityQualityOfService;
+} OBJECT_ATTRIBUTES, *POBJECT_ATTRIBUTES;
+
+
+typedef struct _FILE_DIRECTORY_INFORMATION {
+  ULONG         NextEntryOffset;
+  ULONG         FileIndex;
+  LARGE_INTEGER CreationTime;
+  LARGE_INTEGER LastAccessTime;
+  LARGE_INTEGER LastWriteTime;
+  LARGE_INTEGER ChangeTime;
+  LARGE_INTEGER EndOfFile;
+  LARGE_INTEGER AllocationSize;
+  ULONG         FileAttributes;
+  ULONG         FileNameLength;
+  WCHAR         FileName[1];
+} FILE_DIRECTORY_INFORMATION, *PFILE_DIRECTORY_INFORMATION;
+
+
+#define FILE_SHARE_VALID_FLAGS            0x00000007
+
+// copied from ntstatus.h
+#define STATUS_SUCCESS ((NTSTATUS)0x00000000L)
+#define STATUS_BUFFER_OVERFLOW ((NTSTATUS)0x80000005L)
+#define STATUS_NO_MORE_FILES ((NTSTATUS)0x80000006L)
+#define STATUS_NO_SUCH_FILE ((NTSTATUS)0xC000000FL)
+
+typedef struct _IO_STATUS_BLOCK IO_STATUS_BLOCK;
+
+typedef struct _IO_STATUS_BLOCK *PIO_STATUS_BLOCK;
+// typedef VOID (NTAPI *PIO_APC_ROUTINE )(__in PVOID ApcContext, __in
+// PIO_STATUS_BLOCK IoStatusBlock, __in ULONG Reserved);
+typedef VOID(NTAPI *PIO_APC_ROUTINE)(PVOID ApcContext,
+  PIO_STATUS_BLOCK IoStatusBlock,
+  ULONG Reserved);
+
+
+typedef enum _FILE_INFORMATION_CLASS {
+  FileDirectoryInformation = 1
+} FILE_INFORMATION_CLASS;
+
+typedef NTSTATUS(WINAPI *NtQueryDirectoryFile_type)(
+  HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BLOCK, PVOID, ULONG,
+  FILE_INFORMATION_CLASS, BOOLEAN, PUNICODE_STRING, BOOLEAN);
+
+typedef NTSTATUS(WINAPI *NtOpenFile_type)(PHANDLE, ACCESS_MASK,
+  POBJECT_ATTRIBUTES, PIO_STATUS_BLOCK,
+  ULONG, ULONG);
+
+typedef NTSTATUS(WINAPI *NtClose_type)(HANDLE);
+
+
+NtOpenFile_type NtOpenFile = nullptr;
+NtQueryDirectoryFile_type NtQueryDirectoryFile = nullptr;
+extern NtClose_type NtClose = nullptr;
+
+
+#define FILE_DIRECTORY_FILE                     0x00000001
+#define FILE_WRITE_THROUGH                      0x00000002
+#define FILE_SEQUENTIAL_ONLY                    0x00000004
+#define FILE_NO_INTERMEDIATE_BUFFERING          0x00000008
+
+#define FILE_SYNCHRONOUS_IO_ALERT               0x00000010
+#define FILE_SYNCHRONOUS_IO_NONALERT            0x00000020
+#define FILE_NON_DIRECTORY_FILE                 0x00000040
+#define FILE_CREATE_TREE_CONNECTION             0x00000080
+
+#define FILE_COMPLETE_IF_OPLOCKED               0x00000100
+#define FILE_NO_EA_KNOWLEDGE                    0x00000200
+#define FILE_OPEN_REMOTE_INSTANCE               0x00000400
+#define FILE_RANDOM_ACCESS                      0x00000800
+
+#define FILE_DELETE_ON_CLOSE                    0x00001000
+#define FILE_OPEN_BY_FILE_ID                    0x00002000
+#define FILE_OPEN_FOR_BACKUP_INTENT             0x00004000
+#define FILE_NO_COMPRESSION                     0x00008000
+
+#if (_WIN32_WINNT >= _WIN32_WINNT_WIN7)
+#define FILE_OPEN_REQUIRING_OPLOCK              0x00010000
+#endif
+
+#define FILE_RESERVE_OPFILTER                   0x00100000
+#define FILE_OPEN_REPARSE_POINT                 0x00200000
+#define FILE_OPEN_NO_RECALL                     0x00400000
+#define FILE_OPEN_FOR_FREE_SPACE_QUERY          0x00800000
+
+#define FILE_VALID_OPTION_FLAGS                 0x00ffffff
+#define FILE_VALID_PIPE_OPTION_FLAGS            0x00000032
+#define FILE_VALID_MAILSLOT_OPTION_FLAGS        0x00000032
+#define FILE_VALID_SET_FLAGS                    0x00000036
+
+typedef struct _IO_STATUS_BLOCK {
+#pragma warning(push)
+#pragma warning(disable: 4201) // we'll always use the Microsoft compiler
+  union {
+    NTSTATUS Status;
+    PVOID Pointer;
+  } DUMMYUNIONNAME;
+#pragma warning(pop)
+
+  ULONG_PTR Information;
+} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK;
+
+
+
+namespace env
+{
+
+std::wstring_view toStringView(const UNICODE_STRING* s)
+{
+  if (s && s->Buffer) {
+    return {s->Buffer, (s->Length / sizeof(wchar_t))};
+  } else {
+    return {};
+  }
+}
+
+std::wstring_view toStringView(POBJECT_ATTRIBUTES poa)
+{
+  if (poa->ObjectName) {
+    return toStringView(poa->ObjectName);
+  }
+
+  return {};
+}
+
+QString toString(POBJECT_ATTRIBUTES poa)
+{
+  const auto sv = toStringView(poa);
+  return QString::fromWCharArray(sv.data(), static_cast<int>(sv.size()));
+}
+
+
+class HandleCloserThread
+{
+public:
+  HandleCloserThread()
+    : m_ready(false)
+  {
+    m_handles.reserve(50'000);
+  }
+
+  void shrink()
+  {
+    m_handles.shrink_to_fit();
+  }
+
+  void add(HANDLE h)
+  {
+    m_handles.push_back(h);
+  }
+
+  void wakeup()
+  {
+    m_ready = true;
+    m_cv.notify_one();
+  }
+
+  void run()
+  {
+    MOShared::SetThisThreadName("HandleCloserThread");
+
+    std::unique_lock lock(m_mutex);
+    m_cv.wait(lock, [&]{ return m_ready; });
+
+    closeHandles();
+  }
+
+private:
+  std::vector<HANDLE> m_handles;
+  std::condition_variable m_cv;
+  std::mutex m_mutex;
+  bool m_ready;
+
+  void closeHandles()
+  {
+    for (auto& h : m_handles) {
+      NtClose(h);
+    }
+
+    m_handles.clear();
+    m_ready = false;
+  }
+};
+
+constexpr std::size_t AllocSize = 1024 * 1024;
+static ThreadPool<HandleCloserThread> g_handleClosers;
+
+void setHandleCloserThreadCount(std::size_t n)
+{
+  g_handleClosers.setMax(n);
+}
+
+void forEachEntryImpl(
+  void* cx, HandleCloserThread& hc, std::vector<std::unique_ptr<unsigned char[]>>& buffers,
+  POBJECT_ATTRIBUTES poa, std::size_t depth,
+  DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF)
+{
+  IO_STATUS_BLOCK iosb;
+  UNICODE_STRING ObjectName;
+  OBJECT_ATTRIBUTES oa = { sizeof(oa), 0, &ObjectName };
+  NTSTATUS status;
+
+  status = NtOpenFile(
+    &oa.RootDirectory, FILE_GENERIC_READ, poa, &iosb, FILE_SHARE_VALID_FLAGS,
+    FILE_SYNCHRONOUS_IO_NONALERT|FILE_OPEN_REPARSE_POINT|FILE_OPEN_FOR_BACKUP_INTENT);
+
+  if (status < 0) {
+    log::error(
+      "NtOpenFile() failed for '{}', {}",
+      toString(poa), formatSystemMessage(status));
+
+    return;
+  }
+
+  hc.add(oa.RootDirectory);
+  unsigned char* buffer;
+
+  if (depth >= buffers.size()) {
+    buffers.emplace_back(std::make_unique<unsigned char[]>(AllocSize));
+    buffer = buffers.back().get();
+  } else {
+    buffer = buffers[depth].get();
+  }
+
+  union
+  {
+    PVOID pv;
+    PBYTE pb;
+    PFILE_DIRECTORY_INFORMATION DirInfo;
+  };
+
+  for (;;) {
+    status = NtQueryDirectoryFile(
+      oa.RootDirectory, NULL, NULL, NULL, &iosb,
+      buffer, AllocSize, FileDirectoryInformation, FALSE, NULL, FALSE);
+
+    if (status == STATUS_NO_MORE_FILES) {
+      break;
+    } else if (status < 0) {
+      log::error(
+        "NtQueryDirectoryFile() failed for '{}', {}",
+        toString(poa), formatSystemMessage(status));
+      break;
+    }
+
+    ULONG NextEntryOffset = 0;
+
+    pv = buffer;
+
+    auto isDotDir = [](auto* o) {
+      if (o->Length == 2 && o->Buffer[0] == '.') {
+        return true;
+      }
+
+      if (o->Length == 4 && o->Buffer[0] == '.' && o->Buffer[1] == '.') {
+        return true;
+      }
+
+      return false;
+    };
+
+    std::size_t count = 0;
+
+    for (;;) {
+      ++count;
+      pb += NextEntryOffset;
+
+      ObjectName.Buffer = DirInfo->FileName;
+      ObjectName.Length = (USHORT)DirInfo->FileNameLength;
+
+      if (!isDotDir(&ObjectName)) {
+        ObjectName.MaximumLength = ObjectName.Length;
+
+        if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+          dirStartF(cx, toStringView(&oa));
+          forEachEntryImpl(cx, hc, buffers, &oa, depth+1, dirStartF, dirEndF, fileF);
+          dirEndF(cx, toStringView(&oa));
+        } else {
+          FILETIME ft;
+          ft.dwLowDateTime = DirInfo->LastWriteTime.LowPart;
+          ft.dwHighDateTime = DirInfo->LastWriteTime.HighPart;
+          fileF(cx, toStringView(&oa), ft);
+        }
+      }
+
+      NextEntryOffset = DirInfo->NextEntryOffset;
+
+      if (NextEntryOffset == 0) {
+        break;
+      }
+    }
+
+    if (AllocSize - iosb.Information > (ULONG)FIELD_OFFSET(FILE_DIRECTORY_INFORMATION, FileName[256])) {
+      // NO_MORE_FILES
+      break;
+    }
+  }
+}
+
+
+void DirectoryWalker::forEachEntry(
+  const std::wstring& path, void* cx,
+  DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF)
+{
+  auto& hc = g_handleClosers.request();
+
+  if (!NtOpenFile) {
+    LibraryPtr m(::LoadLibraryW(L"ntdll.dll"));
+    NtOpenFile = (NtOpenFile_type)::GetProcAddress(m.get(), "NtOpenFile");
+    NtQueryDirectoryFile = (NtQueryDirectoryFile_type)::GetProcAddress(m.get(), "NtQueryDirectoryFile");
+    NtClose = (NtClose_type)::GetProcAddress(m.get(), "NtClose");
+  }
+
+  const std::wstring ntpath = std::wstring(L"\\??\\") + path;
+
+  UNICODE_STRING ObjectName = {};
+  ObjectName.Buffer = const_cast<wchar_t*>(ntpath.c_str());
+  ObjectName.Length = (USHORT)ntpath.size() * sizeof(wchar_t);
+  ObjectName.MaximumLength = ObjectName.Length;
+
+  OBJECT_ATTRIBUTES oa = {};
+  oa.Length = sizeof(oa);
+  oa.ObjectName = &ObjectName;
+
+  forEachEntryImpl(cx, hc, m_buffers, &oa, 0, dirStartF, dirEndF, fileF);
+  hc.wakeup();
+}
+
+
+void forEachEntry(
+  const std::wstring& path, void* cx,
+  DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF)
+{
+  DirectoryWalker().forEachEntry(path, cx, dirStartF, dirEndF, fileF);
+}
+
+Directory getFilesAndDirs(const std::wstring& path)
+{
+  struct Context
+  {
+    std::stack<Directory*> current;
+  };
+
+  Directory root;
+
+  Context cx;
+  cx.current.push(&root);
+
+  env::forEachEntry(path, &cx,
+    [](void* pcx, std::wstring_view path) {
+      Context* cx = (Context*)pcx;
+
+      cx->current.top()->dirs.push_back(Directory(path));
+      cx->current.push(&cx->current.top()->dirs.back());
+    },
+
+    [](void* pcx, std::wstring_view path) {
+      Context* cx = (Context*)pcx;
+      cx->current.pop();
+    },
+
+    [](void* pcx, std::wstring_view path, FILETIME ft) {
+      Context* cx = (Context*)pcx;
+
+      cx->current.top()->files.push_back(File(path, ft));
+    }
+  );
+
+  return root;
+}
+
+File::File(std::wstring_view n, FILETIME ft) :
+  name(n.begin(), n.end()),
+  lcname(MOShared::ToLowerCopy(name)),
+  lastModified(ft)
+{
+}
+
+Directory::Directory()
+{
+}
+
+Directory::Directory(std::wstring_view n)
+  : name(n.begin(), n.end()), lcname(MOShared::ToLowerCopy(name))
+{
+}
+
+
+void getFilesAndDirsWithFindImpl(const std::wstring& path, Directory& d)
+{
+  const std::wstring searchString = path + L"\\*";
+
+  WIN32_FIND_DATAW findData;
+
+  HANDLE searchHandle = ::FindFirstFileExW(
+    searchString.c_str(), FindExInfoBasic, &findData, FindExSearchNameMatch,
+    nullptr, FIND_FIRST_EX_LARGE_FETCH);
+
+  if (searchHandle != INVALID_HANDLE_VALUE) {
+    BOOL result = true;
+
+    while (result) {
+      if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+        if ((wcscmp(findData.cFileName, L".") != 0) &&
+          (wcscmp(findData.cFileName, L"..") != 0)) {
+          const std::wstring newPath = path + L"\\" + findData.cFileName;
+          d.dirs.push_back(Directory(findData.cFileName));
+          getFilesAndDirsWithFindImpl(newPath, d.dirs.back());
+        }
+      } else {
+        d.files.push_back(File(findData.cFileName, findData.ftLastWriteTime));
+      }
+
+      result = ::FindNextFileW(searchHandle, &findData);
+    }
+  }
+
+  ::FindClose(searchHandle);
+}
+
+Directory getFilesAndDirsWithFind(const std::wstring& path)
+{
+  Directory d;
+  getFilesAndDirsWithFindImpl(path, d);
+  return d;
+}
+
+} // namespace
diff --git a/src/envfs.h b/src/envfs.h
new file mode 100644
index 00000000..8790b071
--- /dev/null
+++ b/src/envfs.h
@@ -0,0 +1,193 @@
+#ifndef ENV_ENVFS_H
+#define ENV_ENVFS_H
+
+#include <thread>
+
+namespace env
+{
+
+struct File
+{
+  std::wstring name;
+  std::wstring lcname;
+  FILETIME lastModified;
+
+  File(std::wstring_view name, FILETIME ft);
+};
+
+struct Directory
+{
+  std::wstring name;
+  std::wstring lcname;
+
+  std::vector<Directory> dirs;
+  std::vector<File> files;
+
+  Directory();
+  Directory(std::wstring_view name);
+};
+
+
+template <class T>
+class ThreadPool
+{
+public:
+  ThreadPool(std::size_t max=1)
+  {
+    setMax(max);
+  }
+
+  ~ThreadPool()
+  {
+    stopAndJoin();
+  }
+
+  void setMax(std::size_t n)
+  {
+    m_threads.resize(n);
+  }
+
+  void stopAndJoin()
+  {
+    for (auto& ti : m_threads) {
+      ti.stop = true;
+      ti.wakeup();
+    }
+
+    for (auto& ti : m_threads) {
+      if (ti.thread.joinable()) {
+        ti.thread.join();
+      }
+    }
+  }
+
+  void waitForAll()
+  {
+    for (;;) {
+      bool done = true;
+
+      for (auto& ti : m_threads) {
+        if (ti.busy) {
+          done = false;
+          break;
+        }
+      }
+
+      if (done) {
+        break;
+      }
+
+      std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+  }
+
+  T& request()
+  {
+    if (m_threads.empty()) {
+      std::terminate();
+    }
+
+    for (;;) {
+      for (auto& ti : m_threads) {
+        bool expected = false;
+
+        if (ti.busy.compare_exchange_strong(expected, true)) {
+          ti.wakeup();
+          return ti.o;
+        }
+      }
+
+      std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+  }
+
+  template <class F>
+  void forEach(F&& f)
+  {
+    for (auto& ti : m_threads) {
+      f(ti.o);
+    }
+  }
+
+private:
+  struct ThreadInfo
+  {
+    std::thread thread;
+    std::atomic<bool> busy;
+    T o;
+
+    std::condition_variable cv;
+    std::mutex mutex;
+    bool ready;
+
+    std::atomic<bool> stop;
+
+    ThreadInfo()
+      : busy(true), ready(false), stop(false)
+    {
+      thread = std::thread([&]{ run(); });
+    }
+
+    void wakeup()
+    {
+      {
+        std::scoped_lock lock(mutex);
+        ready = true;
+      }
+
+      cv.notify_one();
+    }
+
+    void run()
+    {
+      busy = false;
+
+      while (!stop) {
+        std::unique_lock lock(mutex);
+        cv.wait(lock, [&]{ return ready; });
+
+        if (stop) {
+          break;
+        }
+
+        o.run();
+
+        ready = false;
+        busy = false;
+      }
+    }
+  };
+
+  std::list<ThreadInfo> m_threads;
+};
+
+
+using DirStartF = void (void*, std::wstring_view);
+using DirEndF = void (void*, std::wstring_view);
+using FileF = void (void*, std::wstring_view, FILETIME);
+
+void setHandleCloserThreadCount(std::size_t n);
+
+
+class DirectoryWalker
+{
+public:
+  void forEachEntry(
+    const std::wstring& path, void* cx,
+    DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF);
+
+private:
+  std::vector<std::unique_ptr<unsigned char[]>> m_buffers;
+};
+
+
+void forEachEntry(
+  const std::wstring& path, void* cx,
+  DirStartF* dirStartF, DirEndF* dirEndF, FileF* fileF);
+
+Directory getFilesAndDirs(const std::wstring& path);
+Directory getFilesAndDirsWithFind(const std::wstring& path);
+
+} // namespace
+
+#endif // ENV_ENVFS_H
diff --git a/src/filetree.cpp b/src/filetree.cpp
index 19f2555a..cdcf2feb 100644
--- a/src/filetree.cpp
+++ b/src/filetree.cpp
@@ -3,7 +3,11 @@
 #include "filetreeitem.h"
 #include "organizercore.h"
 #include "envshell.h"
+#include "shared/fileentry.h"
+#include "shared/directoryentry.h"
+#include "shared/filesorigin.h"
 #include <log.h>
+#include <widgetutility.h>
 
 using namespace MOShared;
 using namespace MOBase;
@@ -124,6 +128,8 @@ FileTree::FileTree(OrganizerCore& core, PluginContainer& pc, QTreeView* tree)
   m_tree->setModel(m_model);
   m_tree->header()->resizeSection(0, 200);
 
+  MOBase::setCustomizableColumns(m_tree);
+
   connect(
     m_tree, &QTreeView::customContextMenuRequested,
     [&](auto pos){ onContextMenu(pos); });
@@ -435,69 +441,7 @@ void FileTree::dumpToFile() const
     return;
   }
 
-  QFile out(file);
-
-  if (!out.open(QIODevice::WriteOnly)) {
-    QMessageBox::critical(
-      m_tree->window(), tr("Error"), tr("Failed to open file '%1': %2")
-      .arg(file)
-      .arg(out.errorString()));
-
-    return;
-  }
-
-  try
-  {
-    dumpToFile(out, "Data", *m_core.directoryStructure());
-  }
-  catch(DumpFailed&)
-  {
-    // try to remove it silently
-    if (out.exists()) {
-      if (!out.remove()) {
-        log::error("failed to remove '{}', ignoring", file);
-      }
-    }
-  }
-}
-
-void FileTree::dumpToFile(
-  QFile& out, const QString& parentPath, const DirectoryEntry& entry) const
-{
-  entry.forEachFile([&](auto&& file) {
-    bool isArchive = false;
-    const int originID = file.getOrigin(isArchive);
-
-    if (isArchive) {
-      // TODO: don't list files from archives. maybe make this an option?
-      return true;
-    }
-
-    const auto& origin = m_core.directoryStructure()->getOriginByID(originID);
-    const auto originName = QString::fromStdWString(origin.getName());
-
-    const QString path =
-      parentPath + "\\" + QString::fromStdWString(file.getName());
-
-    if (out.write(path.toUtf8() + "\t(" + originName.toUtf8() + ")\r\n") == -1) {
-      QMessageBox::critical(
-        m_tree->window(), tr("Error"), tr("Failed to write to file %1: %2")
-          .arg(out.fileName())
-          .arg(out.errorString()));
-
-      throw DumpFailed();
-    }
-
-    return true;
-  });
-
-  entry.forEachDirectory([&](auto&& dir) {
-    const auto newParentPath =
-      parentPath + "\\" + QString::fromStdWString(dir.getName());
-
-    dumpToFile(out, newParentPath, dir);
-    return true;
-  });
+  m_core.directoryStructure()->dump(file.toStdWString());
 }
 
 void FileTree::onExpandedChanged(const QModelIndex& index, bool expanded)
@@ -522,6 +466,21 @@ void FileTree::onContextMenu(const QPoint &pos)
   const auto m = QApplication::keyboardModifiers();
 
   if (m & Qt::ShiftModifier) {
+    // shift+right-click won't select individual items because it interferes
+    // with regular shift selection; this makes it behave like explorer:
+    //   - if the right-clicked item is currently part of the selection, show
+    //     the menu for the selection
+    //   - if not, select this item only and show the menu for it
+    const auto index = m_tree->indexAt(pos);
+
+    if (!m_tree->selectionModel()->isSelected(index)) {
+      m_tree->selectionModel()->select(
+        index,
+        QItemSelectionModel::ClearAndSelect |
+        QItemSelectionModel::Rows |
+        QItemSelectionModel::Current);
+    }
+
     // if no shell menu was available, continue on and show the regular
     // context menu
     if (showShellMenu(pos)) {
@@ -604,6 +563,12 @@ bool FileTree::showShellMenu(QPoint pos)
       itor = menus.emplace(item->originID(), mw).first;
     }
 
+    if (!QFile::exists(item->realPath())) {
+      log::error("{}",
+        tr("File '%1' does not exist, you may need to refresh.")
+        .arg(item->realPath()));
+    }
+
     itor->second.addFile(item->realPath());
     ++totalFiles;
 
@@ -641,6 +606,12 @@ bool FileTree::showShellMenu(QPoint pos)
           continue;
         }
 
+        if (!QFile::exists(QString::fromStdWString(fullPath))) {
+          log::error("{}",
+            tr("File '%1' does not exist, you may need to refresh.")
+            .arg(QString::fromStdWString(fullPath)));
+        }
+
         itor->second.addFile(QString::fromStdWString(fullPath));
       }
     }
diff --git a/src/filetree.h b/src/filetree.h
index c1458179..2669e53b 100644
--- a/src/filetree.h
+++ b/src/filetree.h
@@ -65,10 +65,6 @@ private:
   void toggleVisibility(bool b, FileTreeItem* item=nullptr);
 
   QModelIndex proxiedIndex(const QModelIndex& index);
-
-  void dumpToFile(
-    QFile& out, const QString& parentPath,
-    const MOShared::DirectoryEntry& entry) const;
 };
 
 #endif // MODORGANIZER_FILETREE_INCLUDED
diff --git a/src/filetreeitem.h b/src/filetreeitem.h
index 390d5499..2092782e 100644
--- a/src/filetreeitem.h
+++ b/src/filetreeitem.h
@@ -1,7 +1,7 @@
 #ifndef MODORGANIZER_FILETREEITEM_INCLUDED
 #define MODORGANIZER_FILETREEITEM_INCLUDED
 
-#include "directoryentry.h"
+#include "shared/fileregisterfwd.h"
 #include <QFileIconProvider>
 
 class FileTreeModel;
@@ -126,7 +126,7 @@ public:
     return m_wsLcFile;
   }
 
-  const MOShared::DirectoryEntry::FileKey& key() const
+  const MOShared::DirectoryEntryFileKey& key() const
   {
     return m_key;
   }
@@ -288,7 +288,7 @@ private:
 
   const QString m_virtualParentPath;
   const std::wstring m_wsFile, m_wsLcFile;
-  const MOShared::DirectoryEntry::FileKey m_key;
+  const MOShared::DirectoryEntryFileKey m_key;
   const QString m_file;
   const bool m_isDirectory;
 
diff --git a/src/filetreemodel.cpp b/src/filetreemodel.cpp
index 2130bf89..1a5433c5 100644
--- a/src/filetreemodel.cpp
+++ b/src/filetreemodel.cpp
@@ -1,5 +1,9 @@
 #include "filetreemodel.h"
 #include "organizercore.h"
+#include "filesorigin.h"
+#include "util.h"
+#include "shared/directoryentry.h"
+#include "shared/fileentry.h"
 #include <log.h>
 
 using namespace MOBase;
@@ -637,7 +641,7 @@ bool FileTreeModel::updateFiles(
   // removeDisappearingFiles() will add files that are in the tree and still on
   // the filesystem to this set; addNewFiless() will use this to figure out if
   // a file is new or not
-  std::unordered_set<FileEntry::Index> seen;
+  std::unordered_set<FileIndex> seen;
 
   int firstFileRow = 0;
 
@@ -647,7 +651,7 @@ bool FileTreeModel::updateFiles(
 
 void FileTreeModel::removeDisappearingFiles(
   FileTreeItem& parentItem, const MOShared::DirectoryEntry& parentEntry,
-  int& firstFileRow, std::unordered_set<FileEntry::Index>& seen)
+  int& firstFileRow, std::unordered_set<FileIndex>& seen)
 {
   auto& children = parentItem.children();
   auto itor = children.begin();
@@ -708,7 +712,7 @@ void FileTreeModel::removeDisappearingFiles(
 bool FileTreeModel::addNewFiles(
   FileTreeItem& parentItem, const MOShared::DirectoryEntry& parentEntry,
   const std::wstring& parentPath, const int firstFileRow,
-  const std::unordered_set<FileEntry::Index>& seen)
+  const std::unordered_set<FileIndex>& seen)
 {
   // keeps track of the contiguous files that need to be added to
   // avoid calling beginAddRows(), etc. for each item
@@ -958,7 +962,7 @@ std::wstring FileTreeModel::makeModName(
 {
   static const std::wstring Unmanaged = UnmanagedModName().toStdWString();
 
-  const auto origin = m_core.directoryStructure()->getOriginByID(originID);
+  const auto& origin = m_core.directoryStructure()->getOriginByID(originID);
 
   if (origin.getID() == 0) {
     return Unmanaged;
diff --git a/src/filetreemodel.h b/src/filetreemodel.h
index 093407b0..5bfb75aa 100644
--- a/src/filetreemodel.h
+++ b/src/filetreemodel.h
@@ -3,7 +3,7 @@
 
 #include "filetreeitem.h"
 #include "iconfetcher.h"
-#include "directoryentry.h"
+#include "shared/fileregisterfwd.h"
 #include <unordered_set>
 
 class OrganizerCore;
@@ -126,12 +126,12 @@ private:
 
   void removeDisappearingFiles(
     FileTreeItem& parentItem, const MOShared::DirectoryEntry& parentEntry,
-    int& firstFileRow, std::unordered_set<MOShared::FileEntry::Index>& seen);
+    int& firstFileRow, std::unordered_set<MOShared::FileIndex>& seen);
 
   bool addNewFiles(
     FileTreeItem& parentItem, const MOShared::DirectoryEntry& parentEntry,
     const std::wstring& parentPath, int firstFileRow,
-    const std::unordered_set<MOShared::FileEntry::Index>& seen);
+    const std::unordered_set<MOShared::FileIndex>& seen);
 
 
   FileTreeItem::Ptr createDirectoryItem(
diff --git a/src/loglist.h b/src/loglist.h
index 56b95d65..b26f1561 100644
--- a/src/loglist.h
+++ b/src/loglist.h
@@ -20,8 +20,9 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #ifndef LOGBUFFER_H

 #define LOGBUFFER_H

 

-#include <QTreeView>

 #include <log.h>

+#include <QTreeView>

+#include <deque>

 

 class OrganizerCore;

 

diff --git a/src/main.cpp b/src/main.cpp
index 4b291e75..105299a8 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -48,6 +48,7 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "organizercore.h"
 #include "env.h"
 #include "envmodule.h"
+#include "util.h"
 
 #include <eh.h>
 #include <windows_error.h>
@@ -885,6 +886,8 @@ int main(int argc, char *argv[])
   MOApplication application(argc, argv);
   QStringList arguments = application.arguments();
 
+  SetThisThreadName("main");
+
   setupPath();
 
   bool forcePrimary = false;
diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp
index 8f2dd250..21a6fa41 100644
--- a/src/mainwindow.cpp
+++ b/src/mainwindow.cpp
@@ -20,8 +20,6 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "mainwindow.h"
 #include "ui_mainwindow.h"
 
-#include "directoryentry.h"
-#include "directoryrefresher.h"
 #include "executableinfo.h"
 #include "executableslist.h"
 #include "guessedvalue.h"
@@ -89,6 +87,11 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "envshortcut.h"
 #include "browserdialog.h"
 
+#include "directoryrefresher.h"
+#include "shared/directoryentry.h"
+#include "shared/fileentry.h"
+#include "shared/filesorigin.h"
+
 #include <QAbstractItemDelegate>
 #include <QAbstractProxyModel>
 #include <QAction>
@@ -364,7 +367,10 @@ MainWindow::MainWindow(Settings &settings
   connect(ui->espFilterEdit, SIGNAL(textChanged(QString)), this, SLOT(espFilterChanged(QString)));
 
   connect(m_OrganizerCore.directoryRefresher(), SIGNAL(refreshed()), this, SLOT(directory_refreshed()));
-  connect(m_OrganizerCore.directoryRefresher(), SIGNAL(progress(int)), this, SLOT(refresher_progress(int)));
+  connect(
+    m_OrganizerCore.directoryRefresher(),
+    &DirectoryRefresher::progress,
+    this, &MainWindow::refresherProgress);
   connect(m_OrganizerCore.directoryRefresher(), SIGNAL(error(QString)), this, SLOT(showError(QString)));
 
   connect(&m_SavesWatcher, SIGNAL(directoryChanged(QString)), this, SLOT(refreshSavesIfOpen()));
@@ -1870,7 +1876,7 @@ void MainWindow::updateBSAList(const QStringList &defaultArchives, const QString
   std::vector<std::pair<UINT32, QTreeWidgetItem*>> items;
 
   BSAInvalidation * invalidation = m_OrganizerCore.managedGame()->feature<BSAInvalidation>();
-  std::vector<FileEntry::Ptr> files = m_OrganizerCore.directoryStructure()->getFiles();
+  std::vector<FileEntryPtr> files = m_OrganizerCore.directoryStructure()->getFiles();
 
   QStringList plugins = m_OrganizerCore.findFiles("", [](const QString &fileName) -> bool {
     return fileName.endsWith(".esp", Qt::CaseInsensitive)
@@ -1889,7 +1895,7 @@ void MainWindow::updateBSAList(const QStringList &defaultArchives, const QString
     return false;
   };
 
-  for (FileEntry::Ptr current : files) {
+  for (FileEntryPtr current : files) {
     QFileInfo fileInfo(ToQString(current->getName().c_str()));
 
     if (fileInfo.suffix().toLower() == "bsa" || fileInfo.suffix().toLower() == "ba2") {
@@ -1942,7 +1948,7 @@ void MainWindow::updateBSAList(const QStringList &defaultArchives, const QString
   for (auto iter = items.begin(); iter != items.end(); ++iter) {
     int originID = iter->second->data(1, Qt::UserRole).toInt();
 
-    FilesOrigin origin = m_OrganizerCore.directoryStructure()->getOriginByID(originID);
+    const FilesOrigin& origin = m_OrganizerCore.directoryStructure()->getOriginByID(originID);
 
     QString modName;
     const unsigned int modIndex = ModInfo::getIndex(ToQString(origin.getName()));
@@ -2075,6 +2081,8 @@ void MainWindow::readSettings()
   s.geometry().restoreVisibility(ui->menuBar);
   s.geometry().restoreVisibility(ui->statusBar);
 
+  FilterWidget::setOptions(s.interface().filterOptions());
+
   {
     // special case in case someone puts 0 in the INI
     auto v = s.widgets().index(ui->executablesListBox);
@@ -2159,6 +2167,8 @@ void MainWindow::storeSettings()
 
   m_Filters->saveState(s);
   m_DataTab->saveState(s);
+
+  s.interface().setFilterOptions(FilterWidget::options());
 }
 
 QWidget* MainWindow::qtWidget()
@@ -2375,10 +2385,15 @@ void MainWindow::setESPListSorting(int index)
   }
 }
 
-void MainWindow::refresher_progress(int percent)
+void MainWindow::refresherProgress(const DirectoryRefreshProgress* p)
 {
-  setEnabled(percent == 100);
-  ui->statusBar->setProgress(percent);
+  if (p->finished()) {
+    setEnabled(true);
+    ui->statusBar->setProgress(100);
+  } else {
+    setEnabled(false);
+    ui->statusBar->setProgress(p->percentDone());
+  }
 }
 
 void MainWindow::directory_refreshed()
@@ -2500,7 +2515,7 @@ void MainWindow::modRenamed(const QString &oldName, const QString &newName)
 
 void MainWindow::fileMoved(const QString &filePath, const QString &oldOriginName, const QString &newOriginName)
 {
-  const FileEntry::Ptr filePtr = m_OrganizerCore.directoryStructure()->findFile(ToWString(filePath));
+  const FileEntryPtr filePtr = m_OrganizerCore.directoryStructure()->findFile(ToWString(filePath));
   if (filePtr.get() != nullptr) {
     try {
       if (m_OrganizerCore.directoryStructure()->originExists(ToWString(newOriginName))) {
@@ -4112,9 +4127,7 @@ void MainWindow::saveArchiveList()
         }
       }
     }
-    if (archiveFile.commitIfDifferent(m_ArchiveListHash)) {
-      log::debug("{} saved", QDir::toNativeSeparators(m_OrganizerCore.currentProfile()->getArchivesFileName()));
-    }
+    archiveFile.commitIfDifferent(m_ArchiveListHash);
   } else {
     log::warn("archive list not initialised");
   }
@@ -5144,7 +5157,11 @@ void MainWindow::originModified(int originID)
 {
   FilesOrigin &origin = m_OrganizerCore.directoryStructure()->getOriginByID(originID);
   origin.enable(false);
-  m_OrganizerCore.directoryStructure()->addFromOrigin(origin.getName(), origin.getPath(), origin.getPriority());
+
+  DirectoryStats dummy;
+  m_OrganizerCore.directoryStructure()->addFromOrigin(
+    origin.getName(), origin.getPath(), origin.getPriority(), dummy);
+
   DirectoryRefresher::cleanStructure(m_OrganizerCore.directoryStructure());
 }
 
@@ -5814,38 +5831,6 @@ void MainWindow::extractBSATriggered()
   }
 }
 
-
-void MainWindow::displayColumnSelection(const QPoint &pos)
-{
-  QMenu menu;
-
-  // display a list of all headers as checkboxes
-  QAbstractItemModel *model = ui->modList->header()->model();
-  for (int i = 1; i < model->columnCount(); ++i) {
-    QString columnName = model->headerData(i, Qt::Horizontal).toString();
-    QCheckBox *checkBox = new QCheckBox(&menu);
-    checkBox->setText(columnName);
-    checkBox->setChecked(!ui->modList->header()->isSectionHidden(i));
-    QWidgetAction *checkableAction = new QWidgetAction(&menu);
-    checkableAction->setDefaultWidget(checkBox);
-    menu.addAction(checkableAction);
-  }
-  menu.exec(pos);
-
-  // view/hide columns depending on check-state
-  int i = 1;
-  for (const QAction *action : menu.actions()) {
-    const QWidgetAction *widgetAction = qobject_cast<const QWidgetAction*>(action);
-    if (widgetAction != nullptr) {
-      const QCheckBox *checkBox = qobject_cast<const QCheckBox*>(widgetAction->defaultWidget());
-      if (checkBox != nullptr) {
-        ui->modList->header()->setSectionHidden(i, !checkBox->isChecked());
-      }
-    }
-    ++i;
-  }
-}
-
 void MainWindow::on_bsaList_customContextMenuRequested(const QPoint &pos)
 {
   m_ContextItem = ui->bsaList->itemAt(pos);
diff --git a/src/mainwindow.h b/src/mainwindow.h
index 1016c061..b839e85e 100644
--- a/src/mainwindow.h
+++ b/src/mainwindow.h
@@ -31,6 +31,7 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "tutorialcontrol.h"
 #include "plugincontainer.h" //class PluginContainer;
 #include "iplugingame.h" //namespace MOBase { class IPluginGame; }
+#include "shared/fileregisterfwd.h"
 #include <log.h>
 
 class Executable;
@@ -153,12 +154,10 @@ public:
   ModInfo::Ptr previousModInList();
 
 public slots:
-
-  void displayColumnSelection(const QPoint &pos);
-
   void modorder_changed();
   void esplist_changed();
-  void refresher_progress(int percent);
+  void refresherProgress(const DirectoryRefreshProgress* p);
+
   void directory_refreshed();
 
   void toolPluginInvoke();
diff --git a/src/modinfodialog.cpp b/src/modinfodialog.cpp
index bfb8d2c7..c3239e0d 100644
--- a/src/modinfodialog.cpp
+++ b/src/modinfodialog.cpp
@@ -29,6 +29,8 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "modinfodialogcategories.h"
 #include "modinfodialognexus.h"
 #include "modinfodialogfiletree.h"
+#include "shared/directoryentry.h"
+#include "shared/filesorigin.h"
 #include <filesystem>
 
 using namespace MOBase;
diff --git a/src/modinfodialogconflicts.cpp b/src/modinfodialogconflicts.cpp
index 63d89a1a..b61dbb7b 100644
--- a/src/modinfodialogconflicts.cpp
+++ b/src/modinfodialogconflicts.cpp
@@ -4,6 +4,9 @@
 #include "utility.h"
 #include "settings.h"
 #include "organizercore.h"
+#include "shared/directoryentry.h"
+#include "shared/filesorigin.h"
+#include "shared/fileentry.h"
 
 using namespace MOShared;
 using namespace MOBase;
@@ -17,7 +20,7 @@ class ConflictItem
 public:
   ConflictItem(
     QString before, QString relativeName, QString after,
-    FileEntry::Index index,  QString fileName,
+    FileIndex index,  QString fileName,
     bool hasAltOrigins, QString altOrigin,  bool archive) :
       m_before(std::move(before)),
       m_relativeName(std::move(relativeName)),
@@ -65,7 +68,7 @@ public:
     return m_isArchive;
   }
 
-  FileEntry::Index fileIndex() const
+  FileIndex fileIndex() const
   {
     return m_index;
   }
@@ -104,7 +107,7 @@ private:
   QString m_before;
   QString m_relativeName;
   QString m_after;
-  FileEntry::Index m_index;
+  FileIndex m_index;
   QString m_fileName;
   bool m_hasAltOrigins;
   QString m_altOrigin;
@@ -1006,8 +1009,8 @@ bool GeneralConflictsTab::update()
 }
 
 ConflictItem GeneralConflictsTab::createOverwriteItem(
-  FileEntry::Index index, bool archive, QString fileName, QString relativeName,
-  const FileEntry::AlternativesVector& alternatives)
+  FileIndex index, bool archive, QString fileName, QString relativeName,
+  const MOShared::AlternativesVector& alternatives)
 {
   const auto& ds = *m_core.directoryStructure();
   std::wstring altString;
@@ -1028,7 +1031,7 @@ ConflictItem GeneralConflictsTab::createOverwriteItem(
 }
 
 ConflictItem GeneralConflictsTab::createNoConflictItem(
-  FileEntry::Index index, bool archive, QString fileName, QString relativeName)
+  FileIndex index, bool archive, QString fileName, QString relativeName)
 {
   return ConflictItem(
     QString(), std::move(relativeName), QString(), index,
@@ -1036,7 +1039,7 @@ ConflictItem GeneralConflictsTab::createNoConflictItem(
 }
 
 ConflictItem GeneralConflictsTab::createOverwrittenItem(
-  FileEntry::Index index, int fileOrigin, bool archive,
+  FileIndex index, int fileOrigin, bool archive,
   QString fileName, QString relativeName)
 {
   const auto& ds = *m_core.directoryStructure();
@@ -1205,9 +1208,9 @@ void AdvancedConflictsTab::update()
 }
 
 std::optional<ConflictItem> AdvancedConflictsTab::createItem(
-  FileEntry::Index index, int fileOrigin, bool archive,
+  FileIndex index, int fileOrigin, bool archive,
   QString fileName, QString relativeName,
-  const MOShared::FileEntry::AlternativesVector& alternatives)
+  const MOShared::AlternativesVector& alternatives)
 {
   const auto& ds = *m_core.directoryStructure();
 
diff --git a/src/modinfodialogconflicts.h b/src/modinfodialogconflicts.h
index c4845019..945f464d 100644
--- a/src/modinfodialogconflicts.h
+++ b/src/modinfodialogconflicts.h
@@ -4,7 +4,7 @@
 #include "modinfodialogtab.h"
 #include "expanderwidget.h"
 #include "filterwidget.h"
-#include "directoryentry.h"
+#include "shared/fileregisterfwd.h"
 #include <QTreeWidget>
 #include <optional>
 
@@ -52,16 +52,16 @@ private:
   FilterWidget m_filterNoConflicts;
 
   ConflictItem createOverwriteItem(
-    MOShared::FileEntry::Index index, bool archive,
+    MOShared::FileIndex index, bool archive,
     QString fileName, QString relativeName,
-    const MOShared::FileEntry::AlternativesVector& alternatives);
+    const MOShared::AlternativesVector& alternatives);
 
   ConflictItem createNoConflictItem(
-    MOShared::FileEntry::Index index, bool archive,
+    MOShared::FileIndex index, bool archive,
     QString fileName, QString relativeName);
 
   ConflictItem createOverwrittenItem(
-    MOShared::FileEntry::Index index, int fileOrigin, bool archive,
+    MOShared::FileIndex index, int fileOrigin, bool archive,
     QString fileName, QString relativeName);
 
   void onOverwriteActivated(const QModelIndex& index);
@@ -94,9 +94,9 @@ private:
   ConflictListModel* m_model;
 
   std::optional<ConflictItem> createItem(
-    MOShared::FileEntry::Index index, int fileOrigin, bool archive,
+    MOShared::FileIndex index, int fileOrigin, bool archive,
     QString fileName, QString relativeName,
-    const MOShared::FileEntry::AlternativesVector& alternatives);
+    const MOShared::AlternativesVector& alternatives);
 };
 
 
diff --git a/src/modinfodialogimages.cpp b/src/modinfodialogimages.cpp
index 48b3bb3b..8a22045e 100644
--- a/src/modinfodialogimages.cpp
+++ b/src/modinfodialogimages.cpp
@@ -186,11 +186,11 @@ void ImagesTab::switchToFiltered()
 
   const bool hasTextFilter = !m_filter.empty();
 
-  for (auto& f : m_files.allFiles()) {
+  for (File& f : m_files.allFiles()) {
     if (hasTextFilter) {
       // check filter widget
-      const auto m = m_filter.matches([&](auto&& what) {
-        return f.path().contains(what, Qt::CaseInsensitive);
+      const auto m = m_filter.matches([&](const QRegularExpression& regex) {
+        return regex.match(f.filename()).hasMatch();
       });
 
       if (!m) {
@@ -201,7 +201,7 @@ void ImagesTab::switchToFiltered()
 
     if (!m_ddsEnabled) {
       // skip .dds files
-      if (f.path().endsWith(".dds", Qt::CaseInsensitive)) {
+      if (f.filename().endsWith(".dds", Qt::CaseInsensitive)) {
         continue;
       }
     }
diff --git a/src/modinfodialogtab.cpp b/src/modinfodialogtab.cpp
index 9748d059..d662e2b2 100644
--- a/src/modinfodialogtab.cpp
+++ b/src/modinfodialogtab.cpp
@@ -1,8 +1,8 @@
 #include "modinfodialogtab.h"
 #include "ui_modinfodialog.h"
 #include "texteditor.h"
-#include "directoryentry.h"
 #include "modinfo.h"
+#include "shared/filesorigin.h"
 
 ModInfoDialogTab::ModInfoDialogTab(ModInfoDialogTabContext cx) :
   ui(cx.ui), m_core(cx.core), m_plugin(cx.plugin), m_parent(cx.parent),
diff --git a/src/modinfowithconflictinfo.cpp b/src/modinfowithconflictinfo.cpp
index 861782d9..2b4fa11c 100644
--- a/src/modinfowithconflictinfo.cpp
+++ b/src/modinfowithconflictinfo.cpp
@@ -1,7 +1,8 @@
 #include "modinfowithconflictinfo.h"
-
-#include "directoryentry.h"
 #include "utility.h"
+#include "shared/directoryentry.h"
+#include "shared/filesorigin.h"
+#include "shared/fileentry.h"
 #include <filesystem>
 
 using namespace MOBase;
@@ -98,11 +99,11 @@ void ModInfoWithConflictInfo::doConflictCheck() const
 
   if ((*m_DirectoryStructure)->originExists(name)) {
     FilesOrigin &origin = (*m_DirectoryStructure)->getOriginByName(name);
-    std::vector<FileEntry::Ptr> files = origin.getFiles();
+    std::vector<FileEntryPtr> files = origin.getFiles();
     std::set<const DirectoryEntry*> checkedDirs;
 
     // for all files in this origin
-    for (FileEntry::Ptr file : files) {
+    for (FileEntryPtr file : files) {
 
       // skip hiidden file check if already found one
       if (!hasHiddenFiles) {
@@ -267,7 +268,7 @@ bool ModInfoWithConflictInfo::isRedundant() const
   std::wstring name = ToWString(this->name());
   if ((*m_DirectoryStructure)->originExists(name)) {
     FilesOrigin &origin = (*m_DirectoryStructure)->getOriginByName(name);
-    std::vector<FileEntry::Ptr> files = origin.getFiles();
+    std::vector<FileEntryPtr> files = origin.getFiles();
     bool ignore = false;
     for (auto iter = files.begin(); iter != files.end(); ++iter) {
       if ((*iter)->getOrigin(ignore) == origin.getID()) {
diff --git a/src/modlist.cpp b/src/modlist.cpp
index 31eb8387..e84910fd 100644
--- a/src/modlist.cpp
+++ b/src/modlist.cpp
@@ -27,6 +27,10 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "pluginlist.h"
 #include "settings.h"
 #include "modinforegular.h"
+#include "shared/directoryentry.h"
+#include "shared/fileentry.h"
+#include "shared/filesorigin.h"
+
 #include <appconfig.h>
 #include <utility.h>
 #include <report.h>
@@ -872,7 +876,7 @@ void ModList::highlightMods(const QItemSelectionModel *selection, const MOShared
   for (QModelIndex idx : selection->selectedRows(PluginList::COL_NAME)) {
     QString modName = idx.data().toString();
 
-    const MOShared::FileEntry::Ptr fileEntry = directoryEntry.findFile(modName.toStdWString());
+    const MOShared::FileEntryPtr fileEntry = directoryEntry.findFile(modName.toStdWString());
     if (fileEntry.get() != nullptr) {
       bool archive = false;
       std::vector<std::pair<int, std::pair<std::wstring, int>>> origins;
@@ -881,7 +885,7 @@ void ModList::highlightMods(const QItemSelectionModel *selection, const MOShared
         origins.insert(origins.end(), std::pair<int, std::pair<std::wstring, int>>(fileEntry->getOrigin(archive), fileEntry->getArchive()));
       }
       for (auto originInfo : origins) {
-        MOShared::FilesOrigin &origin = directoryEntry.getOriginByID(originInfo.first);
+        MOShared::FilesOrigin& origin = directoryEntry.getOriginByID(originInfo.first);
         for (unsigned int i = 0; i < ModInfo::getNumMods(); ++i) {
           if (ModInfo::getByIndex(i)->internalName() == QString::fromStdWString(origin.getName())) {
             ModInfo::getByIndex(i)->setPluginSelected(true);
@@ -1434,15 +1438,7 @@ bool ModList::toggleSelection(QAbstractItemView *itemView)
 
 bool ModList::eventFilter(QObject *obj, QEvent *event)
 {
-  if (event->type() == QEvent::ContextMenu) {
-    QContextMenuEvent *contextEvent = static_cast<QContextMenuEvent*>(event);
-    QWidget *object = qobject_cast<QWidget*>(obj);
-    if ((object != nullptr) && (contextEvent->reason() == QContextMenuEvent::Mouse)) {
-      emit requestColumnSelect(object->mapToGlobal(contextEvent->pos()));
-
-      return true;
-    }
-  } else if ((event->type() == QEvent::KeyPress) && (m_Profile != nullptr)) {
+  if ((event->type() == QEvent::KeyPress) && (m_Profile != nullptr)) {
     QAbstractItemView *itemView = qobject_cast<QAbstractItemView*>(obj);
     QKeyEvent *keyEvent = static_cast<QKeyEvent*>(event);
 
diff --git a/src/modlist.h b/src/modlist.h
index 9a44b761..d8980dec 100644
--- a/src/modlist.h
+++ b/src/modlist.h
@@ -27,7 +27,6 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "profile.h"

 

 #include <imodlist.h>

-#include <directoryentry.h>

 

 #include <QFile>

 #include <QListWidget>

@@ -202,12 +201,6 @@ signals:
   void resizeHeaders();

 

   /**

-   * @brief requestColumnSelect is emitted whenever the user requested a menu to select visible columns

-   * @param pos the position to display the menu at

-   */

-  void requestColumnSelect(QPoint pos);

-

-  /**

    * @brief emitted to remove a file origin

    * @param name name of the orign to remove

    */

diff --git a/src/modlistview.cpp b/src/modlistview.cpp
index fe2d3e57..c6dbc6ea 100644
--- a/src/modlistview.cpp
+++ b/src/modlistview.cpp
@@ -1,4 +1,5 @@
 #include "modlistview.h"

+#include <widgetutility.h>

 #include <QUrl>

 #include <QMimeData>

 #include <QProxyStyle>

@@ -39,6 +40,7 @@ ModListView::ModListView(QWidget *parent)
   , m_Scrollbar(new ViewMarkingScrollBar(this->model(), this))

 {

   setVerticalScrollBar(m_Scrollbar);

+  MOBase::setCustomizableColumns(this);

 }

 

 void ModListView::dragEnterEvent(QDragEnterEvent *event)

diff --git a/src/organizercore.cpp b/src/organizercore.cpp
index 04d78ca8..80df044a 100644
--- a/src/organizercore.cpp
+++ b/src/organizercore.cpp
@@ -22,7 +22,6 @@
 #include <ipluginmodpage.h>
 #include <dataarchives.h>
 #include <localsavegames.h>
-#include <directoryentry.h>
 #include <scopeguard.h>
 #include <utility.h>
 #include <usvfs.h>
@@ -34,6 +33,11 @@
 #include "previewdialog.h"
 #include "env.h"
 #include "envmodule.h"
+#include "envfs.h"
+#include "directoryrefresher.h"
+#include "shared/directoryentry.h"
+#include "shared/filesorigin.h"
+#include "shared/fileentry.h"
 
 #include <QApplication>
 #include <QCoreApplication>
@@ -88,24 +92,19 @@ QStringList toStringList(InputIterator current, InputIterator end)
 OrganizerCore::OrganizerCore(Settings &settings)
   : m_UserInterface(nullptr)
   , m_PluginContainer(nullptr)
-  , m_GameName()
   , m_CurrentProfile(nullptr)
   , m_Settings(settings)
   , m_Updater(NexusInterface::instance(m_PluginContainer))
-  , m_AboutToRun()
-  , m_FinishedRun()
-  , m_ModInstalled()
   , m_ModList(m_PluginContainer, this)
   , m_PluginList(this)
-  , m_DirectoryRefresher()
+  , m_DirectoryRefresher(new DirectoryRefresher(settings.refreshThreadCount()))
   , m_DirectoryStructure(new DirectoryEntry(L"data", nullptr, 0))
   , m_DownloadManager(NexusInterface::instance(m_PluginContainer), this)
-  , m_InstallationManager()
-  , m_RefresherThread()
   , m_DirectoryUpdate(false)
   , m_ArchivesInit(false)
   , m_PluginListsWriter(std::bind(&OrganizerCore::savePluginList, this))
 {
+  env::setHandleCloserThreadCount(settings.refreshThreadCount());
   m_DownloadManager.setOutputDirectory(m_Settings.paths().downloads(), false);
 
   NexusInterface::instance(m_PluginContainer)->setCacheDirectory(
@@ -116,7 +115,7 @@ OrganizerCore::OrganizerCore(Settings &settings)
 
   connect(&m_DownloadManager, SIGNAL(downloadSpeed(QString, int)), this,
           SLOT(downloadSpeed(QString, int)));
-  connect(&m_DirectoryRefresher, SIGNAL(refreshed()), this,
+  connect(m_DirectoryRefresher.get(), SIGNAL(refreshed()), this,
           SLOT(directory_refreshed()));
 
   connect(&m_ModList, SIGNAL(removeOrigin(QString)), this,
@@ -141,7 +140,7 @@ OrganizerCore::OrganizerCore(Settings &settings)
 
   // make directory refresher run in a separate thread
   m_RefresherThread.start();
-  m_DirectoryRefresher.moveToThread(&m_RefresherThread);
+  m_DirectoryRefresher->moveToThread(&m_RefresherThread);
 }
 
 OrganizerCore::~OrganizerCore()
@@ -149,6 +148,10 @@ OrganizerCore::~OrganizerCore()
   m_RefresherThread.exit();
   m_RefresherThread.wait();
 
+  if (m_StructureDeleter.joinable()) {
+    m_StructureDeleter.join();
+  }
+
   saveCurrentProfile();
 
   // profile has to be cleaned up before the modinfo-buffer is cleared
@@ -234,8 +237,6 @@ void OrganizerCore::setUserInterface(IUserInterface* ui)
             SLOT(removeMod_clicked()));
     connect(&m_ModList, SIGNAL(clearOverwrite()), w,
       SLOT(clearOverwrite()));
-    connect(&m_ModList, SIGNAL(requestColumnSelect(QPoint)), w,
-            SLOT(displayColumnSelection(QPoint)));
     connect(&m_ModList, SIGNAL(fileMoved(QString, QString, QString)), w,
             SLOT(fileMoved(QString, QString, QString)));
     connect(&m_ModList, SIGNAL(modorder_changed()), w,
@@ -813,7 +814,7 @@ QString OrganizerCore::resolvePath(const QString &fileName) const
   if (m_DirectoryStructure == nullptr) {
     return QString();
   }
-  const FileEntry::Ptr file
+  const FileEntryPtr file
       = m_DirectoryStructure->searchFile(ToWString(fileName), nullptr);
   if (file.get() != nullptr) {
     return ToQString(file->getFullPath());
@@ -847,8 +848,8 @@ QStringList OrganizerCore::findFiles(
   if (!path.isEmpty())
     dir = dir->findSubDirectoryRecursive(ToWString(path));
   if (dir != nullptr) {
-    std::vector<FileEntry::Ptr> files = dir->getFiles();
-    foreach (FileEntry::Ptr file, files) {
+    std::vector<FileEntryPtr> files = dir->getFiles();
+    foreach (FileEntryPtr file, files) {
       if (filter(ToQString(file->getFullPath()))) {
         result.append(ToQString(file->getFullPath()));
       }
@@ -860,7 +861,7 @@ QStringList OrganizerCore::findFiles(
 QStringList OrganizerCore::getFileOrigins(const QString &fileName) const
 {
   QStringList result;
-  const FileEntry::Ptr file = m_DirectoryStructure->searchFile(ToWString(fileName), nullptr);
+  const FileEntryPtr file = m_DirectoryStructure->searchFile(ToWString(fileName), nullptr);
 
   if (file.get() != nullptr) {
     result.append(ToQString(
@@ -883,8 +884,8 @@ QList<MOBase::IOrganizer::FileInfo> OrganizerCore::findFileInfos(
   if (!path.isEmpty())
     dir = dir->findSubDirectoryRecursive(ToWString(path));
   if (dir != nullptr) {
-    std::vector<FileEntry::Ptr> files = dir->getFiles();
-    foreach (FileEntry::Ptr file, files) {
+    std::vector<FileEntryPtr> files = dir->getFiles();
+    foreach (FileEntryPtr file, files) {
       IOrganizer::FileInfo info;
       info.filePath    = ToQString(file->getFullPath());
       bool fromArchive = false;
@@ -966,7 +967,7 @@ bool OrganizerCore::previewFileWithAlternatives(
 
 
 
-  const FileEntry::Ptr file = directoryStructure()->searchFile(ToWString(fileName), nullptr);
+  const FileEntryPtr file = directoryStructure()->searchFile(ToWString(fileName), nullptr);
 
   if (file.get() == nullptr) {
     reportError(tr("file not found: %1").arg(qUtf8Printable(fileName)));
@@ -1180,7 +1181,7 @@ void OrganizerCore::updateModsActiveState(const QList<unsigned int> &modIndices,
     QDir dir(modInfo->absolutePath());
     for (const QString &esm :
       dir.entryList(QStringList() << "*.esm", QDir::Files)) {
-      const FileEntry::Ptr file = m_DirectoryStructure->findFile(ToWString(esm));
+      const FileEntryPtr file = m_DirectoryStructure->findFile(ToWString(esm));
       if (file.get() == nullptr) {
         log::warn("failed to activate {}", esm);
         continue;
@@ -1196,7 +1197,7 @@ void OrganizerCore::updateModsActiveState(const QList<unsigned int> &modIndices,
 
     for (const QString &esl :
       dir.entryList(QStringList() << "*.esl", QDir::Files)) {
-      const FileEntry::Ptr file = m_DirectoryStructure->findFile(ToWString(esl));
+      const FileEntryPtr file = m_DirectoryStructure->findFile(ToWString(esl));
       if (file.get() == nullptr) {
         log::warn("failed to activate {}", esl);
         continue;
@@ -1212,7 +1213,7 @@ void OrganizerCore::updateModsActiveState(const QList<unsigned int> &modIndices,
     }
     QStringList esps = dir.entryList(QStringList() << "*.esp", QDir::Files);
     for (const QString &esp : esps) {
-      const FileEntry::Ptr file = m_DirectoryStructure->findFile(ToWString(esp));
+      const FileEntryPtr file = m_DirectoryStructure->findFile(ToWString(esp));
       if (file.get() == nullptr) {
         log::warn("failed to activate {}", esp);
         continue;
@@ -1247,13 +1248,17 @@ void OrganizerCore::updateModInDirectoryStructure(unsigned int index,
 
 void OrganizerCore::updateModsInDirectoryStructure(QMap<unsigned int, ModInfo::Ptr> modInfo)
 {
+  std::vector<DirectoryRefresher::EntryInfo> entries;
+
   for (auto idx : modInfo.keys()) {
-    // add files of the bsa to the directory structure
-    m_DirectoryRefresher.addModFilesToStructure(
-      m_DirectoryStructure, modInfo[idx]->name(),
-      m_CurrentProfile->getModPriority(idx), modInfo[idx]->absolutePath(),
-      modInfo[idx]->stealFiles());
+    entries.push_back({
+      modInfo[idx]->name(), modInfo[idx]->absolutePath(),
+      modInfo[idx]->stealFiles(), {}, m_CurrentProfile->getModPriority(idx)});
   }
+
+  m_DirectoryRefresher->addMultipleModsFilesToStructure(
+    m_DirectoryStructure, entries);
+
   DirectoryRefresher::cleanStructure(m_DirectoryStructure);
   // need to refresh plugin list now so we can activate esps
   refreshESPList(true);
@@ -1270,13 +1275,13 @@ void OrganizerCore::updateModsInDirectoryStructure(QMap<unsigned int, ModInfo::P
   }
 
   std::vector<QString> archives = enabledArchives();
-  m_DirectoryRefresher.setMods(
+  m_DirectoryRefresher->setMods(
     m_CurrentProfile->getActiveMods(),
     std::set<QString>(archives.begin(), archives.end()));
 
   // finally also add files from bsas to the directory structure
   for (auto idx : modInfo.keys()) {
-    m_DirectoryRefresher.addModBSAToStructure(
+    m_DirectoryRefresher->addModBSAToStructure(
       m_DirectoryStructure, modInfo[idx]->name(),
       m_CurrentProfile->getModPriority(idx), modInfo[idx]->absolutePath(),
       modInfo[idx]->archives());
@@ -1382,46 +1387,74 @@ std::vector<QString> OrganizerCore::enabledArchives()
 
 void OrganizerCore::refreshDirectoryStructure()
 {
-  if (!m_DirectoryUpdate) {
-    m_CurrentProfile->writeModlistNow(true);
+  if (m_DirectoryUpdate) {
+    log::debug("can't refresh, already in progress");
+    return;
+  }
 
-    m_DirectoryUpdate = true;
-    std::vector<std::tuple<QString, QString, int>> activeModList
-        = m_CurrentProfile->getActiveMods();
-    auto archives = enabledArchives();
-    m_DirectoryRefresher.setMods(
-        activeModList, std::set<QString>(archives.begin(), archives.end()));
+  log::debug("refreshing structure");
+  m_DirectoryUpdate = true;
 
-    QTimer::singleShot(0, &m_DirectoryRefresher, SLOT(refresh()));
-  }
+  m_CurrentProfile->writeModlistNow(true);
+  const auto activeModList = m_CurrentProfile->getActiveMods();
+  const auto archives = enabledArchives();
+
+  m_DirectoryRefresher->setMods(
+      activeModList, std::set<QString>(archives.begin(), archives.end()));
+
+  // runs refresh() in a thread
+  QTimer::singleShot(0, m_DirectoryRefresher.get(), SLOT(refresh()));
 }
 
 void OrganizerCore::directory_refreshed()
 {
-  DirectoryEntry *newStructure = m_DirectoryRefresher.getDirectoryStructure();
+  log::debug("structure refreshed");
+
+  DirectoryEntry *newStructure = m_DirectoryRefresher->stealDirectoryStructure();
   Q_ASSERT(newStructure != m_DirectoryStructure);
-  if (newStructure != nullptr) {
-    std::swap(m_DirectoryStructure, newStructure);
-    delete newStructure;
-  } else {
+
+  if (newStructure == nullptr) {
     // TODO: don't know why this happens, this slot seems to get called twice
     // with only one emit
     return;
   }
+
+  std::swap(m_DirectoryStructure, newStructure);
+
+  if (m_StructureDeleter.joinable()) {
+    m_StructureDeleter.join();
+  }
+
+  m_StructureDeleter = std::thread([=]{
+    log::debug("structure deleter thread start");
+    delete newStructure;
+    log::debug("structure deleter thread done");
+  });
+
   m_DirectoryUpdate = false;
 
+  log::debug("clearing caches");
   for (int i = 0; i < m_ModList.rowCount(); ++i) {
     ModInfo::Ptr modInfo = ModInfo::getByIndex(i);
     modInfo->clearCaches();
   }
-  for (auto task : m_PostRefreshTasks) {
-    task();
+
+  if (!m_PostRefreshTasks.empty()) {
+    log::debug("running {} post refresh tasks", m_PostRefreshTasks.size());
+
+    for (auto task : m_PostRefreshTasks) {
+      task();
+    }
+
+    m_PostRefreshTasks.clear();
   }
-  m_PostRefreshTasks.clear();
 
   if (m_CurrentProfile != nullptr) {
+    log::debug("refreshing lists");
     refreshLists();
   }
+
+  log::debug("refresh done");
 }
 
 void OrganizerCore::profileRefresh()
@@ -1867,7 +1900,7 @@ std::vector<Mapping> OrganizerCore::fileMapping(
 {
   std::vector<Mapping> result;
 
-  for (FileEntry::Ptr current : directoryEntry->getFiles()) {
+  for (FileEntryPtr current : directoryEntry->getFiles()) {
     bool isArchive = false;
     int origin = current->getOrigin(isArchive);
     if (isArchive || (origin == 0)) {
diff --git a/src/organizercore.h b/src/organizercore.h
index a63dc959..a4d1a799 100644
--- a/src/organizercore.h
+++ b/src/organizercore.h
@@ -1,13 +1,11 @@
 #ifndef ORGANIZERCORE_H

 #define ORGANIZERCORE_H

 

-

 #include "selfupdater.h"

 #include "settings.h"

 #include "modlist.h"

 #include "modinfo.h"

 #include "pluginlist.h"

-#include "directoryrefresher.h"

 #include "installationmanager.h"

 #include "downloadmanager.h"

 #include "executableslist.h"

@@ -15,7 +13,6 @@
 #include "moshortcut.h"

 #include "processrunner.h"

 #include "uilocker.h"

-#include <directoryentry.h>

 #include <imoinfo.h>

 #include <iplugindiagnose.h>

 #include <versioninfo.h>

@@ -24,17 +21,6 @@
 #include "executableinfo.h"

 #include <log.h>

 

-class ModListSortProxy;

-class PluginListSortProxy;

-class Profile;

-class IUserInterface;

-

-namespace MOBase {

-  template <typename T> class GuessedValue;

-  class IModInterface;

-}

-namespace MOShared { class DirectoryEntry; }

-

 #include <QDir>

 #include <QFileInfo>

 #include <QList>

@@ -45,21 +31,25 @@ namespace MOShared { class DirectoryEntry; }
 #include <QThread>

 #include <QVariant>

 

-class QNetworkReply;

-class QUrl;

-class QWidget;

-

-#include <Windows.h> //for HANDLE, LPDWORD

-

-#include <functional>

-#include <vector>

-

+class ModListSortProxy;

+class PluginListSortProxy;

+class Profile;

+class IUserInterface;

 class PluginContainer;

+class DirectoryRefresher;

 

-namespace MOBase {

+namespace MOBase

+{

+  template <typename T> class GuessedValue;

+  class IModInterface;

   class IPluginGame;

 }

 

+namespace MOShared

+{

+  class DirectoryEntry;

+}

+

 

 class OrganizerCore : public QObject, public MOBase::IPluginDiagnose

 {

@@ -114,7 +104,7 @@ public:
   SelfUpdater *updater() { return &m_Updater; }

   InstallationManager *installationManager();

   MOShared::DirectoryEntry *directoryStructure() { return m_DirectoryStructure; }

-  DirectoryRefresher *directoryRefresher() { return &m_DirectoryRefresher; }

+  DirectoryRefresher *directoryRefresher() { return m_DirectoryRefresher.get(); }

   ExecutablesList *executablesList() { return &m_ExecutablesList; }

   void setExecutablesList(const ExecutablesList &executablesList) {

     m_ExecutablesList = executablesList;

@@ -338,7 +328,7 @@ private:
   QStringList m_DefaultArchives;

   QStringList m_ActiveArchives;

 

-  DirectoryRefresher m_DirectoryRefresher;

+  std::unique_ptr<DirectoryRefresher> m_DirectoryRefresher;

   MOShared::DirectoryEntry *m_DirectoryStructure;

 

   DownloadManager m_DownloadManager;

@@ -346,6 +336,8 @@ private:
 

   QThread m_RefresherThread;

 

+  std::thread m_StructureDeleter;

+

   bool m_DirectoryUpdate;

   bool m_ArchivesInit;

   bool m_ArchiveParsing{ m_Settings.archiveParsing() };

diff --git a/src/pch.h b/src/pch.h
index 030ee634..c66550be 100644
--- a/src/pch.h
+++ b/src/pch.h
@@ -5,14 +5,17 @@
 #include <cstdarg>
 #include <cstdint>
 #include <ctime>
+#include <deque>
 #include <exception>
 #include <functional>
+#include <fstream>
 #include <iomanip>
 #include <iostream>
 #include <limits.h>
 #include <list>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <optional>
 #include <regex>
 #include <set>
@@ -21,6 +24,7 @@
 #include <stdexcept>
 #include <string.h>
 #include <string>
+#include <thread>
 #include <tuple>
 #include <unordered_set>
 #include <utility>
diff --git a/src/pluginlist.cpp b/src/pluginlist.cpp
index 266fe35c..4b2eedbd 100644
--- a/src/pluginlist.cpp
+++ b/src/pluginlist.cpp
@@ -23,6 +23,10 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "modinfo.h"

 #include "modlist.h"

 #include "viewmarkingscrollbar.h"

+#include "shared/directoryentry.h"

+#include "shared/filesorigin.h"

+#include "shared/fileentry.h"

+

 #include <utility.h>

 #include <iplugingame.h>

 #include <espfile.h>

@@ -144,10 +148,10 @@ void PluginList::highlightPlugins(const QItemSelectionModel *selection, const MO
     if (!selectedMod.isNull() && profile.modEnabled(modIndex)) {

       QDir dir(selectedMod->absolutePath());

       QStringList plugins = dir.entryList(QStringList() << "*.esp" << "*.esm" << "*.esl");

-      MOShared::FilesOrigin origin = directoryEntry.getOriginByName(selectedMod->internalName().toStdWString());

+      const MOShared::FilesOrigin& origin = directoryEntry.getOriginByName(selectedMod->internalName().toStdWString());

       if (plugins.size() > 0) {

         for (auto plugin : plugins) {

-          MOShared::FileEntry::Ptr file = directoryEntry.findFile(plugin.toStdWString());

+          MOShared::FileEntryPtr file = directoryEntry.findFile(plugin.toStdWString());

           if (file && file->getOrigin() != origin.getID()) {

             const std::vector<std::pair<int, std::pair<std::wstring, int>>> alternatives = file->getAlternatives();

             if (std::find_if(alternatives.begin(), alternatives.end(), [&](const std::pair<int, std::pair<std::wstring, int>>& element) { return element.first == origin.getID(); }) == alternatives.end())

@@ -185,8 +189,8 @@ void PluginList::refresh(const QString &profileName
 

   QStringList availablePlugins;

 

-  std::vector<FileEntry::Ptr> files = baseDirectory.getFiles();

-  for (FileEntry::Ptr current : files) {

+  std::vector<FileEntryPtr> files = baseDirectory.getFiles();

+  for (FileEntryPtr current : files) {

     if (current.get() == nullptr) {

       continue;

     }

@@ -217,7 +221,7 @@ void PluginList::refresh(const QString &profileName
         bool hasIni = baseDirectory.findFile(ToWString(iniPath)).get() != nullptr;

         std::set<QString> loadedArchives;

         QString candidateName;

-        for (FileEntry::Ptr archiveCandidate : files) {

+        for (FileEntryPtr archiveCandidate : files) {

           candidateName = ToQString(archiveCandidate->getName());

           if (candidateName.startsWith(baseName, Qt::CaseInsensitive) &&

              (candidateName.endsWith(".bsa", Qt::CaseInsensitive) ||

@@ -507,7 +511,6 @@ void PluginList::writeLockedOrder(const QString &fileName) const
     file->write(QString("%1|%2\r\n").arg(iter->first).arg(iter->second).toUtf8());

   }

   file.commit();

-  log::debug("{} saved", QDir::toNativeSeparators(fileName));

 }

 

 

@@ -531,9 +534,7 @@ void PluginList::saveTo(const QString &lockedOrderFileName
         deleterFile->write("\r\n");

       }

     }

-    if (deleterFile.commitIfDifferent(m_LastSaveHash[deleterFileName])) {

-      log::debug("{} saved", QDir::toNativeSeparators(deleterFileName));

-    }

+    deleterFile.commitIfDifferent(m_LastSaveHash[deleterFileName]);

   } else if (QFile::exists(deleterFileName)) {

     shellDelete(QStringList() << deleterFileName);

   }

@@ -551,7 +552,7 @@ bool PluginList::saveLoadOrder(DirectoryEntry &directoryStructure)
 

   for (ESPInfo &esp : m_ESPs) {

     std::wstring espName = ToWString(esp.name);

-    const FileEntry::Ptr fileEntry = directoryStructure.findFile(espName);

+    const FileEntryPtr fileEntry = directoryStructure.findFile(espName);

     if (fileEntry.get() != nullptr) {

       QString fileName;

       bool archive = false;

diff --git a/src/pluginlist.h b/src/pluginlist.h
index fb6d0543..cdab18b5 100644
--- a/src/pluginlist.h
+++ b/src/pluginlist.h
@@ -20,7 +20,6 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #ifndef PLUGINLIST_H

 #define PLUGINLIST_H

 

-#include <directoryentry.h>

 #include <ipluginlist.h>

 #include "profile.h"

 #include "loot.h"

diff --git a/src/pluginlistview.cpp b/src/pluginlistview.cpp
index 0fcf8183..4217971d 100644
--- a/src/pluginlistview.cpp
+++ b/src/pluginlistview.cpp
@@ -1,4 +1,5 @@
 #include "pluginlistview.h"
+#include <widgetutility.h>
 #include <QUrl>
 #include <QMimeData>
 #include <QProxyStyle>
@@ -40,6 +41,7 @@ PluginListView::PluginListView(QWidget *parent)
   , m_Scrollbar(new ViewMarkingScrollBar(this->model(), this))
 {
   setVerticalScrollBar(m_Scrollbar);
+  MOBase::setCustomizableColumns(this);
 }
 
 void PluginListView::dragEnterEvent(QDragEnterEvent *event)
diff --git a/src/profile.cpp b/src/profile.cpp
index f041a241..19c0d750 100644
--- a/src/profile.cpp
+++ b/src/profile.cpp
@@ -250,9 +250,7 @@ void Profile::doWriteModlist()
       }
     }
 
-    if (file.commitIfDifferent(m_LastModlistHash)) {
-      log::debug("{} saved", QDir::toNativeSeparators(fileName));
-    }
+    file.commitIfDifferent(m_LastModlistHash);
   } catch (const std::exception &e) {
     reportError(tr("failed to write mod list: %1").arg(e.what()));
     return;
@@ -292,8 +290,6 @@ void Profile::createTweakedIniFile()
     reportError(tr("failed to create tweaked ini: %1")
       .arg(QString::fromStdWString(formatSystemMessage(e))));
   }
-
-  log::debug("{} saved", QDir::toNativeSeparators(tweakedIni));
 }
 
 // static
diff --git a/src/settings.cpp b/src/settings.cpp
index 4431e7d9..f0496fe8 100644
--- a/src/settings.cpp
+++ b/src/settings.cpp
@@ -209,6 +209,16 @@ void Settings::setUseSplash(bool b)
   set(m_Settings, "Settings", "use_splash", b);
 }
 
+std::size_t Settings::refreshThreadCount() const
+{
+  return get<std::size_t>(m_Settings, "Settings", "refresh_thread_count", 10);
+}
+
+void Settings::setRefreshThreadCount(std::size_t n) const
+{
+  return set(m_Settings, "Settings", "refresh_thread_count", n);
+}
+
 std::optional<QVersionNumber> Settings::version() const
 {
   if (auto v=getOptional<QString>(m_Settings, "General", "version")) {
@@ -1992,6 +2002,24 @@ void InterfaceSettings::setDoubleClicksOpenPreviews(bool b)
   set(m_Settings, "Settings", "double_click_previews", b);
 }
 
+FilterWidget::Options InterfaceSettings::filterOptions() const
+{
+  FilterWidget::Options o;
+
+  o.useRegex = get<bool>(m_Settings, "Settings", "filter_regex", false);
+  o.regexCaseSensitive = get<bool>(m_Settings, "Settings", "regex_case_sensitive", false);
+  o.regexExtended = get<bool>(m_Settings, "Settings", "regex_extended", false);
+
+  return o;
+}
+
+void InterfaceSettings::setFilterOptions(const FilterWidget::Options& o)
+{
+  set(m_Settings, "Settings", "filter_regex", o.useRegex);
+  set(m_Settings, "Settings", "regex_case_sensitive", o.regexCaseSensitive);
+  set(m_Settings, "Settings", "regex_extended", o.regexExtended);
+}
+
 
 DiagnosticsSettings::DiagnosticsSettings(QSettings& settings)
   : m_Settings(settings)
diff --git a/src/settings.h b/src/settings.h
index 678226c1..b2cb6be5 100644
--- a/src/settings.h
+++ b/src/settings.h
@@ -21,6 +21,7 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #define SETTINGS_H
 
 #include "loadmechanism.h"
+#include <filterwidget.h>
 #include <lootcli/lootcli.h>
 #include <questionboxmemory.h>
 #include <log.h>
@@ -615,6 +616,11 @@ public:
   bool doubleClicksOpenPreviews() const;
   void setDoubleClicksOpenPreviews(bool b);
 
+  // filter widget options
+  //
+  MOBase::FilterWidget::Options filterOptions() const;
+  void setFilterOptions(const MOBase::FilterWidget::Options& o);
+
 private:
   QSettings& m_Settings;
 };
@@ -725,6 +731,11 @@ public:
   bool useSplash() const;
   void setUseSplash(bool b);
 
+  // number of threads to use when refreshing
+  //
+  std::size_t refreshThreadCount() const;
+  void setRefreshThreadCount(std::size_t n) const;
+
   GameSettings& game();
   const GameSettings& game() const;
 
diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp
index 6e44cc91..639392d2 100644
--- a/src/shared/directoryentry.cpp
+++ b/src/shared/directoryentry.cpp
@@ -18,35 +18,39 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 */

 

 #include "directoryentry.h"

+#include "originconnection.h"

+#include "filesorigin.h"

+#include "fileentry.h"

+#include "envfs.h"

+#include "util.h"

 #include "windows_error.h"

-#include "error_report.h"

 #include <log.h>

-#include <bsatk.h>

-#include <boost/bind.hpp>

-#include <boost/scoped_array.hpp>

-#define WIN32_LEAN_AND_MEAN

-#include <Windows.h>

-#include <sstream>

-#include <ctime>

-#include <algorithm>

-#include <map>

-#include <atomic>

+#include <utility.h>

 

 namespace MOShared

 {

 

 using namespace MOBase;

-static const int MAXPATH_UNICODE = 32767;

+const int MAXPATH_UNICODE = 32767;

 

-static std::wstring tail(const std::wstring &source, const size_t count)

+template <class F>

+void elapsedImpl(std::chrono::nanoseconds& out, F&& f)

 {

-  if (count >= source.length()) {

-    return source;

+  if constexpr (DirectoryStats::EnableInstrumentation) {

+    const auto start = std::chrono::high_resolution_clock::now();

+    f();

+    const auto end = std::chrono::high_resolution_clock::now();

+    out += (end - start);

+  } else {

+    f();

   }

-

-  return source.substr(source.length() - count);

 }

 

+// elapsed() is not optimized out when EnableInstrumentation is false even

+// though it's equivalent that this macro

+#define elapsed(OUT, F) (F)();

+//#define elapsed(OUT, F) elapsedImpl(OUT, F);

+

 static bool SupportOptimizedFind()

 {

   // large fetch and basic info for FindFirstFileEx is supported on win server 2008 r2, win 7 and newer

@@ -63,681 +67,186 @@ static bool SupportOptimizedFind()
   return (::VerifyVersionInfo(&versionInfo, VER_MAJORVERSION | VER_MINORVERSION, mask) == TRUE);

 }

 

-static bool DirCompareByName(const DirectoryEntry *lhs, const DirectoryEntry *rhs)

+static bool DirCompareByName(const DirectoryEntry* lhs, const DirectoryEntry* rhs)

 {

   return _wcsicmp(lhs->getName().c_str(), rhs->getName().c_str()) < 0;

 }

 

 

-class OriginConnection

-{

-public:

-  typedef int Index;

-  static const int INVALID_INDEX = INT_MIN;

-

-  OriginConnection()

-    : m_NextID(0)

-  {

-  }

-

-  FilesOrigin& createOrigin(

-    const std::wstring &originName, const std::wstring &directory, int priority,

-    boost::shared_ptr<FileRegister> fileRegister,

-    boost::shared_ptr<OriginConnection> originConnection)

-  {

-    int newID = createID();

-

-    m_Origins[newID] = FilesOrigin(newID, originName, directory, priority, fileRegister, originConnection);

-    m_OriginsNameMap[originName] = newID;

-    m_OriginsPriorityMap[priority] = newID;

-

-    return m_Origins[newID];

-  }

-

-  bool exists(const std::wstring &name)

-  {

-    return m_OriginsNameMap.find(name) != m_OriginsNameMap.end();

-  }

-

-  FilesOrigin &getByID(Index ID)

-  {

-    return m_Origins[ID];

-  }

-

-  const FilesOrigin* findByID(Index ID) const

-  {

-    auto itor = m_Origins.find(ID);

-

-    if (itor == m_Origins.end()) {

-      return nullptr;

-    } else {

-      return &itor->second;

-    }

-  }

-

-  FilesOrigin &getByName(const std::wstring &name)

-  {

-    std::map<std::wstring, int>::iterator iter = m_OriginsNameMap.find(name);

-

-    if (iter != m_OriginsNameMap.end()) {

-      return m_Origins[iter->second];

-    } else {

-      std::ostringstream stream;

-      stream << QObject::tr("invalid origin name: ").toStdString() << ToString(name, true);

-      throw std::runtime_error(stream.str());

-    }

-  }

-

-  void changePriorityLookup(int oldPriority, int newPriority)

-  {

-    auto iter = m_OriginsPriorityMap.find(oldPriority);

-

-    if (iter != m_OriginsPriorityMap.end()) {

-      Index idx = iter->second;

-      m_OriginsPriorityMap.erase(iter);

-      m_OriginsPriorityMap[newPriority] = idx;

-    }

-  }

-

-  void changeNameLookup(const std::wstring &oldName, const std::wstring &newName)

-  {

-    auto iter = m_OriginsNameMap.find(oldName);

-

-    if (iter != m_OriginsNameMap.end()) {

-      Index idx = iter->second;

-      m_OriginsNameMap.erase(iter);

-      m_OriginsNameMap[newName] = idx;

-    } else {

-      log::error(QObject::tr("failed to change name lookup from {} to {}").toStdString(), oldName, newName);

-    }

-  }

-

-private:

-  Index m_NextID;

-  std::map<Index, FilesOrigin> m_Origins;

-  std::map<std::wstring, Index> m_OriginsNameMap;

-  std::map<int, Index> m_OriginsPriorityMap;

-

-  Index createID()

-  {

-    return m_NextID++;

-  }

-};

-

-

-FileEntry::FileEntry() :

-  m_Index(UINT_MAX), m_Name(), m_Origin(-1), m_Parent(nullptr),

-  m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize),

-  m_LastAccessed(time(nullptr))

+DirectoryEntry::DirectoryEntry(

+  std::wstring name, DirectoryEntry* parent, int originID) :

+    m_OriginConnection(new OriginConnection),

+    m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(true)

 {

+  m_FileRegister.reset(new FileRegister(m_OriginConnection));

+  m_Origins.insert(originID);

 }

 

-FileEntry::FileEntry(Index index, const std::wstring &name, DirectoryEntry *parent) :

-  m_Index(index), m_Name(name), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent),

-  m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize),

-  m_LastAccessed(time(nullptr))

+DirectoryEntry::DirectoryEntry(

+  std::wstring name, DirectoryEntry* parent, int originID,

+  boost::shared_ptr<FileRegister> fileRegister,

+  boost::shared_ptr<OriginConnection> originConnection) :

+    m_FileRegister(fileRegister), m_OriginConnection(originConnection),

+    m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(false)

 {

+  m_Origins.insert(originID);

 }

 

-void FileEntry::addOrigin(

-  int origin, FILETIME fileTime, const std::wstring &archive, int order)

+DirectoryEntry::~DirectoryEntry()

 {

-  m_LastAccessed = time(nullptr);

-  if (m_Parent != nullptr) {

-    m_Parent->propagateOrigin(origin);

-  }

-

-  if (m_Origin == -1) {

-    // If this file has no previous origin, this mod is now the origin with no

-    // alternatives

-    m_Origin = origin;

-    m_FileTime = fileTime;

-    m_Archive = std::pair<std::wstring, int>(archive, order);

-  }

-  else if (

-    (m_Parent != nullptr) && (

-      (m_Parent->getOriginByID(origin).getPriority() > m_Parent->getOriginByID(m_Origin).getPriority()) ||

-      (archive.size() == 0 && m_Archive.first.size() > 0 ))

-    ) {

-    // If this mod has a higher priority than the origin mod OR

-    // this mod has a loose file and the origin mod has an archived file,

-    // this mod is now the origin and the previous origin is the first alternative

-

-    auto itor = std::find_if(

-      m_Alternatives.begin(), m_Alternatives.end(),

-      [&](auto&& i) { return i.first == m_Origin; });

-

-    if (itor == m_Alternatives.end()) {

-      m_Alternatives.push_back({m_Origin, m_Archive});

-    }

-

-    m_Origin = origin;

-    m_FileTime = fileTime;

-    m_Archive = std::pair<std::wstring, int>(archive, order);

-  }

-  else {

-    // This mod is just an alternative

-    bool found = false;

-

-    if (m_Origin == origin) {

-      // already an origin

-      return;

-    }

-

-    for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) {

-      if (iter->first == origin) {

-        // already an origin

-        return;

-      }

-

-      if ((m_Parent != nullptr) &&

-        (m_Parent->getOriginByID(iter->first).getPriority() < m_Parent->getOriginByID(origin).getPriority())) {

-        m_Alternatives.insert(iter, {origin, {archive, order}});

-        found = true;

-        break;

-      }

-    }

-

-    if (!found) {

-      m_Alternatives.push_back({origin, {archive, order}});

-    }

-  }

+  clear();

 }

 

-bool FileEntry::removeOrigin(int origin)

+void DirectoryEntry::clear()

 {

-  if (m_Origin == origin) {

-    if (!m_Alternatives.empty()) {

-      // find alternative with the highest priority

-      auto currentIter = m_Alternatives.begin();

-      for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) {

-        if (iter->first != origin) {

-          //Both files are not from archives.

-          if (!iter->second.first.size() && !currentIter->second.first.size()) {

-            if ((m_Parent->getOriginByID(iter->first).getPriority() > m_Parent->getOriginByID(currentIter->first).getPriority())) {

-              currentIter = iter;

-            }

-          }

-          else {

-            //Both files are from archives

-            if (iter->second.first.size() && currentIter->second.first.size()) {

-              if (iter->second.second > currentIter->second.second) {

-                currentIter = iter;

-              }

-            }

-            else {

-              //Only one of the two is an archive, so we change currentIter only if he is the archive one.

-              if (currentIter->second.first.size()) {

-                currentIter = iter;

-              }

-            }

-          }

-        }

-      }

-

-      int currentID = currentIter->first;

-      m_Archive = currentIter->second;

-      m_Alternatives.erase(currentIter);

-

-      m_Origin = currentID;

-    } else {

-      m_Origin = -1;

-      m_Archive = std::pair<std::wstring, int>(L"", -1);

-      return true;

-    }

-  } else {

-    auto newEnd = std::remove_if(

-      m_Alternatives.begin(), m_Alternatives.end(),

-      [&](auto &i) { return i.first == origin; });

-

-    if (newEnd != m_Alternatives.end()) {

-      m_Alternatives.erase(newEnd, m_Alternatives.end());

-    }

+  for (auto itor=m_SubDirectories.rbegin(); itor!=m_SubDirectories.rend(); ++itor) {

+    delete *itor;

   }

-  return false;

-}

-

-void FileEntry::sortOrigins()

-{

-  m_Alternatives.push_back({m_Origin, m_Archive});

-

-  std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) {

-    if (!LHS.second.first.size() && !RHS.second.first.size()) {

-      int l = m_Parent->getOriginByID(LHS.first).getPriority();

-      if (l < 0) {

-        l = INT_MAX;

-      }

 

-      int r = m_Parent->getOriginByID(RHS.first).getPriority();

-      if (r < 0) {

-        r = INT_MAX;

-      }

-

-      return l < r;

-    }

-

-    if (LHS.second.first.size() && RHS.second.first.size()) {

-      int l = LHS.second.second; if (l < 0) l = INT_MAX;

-      int r = RHS.second.second; if (r < 0) r = INT_MAX;

-

-      return l < r;

-    }

-

-    if (RHS.second.first.size()) {

-        return false;

-    }

-

-    return true;

-  });

-

-  if (!m_Alternatives.empty()) {

-    m_Origin = m_Alternatives.back().first;

-    m_Archive = m_Alternatives.back().second;

-    m_Alternatives.pop_back();

-  }

+  m_Files.clear();

+  m_FilesLookup.clear();

+  m_SubDirectories.clear();

+  m_SubDirectoriesLookup.clear();

 }

 

-bool FileEntry::isFromArchive(std::wstring archiveName) const

+void DirectoryEntry::addFromOrigin(

+  const std::wstring &originName, const std::wstring &directory, int priority,

+  DirectoryStats& stats)

 {

-  if (archiveName.length() == 0) {

-    return m_Archive.first.length() != 0;

-  }

-

-  if (m_Archive.first.compare(archiveName) == 0) {

-    return true;

-  }

-

-  for (auto alternative : m_Alternatives) {

-    if (alternative.second.first.compare(archiveName) == 0) {

-      return true;

-    }

-  }

-

-  return false;

+  env::DirectoryWalker walker;

+  addFromOrigin(walker, originName, directory, priority, stats);

 }

 

-std::wstring FileEntry::getFullPath(int originID) const

+void DirectoryEntry::addFromOrigin(

+  env::DirectoryWalker& walker, const std::wstring &originName,

+  const std::wstring &directory, int priority, DirectoryStats& stats)

 {

-  if (originID == -1) {

-    bool ignore = false;

-    originID = getOrigin(ignore);

-  }

+  FilesOrigin &origin = createOrigin(originName, directory, priority, stats);

 

-  // base directory for origin

-  const auto* o = m_Parent->findOriginByID(originID);

-  if (!o) {

-    return {};

+  if (!directory.empty()) {

+    addFiles(walker, origin, directory, stats);

   }

 

-  std::wstring result = o->getPath();

-

-  // all intermediate directories

-  recurseParents(result, m_Parent);

-

-  return result + L"\\" + m_Name;

+  m_Populated = true;

 }

 

-std::wstring FileEntry::getRelativePath() const

+void DirectoryEntry::addFromList(

+  const std::wstring &originName, const std::wstring &directory,

+  env::Directory& root, int priority, DirectoryStats& stats)

 {

-  std::wstring result;

-

-  // all intermediate directories

-  recurseParents(result, m_Parent);

+  stats = {};

 

-  return result + L"\\" + m_Name;

+  FilesOrigin &origin = createOrigin(originName, directory, priority, stats);

+  addDir(origin, root, stats);

 }

 

-bool FileEntry::recurseParents(std::wstring &path, const DirectoryEntry *parent) const

+void DirectoryEntry::addDir(

+  FilesOrigin& origin, env::Directory& d, DirectoryStats& stats)

 {

-  if (parent == nullptr) {

-    return false;

-  } else {

-    // don't append the topmost parent because it is the virtual data-root

-    if (recurseParents(path, parent->getParent())) {

-      path.append(L"\\").append(parent->getName());

+  elapsed(stats.dirTimes, [&]{

+    for (auto& sd : d.dirs) {

+      auto* sdirEntry = getSubDirectory(sd, true, stats, origin.getID());

+      sdirEntry->addDir(origin, sd, stats);

     }

+  });

 

-    return true;

-  }

-}

-

-

-FilesOrigin::FilesOrigin()

-  : m_ID(0), m_Disabled(false), m_Name(), m_Path(), m_Priority(0)

-{

-}

-

-FilesOrigin::FilesOrigin(const FilesOrigin &reference)

-  : m_ID(reference.m_ID)

-  , m_Disabled(reference.m_Disabled)

-  , m_Name(reference.m_Name)

-  , m_Path(reference.m_Path)

-  , m_Priority(reference.m_Priority)

-  , m_FileRegister(reference.m_FileRegister)

-  , m_OriginConnection(reference.m_OriginConnection)

-{

-}

-

-FilesOrigin::FilesOrigin(

-  int ID, const std::wstring &name, const std::wstring &path, int priority,

-  boost::shared_ptr<MOShared::FileRegister> fileRegister,

-  boost::shared_ptr<MOShared::OriginConnection> originConnection) :

-    m_ID(ID), m_Disabled(false), m_Name(name), m_Path(path),

-    m_Priority(priority), m_FileRegister(fileRegister),

-    m_OriginConnection(originConnection)

-{

-}

-

-void FilesOrigin::setPriority(int priority)

-{

-  m_OriginConnection.lock()->changePriorityLookup(m_Priority, priority);

-

-  m_Priority = priority;

-}

-

-void FilesOrigin::setName(const std::wstring &name)

-{

-  m_OriginConnection.lock()->changeNameLookup(m_Name, name);

+  elapsed(stats.fileTimes, [&]{

+    for (auto& f : d.files) {

+      insert(f, origin, L"", -1, stats);

+    }

+  });

 

-  // change path too

-  if (tail(m_Path, m_Name.length()) == m_Name) {

-    m_Path = m_Path.substr(0, m_Path.length() - m_Name.length()).append(name);

-  }

+  elapsed(stats.sortTimes, [&]{

+    std::sort(

+      m_SubDirectories.begin(),

+      m_SubDirectories.end(),

+      &DirCompareByName);

+  });

 

-  m_Name = name;

+  m_Populated = true;

 }

 

-std::vector<FileEntry::Ptr> FilesOrigin::getFiles() const

+void DirectoryEntry::addFromAllBSAs(

+  const std::wstring& originName, const std::wstring& directory,

+  int priority, const std::vector<std::wstring>& archives,

+  const std::set<std::wstring>& enabledArchives,

+  const std::vector<std::wstring>& loadOrder,

+  DirectoryStats& stats)

 {

-  std::vector<FileEntry::Ptr> result;

+  for (const auto& archive : archives) {

+    const std::filesystem::path archivePath(archive);

+    const auto filename = archivePath.filename().native();

 

-  for (FileEntry::Index fileIdx : m_Files) {

-    if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) {

-      result.push_back(p);

+    if (!enabledArchives.contains(filename)) {

+      continue;

     }

-  }

 

-  return result;

-}

+    const auto filenameLc = ToLowerCopy(filename);

 

-FileEntry::Ptr FilesOrigin::findFile(FileEntry::Index index) const

-{

-  return m_FileRegister.lock()->getFile(index);

-}

+    int order = -1;

 

-void FilesOrigin::enable(bool enabled, time_t notAfter)

-{

-  if (!enabled) {

-    std::set<FileEntry::Index> copy = m_Files;

-    m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter);

-    m_Files.clear();

-  }

-

-  m_Disabled = !enabled;

-}

-

-void FilesOrigin::removeFile(FileEntry::Index index)

-{

-  auto iter = m_Files.find(index);

-

-  if (iter != m_Files.end()) {

-    m_Files.erase(iter);

-  }

-}

+    for (auto plugin : loadOrder)

+    {

+      const auto pluginNameLc =

+        ToLowerCopy(std::filesystem::path(plugin).stem().native());

 

-bool FilesOrigin::containsArchive(std::wstring archiveName)

-{

-  for (FileEntry::Index fileIdx : m_Files) {

-    if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) {

-      if (p->isFromArchive(archiveName)) {

-        return true;

+      if (filenameLc.starts_with(pluginNameLc + L" - ") ||

+          filenameLc.starts_with(pluginNameLc + L".")) {

+        auto itor = std::find(loadOrder.begin(), loadOrder.end(), plugin);

+        if (itor != loadOrder.end()) {

+          order = std::distance(loadOrder.begin(), itor);

+        }

       }

     }

-  }

 

-  return false;

-}

-

-

-FileRegister::FileRegister(boost::shared_ptr<OriginConnection> originConnection)

-  : m_OriginConnection(originConnection)

-{

-}

-

-bool FileRegister::indexValid(FileEntry::Index index) const

-{

-  return (m_Files.find(index) != m_Files.end());

-}

-

-FileEntry::Ptr FileRegister::createFile(const std::wstring &name, DirectoryEntry *parent)

-{

-  FileEntry::Index index = generateIndex();

-

-  auto r = m_Files.insert_or_assign(

-    index, FileEntry::Ptr(new FileEntry(index, name, parent)));

-

-  return r.first->second;

-}

-

-FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const

-{

-  auto iter = m_Files.find(index);

-

-  if (iter != m_Files.end()) {

-    return iter->second;

-  } else {

-    return FileEntry::Ptr();

-  }

-}

-

-bool FileRegister::removeFile(FileEntry::Index index)

-{

-  auto iter = m_Files.find(index);

-

-  if (iter != m_Files.end()) {

-    unregisterFile(iter->second);

-    m_Files.erase(index);

-    return true;

-  } else {

-    log::error(QObject::tr("invalid file index for remove: {}").toStdString(), index);

-    return false;

-  }

-}

-

-void FileRegister::removeOrigin(FileEntry::Index index, int originID)

-{

-  auto iter = m_Files.find(index);

-

-  if (iter != m_Files.end()) {

-    if (iter->second->removeOrigin(originID)) {

-      unregisterFile(iter->second);

-      m_Files.erase(iter);

-    }

-  } else {

-    log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index);

+    addFromBSA(

+      originName, directory, archivePath.native(),

+      priority, order, stats);

   }

 }

 

-void FileRegister::removeOriginMulti(

-  std::set<FileEntry::Index> indices, int originID, time_t notAfter)

+void DirectoryEntry::addFromBSA(

+  const std::wstring& originName, const std::wstring& directory,

+  const std::wstring& archivePath, int priority, int order, DirectoryStats& stats)

 {

-  std::vector<FileEntry::Ptr> removedFiles;

-

-  for (auto iter = indices.begin(); iter != indices.end(); ) {

-    auto pos = m_Files.find(*iter);

-

-    if (pos != m_Files.end()

-        && (pos->second->lastAccessed() < notAfter)

-        && pos->second->removeOrigin(originID)) {

-      removedFiles.push_back(pos->second);

-      m_Files.erase(pos);

-      ++iter;

-    } else {

-      indices.erase(iter++);

-    }

-  }

-

-  // optimization: this is only called when disabling an origin and in this case

-  // we don't have to remove the file from the origin

-

-  // need to remove files from their parent directories. multiple ways to go

-  // about this:

-  //   a) for each file, search its parents file-list (preferably by name) and

-  //      remove what is found

-  //   b) gather the parent directories, go through the file list for each once

-  //      and remove all files that have been removed

-  //

-  // the latter should be faster when there are many files in few directories.

-  // since this is called only when disabling an origin that is probably

-  // frequently the case

-

-  std::set<DirectoryEntry*> parents;

-  for (const FileEntry::Ptr &file : removedFiles) {

-    if (file->getParent() != nullptr) {

-      parents.insert(file->getParent());

-    }

-  }

+  FilesOrigin& origin = createOrigin(originName, directory, priority, stats);

+  const auto archiveName = std::filesystem::path(archivePath).filename().native();

 

-  for (DirectoryEntry *parent : parents) {

-    parent->removeFiles(indices);

+  if (containsArchive(archiveName)) {

+    return;

   }

-}

-

-void FileRegister::sortOrigins()

-{

-  for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) {

-    iter->second->sortOrigins();

-  }

-}

-

-FileEntry::Index FileRegister::generateIndex()

-{

-  static std::atomic<FileEntry::Index> sIndex(0);

-  return sIndex++;

-}

-

-void FileRegister::unregisterFile(FileEntry::Ptr file)

-{

-  bool ignore;

 

-  // unregister from origin

-  int originID = file->getOrigin(ignore);

-  m_OriginConnection->getByID(originID).removeFile(file->getIndex());

-  const auto& alternatives = file->getAlternatives();

+  BSA::Archive archive;

+  BSA::EErrorCode res = archive.read(ToString(archivePath, false).c_str(), false);

 

-  for (auto iter = alternatives.begin(); iter != alternatives.end(); ++iter) {

-    m_OriginConnection->getByID(iter->first).removeFile(file->getIndex());

+  if ((res != BSA::ERROR_NONE) && (res != BSA::ERROR_INVALIDHASHES)) {

+    log::error("invalid bsa '{}', error {}", archivePath, res);

+    return;

   }

 

-  // unregister from directory

-  if (file->getParent() != nullptr) {

-    file->getParent()->removeFile(file->getIndex());

-  }

-}

-

-

-DirectoryEntry::DirectoryEntry(

-  const std::wstring &name, DirectoryEntry *parent, int originID) :

-    m_OriginConnection(new OriginConnection),

-    m_Name(name), m_Parent(parent), m_Populated(false), m_TopLevel(true)

-{

-  m_FileRegister.reset(new FileRegister(m_OriginConnection));

-  m_Origins.insert(originID);

-}

-

-DirectoryEntry::DirectoryEntry(

-  const std::wstring &name, DirectoryEntry *parent, int originID,

-  boost::shared_ptr<FileRegister> fileRegister,

-  boost::shared_ptr<OriginConnection> originConnection) :

-    m_FileRegister(fileRegister), m_OriginConnection(originConnection),

-    m_Name(name), m_Parent(parent), m_Populated(false), m_TopLevel(false)

-{

-  m_Origins.insert(originID);

-}

-

-DirectoryEntry::~DirectoryEntry()

-{

-  clear();

-}

-

-void DirectoryEntry::clear()

-{

-  m_Files.clear();

-  m_FilesLookup.clear();

+  std::error_code ec;

+  const auto lwt = std::filesystem::last_write_time(archivePath, ec);

+  FILETIME ft = {};

 

-  for (DirectoryEntry *entry : m_SubDirectories) {

-    delete entry;

+  if (ec) {

+    log::warn(

+      "failed to get last modified date for '{}', {}",

+      archivePath, ec.message());

+  } else {

+    ft = ToFILETIME(lwt);

   }

 

-  m_SubDirectories.clear();

-  m_SubDirectoriesLookup.clear();

-}

-

-void DirectoryEntry::addFromOrigin(

-  const std::wstring &originName, const std::wstring &directory, int priority)

-{

-  FilesOrigin &origin = createOrigin(originName, directory, priority);

-

-  if (directory.length() != 0) {

-    boost::scoped_array<wchar_t> buffer(new wchar_t[MAXPATH_UNICODE + 1]);

-    memset(buffer.get(), L'\0', MAXPATH_UNICODE + 1);

-    int offset = _snwprintf(buffer.get(), MAXPATH_UNICODE, L"%ls", directory.c_str());

-    buffer.get()[offset] = L'\0';

-    addFiles(origin, buffer.get(), offset);

-  }

+  addFiles(origin, archive.getRoot(), ft, archiveName, order, stats);

 

   m_Populated = true;

 }

 

-void DirectoryEntry::addFromBSA(

-  const std::wstring &originName, std::wstring &directory,

-  const std::wstring &fileName, int priority, int order)

-{

-  FilesOrigin &origin = createOrigin(originName, directory, priority);

-

-  WIN32_FILE_ATTRIBUTE_DATA fileData;

-  if (::GetFileAttributesExW(fileName.c_str(), GetFileExInfoStandard, &fileData) == 0) {

-    throw windows_error(QObject::tr("failed to determine file time").toStdString());

-  }

-

-  FILETIME now;

-  ::GetSystemTimeAsFileTime(&now);

-

-  const double clfSecondsPer100ns = 100. * 1.E-9;

-

-  ((ULARGE_INTEGER *)&now)->QuadPart -= ((double)5) / clfSecondsPer100ns;

-

-  size_t namePos = fileName.find_last_of(L"\\/");

-  if (namePos == std::wstring::npos) {

-    namePos = 0;

-  }

-  else {

-    ++namePos;

-  }

-

-  if (!containsArchive(fileName.substr(namePos)) || ::CompareFileTime(&fileData.ftLastWriteTime, &now) > 0) {

-    BSA::Archive archive;

-    BSA::EErrorCode res = archive.read(ToString(fileName, false).c_str(), false);

-

-    if ((res != BSA::ERROR_NONE) && (res != BSA::ERROR_INVALIDHASHES)) {

-      std::ostringstream stream;

-

-      stream

-		<< QObject::tr("invalid bsa file: ").toStdString()

-		<< ToString(fileName, false)

-		<< " error code " << res << " - " << ::GetLastError();

-

-      throw std::runtime_error(stream.str());

-    }

-

-    addFiles(origin, archive.getRoot(), fileData.ftLastWriteTime, fileName.substr(namePos), order);

-    m_Populated = true;

-  }

-}

-

 void DirectoryEntry::propagateOrigin(int origin)

 {

-  m_Origins.insert(origin);

+  {

+    std::scoped_lock lock(m_OriginsMutex);

+    m_Origins.insert(origin);

+  }

 

   if (m_Parent != nullptr) {

     m_Parent->propagateOrigin(origin);

@@ -769,7 +278,7 @@ int DirectoryEntry::anyOrigin() const
   bool ignore;

 

   for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) {

-    FileEntry::Ptr entry = m_FileRegister->getFile(iter->second);

+    FileEntryPtr entry = m_FileRegister->getFile(iter->second);

     if ((entry.get() != nullptr) && !entry->isFromArchive()) {

       return entry->getOrigin(ignore);

     }

@@ -777,9 +286,9 @@ int DirectoryEntry::anyOrigin() const
 

   // if we got here, no file directly within this directory is a valid indicator for a mod, thus

   // we continue looking in subdirectories

-  for (DirectoryEntry *entry : m_SubDirectories) {

+  for (DirectoryEntry* entry : m_SubDirectories) {

     int res = entry->anyOrigin();

-    if (res != -1){

+    if (res != InvalidOriginID){

       return res;

     }

   }

@@ -787,9 +296,9 @@ int DirectoryEntry::anyOrigin() const
   return *(m_Origins.begin());

 }

 

-std::vector<FileEntry::Ptr> DirectoryEntry::getFiles() const

+std::vector<FileEntryPtr> DirectoryEntry::getFiles() const

 {

-  std::vector<FileEntry::Ptr> result;

+  std::vector<FileEntryPtr> result;

 

   for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) {

     result.push_back(m_FileRegister->getFile(iter->second));

@@ -798,7 +307,7 @@ std::vector<FileEntry::Ptr> DirectoryEntry::getFiles() const
   return result;

 }

 

-DirectoryEntry *DirectoryEntry::findSubDirectory(

+DirectoryEntry* DirectoryEntry::findSubDirectory(

   const std::wstring &name, bool alreadyLowerCase) const

 {

   SubDirectoriesLookup::const_iterator itor;

@@ -816,37 +325,38 @@ DirectoryEntry *DirectoryEntry::findSubDirectory(
   return itor->second;

 }

 

-DirectoryEntry *DirectoryEntry::findSubDirectoryRecursive(const std::wstring &path)

+DirectoryEntry* DirectoryEntry::findSubDirectoryRecursive(const std::wstring &path)

 {

-  return getSubDirectoryRecursive(path, false, -1);

+  DirectoryStats dummy;

+  return getSubDirectoryRecursive(path, false, dummy, InvalidOriginID);

 }

 

-const FileEntry::Ptr DirectoryEntry::findFile(

+const FileEntryPtr DirectoryEntry::findFile(

   const std::wstring &name, bool alreadyLowerCase) const

 {

   FilesLookup::const_iterator iter;

 

   if (alreadyLowerCase) {

-    iter = m_FilesLookup.find(FileKey(name));

+    iter = m_FilesLookup.find(DirectoryEntryFileKey(name));

   } else {

-    iter = m_FilesLookup.find(FileKey(ToLowerCopy(name)));

+    iter = m_FilesLookup.find(DirectoryEntryFileKey(ToLowerCopy(name)));

   }

 

   if (iter != m_FilesLookup.end()) {

     return m_FileRegister->getFile(iter->second);

   } else {

-    return FileEntry::Ptr();

+    return FileEntryPtr();

   }

 }

 

-const FileEntry::Ptr DirectoryEntry::findFile(const FileKey& key) const

+const FileEntryPtr DirectoryEntry::findFile(const DirectoryEntryFileKey& key) const

 {

   auto iter = m_FilesLookup.find(key);

 

   if (iter != m_FilesLookup.end()) {

     return m_FileRegister->getFile(iter->second);

   } else {

-    return FileEntry::Ptr();

+    return FileEntryPtr();

   }

 }

 

@@ -858,7 +368,7 @@ bool DirectoryEntry::hasFile(const std::wstring& name) const
 bool DirectoryEntry::containsArchive(std::wstring archiveName)

 {

   for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) {

-    FileEntry::Ptr entry = m_FileRegister->getFile(iter->second);

+    FileEntryPtr entry = m_FileRegister->getFile(iter->second);

     if (entry->isFromArchive(archiveName)) {

       return true;

     }

@@ -867,8 +377,8 @@ bool DirectoryEntry::containsArchive(std::wstring archiveName)
   return false;

 }

 

-const FileEntry::Ptr DirectoryEntry::searchFile(

-  const std::wstring &path, const DirectoryEntry **directory) const

+const FileEntryPtr DirectoryEntry::searchFile(

+  const std::wstring &path, const DirectoryEntry** directory) const

 {

   if (directory != nullptr) {

     *directory = nullptr;

@@ -880,7 +390,7 @@ const FileEntry::Ptr DirectoryEntry::searchFile(
       *directory = this;

     }

 

-    return FileEntry::Ptr();

+    return FileEntryPtr();

   }

 

   const size_t len =  path.find_first_of(L"\\/");

@@ -892,7 +402,7 @@ const FileEntry::Ptr DirectoryEntry::searchFile(
     if (iter != m_Files.end()) {

       return m_FileRegister->getFile(iter->second);

     } else if (directory != nullptr) {

-      DirectoryEntry *temp = findSubDirectory(path);

+      DirectoryEntry* temp = findSubDirectory(path);

       if (temp != nullptr) {

         *directory = temp;

       }

@@ -900,41 +410,27 @@ const FileEntry::Ptr DirectoryEntry::searchFile(
   } else {

     // file is in a subdirectory, recurse into the matching subdirectory

     std::wstring pathComponent = path.substr(0, len);

-    DirectoryEntry *temp = findSubDirectory(pathComponent);

+    DirectoryEntry* temp = findSubDirectory(pathComponent);

 

     if (temp != nullptr) {

       if (len >= path.size()) {

         log::error(QObject::tr("unexpected end of path").toStdString());

-        return FileEntry::Ptr();

+        return FileEntryPtr();

       }

 

       return temp->searchFile(path.substr(len + 1), directory);

     }

   }

 

-  return FileEntry::Ptr();

-}

-

-void DirectoryEntry::insertFile(

-  const std::wstring &filePath, FilesOrigin &origin, FILETIME fileTime)

-{

-  size_t pos = filePath.find_first_of(L"\\/");

-

-  if (pos == std::string::npos) {

-    this->insert(filePath, origin, fileTime, std::wstring(), -1);

-  } else {

-    std::wstring dirName = filePath.substr(0, pos);

-    std::wstring rest = filePath.substr(pos + 1);

-    getSubDirectoryRecursive(dirName, true, origin.getID())->insertFile(rest, origin, fileTime);

-  }

+  return FileEntryPtr();

 }

 

-void DirectoryEntry::removeFile(FileEntry::Index index)

+void DirectoryEntry::removeFile(FileIndex index)

 {

   removeFileFromList(index);

 }

 

-bool DirectoryEntry::removeFile(const std::wstring &filePath, int *origin)

+bool DirectoryEntry::removeFile(const std::wstring &filePath, int* origin)

 {

   size_t pos = filePath.find_first_of(L"\\/");

 

@@ -944,7 +440,9 @@ bool DirectoryEntry::removeFile(const std::wstring &filePath, int *origin)
 

   std::wstring dirName = filePath.substr(0, pos);

   std::wstring rest = filePath.substr(pos + 1);

-  DirectoryEntry *entry = getSubDirectoryRecursive(dirName, false);

+

+  DirectoryStats dummy;

+  DirectoryEntry* entry = getSubDirectoryRecursive(dirName, false, dummy);

 

   if (entry != nullptr) {

     return entry->removeFile(rest, origin);

@@ -959,7 +457,7 @@ void DirectoryEntry::removeDir(const std::wstring &path)
 

   if (pos == std::string::npos) {

     for (auto iter = m_SubDirectories.begin(); iter != m_SubDirectories.end(); ++iter) {

-      DirectoryEntry *entry = *iter;

+      DirectoryEntry* entry = *iter;

 

       if (CaseInsensitiveEqual(entry->getName(), path)) {

         entry->removeDirRecursive();

@@ -971,7 +469,9 @@ void DirectoryEntry::removeDir(const std::wstring &path)
   } else {

     std::wstring dirName = path.substr(0, pos);

     std::wstring rest = path.substr(pos + 1);

-    DirectoryEntry *entry = getSubDirectoryRecursive(dirName, false);

+

+    DirectoryStats dummy;

+    DirectoryEntry* entry = getSubDirectoryRecursive(dirName, false, dummy);

 

     if (entry != nullptr) {

       entry->removeDir(rest);

@@ -979,7 +479,7 @@ void DirectoryEntry::removeDir(const std::wstring &path)
   }

 }

 

-bool DirectoryEntry::remove(const std::wstring &fileName, int *origin)

+bool DirectoryEntry::remove(const std::wstring &fileName, int* origin)

 {

   const auto lcFileName = ToLowerCopy(fileName);

 

@@ -988,7 +488,7 @@ bool DirectoryEntry::remove(const std::wstring &fileName, int *origin)
 

   if (iter != m_Files.end()) {

     if (origin != nullptr) {

-      FileEntry::Ptr entry = m_FileRegister->getFile(iter->second);

+      FileEntryPtr entry = m_FileRegister->getFile(iter->second);

       if (entry.get() != nullptr) {

         bool ignore;

         *origin = entry->getOrigin(ignore);

@@ -1007,98 +507,204 @@ bool DirectoryEntry::hasContentsFromOrigin(int originID) const
 }

 

 FilesOrigin &DirectoryEntry::createOrigin(

-  const std::wstring &originName, const std::wstring &directory, int priority)

+  const std::wstring &originName, const std::wstring &directory, int priority,

+  DirectoryStats& stats)

 {

-  if (m_OriginConnection->exists(originName)) {

-    FilesOrigin &origin = m_OriginConnection->getByName(originName);

-    origin.enable(true);

-    return origin;

+  auto r = m_OriginConnection->getOrCreate(

+    originName, directory, priority,

+    m_FileRegister, m_OriginConnection, stats);

+

+  if (r.second) {

+    ++stats.originCreate;

   } else {

-    return m_OriginConnection->createOrigin(

-      originName, directory, priority, m_FileRegister, m_OriginConnection);

+    ++stats.originExists;

   }

+

+  return r.first;

 }

 

-void DirectoryEntry::removeFiles(const std::set<FileEntry::Index> &indices)

+void DirectoryEntry::removeFiles(const std::set<FileIndex> &indices)

 {

   removeFilesFromList(indices);

 }

 

-FileEntry::Ptr DirectoryEntry::insert(

-  const std::wstring &fileName, FilesOrigin &origin, FILETIME fileTime,

-  const std::wstring &archive, int order)

+FileEntryPtr DirectoryEntry::insert(

+  std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime,

+  std::wstring_view archive, int order, DirectoryStats& stats)

 {

   std::wstring fileNameLower = ToLowerCopy(fileName);

+  FileEntryPtr fe;

 

-  auto iter = m_Files.find(fileNameLower);

-  FileEntry::Ptr file;

+  DirectoryEntryFileKey key(std::move(fileNameLower));

 

-  if (iter != m_Files.end()) {

-    file = m_FileRegister->getFile(iter->second);

-  } else {

-    file = m_FileRegister->createFile(fileName, this);

-    addFileToList(std::move(fileNameLower), file->getIndex());

+  {

+    std::unique_lock lock(m_FilesMutex);

+

+    FilesLookup::iterator itor;

+

+    elapsed(stats.filesLookupTimes, [&]{

+      itor = m_FilesLookup.find(key);

+    });

+

+    if (itor != m_FilesLookup.end()) {

+      lock.unlock();

+      ++stats.fileExists;

+      fe = m_FileRegister->getFile(itor->second);

+    } else {

+      ++stats.fileCreate;

+      fe = m_FileRegister->createFile(

+        std::wstring(fileName.begin(), fileName.end()), this, stats);

 

-    // fileNameLower has moved from this point

+      elapsed(stats.addFileTimes, [&] {

+        addFileToList(std::move(key.value), fe->getIndex());

+      });

+

+      // fileNameLower has moved from this point

+    }

   }

 

-  file->addOrigin(origin.getID(), fileTime, archive, order);

-  origin.addFile(file->getIndex());

+  elapsed(stats.addOriginToFileTimes, [&]{

+    fe->addOrigin(origin.getID(), fileTime, archive, order);

+  });

 

-  return file;

+  elapsed(stats.addFileToOriginTimes, [&]{

+    origin.addFile(fe->getIndex());

+  });

+

+  return fe;

 }

 

-void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOffset)

+FileEntryPtr DirectoryEntry::insert(

+  env::File& file, FilesOrigin &origin, std::wstring_view archive, int order,

+  DirectoryStats& stats)

 {

-  WIN32_FIND_DATAW findData;

+  FileEntryPtr fe;

 

-  _snwprintf_s(buffer + bufferOffset, MAXPATH_UNICODE - bufferOffset, _TRUNCATE, L"\\*");

+  {

+    std::unique_lock lock(m_FilesMutex);

 

-  HANDLE searchHandle = nullptr;

+    FilesMap::iterator itor;

 

-  if (SupportOptimizedFind()) {

-    searchHandle = ::FindFirstFileExW(

-      buffer, FindExInfoBasic, &findData, FindExSearchNameMatch, nullptr,

-      FIND_FIRST_EX_LARGE_FETCH);

-  } else {

-    searchHandle = ::FindFirstFileExW(

-      buffer, FindExInfoStandard, &findData, FindExSearchNameMatch, nullptr, 0);

+    elapsed(stats.filesLookupTimes, [&]{

+      itor = m_Files.find(file.lcname);

+    });

+

+    if (itor != m_Files.end()) {

+      lock.unlock();

+      ++stats.fileExists;

+      fe = m_FileRegister->getFile(itor->second);

+    } else {

+      ++stats.fileCreate;

+      fe = m_FileRegister->createFile(std::move(file.name), this, stats);

+      // file.name has been moved from this point

+

+      elapsed(stats.addFileTimes, [&]{

+        addFileToList(std::move(file.lcname), fe->getIndex());

+      });

+

+      // file.lcname has been moved from this point

+    }

   }

 

-  if (searchHandle != INVALID_HANDLE_VALUE) {

-    BOOL result = true;

+  elapsed(stats.addOriginToFileTimes, [&]{

+    fe->addOrigin(origin.getID(), file.lastModified, archive, order);

+  });

+

+  elapsed(stats.addFileToOriginTimes, [&]{

+    origin.addFile(fe->getIndex());

+  });

 

-    while (result) {

-      if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {

-        if ((wcscmp(findData.cFileName, L".") != 0) &&

-            (wcscmp(findData.cFileName, L"..") != 0)) {

-          int offset = _snwprintf(buffer + bufferOffset, MAXPATH_UNICODE, L"\\%ls", findData.cFileName);

+  return fe;

+}

 

-          // recurse into subdirectories

-          DirectoryEntry* sd = getSubDirectory(findData.cFileName, true, origin.getID());

-          sd->addFiles(origin, buffer, bufferOffset + offset);

-        }

-      } else {

-        insert(findData.cFileName, origin, findData.ftLastWriteTime, L"", -1);

-      }

+struct DirectoryEntry::Context

+{

+  FilesOrigin& origin;

+  DirectoryStats& stats;

+  std::stack<DirectoryEntry*> current;

+};

+

+void DirectoryEntry::addFiles(

+  env::DirectoryWalker& walker, FilesOrigin &origin,

+  const std::wstring& path, DirectoryStats& stats)

+{

+  Context cx = {origin, stats};

+  cx.current.push(this);

+

+  walker.forEachEntry(path, &cx,

+    [](void* pcx, std::wstring_view path)

+    {

+      onDirectoryStart((Context*)pcx, path);

+    },

+

+    [](void* pcx, std::wstring_view path)

+    {

+      onDirectoryEnd((Context*)pcx, path);

+    },

 

-      result = ::FindNextFileW(searchHandle, &findData);

+    [](void* pcx, std::wstring_view path, FILETIME ft)

+    {

+      onFile((Context*)pcx, path, ft);

     }

+  );

+

+  {

+    std::scoped_lock lock(m_SubDirMutex);

+

+    std::sort(

+      m_SubDirectories.begin(),

+      m_SubDirectories.end(),

+      &DirCompareByName);

   }

+}

+

+void DirectoryEntry::onDirectoryStart(Context* cx, std::wstring_view path)

+{

+  elapsed(cx->stats.dirTimes, [&] {

+    auto* sd = cx->current.top()->getSubDirectory(

+      path, true, cx->stats, cx->origin.getID());

+

+    cx->current.push(sd);

+  });

+}

+

+void DirectoryEntry::onDirectoryEnd(Context* cx, std::wstring_view path)

+{

+  elapsed(cx->stats.dirTimes, [&] {

+    auto* current = cx->current.top();

+

+    {

+      std::scoped_lock lock(current->m_SubDirMutex);

+

+      std::sort(

+        current->m_SubDirectories.begin(),

+        current->m_SubDirectories.end(),

+        &DirCompareByName);

+    }

 

-  std::sort(m_SubDirectories.begin(), m_SubDirectories.end(), &DirCompareByName);

-  ::FindClose(searchHandle);

+    cx->current.pop();

+  });

+}

+

+void DirectoryEntry::onFile(Context* cx, std::wstring_view path, FILETIME ft)

+{

+  elapsed(cx->stats.fileTimes, [&]{

+    cx->current.top()->insert(path, cx->origin, ft, L"", -1, cx->stats);

+  });

 }

 

 void DirectoryEntry::addFiles(

-  FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime,

-  const std::wstring &archiveName, int order)

+  FilesOrigin& origin, const BSA::Folder::Ptr archiveFolder, FILETIME fileTime,

+  const std::wstring& archiveName, int order, DirectoryStats& stats)

 {

   // add files

-  for (unsigned int fileIdx = 0; fileIdx < archiveFolder->getNumFiles(); ++fileIdx) {

-    BSA::File::Ptr file = archiveFolder->getFile(fileIdx);

+  const auto fileCount = archiveFolder->getNumFiles();

+  for (unsigned int i=0; i<fileCount; ++i) {

+    const BSA::File::Ptr file = archiveFolder->getFile(i);

 

-    auto f = insert(ToWString(file->getName(), true), origin, fileTime, archiveName, order);

+    auto f = insert(

+      ToWString(file->getName(), true), origin, fileTime,

+      archiveName, order, stats);

 

     if (f) {

       if (file->getUncompressedFileSize() > 0) {

@@ -1110,28 +716,45 @@ void DirectoryEntry::addFiles(
   }

 

   // recurse into subdirectories

-  for (unsigned int folderIdx = 0; folderIdx < archiveFolder->getNumSubFolders(); ++folderIdx) {

-    BSA::Folder::Ptr folder = archiveFolder->getSubFolder(folderIdx);

-    DirectoryEntry *folderEntry = getSubDirectoryRecursive(ToWString(folder->getName(), true), true, origin.getID());

+  const auto dirCount = archiveFolder->getNumSubFolders();

+  for (unsigned int i=0; i<dirCount; ++i) {

+    const BSA::Folder::Ptr folder = archiveFolder->getSubFolder(i);

+

+    DirectoryEntry* folderEntry = getSubDirectoryRecursive(

+      ToWString(folder->getName(), true), true, stats, origin.getID());

 

-    folderEntry->addFiles(origin, folder, fileTime, archiveName, order);

+    folderEntry->addFiles(origin, folder, fileTime, archiveName, order, stats);

   }

 }

 

-DirectoryEntry *DirectoryEntry::getSubDirectory(

-  const std::wstring &name, bool create, int originID)

+DirectoryEntry* DirectoryEntry::getSubDirectory(

+  std::wstring_view name, bool create, DirectoryStats& stats, int originID)

 {

-  for (DirectoryEntry *entry : m_SubDirectories) {

-    if (CaseInsensitiveEqual(entry->getName(), name)) {

-      return entry;

-    }

+  std::wstring nameLc = ToLowerCopy(name);

+

+  std::scoped_lock lock(m_SubDirMutex);

+

+  SubDirectoriesLookup::iterator itor;

+  elapsed(stats.subdirLookupTimes, [&] {

+    itor = m_SubDirectoriesLookup.find(nameLc);

+  });

+

+  if (itor != m_SubDirectoriesLookup.end()) {

+    ++stats.subdirExists;

+    return itor->second;

   }

 

   if (create) {

+    ++stats.subdirCreate;

+

     auto* entry = new DirectoryEntry(

-      name, this, originID, m_FileRegister, m_OriginConnection);

+      std::wstring(name.begin(), name.end()), this, originID,

+      m_FileRegister, m_OriginConnection);

 

-    addDirectoryToList(entry);

+    elapsed(stats.addDirectoryTimes, [&] {

+      addDirectoryToList(entry, std::move(nameLc));

+      // nameLc is moved from this point

+    });

 

     return entry;

   } else {

@@ -1139,8 +762,44 @@ DirectoryEntry *DirectoryEntry::getSubDirectory(
   }

 }

 

-DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive(

-  const std::wstring &path, bool create, int originID)

+DirectoryEntry* DirectoryEntry::getSubDirectory(

+  env::Directory& dir, bool create, DirectoryStats& stats, int originID)

+{

+  SubDirectoriesLookup::iterator itor;

+

+  std::scoped_lock lock(m_SubDirMutex);

+

+  elapsed(stats.subdirLookupTimes, [&] {

+    itor = m_SubDirectoriesLookup.find(dir.lcname);

+  });

+

+  if (itor != m_SubDirectoriesLookup.end()) {

+    ++stats.subdirExists;

+    return itor->second;

+  }

+

+  if (create) {

+    ++stats.subdirCreate;

+

+    auto* entry = new DirectoryEntry(

+      std::move(dir.name), this, originID,

+      m_FileRegister, m_OriginConnection);

+    // dir.name is moved from this point

+

+    elapsed(stats.addDirectoryTimes, [&]{

+      addDirectoryToList(entry, std::move(dir.lcname));

+    });

+

+    // dir.lcname is moved from this point

+

+    return entry;

+  } else {

+    return nullptr;

+  }

+}

+

+DirectoryEntry* DirectoryEntry::getSubDirectoryRecursive(

+  const std::wstring& path, bool create, DirectoryStats& stats, int originID)

 {

   if (path.length() == 0) {

     // path ended with a backslash?

@@ -1150,14 +809,16 @@ DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive(
   const size_t pos = path.find_first_of(L"\\/");

 

   if (pos == std::wstring::npos) {

-    return getSubDirectory(path, create);

+    return getSubDirectory(path, create, stats);

   } else {

-    DirectoryEntry *nextChild = getSubDirectory(path.substr(0, pos), create, originID);

+    DirectoryEntry* nextChild = getSubDirectory(

+      path.substr(0, pos), create, stats, originID);

 

     if (nextChild == nullptr) {

       return nullptr;

     } else {

-      return nextChild->getSubDirectoryRecursive(path.substr(pos + 1), create, originID);

+      return nextChild->getSubDirectoryRecursive(

+        path.substr(pos + 1), create, stats, originID);

     }

   }

 }

@@ -1170,7 +831,7 @@ void DirectoryEntry::removeDirRecursive()
 

   m_FilesLookup.clear();

 

-  for (DirectoryEntry *entry : m_SubDirectories) {

+  for (DirectoryEntry* entry : m_SubDirectories) {

     entry->removeDirRecursive();

     delete entry;

   }

@@ -1179,10 +840,10 @@ void DirectoryEntry::removeDirRecursive()
   m_SubDirectoriesLookup.clear();

 }

 

-void DirectoryEntry::addDirectoryToList(DirectoryEntry* e)

+void DirectoryEntry::addDirectoryToList(DirectoryEntry* e, std::wstring nameLc)

 {

   m_SubDirectories.push_back(e);

-  m_SubDirectoriesLookup.emplace(ToLowerCopy(e->getName()), e);

+  m_SubDirectoriesLookup.emplace(std::move(nameLc), e);

 }

 

 void DirectoryEntry::removeDirectoryFromList(SubDirectories::iterator itor)

@@ -1204,7 +865,7 @@ void DirectoryEntry::removeDirectoryFromList(SubDirectories::iterator itor)
   m_SubDirectories.erase(itor);

 }

 

-void DirectoryEntry::removeFileFromList(FileEntry::Index index)

+void DirectoryEntry::removeFileFromList(FileIndex index)

 {

   auto removeFrom = [&](auto& list) {

     auto iter = std::find_if(

@@ -1234,7 +895,7 @@ void DirectoryEntry::removeFileFromList(FileEntry::Index index)
   removeFrom(m_Files);

 }

 

-void DirectoryEntry::removeFilesFromList(const std::set<FileEntry::Index>& indices)

+void DirectoryEntry::removeFilesFromList(const std::set<FileIndex>& indices)

 {

   for (auto iter = m_Files.begin(); iter != m_Files.end();) {

     if (indices.find(iter->second) != indices.end()) {

@@ -1253,12 +914,79 @@ void DirectoryEntry::removeFilesFromList(const std::set<FileEntry::Index>& indic
   }

 }

 

-void DirectoryEntry::addFileToList(

-  std::wstring fileNameLower, FileEntry::Index index)

+void DirectoryEntry::addFileToList(std::wstring fileNameLower, FileIndex index)

 {

   m_FilesLookup.emplace(fileNameLower, index);

   m_Files.emplace(std::move(fileNameLower), index);

   // fileNameLower has been moved from this point

 }

 

+struct DumpFailed : public std::runtime_error

+{

+  using runtime_error::runtime_error;

+};

+

+void DirectoryEntry::dump(const std::wstring& file) const

+{

+  try

+  {

+    std::FILE* f = nullptr;

+    auto e = _wfopen_s(&f, file.c_str(), L"wb");

+

+    if (e != 0 || !f) {

+      throw DumpFailed(fmt::format(

+        "failed to open, {} ({})", std::strerror(e), e));

+    }

+

+    Guard g([&]{ std::fclose(f); });

+

+    dump(f, L"Data");

+  }

+  catch(DumpFailed& e)

+  {

+    log::error(

+      "failed to write list to '{}': {}",

+      QString::fromStdWString(file).toStdString(), e.what());

+  }

+}

+

+void DirectoryEntry::dump(std::FILE* f, const std::wstring& parentPath) const

+{

+  {

+    std::scoped_lock lock(m_FilesMutex);

+

+    for (auto&& index : m_Files) {

+      const auto file = m_FileRegister->getFile(index.second);

+      if (!file) {

+        continue;

+      }

+

+      if (file->isFromArchive()) {

+        // TODO: don't list files from archives. maybe make this an option?

+        continue;

+      }

+

+      const auto& o = m_OriginConnection->getByID(file->getOrigin());

+      const auto path = parentPath + L"\\" + file->getName();

+      const auto line = path + L"\t(" + o.getName() + L")\r\n";

+

+      const auto lineu8 = MOShared::ToString(line, true);

+

+      if (std::fwrite(lineu8.data(), lineu8.size(), 1, f) != 1) {

+        const auto e = errno;

+        throw DumpFailed(fmt::format(

+          "failed to write, {} ({})", std::strerror(e), e));

+      }

+    }

+  }

+

+  {

+    std::scoped_lock lock(m_SubDirMutex);

+    for (auto&& d : m_SubDirectories) {

+      const auto path = parentPath + L"\\" + d->m_Name;

+      d->dump(f, path);

+    }

+  }

+}

+

 } // namespace MOShared

diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h
index e26011d7..a28ceeae 100644
--- a/src/shared/directoryentry.h
+++ b/src/shared/directoryentry.h
@@ -17,25 +17,18 @@ You should have received a copy of the GNU General Public License
 along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.

 */

 

-#ifndef DIRECTORYENTRY_H

-#define DIRECTORYENTRY_H

+#ifndef MO_REGISTER_DIRECTORYENTRY_INCLUDED

+#define MO_REGISTER_DIRECTORYENTRY_INCLUDED

 

-

-#include <string>

-#include <set>

-#include <vector>

-#include <map>

-#include <cassert>

-#define WIN32_MEAN_AND_LEAN

-#include <Windows.h>

+#include "fileregister.h"

 #include <bsatk.h>

-#ifndef Q_MOC_RUN

-#include <boost/shared_ptr.hpp>

-#include <boost/weak_ptr.hpp>

-#endif

-#include "util.h"

 

-namespace MOShared { struct DirectoryEntryFileKey; }

+namespace env

+{

+  class DirectoryWalker;

+  struct Directory;

+  struct File;

+}

 

 namespace std

 {

@@ -53,274 +46,23 @@ namespace std
 namespace MOShared

 {

 

-class DirectoryEntry;

-class OriginConnection;

-class FileRegister;

-

-

-class FileEntry

-{

-public:

-  static constexpr uint64_t NoFileSize =

-    std::numeric_limits<uint64_t>::max();

-

-  typedef unsigned int Index;

-  typedef boost::shared_ptr<FileEntry> Ptr;

-

-  // a vector of {originId, {archiveName, order}}

-  //

-  // if a file is in an archive, archiveName is the name of the bsa and order

-  // is the order of the associated plugin in the plugins list

-  //

-  // is a file is not in an archive, archiveName is empty and order is usually

-  // -1

-  typedef std::vector<std::pair<int, std::pair<std::wstring, int>>>

-    AlternativesVector;

-

-  FileEntry();

-  FileEntry(Index index, const std::wstring &name, DirectoryEntry *parent);

-

-  Index getIndex() const

-  {

-    return m_Index;

-  }

-

-  time_t lastAccessed() const

-  {

-    return m_LastAccessed;

-  }

-

-  void addOrigin(

-    int origin, FILETIME fileTime, const std::wstring &archive, int order);

-

-  // remove the specified origin from the list of origins that contain this

-  // file. if no origin is left, the file is effectively deleted and true is

-  // returned. otherwise, false is returned

-  bool removeOrigin(int origin);

-

-  void sortOrigins();

-

-  // gets the list of alternative origins (origins with lower priority than

-  // the primary one). if sortOrigins has been called, it is sorted by priority

-  // (ascending)

-  const AlternativesVector &getAlternatives() const

-  {

-    return m_Alternatives;

-  }

-

-  const std::wstring &getName() const

-  {

-    return m_Name;

-  }

-

-  int getOrigin() const

-  {

-    return m_Origin;

-  }

-

-  int getOrigin(bool &archive) const

-  {

-    archive = (m_Archive.first.length() != 0);

-    return m_Origin;

-  }

-

-  const std::pair<std::wstring, int> &getArchive() const

-  {

-    return m_Archive;

-  }

-

-  bool isFromArchive(std::wstring archiveName = L"") const;

-

-  // if originID is -1, uses the main origin; if this file doesn't exist in the

-  // given origin, returns an empty string

-  //

-  std::wstring getFullPath(int originID=-1) const;

-

-  std::wstring getRelativePath() const;

-

-  DirectoryEntry *getParent()

-  {

-    return m_Parent;

-  }

-

-  void setFileTime(FILETIME fileTime) const

-  {

-    m_FileTime = fileTime;

-  }

-

-  FILETIME getFileTime() const

-  {

-    return m_FileTime;

-  }

-

-  void setFileSize(uint64_t size, uint64_t compressedSize)

-  {

-    m_FileSize = size;

-    m_CompressedFileSize = compressedSize;

-  }

-

-  uint64_t getFileSize() const

-  {

-    return m_FileSize;

-  }

-

-  uint64_t getCompressedFileSize() const

-  {

-    return m_CompressedFileSize;

-  }

-

-private:

-  Index m_Index;

-  std::wstring m_Name;

-  int m_Origin;

-  std::pair<std::wstring, int> m_Archive;

-  AlternativesVector m_Alternatives;

-  DirectoryEntry *m_Parent;

-  mutable FILETIME m_FileTime;

-  uint64_t m_FileSize, m_CompressedFileSize;

-

-  time_t m_LastAccessed;

-

-  bool recurseParents(std::wstring &path, const DirectoryEntry *parent) const;

-};

-

-

-// represents a mod or the data directory, providing files to the tree

-class FilesOrigin

-{

-  friend class OriginConnection;

-

-public:

-  FilesOrigin();

-  FilesOrigin(const FilesOrigin &reference);

-

-  // sets priority for this origin, but it will overwrite the existing mapping

-  // for this priority, the previous origin will no longer be referenced

-  void setPriority(int priority);

-

-  int getPriority() const

-  {

-    return m_Priority;

-  }

-

-  void setName(const std::wstring &name);

-  const std::wstring &getName() const

-  {

-    return m_Name;

-  }

-

-  int getID() const

-  {

-    return m_ID;

-  }

-

-  const std::wstring &getPath() const

-  {

-    return m_Path;

-  }

-

-  std::vector<FileEntry::Ptr> getFiles() const;

-  FileEntry::Ptr findFile(FileEntry::Index index) const;

-

-  void enable(bool enabled, time_t notAfter = LONG_MAX);

-  bool isDisabled() const

-  {

-    return m_Disabled;

-  }

-

-  void addFile(FileEntry::Index index)

-  {

-    m_Files.insert(index);

-  }

-

-  void removeFile(FileEntry::Index index);

-

-  bool containsArchive(std::wstring archiveName);

-

-private:

-  int m_ID;

-  bool m_Disabled;

-  std::set<FileEntry::Index> m_Files;

-  std::wstring m_Name;

-  std::wstring m_Path;

-  int m_Priority;

-  boost::weak_ptr<FileRegister> m_FileRegister;

-  boost::weak_ptr<OriginConnection> m_OriginConnection;

-

-  FilesOrigin(

-    int ID, const std::wstring &name, const std::wstring &path, int priority,

-    boost::shared_ptr<FileRegister> fileRegister,

-    boost::shared_ptr<OriginConnection> originConnection);

-};

-

-

-class FileRegister

-{

-public:

-  FileRegister(boost::shared_ptr<OriginConnection> originConnection);

-

-  bool indexValid(FileEntry::Index index) const;

-

-  FileEntry::Ptr createFile(const std::wstring &name, DirectoryEntry *parent);

-  FileEntry::Ptr getFile(FileEntry::Index index) const;

-

-  size_t size() const

-  {

-    return m_Files.size();

-  }

-

-  bool removeFile(FileEntry::Index index);

-  void removeOrigin(FileEntry::Index index, int originID);

-  void removeOriginMulti(std::set<FileEntry::Index> indices, int originID, time_t notAfter);

-

-  void sortOrigins();

-

-private:

-  std::map<FileEntry::Index, FileEntry::Ptr> m_Files;

-  boost::shared_ptr<OriginConnection> m_OriginConnection;

-

-  FileEntry::Index generateIndex();

-  void unregisterFile(FileEntry::Ptr file);

-};

-

-

-struct DirectoryEntryFileKey

-{

-  DirectoryEntryFileKey(std::wstring v)

-    : value(std::move(v)), hash(getHash(value))

-  {

-  }

-

-  bool operator==(const DirectoryEntryFileKey& o) const

-  {

-    return (value == o.value);

-  }

-

-  static std::size_t getHash(const std::wstring& value)

-  {

-    return std::hash<std::wstring>()(value);

-  }

-

-  const std::wstring value;

-  const std::size_t hash;

-};

-

-

 class DirectoryEntry

 {

 public:

-  using FileKey = DirectoryEntryFileKey;

-

   DirectoryEntry(

-    const std::wstring &name, DirectoryEntry *parent, int originID);

+    std::wstring name, DirectoryEntry* parent, OriginID originID);

 

   DirectoryEntry(

-    const std::wstring &name, DirectoryEntry *parent, int originID,

+    std::wstring name, DirectoryEntry* parent, OriginID originID,

     boost::shared_ptr<FileRegister> fileRegister,

     boost::shared_ptr<OriginConnection> originConnection);

 

   ~DirectoryEntry();

 

+  // noncopyable

+  DirectoryEntry(const DirectoryEntry&) = delete;

+  DirectoryEntry& operator=(const DirectoryEntry&) = delete;

+

   void clear();

 

   bool isPopulated() const

@@ -343,7 +85,7 @@ public:
     return !m_Files.empty();

   }

 

-  const DirectoryEntry *getParent() const

+  const DirectoryEntry* getParent() const

   {

     return m_Parent;

   }

@@ -351,16 +93,32 @@ public:
   // add files to this directory (and subdirectories) from the specified origin.

   // That origin may exist or not

   void addFromOrigin(

-    const std::wstring &originName,

-    const std::wstring &directory, int priority);

+    const std::wstring& originName,

+    const std::wstring& directory, int priority, DirectoryStats& stats);

+

+  void addFromOrigin(

+    env::DirectoryWalker& walker, const std::wstring& originName,

+    const std::wstring& directory, int priority, DirectoryStats& stats);

+

+  void addFromAllBSAs(

+    const std::wstring& originName, const std::wstring& directory,

+    int priority, const std::vector<std::wstring>& archives,

+    const std::set<std::wstring>& enabledArchives,

+    const std::vector<std::wstring>& loadOrder,

+    DirectoryStats& stats);

 

   void addFromBSA(

-    const std::wstring &originName, std::wstring &directory,

-    const std::wstring &fileName, int priority, int order);

+    const std::wstring& originName, const std::wstring& directory,

+    const std::wstring& archivePath, int priority, int order,

+    DirectoryStats& stats);

+

+  void addFromList(

+    const std::wstring& originName, const std::wstring& directory,

+    env::Directory& root, int priority, DirectoryStats& stats);

 

-  void propagateOrigin(int origin);

+  void propagateOrigin(OriginID origin);

 

-  const std::wstring &getName() const

+  const std::wstring& getName() const

   {

     return m_Name;

   }

@@ -370,18 +128,18 @@ public:
     return m_FileRegister;

   }

 

-  bool originExists(const std::wstring &name) const;

-  FilesOrigin &getOriginByID(int ID) const;

-  FilesOrigin &getOriginByName(const std::wstring &name) const;

-  const FilesOrigin* findOriginByID(int ID) const;

+  bool originExists(const std::wstring& name) const;

+  FilesOrigin& getOriginByID(OriginID ID) const;

+  FilesOrigin& getOriginByName(const std::wstring& name) const;

+  const FilesOrigin* findOriginByID(OriginID ID) const;

 

-  int anyOrigin() const;

+  OriginID anyOrigin() const;

 

-  std::vector<FileEntry::Ptr> getFiles() const;

+  std::vector<FileEntryPtr> getFiles() const;

 

   void getSubDirectories(

-    std::vector<DirectoryEntry*>::const_iterator &begin,

-    std::vector<DirectoryEntry*>::const_iterator &end) const

+    std::vector<DirectoryEntry*>::const_iterator& begin,

+    std::vector<DirectoryEntry*>::const_iterator& end) const

   {

     begin = m_SubDirectories.begin();

     end = m_SubDirectories.end();

@@ -424,22 +182,22 @@ public:
     }

   }

 

-  FileEntry::Ptr getFileByIndex(FileEntry::Index index) const

+  FileEntryPtr getFileByIndex(FileIndex index) const

   {

     return m_FileRegister->getFile(index);

   }

 

-  DirectoryEntry *findSubDirectory(

-    const std::wstring &name, bool alreadyLowerCase=false) const;

+  DirectoryEntry* findSubDirectory(

+    const std::wstring& name, bool alreadyLowerCase=false) const;

 

-  DirectoryEntry *findSubDirectoryRecursive(const std::wstring &path);

+  DirectoryEntry* findSubDirectoryRecursive(const std::wstring& path);

 

   /** retrieve a file in this directory by name.

     * @param name name of the file

     * @return fileentry object for the file or nullptr if no file matches

     */

-  const FileEntry::Ptr findFile(const std::wstring &name, bool alreadyLowerCase=false) const;

-  const FileEntry::Ptr findFile(const FileKey& key) const;

+  const FileEntryPtr findFile(const std::wstring& name, bool alreadyLowerCase=false) const;

+  const FileEntryPtr findFile(const DirectoryEntryFileKey& key) const;

 

   bool hasFile(const std::wstring& name) const;

   bool containsArchive(std::wstring archiveName);

@@ -450,36 +208,36 @@ public:
   // if directory is not nullptr, the referenced variable will be set to the

   // path containing the file

   //

-  const FileEntry::Ptr searchFile(

-    const std::wstring &path, const DirectoryEntry **directory=nullptr) const;

+  const FileEntryPtr searchFile(

+    const std::wstring& path, const DirectoryEntry** directory=nullptr) const;

 

-  void insertFile(const std::wstring &filePath, FilesOrigin &origin, FILETIME fileTime);

-

-  void removeFile(FileEntry::Index index);

+  void removeFile(FileIndex index);

 

   // remove the specified file from the tree. This can be a path leading to a

   // file in a subdirectory

-  bool removeFile(const std::wstring &filePath, int *origin = nullptr);

+  bool removeFile(const std::wstring& filePath, OriginID* origin = nullptr);

 

   /**

    * @brief remove the specified directory

    * @param path directory to remove

    */

-  void removeDir(const std::wstring &path);

+  void removeDir(const std::wstring& path);

+

+  bool remove(const std::wstring& fileName, OriginID* origin);

 

-  bool remove(const std::wstring &fileName, int *origin);

+  bool hasContentsFromOrigin(OriginID originID) const;

 

-  bool hasContentsFromOrigin(int originID) const;

+  FilesOrigin& createOrigin(

+    const std::wstring& originName,

+    const std::wstring& directory, int priority, DirectoryStats& stats);

 

-  FilesOrigin &createOrigin(

-    const std::wstring &originName,

-    const std::wstring &directory, int priority);

+  void removeFiles(const std::set<FileIndex>& indices);

 

-  void removeFiles(const std::set<FileEntry::Index> &indices);

+  void dump(const std::wstring& file) const;

 

 private:

-  using FilesMap = std::map<std::wstring, FileEntry::Index>;

-  using FilesLookup = std::unordered_map<FileKey, FileEntry::Index>;

+  using FilesMap = std::map<std::wstring, FileIndex>;

+  using FilesLookup = std::unordered_map<DirectoryEntryFileKey, FileIndex>;

   using SubDirectories = std::vector<DirectoryEntry*>;

   using SubDirectoriesLookup = std::unordered_map<std::wstring, DirectoryEntry*>;

 

@@ -492,39 +250,60 @@ private:
   SubDirectories m_SubDirectories;

   SubDirectoriesLookup m_SubDirectoriesLookup;

 

-  DirectoryEntry *m_Parent;

-  std::set<int> m_Origins;

+  DirectoryEntry* m_Parent;

+  std::set<OriginID> m_Origins;

   bool m_Populated;

   bool m_TopLevel;

+  mutable std::mutex m_SubDirMutex;

+  mutable std::mutex m_FilesMutex;

+  mutable std::mutex m_OriginsMutex;

 

 

-  DirectoryEntry(const DirectoryEntry &reference);

+  FileEntryPtr insert(

+    std::wstring_view fileName, FilesOrigin& origin, FILETIME fileTime,

+    std::wstring_view archive, int order, DirectoryStats& stats);

 

-  FileEntry::Ptr insert(

-    const std::wstring &fileName, FilesOrigin &origin, FILETIME fileTime,

-    const std::wstring &archive, int order);

+  FileEntryPtr insert(

+    env::File& file, FilesOrigin& origin,

+    std::wstring_view archive, int order, DirectoryStats& stats);

 

   void addFiles(

-    FilesOrigin &origin, wchar_t *buffer, int bufferOffset);

+    env::DirectoryWalker& walker, FilesOrigin& origin,

+    const std::wstring& path, DirectoryStats& stats);

 

   void addFiles(

-    FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime,

-    const std::wstring &archiveName, int order);

+    FilesOrigin& origin, BSA::Folder::Ptr archiveFolder, FILETIME fileTime,

+    const std::wstring& archiveName, int order, DirectoryStats& stats);

 

-  DirectoryEntry *getSubDirectory(

-    const std::wstring &name, bool create, int originID = -1);

+  void addDir(FilesOrigin& origin, env::Directory& d, DirectoryStats& stats);

 

-  DirectoryEntry *getSubDirectoryRecursive(

-    const std::wstring &path, bool create, int originID = -1);

+  DirectoryEntry* getSubDirectory(

+    std::wstring_view name, bool create, DirectoryStats& stats,

+    OriginID originID = InvalidOriginID);

+

+  DirectoryEntry* getSubDirectory(

+    env::Directory& dir, bool create, DirectoryStats& stats,

+    OriginID originID = InvalidOriginID);

+

+  DirectoryEntry* getSubDirectoryRecursive(

+    const std::wstring& path, bool create, DirectoryStats& stats,

+    OriginID originID = InvalidOriginID);

 

   void removeDirRecursive();

 

-  void addDirectoryToList(DirectoryEntry* e);

+  void addDirectoryToList(DirectoryEntry* e, std::wstring nameLc);

   void removeDirectoryFromList(SubDirectories::iterator itor);

 

-  void addFileToList(std::wstring fileNameLower, FileEntry::Index index);

-  void removeFileFromList(FileEntry::Index index);

-  void removeFilesFromList(const std::set<FileEntry::Index>& indices);

+  void addFileToList(std::wstring fileNameLower, FileIndex index);

+  void removeFileFromList(FileIndex index);

+  void removeFilesFromList(const std::set<FileIndex>& indices);

+

+  struct Context;

+  static void onDirectoryStart(Context* cx, std::wstring_view path);

+  static void onDirectoryEnd(Context* cx, std::wstring_view path);

+  static void onFile(Context* cx, std::wstring_view path, FILETIME ft);

+

+  void dump(std::FILE* f, const std::wstring& parentPath) const;

 };

 

 } // namespace MOShared

@@ -540,4 +319,4 @@ namespace std
   }

 }

 

-#endif // DIRECTORYENTRY_H

+#endif // MO_REGISTER_DIRECTORYENTRY_INCLUDED

diff --git a/src/shared/fileentry.cpp b/src/shared/fileentry.cpp
new file mode 100644
index 00000000..e57c3cc5
--- /dev/null
+++ b/src/shared/fileentry.cpp
@@ -0,0 +1,251 @@
+#include "fileentry.h"
+#include "directoryentry.h"
+#include "filesorigin.h"
+
+namespace MOShared
+{
+
+FileEntry::FileEntry() :
+  m_Index(InvalidFileIndex), m_Name(), m_Origin(-1), m_Parent(nullptr),
+  m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize)
+{
+}
+
+FileEntry::FileEntry(FileIndex index, std::wstring name, DirectoryEntry *parent) :
+  m_Index(index), m_Name(std::move(name)), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent),
+  m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize)
+{
+}
+
+void FileEntry::addOrigin(
+  OriginID origin, FILETIME fileTime, std::wstring_view archive, int order)
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  if (m_Parent != nullptr) {
+    m_Parent->propagateOrigin(origin);
+  }
+
+  if (m_Origin == -1) {
+    // If this file has no previous origin, this mod is now the origin with no
+    // alternatives
+    m_Origin = origin;
+    m_FileTime = fileTime;
+    m_Archive = std::pair<std::wstring, int>(std::wstring(archive.begin(), archive.end()), order);
+  }
+  else if (
+    (m_Parent != nullptr) && (
+    (m_Parent->getOriginByID(origin).getPriority() > m_Parent->getOriginByID(m_Origin).getPriority()) ||
+      (archive.size() == 0 && m_Archive.first.size() > 0 ))
+    ) {
+    // If this mod has a higher priority than the origin mod OR
+    // this mod has a loose file and the origin mod has an archived file,
+    // this mod is now the origin and the previous origin is the first alternative
+
+    auto itor = std::find_if(
+      m_Alternatives.begin(), m_Alternatives.end(),
+      [&](auto&& i) { return i.first == m_Origin; });
+
+    if (itor == m_Alternatives.end()) {
+      m_Alternatives.push_back({m_Origin, m_Archive});
+    }
+
+    m_Origin = origin;
+    m_FileTime = fileTime;
+    m_Archive = std::pair<std::wstring, int>(std::wstring(archive.begin(), archive.end()), order);
+  }
+  else {
+    // This mod is just an alternative
+    bool found = false;
+
+    if (m_Origin == origin) {
+      // already an origin
+      return;
+    }
+
+    for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) {
+      if (iter->first == origin) {
+        // already an origin
+        return;
+      }
+
+      if ((m_Parent != nullptr) &&
+        (m_Parent->getOriginByID(iter->first).getPriority() < m_Parent->getOriginByID(origin).getPriority())) {
+        m_Alternatives.insert(iter, {origin, {std::wstring(archive.begin(), archive.end()), order}});
+        found = true;
+        break;
+      }
+    }
+
+    if (!found) {
+      m_Alternatives.push_back({origin, {std::wstring(archive.begin(), archive.end()), order}});
+    }
+  }
+}
+
+bool FileEntry::removeOrigin(OriginID origin)
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  if (m_Origin == origin) {
+    if (!m_Alternatives.empty()) {
+      // find alternative with the highest priority
+      auto currentIter = m_Alternatives.begin();
+      for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) {
+        if (iter->first != origin) {
+          //Both files are not from archives.
+          if (!iter->second.first.size() && !currentIter->second.first.size()) {
+            if ((m_Parent->getOriginByID(iter->first).getPriority() > m_Parent->getOriginByID(currentIter->first).getPriority())) {
+              currentIter = iter;
+            }
+          }
+          else {
+            //Both files are from archives
+            if (iter->second.first.size() && currentIter->second.first.size()) {
+              if (iter->second.second > currentIter->second.second) {
+                currentIter = iter;
+              }
+            }
+            else {
+              //Only one of the two is an archive, so we change currentIter only if he is the archive one.
+              if (currentIter->second.first.size()) {
+                currentIter = iter;
+              }
+            }
+          }
+        }
+      }
+
+      OriginID currentID = currentIter->first;
+      m_Archive = currentIter->second;
+      m_Alternatives.erase(currentIter);
+
+      m_Origin = currentID;
+    } else {
+      m_Origin = -1;
+      m_Archive = std::pair<std::wstring, int>(L"", -1);
+      return true;
+    }
+  } else {
+    auto newEnd = std::remove_if(
+      m_Alternatives.begin(), m_Alternatives.end(),
+      [&](auto &i) { return i.first == origin; });
+
+    if (newEnd != m_Alternatives.end()) {
+      m_Alternatives.erase(newEnd, m_Alternatives.end());
+    }
+  }
+  return false;
+}
+
+void FileEntry::sortOrigins()
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  m_Alternatives.push_back({m_Origin, m_Archive});
+
+  std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) {
+    if (!LHS.second.first.size() && !RHS.second.first.size()) {
+      int l = m_Parent->getOriginByID(LHS.first).getPriority();
+      if (l < 0) {
+        l = INT_MAX;
+      }
+
+      int r = m_Parent->getOriginByID(RHS.first).getPriority();
+      if (r < 0) {
+        r = INT_MAX;
+      }
+
+      return l < r;
+    }
+
+    if (LHS.second.first.size() && RHS.second.first.size()) {
+      int l = LHS.second.second; if (l < 0) l = INT_MAX;
+      int r = RHS.second.second; if (r < 0) r = INT_MAX;
+
+      return l < r;
+    }
+
+    if (RHS.second.first.size()) {
+      return false;
+    }
+
+    return true;
+    });
+
+  if (!m_Alternatives.empty()) {
+    m_Origin = m_Alternatives.back().first;
+    m_Archive = m_Alternatives.back().second;
+    m_Alternatives.pop_back();
+  }
+}
+
+bool FileEntry::isFromArchive(std::wstring archiveName) const
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  if (archiveName.length() == 0) {
+    return m_Archive.first.length() != 0;
+  }
+
+  if (m_Archive.first.compare(archiveName) == 0) {
+    return true;
+  }
+
+  for (auto alternative : m_Alternatives) {
+    if (alternative.second.first.compare(archiveName) == 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+std::wstring FileEntry::getFullPath(OriginID originID) const
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  if (originID == InvalidOriginID) {
+    bool ignore = false;
+    originID = getOrigin(ignore);
+  }
+
+  // base directory for origin
+  const auto* o = m_Parent->findOriginByID(originID);
+  if (!o) {
+    return {};
+  }
+
+  std::wstring result = o->getPath();
+
+  // all intermediate directories
+  recurseParents(result, m_Parent);
+
+  return result + L"\\" + m_Name;
+}
+
+std::wstring FileEntry::getRelativePath() const
+{
+  std::wstring result;
+
+  // all intermediate directories
+  recurseParents(result, m_Parent);
+
+  return result + L"\\" + m_Name;
+}
+
+bool FileEntry::recurseParents(std::wstring &path, const DirectoryEntry *parent) const
+{
+  if (parent == nullptr) {
+    return false;
+  } else {
+    // don't append the topmost parent because it is the virtual data-root
+    if (recurseParents(path, parent->getParent())) {
+      path.append(L"\\").append(parent->getName());
+    }
+
+    return true;
+  }
+}
+
+} // namespace
diff --git a/src/shared/fileentry.h b/src/shared/fileentry.h
new file mode 100644
index 00000000..aceeec57
--- /dev/null
+++ b/src/shared/fileentry.h
@@ -0,0 +1,122 @@
+#ifndef MO_REGISTER_FILEENTRY_INCLUDED
+#define MO_REGISTER_FILEENTRY_INCLUDED
+
+#include "fileregisterfwd.h"
+
+namespace MOShared
+{
+
+class FileEntry
+{
+public:
+  static constexpr uint64_t NoFileSize =
+    std::numeric_limits<uint64_t>::max();
+
+  FileEntry();
+  FileEntry(FileIndex index, std::wstring name, DirectoryEntry *parent);
+
+  // noncopyable
+  FileEntry(const FileEntry&) = delete;
+  FileEntry& operator=(const FileEntry&) = delete;
+
+  FileIndex getIndex() const
+  {
+    return m_Index;
+  }
+
+  void addOrigin(
+    OriginID origin, FILETIME fileTime, std::wstring_view archive, int order);
+
+  // remove the specified origin from the list of origins that contain this
+  // file. if no origin is left, the file is effectively deleted and true is
+  // returned. otherwise, false is returned
+  bool removeOrigin(OriginID origin);
+
+  void sortOrigins();
+
+  // gets the list of alternative origins (origins with lower priority than
+  // the primary one). if sortOrigins has been called, it is sorted by priority
+  // (ascending)
+  const AlternativesVector &getAlternatives() const
+  {
+    return m_Alternatives;
+  }
+
+  const std::wstring &getName() const
+  {
+    return m_Name;
+  }
+
+  OriginID getOrigin() const
+  {
+    return m_Origin;
+  }
+
+  OriginID getOrigin(bool &archive) const
+  {
+    archive = (m_Archive.first.length() != 0);
+    return m_Origin;
+  }
+
+  const std::pair<std::wstring, int> &getArchive() const
+  {
+    return m_Archive;
+  }
+
+  bool isFromArchive(std::wstring archiveName = L"") const;
+
+  // if originID is -1, uses the main origin; if this file doesn't exist in the
+  // given origin, returns an empty string
+  //
+  std::wstring getFullPath(OriginID originID=InvalidOriginID) const;
+
+  std::wstring getRelativePath() const;
+
+  DirectoryEntry *getParent()
+  {
+    return m_Parent;
+  }
+
+  void setFileTime(FILETIME fileTime) const
+  {
+    m_FileTime = fileTime;
+  }
+
+  FILETIME getFileTime() const
+  {
+    return m_FileTime;
+  }
+
+  void setFileSize(uint64_t size, uint64_t compressedSize)
+  {
+    m_FileSize = size;
+    m_CompressedFileSize = compressedSize;
+  }
+
+  uint64_t getFileSize() const
+  {
+    return m_FileSize;
+  }
+
+  uint64_t getCompressedFileSize() const
+  {
+    return m_CompressedFileSize;
+  }
+
+private:
+  FileIndex m_Index;
+  std::wstring m_Name;
+  OriginID m_Origin;
+  std::pair<std::wstring, int> m_Archive;
+  AlternativesVector m_Alternatives;
+  DirectoryEntry *m_Parent;
+  mutable FILETIME m_FileTime;
+  uint64_t m_FileSize, m_CompressedFileSize;
+  mutable std::mutex m_OriginsMutex;
+
+  bool recurseParents(std::wstring &path, const DirectoryEntry *parent) const;
+};
+
+} // namespace
+
+#endif // MO_REGISTER_FILEENTRY_INCLUDED
diff --git a/src/shared/fileregister.cpp b/src/shared/fileregister.cpp
new file mode 100644
index 00000000..b56599d1
--- /dev/null
+++ b/src/shared/fileregister.cpp
@@ -0,0 +1,184 @@
+#include "fileregister.h"
+#include "fileentry.h"
+#include "directoryentry.h"
+#include "originconnection.h"
+#include "filesorigin.h"
+#include <log.h>
+
+namespace MOShared
+{
+
+using namespace MOBase;
+
+FileRegister::FileRegister(boost::shared_ptr<OriginConnection> originConnection)
+  : m_OriginConnection(originConnection), m_NextIndex(0)
+{
+}
+
+bool FileRegister::indexValid(FileIndex index) const
+{
+  std::scoped_lock lock(m_Mutex);
+
+  if (index < m_Files.size()) {
+    return (m_Files[index].get() != nullptr);
+  }
+
+  return false;
+}
+
+FileEntryPtr FileRegister::createFile(
+  std::wstring name, DirectoryEntry *parent, DirectoryStats& stats)
+{
+  const auto index = generateIndex();
+  auto p = FileEntryPtr(new FileEntry(index, std::move(name), parent));
+
+  {
+    std::scoped_lock lock(m_Mutex);
+
+    if (index >= m_Files.size()) {
+      m_Files.resize(index + 1);
+    }
+
+    m_Files[index] = p;
+  }
+
+  return p;
+}
+
+FileIndex FileRegister::generateIndex()
+{
+  return m_NextIndex++;
+}
+
+FileEntryPtr FileRegister::getFile(FileIndex index) const
+{
+  std::scoped_lock lock(m_Mutex);
+
+  if (index < m_Files.size()) {
+    return m_Files[index];
+  } else {
+    return {};
+  }
+}
+
+bool FileRegister::removeFile(FileIndex index)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  if (index < m_Files.size()) {
+    FileEntryPtr p;
+    m_Files[index].swap(p);
+
+    if (p) {
+      unregisterFile(p);
+      return true;
+    }
+  }
+
+  log::error(QObject::tr("invalid file index for remove: {}").toStdString(), index);
+  return false;
+}
+
+void FileRegister::removeOrigin(FileIndex index, OriginID originID)
+{
+  std::unique_lock lock(m_Mutex);
+
+  if (index < m_Files.size()) {
+    FileEntryPtr& p = m_Files[index];
+
+    if (p) {
+      if (p->removeOrigin(originID)) {
+        m_Files[index] = {};
+        lock.unlock();
+        unregisterFile(p);
+        return;
+      }
+    }
+  }
+
+  log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index);
+}
+
+void FileRegister::removeOriginMulti(
+  std::set<FileIndex> indices, OriginID originID)
+{
+  std::vector<FileEntryPtr> removedFiles;
+
+  {
+    std::scoped_lock lock(m_Mutex);
+
+    for (auto iter = indices.begin(); iter != indices.end(); ) {
+      const auto index = *iter;
+
+      if (index < m_Files.size()) {
+        const auto& p = m_Files[index];
+
+        if (p && p->removeOrigin(originID)) {
+          removedFiles.push_back(p);
+          m_Files[index] = {};
+          ++iter;
+          continue;
+        }
+      }
+
+      iter = indices.erase(iter);
+    }
+  }
+
+  // optimization: this is only called when disabling an origin and in this case
+  // we don't have to remove the file from the origin
+
+  // need to remove files from their parent directories. multiple ways to go
+  // about this:
+  //   a) for each file, search its parents file-list (preferably by name) and
+  //      remove what is found
+  //   b) gather the parent directories, go through the file list for each once
+  //      and remove all files that have been removed
+  //
+  // the latter should be faster when there are many files in few directories.
+  // since this is called only when disabling an origin that is probably
+  // frequently the case
+
+  std::set<DirectoryEntry*> parents;
+  for (const FileEntryPtr &file : removedFiles) {
+    if (file->getParent() != nullptr) {
+      parents.insert(file->getParent());
+    }
+  }
+
+  for (DirectoryEntry *parent : parents) {
+    parent->removeFiles(indices);
+  }
+}
+
+void FileRegister::sortOrigins()
+{
+  std::scoped_lock lock(m_Mutex);
+
+  for (auto&& p : m_Files) {
+    if (p) {
+      p->sortOrigins();
+    }
+  }
+}
+
+void FileRegister::unregisterFile(FileEntryPtr file)
+{
+  bool ignore;
+
+  // unregister from origin
+  OriginID originID = file->getOrigin(ignore);
+  m_OriginConnection->getByID(originID).removeFile(file->getIndex());
+  const auto& alternatives = file->getAlternatives();
+
+  for (auto iter = alternatives.begin(); iter != alternatives.end(); ++iter) {
+    m_OriginConnection->getByID(iter->first).removeFile(file->getIndex());
+  }
+
+  // unregister from directory
+  if (file->getParent() != nullptr) {
+    file->getParent()->removeFile(file->getIndex());
+  }
+}
+
+} // namespace
diff --git a/src/shared/fileregister.h b/src/shared/fileregister.h
new file mode 100644
index 00000000..c27dc93b
--- /dev/null
+++ b/src/shared/fileregister.h
@@ -0,0 +1,53 @@
+#ifndef MO_REGISTER_FILESREGISTER_INCLUDED
+#define MO_REGISTER_FILESREGISTER_INCLUDED
+
+#include "fileregisterfwd.h"
+#include <mutex>
+#include <boost/shared_ptr.hpp>
+
+namespace MOShared
+{
+
+class FileRegister
+{
+public:
+  FileRegister(boost::shared_ptr<OriginConnection> originConnection);
+
+  // noncopyable
+  FileRegister(const FileRegister&) = delete;
+  FileRegister& operator=(const FileRegister&) = delete;
+
+  bool indexValid(FileIndex index) const;
+
+  FileEntryPtr createFile(
+    std::wstring name, DirectoryEntry *parent, DirectoryStats& stats);
+
+  FileEntryPtr getFile(FileIndex index) const;
+
+  size_t highestCount() const
+  {
+    std::scoped_lock lock(m_Mutex);
+    return m_Files.size();
+  }
+
+  bool removeFile(FileIndex index);
+  void removeOrigin(FileIndex index, OriginID originID);
+  void removeOriginMulti(std::set<FileIndex> indices, OriginID originID);
+
+  void sortOrigins();
+
+private:
+  using FileMap = std::deque<FileEntryPtr>;
+
+  mutable std::mutex m_Mutex;
+  FileMap m_Files;
+  boost::shared_ptr<OriginConnection> m_OriginConnection;
+  std::atomic<FileIndex> m_NextIndex;
+
+  void unregisterFile(FileEntryPtr file);
+  FileIndex generateIndex();
+};
+
+} // namespace
+
+#endif // MO_REGISTER_FILESREGISTER_INCLUDED
diff --git a/src/shared/fileregisterfwd.h b/src/shared/fileregisterfwd.h
new file mode 100644
index 00000000..720e6e30
--- /dev/null
+++ b/src/shared/fileregisterfwd.h
@@ -0,0 +1,96 @@
+#ifndef MO_REGISTER_FILEREGISTERFWD_INCLUDED
+#define MO_REGISTER_FILEREGISTERFWD_INCLUDED
+
+class DirectoryRefreshProgress;
+
+namespace MOShared
+{
+
+struct DirectoryEntryFileKey
+{
+  DirectoryEntryFileKey(std::wstring v)
+    : value(std::move(v)), hash(getHash(value))
+  {
+  }
+
+  bool operator==(const DirectoryEntryFileKey& o) const
+  {
+    return (value == o.value);
+  }
+
+  static std::size_t getHash(const std::wstring& value)
+  {
+    return std::hash<std::wstring>()(value);
+  }
+
+  std::wstring value;
+  const std::size_t hash;
+};
+
+
+class DirectoryEntry;
+class OriginConnection;
+class FileRegister;
+class FilesOrigin;
+class FileEntry;
+struct DirectoryStats;
+
+using FileEntryPtr = boost::shared_ptr<FileEntry>;
+using FileIndex = unsigned int;
+using OriginID = int;
+
+constexpr FileIndex InvalidFileIndex = UINT_MAX;
+constexpr OriginID InvalidOriginID = -1;
+
+
+// a vector of {originId, {archiveName, order}}
+//
+// if a file is in an archive, archiveName is the name of the bsa and order
+// is the order of the associated plugin in the plugins list
+//
+// is a file is not in an archive, archiveName is empty and order is usually
+// -1
+using AlternativesVector = std::vector<std::pair<OriginID, std::pair<std::wstring, int>>>;
+
+struct DirectoryStats
+{
+  static constexpr bool EnableInstrumentation = false;
+
+  std::string mod;
+
+  std::chrono::nanoseconds dirTimes;
+  std::chrono::nanoseconds fileTimes;
+  std::chrono::nanoseconds sortTimes;
+
+  std::chrono::nanoseconds subdirLookupTimes;
+  std::chrono::nanoseconds addDirectoryTimes;
+
+  std::chrono::nanoseconds filesLookupTimes;
+  std::chrono::nanoseconds addFileTimes;
+  std::chrono::nanoseconds addOriginToFileTimes;
+  std::chrono::nanoseconds addFileToOriginTimes;
+  std::chrono::nanoseconds addFileToRegisterTimes;
+
+  int64_t originExists;
+  int64_t originCreate;
+  int64_t originsNeededEnabled;
+
+  int64_t subdirExists;
+  int64_t subdirCreate;
+
+  int64_t fileExists;
+  int64_t fileCreate;
+  int64_t filesInsertedInRegister;
+  int64_t filesAssignedInRegister;
+
+  DirectoryStats();
+
+  DirectoryStats& operator+=(const DirectoryStats& o);
+
+  static std::string csvHeader();
+  std::string toCsv() const;
+};
+
+} // namespace
+
+#endif // MO_REGISTER_FILEREGISTERFWD_INCLUDED
diff --git a/src/shared/filesorigin.cpp b/src/shared/filesorigin.cpp
new file mode 100644
index 00000000..7476b6c5
--- /dev/null
+++ b/src/shared/filesorigin.cpp
@@ -0,0 +1,126 @@
+#include "filesorigin.h"
+#include "originconnection.h"
+#include "fileregister.h"
+#include "fileentry.h"
+
+namespace MOShared
+{
+
+std::wstring tail(const std::wstring &source, const size_t count)
+{
+  if (count >= source.length()) {
+    return source;
+  }
+
+  return source.substr(source.length() - count);
+}
+
+
+FilesOrigin::FilesOrigin()
+  : m_ID(0), m_Disabled(false), m_Name(), m_Path(), m_Priority(0)
+{
+}
+
+FilesOrigin::FilesOrigin(
+  OriginID ID, const std::wstring &name, const std::wstring &path, int priority,
+  boost::shared_ptr<MOShared::FileRegister> fileRegister,
+  boost::shared_ptr<MOShared::OriginConnection> originConnection) :
+  m_ID(ID), m_Disabled(false), m_Name(name), m_Path(path),
+  m_Priority(priority), m_FileRegister(fileRegister),
+  m_OriginConnection(originConnection)
+{
+}
+
+void FilesOrigin::setPriority(int priority)
+{
+  m_OriginConnection.lock()->changePriorityLookup(m_Priority, priority);
+
+  m_Priority = priority;
+}
+
+void FilesOrigin::setName(const std::wstring &name)
+{
+  m_OriginConnection.lock()->changeNameLookup(m_Name, name);
+
+  // change path too
+  if (tail(m_Path, m_Name.length()) == m_Name) {
+    m_Path = m_Path.substr(0, m_Path.length() - m_Name.length()).append(name);
+  }
+
+  m_Name = name;
+}
+
+std::vector<FileEntryPtr> FilesOrigin::getFiles() const
+{
+  std::vector<FileEntryPtr> result;
+
+  {
+    std::scoped_lock lock(m_Mutex);
+
+    for (FileIndex fileIdx : m_Files) {
+      if (FileEntryPtr p = m_FileRegister.lock()->getFile(fileIdx)) {
+        result.push_back(p);
+      }
+    }
+  }
+
+  return result;
+}
+
+FileEntryPtr FilesOrigin::findFile(FileIndex index) const
+{
+  return m_FileRegister.lock()->getFile(index);
+}
+
+void FilesOrigin::enable(bool enabled)
+{
+  DirectoryStats dummy;
+  enable(enabled, dummy);
+}
+
+void FilesOrigin::enable(bool enabled, DirectoryStats& stats)
+{
+  if (!enabled) {
+    ++stats.originsNeededEnabled;
+
+    std::set<FileIndex> copy;
+
+    {
+      std::scoped_lock lock(m_Mutex);
+      copy = m_Files;
+      m_Files.clear();
+    }
+
+    m_FileRegister.lock()->removeOriginMulti(copy, m_ID);
+  }
+
+  m_Disabled = !enabled;
+}
+
+void FilesOrigin::removeFile(FileIndex index)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto iter = m_Files.find(index);
+
+  if (iter != m_Files.end()) {
+    m_Files.erase(iter);
+  }
+}
+
+bool FilesOrigin::containsArchive(std::wstring archiveName)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  for (FileIndex fileIdx : m_Files) {
+    if (FileEntryPtr p = m_FileRegister.lock()->getFile(fileIdx)) {
+      if (p->isFromArchive(archiveName)) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+} //  namespace
diff --git a/src/shared/filesorigin.h b/src/shared/filesorigin.h
new file mode 100644
index 00000000..222a834f
--- /dev/null
+++ b/src/shared/filesorigin.h
@@ -0,0 +1,85 @@
+#ifndef MO_REGISTER_FILESORIGIN_INCLUDED
+#define MO_REGISTER_FILESORIGIN_INCLUDED
+
+#include "fileregisterfwd.h"
+
+namespace MOShared
+{
+
+// represents a mod or the data directory, providing files to the tree
+class FilesOrigin
+{
+public:
+  FilesOrigin();
+
+  FilesOrigin(
+    OriginID ID, const std::wstring &name, const std::wstring &path,
+    int priority,
+    boost::shared_ptr<FileRegister> fileRegister,
+    boost::shared_ptr<OriginConnection> originConnection);
+
+  // noncopyable
+  FilesOrigin(const FilesOrigin&) = delete;
+  FilesOrigin& operator=(const FilesOrigin&) = delete;
+
+  // sets priority for this origin, but it will overwrite the existing mapping
+  // for this priority, the previous origin will no longer be referenced
+  void setPriority(int priority);
+
+  int getPriority() const
+  {
+    return m_Priority;
+  }
+
+  void setName(const std::wstring &name);
+  const std::wstring &getName() const
+  {
+    return m_Name;
+  }
+
+  OriginID getID() const
+  {
+    return m_ID;
+  }
+
+  const std::wstring &getPath() const
+  {
+    return m_Path;
+  }
+
+  std::vector<FileEntryPtr> getFiles() const;
+  FileEntryPtr findFile(FileIndex index) const;
+
+  void enable(bool enabled, DirectoryStats& stats);
+  void enable(bool enabled);
+
+  bool isDisabled() const
+  {
+    return m_Disabled;
+  }
+
+  void addFile(FileIndex index)
+  {
+    std::scoped_lock lock(m_Mutex);
+    m_Files.insert(index);
+  }
+
+  void removeFile(FileIndex index);
+
+  bool containsArchive(std::wstring archiveName);
+
+private:
+  OriginID m_ID;
+  bool m_Disabled;
+  std::set<FileIndex> m_Files;
+  std::wstring m_Name;
+  std::wstring m_Path;
+  int m_Priority;
+  boost::weak_ptr<FileRegister> m_FileRegister;
+  boost::weak_ptr<OriginConnection> m_OriginConnection;
+  mutable std::mutex m_Mutex;
+};
+
+} // namespace
+
+#endif // MO_REGISTER_FILESORIGIN_INCLUDED
diff --git a/src/shared/originconnection.cpp b/src/shared/originconnection.cpp
new file mode 100644
index 00000000..c1c096f0
--- /dev/null
+++ b/src/shared/originconnection.cpp
@@ -0,0 +1,145 @@
+#include "originconnection.h"
+#include "filesorigin.h"
+#include "util.h"
+#include <log.h>
+
+namespace MOShared
+{
+
+using namespace MOBase;
+
+OriginConnection::OriginConnection()
+  : m_NextID(0)
+{
+}
+
+std::pair<FilesOrigin&, bool> OriginConnection::getOrCreate(
+  const std::wstring &originName, const std::wstring &directory, int priority,
+  const boost::shared_ptr<FileRegister>& fileRegister,
+  const boost::shared_ptr<OriginConnection>& originConnection,
+  DirectoryStats& stats)
+{
+  std::unique_lock lock(m_Mutex);
+
+  auto itor = m_OriginsNameMap.find(originName);
+
+  if (itor == m_OriginsNameMap.end()) {
+    FilesOrigin& origin = createOriginNoLock(
+      originName, directory, priority, fileRegister, originConnection);
+
+    return {origin, true};
+  } else {
+    FilesOrigin& origin = m_Origins[itor->second];
+    lock.unlock();
+
+    origin.enable(true, stats);
+    return {origin, false};
+  }
+}
+
+FilesOrigin& OriginConnection::createOrigin(
+  const std::wstring &originName, const std::wstring &directory, int priority,
+  boost::shared_ptr<FileRegister> fileRegister,
+  boost::shared_ptr<OriginConnection> originConnection)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  return createOriginNoLock(
+    originName, directory, priority, fileRegister, originConnection);
+}
+
+bool OriginConnection::exists(const std::wstring &name)
+{
+  std::scoped_lock lock(m_Mutex);
+  return m_OriginsNameMap.find(name) != m_OriginsNameMap.end();
+}
+
+FilesOrigin& OriginConnection::getByID(OriginID ID)
+{
+  std::scoped_lock lock(m_Mutex);
+  return m_Origins[ID];
+}
+
+const FilesOrigin* OriginConnection::findByID(OriginID ID) const
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto itor = m_Origins.find(ID);
+
+  if (itor == m_Origins.end()) {
+    return nullptr;
+  } else {
+    return &itor->second;
+  }
+}
+
+FilesOrigin& OriginConnection::getByName(const std::wstring &name)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto iter = m_OriginsNameMap.find(name);
+
+  if (iter != m_OriginsNameMap.end()) {
+    return m_Origins[iter->second];
+  } else {
+    std::ostringstream stream;
+    stream << QObject::tr("invalid origin name: ").toStdString() << ToString(name, true);
+    throw std::runtime_error(stream.str());
+  }
+}
+
+void OriginConnection::changePriorityLookup(int oldPriority, int newPriority)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto iter = m_OriginsPriorityMap.find(oldPriority);
+
+  if (iter != m_OriginsPriorityMap.end()) {
+    OriginID idx = iter->second;
+    m_OriginsPriorityMap.erase(iter);
+    m_OriginsPriorityMap[newPriority] = idx;
+  }
+}
+
+void OriginConnection::changeNameLookup(const std::wstring &oldName, const std::wstring &newName)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto iter = m_OriginsNameMap.find(oldName);
+
+  if (iter != m_OriginsNameMap.end()) {
+    OriginID idx = iter->second;
+    m_OriginsNameMap.erase(iter);
+    m_OriginsNameMap[newName] = idx;
+  } else {
+    log::error(QObject::tr("failed to change name lookup from {} to {}").toStdString(), oldName, newName);
+  }
+}
+
+OriginID OriginConnection::createID()
+{
+  return m_NextID++;
+}
+
+FilesOrigin& OriginConnection::createOriginNoLock(
+  const std::wstring &originName, const std::wstring &directory, int priority,
+  boost::shared_ptr<FileRegister> fileRegister,
+  boost::shared_ptr<OriginConnection> originConnection)
+{
+  OriginID newID = createID();
+
+  auto itor = m_Origins.emplace(
+    std::piecewise_construct,
+    std::forward_as_tuple(newID),
+    std::forward_as_tuple(
+      newID, originName, directory, priority,
+      fileRegister, originConnection))
+    .first;
+
+  m_OriginsNameMap.insert({originName, newID});
+  m_OriginsPriorityMap.insert({priority, newID});
+
+  return itor->second;
+}
+
+} // namespace
diff --git a/src/shared/originconnection.h b/src/shared/originconnection.h
new file mode 100644
index 00000000..5fecd63a
--- /dev/null
+++ b/src/shared/originconnection.h
@@ -0,0 +1,56 @@
+#ifndef MO_REGISTER_ORIGINCONNECTION_INCLUDED
+#define MO_REGISTER_ORIGINCONNECTION_INCLUDED
+
+#include "fileregisterfwd.h"
+
+namespace MOShared
+{
+
+class OriginConnection
+{
+public:
+  OriginConnection();
+
+  // noncopyable
+  OriginConnection(const OriginConnection&) = delete;
+  OriginConnection& operator=(const OriginConnection&) = delete;
+
+  std::pair<FilesOrigin&, bool> getOrCreate(
+    const std::wstring &originName, const std::wstring &directory, int priority,
+    const boost::shared_ptr<FileRegister>& fileRegister,
+    const boost::shared_ptr<OriginConnection>& originConnection,
+    DirectoryStats& stats);
+
+  FilesOrigin& createOrigin(
+    const std::wstring &originName, const std::wstring &directory, int priority,
+    boost::shared_ptr<FileRegister> fileRegister,
+    boost::shared_ptr<OriginConnection> originConnection);
+
+  bool exists(const std::wstring &name);
+
+  FilesOrigin &getByID(OriginID ID);
+  const FilesOrigin* findByID(OriginID ID) const;
+  FilesOrigin &getByName(const std::wstring &name);
+
+  void changePriorityLookup(int oldPriority, int newPriority);
+
+  void changeNameLookup(const std::wstring &oldName, const std::wstring &newName);
+
+private:
+  std::atomic<OriginID> m_NextID;
+  std::map<OriginID, FilesOrigin> m_Origins;
+  std::map<std::wstring, OriginID> m_OriginsNameMap;
+  std::map<int, OriginID> m_OriginsPriorityMap;
+  mutable std::mutex m_Mutex;
+
+  OriginID createID();
+
+  FilesOrigin& createOriginNoLock(
+    const std::wstring &originName, const std::wstring &directory, int priority,
+    boost::shared_ptr<FileRegister> fileRegister,
+    boost::shared_ptr<OriginConnection> originConnection);
+};
+
+} // namespace
+
+#endif // MO_REGISTER_ORIGINCONNECTION_INCLUDED
diff --git a/src/shared/util.cpp b/src/shared/util.cpp
index aa4ad4b3..483b36a9 100644
--- a/src/shared/util.cpp
+++ b/src/shared/util.cpp
@@ -130,6 +130,13 @@ std::wstring ToLowerCopy(const std::wstring& text)
   return result;

 }

 

+std::wstring ToLowerCopy(std::wstring_view text)

+{

+  std::wstring result(text.begin(), text.end());

+  ToLowerInPlace(result);

+  return result;

+}

+

 bool CaseInsenstiveComparePred(wchar_t lhs, wchar_t rhs)

 {

   return std::tolower(lhs, loc) == std::tolower(rhs, loc);

@@ -375,26 +382,6 @@ void checkDuplicateShortcuts(const QMenu& m)
 } // namespace MOShared

 

 

-TimeThis::TimeThis(QString what)

-  : m_what(std::move(what)), m_start(Clock::now())

-{

-}

-

-TimeThis::~TimeThis()

-{

-  using namespace std::chrono;

-

-  const auto end = Clock::now();

-  const auto d = duration_cast<milliseconds>(end - m_start).count();

-

-  if (m_what.isEmpty()) {

-    log::debug("{} ms", d);

-  } else {

-    log::debug("{} {} ms", m_what, d);

-  }

-}

-

-

 static bool g_exiting = false;

 static bool g_canClose = false;

 

diff --git a/src/shared/util.h b/src/shared/util.h
index 05522c2d..2761b64f 100644
--- a/src/shared/util.h
+++ b/src/shared/util.h
@@ -20,7 +20,9 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #ifndef UTIL_H

 #define UTIL_H

 

+#include <log.h>

 #include <string>

+#include <filesystem>

 #include <versioninfo.h>

 

 class Executable;

@@ -41,6 +43,7 @@ std::string ToLowerCopy(const std::string& text);
 

 std::wstring& ToLowerInPlace(std::wstring& text);

 std::wstring ToLowerCopy(const std::wstring& text);

+std::wstring ToLowerCopy(std::wstring_view text);

 

 bool CaseInsensitiveEqual(const std::wstring &lhs, const std::wstring &rhs);

 

@@ -50,21 +53,16 @@ QString getUsvfsVersionString();
 void SetThisThreadName(const QString& s);

 void checkDuplicateShortcuts(const QMenu& m);

 

-} // namespace MOShared

-

-

-class TimeThis

+inline FILETIME ToFILETIME(std::filesystem::file_time_type t)

 {

-public:

-  TimeThis(QString what={});

-  ~TimeThis();

+  FILETIME ft;

+  static_assert(sizeof(t) == sizeof(ft));

 

-private:

-  using Clock = std::chrono::high_resolution_clock;

+  std::memcpy(&ft, &t, sizeof(FILETIME));

+  return ft;

+}

 

-  QString m_what;

-  Clock::time_point m_start;

-};

+} // namespace MOShared

 

 

 enum class Exit

diff --git a/src/stylesheets/Night Eyes.qss b/src/stylesheets/Night Eyes.qss
index 14f7434e..135f0c09 100644
--- a/src/stylesheets/Night Eyes.qss
+++ b/src/stylesheets/Night Eyes.qss
@@ -691,4 +691,12 @@ QProgressBar
 	border-radius: 5px;
 	text-align: center;
 	margin: 0px;
-}
\ No newline at end of file
+}
+
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/Parchment v1.1 by Bob.qss b/src/stylesheets/Parchment v1.1 by Bob.qss
index 61c2f84d..269869ad 100644
--- a/src/stylesheets/Parchment v1.1 by Bob.qss
+++ b/src/stylesheets/Parchment v1.1 by Bob.qss
@@ -145,6 +145,10 @@ QWebView {
 	border-radius: 6px;
 }
 
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 /* Group Boxes */
 
 QGroupBox {
@@ -596,4 +600,8 @@ QSlider::handle:horizontal:hover {
 QTableView {
 	gridline-color: #DBD399;
 	border: 0px;
-}
\ No newline at end of file
+}
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/Transparent-Style-101-Green.qss b/src/stylesheets/Transparent-Style-101-Green.qss
index 3bbdb5db..0d35ec7e 100644
--- a/src/stylesheets/Transparent-Style-101-Green.qss
+++ b/src/stylesheets/Transparent-Style-101-Green.qss
@@ -6,13 +6,13 @@
 }
  QWidget {
      color: #fff;
-     background-color: rgb(0, 204, 0, 5);
-     alternate-background-color: rgb(0, 204, 0, 10);
+     background-color: rgba(0, 204, 0, 5);
+     alternate-background-color: rgba(0, 204, 0, 10);
 }
  QAbstractItemView {
      color: #fff;
-     background-color: rgb(0, 204, 0, 5);
-     alternate-background-color: rgb(0, 204, 0, 10);
+     background-color: rgba(0, 204, 0, 5);
+     alternate-background-color: rgba(0, 204, 0, 10);
      border: 1px solid #001a00;
      border-style:outset;
      border-radius: 3px;
@@ -29,7 +29,7 @@
      border-image: url(./Transparent-Style/Green/vault-tec-small.png) 0 0 0 0 stretch stretch;
 }
  #nameLabel {
-     background-color: rgb(0, 0, 0, 20);
+     background-color: rgba(0, 0, 0, 20);
      border-top:1px solid black;
      border-bottom:1px solid black;
 }
@@ -269,7 +269,7 @@
      padding-right: 5%;
 }
  QPushButton:open {
-     background-color: rgb(0, 204, 0, 100);
+     background-color: rgba(0, 204, 0, 100);
 }
  QPushButton:!enabled {
      border: 1px grey;
@@ -331,29 +331,34 @@
      border-radius: 5px;
      border-top:1px solid #b9d7fc;
      border-bottom:1px solid #b9d7fc;
-     background-color: rgb(0, 204, 0, 100);
+     background-color: rgba(0, 204, 0, 100);
 }
  QTreeView::branch:has-siblings:!adjoins-item {
-     background:url(Vault-101/Vault-101-item-line-v.png) center center no-repeat;
+     background:url(Transparent-Style/Green/Vault-101-item-line-v.png) center center no-repeat;
 }
  QTreeView::branch:has-siblings {
-     background:url(Vault-101/Vault-101-itemB-line.png) center center no-repeat;
+     background:url(Transparent-Style/Green/Vault-101-itemB-line.png) center center no-repeat;
 }
  QTreeView::branch:!has-children:!has-siblings:adjoins-item {
-     background:url(Vault-101/Vault-101-itemB-end.png) center center no-repeat;
+     background:url(Transparent-Style/Green/Vault-101-itemB-end.png) center center no-repeat;
 }
  QTreeView::branch:closed:has-children:has-siblings {
-     background:url(Vault-101/Vault-101-itemB-close.png) center center no-repeat;
+     background:url(Transparent-Style/Green/Vault-101-itemB-close.png) center center no-repeat;
 }
  QTreeView::branch:closed:has-children:!has-siblings {
-     background:url(Vault-101/Vault-101-itemB-close-last.png) center center no-repeat;
+     background:url(Transparent-Style/Green/Vault-101-itemB-close-last.png) center center no-repeat;
 }
  QTreeView::branch:open:has-children:has-siblings {
-     background:url(Vault-101/Vault-101-itemB-open.png) center center no-repeat;
+     background:url(Transparent-Style/Green/Vault-101-itemB-open.png) center center no-repeat;
 }
  QTreeView::branch:open:has-children:!has-siblings {
-     background:url(Vault-101/Vault-101-itemB-open-last.png) center center no-repeat;
+     background:url(Transparent-Style/Green/Vault-101-itemB-open-last.png) center center no-repeat;
 }
+
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 /* **************************************************************** */
 /* Checkboxes and Radio buttons common #QCheckBox, #QRadioButton */
 /* common */
@@ -477,7 +482,7 @@ QStatusBar::item {border: None;}
      border-radius:5px;
      margin: 0px;
      padding-left:2px;
-     selection-background-color: rgb(0, 204, 0, 10);
+     selection-background-color: rgba(0, 204, 0, 10);
 }
 /* Font size */
  QLabel, QMenu, QPushButton, QTabBar, QTextEdit, QLineEdit, QWebView, QComboBox, QComboBox:editable, QAbstractSpinBox, QGroupBox, QCheckBox, QRadioButton {
@@ -495,3 +500,7 @@ LinkLabel {
      font-family: Source Sans Pro;
      font-size: 14px;
 }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/Transparent-Style-BOS.qss b/src/stylesheets/Transparent-Style-BOS.qss
index 33eed327..0b4ba23d 100644
--- a/src/stylesheets/Transparent-Style-BOS.qss
+++ b/src/stylesheets/Transparent-Style-BOS.qss
@@ -12,8 +12,8 @@
 }
  QAbstractItemView{
      color: #dcdcdc;
-     background-color: rgb(45,45,48,30);
-     alternate-background-color: rgb(38,38,38,50);
+     background-color: rgba(45,45,48,30);
+     alternate-background-color: rgba(38,38,38,50);
      border: 1px solid #3F3F46;
     /* Grey - Blue*/
      border-style:outset;
@@ -507,6 +507,11 @@ QMenu::item:selected{
  QTreeView::branch:open:has-children:!has-siblings{
      background:url(Transparent-Style/vault-101-itemB-open-last.png) center center no-repeat;
 }
+
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 /* **************************************************************** */
 /* Checkboxes and Radio buttons common #QCheckBox, #QRadioButton */
 /* common */
@@ -747,3 +752,7 @@ QStatusBar::item {border: None;}
 LinkLabel {
   qproperty-linkColor: #3399FF;
 }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/Transparent-Style-Skyrim.qss b/src/stylesheets/Transparent-Style-Skyrim.qss
index 308580a5..cbc43033 100644
--- a/src/stylesheets/Transparent-Style-Skyrim.qss
+++ b/src/stylesheets/Transparent-Style-Skyrim.qss
@@ -12,8 +12,8 @@
 }
  QAbstractItemView{
      color: #dcdcdc;
-     background-color: rgb(45,45,48,30);
-     alternate-background-color: rgb(38,38,38,50);
+     background-color: rgba(45,45,48,30);
+     alternate-background-color: rgba(38,38,38,50);
      border: 1px solid #3F3F46;
     /* Grey - Blue*/
      border-style:outset;
@@ -507,6 +507,11 @@ QMenu::item:selected{
  QTreeView::branch:open:has-children:!has-siblings{
      background:url(Transparent-Style/vault-101-itemB-open-last.png) center center no-repeat;
 }
+
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 /* **************************************************************** */
 /* Checkboxes and Radio buttons common #QCheckBox, #QRadioButton */
 /* common */
@@ -747,3 +752,7 @@ QStatusBar::item {border: None;}
 LinkLabel {
   qproperty-linkColor: #3399FF;
 }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/dark.qss b/src/stylesheets/dark.qss
index 91f808bc..1bf5c819 100644
--- a/src/stylesheets/dark.qss
+++ b/src/stylesheets/dark.qss
@@ -356,6 +356,10 @@ QTreeView, QListView
   alternate-background-color: #2F3031;

 }

 

+QAbstractItemView[filtered=true] {

+	border: 2px solid #f00 !important;

+}

+

 QTreeView::branch:has-children:!has-siblings:closed,

 QTreeView::branch:closed:has-children:has-siblings

 {

@@ -385,3 +389,7 @@ DownloadListWidget[downloadView=compact]::item {
 LinkLabel {

   qproperty-linkColor: #3399FF;

 }

+

+QLineEdit[valid-filter=false] {

+	background-color: #661111 !important;

+}

diff --git a/src/stylesheets/dracula.qss b/src/stylesheets/dracula.qss
index a78d2414..9beba52d 100644
--- a/src/stylesheets/dracula.qss
+++ b/src/stylesheets/dracula.qss
@@ -373,6 +373,10 @@ QTreeView::branch:open:has-children:has-siblings {
     image: url(:/stylesheet/branch-open.png);
 }
 
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 /*
  * Tool bar
  */
@@ -435,3 +439,7 @@ QProgressBar::chunk
 {
   background-color: #427683;
 }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/skyrim.qss b/src/stylesheets/skyrim.qss
index a516f5ff..ffd42640 100644
--- a/src/stylesheets/skyrim.qss
+++ b/src/stylesheets/skyrim.qss
@@ -79,6 +79,10 @@ QListView,
 QTreeView {
   show-decoration-selected: 1; }
 
+QAbstractItemView[filtered=true] {
+	background-color: #f00 !important;
+}
+
 QAbstractItemView::item:hover,
 QListView::item:hover,
 QTreeView::branch:hover,
@@ -659,3 +663,7 @@ QDialog#CredentialsDialog {
 
 #pluginsList {
   min-width: 260px; }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/vs15 Dark-Green.qss b/src/stylesheets/vs15 Dark-Green.qss
index 0a3150ad..a4c6a2cc 100644
--- a/src/stylesheets/vs15 Dark-Green.qss
+++ b/src/stylesheets/vs15 Dark-Green.qss
@@ -71,6 +71,10 @@ QAbstractItemView {
     background-color: #009933;
     color: #F1F1F1; }
 
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 QAbstractItemView,
 QListView,
 QTreeView {
@@ -755,3 +759,7 @@ QDialog#QueryOverwriteDialog {
 
 QDialog#ModInfoDialog {
   min-width: 850px; }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/vs15 Dark-Orange.qss b/src/stylesheets/vs15 Dark-Orange.qss
index 0e1d96b1..c909d103 100644
--- a/src/stylesheets/vs15 Dark-Orange.qss
+++ b/src/stylesheets/vs15 Dark-Orange.qss
@@ -72,6 +72,10 @@ QAbstractItemView {
     background-color: #CC6600;
     color: #F1F1F1; }
 
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 QAbstractItemView,
 QListView,
 QTreeView {
@@ -756,3 +760,7 @@ QDialog#QueryOverwriteDialog {
 
 QDialog#ModInfoDialog {
   min-width: 850px; }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/vs15 Dark-Purple.qss b/src/stylesheets/vs15 Dark-Purple.qss
index c19d188b..2cff0e06 100644
--- a/src/stylesheets/vs15 Dark-Purple.qss
+++ b/src/stylesheets/vs15 Dark-Purple.qss
@@ -72,6 +72,10 @@ QAbstractItemView {
     background-color: #7E2AD2;
     color: #F1F1F1; }
 
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 QAbstractItemView,
 QListView,
 QTreeView {
@@ -756,3 +760,7 @@ QDialog#QueryOverwriteDialog {
 
 QDialog#ModInfoDialog {
   min-width: 850px; }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/vs15 Dark-Red.qss b/src/stylesheets/vs15 Dark-Red.qss
index a6221dff..2df39fc4 100644
--- a/src/stylesheets/vs15 Dark-Red.qss
+++ b/src/stylesheets/vs15 Dark-Red.qss
@@ -72,6 +72,10 @@ QAbstractItemView {
     background-color: #990000;
     color: #F1F1F1; }
 
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 QAbstractItemView,
 QListView,
 QTreeView {
@@ -756,3 +760,7 @@ QDialog#QueryOverwriteDialog {
 
 QDialog#ModInfoDialog {
   min-width: 850px; }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/vs15 Dark-Yellow.qss b/src/stylesheets/vs15 Dark-Yellow.qss
index fb2e0422..63688511 100644
--- a/src/stylesheets/vs15 Dark-Yellow.qss
+++ b/src/stylesheets/vs15 Dark-Yellow.qss
@@ -71,6 +71,10 @@ QAbstractItemView {
     background-color: #9A9A00;
     color: #F1F1F1; }
 
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 QAbstractItemView,
 QListView,
 QTreeView {
@@ -755,3 +759,7 @@ QDialog#QueryOverwriteDialog {
 
 QDialog#ModInfoDialog {
   min-width: 850px; }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/stylesheets/vs15 Dark.qss b/src/stylesheets/vs15 Dark.qss
index 17def4dd..af29a696 100644
--- a/src/stylesheets/vs15 Dark.qss
+++ b/src/stylesheets/vs15 Dark.qss
@@ -71,6 +71,10 @@ QAbstractItemView {
     background-color: #3399FF;
     color: #F1F1F1; }
 
+QAbstractItemView[filtered=true] {
+	border: 2px solid #f00 !important;
+}
+
 QAbstractItemView,
 QListView,
 QTreeView {
@@ -755,3 +759,7 @@ QDialog#QueryOverwriteDialog {
 
 QDialog#ModInfoDialog {
   min-width: 850px; }
+
+QLineEdit[valid-filter=false] {
+	background-color: #661111 !important;
+}
diff --git a/src/syncoverwritedialog.cpp b/src/syncoverwritedialog.cpp
index b1643b2d..0afb1121 100644
--- a/src/syncoverwritedialog.cpp
+++ b/src/syncoverwritedialog.cpp
@@ -18,8 +18,11 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 */

 

 #include "syncoverwritedialog.h"

-

+#include "shared/directoryentry.h"

+#include "shared/fileentry.h"

+#include "shared/filesorigin.h"

 #include "ui_syncoverwritedialog.h"

+

 #include <utility.h>

 #include <report.h>

 #include <log.h>

@@ -92,7 +95,7 @@ void SyncOverwriteDialog::readTree(const QString &path, DirectoryEntry *director
         newItem = nullptr;

       }

     } else {

-      const FileEntry::Ptr entry = directoryStructure->findFile(ToWString(file));

+      const FileEntryPtr entry = directoryStructure->findFile(ToWString(file));

       QComboBox* combo = new QComboBox(ui->syncTree);

       combo->addItem(tr("<don't sync>"), -1);

       if (entry.get() != nullptr) {

diff --git a/src/syncoverwritedialog.h b/src/syncoverwritedialog.h
index 79b30a8d..cb4c92fb 100644
--- a/src/syncoverwritedialog.h
+++ b/src/syncoverwritedialog.h
@@ -20,11 +20,9 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #ifndef SYNCOVERWRITEDIALOG_H

 #define SYNCOVERWRITEDIALOG_H

 

-

 #include "tutorabledialog.h"

+#include "shared/fileregisterfwd.h"

 #include <QTreeWidgetItem>

-#include <directoryentry.h>

-

 

 namespace Ui {

 class SyncOverwriteDialog;

@@ -33,9 +31,12 @@ class SyncOverwriteDialog;
 class SyncOverwriteDialog : public MOBase::TutorableDialog

 {

   Q_OBJECT

-  

+

 public:

-  explicit SyncOverwriteDialog(const QString &path, MOShared::DirectoryEntry *directoryStructure, QWidget *parent = 0);

+  explicit SyncOverwriteDialog(

+    const QString &path, MOShared::DirectoryEntry *directoryStructure,

+    QWidget *parent = 0);

+

   ~SyncOverwriteDialog();

 

   void apply(const QString &modDirectory);

diff --git a/src/usvfsconnector.cpp b/src/usvfsconnector.cpp
index 59880754..ed6e31ce 100644
--- a/src/usvfsconnector.cpp
+++ b/src/usvfsconnector.cpp
@@ -70,6 +70,8 @@ LogWorker::~LogWorker()
 
 void LogWorker::process()
 {
+  MOShared::SetThisThreadName("LogWorker");
+
   int noLogCycles = 0;
   while (!m_QuitRequested) {
     if (GetLogMessages(&m_Buffer[0], m_Buffer.size(), false)) {
diff --git a/src/version.rc b/src/version.rc
index a5fbb7a6..4c77bfb6 100644
--- a/src/version.rc
+++ b/src/version.rc
@@ -4,7 +4,7 @@
 // Otherwise, if letters are used in VER_FILEVERSION_STR, uses the full MOBase::VersionInfo parser
 // Otherwise, uses the numbers from VER_FILEVERSION and sets the release type as pre-alpha
 #define VER_FILEVERSION     2,3,0
-#define VER_FILEVERSION_STR "2.3.0alpha5\0"
+#define VER_FILEVERSION_STR "2.3.0alpha6\0"
 
 VS_VERSION_INFO VERSIONINFO
 FILEVERSION     VER_FILEVERSION