Merge pull request #1003 from isanae/file-list-improvements

Refresh optimizations

13 files changed, 1715 insertions, 1105 deletions

diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp
index 6e44cc91..639392d2 100644
--- a/src/shared/directoryentry.cpp
+++ b/src/shared/directoryentry.cpp
@@ -18,35 +18,39 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 */

 

 #include "directoryentry.h"

+#include "originconnection.h"

+#include "filesorigin.h"

+#include "fileentry.h"

+#include "envfs.h"

+#include "util.h"

 #include "windows_error.h"

-#include "error_report.h"

 #include <log.h>

-#include <bsatk.h>

-#include <boost/bind.hpp>

-#include <boost/scoped_array.hpp>

-#define WIN32_LEAN_AND_MEAN

-#include <Windows.h>

-#include <sstream>

-#include <ctime>

-#include <algorithm>

-#include <map>

-#include <atomic>

+#include <utility.h>

 

 namespace MOShared

 {

 

 using namespace MOBase;

-static const int MAXPATH_UNICODE = 32767;

+const int MAXPATH_UNICODE = 32767;

 

-static std::wstring tail(const std::wstring &source, const size_t count)

+template <class F>

+void elapsedImpl(std::chrono::nanoseconds& out, F&& f)

 {

-  if (count >= source.length()) {

-    return source;

+  if constexpr (DirectoryStats::EnableInstrumentation) {

+    const auto start = std::chrono::high_resolution_clock::now();

+    f();

+    const auto end = std::chrono::high_resolution_clock::now();

+    out += (end - start);

+  } else {

+    f();

   }

-

-  return source.substr(source.length() - count);

 }

 

+// elapsed() is not optimized out when EnableInstrumentation is false even

+// though it's equivalent that this macro

+#define elapsed(OUT, F) (F)();

+//#define elapsed(OUT, F) elapsedImpl(OUT, F);

+

 static bool SupportOptimizedFind()

 {

   // large fetch and basic info for FindFirstFileEx is supported on win server 2008 r2, win 7 and newer

@@ -63,681 +67,186 @@ static bool SupportOptimizedFind()
   return (::VerifyVersionInfo(&versionInfo, VER_MAJORVERSION | VER_MINORVERSION, mask) == TRUE);

 }

 

-static bool DirCompareByName(const DirectoryEntry *lhs, const DirectoryEntry *rhs)

+static bool DirCompareByName(const DirectoryEntry* lhs, const DirectoryEntry* rhs)

 {

   return _wcsicmp(lhs->getName().c_str(), rhs->getName().c_str()) < 0;

 }

 

 

-class OriginConnection

-{

-public:

-  typedef int Index;

-  static const int INVALID_INDEX = INT_MIN;

-

-  OriginConnection()

-    : m_NextID(0)

-  {

-  }

-

-  FilesOrigin& createOrigin(

-    const std::wstring &originName, const std::wstring &directory, int priority,

-    boost::shared_ptr<FileRegister> fileRegister,

-    boost::shared_ptr<OriginConnection> originConnection)

-  {

-    int newID = createID();

-

-    m_Origins[newID] = FilesOrigin(newID, originName, directory, priority, fileRegister, originConnection);

-    m_OriginsNameMap[originName] = newID;

-    m_OriginsPriorityMap[priority] = newID;

-

-    return m_Origins[newID];

-  }

-

-  bool exists(const std::wstring &name)

-  {

-    return m_OriginsNameMap.find(name) != m_OriginsNameMap.end();

-  }

-

-  FilesOrigin &getByID(Index ID)

-  {

-    return m_Origins[ID];

-  }

-

-  const FilesOrigin* findByID(Index ID) const

-  {

-    auto itor = m_Origins.find(ID);

-

-    if (itor == m_Origins.end()) {

-      return nullptr;

-    } else {

-      return &itor->second;

-    }

-  }

-

-  FilesOrigin &getByName(const std::wstring &name)

-  {

-    std::map<std::wstring, int>::iterator iter = m_OriginsNameMap.find(name);

-

-    if (iter != m_OriginsNameMap.end()) {

-      return m_Origins[iter->second];

-    } else {

-      std::ostringstream stream;

-      stream << QObject::tr("invalid origin name: ").toStdString() << ToString(name, true);

-      throw std::runtime_error(stream.str());

-    }

-  }

-

-  void changePriorityLookup(int oldPriority, int newPriority)

-  {

-    auto iter = m_OriginsPriorityMap.find(oldPriority);

-

-    if (iter != m_OriginsPriorityMap.end()) {

-      Index idx = iter->second;

-      m_OriginsPriorityMap.erase(iter);

-      m_OriginsPriorityMap[newPriority] = idx;

-    }

-  }

-

-  void changeNameLookup(const std::wstring &oldName, const std::wstring &newName)

-  {

-    auto iter = m_OriginsNameMap.find(oldName);

-

-    if (iter != m_OriginsNameMap.end()) {

-      Index idx = iter->second;

-      m_OriginsNameMap.erase(iter);

-      m_OriginsNameMap[newName] = idx;

-    } else {

-      log::error(QObject::tr("failed to change name lookup from {} to {}").toStdString(), oldName, newName);

-    }

-  }

-

-private:

-  Index m_NextID;

-  std::map<Index, FilesOrigin> m_Origins;

-  std::map<std::wstring, Index> m_OriginsNameMap;

-  std::map<int, Index> m_OriginsPriorityMap;

-

-  Index createID()

-  {

-    return m_NextID++;

-  }

-};

-

-

-FileEntry::FileEntry() :

-  m_Index(UINT_MAX), m_Name(), m_Origin(-1), m_Parent(nullptr),

-  m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize),

-  m_LastAccessed(time(nullptr))

+DirectoryEntry::DirectoryEntry(

+  std::wstring name, DirectoryEntry* parent, int originID) :

+    m_OriginConnection(new OriginConnection),

+    m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(true)

 {

+  m_FileRegister.reset(new FileRegister(m_OriginConnection));

+  m_Origins.insert(originID);

 }

 

-FileEntry::FileEntry(Index index, const std::wstring &name, DirectoryEntry *parent) :

-  m_Index(index), m_Name(name), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent),

-  m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize),

-  m_LastAccessed(time(nullptr))

+DirectoryEntry::DirectoryEntry(

+  std::wstring name, DirectoryEntry* parent, int originID,

+  boost::shared_ptr<FileRegister> fileRegister,

+  boost::shared_ptr<OriginConnection> originConnection) :

+    m_FileRegister(fileRegister), m_OriginConnection(originConnection),

+    m_Name(std::move(name)), m_Parent(parent), m_Populated(false), m_TopLevel(false)

 {

+  m_Origins.insert(originID);

 }

 

-void FileEntry::addOrigin(

-  int origin, FILETIME fileTime, const std::wstring &archive, int order)

+DirectoryEntry::~DirectoryEntry()

 {

-  m_LastAccessed = time(nullptr);

-  if (m_Parent != nullptr) {

-    m_Parent->propagateOrigin(origin);

-  }

-

-  if (m_Origin == -1) {

-    // If this file has no previous origin, this mod is now the origin with no

-    // alternatives

-    m_Origin = origin;

-    m_FileTime = fileTime;

-    m_Archive = std::pair<std::wstring, int>(archive, order);

-  }

-  else if (

-    (m_Parent != nullptr) && (

-      (m_Parent->getOriginByID(origin).getPriority() > m_Parent->getOriginByID(m_Origin).getPriority()) ||

-      (archive.size() == 0 && m_Archive.first.size() > 0 ))

-    ) {

-    // If this mod has a higher priority than the origin mod OR

-    // this mod has a loose file and the origin mod has an archived file,

-    // this mod is now the origin and the previous origin is the first alternative

-

-    auto itor = std::find_if(

-      m_Alternatives.begin(), m_Alternatives.end(),

-      [&](auto&& i) { return i.first == m_Origin; });

-

-    if (itor == m_Alternatives.end()) {

-      m_Alternatives.push_back({m_Origin, m_Archive});

-    }

-

-    m_Origin = origin;

-    m_FileTime = fileTime;

-    m_Archive = std::pair<std::wstring, int>(archive, order);

-  }

-  else {

-    // This mod is just an alternative

-    bool found = false;

-

-    if (m_Origin == origin) {

-      // already an origin

-      return;

-    }

-

-    for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) {

-      if (iter->first == origin) {

-        // already an origin

-        return;

-      }

-

-      if ((m_Parent != nullptr) &&

-        (m_Parent->getOriginByID(iter->first).getPriority() < m_Parent->getOriginByID(origin).getPriority())) {

-        m_Alternatives.insert(iter, {origin, {archive, order}});

-        found = true;

-        break;

-      }

-    }

-

-    if (!found) {

-      m_Alternatives.push_back({origin, {archive, order}});

-    }

-  }

+  clear();

 }

 

-bool FileEntry::removeOrigin(int origin)

+void DirectoryEntry::clear()

 {

-  if (m_Origin == origin) {

-    if (!m_Alternatives.empty()) {

-      // find alternative with the highest priority

-      auto currentIter = m_Alternatives.begin();

-      for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) {

-        if (iter->first != origin) {

-          //Both files are not from archives.

-          if (!iter->second.first.size() && !currentIter->second.first.size()) {

-            if ((m_Parent->getOriginByID(iter->first).getPriority() > m_Parent->getOriginByID(currentIter->first).getPriority())) {

-              currentIter = iter;

-            }

-          }

-          else {

-            //Both files are from archives

-            if (iter->second.first.size() && currentIter->second.first.size()) {

-              if (iter->second.second > currentIter->second.second) {

-                currentIter = iter;

-              }

-            }

-            else {

-              //Only one of the two is an archive, so we change currentIter only if he is the archive one.

-              if (currentIter->second.first.size()) {

-                currentIter = iter;

-              }

-            }

-          }

-        }

-      }

-

-      int currentID = currentIter->first;

-      m_Archive = currentIter->second;

-      m_Alternatives.erase(currentIter);

-

-      m_Origin = currentID;

-    } else {

-      m_Origin = -1;

-      m_Archive = std::pair<std::wstring, int>(L"", -1);

-      return true;

-    }

-  } else {

-    auto newEnd = std::remove_if(

-      m_Alternatives.begin(), m_Alternatives.end(),

-      [&](auto &i) { return i.first == origin; });

-

-    if (newEnd != m_Alternatives.end()) {

-      m_Alternatives.erase(newEnd, m_Alternatives.end());

-    }

+  for (auto itor=m_SubDirectories.rbegin(); itor!=m_SubDirectories.rend(); ++itor) {

+    delete *itor;

   }

-  return false;

-}

-

-void FileEntry::sortOrigins()

-{

-  m_Alternatives.push_back({m_Origin, m_Archive});

-

-  std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) {

-    if (!LHS.second.first.size() && !RHS.second.first.size()) {

-      int l = m_Parent->getOriginByID(LHS.first).getPriority();

-      if (l < 0) {

-        l = INT_MAX;

-      }

 

-      int r = m_Parent->getOriginByID(RHS.first).getPriority();

-      if (r < 0) {

-        r = INT_MAX;

-      }

-

-      return l < r;

-    }

-

-    if (LHS.second.first.size() && RHS.second.first.size()) {

-      int l = LHS.second.second; if (l < 0) l = INT_MAX;

-      int r = RHS.second.second; if (r < 0) r = INT_MAX;

-

-      return l < r;

-    }

-

-    if (RHS.second.first.size()) {

-        return false;

-    }

-

-    return true;

-  });

-

-  if (!m_Alternatives.empty()) {

-    m_Origin = m_Alternatives.back().first;

-    m_Archive = m_Alternatives.back().second;

-    m_Alternatives.pop_back();

-  }

+  m_Files.clear();

+  m_FilesLookup.clear();

+  m_SubDirectories.clear();

+  m_SubDirectoriesLookup.clear();

 }

 

-bool FileEntry::isFromArchive(std::wstring archiveName) const

+void DirectoryEntry::addFromOrigin(

+  const std::wstring &originName, const std::wstring &directory, int priority,

+  DirectoryStats& stats)

 {

-  if (archiveName.length() == 0) {

-    return m_Archive.first.length() != 0;

-  }

-

-  if (m_Archive.first.compare(archiveName) == 0) {

-    return true;

-  }

-

-  for (auto alternative : m_Alternatives) {

-    if (alternative.second.first.compare(archiveName) == 0) {

-      return true;

-    }

-  }

-

-  return false;

+  env::DirectoryWalker walker;

+  addFromOrigin(walker, originName, directory, priority, stats);

 }

 

-std::wstring FileEntry::getFullPath(int originID) const

+void DirectoryEntry::addFromOrigin(

+  env::DirectoryWalker& walker, const std::wstring &originName,

+  const std::wstring &directory, int priority, DirectoryStats& stats)

 {

-  if (originID == -1) {

-    bool ignore = false;

-    originID = getOrigin(ignore);

-  }

+  FilesOrigin &origin = createOrigin(originName, directory, priority, stats);

 

-  // base directory for origin

-  const auto* o = m_Parent->findOriginByID(originID);

-  if (!o) {

-    return {};

+  if (!directory.empty()) {

+    addFiles(walker, origin, directory, stats);

   }

 

-  std::wstring result = o->getPath();

-

-  // all intermediate directories

-  recurseParents(result, m_Parent);

-

-  return result + L"\\" + m_Name;

+  m_Populated = true;

 }

 

-std::wstring FileEntry::getRelativePath() const

+void DirectoryEntry::addFromList(

+  const std::wstring &originName, const std::wstring &directory,

+  env::Directory& root, int priority, DirectoryStats& stats)

 {

-  std::wstring result;

-

-  // all intermediate directories

-  recurseParents(result, m_Parent);

+  stats = {};

 

-  return result + L"\\" + m_Name;

+  FilesOrigin &origin = createOrigin(originName, directory, priority, stats);

+  addDir(origin, root, stats);

 }

 

-bool FileEntry::recurseParents(std::wstring &path, const DirectoryEntry *parent) const

+void DirectoryEntry::addDir(

+  FilesOrigin& origin, env::Directory& d, DirectoryStats& stats)

 {

-  if (parent == nullptr) {

-    return false;

-  } else {

-    // don't append the topmost parent because it is the virtual data-root

-    if (recurseParents(path, parent->getParent())) {

-      path.append(L"\\").append(parent->getName());

+  elapsed(stats.dirTimes, [&]{

+    for (auto& sd : d.dirs) {

+      auto* sdirEntry = getSubDirectory(sd, true, stats, origin.getID());

+      sdirEntry->addDir(origin, sd, stats);

     }

+  });

 

-    return true;

-  }

-}

-

-

-FilesOrigin::FilesOrigin()

-  : m_ID(0), m_Disabled(false), m_Name(), m_Path(), m_Priority(0)

-{

-}

-

-FilesOrigin::FilesOrigin(const FilesOrigin &reference)

-  : m_ID(reference.m_ID)

-  , m_Disabled(reference.m_Disabled)

-  , m_Name(reference.m_Name)

-  , m_Path(reference.m_Path)

-  , m_Priority(reference.m_Priority)

-  , m_FileRegister(reference.m_FileRegister)

-  , m_OriginConnection(reference.m_OriginConnection)

-{

-}

-

-FilesOrigin::FilesOrigin(

-  int ID, const std::wstring &name, const std::wstring &path, int priority,

-  boost::shared_ptr<MOShared::FileRegister> fileRegister,

-  boost::shared_ptr<MOShared::OriginConnection> originConnection) :

-    m_ID(ID), m_Disabled(false), m_Name(name), m_Path(path),

-    m_Priority(priority), m_FileRegister(fileRegister),

-    m_OriginConnection(originConnection)

-{

-}

-

-void FilesOrigin::setPriority(int priority)

-{

-  m_OriginConnection.lock()->changePriorityLookup(m_Priority, priority);

-

-  m_Priority = priority;

-}

-

-void FilesOrigin::setName(const std::wstring &name)

-{

-  m_OriginConnection.lock()->changeNameLookup(m_Name, name);

+  elapsed(stats.fileTimes, [&]{

+    for (auto& f : d.files) {

+      insert(f, origin, L"", -1, stats);

+    }

+  });

 

-  // change path too

-  if (tail(m_Path, m_Name.length()) == m_Name) {

-    m_Path = m_Path.substr(0, m_Path.length() - m_Name.length()).append(name);

-  }

+  elapsed(stats.sortTimes, [&]{

+    std::sort(

+      m_SubDirectories.begin(),

+      m_SubDirectories.end(),

+      &DirCompareByName);

+  });

 

-  m_Name = name;

+  m_Populated = true;

 }

 

-std::vector<FileEntry::Ptr> FilesOrigin::getFiles() const

+void DirectoryEntry::addFromAllBSAs(

+  const std::wstring& originName, const std::wstring& directory,

+  int priority, const std::vector<std::wstring>& archives,

+  const std::set<std::wstring>& enabledArchives,

+  const std::vector<std::wstring>& loadOrder,

+  DirectoryStats& stats)

 {

-  std::vector<FileEntry::Ptr> result;

+  for (const auto& archive : archives) {

+    const std::filesystem::path archivePath(archive);

+    const auto filename = archivePath.filename().native();

 

-  for (FileEntry::Index fileIdx : m_Files) {

-    if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) {

-      result.push_back(p);

+    if (!enabledArchives.contains(filename)) {

+      continue;

     }

-  }

 

-  return result;

-}

+    const auto filenameLc = ToLowerCopy(filename);

 

-FileEntry::Ptr FilesOrigin::findFile(FileEntry::Index index) const

-{

-  return m_FileRegister.lock()->getFile(index);

-}

+    int order = -1;

 

-void FilesOrigin::enable(bool enabled, time_t notAfter)

-{

-  if (!enabled) {

-    std::set<FileEntry::Index> copy = m_Files;

-    m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter);

-    m_Files.clear();

-  }

-

-  m_Disabled = !enabled;

-}

-

-void FilesOrigin::removeFile(FileEntry::Index index)

-{

-  auto iter = m_Files.find(index);

-

-  if (iter != m_Files.end()) {

-    m_Files.erase(iter);

-  }

-}

+    for (auto plugin : loadOrder)

+    {

+      const auto pluginNameLc =

+        ToLowerCopy(std::filesystem::path(plugin).stem().native());

 

-bool FilesOrigin::containsArchive(std::wstring archiveName)

-{

-  for (FileEntry::Index fileIdx : m_Files) {

-    if (FileEntry::Ptr p = m_FileRegister.lock()->getFile(fileIdx)) {

-      if (p->isFromArchive(archiveName)) {

-        return true;

+      if (filenameLc.starts_with(pluginNameLc + L" - ") ||

+          filenameLc.starts_with(pluginNameLc + L".")) {

+        auto itor = std::find(loadOrder.begin(), loadOrder.end(), plugin);

+        if (itor != loadOrder.end()) {

+          order = std::distance(loadOrder.begin(), itor);

+        }

       }

     }

-  }

 

-  return false;

-}

-

-

-FileRegister::FileRegister(boost::shared_ptr<OriginConnection> originConnection)

-  : m_OriginConnection(originConnection)

-{

-}

-

-bool FileRegister::indexValid(FileEntry::Index index) const

-{

-  return (m_Files.find(index) != m_Files.end());

-}

-

-FileEntry::Ptr FileRegister::createFile(const std::wstring &name, DirectoryEntry *parent)

-{

-  FileEntry::Index index = generateIndex();

-

-  auto r = m_Files.insert_or_assign(

-    index, FileEntry::Ptr(new FileEntry(index, name, parent)));

-

-  return r.first->second;

-}

-

-FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const

-{

-  auto iter = m_Files.find(index);

-

-  if (iter != m_Files.end()) {

-    return iter->second;

-  } else {

-    return FileEntry::Ptr();

-  }

-}

-

-bool FileRegister::removeFile(FileEntry::Index index)

-{

-  auto iter = m_Files.find(index);

-

-  if (iter != m_Files.end()) {

-    unregisterFile(iter->second);

-    m_Files.erase(index);

-    return true;

-  } else {

-    log::error(QObject::tr("invalid file index for remove: {}").toStdString(), index);

-    return false;

-  }

-}

-

-void FileRegister::removeOrigin(FileEntry::Index index, int originID)

-{

-  auto iter = m_Files.find(index);

-

-  if (iter != m_Files.end()) {

-    if (iter->second->removeOrigin(originID)) {

-      unregisterFile(iter->second);

-      m_Files.erase(iter);

-    }

-  } else {

-    log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index);

+    addFromBSA(

+      originName, directory, archivePath.native(),

+      priority, order, stats);

   }

 }

 

-void FileRegister::removeOriginMulti(

-  std::set<FileEntry::Index> indices, int originID, time_t notAfter)

+void DirectoryEntry::addFromBSA(

+  const std::wstring& originName, const std::wstring& directory,

+  const std::wstring& archivePath, int priority, int order, DirectoryStats& stats)

 {

-  std::vector<FileEntry::Ptr> removedFiles;

-

-  for (auto iter = indices.begin(); iter != indices.end(); ) {

-    auto pos = m_Files.find(*iter);

-

-    if (pos != m_Files.end()

-        && (pos->second->lastAccessed() < notAfter)

-        && pos->second->removeOrigin(originID)) {

-      removedFiles.push_back(pos->second);

-      m_Files.erase(pos);

-      ++iter;

-    } else {

-      indices.erase(iter++);

-    }

-  }

-

-  // optimization: this is only called when disabling an origin and in this case

-  // we don't have to remove the file from the origin

-

-  // need to remove files from their parent directories. multiple ways to go

-  // about this:

-  //   a) for each file, search its parents file-list (preferably by name) and

-  //      remove what is found

-  //   b) gather the parent directories, go through the file list for each once

-  //      and remove all files that have been removed

-  //

-  // the latter should be faster when there are many files in few directories.

-  // since this is called only when disabling an origin that is probably

-  // frequently the case

-

-  std::set<DirectoryEntry*> parents;

-  for (const FileEntry::Ptr &file : removedFiles) {

-    if (file->getParent() != nullptr) {

-      parents.insert(file->getParent());

-    }

-  }

+  FilesOrigin& origin = createOrigin(originName, directory, priority, stats);

+  const auto archiveName = std::filesystem::path(archivePath).filename().native();

 

-  for (DirectoryEntry *parent : parents) {

-    parent->removeFiles(indices);

+  if (containsArchive(archiveName)) {

+    return;

   }

-}

-

-void FileRegister::sortOrigins()

-{

-  for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) {

-    iter->second->sortOrigins();

-  }

-}

-

-FileEntry::Index FileRegister::generateIndex()

-{

-  static std::atomic<FileEntry::Index> sIndex(0);

-  return sIndex++;

-}

-

-void FileRegister::unregisterFile(FileEntry::Ptr file)

-{

-  bool ignore;

 

-  // unregister from origin

-  int originID = file->getOrigin(ignore);

-  m_OriginConnection->getByID(originID).removeFile(file->getIndex());

-  const auto& alternatives = file->getAlternatives();

+  BSA::Archive archive;

+  BSA::EErrorCode res = archive.read(ToString(archivePath, false).c_str(), false);

 

-  for (auto iter = alternatives.begin(); iter != alternatives.end(); ++iter) {

-    m_OriginConnection->getByID(iter->first).removeFile(file->getIndex());

+  if ((res != BSA::ERROR_NONE) && (res != BSA::ERROR_INVALIDHASHES)) {

+    log::error("invalid bsa '{}', error {}", archivePath, res);

+    return;

   }

 

-  // unregister from directory

-  if (file->getParent() != nullptr) {

-    file->getParent()->removeFile(file->getIndex());

-  }

-}

-

-

-DirectoryEntry::DirectoryEntry(

-  const std::wstring &name, DirectoryEntry *parent, int originID) :

-    m_OriginConnection(new OriginConnection),

-    m_Name(name), m_Parent(parent), m_Populated(false), m_TopLevel(true)

-{

-  m_FileRegister.reset(new FileRegister(m_OriginConnection));

-  m_Origins.insert(originID);

-}

-

-DirectoryEntry::DirectoryEntry(

-  const std::wstring &name, DirectoryEntry *parent, int originID,

-  boost::shared_ptr<FileRegister> fileRegister,

-  boost::shared_ptr<OriginConnection> originConnection) :

-    m_FileRegister(fileRegister), m_OriginConnection(originConnection),

-    m_Name(name), m_Parent(parent), m_Populated(false), m_TopLevel(false)

-{

-  m_Origins.insert(originID);

-}

-

-DirectoryEntry::~DirectoryEntry()

-{

-  clear();

-}

-

-void DirectoryEntry::clear()

-{

-  m_Files.clear();

-  m_FilesLookup.clear();

+  std::error_code ec;

+  const auto lwt = std::filesystem::last_write_time(archivePath, ec);

+  FILETIME ft = {};

 

-  for (DirectoryEntry *entry : m_SubDirectories) {

-    delete entry;

+  if (ec) {

+    log::warn(

+      "failed to get last modified date for '{}', {}",

+      archivePath, ec.message());

+  } else {

+    ft = ToFILETIME(lwt);

   }

 

-  m_SubDirectories.clear();

-  m_SubDirectoriesLookup.clear();

-}

-

-void DirectoryEntry::addFromOrigin(

-  const std::wstring &originName, const std::wstring &directory, int priority)

-{

-  FilesOrigin &origin = createOrigin(originName, directory, priority);

-

-  if (directory.length() != 0) {

-    boost::scoped_array<wchar_t> buffer(new wchar_t[MAXPATH_UNICODE + 1]);

-    memset(buffer.get(), L'\0', MAXPATH_UNICODE + 1);

-    int offset = _snwprintf(buffer.get(), MAXPATH_UNICODE, L"%ls", directory.c_str());

-    buffer.get()[offset] = L'\0';

-    addFiles(origin, buffer.get(), offset);

-  }

+  addFiles(origin, archive.getRoot(), ft, archiveName, order, stats);

 

   m_Populated = true;

 }

 

-void DirectoryEntry::addFromBSA(

-  const std::wstring &originName, std::wstring &directory,

-  const std::wstring &fileName, int priority, int order)

-{

-  FilesOrigin &origin = createOrigin(originName, directory, priority);

-

-  WIN32_FILE_ATTRIBUTE_DATA fileData;

-  if (::GetFileAttributesExW(fileName.c_str(), GetFileExInfoStandard, &fileData) == 0) {

-    throw windows_error(QObject::tr("failed to determine file time").toStdString());

-  }

-

-  FILETIME now;

-  ::GetSystemTimeAsFileTime(&now);

-

-  const double clfSecondsPer100ns = 100. * 1.E-9;

-

-  ((ULARGE_INTEGER *)&now)->QuadPart -= ((double)5) / clfSecondsPer100ns;

-

-  size_t namePos = fileName.find_last_of(L"\\/");

-  if (namePos == std::wstring::npos) {

-    namePos = 0;

-  }

-  else {

-    ++namePos;

-  }

-

-  if (!containsArchive(fileName.substr(namePos)) || ::CompareFileTime(&fileData.ftLastWriteTime, &now) > 0) {

-    BSA::Archive archive;

-    BSA::EErrorCode res = archive.read(ToString(fileName, false).c_str(), false);

-

-    if ((res != BSA::ERROR_NONE) && (res != BSA::ERROR_INVALIDHASHES)) {

-      std::ostringstream stream;

-

-      stream

-		<< QObject::tr("invalid bsa file: ").toStdString()

-		<< ToString(fileName, false)

-		<< " error code " << res << " - " << ::GetLastError();

-

-      throw std::runtime_error(stream.str());

-    }

-

-    addFiles(origin, archive.getRoot(), fileData.ftLastWriteTime, fileName.substr(namePos), order);

-    m_Populated = true;

-  }

-}

-

 void DirectoryEntry::propagateOrigin(int origin)

 {

-  m_Origins.insert(origin);

+  {

+    std::scoped_lock lock(m_OriginsMutex);

+    m_Origins.insert(origin);

+  }

 

   if (m_Parent != nullptr) {

     m_Parent->propagateOrigin(origin);

@@ -769,7 +278,7 @@ int DirectoryEntry::anyOrigin() const
   bool ignore;

 

   for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) {

-    FileEntry::Ptr entry = m_FileRegister->getFile(iter->second);

+    FileEntryPtr entry = m_FileRegister->getFile(iter->second);

     if ((entry.get() != nullptr) && !entry->isFromArchive()) {

       return entry->getOrigin(ignore);

     }

@@ -777,9 +286,9 @@ int DirectoryEntry::anyOrigin() const
 

   // if we got here, no file directly within this directory is a valid indicator for a mod, thus

   // we continue looking in subdirectories

-  for (DirectoryEntry *entry : m_SubDirectories) {

+  for (DirectoryEntry* entry : m_SubDirectories) {

     int res = entry->anyOrigin();

-    if (res != -1){

+    if (res != InvalidOriginID){

       return res;

     }

   }

@@ -787,9 +296,9 @@ int DirectoryEntry::anyOrigin() const
   return *(m_Origins.begin());

 }

 

-std::vector<FileEntry::Ptr> DirectoryEntry::getFiles() const

+std::vector<FileEntryPtr> DirectoryEntry::getFiles() const

 {

-  std::vector<FileEntry::Ptr> result;

+  std::vector<FileEntryPtr> result;

 

   for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) {

     result.push_back(m_FileRegister->getFile(iter->second));

@@ -798,7 +307,7 @@ std::vector<FileEntry::Ptr> DirectoryEntry::getFiles() const
   return result;

 }

 

-DirectoryEntry *DirectoryEntry::findSubDirectory(

+DirectoryEntry* DirectoryEntry::findSubDirectory(

   const std::wstring &name, bool alreadyLowerCase) const

 {

   SubDirectoriesLookup::const_iterator itor;

@@ -816,37 +325,38 @@ DirectoryEntry *DirectoryEntry::findSubDirectory(
   return itor->second;

 }

 

-DirectoryEntry *DirectoryEntry::findSubDirectoryRecursive(const std::wstring &path)

+DirectoryEntry* DirectoryEntry::findSubDirectoryRecursive(const std::wstring &path)

 {

-  return getSubDirectoryRecursive(path, false, -1);

+  DirectoryStats dummy;

+  return getSubDirectoryRecursive(path, false, dummy, InvalidOriginID);

 }

 

-const FileEntry::Ptr DirectoryEntry::findFile(

+const FileEntryPtr DirectoryEntry::findFile(

   const std::wstring &name, bool alreadyLowerCase) const

 {

   FilesLookup::const_iterator iter;

 

   if (alreadyLowerCase) {

-    iter = m_FilesLookup.find(FileKey(name));

+    iter = m_FilesLookup.find(DirectoryEntryFileKey(name));

   } else {

-    iter = m_FilesLookup.find(FileKey(ToLowerCopy(name)));

+    iter = m_FilesLookup.find(DirectoryEntryFileKey(ToLowerCopy(name)));

   }

 

   if (iter != m_FilesLookup.end()) {

     return m_FileRegister->getFile(iter->second);

   } else {

-    return FileEntry::Ptr();

+    return FileEntryPtr();

   }

 }

 

-const FileEntry::Ptr DirectoryEntry::findFile(const FileKey& key) const

+const FileEntryPtr DirectoryEntry::findFile(const DirectoryEntryFileKey& key) const

 {

   auto iter = m_FilesLookup.find(key);

 

   if (iter != m_FilesLookup.end()) {

     return m_FileRegister->getFile(iter->second);

   } else {

-    return FileEntry::Ptr();

+    return FileEntryPtr();

   }

 }

 

@@ -858,7 +368,7 @@ bool DirectoryEntry::hasFile(const std::wstring& name) const
 bool DirectoryEntry::containsArchive(std::wstring archiveName)

 {

   for (auto iter = m_Files.begin(); iter != m_Files.end(); ++iter) {

-    FileEntry::Ptr entry = m_FileRegister->getFile(iter->second);

+    FileEntryPtr entry = m_FileRegister->getFile(iter->second);

     if (entry->isFromArchive(archiveName)) {

       return true;

     }

@@ -867,8 +377,8 @@ bool DirectoryEntry::containsArchive(std::wstring archiveName)
   return false;

 }

 

-const FileEntry::Ptr DirectoryEntry::searchFile(

-  const std::wstring &path, const DirectoryEntry **directory) const

+const FileEntryPtr DirectoryEntry::searchFile(

+  const std::wstring &path, const DirectoryEntry** directory) const

 {

   if (directory != nullptr) {

     *directory = nullptr;

@@ -880,7 +390,7 @@ const FileEntry::Ptr DirectoryEntry::searchFile(
       *directory = this;

     }

 

-    return FileEntry::Ptr();

+    return FileEntryPtr();

   }

 

   const size_t len =  path.find_first_of(L"\\/");

@@ -892,7 +402,7 @@ const FileEntry::Ptr DirectoryEntry::searchFile(
     if (iter != m_Files.end()) {

       return m_FileRegister->getFile(iter->second);

     } else if (directory != nullptr) {

-      DirectoryEntry *temp = findSubDirectory(path);

+      DirectoryEntry* temp = findSubDirectory(path);

       if (temp != nullptr) {

         *directory = temp;

       }

@@ -900,41 +410,27 @@ const FileEntry::Ptr DirectoryEntry::searchFile(
   } else {

     // file is in a subdirectory, recurse into the matching subdirectory

     std::wstring pathComponent = path.substr(0, len);

-    DirectoryEntry *temp = findSubDirectory(pathComponent);

+    DirectoryEntry* temp = findSubDirectory(pathComponent);

 

     if (temp != nullptr) {

       if (len >= path.size()) {

         log::error(QObject::tr("unexpected end of path").toStdString());

-        return FileEntry::Ptr();

+        return FileEntryPtr();

       }

 

       return temp->searchFile(path.substr(len + 1), directory);

     }

   }

 

-  return FileEntry::Ptr();

-}

-

-void DirectoryEntry::insertFile(

-  const std::wstring &filePath, FilesOrigin &origin, FILETIME fileTime)

-{

-  size_t pos = filePath.find_first_of(L"\\/");

-

-  if (pos == std::string::npos) {

-    this->insert(filePath, origin, fileTime, std::wstring(), -1);

-  } else {

-    std::wstring dirName = filePath.substr(0, pos);

-    std::wstring rest = filePath.substr(pos + 1);

-    getSubDirectoryRecursive(dirName, true, origin.getID())->insertFile(rest, origin, fileTime);

-  }

+  return FileEntryPtr();

 }

 

-void DirectoryEntry::removeFile(FileEntry::Index index)

+void DirectoryEntry::removeFile(FileIndex index)

 {

   removeFileFromList(index);

 }

 

-bool DirectoryEntry::removeFile(const std::wstring &filePath, int *origin)

+bool DirectoryEntry::removeFile(const std::wstring &filePath, int* origin)

 {

   size_t pos = filePath.find_first_of(L"\\/");

 

@@ -944,7 +440,9 @@ bool DirectoryEntry::removeFile(const std::wstring &filePath, int *origin)
 

   std::wstring dirName = filePath.substr(0, pos);

   std::wstring rest = filePath.substr(pos + 1);

-  DirectoryEntry *entry = getSubDirectoryRecursive(dirName, false);

+

+  DirectoryStats dummy;

+  DirectoryEntry* entry = getSubDirectoryRecursive(dirName, false, dummy);

 

   if (entry != nullptr) {

     return entry->removeFile(rest, origin);

@@ -959,7 +457,7 @@ void DirectoryEntry::removeDir(const std::wstring &path)
 

   if (pos == std::string::npos) {

     for (auto iter = m_SubDirectories.begin(); iter != m_SubDirectories.end(); ++iter) {

-      DirectoryEntry *entry = *iter;

+      DirectoryEntry* entry = *iter;

 

       if (CaseInsensitiveEqual(entry->getName(), path)) {

         entry->removeDirRecursive();

@@ -971,7 +469,9 @@ void DirectoryEntry::removeDir(const std::wstring &path)
   } else {

     std::wstring dirName = path.substr(0, pos);

     std::wstring rest = path.substr(pos + 1);

-    DirectoryEntry *entry = getSubDirectoryRecursive(dirName, false);

+

+    DirectoryStats dummy;

+    DirectoryEntry* entry = getSubDirectoryRecursive(dirName, false, dummy);

 

     if (entry != nullptr) {

       entry->removeDir(rest);

@@ -979,7 +479,7 @@ void DirectoryEntry::removeDir(const std::wstring &path)
   }

 }

 

-bool DirectoryEntry::remove(const std::wstring &fileName, int *origin)

+bool DirectoryEntry::remove(const std::wstring &fileName, int* origin)

 {

   const auto lcFileName = ToLowerCopy(fileName);

 

@@ -988,7 +488,7 @@ bool DirectoryEntry::remove(const std::wstring &fileName, int *origin)
 

   if (iter != m_Files.end()) {

     if (origin != nullptr) {

-      FileEntry::Ptr entry = m_FileRegister->getFile(iter->second);

+      FileEntryPtr entry = m_FileRegister->getFile(iter->second);

       if (entry.get() != nullptr) {

         bool ignore;

         *origin = entry->getOrigin(ignore);

@@ -1007,98 +507,204 @@ bool DirectoryEntry::hasContentsFromOrigin(int originID) const
 }

 

 FilesOrigin &DirectoryEntry::createOrigin(

-  const std::wstring &originName, const std::wstring &directory, int priority)

+  const std::wstring &originName, const std::wstring &directory, int priority,

+  DirectoryStats& stats)

 {

-  if (m_OriginConnection->exists(originName)) {

-    FilesOrigin &origin = m_OriginConnection->getByName(originName);

-    origin.enable(true);

-    return origin;

+  auto r = m_OriginConnection->getOrCreate(

+    originName, directory, priority,

+    m_FileRegister, m_OriginConnection, stats);

+

+  if (r.second) {

+    ++stats.originCreate;

   } else {

-    return m_OriginConnection->createOrigin(

-      originName, directory, priority, m_FileRegister, m_OriginConnection);

+    ++stats.originExists;

   }

+

+  return r.first;

 }

 

-void DirectoryEntry::removeFiles(const std::set<FileEntry::Index> &indices)

+void DirectoryEntry::removeFiles(const std::set<FileIndex> &indices)

 {

   removeFilesFromList(indices);

 }

 

-FileEntry::Ptr DirectoryEntry::insert(

-  const std::wstring &fileName, FilesOrigin &origin, FILETIME fileTime,

-  const std::wstring &archive, int order)

+FileEntryPtr DirectoryEntry::insert(

+  std::wstring_view fileName, FilesOrigin &origin, FILETIME fileTime,

+  std::wstring_view archive, int order, DirectoryStats& stats)

 {

   std::wstring fileNameLower = ToLowerCopy(fileName);

+  FileEntryPtr fe;

 

-  auto iter = m_Files.find(fileNameLower);

-  FileEntry::Ptr file;

+  DirectoryEntryFileKey key(std::move(fileNameLower));

 

-  if (iter != m_Files.end()) {

-    file = m_FileRegister->getFile(iter->second);

-  } else {

-    file = m_FileRegister->createFile(fileName, this);

-    addFileToList(std::move(fileNameLower), file->getIndex());

+  {

+    std::unique_lock lock(m_FilesMutex);

+

+    FilesLookup::iterator itor;

+

+    elapsed(stats.filesLookupTimes, [&]{

+      itor = m_FilesLookup.find(key);

+    });

+

+    if (itor != m_FilesLookup.end()) {

+      lock.unlock();

+      ++stats.fileExists;

+      fe = m_FileRegister->getFile(itor->second);

+    } else {

+      ++stats.fileCreate;

+      fe = m_FileRegister->createFile(

+        std::wstring(fileName.begin(), fileName.end()), this, stats);

 

-    // fileNameLower has moved from this point

+      elapsed(stats.addFileTimes, [&] {

+        addFileToList(std::move(key.value), fe->getIndex());

+      });

+

+      // fileNameLower has moved from this point

+    }

   }

 

-  file->addOrigin(origin.getID(), fileTime, archive, order);

-  origin.addFile(file->getIndex());

+  elapsed(stats.addOriginToFileTimes, [&]{

+    fe->addOrigin(origin.getID(), fileTime, archive, order);

+  });

 

-  return file;

+  elapsed(stats.addFileToOriginTimes, [&]{

+    origin.addFile(fe->getIndex());

+  });

+

+  return fe;

 }

 

-void DirectoryEntry::addFiles(FilesOrigin &origin, wchar_t *buffer, int bufferOffset)

+FileEntryPtr DirectoryEntry::insert(

+  env::File& file, FilesOrigin &origin, std::wstring_view archive, int order,

+  DirectoryStats& stats)

 {

-  WIN32_FIND_DATAW findData;

+  FileEntryPtr fe;

 

-  _snwprintf_s(buffer + bufferOffset, MAXPATH_UNICODE - bufferOffset, _TRUNCATE, L"\\*");

+  {

+    std::unique_lock lock(m_FilesMutex);

 

-  HANDLE searchHandle = nullptr;

+    FilesMap::iterator itor;

 

-  if (SupportOptimizedFind()) {

-    searchHandle = ::FindFirstFileExW(

-      buffer, FindExInfoBasic, &findData, FindExSearchNameMatch, nullptr,

-      FIND_FIRST_EX_LARGE_FETCH);

-  } else {

-    searchHandle = ::FindFirstFileExW(

-      buffer, FindExInfoStandard, &findData, FindExSearchNameMatch, nullptr, 0);

+    elapsed(stats.filesLookupTimes, [&]{

+      itor = m_Files.find(file.lcname);

+    });

+

+    if (itor != m_Files.end()) {

+      lock.unlock();

+      ++stats.fileExists;

+      fe = m_FileRegister->getFile(itor->second);

+    } else {

+      ++stats.fileCreate;

+      fe = m_FileRegister->createFile(std::move(file.name), this, stats);

+      // file.name has been moved from this point

+

+      elapsed(stats.addFileTimes, [&]{

+        addFileToList(std::move(file.lcname), fe->getIndex());

+      });

+

+      // file.lcname has been moved from this point

+    }

   }

 

-  if (searchHandle != INVALID_HANDLE_VALUE) {

-    BOOL result = true;

+  elapsed(stats.addOriginToFileTimes, [&]{

+    fe->addOrigin(origin.getID(), file.lastModified, archive, order);

+  });

+

+  elapsed(stats.addFileToOriginTimes, [&]{

+    origin.addFile(fe->getIndex());

+  });

 

-    while (result) {

-      if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {

-        if ((wcscmp(findData.cFileName, L".") != 0) &&

-            (wcscmp(findData.cFileName, L"..") != 0)) {

-          int offset = _snwprintf(buffer + bufferOffset, MAXPATH_UNICODE, L"\\%ls", findData.cFileName);

+  return fe;

+}

 

-          // recurse into subdirectories

-          DirectoryEntry* sd = getSubDirectory(findData.cFileName, true, origin.getID());

-          sd->addFiles(origin, buffer, bufferOffset + offset);

-        }

-      } else {

-        insert(findData.cFileName, origin, findData.ftLastWriteTime, L"", -1);

-      }

+struct DirectoryEntry::Context

+{

+  FilesOrigin& origin;

+  DirectoryStats& stats;

+  std::stack<DirectoryEntry*> current;

+};

+

+void DirectoryEntry::addFiles(

+  env::DirectoryWalker& walker, FilesOrigin &origin,

+  const std::wstring& path, DirectoryStats& stats)

+{

+  Context cx = {origin, stats};

+  cx.current.push(this);

+

+  walker.forEachEntry(path, &cx,

+    [](void* pcx, std::wstring_view path)

+    {

+      onDirectoryStart((Context*)pcx, path);

+    },

+

+    [](void* pcx, std::wstring_view path)

+    {

+      onDirectoryEnd((Context*)pcx, path);

+    },

 

-      result = ::FindNextFileW(searchHandle, &findData);

+    [](void* pcx, std::wstring_view path, FILETIME ft)

+    {

+      onFile((Context*)pcx, path, ft);

     }

+  );

+

+  {

+    std::scoped_lock lock(m_SubDirMutex);

+

+    std::sort(

+      m_SubDirectories.begin(),

+      m_SubDirectories.end(),

+      &DirCompareByName);

   }

+}

+

+void DirectoryEntry::onDirectoryStart(Context* cx, std::wstring_view path)

+{

+  elapsed(cx->stats.dirTimes, [&] {

+    auto* sd = cx->current.top()->getSubDirectory(

+      path, true, cx->stats, cx->origin.getID());

+

+    cx->current.push(sd);

+  });

+}

+

+void DirectoryEntry::onDirectoryEnd(Context* cx, std::wstring_view path)

+{

+  elapsed(cx->stats.dirTimes, [&] {

+    auto* current = cx->current.top();

+

+    {

+      std::scoped_lock lock(current->m_SubDirMutex);

+

+      std::sort(

+        current->m_SubDirectories.begin(),

+        current->m_SubDirectories.end(),

+        &DirCompareByName);

+    }

 

-  std::sort(m_SubDirectories.begin(), m_SubDirectories.end(), &DirCompareByName);

-  ::FindClose(searchHandle);

+    cx->current.pop();

+  });

+}

+

+void DirectoryEntry::onFile(Context* cx, std::wstring_view path, FILETIME ft)

+{

+  elapsed(cx->stats.fileTimes, [&]{

+    cx->current.top()->insert(path, cx->origin, ft, L"", -1, cx->stats);

+  });

 }

 

 void DirectoryEntry::addFiles(

-  FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime,

-  const std::wstring &archiveName, int order)

+  FilesOrigin& origin, const BSA::Folder::Ptr archiveFolder, FILETIME fileTime,

+  const std::wstring& archiveName, int order, DirectoryStats& stats)

 {

   // add files

-  for (unsigned int fileIdx = 0; fileIdx < archiveFolder->getNumFiles(); ++fileIdx) {

-    BSA::File::Ptr file = archiveFolder->getFile(fileIdx);

+  const auto fileCount = archiveFolder->getNumFiles();

+  for (unsigned int i=0; i<fileCount; ++i) {

+    const BSA::File::Ptr file = archiveFolder->getFile(i);

 

-    auto f = insert(ToWString(file->getName(), true), origin, fileTime, archiveName, order);

+    auto f = insert(

+      ToWString(file->getName(), true), origin, fileTime,

+      archiveName, order, stats);

 

     if (f) {

       if (file->getUncompressedFileSize() > 0) {

@@ -1110,28 +716,45 @@ void DirectoryEntry::addFiles(
   }

 

   // recurse into subdirectories

-  for (unsigned int folderIdx = 0; folderIdx < archiveFolder->getNumSubFolders(); ++folderIdx) {

-    BSA::Folder::Ptr folder = archiveFolder->getSubFolder(folderIdx);

-    DirectoryEntry *folderEntry = getSubDirectoryRecursive(ToWString(folder->getName(), true), true, origin.getID());

+  const auto dirCount = archiveFolder->getNumSubFolders();

+  for (unsigned int i=0; i<dirCount; ++i) {

+    const BSA::Folder::Ptr folder = archiveFolder->getSubFolder(i);

+

+    DirectoryEntry* folderEntry = getSubDirectoryRecursive(

+      ToWString(folder->getName(), true), true, stats, origin.getID());

 

-    folderEntry->addFiles(origin, folder, fileTime, archiveName, order);

+    folderEntry->addFiles(origin, folder, fileTime, archiveName, order, stats);

   }

 }

 

-DirectoryEntry *DirectoryEntry::getSubDirectory(

-  const std::wstring &name, bool create, int originID)

+DirectoryEntry* DirectoryEntry::getSubDirectory(

+  std::wstring_view name, bool create, DirectoryStats& stats, int originID)

 {

-  for (DirectoryEntry *entry : m_SubDirectories) {

-    if (CaseInsensitiveEqual(entry->getName(), name)) {

-      return entry;

-    }

+  std::wstring nameLc = ToLowerCopy(name);

+

+  std::scoped_lock lock(m_SubDirMutex);

+

+  SubDirectoriesLookup::iterator itor;

+  elapsed(stats.subdirLookupTimes, [&] {

+    itor = m_SubDirectoriesLookup.find(nameLc);

+  });

+

+  if (itor != m_SubDirectoriesLookup.end()) {

+    ++stats.subdirExists;

+    return itor->second;

   }

 

   if (create) {

+    ++stats.subdirCreate;

+

     auto* entry = new DirectoryEntry(

-      name, this, originID, m_FileRegister, m_OriginConnection);

+      std::wstring(name.begin(), name.end()), this, originID,

+      m_FileRegister, m_OriginConnection);

 

-    addDirectoryToList(entry);

+    elapsed(stats.addDirectoryTimes, [&] {

+      addDirectoryToList(entry, std::move(nameLc));

+      // nameLc is moved from this point

+    });

 

     return entry;

   } else {

@@ -1139,8 +762,44 @@ DirectoryEntry *DirectoryEntry::getSubDirectory(
   }

 }

 

-DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive(

-  const std::wstring &path, bool create, int originID)

+DirectoryEntry* DirectoryEntry::getSubDirectory(

+  env::Directory& dir, bool create, DirectoryStats& stats, int originID)

+{

+  SubDirectoriesLookup::iterator itor;

+

+  std::scoped_lock lock(m_SubDirMutex);

+

+  elapsed(stats.subdirLookupTimes, [&] {

+    itor = m_SubDirectoriesLookup.find(dir.lcname);

+  });

+

+  if (itor != m_SubDirectoriesLookup.end()) {

+    ++stats.subdirExists;

+    return itor->second;

+  }

+

+  if (create) {

+    ++stats.subdirCreate;

+

+    auto* entry = new DirectoryEntry(

+      std::move(dir.name), this, originID,

+      m_FileRegister, m_OriginConnection);

+    // dir.name is moved from this point

+

+    elapsed(stats.addDirectoryTimes, [&]{

+      addDirectoryToList(entry, std::move(dir.lcname));

+    });

+

+    // dir.lcname is moved from this point

+

+    return entry;

+  } else {

+    return nullptr;

+  }

+}

+

+DirectoryEntry* DirectoryEntry::getSubDirectoryRecursive(

+  const std::wstring& path, bool create, DirectoryStats& stats, int originID)

 {

   if (path.length() == 0) {

     // path ended with a backslash?

@@ -1150,14 +809,16 @@ DirectoryEntry *DirectoryEntry::getSubDirectoryRecursive(
   const size_t pos = path.find_first_of(L"\\/");

 

   if (pos == std::wstring::npos) {

-    return getSubDirectory(path, create);

+    return getSubDirectory(path, create, stats);

   } else {

-    DirectoryEntry *nextChild = getSubDirectory(path.substr(0, pos), create, originID);

+    DirectoryEntry* nextChild = getSubDirectory(

+      path.substr(0, pos), create, stats, originID);

 

     if (nextChild == nullptr) {

       return nullptr;

     } else {

-      return nextChild->getSubDirectoryRecursive(path.substr(pos + 1), create, originID);

+      return nextChild->getSubDirectoryRecursive(

+        path.substr(pos + 1), create, stats, originID);

     }

   }

 }

@@ -1170,7 +831,7 @@ void DirectoryEntry::removeDirRecursive()
 

   m_FilesLookup.clear();

 

-  for (DirectoryEntry *entry : m_SubDirectories) {

+  for (DirectoryEntry* entry : m_SubDirectories) {

     entry->removeDirRecursive();

     delete entry;

   }

@@ -1179,10 +840,10 @@ void DirectoryEntry::removeDirRecursive()
   m_SubDirectoriesLookup.clear();

 }

 

-void DirectoryEntry::addDirectoryToList(DirectoryEntry* e)

+void DirectoryEntry::addDirectoryToList(DirectoryEntry* e, std::wstring nameLc)

 {

   m_SubDirectories.push_back(e);

-  m_SubDirectoriesLookup.emplace(ToLowerCopy(e->getName()), e);

+  m_SubDirectoriesLookup.emplace(std::move(nameLc), e);

 }

 

 void DirectoryEntry::removeDirectoryFromList(SubDirectories::iterator itor)

@@ -1204,7 +865,7 @@ void DirectoryEntry::removeDirectoryFromList(SubDirectories::iterator itor)
   m_SubDirectories.erase(itor);

 }

 

-void DirectoryEntry::removeFileFromList(FileEntry::Index index)

+void DirectoryEntry::removeFileFromList(FileIndex index)

 {

   auto removeFrom = [&](auto& list) {

     auto iter = std::find_if(

@@ -1234,7 +895,7 @@ void DirectoryEntry::removeFileFromList(FileEntry::Index index)
   removeFrom(m_Files);

 }

 

-void DirectoryEntry::removeFilesFromList(const std::set<FileEntry::Index>& indices)

+void DirectoryEntry::removeFilesFromList(const std::set<FileIndex>& indices)

 {

   for (auto iter = m_Files.begin(); iter != m_Files.end();) {

     if (indices.find(iter->second) != indices.end()) {

@@ -1253,12 +914,79 @@ void DirectoryEntry::removeFilesFromList(const std::set<FileEntry::Index>& indic
   }

 }

 

-void DirectoryEntry::addFileToList(

-  std::wstring fileNameLower, FileEntry::Index index)

+void DirectoryEntry::addFileToList(std::wstring fileNameLower, FileIndex index)

 {

   m_FilesLookup.emplace(fileNameLower, index);

   m_Files.emplace(std::move(fileNameLower), index);

   // fileNameLower has been moved from this point

 }

 

+struct DumpFailed : public std::runtime_error

+{

+  using runtime_error::runtime_error;

+};

+

+void DirectoryEntry::dump(const std::wstring& file) const

+{

+  try

+  {

+    std::FILE* f = nullptr;

+    auto e = _wfopen_s(&f, file.c_str(), L"wb");

+

+    if (e != 0 || !f) {

+      throw DumpFailed(fmt::format(

+        "failed to open, {} ({})", std::strerror(e), e));

+    }

+

+    Guard g([&]{ std::fclose(f); });

+

+    dump(f, L"Data");

+  }

+  catch(DumpFailed& e)

+  {

+    log::error(

+      "failed to write list to '{}': {}",

+      QString::fromStdWString(file).toStdString(), e.what());

+  }

+}

+

+void DirectoryEntry::dump(std::FILE* f, const std::wstring& parentPath) const

+{

+  {

+    std::scoped_lock lock(m_FilesMutex);

+

+    for (auto&& index : m_Files) {

+      const auto file = m_FileRegister->getFile(index.second);

+      if (!file) {

+        continue;

+      }

+

+      if (file->isFromArchive()) {

+        // TODO: don't list files from archives. maybe make this an option?

+        continue;

+      }

+

+      const auto& o = m_OriginConnection->getByID(file->getOrigin());

+      const auto path = parentPath + L"\\" + file->getName();

+      const auto line = path + L"\t(" + o.getName() + L")\r\n";

+

+      const auto lineu8 = MOShared::ToString(line, true);

+

+      if (std::fwrite(lineu8.data(), lineu8.size(), 1, f) != 1) {

+        const auto e = errno;

+        throw DumpFailed(fmt::format(

+          "failed to write, {} ({})", std::strerror(e), e));

+      }

+    }

+  }

+

+  {

+    std::scoped_lock lock(m_SubDirMutex);

+    for (auto&& d : m_SubDirectories) {

+      const auto path = parentPath + L"\\" + d->m_Name;

+      d->dump(f, path);

+    }

+  }

+}

+

 } // namespace MOShared

diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h
index e26011d7..a28ceeae 100644
--- a/src/shared/directoryentry.h
+++ b/src/shared/directoryentry.h
@@ -17,25 +17,18 @@ You should have received a copy of the GNU General Public License
 along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.

 */

 

-#ifndef DIRECTORYENTRY_H

-#define DIRECTORYENTRY_H

+#ifndef MO_REGISTER_DIRECTORYENTRY_INCLUDED

+#define MO_REGISTER_DIRECTORYENTRY_INCLUDED

 

-

-#include <string>

-#include <set>

-#include <vector>

-#include <map>

-#include <cassert>

-#define WIN32_MEAN_AND_LEAN

-#include <Windows.h>

+#include "fileregister.h"

 #include <bsatk.h>

-#ifndef Q_MOC_RUN

-#include <boost/shared_ptr.hpp>

-#include <boost/weak_ptr.hpp>

-#endif

-#include "util.h"

 

-namespace MOShared { struct DirectoryEntryFileKey; }

+namespace env

+{

+  class DirectoryWalker;

+  struct Directory;

+  struct File;

+}

 

 namespace std

 {

@@ -53,274 +46,23 @@ namespace std
 namespace MOShared

 {

 

-class DirectoryEntry;

-class OriginConnection;

-class FileRegister;

-

-

-class FileEntry

-{

-public:

-  static constexpr uint64_t NoFileSize =

-    std::numeric_limits<uint64_t>::max();

-

-  typedef unsigned int Index;

-  typedef boost::shared_ptr<FileEntry> Ptr;

-

-  // a vector of {originId, {archiveName, order}}

-  //

-  // if a file is in an archive, archiveName is the name of the bsa and order

-  // is the order of the associated plugin in the plugins list

-  //

-  // is a file is not in an archive, archiveName is empty and order is usually

-  // -1

-  typedef std::vector<std::pair<int, std::pair<std::wstring, int>>>

-    AlternativesVector;

-

-  FileEntry();

-  FileEntry(Index index, const std::wstring &name, DirectoryEntry *parent);

-

-  Index getIndex() const

-  {

-    return m_Index;

-  }

-

-  time_t lastAccessed() const

-  {

-    return m_LastAccessed;

-  }

-

-  void addOrigin(

-    int origin, FILETIME fileTime, const std::wstring &archive, int order);

-

-  // remove the specified origin from the list of origins that contain this

-  // file. if no origin is left, the file is effectively deleted and true is

-  // returned. otherwise, false is returned

-  bool removeOrigin(int origin);

-

-  void sortOrigins();

-

-  // gets the list of alternative origins (origins with lower priority than

-  // the primary one). if sortOrigins has been called, it is sorted by priority

-  // (ascending)

-  const AlternativesVector &getAlternatives() const

-  {

-    return m_Alternatives;

-  }

-

-  const std::wstring &getName() const

-  {

-    return m_Name;

-  }

-

-  int getOrigin() const

-  {

-    return m_Origin;

-  }

-

-  int getOrigin(bool &archive) const

-  {

-    archive = (m_Archive.first.length() != 0);

-    return m_Origin;

-  }

-

-  const std::pair<std::wstring, int> &getArchive() const

-  {

-    return m_Archive;

-  }

-

-  bool isFromArchive(std::wstring archiveName = L"") const;

-

-  // if originID is -1, uses the main origin; if this file doesn't exist in the

-  // given origin, returns an empty string

-  //

-  std::wstring getFullPath(int originID=-1) const;

-

-  std::wstring getRelativePath() const;

-

-  DirectoryEntry *getParent()

-  {

-    return m_Parent;

-  }

-

-  void setFileTime(FILETIME fileTime) const

-  {

-    m_FileTime = fileTime;

-  }

-

-  FILETIME getFileTime() const

-  {

-    return m_FileTime;

-  }

-

-  void setFileSize(uint64_t size, uint64_t compressedSize)

-  {

-    m_FileSize = size;

-    m_CompressedFileSize = compressedSize;

-  }

-

-  uint64_t getFileSize() const

-  {

-    return m_FileSize;

-  }

-

-  uint64_t getCompressedFileSize() const

-  {

-    return m_CompressedFileSize;

-  }

-

-private:

-  Index m_Index;

-  std::wstring m_Name;

-  int m_Origin;

-  std::pair<std::wstring, int> m_Archive;

-  AlternativesVector m_Alternatives;

-  DirectoryEntry *m_Parent;

-  mutable FILETIME m_FileTime;

-  uint64_t m_FileSize, m_CompressedFileSize;

-

-  time_t m_LastAccessed;

-

-  bool recurseParents(std::wstring &path, const DirectoryEntry *parent) const;

-};

-

-

-// represents a mod or the data directory, providing files to the tree

-class FilesOrigin

-{

-  friend class OriginConnection;

-

-public:

-  FilesOrigin();

-  FilesOrigin(const FilesOrigin &reference);

-

-  // sets priority for this origin, but it will overwrite the existing mapping

-  // for this priority, the previous origin will no longer be referenced

-  void setPriority(int priority);

-

-  int getPriority() const

-  {

-    return m_Priority;

-  }

-

-  void setName(const std::wstring &name);

-  const std::wstring &getName() const

-  {

-    return m_Name;

-  }

-

-  int getID() const

-  {

-    return m_ID;

-  }

-

-  const std::wstring &getPath() const

-  {

-    return m_Path;

-  }

-

-  std::vector<FileEntry::Ptr> getFiles() const;

-  FileEntry::Ptr findFile(FileEntry::Index index) const;

-

-  void enable(bool enabled, time_t notAfter = LONG_MAX);

-  bool isDisabled() const

-  {

-    return m_Disabled;

-  }

-

-  void addFile(FileEntry::Index index)

-  {

-    m_Files.insert(index);

-  }

-

-  void removeFile(FileEntry::Index index);

-

-  bool containsArchive(std::wstring archiveName);

-

-private:

-  int m_ID;

-  bool m_Disabled;

-  std::set<FileEntry::Index> m_Files;

-  std::wstring m_Name;

-  std::wstring m_Path;

-  int m_Priority;

-  boost::weak_ptr<FileRegister> m_FileRegister;

-  boost::weak_ptr<OriginConnection> m_OriginConnection;

-

-  FilesOrigin(

-    int ID, const std::wstring &name, const std::wstring &path, int priority,

-    boost::shared_ptr<FileRegister> fileRegister,

-    boost::shared_ptr<OriginConnection> originConnection);

-};

-

-

-class FileRegister

-{

-public:

-  FileRegister(boost::shared_ptr<OriginConnection> originConnection);

-

-  bool indexValid(FileEntry::Index index) const;

-

-  FileEntry::Ptr createFile(const std::wstring &name, DirectoryEntry *parent);

-  FileEntry::Ptr getFile(FileEntry::Index index) const;

-

-  size_t size() const

-  {

-    return m_Files.size();

-  }

-

-  bool removeFile(FileEntry::Index index);

-  void removeOrigin(FileEntry::Index index, int originID);

-  void removeOriginMulti(std::set<FileEntry::Index> indices, int originID, time_t notAfter);

-

-  void sortOrigins();

-

-private:

-  std::map<FileEntry::Index, FileEntry::Ptr> m_Files;

-  boost::shared_ptr<OriginConnection> m_OriginConnection;

-

-  FileEntry::Index generateIndex();

-  void unregisterFile(FileEntry::Ptr file);

-};

-

-

-struct DirectoryEntryFileKey

-{

-  DirectoryEntryFileKey(std::wstring v)

-    : value(std::move(v)), hash(getHash(value))

-  {

-  }

-

-  bool operator==(const DirectoryEntryFileKey& o) const

-  {

-    return (value == o.value);

-  }

-

-  static std::size_t getHash(const std::wstring& value)

-  {

-    return std::hash<std::wstring>()(value);

-  }

-

-  const std::wstring value;

-  const std::size_t hash;

-};

-

-

 class DirectoryEntry

 {

 public:

-  using FileKey = DirectoryEntryFileKey;

-

   DirectoryEntry(

-    const std::wstring &name, DirectoryEntry *parent, int originID);

+    std::wstring name, DirectoryEntry* parent, OriginID originID);

 

   DirectoryEntry(

-    const std::wstring &name, DirectoryEntry *parent, int originID,

+    std::wstring name, DirectoryEntry* parent, OriginID originID,

     boost::shared_ptr<FileRegister> fileRegister,

     boost::shared_ptr<OriginConnection> originConnection);

 

   ~DirectoryEntry();

 

+  // noncopyable

+  DirectoryEntry(const DirectoryEntry&) = delete;

+  DirectoryEntry& operator=(const DirectoryEntry&) = delete;

+

   void clear();

 

   bool isPopulated() const

@@ -343,7 +85,7 @@ public:
     return !m_Files.empty();

   }

 

-  const DirectoryEntry *getParent() const

+  const DirectoryEntry* getParent() const

   {

     return m_Parent;

   }

@@ -351,16 +93,32 @@ public:
   // add files to this directory (and subdirectories) from the specified origin.

   // That origin may exist or not

   void addFromOrigin(

-    const std::wstring &originName,

-    const std::wstring &directory, int priority);

+    const std::wstring& originName,

+    const std::wstring& directory, int priority, DirectoryStats& stats);

+

+  void addFromOrigin(

+    env::DirectoryWalker& walker, const std::wstring& originName,

+    const std::wstring& directory, int priority, DirectoryStats& stats);

+

+  void addFromAllBSAs(

+    const std::wstring& originName, const std::wstring& directory,

+    int priority, const std::vector<std::wstring>& archives,

+    const std::set<std::wstring>& enabledArchives,

+    const std::vector<std::wstring>& loadOrder,

+    DirectoryStats& stats);

 

   void addFromBSA(

-    const std::wstring &originName, std::wstring &directory,

-    const std::wstring &fileName, int priority, int order);

+    const std::wstring& originName, const std::wstring& directory,

+    const std::wstring& archivePath, int priority, int order,

+    DirectoryStats& stats);

+

+  void addFromList(

+    const std::wstring& originName, const std::wstring& directory,

+    env::Directory& root, int priority, DirectoryStats& stats);

 

-  void propagateOrigin(int origin);

+  void propagateOrigin(OriginID origin);

 

-  const std::wstring &getName() const

+  const std::wstring& getName() const

   {

     return m_Name;

   }

@@ -370,18 +128,18 @@ public:
     return m_FileRegister;

   }

 

-  bool originExists(const std::wstring &name) const;

-  FilesOrigin &getOriginByID(int ID) const;

-  FilesOrigin &getOriginByName(const std::wstring &name) const;

-  const FilesOrigin* findOriginByID(int ID) const;

+  bool originExists(const std::wstring& name) const;

+  FilesOrigin& getOriginByID(OriginID ID) const;

+  FilesOrigin& getOriginByName(const std::wstring& name) const;

+  const FilesOrigin* findOriginByID(OriginID ID) const;

 

-  int anyOrigin() const;

+  OriginID anyOrigin() const;

 

-  std::vector<FileEntry::Ptr> getFiles() const;

+  std::vector<FileEntryPtr> getFiles() const;

 

   void getSubDirectories(

-    std::vector<DirectoryEntry*>::const_iterator &begin,

-    std::vector<DirectoryEntry*>::const_iterator &end) const

+    std::vector<DirectoryEntry*>::const_iterator& begin,

+    std::vector<DirectoryEntry*>::const_iterator& end) const

   {

     begin = m_SubDirectories.begin();

     end = m_SubDirectories.end();

@@ -424,22 +182,22 @@ public:
     }

   }

 

-  FileEntry::Ptr getFileByIndex(FileEntry::Index index) const

+  FileEntryPtr getFileByIndex(FileIndex index) const

   {

     return m_FileRegister->getFile(index);

   }

 

-  DirectoryEntry *findSubDirectory(

-    const std::wstring &name, bool alreadyLowerCase=false) const;

+  DirectoryEntry* findSubDirectory(

+    const std::wstring& name, bool alreadyLowerCase=false) const;

 

-  DirectoryEntry *findSubDirectoryRecursive(const std::wstring &path);

+  DirectoryEntry* findSubDirectoryRecursive(const std::wstring& path);

 

   /** retrieve a file in this directory by name.

     * @param name name of the file

     * @return fileentry object for the file or nullptr if no file matches

     */

-  const FileEntry::Ptr findFile(const std::wstring &name, bool alreadyLowerCase=false) const;

-  const FileEntry::Ptr findFile(const FileKey& key) const;

+  const FileEntryPtr findFile(const std::wstring& name, bool alreadyLowerCase=false) const;

+  const FileEntryPtr findFile(const DirectoryEntryFileKey& key) const;

 

   bool hasFile(const std::wstring& name) const;

   bool containsArchive(std::wstring archiveName);

@@ -450,36 +208,36 @@ public:
   // if directory is not nullptr, the referenced variable will be set to the

   // path containing the file

   //

-  const FileEntry::Ptr searchFile(

-    const std::wstring &path, const DirectoryEntry **directory=nullptr) const;

+  const FileEntryPtr searchFile(

+    const std::wstring& path, const DirectoryEntry** directory=nullptr) const;

 

-  void insertFile(const std::wstring &filePath, FilesOrigin &origin, FILETIME fileTime);

-

-  void removeFile(FileEntry::Index index);

+  void removeFile(FileIndex index);

 

   // remove the specified file from the tree. This can be a path leading to a

   // file in a subdirectory

-  bool removeFile(const std::wstring &filePath, int *origin = nullptr);

+  bool removeFile(const std::wstring& filePath, OriginID* origin = nullptr);

 

   /**

    * @brief remove the specified directory

    * @param path directory to remove

    */

-  void removeDir(const std::wstring &path);

+  void removeDir(const std::wstring& path);

+

+  bool remove(const std::wstring& fileName, OriginID* origin);

 

-  bool remove(const std::wstring &fileName, int *origin);

+  bool hasContentsFromOrigin(OriginID originID) const;

 

-  bool hasContentsFromOrigin(int originID) const;

+  FilesOrigin& createOrigin(

+    const std::wstring& originName,

+    const std::wstring& directory, int priority, DirectoryStats& stats);

 

-  FilesOrigin &createOrigin(

-    const std::wstring &originName,

-    const std::wstring &directory, int priority);

+  void removeFiles(const std::set<FileIndex>& indices);

 

-  void removeFiles(const std::set<FileEntry::Index> &indices);

+  void dump(const std::wstring& file) const;

 

 private:

-  using FilesMap = std::map<std::wstring, FileEntry::Index>;

-  using FilesLookup = std::unordered_map<FileKey, FileEntry::Index>;

+  using FilesMap = std::map<std::wstring, FileIndex>;

+  using FilesLookup = std::unordered_map<DirectoryEntryFileKey, FileIndex>;

   using SubDirectories = std::vector<DirectoryEntry*>;

   using SubDirectoriesLookup = std::unordered_map<std::wstring, DirectoryEntry*>;

 

@@ -492,39 +250,60 @@ private:
   SubDirectories m_SubDirectories;

   SubDirectoriesLookup m_SubDirectoriesLookup;

 

-  DirectoryEntry *m_Parent;

-  std::set<int> m_Origins;

+  DirectoryEntry* m_Parent;

+  std::set<OriginID> m_Origins;

   bool m_Populated;

   bool m_TopLevel;

+  mutable std::mutex m_SubDirMutex;

+  mutable std::mutex m_FilesMutex;

+  mutable std::mutex m_OriginsMutex;

 

 

-  DirectoryEntry(const DirectoryEntry &reference);

+  FileEntryPtr insert(

+    std::wstring_view fileName, FilesOrigin& origin, FILETIME fileTime,

+    std::wstring_view archive, int order, DirectoryStats& stats);

 

-  FileEntry::Ptr insert(

-    const std::wstring &fileName, FilesOrigin &origin, FILETIME fileTime,

-    const std::wstring &archive, int order);

+  FileEntryPtr insert(

+    env::File& file, FilesOrigin& origin,

+    std::wstring_view archive, int order, DirectoryStats& stats);

 

   void addFiles(

-    FilesOrigin &origin, wchar_t *buffer, int bufferOffset);

+    env::DirectoryWalker& walker, FilesOrigin& origin,

+    const std::wstring& path, DirectoryStats& stats);

 

   void addFiles(

-    FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime,

-    const std::wstring &archiveName, int order);

+    FilesOrigin& origin, BSA::Folder::Ptr archiveFolder, FILETIME fileTime,

+    const std::wstring& archiveName, int order, DirectoryStats& stats);

 

-  DirectoryEntry *getSubDirectory(

-    const std::wstring &name, bool create, int originID = -1);

+  void addDir(FilesOrigin& origin, env::Directory& d, DirectoryStats& stats);

 

-  DirectoryEntry *getSubDirectoryRecursive(

-    const std::wstring &path, bool create, int originID = -1);

+  DirectoryEntry* getSubDirectory(

+    std::wstring_view name, bool create, DirectoryStats& stats,

+    OriginID originID = InvalidOriginID);

+

+  DirectoryEntry* getSubDirectory(

+    env::Directory& dir, bool create, DirectoryStats& stats,

+    OriginID originID = InvalidOriginID);

+

+  DirectoryEntry* getSubDirectoryRecursive(

+    const std::wstring& path, bool create, DirectoryStats& stats,

+    OriginID originID = InvalidOriginID);

 

   void removeDirRecursive();

 

-  void addDirectoryToList(DirectoryEntry* e);

+  void addDirectoryToList(DirectoryEntry* e, std::wstring nameLc);

   void removeDirectoryFromList(SubDirectories::iterator itor);

 

-  void addFileToList(std::wstring fileNameLower, FileEntry::Index index);

-  void removeFileFromList(FileEntry::Index index);

-  void removeFilesFromList(const std::set<FileEntry::Index>& indices);

+  void addFileToList(std::wstring fileNameLower, FileIndex index);

+  void removeFileFromList(FileIndex index);

+  void removeFilesFromList(const std::set<FileIndex>& indices);

+

+  struct Context;

+  static void onDirectoryStart(Context* cx, std::wstring_view path);

+  static void onDirectoryEnd(Context* cx, std::wstring_view path);

+  static void onFile(Context* cx, std::wstring_view path, FILETIME ft);

+

+  void dump(std::FILE* f, const std::wstring& parentPath) const;

 };

 

 } // namespace MOShared

@@ -540,4 +319,4 @@ namespace std
   }

 }

 

-#endif // DIRECTORYENTRY_H

+#endif // MO_REGISTER_DIRECTORYENTRY_INCLUDED

diff --git a/src/shared/fileentry.cpp b/src/shared/fileentry.cpp
new file mode 100644
index 00000000..e57c3cc5
--- /dev/null
+++ b/src/shared/fileentry.cpp
@@ -0,0 +1,251 @@
+#include "fileentry.h"
+#include "directoryentry.h"
+#include "filesorigin.h"
+
+namespace MOShared
+{
+
+FileEntry::FileEntry() :
+  m_Index(InvalidFileIndex), m_Name(), m_Origin(-1), m_Parent(nullptr),
+  m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize)
+{
+}
+
+FileEntry::FileEntry(FileIndex index, std::wstring name, DirectoryEntry *parent) :
+  m_Index(index), m_Name(std::move(name)), m_Origin(-1), m_Archive(L"", -1), m_Parent(parent),
+  m_FileSize(NoFileSize), m_CompressedFileSize(NoFileSize)
+{
+}
+
+void FileEntry::addOrigin(
+  OriginID origin, FILETIME fileTime, std::wstring_view archive, int order)
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  if (m_Parent != nullptr) {
+    m_Parent->propagateOrigin(origin);
+  }
+
+  if (m_Origin == -1) {
+    // If this file has no previous origin, this mod is now the origin with no
+    // alternatives
+    m_Origin = origin;
+    m_FileTime = fileTime;
+    m_Archive = std::pair<std::wstring, int>(std::wstring(archive.begin(), archive.end()), order);
+  }
+  else if (
+    (m_Parent != nullptr) && (
+    (m_Parent->getOriginByID(origin).getPriority() > m_Parent->getOriginByID(m_Origin).getPriority()) ||
+      (archive.size() == 0 && m_Archive.first.size() > 0 ))
+    ) {
+    // If this mod has a higher priority than the origin mod OR
+    // this mod has a loose file and the origin mod has an archived file,
+    // this mod is now the origin and the previous origin is the first alternative
+
+    auto itor = std::find_if(
+      m_Alternatives.begin(), m_Alternatives.end(),
+      [&](auto&& i) { return i.first == m_Origin; });
+
+    if (itor == m_Alternatives.end()) {
+      m_Alternatives.push_back({m_Origin, m_Archive});
+    }
+
+    m_Origin = origin;
+    m_FileTime = fileTime;
+    m_Archive = std::pair<std::wstring, int>(std::wstring(archive.begin(), archive.end()), order);
+  }
+  else {
+    // This mod is just an alternative
+    bool found = false;
+
+    if (m_Origin == origin) {
+      // already an origin
+      return;
+    }
+
+    for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) {
+      if (iter->first == origin) {
+        // already an origin
+        return;
+      }
+
+      if ((m_Parent != nullptr) &&
+        (m_Parent->getOriginByID(iter->first).getPriority() < m_Parent->getOriginByID(origin).getPriority())) {
+        m_Alternatives.insert(iter, {origin, {std::wstring(archive.begin(), archive.end()), order}});
+        found = true;
+        break;
+      }
+    }
+
+    if (!found) {
+      m_Alternatives.push_back({origin, {std::wstring(archive.begin(), archive.end()), order}});
+    }
+  }
+}
+
+bool FileEntry::removeOrigin(OriginID origin)
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  if (m_Origin == origin) {
+    if (!m_Alternatives.empty()) {
+      // find alternative with the highest priority
+      auto currentIter = m_Alternatives.begin();
+      for (auto iter = m_Alternatives.begin(); iter != m_Alternatives.end(); ++iter) {
+        if (iter->first != origin) {
+          //Both files are not from archives.
+          if (!iter->second.first.size() && !currentIter->second.first.size()) {
+            if ((m_Parent->getOriginByID(iter->first).getPriority() > m_Parent->getOriginByID(currentIter->first).getPriority())) {
+              currentIter = iter;
+            }
+          }
+          else {
+            //Both files are from archives
+            if (iter->second.first.size() && currentIter->second.first.size()) {
+              if (iter->second.second > currentIter->second.second) {
+                currentIter = iter;
+              }
+            }
+            else {
+              //Only one of the two is an archive, so we change currentIter only if he is the archive one.
+              if (currentIter->second.first.size()) {
+                currentIter = iter;
+              }
+            }
+          }
+        }
+      }
+
+      OriginID currentID = currentIter->first;
+      m_Archive = currentIter->second;
+      m_Alternatives.erase(currentIter);
+
+      m_Origin = currentID;
+    } else {
+      m_Origin = -1;
+      m_Archive = std::pair<std::wstring, int>(L"", -1);
+      return true;
+    }
+  } else {
+    auto newEnd = std::remove_if(
+      m_Alternatives.begin(), m_Alternatives.end(),
+      [&](auto &i) { return i.first == origin; });
+
+    if (newEnd != m_Alternatives.end()) {
+      m_Alternatives.erase(newEnd, m_Alternatives.end());
+    }
+  }
+  return false;
+}
+
+void FileEntry::sortOrigins()
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  m_Alternatives.push_back({m_Origin, m_Archive});
+
+  std::sort(m_Alternatives.begin(), m_Alternatives.end(), [&](auto&& LHS, auto&& RHS) {
+    if (!LHS.second.first.size() && !RHS.second.first.size()) {
+      int l = m_Parent->getOriginByID(LHS.first).getPriority();
+      if (l < 0) {
+        l = INT_MAX;
+      }
+
+      int r = m_Parent->getOriginByID(RHS.first).getPriority();
+      if (r < 0) {
+        r = INT_MAX;
+      }
+
+      return l < r;
+    }
+
+    if (LHS.second.first.size() && RHS.second.first.size()) {
+      int l = LHS.second.second; if (l < 0) l = INT_MAX;
+      int r = RHS.second.second; if (r < 0) r = INT_MAX;
+
+      return l < r;
+    }
+
+    if (RHS.second.first.size()) {
+      return false;
+    }
+
+    return true;
+    });
+
+  if (!m_Alternatives.empty()) {
+    m_Origin = m_Alternatives.back().first;
+    m_Archive = m_Alternatives.back().second;
+    m_Alternatives.pop_back();
+  }
+}
+
+bool FileEntry::isFromArchive(std::wstring archiveName) const
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  if (archiveName.length() == 0) {
+    return m_Archive.first.length() != 0;
+  }
+
+  if (m_Archive.first.compare(archiveName) == 0) {
+    return true;
+  }
+
+  for (auto alternative : m_Alternatives) {
+    if (alternative.second.first.compare(archiveName) == 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+std::wstring FileEntry::getFullPath(OriginID originID) const
+{
+  std::scoped_lock lock(m_OriginsMutex);
+
+  if (originID == InvalidOriginID) {
+    bool ignore = false;
+    originID = getOrigin(ignore);
+  }
+
+  // base directory for origin
+  const auto* o = m_Parent->findOriginByID(originID);
+  if (!o) {
+    return {};
+  }
+
+  std::wstring result = o->getPath();
+
+  // all intermediate directories
+  recurseParents(result, m_Parent);
+
+  return result + L"\\" + m_Name;
+}
+
+std::wstring FileEntry::getRelativePath() const
+{
+  std::wstring result;
+
+  // all intermediate directories
+  recurseParents(result, m_Parent);
+
+  return result + L"\\" + m_Name;
+}
+
+bool FileEntry::recurseParents(std::wstring &path, const DirectoryEntry *parent) const
+{
+  if (parent == nullptr) {
+    return false;
+  } else {
+    // don't append the topmost parent because it is the virtual data-root
+    if (recurseParents(path, parent->getParent())) {
+      path.append(L"\\").append(parent->getName());
+    }
+
+    return true;
+  }
+}
+
+} // namespace
diff --git a/src/shared/fileentry.h b/src/shared/fileentry.h
new file mode 100644
index 00000000..aceeec57
--- /dev/null
+++ b/src/shared/fileentry.h
@@ -0,0 +1,122 @@
+#ifndef MO_REGISTER_FILEENTRY_INCLUDED
+#define MO_REGISTER_FILEENTRY_INCLUDED
+
+#include "fileregisterfwd.h"
+
+namespace MOShared
+{
+
+class FileEntry
+{
+public:
+  static constexpr uint64_t NoFileSize =
+    std::numeric_limits<uint64_t>::max();
+
+  FileEntry();
+  FileEntry(FileIndex index, std::wstring name, DirectoryEntry *parent);
+
+  // noncopyable
+  FileEntry(const FileEntry&) = delete;
+  FileEntry& operator=(const FileEntry&) = delete;
+
+  FileIndex getIndex() const
+  {
+    return m_Index;
+  }
+
+  void addOrigin(
+    OriginID origin, FILETIME fileTime, std::wstring_view archive, int order);
+
+  // remove the specified origin from the list of origins that contain this
+  // file. if no origin is left, the file is effectively deleted and true is
+  // returned. otherwise, false is returned
+  bool removeOrigin(OriginID origin);
+
+  void sortOrigins();
+
+  // gets the list of alternative origins (origins with lower priority than
+  // the primary one). if sortOrigins has been called, it is sorted by priority
+  // (ascending)
+  const AlternativesVector &getAlternatives() const
+  {
+    return m_Alternatives;
+  }
+
+  const std::wstring &getName() const
+  {
+    return m_Name;
+  }
+
+  OriginID getOrigin() const
+  {
+    return m_Origin;
+  }
+
+  OriginID getOrigin(bool &archive) const
+  {
+    archive = (m_Archive.first.length() != 0);
+    return m_Origin;
+  }
+
+  const std::pair<std::wstring, int> &getArchive() const
+  {
+    return m_Archive;
+  }
+
+  bool isFromArchive(std::wstring archiveName = L"") const;
+
+  // if originID is -1, uses the main origin; if this file doesn't exist in the
+  // given origin, returns an empty string
+  //
+  std::wstring getFullPath(OriginID originID=InvalidOriginID) const;
+
+  std::wstring getRelativePath() const;
+
+  DirectoryEntry *getParent()
+  {
+    return m_Parent;
+  }
+
+  void setFileTime(FILETIME fileTime) const
+  {
+    m_FileTime = fileTime;
+  }
+
+  FILETIME getFileTime() const
+  {
+    return m_FileTime;
+  }
+
+  void setFileSize(uint64_t size, uint64_t compressedSize)
+  {
+    m_FileSize = size;
+    m_CompressedFileSize = compressedSize;
+  }
+
+  uint64_t getFileSize() const
+  {
+    return m_FileSize;
+  }
+
+  uint64_t getCompressedFileSize() const
+  {
+    return m_CompressedFileSize;
+  }
+
+private:
+  FileIndex m_Index;
+  std::wstring m_Name;
+  OriginID m_Origin;
+  std::pair<std::wstring, int> m_Archive;
+  AlternativesVector m_Alternatives;
+  DirectoryEntry *m_Parent;
+  mutable FILETIME m_FileTime;
+  uint64_t m_FileSize, m_CompressedFileSize;
+  mutable std::mutex m_OriginsMutex;
+
+  bool recurseParents(std::wstring &path, const DirectoryEntry *parent) const;
+};
+
+} // namespace
+
+#endif // MO_REGISTER_FILEENTRY_INCLUDED
diff --git a/src/shared/fileregister.cpp b/src/shared/fileregister.cpp
new file mode 100644
index 00000000..b56599d1
--- /dev/null
+++ b/src/shared/fileregister.cpp
@@ -0,0 +1,184 @@
+#include "fileregister.h"
+#include "fileentry.h"
+#include "directoryentry.h"
+#include "originconnection.h"
+#include "filesorigin.h"
+#include <log.h>
+
+namespace MOShared
+{
+
+using namespace MOBase;
+
+FileRegister::FileRegister(boost::shared_ptr<OriginConnection> originConnection)
+  : m_OriginConnection(originConnection), m_NextIndex(0)
+{
+}
+
+bool FileRegister::indexValid(FileIndex index) const
+{
+  std::scoped_lock lock(m_Mutex);
+
+  if (index < m_Files.size()) {
+    return (m_Files[index].get() != nullptr);
+  }
+
+  return false;
+}
+
+FileEntryPtr FileRegister::createFile(
+  std::wstring name, DirectoryEntry *parent, DirectoryStats& stats)
+{
+  const auto index = generateIndex();
+  auto p = FileEntryPtr(new FileEntry(index, std::move(name), parent));
+
+  {
+    std::scoped_lock lock(m_Mutex);
+
+    if (index >= m_Files.size()) {
+      m_Files.resize(index + 1);
+    }
+
+    m_Files[index] = p;
+  }
+
+  return p;
+}
+
+FileIndex FileRegister::generateIndex()
+{
+  return m_NextIndex++;
+}
+
+FileEntryPtr FileRegister::getFile(FileIndex index) const
+{
+  std::scoped_lock lock(m_Mutex);
+
+  if (index < m_Files.size()) {
+    return m_Files[index];
+  } else {
+    return {};
+  }
+}
+
+bool FileRegister::removeFile(FileIndex index)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  if (index < m_Files.size()) {
+    FileEntryPtr p;
+    m_Files[index].swap(p);
+
+    if (p) {
+      unregisterFile(p);
+      return true;
+    }
+  }
+
+  log::error(QObject::tr("invalid file index for remove: {}").toStdString(), index);
+  return false;
+}
+
+void FileRegister::removeOrigin(FileIndex index, OriginID originID)
+{
+  std::unique_lock lock(m_Mutex);
+
+  if (index < m_Files.size()) {
+    FileEntryPtr& p = m_Files[index];
+
+    if (p) {
+      if (p->removeOrigin(originID)) {
+        m_Files[index] = {};
+        lock.unlock();
+        unregisterFile(p);
+        return;
+      }
+    }
+  }
+
+  log::error(QObject::tr("invalid file index for remove (for origin): {}").toStdString(), index);
+}
+
+void FileRegister::removeOriginMulti(
+  std::set<FileIndex> indices, OriginID originID)
+{
+  std::vector<FileEntryPtr> removedFiles;
+
+  {
+    std::scoped_lock lock(m_Mutex);
+
+    for (auto iter = indices.begin(); iter != indices.end(); ) {
+      const auto index = *iter;
+
+      if (index < m_Files.size()) {
+        const auto& p = m_Files[index];
+
+        if (p && p->removeOrigin(originID)) {
+          removedFiles.push_back(p);
+          m_Files[index] = {};
+          ++iter;
+          continue;
+        }
+      }
+
+      iter = indices.erase(iter);
+    }
+  }
+
+  // optimization: this is only called when disabling an origin and in this case
+  // we don't have to remove the file from the origin
+
+  // need to remove files from their parent directories. multiple ways to go
+  // about this:
+  //   a) for each file, search its parents file-list (preferably by name) and
+  //      remove what is found
+  //   b) gather the parent directories, go through the file list for each once
+  //      and remove all files that have been removed
+  //
+  // the latter should be faster when there are many files in few directories.
+  // since this is called only when disabling an origin that is probably
+  // frequently the case
+
+  std::set<DirectoryEntry*> parents;
+  for (const FileEntryPtr &file : removedFiles) {
+    if (file->getParent() != nullptr) {
+      parents.insert(file->getParent());
+    }
+  }
+
+  for (DirectoryEntry *parent : parents) {
+    parent->removeFiles(indices);
+  }
+}
+
+void FileRegister::sortOrigins()
+{
+  std::scoped_lock lock(m_Mutex);
+
+  for (auto&& p : m_Files) {
+    if (p) {
+      p->sortOrigins();
+    }
+  }
+}
+
+void FileRegister::unregisterFile(FileEntryPtr file)
+{
+  bool ignore;
+
+  // unregister from origin
+  OriginID originID = file->getOrigin(ignore);
+  m_OriginConnection->getByID(originID).removeFile(file->getIndex());
+  const auto& alternatives = file->getAlternatives();
+
+  for (auto iter = alternatives.begin(); iter != alternatives.end(); ++iter) {
+    m_OriginConnection->getByID(iter->first).removeFile(file->getIndex());
+  }
+
+  // unregister from directory
+  if (file->getParent() != nullptr) {
+    file->getParent()->removeFile(file->getIndex());
+  }
+}
+
+} // namespace
diff --git a/src/shared/fileregister.h b/src/shared/fileregister.h
new file mode 100644
index 00000000..c27dc93b
--- /dev/null
+++ b/src/shared/fileregister.h
@@ -0,0 +1,53 @@
+#ifndef MO_REGISTER_FILESREGISTER_INCLUDED
+#define MO_REGISTER_FILESREGISTER_INCLUDED
+
+#include "fileregisterfwd.h"
+#include <mutex>
+#include <boost/shared_ptr.hpp>
+
+namespace MOShared
+{
+
+class FileRegister
+{
+public:
+  FileRegister(boost::shared_ptr<OriginConnection> originConnection);
+
+  // noncopyable
+  FileRegister(const FileRegister&) = delete;
+  FileRegister& operator=(const FileRegister&) = delete;
+
+  bool indexValid(FileIndex index) const;
+
+  FileEntryPtr createFile(
+    std::wstring name, DirectoryEntry *parent, DirectoryStats& stats);
+
+  FileEntryPtr getFile(FileIndex index) const;
+
+  size_t highestCount() const
+  {
+    std::scoped_lock lock(m_Mutex);
+    return m_Files.size();
+  }
+
+  bool removeFile(FileIndex index);
+  void removeOrigin(FileIndex index, OriginID originID);
+  void removeOriginMulti(std::set<FileIndex> indices, OriginID originID);
+
+  void sortOrigins();
+
+private:
+  using FileMap = std::deque<FileEntryPtr>;
+
+  mutable std::mutex m_Mutex;
+  FileMap m_Files;
+  boost::shared_ptr<OriginConnection> m_OriginConnection;
+  std::atomic<FileIndex> m_NextIndex;
+
+  void unregisterFile(FileEntryPtr file);
+  FileIndex generateIndex();
+};
+
+} // namespace
+
+#endif // MO_REGISTER_FILESREGISTER_INCLUDED
diff --git a/src/shared/fileregisterfwd.h b/src/shared/fileregisterfwd.h
new file mode 100644
index 00000000..720e6e30
--- /dev/null
+++ b/src/shared/fileregisterfwd.h
@@ -0,0 +1,96 @@
+#ifndef MO_REGISTER_FILEREGISTERFWD_INCLUDED
+#define MO_REGISTER_FILEREGISTERFWD_INCLUDED
+
+class DirectoryRefreshProgress;
+
+namespace MOShared
+{
+
+struct DirectoryEntryFileKey
+{
+  DirectoryEntryFileKey(std::wstring v)
+    : value(std::move(v)), hash(getHash(value))
+  {
+  }
+
+  bool operator==(const DirectoryEntryFileKey& o) const
+  {
+    return (value == o.value);
+  }
+
+  static std::size_t getHash(const std::wstring& value)
+  {
+    return std::hash<std::wstring>()(value);
+  }
+
+  std::wstring value;
+  const std::size_t hash;
+};
+
+
+class DirectoryEntry;
+class OriginConnection;
+class FileRegister;
+class FilesOrigin;
+class FileEntry;
+struct DirectoryStats;
+
+using FileEntryPtr = boost::shared_ptr<FileEntry>;
+using FileIndex = unsigned int;
+using OriginID = int;
+
+constexpr FileIndex InvalidFileIndex = UINT_MAX;
+constexpr OriginID InvalidOriginID = -1;
+
+
+// a vector of {originId, {archiveName, order}}
+//
+// if a file is in an archive, archiveName is the name of the bsa and order
+// is the order of the associated plugin in the plugins list
+//
+// is a file is not in an archive, archiveName is empty and order is usually
+// -1
+using AlternativesVector = std::vector<std::pair<OriginID, std::pair<std::wstring, int>>>;
+
+struct DirectoryStats
+{
+  static constexpr bool EnableInstrumentation = false;
+
+  std::string mod;
+
+  std::chrono::nanoseconds dirTimes;
+  std::chrono::nanoseconds fileTimes;
+  std::chrono::nanoseconds sortTimes;
+
+  std::chrono::nanoseconds subdirLookupTimes;
+  std::chrono::nanoseconds addDirectoryTimes;
+
+  std::chrono::nanoseconds filesLookupTimes;
+  std::chrono::nanoseconds addFileTimes;
+  std::chrono::nanoseconds addOriginToFileTimes;
+  std::chrono::nanoseconds addFileToOriginTimes;
+  std::chrono::nanoseconds addFileToRegisterTimes;
+
+  int64_t originExists;
+  int64_t originCreate;
+  int64_t originsNeededEnabled;
+
+  int64_t subdirExists;
+  int64_t subdirCreate;
+
+  int64_t fileExists;
+  int64_t fileCreate;
+  int64_t filesInsertedInRegister;
+  int64_t filesAssignedInRegister;
+
+  DirectoryStats();
+
+  DirectoryStats& operator+=(const DirectoryStats& o);
+
+  static std::string csvHeader();
+  std::string toCsv() const;
+};
+
+} // namespace
+
+#endif // MO_REGISTER_FILEREGISTERFWD_INCLUDED
diff --git a/src/shared/filesorigin.cpp b/src/shared/filesorigin.cpp
new file mode 100644
index 00000000..7476b6c5
--- /dev/null
+++ b/src/shared/filesorigin.cpp
@@ -0,0 +1,126 @@
+#include "filesorigin.h"
+#include "originconnection.h"
+#include "fileregister.h"
+#include "fileentry.h"
+
+namespace MOShared
+{
+
+std::wstring tail(const std::wstring &source, const size_t count)
+{
+  if (count >= source.length()) {
+    return source;
+  }
+
+  return source.substr(source.length() - count);
+}
+
+
+FilesOrigin::FilesOrigin()
+  : m_ID(0), m_Disabled(false), m_Name(), m_Path(), m_Priority(0)
+{
+}
+
+FilesOrigin::FilesOrigin(
+  OriginID ID, const std::wstring &name, const std::wstring &path, int priority,
+  boost::shared_ptr<MOShared::FileRegister> fileRegister,
+  boost::shared_ptr<MOShared::OriginConnection> originConnection) :
+  m_ID(ID), m_Disabled(false), m_Name(name), m_Path(path),
+  m_Priority(priority), m_FileRegister(fileRegister),
+  m_OriginConnection(originConnection)
+{
+}
+
+void FilesOrigin::setPriority(int priority)
+{
+  m_OriginConnection.lock()->changePriorityLookup(m_Priority, priority);
+
+  m_Priority = priority;
+}
+
+void FilesOrigin::setName(const std::wstring &name)
+{
+  m_OriginConnection.lock()->changeNameLookup(m_Name, name);
+
+  // change path too
+  if (tail(m_Path, m_Name.length()) == m_Name) {
+    m_Path = m_Path.substr(0, m_Path.length() - m_Name.length()).append(name);
+  }
+
+  m_Name = name;
+}
+
+std::vector<FileEntryPtr> FilesOrigin::getFiles() const
+{
+  std::vector<FileEntryPtr> result;
+
+  {
+    std::scoped_lock lock(m_Mutex);
+
+    for (FileIndex fileIdx : m_Files) {
+      if (FileEntryPtr p = m_FileRegister.lock()->getFile(fileIdx)) {
+        result.push_back(p);
+      }
+    }
+  }
+
+  return result;
+}
+
+FileEntryPtr FilesOrigin::findFile(FileIndex index) const
+{
+  return m_FileRegister.lock()->getFile(index);
+}
+
+void FilesOrigin::enable(bool enabled)
+{
+  DirectoryStats dummy;
+  enable(enabled, dummy);
+}
+
+void FilesOrigin::enable(bool enabled, DirectoryStats& stats)
+{
+  if (!enabled) {
+    ++stats.originsNeededEnabled;
+
+    std::set<FileIndex> copy;
+
+    {
+      std::scoped_lock lock(m_Mutex);
+      copy = m_Files;
+      m_Files.clear();
+    }
+
+    m_FileRegister.lock()->removeOriginMulti(copy, m_ID);
+  }
+
+  m_Disabled = !enabled;
+}
+
+void FilesOrigin::removeFile(FileIndex index)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto iter = m_Files.find(index);
+
+  if (iter != m_Files.end()) {
+    m_Files.erase(iter);
+  }
+}
+
+bool FilesOrigin::containsArchive(std::wstring archiveName)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  for (FileIndex fileIdx : m_Files) {
+    if (FileEntryPtr p = m_FileRegister.lock()->getFile(fileIdx)) {
+      if (p->isFromArchive(archiveName)) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+} //  namespace
diff --git a/src/shared/filesorigin.h b/src/shared/filesorigin.h
new file mode 100644
index 00000000..222a834f
--- /dev/null
+++ b/src/shared/filesorigin.h
@@ -0,0 +1,85 @@
+#ifndef MO_REGISTER_FILESORIGIN_INCLUDED
+#define MO_REGISTER_FILESORIGIN_INCLUDED
+
+#include "fileregisterfwd.h"
+
+namespace MOShared
+{
+
+// represents a mod or the data directory, providing files to the tree
+class FilesOrigin
+{
+public:
+  FilesOrigin();
+
+  FilesOrigin(
+    OriginID ID, const std::wstring &name, const std::wstring &path,
+    int priority,
+    boost::shared_ptr<FileRegister> fileRegister,
+    boost::shared_ptr<OriginConnection> originConnection);
+
+  // noncopyable
+  FilesOrigin(const FilesOrigin&) = delete;
+  FilesOrigin& operator=(const FilesOrigin&) = delete;
+
+  // sets priority for this origin, but it will overwrite the existing mapping
+  // for this priority, the previous origin will no longer be referenced
+  void setPriority(int priority);
+
+  int getPriority() const
+  {
+    return m_Priority;
+  }
+
+  void setName(const std::wstring &name);
+  const std::wstring &getName() const
+  {
+    return m_Name;
+  }
+
+  OriginID getID() const
+  {
+    return m_ID;
+  }
+
+  const std::wstring &getPath() const
+  {
+    return m_Path;
+  }
+
+  std::vector<FileEntryPtr> getFiles() const;
+  FileEntryPtr findFile(FileIndex index) const;
+
+  void enable(bool enabled, DirectoryStats& stats);
+  void enable(bool enabled);
+
+  bool isDisabled() const
+  {
+    return m_Disabled;
+  }
+
+  void addFile(FileIndex index)
+  {
+    std::scoped_lock lock(m_Mutex);
+    m_Files.insert(index);
+  }
+
+  void removeFile(FileIndex index);
+
+  bool containsArchive(std::wstring archiveName);
+
+private:
+  OriginID m_ID;
+  bool m_Disabled;
+  std::set<FileIndex> m_Files;
+  std::wstring m_Name;
+  std::wstring m_Path;
+  int m_Priority;
+  boost::weak_ptr<FileRegister> m_FileRegister;
+  boost::weak_ptr<OriginConnection> m_OriginConnection;
+  mutable std::mutex m_Mutex;
+};
+
+} // namespace
+
+#endif // MO_REGISTER_FILESORIGIN_INCLUDED
diff --git a/src/shared/originconnection.cpp b/src/shared/originconnection.cpp
new file mode 100644
index 00000000..c1c096f0
--- /dev/null
+++ b/src/shared/originconnection.cpp
@@ -0,0 +1,145 @@
+#include "originconnection.h"
+#include "filesorigin.h"
+#include "util.h"
+#include <log.h>
+
+namespace MOShared
+{
+
+using namespace MOBase;
+
+OriginConnection::OriginConnection()
+  : m_NextID(0)
+{
+}
+
+std::pair<FilesOrigin&, bool> OriginConnection::getOrCreate(
+  const std::wstring &originName, const std::wstring &directory, int priority,
+  const boost::shared_ptr<FileRegister>& fileRegister,
+  const boost::shared_ptr<OriginConnection>& originConnection,
+  DirectoryStats& stats)
+{
+  std::unique_lock lock(m_Mutex);
+
+  auto itor = m_OriginsNameMap.find(originName);
+
+  if (itor == m_OriginsNameMap.end()) {
+    FilesOrigin& origin = createOriginNoLock(
+      originName, directory, priority, fileRegister, originConnection);
+
+    return {origin, true};
+  } else {
+    FilesOrigin& origin = m_Origins[itor->second];
+    lock.unlock();
+
+    origin.enable(true, stats);
+    return {origin, false};
+  }
+}
+
+FilesOrigin& OriginConnection::createOrigin(
+  const std::wstring &originName, const std::wstring &directory, int priority,
+  boost::shared_ptr<FileRegister> fileRegister,
+  boost::shared_ptr<OriginConnection> originConnection)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  return createOriginNoLock(
+    originName, directory, priority, fileRegister, originConnection);
+}
+
+bool OriginConnection::exists(const std::wstring &name)
+{
+  std::scoped_lock lock(m_Mutex);
+  return m_OriginsNameMap.find(name) != m_OriginsNameMap.end();
+}
+
+FilesOrigin& OriginConnection::getByID(OriginID ID)
+{
+  std::scoped_lock lock(m_Mutex);
+  return m_Origins[ID];
+}
+
+const FilesOrigin* OriginConnection::findByID(OriginID ID) const
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto itor = m_Origins.find(ID);
+
+  if (itor == m_Origins.end()) {
+    return nullptr;
+  } else {
+    return &itor->second;
+  }
+}
+
+FilesOrigin& OriginConnection::getByName(const std::wstring &name)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto iter = m_OriginsNameMap.find(name);
+
+  if (iter != m_OriginsNameMap.end()) {
+    return m_Origins[iter->second];
+  } else {
+    std::ostringstream stream;
+    stream << QObject::tr("invalid origin name: ").toStdString() << ToString(name, true);
+    throw std::runtime_error(stream.str());
+  }
+}
+
+void OriginConnection::changePriorityLookup(int oldPriority, int newPriority)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto iter = m_OriginsPriorityMap.find(oldPriority);
+
+  if (iter != m_OriginsPriorityMap.end()) {
+    OriginID idx = iter->second;
+    m_OriginsPriorityMap.erase(iter);
+    m_OriginsPriorityMap[newPriority] = idx;
+  }
+}
+
+void OriginConnection::changeNameLookup(const std::wstring &oldName, const std::wstring &newName)
+{
+  std::scoped_lock lock(m_Mutex);
+
+  auto iter = m_OriginsNameMap.find(oldName);
+
+  if (iter != m_OriginsNameMap.end()) {
+    OriginID idx = iter->second;
+    m_OriginsNameMap.erase(iter);
+    m_OriginsNameMap[newName] = idx;
+  } else {
+    log::error(QObject::tr("failed to change name lookup from {} to {}").toStdString(), oldName, newName);
+  }
+}
+
+OriginID OriginConnection::createID()
+{
+  return m_NextID++;
+}
+
+FilesOrigin& OriginConnection::createOriginNoLock(
+  const std::wstring &originName, const std::wstring &directory, int priority,
+  boost::shared_ptr<FileRegister> fileRegister,
+  boost::shared_ptr<OriginConnection> originConnection)
+{
+  OriginID newID = createID();
+
+  auto itor = m_Origins.emplace(
+    std::piecewise_construct,
+    std::forward_as_tuple(newID),
+    std::forward_as_tuple(
+      newID, originName, directory, priority,
+      fileRegister, originConnection))
+    .first;
+
+  m_OriginsNameMap.insert({originName, newID});
+  m_OriginsPriorityMap.insert({priority, newID});
+
+  return itor->second;
+}
+
+} // namespace
diff --git a/src/shared/originconnection.h b/src/shared/originconnection.h
new file mode 100644
index 00000000..5fecd63a
--- /dev/null
+++ b/src/shared/originconnection.h
@@ -0,0 +1,56 @@
+#ifndef MO_REGISTER_ORIGINCONNECTION_INCLUDED
+#define MO_REGISTER_ORIGINCONNECTION_INCLUDED
+
+#include "fileregisterfwd.h"
+
+namespace MOShared
+{
+
+class OriginConnection
+{
+public:
+  OriginConnection();
+
+  // noncopyable
+  OriginConnection(const OriginConnection&) = delete;
+  OriginConnection& operator=(const OriginConnection&) = delete;
+
+  std::pair<FilesOrigin&, bool> getOrCreate(
+    const std::wstring &originName, const std::wstring &directory, int priority,
+    const boost::shared_ptr<FileRegister>& fileRegister,
+    const boost::shared_ptr<OriginConnection>& originConnection,
+    DirectoryStats& stats);
+
+  FilesOrigin& createOrigin(
+    const std::wstring &originName, const std::wstring &directory, int priority,
+    boost::shared_ptr<FileRegister> fileRegister,
+    boost::shared_ptr<OriginConnection> originConnection);
+
+  bool exists(const std::wstring &name);
+
+  FilesOrigin &getByID(OriginID ID);
+  const FilesOrigin* findByID(OriginID ID) const;
+  FilesOrigin &getByName(const std::wstring &name);
+
+  void changePriorityLookup(int oldPriority, int newPriority);
+
+  void changeNameLookup(const std::wstring &oldName, const std::wstring &newName);
+
+private:
+  std::atomic<OriginID> m_NextID;
+  std::map<OriginID, FilesOrigin> m_Origins;
+  std::map<std::wstring, OriginID> m_OriginsNameMap;
+  std::map<int, OriginID> m_OriginsPriorityMap;
+  mutable std::mutex m_Mutex;
+
+  OriginID createID();
+
+  FilesOrigin& createOriginNoLock(
+    const std::wstring &originName, const std::wstring &directory, int priority,
+    boost::shared_ptr<FileRegister> fileRegister,
+    boost::shared_ptr<OriginConnection> originConnection);
+};
+
+} // namespace
+
+#endif // MO_REGISTER_ORIGINCONNECTION_INCLUDED
diff --git a/src/shared/util.cpp b/src/shared/util.cpp
index aa4ad4b3..483b36a9 100644
--- a/src/shared/util.cpp
+++ b/src/shared/util.cpp
@@ -130,6 +130,13 @@ std::wstring ToLowerCopy(const std::wstring& text)
   return result;

 }

 

+std::wstring ToLowerCopy(std::wstring_view text)

+{

+  std::wstring result(text.begin(), text.end());

+  ToLowerInPlace(result);

+  return result;

+}

+

 bool CaseInsenstiveComparePred(wchar_t lhs, wchar_t rhs)

 {

   return std::tolower(lhs, loc) == std::tolower(rhs, loc);

@@ -375,26 +382,6 @@ void checkDuplicateShortcuts(const QMenu& m)
 } // namespace MOShared

 

 

-TimeThis::TimeThis(QString what)

-  : m_what(std::move(what)), m_start(Clock::now())

-{

-}

-

-TimeThis::~TimeThis()

-{

-  using namespace std::chrono;

-

-  const auto end = Clock::now();

-  const auto d = duration_cast<milliseconds>(end - m_start).count();

-

-  if (m_what.isEmpty()) {

-    log::debug("{} ms", d);

-  } else {

-    log::debug("{} {} ms", m_what, d);

-  }

-}

-

-

 static bool g_exiting = false;

 static bool g_canClose = false;

 

diff --git a/src/shared/util.h b/src/shared/util.h
index 05522c2d..2761b64f 100644
--- a/src/shared/util.h
+++ b/src/shared/util.h
@@ -20,7 +20,9 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #ifndef UTIL_H

 #define UTIL_H

 

+#include <log.h>

 #include <string>

+#include <filesystem>

 #include <versioninfo.h>

 

 class Executable;

@@ -41,6 +43,7 @@ std::string ToLowerCopy(const std::string& text);
 

 std::wstring& ToLowerInPlace(std::wstring& text);

 std::wstring ToLowerCopy(const std::wstring& text);

+std::wstring ToLowerCopy(std::wstring_view text);

 

 bool CaseInsensitiveEqual(const std::wstring &lhs, const std::wstring &rhs);

 

@@ -50,21 +53,16 @@ QString getUsvfsVersionString();
 void SetThisThreadName(const QString& s);

 void checkDuplicateShortcuts(const QMenu& m);

 

-} // namespace MOShared

-

-

-class TimeThis

+inline FILETIME ToFILETIME(std::filesystem::file_time_type t)

 {

-public:

-  TimeThis(QString what={});

-  ~TimeThis();

+  FILETIME ft;

+  static_assert(sizeof(t) == sizeof(ft));

 

-private:

-  using Clock = std::chrono::high_resolution_clock;

+  std::memcpy(&ft, &t, sizeof(FILETIME));

+  return ft;

+}

 

-  QString m_what;

-  Clock::time_point m_start;

-};

+} // namespace MOShared

 

 

 enum class Exit