Merge pull request #1003 from isanae/file-list-improvements

Refresh optimizations

1 files changed, 81 insertions, 48 deletions

diff --git a/src/organizercore.cpp b/src/organizercore.cpp
index 04d78ca8..80df044a 100644
--- a/src/organizercore.cpp
+++ b/src/organizercore.cpp
@@ -22,7 +22,6 @@
 #include <ipluginmodpage.h>
 #include <dataarchives.h>
 #include <localsavegames.h>
-#include <directoryentry.h>
 #include <scopeguard.h>
 #include <utility.h>
 #include <usvfs.h>
@@ -34,6 +33,11 @@
 #include "previewdialog.h"
 #include "env.h"
 #include "envmodule.h"
+#include "envfs.h"
+#include "directoryrefresher.h"
+#include "shared/directoryentry.h"
+#include "shared/filesorigin.h"
+#include "shared/fileentry.h"
 
 #include <QApplication>
 #include <QCoreApplication>
@@ -88,24 +92,19 @@ QStringList toStringList(InputIterator current, InputIterator end)
 OrganizerCore::OrganizerCore(Settings &settings)
   : m_UserInterface(nullptr)
   , m_PluginContainer(nullptr)
-  , m_GameName()
   , m_CurrentProfile(nullptr)
   , m_Settings(settings)
   , m_Updater(NexusInterface::instance(m_PluginContainer))
-  , m_AboutToRun()
-  , m_FinishedRun()
-  , m_ModInstalled()
   , m_ModList(m_PluginContainer, this)
   , m_PluginList(this)
-  , m_DirectoryRefresher()
+  , m_DirectoryRefresher(new DirectoryRefresher(settings.refreshThreadCount()))
   , m_DirectoryStructure(new DirectoryEntry(L"data", nullptr, 0))
   , m_DownloadManager(NexusInterface::instance(m_PluginContainer), this)
-  , m_InstallationManager()
-  , m_RefresherThread()
   , m_DirectoryUpdate(false)
   , m_ArchivesInit(false)
   , m_PluginListsWriter(std::bind(&OrganizerCore::savePluginList, this))
 {
+  env::setHandleCloserThreadCount(settings.refreshThreadCount());
   m_DownloadManager.setOutputDirectory(m_Settings.paths().downloads(), false);
 
   NexusInterface::instance(m_PluginContainer)->setCacheDirectory(
@@ -116,7 +115,7 @@ OrganizerCore::OrganizerCore(Settings &settings)
 
   connect(&m_DownloadManager, SIGNAL(downloadSpeed(QString, int)), this,
           SLOT(downloadSpeed(QString, int)));
-  connect(&m_DirectoryRefresher, SIGNAL(refreshed()), this,
+  connect(m_DirectoryRefresher.get(), SIGNAL(refreshed()), this,
           SLOT(directory_refreshed()));
 
   connect(&m_ModList, SIGNAL(removeOrigin(QString)), this,
@@ -141,7 +140,7 @@ OrganizerCore::OrganizerCore(Settings &settings)
 
   // make directory refresher run in a separate thread
   m_RefresherThread.start();
-  m_DirectoryRefresher.moveToThread(&m_RefresherThread);
+  m_DirectoryRefresher->moveToThread(&m_RefresherThread);
 }
 
 OrganizerCore::~OrganizerCore()
@@ -149,6 +148,10 @@ OrganizerCore::~OrganizerCore()
   m_RefresherThread.exit();
   m_RefresherThread.wait();
 
+  if (m_StructureDeleter.joinable()) {
+    m_StructureDeleter.join();
+  }
+
   saveCurrentProfile();
 
   // profile has to be cleaned up before the modinfo-buffer is cleared
@@ -234,8 +237,6 @@ void OrganizerCore::setUserInterface(IUserInterface* ui)
             SLOT(removeMod_clicked()));
     connect(&m_ModList, SIGNAL(clearOverwrite()), w,
       SLOT(clearOverwrite()));
-    connect(&m_ModList, SIGNAL(requestColumnSelect(QPoint)), w,
-            SLOT(displayColumnSelection(QPoint)));
     connect(&m_ModList, SIGNAL(fileMoved(QString, QString, QString)), w,
             SLOT(fileMoved(QString, QString, QString)));
     connect(&m_ModList, SIGNAL(modorder_changed()), w,
@@ -813,7 +814,7 @@ QString OrganizerCore::resolvePath(const QString &fileName) const
   if (m_DirectoryStructure == nullptr) {
     return QString();
   }
-  const FileEntry::Ptr file
+  const FileEntryPtr file
       = m_DirectoryStructure->searchFile(ToWString(fileName), nullptr);
   if (file.get() != nullptr) {
     return ToQString(file->getFullPath());
@@ -847,8 +848,8 @@ QStringList OrganizerCore::findFiles(
   if (!path.isEmpty())
     dir = dir->findSubDirectoryRecursive(ToWString(path));
   if (dir != nullptr) {
-    std::vector<FileEntry::Ptr> files = dir->getFiles();
-    foreach (FileEntry::Ptr file, files) {
+    std::vector<FileEntryPtr> files = dir->getFiles();
+    foreach (FileEntryPtr file, files) {
       if (filter(ToQString(file->getFullPath()))) {
         result.append(ToQString(file->getFullPath()));
       }
@@ -860,7 +861,7 @@ QStringList OrganizerCore::findFiles(
 QStringList OrganizerCore::getFileOrigins(const QString &fileName) const
 {
   QStringList result;
-  const FileEntry::Ptr file = m_DirectoryStructure->searchFile(ToWString(fileName), nullptr);
+  const FileEntryPtr file = m_DirectoryStructure->searchFile(ToWString(fileName), nullptr);
 
   if (file.get() != nullptr) {
     result.append(ToQString(
@@ -883,8 +884,8 @@ QList<MOBase::IOrganizer::FileInfo> OrganizerCore::findFileInfos(
   if (!path.isEmpty())
     dir = dir->findSubDirectoryRecursive(ToWString(path));
   if (dir != nullptr) {
-    std::vector<FileEntry::Ptr> files = dir->getFiles();
-    foreach (FileEntry::Ptr file, files) {
+    std::vector<FileEntryPtr> files = dir->getFiles();
+    foreach (FileEntryPtr file, files) {
       IOrganizer::FileInfo info;
       info.filePath    = ToQString(file->getFullPath());
       bool fromArchive = false;
@@ -966,7 +967,7 @@ bool OrganizerCore::previewFileWithAlternatives(
 
 
 
-  const FileEntry::Ptr file = directoryStructure()->searchFile(ToWString(fileName), nullptr);
+  const FileEntryPtr file = directoryStructure()->searchFile(ToWString(fileName), nullptr);
 
   if (file.get() == nullptr) {
     reportError(tr("file not found: %1").arg(qUtf8Printable(fileName)));
@@ -1180,7 +1181,7 @@ void OrganizerCore::updateModsActiveState(const QList<unsigned int> &modIndices,
     QDir dir(modInfo->absolutePath());
     for (const QString &esm :
       dir.entryList(QStringList() << "*.esm", QDir::Files)) {
-      const FileEntry::Ptr file = m_DirectoryStructure->findFile(ToWString(esm));
+      const FileEntryPtr file = m_DirectoryStructure->findFile(ToWString(esm));
       if (file.get() == nullptr) {
         log::warn("failed to activate {}", esm);
         continue;
@@ -1196,7 +1197,7 @@ void OrganizerCore::updateModsActiveState(const QList<unsigned int> &modIndices,
 
     for (const QString &esl :
       dir.entryList(QStringList() << "*.esl", QDir::Files)) {
-      const FileEntry::Ptr file = m_DirectoryStructure->findFile(ToWString(esl));
+      const FileEntryPtr file = m_DirectoryStructure->findFile(ToWString(esl));
       if (file.get() == nullptr) {
         log::warn("failed to activate {}", esl);
         continue;
@@ -1212,7 +1213,7 @@ void OrganizerCore::updateModsActiveState(const QList<unsigned int> &modIndices,
     }
     QStringList esps = dir.entryList(QStringList() << "*.esp", QDir::Files);
     for (const QString &esp : esps) {
-      const FileEntry::Ptr file = m_DirectoryStructure->findFile(ToWString(esp));
+      const FileEntryPtr file = m_DirectoryStructure->findFile(ToWString(esp));
       if (file.get() == nullptr) {
         log::warn("failed to activate {}", esp);
         continue;
@@ -1247,13 +1248,17 @@ void OrganizerCore::updateModInDirectoryStructure(unsigned int index,
 
 void OrganizerCore::updateModsInDirectoryStructure(QMap<unsigned int, ModInfo::Ptr> modInfo)
 {
+  std::vector<DirectoryRefresher::EntryInfo> entries;
+
   for (auto idx : modInfo.keys()) {
-    // add files of the bsa to the directory structure
-    m_DirectoryRefresher.addModFilesToStructure(
-      m_DirectoryStructure, modInfo[idx]->name(),
-      m_CurrentProfile->getModPriority(idx), modInfo[idx]->absolutePath(),
-      modInfo[idx]->stealFiles());
+    entries.push_back({
+      modInfo[idx]->name(), modInfo[idx]->absolutePath(),
+      modInfo[idx]->stealFiles(), {}, m_CurrentProfile->getModPriority(idx)});
   }
+
+  m_DirectoryRefresher->addMultipleModsFilesToStructure(
+    m_DirectoryStructure, entries);
+
   DirectoryRefresher::cleanStructure(m_DirectoryStructure);
   // need to refresh plugin list now so we can activate esps
   refreshESPList(true);
@@ -1270,13 +1275,13 @@ void OrganizerCore::updateModsInDirectoryStructure(QMap<unsigned int, ModInfo::P
   }
 
   std::vector<QString> archives = enabledArchives();
-  m_DirectoryRefresher.setMods(
+  m_DirectoryRefresher->setMods(
     m_CurrentProfile->getActiveMods(),
     std::set<QString>(archives.begin(), archives.end()));
 
   // finally also add files from bsas to the directory structure
   for (auto idx : modInfo.keys()) {
-    m_DirectoryRefresher.addModBSAToStructure(
+    m_DirectoryRefresher->addModBSAToStructure(
       m_DirectoryStructure, modInfo[idx]->name(),
       m_CurrentProfile->getModPriority(idx), modInfo[idx]->absolutePath(),
       modInfo[idx]->archives());
@@ -1382,46 +1387,74 @@ std::vector<QString> OrganizerCore::enabledArchives()
 
 void OrganizerCore::refreshDirectoryStructure()
 {
-  if (!m_DirectoryUpdate) {
-    m_CurrentProfile->writeModlistNow(true);
+  if (m_DirectoryUpdate) {
+    log::debug("can't refresh, already in progress");
+    return;
+  }
 
-    m_DirectoryUpdate = true;
-    std::vector<std::tuple<QString, QString, int>> activeModList
-        = m_CurrentProfile->getActiveMods();
-    auto archives = enabledArchives();
-    m_DirectoryRefresher.setMods(
-        activeModList, std::set<QString>(archives.begin(), archives.end()));
+  log::debug("refreshing structure");
+  m_DirectoryUpdate = true;
 
-    QTimer::singleShot(0, &m_DirectoryRefresher, SLOT(refresh()));
-  }
+  m_CurrentProfile->writeModlistNow(true);
+  const auto activeModList = m_CurrentProfile->getActiveMods();
+  const auto archives = enabledArchives();
+
+  m_DirectoryRefresher->setMods(
+      activeModList, std::set<QString>(archives.begin(), archives.end()));
+
+  // runs refresh() in a thread
+  QTimer::singleShot(0, m_DirectoryRefresher.get(), SLOT(refresh()));
 }
 
 void OrganizerCore::directory_refreshed()
 {
-  DirectoryEntry *newStructure = m_DirectoryRefresher.getDirectoryStructure();
+  log::debug("structure refreshed");
+
+  DirectoryEntry *newStructure = m_DirectoryRefresher->stealDirectoryStructure();
   Q_ASSERT(newStructure != m_DirectoryStructure);
-  if (newStructure != nullptr) {
-    std::swap(m_DirectoryStructure, newStructure);
-    delete newStructure;
-  } else {
+
+  if (newStructure == nullptr) {
     // TODO: don't know why this happens, this slot seems to get called twice
     // with only one emit
     return;
   }
+
+  std::swap(m_DirectoryStructure, newStructure);
+
+  if (m_StructureDeleter.joinable()) {
+    m_StructureDeleter.join();
+  }
+
+  m_StructureDeleter = std::thread([=]{
+    log::debug("structure deleter thread start");
+    delete newStructure;
+    log::debug("structure deleter thread done");
+  });
+
   m_DirectoryUpdate = false;
 
+  log::debug("clearing caches");
   for (int i = 0; i < m_ModList.rowCount(); ++i) {
     ModInfo::Ptr modInfo = ModInfo::getByIndex(i);
     modInfo->clearCaches();
   }
-  for (auto task : m_PostRefreshTasks) {
-    task();
+
+  if (!m_PostRefreshTasks.empty()) {
+    log::debug("running {} post refresh tasks", m_PostRefreshTasks.size());
+
+    for (auto task : m_PostRefreshTasks) {
+      task();
+    }
+
+    m_PostRefreshTasks.clear();
   }
-  m_PostRefreshTasks.clear();
 
   if (m_CurrentProfile != nullptr) {
+    log::debug("refreshing lists");
     refreshLists();
   }
+
+  log::debug("refresh done");
 }
 
 void OrganizerCore::profileRefresh()
@@ -1867,7 +1900,7 @@ std::vector<Mapping> OrganizerCore::fileMapping(
 {
   std::vector<Mapping> result;
 
-  for (FileEntry::Ptr current : directoryEntry->getFiles()) {
+  for (FileEntryPtr current : directoryEntry->getFiles()) {
     bool isArchive = false;
     int origin = current->getOrigin(isArchive);
     if (isArchive || (origin == 0)) {