optimizations for download manager:

 - use envfs for walking directory
 - minimize string copies, disk access
 - use set of filenames to check for duplicates
 - use file size from envfs

5 files changed, 90 insertions, 37 deletions

diff --git a/src/downloadmanager.cpp b/src/downloadmanager.cpp
index 762e3cdb..c912464d 100644
--- a/src/downloadmanager.cpp
+++ b/src/downloadmanager.cpp
@@ -24,11 +24,13 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "nexusinterface.h"
 #include "nxmaccessmanager.h"
 #include "iplugingame.h"
+#include "envfs.h"
 #include <nxmurl.h>
 #include <taskprogressmanager.h>
 #include "utility.h"
 #include "selectiondialog.h"
 #include "bbcode.h"
+#include "shared/util.h"
 #include <utility.h>
 #include <report.h>
 
@@ -77,7 +79,9 @@ DownloadManager::DownloadInfo *DownloadManager::DownloadInfo::createNew(const Mo
   return info;
 }
 
-DownloadManager::DownloadInfo *DownloadManager::DownloadInfo::createFromMeta(const QString &filePath, bool showHidden, const QString outputDirectory)
+DownloadManager::DownloadInfo *DownloadManager::DownloadInfo::createFromMeta(
+  const QString &filePath, bool showHidden, const QString outputDirectory,
+  std::optional<uint64_t> fileSize)
 {
   DownloadInfo *info = new DownloadInfo;
 
@@ -113,7 +117,7 @@ DownloadManager::DownloadInfo *DownloadManager::DownloadInfo::createFromMeta(con
 
   info->m_DownloadID = s_NextDownloadID++;
   info->m_Output.setFileName(filePath);
-  info->m_TotalSize = QFileInfo(filePath).size();
+  info->m_TotalSize = fileSize ? *fileSize : QFileInfo(filePath).size();
   info->m_PreResumeSize = info->m_TotalSize;
   info->m_CurrentUrl = 0;
   info->m_Urls = metaFile.value("url", "").toString().split(";");
@@ -325,13 +329,15 @@ void DownloadManager::refreshList()
       }
     }
 
-    QStringList supportedExtensions = m_OrganizerCore->installationManager()->getSupportedExtensions();
-    QStringList nameFilters(supportedExtensions);
+    const QStringList supportedExtensions = m_OrganizerCore->installationManager()->getSupportedExtensions();
+    std::vector<std::wstring> nameFilters;
     for (const auto& extension : supportedExtensions) {
-      nameFilters.append("*." + extension);
+      nameFilters.push_back(L"." + extension.toLower().toStdWString());
     }
 
-    nameFilters.append(QString("*").append(UNFINISHED));
+    nameFilters.push_back(QString(UNFINISHED).toLower().toStdWString());
+
+
     QDir dir(QDir::fromNativeSeparators(m_OutputDirectory));
 
     // find orphaned meta files and delete them (sounds cruel but it's better for everyone)
@@ -348,28 +354,65 @@ void DownloadManager::refreshList()
       shellDelete(orphans, true);
     }
 
-    // add existing downloads to list
-    foreach (QString file, dir.entryList(nameFilters, QDir::Files, QDir::Time)) {
-      bool Exists = false;
-      for (QVector<DownloadInfo*>::const_iterator Iter = m_ActiveDownloads.begin(); Iter != m_ActiveDownloads.end() && !Exists; ++Iter) {
-        if (QString::compare((*Iter)->m_FileName, file, Qt::CaseInsensitive) == 0) {
-          Exists = true;
-        } else if (QString::compare(QFileInfo((*Iter)->m_Output.fileName()).fileName(), file, Qt::CaseInsensitive) == 0) {
-          Exists = true;
-        }
-      }
-      if (Exists) {
-        continue;
-      }
 
-      QString fileName = QDir::fromNativeSeparators(m_OutputDirectory) + "/" + file;
+    std::set<std::wstring> seen;
 
-      DownloadInfo *info = DownloadInfo::createFromMeta(fileName, m_ShowHidden, m_OutputDirectory);
-      if (info != nullptr) {
-        m_ActiveDownloads.push_front(info);
-      }
+    struct Context
+    {
+      DownloadManager& self;
+      std::set<std::wstring>& seen;
+      std::vector<std::wstring>& extensions;
+    };
+
+    Context cx = {*this, seen, nameFilters};
+
+    for (auto&& d : m_ActiveDownloads) {
+      cx.seen.insert(d->m_FileName.toLower().toStdWString());
+      cx.seen.insert(QFileInfo(d->m_Output.fileName()).fileName().toLower().toStdWString());
     }
 
+
+    env::forEachEntry(
+      QDir::toNativeSeparators(m_OutputDirectory).toStdWString(), &cx, nullptr, nullptr,
+      [](void* data, std::wstring_view f, FILETIME, uint64_t size) {
+        auto& cx = *static_cast<Context*>(data);
+
+        std::wstring lc = MOShared::ToLowerCopy(f);
+
+        bool interestingExt = false;
+        for (auto&& ext : cx.extensions) {
+          if (lc.ends_with(ext)) {
+            interestingExt = true;
+            break;
+          }
+        }
+
+        if (!interestingExt) {
+          return;
+        }
+
+        if (cx.seen.contains(lc)) {
+          return;
+        }
+
+        QString fileName =
+          QDir::fromNativeSeparators(cx.self.m_OutputDirectory) + "/" +
+          QString::fromWCharArray(f.data(), f.size());
+
+        DownloadInfo *info = DownloadInfo::createFromMeta(
+          fileName, cx.self.m_ShowHidden, cx.self.m_OutputDirectory, size);
+
+        if (info == nullptr) {
+          return;
+        }
+
+        cx.self.m_ActiveDownloads.push_front(info);
+        cx.seen.insert(std::move(lc));
+        cx.seen.insert(QFileInfo(info->m_Output.fileName()).fileName().toLower().toStdWString());
+    });
+
+    log::debug("saw {} downloads", m_ActiveDownloads.size());
+
     emit update(-1);
 
     //let watcher trigger refreshes again
diff --git a/src/downloadmanager.h b/src/downloadmanager.h
index 953ab88b..5e126f4b 100644
--- a/src/downloadmanager.h
+++ b/src/downloadmanager.h
@@ -104,7 +104,9 @@ private:
     bool m_Hidden;
 
     static DownloadInfo *createNew(const MOBase::ModRepositoryFileInfo *fileInfo, const QStringList &URLs);
-    static DownloadInfo *createFromMeta(const QString &filePath, bool showHidden, const QString outputDirectory);
+    static DownloadInfo *createFromMeta(
+      const QString &filePath, bool showHidden, const QString outputDirectory,
+      std::optional<uint64_t> fileSize={});
 
     /**
      * @brief rename the file
diff --git a/src/envfs.cpp b/src/envfs.cpp
index 20ba4229..9317eb70 100644
--- a/src/envfs.cpp
+++ b/src/envfs.cpp
@@ -298,14 +298,17 @@ void forEachEntryImpl(
         ObjectName.MaximumLength = ObjectName.Length;
 
         if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
-          dirStartF(cx, toStringView(&oa));
-          forEachEntryImpl(cx, hc, buffers, &oa, depth+1, dirStartF, dirEndF, fileF);
-          dirEndF(cx, toStringView(&oa));
+          if (dirStartF && dirEndF) {
+            dirStartF(cx, toStringView(&oa));
+            forEachEntryImpl(cx, hc, buffers, &oa, depth+1, dirStartF, dirEndF, fileF);
+            dirEndF(cx, toStringView(&oa));
+          }
         } else {
           FILETIME ft;
           ft.dwLowDateTime = DirInfo->LastWriteTime.LowPart;
           ft.dwHighDateTime = DirInfo->LastWriteTime.HighPart;
-          fileF(cx, toStringView(&oa), ft);
+
+          fileF(cx, toStringView(&oa), ft, DirInfo->AllocationSize.QuadPart);
         }
       }
 
@@ -380,20 +383,20 @@ Directory getFilesAndDirs(const std::wstring& path)
       cx->current.pop();
     },
 
-    [](void* pcx, std::wstring_view path, FILETIME ft) {
+    [](void* pcx, std::wstring_view path, FILETIME ft, uint64_t s) {
       Context* cx = (Context*)pcx;
 
-      cx->current.top()->files.push_back(File(path, ft));
+      cx->current.top()->files.push_back(File(path, ft, s));
     }
   );
 
   return root;
 }
 
-File::File(std::wstring_view n, FILETIME ft) :
+File::File(std::wstring_view n, FILETIME ft, uint64_t s) :
   name(n.begin(), n.end()),
   lcname(MOShared::ToLowerCopy(name)),
-  lastModified(ft)
+  lastModified(ft), size(s)
 {
 }
 
@@ -429,7 +432,11 @@ void getFilesAndDirsWithFindImpl(const std::wstring& path, Directory& d)
           getFilesAndDirsWithFindImpl(newPath, d.dirs.back());
         }
       } else {
-        d.files.push_back(File(findData.cFileName, findData.ftLastWriteTime));
+        const auto size =
+          (findData.nFileSizeHigh * (MAXDWORD+1)) + findData.nFileSizeLow;
+
+        d.files.push_back(File(
+          findData.cFileName, findData.ftLastWriteTime, size));
       }
 
       result = ::FindNextFileW(searchHandle, &findData);
diff --git a/src/envfs.h b/src/envfs.h
index e4d98d71..62540e18 100644
--- a/src/envfs.h
+++ b/src/envfs.h
@@ -12,8 +12,9 @@ struct File
   std::wstring name;
   std::wstring lcname;
   FILETIME lastModified;
+  uint64_t size;
 
-  File(std::wstring_view name, FILETIME ft);
+  File(std::wstring_view name, FILETIME ft, uint64_t size);
 };
 
 struct Directory
@@ -174,7 +175,7 @@ private:
 
 using DirStartF = void (void*, std::wstring_view);
 using DirEndF = void (void*, std::wstring_view);
-using FileF = void (void*, std::wstring_view, FILETIME);
+using FileF = void (void*, std::wstring_view, FILETIME, uint64_t);
 
 void setHandleCloserThreadCount(std::size_t n);
 
diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp
index c4b467d6..0d275844 100644
--- a/src/shared/directoryentry.cpp
+++ b/src/shared/directoryentry.cpp
@@ -636,7 +636,7 @@ void DirectoryEntry::addFiles(
       onDirectoryEnd((Context*)pcx, path);

     },

 

-    [](void* pcx, std::wstring_view path, FILETIME ft)

+    [](void* pcx, std::wstring_view path, FILETIME ft, uint64_t)

     {

       onFile((Context*)pcx, path, ft);

     }