- performance optimizations for detection of file changes in mod directories

- added a workaround for download-link handling in chrome
- MO will now create a profile even if the game-launcher has never been run
- bugfix: files weren't cleanly removed from vfs file register

5 files changed, 100 insertions, 36 deletions

diff --git a/src/shared/directoryentry.cpp b/src/shared/directoryentry.cpp
index aead0cca..e0914870 100644
--- a/src/shared/directoryentry.cpp
+++ b/src/shared/directoryentry.cpp
@@ -21,6 +21,7 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #define WIN32_LEAN_AND_MEAN

 #include <Windows.h>

 #include <sstream>

+#include <ctime>

 #include <algorithm>

 #include <bsatk.h>

 #include "error_report.h"

@@ -28,6 +29,7 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "windows_error.h"

 #include <boost/bind.hpp>

 #include <boost/scoped_array.hpp>

+#include <boost/foreach.hpp>

 #include "util.h"

 #include "leaktrace.h"

 

@@ -133,13 +135,11 @@ private:
 //

 

 

-void FilesOrigin::enable(bool enabled)

+void FilesOrigin::enable(bool enabled, time_t notAfter)

 {

   if (!enabled) {

     std::set<FileEntry::Index> copy = m_Files;

-    for (auto iter = copy.begin(); iter != copy.end(); ++iter) {

-      m_FileRegister.lock()->removeOrigin(*iter, m_ID);

-    }

+    m_FileRegister.lock()->removeOriginMulti(copy, m_ID, notAfter);

     m_Files.clear();

   }

   m_Disabled = !enabled;

@@ -233,6 +233,7 @@ std::vector<FileEntry::Ptr> FilesOrigin::getFiles() const
 

 void FileEntry::addOrigin(int origin, FILETIME fileTime, const std::wstring &archive)

 {

+  m_LastAccessed = time(NULL);

   if (m_Parent != NULL) {

     m_Parent->propagateOrigin(origin);

   }

@@ -324,13 +325,13 @@ static bool ByOriginPriority(DirectoryEntry *entry, int LHS, int RHS)
 

 

 FileEntry::FileEntry()

-  : m_Index(UINT_MAX), m_Name(), m_Parent(NULL)

+  : m_Index(UINT_MAX), m_Name(), m_Parent(NULL), m_LastAccessed(time(NULL))

 {

   LEAK_TRACE;

 }

 

 FileEntry::FileEntry(Index index, const std::wstring &name, DirectoryEntry *parent)

-  : m_Index(index), m_Name(name), m_Parent(parent), m_Origin(-1), m_Archive(L"")

+  : m_Index(index), m_Name(name), m_Parent(parent), m_Origin(-1), m_Archive(L""), m_LastAccessed(time(NULL))

 {

   LEAK_TRACE;

 }

@@ -654,6 +655,18 @@ void DirectoryEntry::removeFile(FileEntry::Index index)
 }

 

 

+void DirectoryEntry::removeFiles(const std::set<FileEntry::Index> &indices)

+{

+  for (auto iter = m_Files.begin(); iter != m_Files.end();) {

+    if (indices.find(iter->second) != indices.end()) {

+      m_Files.erase(iter++);

+    } else {

+      ++iter;

+    }

+  }

+}

+

+

 int DirectoryEntry::anyOrigin() const

 {

   bool ignore;

@@ -737,7 +750,7 @@ const FileEntry::Ptr DirectoryEntry::searchFile(const std::wstring &path, const
 

   if (len == std::string::npos) {

     // no more path components

-    auto iter = m_Files.find(path);

+    auto iter = m_Files.find(ToLower(path));

     if (iter != m_Files.end()) {

       return m_FileRegister->getFile(iter->second);

     } else if (directory != NULL) {

@@ -777,7 +790,7 @@ DirectoryEntry *DirectoryEntry::findSubDirectoryRecursive(const std::wstring &pa
 

 const FileEntry::Ptr DirectoryEntry::findFile(const std::wstring &name) const

 {

-  auto iter = m_Files.find(name);

+  auto iter = m_Files.find(ToLower(name));

   if (iter != m_Files.end()) {

     return m_FileRegister->getFile(iter->second);

   } else {

@@ -865,7 +878,6 @@ FileEntry::Ptr FileRegister::getFile(FileEntry::Index index) const
   return FileEntry::Ptr();

 }

 

-

 void FileRegister::unregisterFile(FileEntry::Ptr file)

 {

   bool ignore;

@@ -890,6 +902,8 @@ void FileRegister::removeFile(FileEntry::Index index)
   if (iter != m_Files.end()) {

     unregisterFile(iter->second);

     m_Files.erase(index);

+  } else {

+    log("invalid file index for remove: %lu", index);

   }

 }

 

@@ -899,7 +913,49 @@ void FileRegister::removeOrigin(FileEntry::Index index, int originID)
   if (iter != m_Files.end()) {

     if (iter->second->removeOrigin(originID)) {

       unregisterFile(iter->second);

+      m_Files.erase(iter);

     }

+  } else {

+    log("invalid file index for remove (for origin): %lu", index);

+  }

+}

+

+void FileRegister::removeOriginMulti(std::set<FileEntry::Index> indices, int originID, time_t notAfter)

+{

+  std::vector<FileEntry::Ptr> removedFiles;

+  for (auto iter = indices.begin(); iter != indices.end();) {

+    auto pos = m_Files.find(*iter);

+    if (pos != m_Files.end()

+        && (pos->second->lastAccessed() < notAfter)

+        && pos->second->removeOrigin(originID)) {

+      removedFiles.push_back(pos->second);

+      m_Files.erase(pos);

+      ++iter;

+    } else {

+      indices.erase(iter++);

+    }

+  }

+

+  if (removedFiles.size() > 0) {

+    log("%d files actually removed", removedFiles.size());

+  }

+

+  // optimization: this is only called when disabling an origin and in this case we don't have

+  // to remove the file from the origin

+

+  // need to remove files from their parent directories. multiple ways to go about this:

+  // a) for each file, search its parents file-list (preferably by name) and remove what is found

+  // b) gather the parent directories, go through the file list for each once and remove all files that have been removed

+  // the latter should be faster when there are many files in few directories. since this is called

+  // only when disabling an origin that is probably frequently the case

+  std::set<DirectoryEntry*> parents;

+  BOOST_FOREACH (const FileEntry::Ptr &file, removedFiles) {

+    if (file->getParent() != NULL) {

+      parents.insert(file->getParent());

+    }

+  }

+  BOOST_FOREACH (DirectoryEntry *parent, parents) {

+    parent->removeFiles(indices);

   }

 }

 

diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h
index 7b656b05..1e904311 100644
--- a/src/shared/directoryentry.h
+++ b/src/shared/directoryentry.h
@@ -34,6 +34,11 @@ along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.
 #include "util.h"

 

 

+

+#include "error_report.h"

+

+

+

 namespace MOShared {

 

 

@@ -60,6 +65,8 @@ public:
 

   Index getIndex() const { return m_Index; }

 

+  time_t lastAccessed() const { return m_LastAccessed; }

+

   void addOrigin(int origin, FILETIME fileTime, const std::wstring &archive);

   // remove the specified origin from the list of origins that contain this file. if no origin is left,

   // the file is effectively deleted and true is returned. otherwise, false is returned

@@ -98,6 +105,8 @@ private:
   DirectoryEntry *m_Parent;

   mutable FILETIME m_FileTime;

 

+  time_t m_LastAccessed;

+

   friend bool operator<(const FileEntry &lhs, const FileEntry &rhs) {

     return _wcsicmp(lhs.m_Name.c_str(), rhs.m_Name.c_str()) < 0;

   }

@@ -130,7 +139,7 @@ public:
 

   std::vector<FileEntry::Ptr> getFiles() const;

 

-  void enable(bool enabled);

+  void enable(bool enabled, time_t notAfter = LONG_MAX);

   bool isDisabled() const { return m_Disabled; }

 

   void addFile(FileEntry::Index index) { m_Files.insert(index); }

@@ -171,8 +180,11 @@ public:
   FileEntry::Ptr createFile(const std::wstring &name, DirectoryEntry *parent);

   FileEntry::Ptr getFile(FileEntry::Index index) const;

 

+  size_t size() const { return m_Files.size(); }

+

   void removeFile(FileEntry::Index index);

   void removeOrigin(FileEntry::Index index, int originID);

+  void removeOriginMulti(std::set<FileEntry::Index> indices, int originID, time_t notAfter);

 

   void sortOrigins();

 

@@ -258,7 +270,7 @@ public:
   void removeDir(const std::wstring &path);

 

   void remove(const std::wstring &fileName, int *origin) {

-    auto iter = m_Files.find(fileName);

+    auto iter = m_Files.find(ToLower(fileName));

     if (iter != m_Files.end()) {

       if (origin != NULL) {

         FileEntry::Ptr entry = m_FileRegister->getFile(iter->second);

@@ -268,6 +280,8 @@ public:
         }

       }

       m_FileRegister->removeFile(iter->second);

+    } else {

+      log ("failed to remove %ls from %ls", fileName.c_str(), m_Name.c_str());

     }

   }

 

@@ -275,20 +289,23 @@ public:
 

   FilesOrigin &createOrigin(const std::wstring &originName, const std::wstring &directory, int priority);

 

+  void removeFiles(const std::set<FileEntry::Index> &indices);

+

 private:

 

   DirectoryEntry(const DirectoryEntry &reference);

   DirectoryEntry &operator=(const DirectoryEntry &reference);

 

   void insert(const std::wstring &fileName, FilesOrigin &origin, FILETIME fileTime, const std::wstring &archive) {

-    auto iter = m_Files.find(fileName);

+    std::wstring fileNameLower = ToLower(fileName);

+    auto iter = m_Files.find(fileNameLower);

     FileEntry::Ptr file;

     if (iter != m_Files.end()) {

       file = m_FileRegister->getFile(iter->second);

     } else {

       file = m_FileRegister->createFile(fileName, this);

       // TODO this has been observed to cause a crash, no clue why

-      m_Files[fileName] = file->getIndex();

+      m_Files[fileNameLower] = file->getIndex();

     }

     file->addOrigin(origin.getID(), fileTime, archive);

     origin.addFile(file->getIndex());

@@ -307,20 +324,11 @@ private:
 

 private:

 

-  struct WStrLess {

-    bool operator()(const std::wstring &LHS, const std::wstring &RHS) const {

-      return _wcsicmp(LHS.c_str(), RHS.c_str()) < 0;

-    }

-  };

-

-

-private:

-

   boost::shared_ptr<FileRegister> m_FileRegister;

   boost::shared_ptr<OriginConnection> m_OriginConnection;

 

   std::wstring m_Name;

-  std::map<std::wstring, FileEntry::Index, WStrLess> m_Files;

+  std::map<std::wstring, FileEntry::Index> m_Files;

   std::vector<DirectoryEntry*> m_SubDirectories;

 

   DirectoryEntry *m_Parent;

diff --git a/src/shared/shared.pro b/src/shared/shared.pro
index c5925c24..84a9c274 100644
--- a/src/shared/shared.pro
+++ b/src/shared/shared.pro
@@ -15,8 +15,8 @@ INCLUDEPATH += ../bsatk "$(BOOSTPATH)"
 

 

 # only for custom leak detection

-#DEFINES += TRACE_LEAKS

-#LIBS += -lDbgHelp

+DEFINES += TRACE_LEAKS

+LIBS += -lDbgHelp

 

 

 CONFIG(debug, debug|release) {

diff --git a/src/shared/skyriminfo.cpp b/src/shared/skyriminfo.cpp
index 5017da38..72452255 100644
--- a/src/shared/skyriminfo.cpp
+++ b/src/shared/skyriminfo.cpp
@@ -273,7 +273,9 @@ void SkyrimInfo::createProfile(const std::wstring &directory, bool useDefaults)
       std::wostringstream source;

       source << getMyGamesDirectory() << L"\\Skyrim\\skyrimprefs.ini";

       if (!::CopyFileW(source.str().c_str(), target.c_str(), true)) {

-        if (::GetLastError() != ERROR_FILE_EXISTS) {

+        log("failed to copy ini file %ls", source.str().c_str());

+        // create empty

+        if (::CreateFileW(target.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL) == INVALID_HANDLE_VALUE) {

           std::ostringstream stream;

           stream << "failed to copy ini file: " << ToString(source.str(), false);

           throw windows_error(stream.str());

diff --git a/src/shared/util.cpp b/src/shared/util.cpp
index 22657e6c..61ca9cdc 100644
--- a/src/shared/util.cpp
+++ b/src/shared/util.cpp
@@ -86,30 +86,28 @@ std::wstring ToWString(const std::string &source, bool utf8)
 

 std::string &ToLower(std::string &text)

 {

-  std::transform(text.begin(), text.end(), text.begin(), tolower);

+  std::transform(text.begin(), text.end(), text.begin(), ::tolower);

   return text;

 }

 

 std::string ToLower(const std::string &text)

 {

-  std::string temp = text;

-

-  std::transform(temp.begin(), temp.end(), temp.begin(), tolower);

-  return temp;

+  std::string result = text;

+  std::transform(result.begin(), result.end(), result.begin(), ::tolower);

+  return result;

 }

 

 std::wstring &ToLower(std::wstring &text)

 {

-  std::transform(text.begin(), text.end(), text.begin(), towlower);

+  std::transform(text.begin(), text.end(), text.begin(), ::towlower);

   return text;

 }

 

 std::wstring ToLower(const std::wstring &text)

 {

-  std::wstring temp = text;

-

-  std::transform(temp.begin(), temp.end(), temp.begin(), towlower);

-  return temp;

+  std::wstring result = text;

+  std::transform(result.begin(), result.end(), result.begin(), ::towlower);

+  return result;

 }

 

 VS_FIXEDFILEINFO GetFileVersion(const std::wstring &fileName)