Merge pull request #1003 from isanae/file-list-improvements

Refresh optimizations

1 files changed, 106 insertions, 327 deletions

diff --git a/src/shared/directoryentry.h b/src/shared/directoryentry.h
index e26011d7..a28ceeae 100644
--- a/src/shared/directoryentry.h
+++ b/src/shared/directoryentry.h
@@ -17,25 +17,18 @@ You should have received a copy of the GNU General Public License
 along with Mod Organizer.  If not, see <http://www.gnu.org/licenses/>.

 */

 

-#ifndef DIRECTORYENTRY_H

-#define DIRECTORYENTRY_H

+#ifndef MO_REGISTER_DIRECTORYENTRY_INCLUDED

+#define MO_REGISTER_DIRECTORYENTRY_INCLUDED

 

-

-#include <string>

-#include <set>

-#include <vector>

-#include <map>

-#include <cassert>

-#define WIN32_MEAN_AND_LEAN

-#include <Windows.h>

+#include "fileregister.h"

 #include <bsatk.h>

-#ifndef Q_MOC_RUN

-#include <boost/shared_ptr.hpp>

-#include <boost/weak_ptr.hpp>

-#endif

-#include "util.h"

 

-namespace MOShared { struct DirectoryEntryFileKey; }

+namespace env

+{

+  class DirectoryWalker;

+  struct Directory;

+  struct File;

+}

 

 namespace std

 {

@@ -53,274 +46,23 @@ namespace std
 namespace MOShared

 {

 

-class DirectoryEntry;

-class OriginConnection;

-class FileRegister;

-

-

-class FileEntry

-{

-public:

-  static constexpr uint64_t NoFileSize =

-    std::numeric_limits<uint64_t>::max();

-

-  typedef unsigned int Index;

-  typedef boost::shared_ptr<FileEntry> Ptr;

-

-  // a vector of {originId, {archiveName, order}}

-  //

-  // if a file is in an archive, archiveName is the name of the bsa and order

-  // is the order of the associated plugin in the plugins list

-  //

-  // is a file is not in an archive, archiveName is empty and order is usually

-  // -1

-  typedef std::vector<std::pair<int, std::pair<std::wstring, int>>>

-    AlternativesVector;

-

-  FileEntry();

-  FileEntry(Index index, const std::wstring &name, DirectoryEntry *parent);

-

-  Index getIndex() const

-  {

-    return m_Index;

-  }

-

-  time_t lastAccessed() const

-  {

-    return m_LastAccessed;

-  }

-

-  void addOrigin(

-    int origin, FILETIME fileTime, const std::wstring &archive, int order);

-

-  // remove the specified origin from the list of origins that contain this

-  // file. if no origin is left, the file is effectively deleted and true is

-  // returned. otherwise, false is returned

-  bool removeOrigin(int origin);

-

-  void sortOrigins();

-

-  // gets the list of alternative origins (origins with lower priority than

-  // the primary one). if sortOrigins has been called, it is sorted by priority

-  // (ascending)

-  const AlternativesVector &getAlternatives() const

-  {

-    return m_Alternatives;

-  }

-

-  const std::wstring &getName() const

-  {

-    return m_Name;

-  }

-

-  int getOrigin() const

-  {

-    return m_Origin;

-  }

-

-  int getOrigin(bool &archive) const

-  {

-    archive = (m_Archive.first.length() != 0);

-    return m_Origin;

-  }

-

-  const std::pair<std::wstring, int> &getArchive() const

-  {

-    return m_Archive;

-  }

-

-  bool isFromArchive(std::wstring archiveName = L"") const;

-

-  // if originID is -1, uses the main origin; if this file doesn't exist in the

-  // given origin, returns an empty string

-  //

-  std::wstring getFullPath(int originID=-1) const;

-

-  std::wstring getRelativePath() const;

-

-  DirectoryEntry *getParent()

-  {

-    return m_Parent;

-  }

-

-  void setFileTime(FILETIME fileTime) const

-  {

-    m_FileTime = fileTime;

-  }

-

-  FILETIME getFileTime() const

-  {

-    return m_FileTime;

-  }

-

-  void setFileSize(uint64_t size, uint64_t compressedSize)

-  {

-    m_FileSize = size;

-    m_CompressedFileSize = compressedSize;

-  }

-

-  uint64_t getFileSize() const

-  {

-    return m_FileSize;

-  }

-

-  uint64_t getCompressedFileSize() const

-  {

-    return m_CompressedFileSize;

-  }

-

-private:

-  Index m_Index;

-  std::wstring m_Name;

-  int m_Origin;

-  std::pair<std::wstring, int> m_Archive;

-  AlternativesVector m_Alternatives;

-  DirectoryEntry *m_Parent;

-  mutable FILETIME m_FileTime;

-  uint64_t m_FileSize, m_CompressedFileSize;

-

-  time_t m_LastAccessed;

-

-  bool recurseParents(std::wstring &path, const DirectoryEntry *parent) const;

-};

-

-

-// represents a mod or the data directory, providing files to the tree

-class FilesOrigin

-{

-  friend class OriginConnection;

-

-public:

-  FilesOrigin();

-  FilesOrigin(const FilesOrigin &reference);

-

-  // sets priority for this origin, but it will overwrite the existing mapping

-  // for this priority, the previous origin will no longer be referenced

-  void setPriority(int priority);

-

-  int getPriority() const

-  {

-    return m_Priority;

-  }

-

-  void setName(const std::wstring &name);

-  const std::wstring &getName() const

-  {

-    return m_Name;

-  }

-

-  int getID() const

-  {

-    return m_ID;

-  }

-

-  const std::wstring &getPath() const

-  {

-    return m_Path;

-  }

-

-  std::vector<FileEntry::Ptr> getFiles() const;

-  FileEntry::Ptr findFile(FileEntry::Index index) const;

-

-  void enable(bool enabled, time_t notAfter = LONG_MAX);

-  bool isDisabled() const

-  {

-    return m_Disabled;

-  }

-

-  void addFile(FileEntry::Index index)

-  {

-    m_Files.insert(index);

-  }

-

-  void removeFile(FileEntry::Index index);

-

-  bool containsArchive(std::wstring archiveName);

-

-private:

-  int m_ID;

-  bool m_Disabled;

-  std::set<FileEntry::Index> m_Files;

-  std::wstring m_Name;

-  std::wstring m_Path;

-  int m_Priority;

-  boost::weak_ptr<FileRegister> m_FileRegister;

-  boost::weak_ptr<OriginConnection> m_OriginConnection;

-

-  FilesOrigin(

-    int ID, const std::wstring &name, const std::wstring &path, int priority,

-    boost::shared_ptr<FileRegister> fileRegister,

-    boost::shared_ptr<OriginConnection> originConnection);

-};

-

-

-class FileRegister

-{

-public:

-  FileRegister(boost::shared_ptr<OriginConnection> originConnection);

-

-  bool indexValid(FileEntry::Index index) const;

-

-  FileEntry::Ptr createFile(const std::wstring &name, DirectoryEntry *parent);

-  FileEntry::Ptr getFile(FileEntry::Index index) const;

-

-  size_t size() const

-  {

-    return m_Files.size();

-  }

-

-  bool removeFile(FileEntry::Index index);

-  void removeOrigin(FileEntry::Index index, int originID);

-  void removeOriginMulti(std::set<FileEntry::Index> indices, int originID, time_t notAfter);

-

-  void sortOrigins();

-

-private:

-  std::map<FileEntry::Index, FileEntry::Ptr> m_Files;

-  boost::shared_ptr<OriginConnection> m_OriginConnection;

-

-  FileEntry::Index generateIndex();

-  void unregisterFile(FileEntry::Ptr file);

-};

-

-

-struct DirectoryEntryFileKey

-{

-  DirectoryEntryFileKey(std::wstring v)

-    : value(std::move(v)), hash(getHash(value))

-  {

-  }

-

-  bool operator==(const DirectoryEntryFileKey& o) const

-  {

-    return (value == o.value);

-  }

-

-  static std::size_t getHash(const std::wstring& value)

-  {

-    return std::hash<std::wstring>()(value);

-  }

-

-  const std::wstring value;

-  const std::size_t hash;

-};

-

-

 class DirectoryEntry

 {

 public:

-  using FileKey = DirectoryEntryFileKey;

-

   DirectoryEntry(

-    const std::wstring &name, DirectoryEntry *parent, int originID);

+    std::wstring name, DirectoryEntry* parent, OriginID originID);

 

   DirectoryEntry(

-    const std::wstring &name, DirectoryEntry *parent, int originID,

+    std::wstring name, DirectoryEntry* parent, OriginID originID,

     boost::shared_ptr<FileRegister> fileRegister,

     boost::shared_ptr<OriginConnection> originConnection);

 

   ~DirectoryEntry();

 

+  // noncopyable

+  DirectoryEntry(const DirectoryEntry&) = delete;

+  DirectoryEntry& operator=(const DirectoryEntry&) = delete;

+

   void clear();

 

   bool isPopulated() const

@@ -343,7 +85,7 @@ public:
     return !m_Files.empty();

   }

 

-  const DirectoryEntry *getParent() const

+  const DirectoryEntry* getParent() const

   {

     return m_Parent;

   }

@@ -351,16 +93,32 @@ public:
   // add files to this directory (and subdirectories) from the specified origin.

   // That origin may exist or not

   void addFromOrigin(

-    const std::wstring &originName,

-    const std::wstring &directory, int priority);

+    const std::wstring& originName,

+    const std::wstring& directory, int priority, DirectoryStats& stats);

+

+  void addFromOrigin(

+    env::DirectoryWalker& walker, const std::wstring& originName,

+    const std::wstring& directory, int priority, DirectoryStats& stats);

+

+  void addFromAllBSAs(

+    const std::wstring& originName, const std::wstring& directory,

+    int priority, const std::vector<std::wstring>& archives,

+    const std::set<std::wstring>& enabledArchives,

+    const std::vector<std::wstring>& loadOrder,

+    DirectoryStats& stats);

 

   void addFromBSA(

-    const std::wstring &originName, std::wstring &directory,

-    const std::wstring &fileName, int priority, int order);

+    const std::wstring& originName, const std::wstring& directory,

+    const std::wstring& archivePath, int priority, int order,

+    DirectoryStats& stats);

+

+  void addFromList(

+    const std::wstring& originName, const std::wstring& directory,

+    env::Directory& root, int priority, DirectoryStats& stats);

 

-  void propagateOrigin(int origin);

+  void propagateOrigin(OriginID origin);

 

-  const std::wstring &getName() const

+  const std::wstring& getName() const

   {

     return m_Name;

   }

@@ -370,18 +128,18 @@ public:
     return m_FileRegister;

   }

 

-  bool originExists(const std::wstring &name) const;

-  FilesOrigin &getOriginByID(int ID) const;

-  FilesOrigin &getOriginByName(const std::wstring &name) const;

-  const FilesOrigin* findOriginByID(int ID) const;

+  bool originExists(const std::wstring& name) const;

+  FilesOrigin& getOriginByID(OriginID ID) const;

+  FilesOrigin& getOriginByName(const std::wstring& name) const;

+  const FilesOrigin* findOriginByID(OriginID ID) const;

 

-  int anyOrigin() const;

+  OriginID anyOrigin() const;

 

-  std::vector<FileEntry::Ptr> getFiles() const;

+  std::vector<FileEntryPtr> getFiles() const;

 

   void getSubDirectories(

-    std::vector<DirectoryEntry*>::const_iterator &begin,

-    std::vector<DirectoryEntry*>::const_iterator &end) const

+    std::vector<DirectoryEntry*>::const_iterator& begin,

+    std::vector<DirectoryEntry*>::const_iterator& end) const

   {

     begin = m_SubDirectories.begin();

     end = m_SubDirectories.end();

@@ -424,22 +182,22 @@ public:
     }

   }

 

-  FileEntry::Ptr getFileByIndex(FileEntry::Index index) const

+  FileEntryPtr getFileByIndex(FileIndex index) const

   {

     return m_FileRegister->getFile(index);

   }

 

-  DirectoryEntry *findSubDirectory(

-    const std::wstring &name, bool alreadyLowerCase=false) const;

+  DirectoryEntry* findSubDirectory(

+    const std::wstring& name, bool alreadyLowerCase=false) const;

 

-  DirectoryEntry *findSubDirectoryRecursive(const std::wstring &path);

+  DirectoryEntry* findSubDirectoryRecursive(const std::wstring& path);

 

   /** retrieve a file in this directory by name.

     * @param name name of the file

     * @return fileentry object for the file or nullptr if no file matches

     */

-  const FileEntry::Ptr findFile(const std::wstring &name, bool alreadyLowerCase=false) const;

-  const FileEntry::Ptr findFile(const FileKey& key) const;

+  const FileEntryPtr findFile(const std::wstring& name, bool alreadyLowerCase=false) const;

+  const FileEntryPtr findFile(const DirectoryEntryFileKey& key) const;

 

   bool hasFile(const std::wstring& name) const;

   bool containsArchive(std::wstring archiveName);

@@ -450,36 +208,36 @@ public:
   // if directory is not nullptr, the referenced variable will be set to the

   // path containing the file

   //

-  const FileEntry::Ptr searchFile(

-    const std::wstring &path, const DirectoryEntry **directory=nullptr) const;

+  const FileEntryPtr searchFile(

+    const std::wstring& path, const DirectoryEntry** directory=nullptr) const;

 

-  void insertFile(const std::wstring &filePath, FilesOrigin &origin, FILETIME fileTime);

-

-  void removeFile(FileEntry::Index index);

+  void removeFile(FileIndex index);

 

   // remove the specified file from the tree. This can be a path leading to a

   // file in a subdirectory

-  bool removeFile(const std::wstring &filePath, int *origin = nullptr);

+  bool removeFile(const std::wstring& filePath, OriginID* origin = nullptr);

 

   /**

    * @brief remove the specified directory

    * @param path directory to remove

    */

-  void removeDir(const std::wstring &path);

+  void removeDir(const std::wstring& path);

+

+  bool remove(const std::wstring& fileName, OriginID* origin);

 

-  bool remove(const std::wstring &fileName, int *origin);

+  bool hasContentsFromOrigin(OriginID originID) const;

 

-  bool hasContentsFromOrigin(int originID) const;

+  FilesOrigin& createOrigin(

+    const std::wstring& originName,

+    const std::wstring& directory, int priority, DirectoryStats& stats);

 

-  FilesOrigin &createOrigin(

-    const std::wstring &originName,

-    const std::wstring &directory, int priority);

+  void removeFiles(const std::set<FileIndex>& indices);

 

-  void removeFiles(const std::set<FileEntry::Index> &indices);

+  void dump(const std::wstring& file) const;

 

 private:

-  using FilesMap = std::map<std::wstring, FileEntry::Index>;

-  using FilesLookup = std::unordered_map<FileKey, FileEntry::Index>;

+  using FilesMap = std::map<std::wstring, FileIndex>;

+  using FilesLookup = std::unordered_map<DirectoryEntryFileKey, FileIndex>;

   using SubDirectories = std::vector<DirectoryEntry*>;

   using SubDirectoriesLookup = std::unordered_map<std::wstring, DirectoryEntry*>;

 

@@ -492,39 +250,60 @@ private:
   SubDirectories m_SubDirectories;

   SubDirectoriesLookup m_SubDirectoriesLookup;

 

-  DirectoryEntry *m_Parent;

-  std::set<int> m_Origins;

+  DirectoryEntry* m_Parent;

+  std::set<OriginID> m_Origins;

   bool m_Populated;

   bool m_TopLevel;

+  mutable std::mutex m_SubDirMutex;

+  mutable std::mutex m_FilesMutex;

+  mutable std::mutex m_OriginsMutex;

 

 

-  DirectoryEntry(const DirectoryEntry &reference);

+  FileEntryPtr insert(

+    std::wstring_view fileName, FilesOrigin& origin, FILETIME fileTime,

+    std::wstring_view archive, int order, DirectoryStats& stats);

 

-  FileEntry::Ptr insert(

-    const std::wstring &fileName, FilesOrigin &origin, FILETIME fileTime,

-    const std::wstring &archive, int order);

+  FileEntryPtr insert(

+    env::File& file, FilesOrigin& origin,

+    std::wstring_view archive, int order, DirectoryStats& stats);

 

   void addFiles(

-    FilesOrigin &origin, wchar_t *buffer, int bufferOffset);

+    env::DirectoryWalker& walker, FilesOrigin& origin,

+    const std::wstring& path, DirectoryStats& stats);

 

   void addFiles(

-    FilesOrigin &origin, BSA::Folder::Ptr archiveFolder, FILETIME &fileTime,

-    const std::wstring &archiveName, int order);

+    FilesOrigin& origin, BSA::Folder::Ptr archiveFolder, FILETIME fileTime,

+    const std::wstring& archiveName, int order, DirectoryStats& stats);

 

-  DirectoryEntry *getSubDirectory(

-    const std::wstring &name, bool create, int originID = -1);

+  void addDir(FilesOrigin& origin, env::Directory& d, DirectoryStats& stats);

 

-  DirectoryEntry *getSubDirectoryRecursive(

-    const std::wstring &path, bool create, int originID = -1);

+  DirectoryEntry* getSubDirectory(

+    std::wstring_view name, bool create, DirectoryStats& stats,

+    OriginID originID = InvalidOriginID);

+

+  DirectoryEntry* getSubDirectory(

+    env::Directory& dir, bool create, DirectoryStats& stats,

+    OriginID originID = InvalidOriginID);

+

+  DirectoryEntry* getSubDirectoryRecursive(

+    const std::wstring& path, bool create, DirectoryStats& stats,

+    OriginID originID = InvalidOriginID);

 

   void removeDirRecursive();

 

-  void addDirectoryToList(DirectoryEntry* e);

+  void addDirectoryToList(DirectoryEntry* e, std::wstring nameLc);

   void removeDirectoryFromList(SubDirectories::iterator itor);

 

-  void addFileToList(std::wstring fileNameLower, FileEntry::Index index);

-  void removeFileFromList(FileEntry::Index index);

-  void removeFilesFromList(const std::set<FileEntry::Index>& indices);

+  void addFileToList(std::wstring fileNameLower, FileIndex index);

+  void removeFileFromList(FileIndex index);

+  void removeFilesFromList(const std::set<FileIndex>& indices);

+

+  struct Context;

+  static void onDirectoryStart(Context* cx, std::wstring_view path);

+  static void onDirectoryEnd(Context* cx, std::wstring_view path);

+  static void onFile(Context* cx, std::wstring_view path, FILETIME ft);

+

+  void dump(std::FILE* f, const std::wstring& parentPath) const;

 };

 

 } // namespace MOShared

@@ -540,4 +319,4 @@ namespace std
   }

 }

 

-#endif // DIRECTORYENTRY_H

+#endif // MO_REGISTER_DIRECTORYENTRY_INCLUDED