From 70bcd97bd23756a92507006e6202eee7af902cd3 Mon Sep 17 00:00:00 2001 From: LostDragonist Date: Thu, 7 Mar 2019 16:32:17 -0600 Subject: Use MD5 when querying info before bothering the user --- src/downloadlist.cpp | 5 +- src/downloadlistwidget.cpp | 9 +++- src/downloadlistwidget.h | 2 + src/downloadmanager.cpp | 114 ++++++++++++++++++++++++++++++++++++++++++++- src/downloadmanager.h | 37 +++++++++------ src/mainwindow.cpp | 1 + src/nexusinterface.cpp | 60 ++++++++++++++++++++++++ src/nexusinterface.h | 18 +++++++ 8 files changed, 225 insertions(+), 21 deletions(-) diff --git a/src/downloadlist.cpp b/src/downloadlist.cpp index 765ee646..dd72abbd 100644 --- a/src/downloadlist.cpp +++ b/src/downloadlist.cpp @@ -104,8 +104,9 @@ QVariant DownloadList::data(const QModelIndex &index, int role) const case DownloadManager::STATE_CANCELED: return tr("Canceled"); case DownloadManager::STATE_PAUSED: return tr("Paused"); case DownloadManager::STATE_ERROR: return tr("Error"); - case DownloadManager::STATE_FETCHINGMODINFO: return tr("Fetching Info 1"); - case DownloadManager::STATE_FETCHINGFILEINFO: return tr("Fetching Info 2"); + case DownloadManager::STATE_FETCHINGMODINFO: return tr("Fetching Info"); + case DownloadManager::STATE_FETCHINGFILEINFO: return tr("Fetching Info"); + case DownloadManager::STATE_FETCHINGMODINFO_MD5: return tr("Fetching Info"); case DownloadManager::STATE_READY: return tr("Downloaded"); case DownloadManager::STATE_INSTALLED: return tr("Installed"); case DownloadManager::STATE_UNINSTALLED: return tr("Uninstalled"); diff --git a/src/downloadlistwidget.cpp b/src/downloadlistwidget.cpp index be9d30e2..f0c4da1a 100644 --- a/src/downloadlistwidget.cpp +++ b/src/downloadlistwidget.cpp @@ -199,8 +199,8 @@ void DownloadListWidget::onCustomContextMenu(const QPoint &point) if (state >= DownloadManager::STATE_READY) { menu.addAction(tr("Install"), this, SLOT(issueInstall())); - if (m_Manager->isInfoIncomplete(m_ContextRow)) - menu.addAction(tr("Query Info"), this, SLOT(issueQueryInfo())); + if (m_Manager->isInfoIncomplete(m_ContextRow)) + menu.addAction(tr("Query Info"), this, SLOT(issueQueryInfoMd5())); else menu.addAction(tr("Visit on Nexus"), this, SLOT(issueVisitOnNexus())); menu.addAction(tr("Open File"), this, SLOT(issueOpenFile())); @@ -252,6 +252,11 @@ void DownloadListWidget::issueQueryInfo() emit queryInfo(m_ContextRow); } +void DownloadListWidget::issueQueryInfoMd5() +{ + emit queryInfoMd5(m_ContextRow); +} + void DownloadListWidget::issueDelete() { if (QMessageBox::question(nullptr, tr("Delete Files?"), diff --git a/src/downloadlistwidget.h b/src/downloadlistwidget.h index 4776d259..ad07b0f1 100644 --- a/src/downloadlistwidget.h +++ b/src/downloadlistwidget.h @@ -78,6 +78,7 @@ public: signals: void installDownload(int index); void queryInfo(int index); + void queryInfoMd5(int index); void removeDownload(int index, bool deleteFile); void restoreDownload(int index); void cancelDownload(int index); @@ -109,6 +110,7 @@ private slots: void issueRemoveFromViewCompleted(); void issueRemoveFromViewUninstalled(); void issueQueryInfo(); + void issueQueryInfoMd5(); private: DownloadManager *m_Manager; diff --git a/src/downloadmanager.cpp b/src/downloadmanager.cpp index c3a6a3e7..3563ecdb 100644 --- a/src/downloadmanager.cpp +++ b/src/downloadmanager.cpp @@ -262,7 +262,8 @@ void DownloadManager::pauseAll() foreach (DownloadInfo *info, m_ActiveDownloads) { if ((info->m_State < STATE_CANCELED) || (info->m_State == STATE_FETCHINGFILEINFO) || - (info->m_State == STATE_FETCHINGMODINFO)) { + (info->m_State == STATE_FETCHINGMODINFO) || + (info->m_State == STATE_FETCHINGMODINFO_MD5)) { done = false; break; } @@ -819,7 +820,9 @@ void DownloadManager::pauseDownload(int index) } else { setState(info, STATE_PAUSED); } - } else if ((info->m_State == STATE_FETCHINGMODINFO) || (info->m_State == STATE_FETCHINGFILEINFO)) { + } else if ((info->m_State == STATE_FETCHINGMODINFO) || + (info->m_State == STATE_FETCHINGFILEINFO) || + (info->m_State == STATE_FETCHINGMODINFO_MD5)) { setState(info, STATE_READY); } } @@ -952,6 +955,46 @@ void DownloadManager::queryInfo(int index) setState(info, STATE_FETCHINGMODINFO); } +void DownloadManager::queryInfoMd5(int index) +{ + if ((index < 0) || (index >= m_ActiveDownloads.size())) { + reportError(tr("query: invalid download index %1").arg(index)); + return; + } + DownloadInfo *info = m_ActiveDownloads[index]; + + if (info->m_FileInfo->repository != "Nexus") { + qWarning("re-querying file info is currently only possible with Nexus"); + return; + } + + if (info->m_State < DownloadManager::STATE_READY) { + // UI shouldn't allow this + return; + } + + info->m_GamesToQuery << m_ManagedGame->gameShortName(); + info->m_GamesToQuery << m_ManagedGame->validShortNames(); + + QFile downloadFile(info->m_FileName); + if (!downloadFile.exists()) { + downloadFile.setFileName(m_OrganizerCore->downloadsPath() + "\\" + info->m_FileName); + } + if (!downloadFile.exists()) { + qDebug("Can't find download file %s", info->m_FileName); + return; + } + if (!downloadFile.open(QIODevice::ReadOnly)) { + qDebug("Can't open download file %s", info->m_FileName); + return; + } + info->m_Hash = QCryptographicHash::hash(downloadFile.readAll(), QCryptographicHash::Md5); + downloadFile.close(); + + info->m_ReQueried = true; + setState(info, STATE_FETCHINGMODINFO_MD5); +} + void DownloadManager::visitOnNexus(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { @@ -1329,6 +1372,9 @@ void DownloadManager::setState(DownloadManager::DownloadInfo *info, DownloadMana case STATE_FETCHINGFILEINFO: { m_RequestIDs.insert(m_NexusInterface->requestFiles(info->m_FileInfo->gameName, info->m_FileInfo->modID, this, info->m_DownloadID, QString())); } break; + case STATE_FETCHINGMODINFO_MD5: { + m_RequestIDs.insert(m_NexusInterface->requestInfoFromMd5(info->m_GamesToQuery[0], info->m_Hash, this, info->m_DownloadID, QString())); + } break; case STATE_READY: { createMetaFile(info); emit downloadComplete(row); @@ -1682,6 +1728,51 @@ void DownloadManager::nxmDownloadURLsAvailable(QString gameName, int modID, int } +void DownloadManager::nxmFileInfoFromMd5Available(QString gameName, QVariant userData, QVariant resultData, int requestID) +{ + std::set::iterator idIter = m_RequestIDs.find(requestID); + if (idIter == m_RequestIDs.end()) { + return; + } else { + m_RequestIDs.erase(idIter); + } + + auto resultlist = resultData.toList(); + auto results = resultlist[0].toMap(); + auto fileDetails = results["file_details"].toMap(); + auto modDetails = results["mod"].toMap(); + + DownloadInfo *info = downloadInfoByID(userData.toInt()); + + info->m_FileInfo->name = fileDetails["name"].toString(); + info->m_FileInfo->fileID = fileDetails["file_id"].toInt(); + info->m_FileInfo->description = fileDetails["description"].toString(); + info->m_FileInfo->version.parse(fileDetails["version"].toString()); + if (!info->m_FileInfo->version.isValid()) + info->m_FileInfo->version.parse(fileDetails["mod_version"].toString()); + info->m_FileInfo->fileCategory = fileDetails["category_id"].toInt(); + + info->m_FileInfo->modID = modDetails["mod_id"].toInt(); + info->m_FileInfo->modName = modDetails["name"].toString(); + info->m_FileInfo->categoryID = modDetails["category_id"].toInt(); + + QString gameShortName = gameName; + QStringList games(m_ManagedGame->validShortNames()); + games += m_ManagedGame->gameShortName(); + for (auto game : games) { + MOBase::IPluginGame *gamePlugin = m_OrganizerCore->getGame(game); + if (gamePlugin->gameNexusName().compare(gameName, Qt::CaseInsensitive) == 0) { + gameShortName = gamePlugin->gameShortName(); + break; + } + } + + info->m_FileInfo->gameName = gameShortName; + + setState(info, STATE_READY); +} + + void DownloadManager::nxmRequestFailed(QString gameName, int modID, int fileID, QVariant userData, int requestID, QNetworkReply::NetworkError error, const QString &errorString) { std::set::iterator idIter = m_RequestIDs.find(requestID); @@ -1691,10 +1782,29 @@ void DownloadManager::nxmRequestFailed(QString gameName, int modID, int fileID, m_RequestIDs.erase(idIter); } + DownloadInfo *userDataInfo = downloadInfoByID(userData.toInt()); + int index = 0; for (QVector::iterator iter = m_ActiveDownloads.begin(); iter != m_ActiveDownloads.end(); ++iter, ++index) { DownloadInfo *info = *iter; + if (info != userDataInfo) + continue; + + // MD5 searches continue until all possible games are done + if (info->m_State == STATE_FETCHINGMODINFO_MD5) { + if (info->m_GamesToQuery.count() >= 2) { + info->m_GamesToQuery.pop_front(); + setState(info, STATE_FETCHINGMODINFO_MD5); + break; + } else { + info->m_State = STATE_READY; + queryInfo(index); + emit update(index); + break; + } + } + if (info->m_FileInfo->modID == modID) { if (info->m_State < STATE_FETCHINGMODINFO) { m_ActiveDownloads.erase(iter); diff --git a/src/downloadmanager.h b/src/downloadmanager.h index 841a9fbc..8033989e 100644 --- a/src/downloadmanager.h +++ b/src/downloadmanager.h @@ -61,6 +61,7 @@ public: STATE_ERROR, STATE_FETCHINGMODINFO, STATE_FETCHINGFILEINFO, + STATE_FETCHINGMODINFO_MD5, STATE_NOFETCH, STATE_READY, STATE_INSTALLED, @@ -86,6 +87,8 @@ private: qint64 m_ResumePos; qint64 m_TotalSize; QDateTime m_Created; // used as a cache in DownloadManager::getFileTime, may not be valid elsewhere + QByteArray m_Hash; + QStringList m_GamesToQuery; int m_Tries; bool m_ReQueried; @@ -153,16 +156,16 @@ public: **/ void setOutputDirectory(const QString &outputDirectory); - /** - * @brief disables feedback from the downlods fileSystemWhatcher untill disableDownloadsWatcherEnd() is called - * - **/ - static void startDisableDirWatcher(); + /** + * @brief disables feedback from the downlods fileSystemWhatcher untill disableDownloadsWatcherEnd() is called + * + **/ + static void startDisableDirWatcher(); - /** - * @brief re-enables feedback from the downlods fileSystemWhatcher after disableDownloadsWatcherStart() was called - **/ - static void endDisableDirWatcher(); + /** + * @brief re-enables feedback from the downlods fileSystemWhatcher after disableDownloadsWatcherStart() was called + **/ + static void endDisableDirWatcher(); /** * @return current download directory @@ -290,7 +293,7 @@ public: * the following states: * started -> downloading -> fetching mod info -> fetching file info -> done * in case of downloads started via nxm-link, file information is fetched first - * + * * @param index index of the file to look up * @return the download state **/ @@ -444,7 +447,9 @@ public slots: void queryInfo(int index); - void visitOnNexus(int index); + void queryInfoMd5(int index); + + void visitOnNexus(int index); void openFile(int index); @@ -458,6 +463,8 @@ public slots: void nxmDownloadURLsAvailable(QString gameName, int modID, int fileID, QVariant userData, QVariant resultData, int requestID); + void nxmFileInfoFromMd5Available(QString gameName, QVariant userData, QVariant resultData, int requestID); + void nxmRequestFailed(QString gameName, int modID, int fileID, QVariant userData, int requestID, QNetworkReply::NetworkError error, const QString &errorString); void managedGameChanged(MOBase::IPluginGame const *gamePlugin); @@ -547,10 +554,10 @@ private: QFileSystemWatcher m_DirWatcher; - //The dirWatcher is actually triggering off normal Mo operations such as deleting downloads or editing .meta files - //so it needs to be disabled during operations that are known to cause the creation or deletion of files in the Downloads folder. - //Notably using QSettings to edit a file creates a temporarily .lock file that causes the Watcher to trigger multiple listRefreshes freezing the ui. - static int m_DirWatcherDisabler; + //The dirWatcher is actually triggering off normal Mo operations such as deleting downloads or editing .meta files + //so it needs to be disabled during operations that are known to cause the creation or deletion of files in the Downloads folder. + //Notably using QSettings to edit a file creates a temporarily .lock file that causes the Watcher to trigger multiple listRefreshes freezing the ui. + static int m_DirWatcherDisabler; std::map m_DownloadFails; diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index 3f6da2a6..0bdefe80 100644 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -5518,6 +5518,7 @@ void MainWindow::initDownloadView() connect(ui->downloadView, SIGNAL(installDownload(int)), &m_OrganizerCore, SLOT(installDownload(int))); connect(ui->downloadView, SIGNAL(queryInfo(int)), m_OrganizerCore.downloadManager(), SLOT(queryInfo(int))); + connect(ui->downloadView, SIGNAL(queryInfoMd5(int)), m_OrganizerCore.downloadManager(), SLOT(queryInfoMd5(int))); connect(ui->downloadView, SIGNAL(visitOnNexus(int)), m_OrganizerCore.downloadManager(), SLOT(visitOnNexus(int))); connect(ui->downloadView, SIGNAL(openFile(int)), m_OrganizerCore.downloadManager(), SLOT(openFile(int))); connect(ui->downloadView, SIGNAL(openInDownloadsFolder(int)), m_OrganizerCore.downloadManager(), SLOT(openInDownloadsFolder(int))); diff --git a/src/nexusinterface.cpp b/src/nexusinterface.cpp index b1f50fd8..33c5983c 100644 --- a/src/nexusinterface.cpp +++ b/src/nexusinterface.cpp @@ -555,6 +555,23 @@ int NexusInterface::requestToggleTracking(QString gameName, int modID, bool trac return -1; } +int NexusInterface::requestInfoFromMd5(QString gameName, QByteArray &hash, QObject *receiver, QVariant userData, + const QString &subModule, MOBase::IPluginGame const *game) +{ + NXMRequestInfo requestInfo(hash, NXMRequestInfo::TYPE_FILEINFO_MD5, userData, subModule, game); + requestInfo.m_Hash = hash; + m_RequestQueue.enqueue(requestInfo); + + connect(this, SIGNAL(nxmFileInfoFromMd5Available(QString, QVariant, QVariant, int)), + receiver, SLOT(nxmFileInfoFromMd5Available(QString, QVariant, QVariant, int)), Qt::UniqueConnection); + + connect(this, SIGNAL(nxmRequestFailed(QString, int, int, QVariant, int, QNetworkReply::NetworkError, QString)), + receiver, SLOT(nxmRequestFailed(QString, int, int, QVariant, int, QNetworkReply::NetworkError, QString)), Qt::UniqueConnection); + + nextRequest(); + return requestInfo.m_ID; +} + IPluginGame* NexusInterface::getGame(QString gameName) const { auto gamePlugins = m_PluginContainer->plugins(); @@ -673,6 +690,9 @@ void NexusInterface::nextRequest() case NXMRequestInfo::TYPE_TRACKEDMODS: { url = QStringLiteral("%1/user/tracked_mods").arg(info.m_URL); } break; + case NXMRequestInfo::TYPE_FILEINFO_MD5: { + url = QStringLiteral("%1/games/%2/mods/md5_search/%3").arg(info.m_URL).arg(info.m_GameName).arg(QString(info.m_Hash.toHex())); + } } } else { url = info.m_URL; @@ -719,6 +739,7 @@ void NexusInterface::requestFinished(std::list::iterator iter) if (reply->error() != QNetworkReply::NoError) { int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (statusCode == 429) { if (reply->rawHeader("x-rl-daily-remaining").toInt() || reply->rawHeader("x-rl-hourly-remaining").toInt()) qWarning("You appear to be making requests to the Nexus API too quickly and are being throttled. Please inform the MO2 team."); @@ -793,6 +814,9 @@ void NexusInterface::requestFinished(std::list::iterator iter) case NXMRequestInfo::TYPE_TRACKEDMODS: { emit nxmTrackedModsAvailable(iter->m_UserData, result, iter->m_ID); } break; + case NXMRequestInfo::TYPE_FILEINFO_MD5: { + emit nxmFileInfoFromMd5Available(iter->m_GameName, iter->m_UserData, result, iter->m_ID); + } break; } m_RemainingDailyRequests = reply->rawHeader("x-rl-daily-remaining").toInt(); @@ -890,6 +914,8 @@ NexusInterface::NXMRequestInfo::NXMRequestInfo(int modID , m_NexusGameID(game->nexusGameID()) , m_GameName(game->gameNexusName()) , m_Endorse(false) + , m_Track(false) + , m_Hash(QByteArray()) { } @@ -915,6 +941,8 @@ NexusInterface::NXMRequestInfo::NXMRequestInfo(int modID , m_NexusGameID(game->nexusGameID()) , m_GameName(game->gameNexusName()) , m_Endorse(false) + , m_Track(false) + , m_Hash(QByteArray()) {} NexusInterface::NXMRequestInfo::NXMRequestInfo(int modID @@ -939,6 +967,8 @@ NexusInterface::NXMRequestInfo::NXMRequestInfo(int modID , m_NexusGameID(game->nexusGameID()) , m_GameName(game->gameNexusName()) , m_Endorse(false) + , m_Track(false) + , m_Hash(QByteArray()) {} NexusInterface::NXMRequestInfo::NXMRequestInfo(Type type @@ -960,6 +990,8 @@ NexusInterface::NXMRequestInfo::NXMRequestInfo(Type type , m_NexusGameID(0) , m_GameName("") , m_Endorse(false) + , m_Track(false) + , m_Hash(QByteArray()) {} @@ -984,4 +1016,32 @@ NexusInterface::NXMRequestInfo::NXMRequestInfo(UpdatePeriod period , m_NexusGameID(game->nexusGameID()) , m_GameName(game->gameNexusName()) , m_Endorse(false) + , m_Track(false) + , m_Hash(QByteArray()) +{} + + +NexusInterface::NXMRequestInfo::NXMRequestInfo(QByteArray &hash + , NexusInterface::NXMRequestInfo::Type type + , QVariant userData + , const QString &subModule + , MOBase::IPluginGame const *game +) + : m_ModID(0) + , m_ModVersion("0") + , m_FileID(0) + , m_Reply(nullptr) + , m_Type(type) + , m_UpdatePeriod(UpdatePeriod::NONE) + , m_UserData(userData) + , m_Timeout(nullptr) + , m_Reroute(false) + , m_ID(s_NextID.fetchAndAddAcquire(1)) + , m_URL(get_management_url()) + , m_SubModule(subModule) + , m_NexusGameID(game->nexusGameID()) + , m_GameName(game->gameNexusName()) + , m_Endorse(false) + , m_Track(false) + , m_Hash(hash) {} diff --git a/src/nexusinterface.h b/src/nexusinterface.h index a0f94562..6f3c9cd1 100644 --- a/src/nexusinterface.h +++ b/src/nexusinterface.h @@ -369,6 +369,20 @@ public: int requestToggleTracking(QString gameName, int modID, bool track, QObject *receiver, QVariant userData, const QString &subModule, MOBase::IPluginGame const *game); + /** + * + */ + int requestInfoFromMd5(QString gameName, QByteArray &hash, QObject *receiver, QVariant userData, const QString &subModule) + { + return requestInfoFromMd5(gameName, hash, receiver, userData, subModule, getGame(gameName)); + } + + /** + * + */ + int requestInfoFromMd5(QString gameName, QByteArray &hash, QObject *receiver, QVariant userData, const QString &subModule, + MOBase::IPluginGame const *game); + /** * @param directory the directory to store cache files **/ @@ -438,6 +452,7 @@ signals: void nxmUpdatesAvailable(QString gameName, int modID, QVariant userData, QVariant resultData, int requestID); void nxmFilesAvailable(QString gameName, int modID, QVariant userData, QVariant resultData, int requestID); void nxmFileInfoAvailable(QString gameName, int modID, int fileID, QVariant userData, QVariant resultData, int requestID); + void nxmFileInfoFromMd5Available(QString gameName, QVariant userData, QVariant resultData, int requestID); void nxmDownloadURLsAvailable(QString gameName, int modID, int fileID, QVariant userData, QVariant resultData, int requestID); void nxmEndorsementsAvailable(QVariant userData, QVariant resultData, int requestID); void nxmEndorsementToggled(QString gameName, int modID, QVariant userData, QVariant resultData, int requestID); @@ -480,6 +495,7 @@ private: TYPE_CHECKUPDATES, TYPE_TOGGLETRACKING, TYPE_TRACKEDMODS, + TYPE_FILEINFO_MD5, } m_Type; UpdatePeriod m_UpdatePeriod; QVariant m_UserData; @@ -492,12 +508,14 @@ private: int m_ID; int m_Endorse; int m_Track; + QByteArray m_Hash; NXMRequestInfo(int modID, Type type, QVariant userData, const QString &subModule, MOBase::IPluginGame const *game); NXMRequestInfo(int modID, QString modVersion, Type type, QVariant userData, const QString &subModule, MOBase::IPluginGame const *game); NXMRequestInfo(int modID, int fileID, Type type, QVariant userData, const QString &subModule, MOBase::IPluginGame const *game); NXMRequestInfo(Type type, QVariant userData, const QString &subModule); NXMRequestInfo(UpdatePeriod period, Type type, QVariant userData, const QString &subModule, MOBase::IPluginGame const *game); + NXMRequestInfo(QByteArray &hash, Type type, QVariant userData, const QString &subModule, MOBase::IPluginGame const *game); private: static QAtomicInt s_NextID; -- cgit v1.3.1