/* Copyright (C) 2012 Sebastian Herbord. All rights reserved. This file is part of Mod Organizer. Mod Organizer is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. Mod Organizer is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with Mod Organizer. If not, see . */ #include "downloadmanager.h" #include "bbcode.h" #include "envfs.h" #include "filesystemutilities.h" #include "iplugingame.h" #include "nexusinterface.h" #include "nxmaccessmanager.h" #include "nxmurl.h" #include "organizercore.h" #include "selectiondialog.h" #include "shared/util.h" #include "utility.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace MOBase; // TODO limit number of downloads, also display download during nxm requests, store // modid/fileid with downloads static const char UNFINISHED[] = ".unfinished"; // Heuristic: does this filename look like a CDN object key that should be // replaced by the server's Content-Disposition filename if available? // Nexus v2 file metadata sometimes returns `file_name` as a CDN-path UUID // (e.g. "91/83/bb/9183bbff-...") rather than the actual archive name. static bool looksLikeCdnObjectKey(const QString& name) { if (name.isEmpty()) { return true; } // Real archive filenames always end in an archive suffix. If the name // has no recognized archive extension, assume it's a CDN key. static const QStringList kArchiveSuffixes = { ".zip", ".rar", ".7z", ".tar", ".gz", ".bz2", ".xz", ".exe", ".msi", ".bsa", ".ba2"}; for (const auto& suf : kArchiveSuffixes) { if (name.endsWith(suf, Qt::CaseInsensitive)) { return false; } } return true; } unsigned int DownloadManager::DownloadInfo::s_NextDownloadID = 1U; int DownloadManager::m_DirWatcherDisabler = 0; DownloadManager::DownloadInfo* DownloadManager::DownloadInfo::createNew(const ModRepositoryFileInfo* fileInfo, const QStringList& URLs, std::optional reservedID) { DownloadInfo* info = new DownloadInfo; info->m_DownloadID = reservedID.value_or(s_NextDownloadID++); info->m_StartTime.start(); info->m_PreResumeSize = 0LL; info->m_Progress = std::make_pair(0, "0.0 B/s "); info->m_ResumePos = 0; info->m_FileInfo = new ModRepositoryFileInfo(*fileInfo); info->m_Urls = URLs; info->m_CurrentUrl = 0; info->m_Tries = AUTOMATIC_RETRIES; info->m_State = STATE_STARTED; info->m_TaskProgressId = TaskProgressManager::instance().getId(); info->m_Reply = nullptr; return info; } DownloadManager::DownloadInfo* DownloadManager::DownloadInfo::createFromMeta(const QString& filePath, bool showHidden, const QString outputDirectory, std::optional fileSize, std::optional reservedID) { DownloadInfo* info = new DownloadInfo; QString const metaFileName = filePath + ".meta"; QFileInfo const metaFileInfo(metaFileName); if (QDir::fromNativeSeparators(metaFileInfo.path()) .compare(QDir::fromNativeSeparators(outputDirectory), Qt::CaseInsensitive) != 0) return nullptr; QSettings const metaFile(metaFileName, QSettings::IniFormat); if (!showHidden && metaFile.value("removed", false).toBool()) { return nullptr; } else { info->m_Hidden = metaFile.value("removed", false).toBool(); } QString const fileName = QFileInfo(filePath).fileName(); if (fileName.endsWith(UNFINISHED)) { info->m_FileName = fileName.mid(0, fileName.length() - static_cast(strlen(UNFINISHED))); info->m_State = STATE_PAUSED; } else { info->m_FileName = fileName; if (metaFile.value("paused", false).toBool()) { info->m_State = STATE_PAUSED; } else if (metaFile.value("uninstalled", false).toBool()) { info->m_State = STATE_UNINSTALLED; } else if (metaFile.value("installed", false).toBool()) { info->m_State = STATE_INSTALLED; } else { info->m_State = STATE_READY; } } info->m_DownloadID = reservedID.value_or(s_NextDownloadID++); info->m_Output.setFileName(filePath); info->m_TotalSize = fileSize ? *fileSize : QFileInfo(filePath).size(); info->m_PreResumeSize = info->m_TotalSize; info->m_CurrentUrl = 0; info->m_Urls = metaFile.value("url", "").toString().split(";"); info->m_Tries = 0; info->m_TaskProgressId = TaskProgressManager::instance().getId(); QString const gameName = metaFile.value("gameName", "").toString(); int const modID = metaFile.value("modID", 0).toInt(); int const fileID = metaFile.value("fileID", 0).toInt(); info->m_FileInfo = new ModRepositoryFileInfo(gameName, modID, fileID); info->m_FileInfo->name = metaFile.value("name", "").toString(); if (info->m_FileInfo->name == "0") { // bug in earlier version info->m_FileInfo->name = ""; } info->m_FileInfo->modName = metaFile.value("modName", "").toString(); info->m_FileInfo->gameName = gameName; info->m_FileInfo->modID = modID; info->m_FileInfo->fileID = fileID; info->m_FileInfo->description = metaFile.value("description").toString(); info->m_FileInfo->version.parse(metaFile.value("version", "0").toString()); info->m_FileInfo->newestVersion.parse( metaFile.value("newestVersion", "0").toString()); info->m_FileInfo->categoryID = metaFile.value("category", 0).toInt(); info->m_FileInfo->fileCategory = metaFile.value("fileCategory", 0).toInt(); info->m_FileInfo->repository = metaFile.value("repository", "Nexus").toString(); info->m_FileInfo->userData = metaFile.value("userData").toMap(); info->m_FileInfo->author = metaFile.value("author", "").toString(); info->m_FileInfo->uploader = metaFile.value("uploader", "").toString(); info->m_FileInfo->uploaderUrl = metaFile.value("uploaderUrl", "").toString(); info->m_Reply = nullptr; return info; } ScopedDisableDirWatcher::ScopedDisableDirWatcher(DownloadManager* downloadManager) { m_downloadManager = downloadManager; DownloadManager::startDisableDirWatcher(); log::debug("Scoped Disable DirWatcher: Started"); } ScopedDisableDirWatcher::~ScopedDisableDirWatcher() { DownloadManager::endDisableDirWatcher(); m_downloadManager = nullptr; log::debug("Scoped Disable DirWatcher: Stopped"); } void DownloadManager::startDisableDirWatcher() { DownloadManager::m_DirWatcherDisabler++; } void DownloadManager::endDisableDirWatcher() { if (DownloadManager::m_DirWatcherDisabler > 0) { if (DownloadManager::m_DirWatcherDisabler == 1) QCoreApplication::processEvents(); DownloadManager::m_DirWatcherDisabler--; } else { DownloadManager::m_DirWatcherDisabler = 0; } } void DownloadManager::DownloadInfo::setName(QString newName, bool renameFile) { QString oldMetaFileName = QString("%1.meta").arg(m_FileName); m_FileName = QFileInfo(newName).fileName(); if ((m_State == DownloadManager::STATE_STARTED) || (m_State == DownloadManager::STATE_DOWNLOADING) || (m_State == DownloadManager::STATE_PAUSED)) { newName.append(UNFINISHED); oldMetaFileName = QString("%1%2.meta").arg(m_FileName).arg(UNFINISHED); } if (renameFile) { if ((newName != m_Output.fileName()) && !m_Output.rename(newName)) { reportError(tr(R"(failed to rename "%1" to "%2")") .arg(m_Output.fileName()) .arg(newName)); return; } QFile metaFile(QFileInfo(newName).path() + "/" + oldMetaFileName); if (metaFile.exists()) metaFile.rename(newName.mid(0).append(".meta")); } if (!m_Output.isOpen()) { // can't set file name if it's open m_Output.setFileName(newName); } } bool DownloadManager::DownloadInfo::isPausedState() const { return m_State == STATE_PAUSED || m_State == STATE_ERROR; } QString DownloadManager::DownloadInfo::currentURL() { return m_Urls[m_CurrentUrl]; } DownloadManager::DownloadManager(NexusInterface* nexusInterface, QObject* parent) : m_NexusInterface(nexusInterface) { m_OrganizerCore = dynamic_cast(parent); connect(&m_DirWatcher, SIGNAL(directoryChanged(QString)), this, SLOT(directoryChanged(QString))); m_TimeoutTimer.setSingleShot(false); connect(&m_TimeoutTimer, &QTimer::timeout, this, &DownloadManager::checkDownloadTimeout); m_TimeoutTimer.start(5 * 1000); } DownloadManager::~DownloadManager() { for (QVector::iterator iter = m_ActiveDownloads.begin(); iter != m_ActiveDownloads.end(); ++iter) { delete *iter; } m_ActiveDownloads.clear(); m_ByID.clear(); } void DownloadManager::setParentWidget(QWidget* w) { m_ParentWidget = w; } bool DownloadManager::downloadsInProgress() { for (QVector::iterator iter = m_ActiveDownloads.begin(); iter != m_ActiveDownloads.end(); ++iter) { if ((*iter)->m_State < STATE_READY) { return true; } } return false; } bool DownloadManager::downloadsInProgressNoPause() { for (QVector::iterator iter = m_ActiveDownloads.begin(); iter != m_ActiveDownloads.end(); ++iter) { if ((*iter)->m_State < STATE_READY && (*iter)->m_State != STATE_PAUSED) { return true; } } return false; } void DownloadManager::pauseAll() { // first loop: pause all downloads for (int i = 0; i < m_ActiveDownloads.count(); ++i) { if (m_ActiveDownloads[i]->m_State < STATE_READY) { pauseDownload(i); } } QThread::msleep(100); bool done = false; QTime const startTime = QTime::currentTime(); // further loops: busy waiting for all downloads to complete. This could be neater... while (!done && (startTime.secsTo(QTime::currentTime()) < 5)) { QCoreApplication::processEvents(); done = true; foreach (DownloadInfo* info, m_ActiveDownloads) { if ((info->m_State < STATE_CANCELED) || (info->m_State == STATE_FETCHINGFILEINFO) || (info->m_State == STATE_FETCHINGMODINFO) || (info->m_State == STATE_FETCHINGMODINFO_MD5)) { done = false; break; } } if (!done) { QThread::msleep(100); } } } void DownloadManager::setOutputDirectory(const QString& outputDirectory, const bool refresh) { QStringList const directories = m_DirWatcher.directories(); if (!directories.empty()) { m_DirWatcher.removePaths(directories); } m_OutputDirectory = QDir::fromNativeSeparators(outputDirectory); if (refresh) { refreshList(); } m_DirWatcher.addPath(m_OutputDirectory); } void DownloadManager::setShowHidden(bool showHidden) { m_ShowHidden = showHidden; refreshList(); } void DownloadManager::setPluginContainer(PluginContainer* pluginContainer) { m_NexusInterface->setPluginContainer(pluginContainer); } void DownloadManager::refreshList() { TimeThis const tt("DownloadManager::refreshList()"); try { emit aboutToUpdate(); // avoid triggering other refreshes ScopedDisableDirWatcher const scopedDirWatcher(this); // remove finished downloads for (QVector::iterator iter = m_ActiveDownloads.begin(); iter != m_ActiveDownloads.end();) { if (((*iter)->m_State == STATE_READY) || ((*iter)->m_State == STATE_INSTALLED) || ((*iter)->m_State == STATE_UNINSTALLED)) { m_ByID.remove((*iter)->m_DownloadID); delete *iter; iter = m_ActiveDownloads.erase(iter); } else { ++iter; } } const QStringList supportedExtensions = m_OrganizerCore->installationManager()->getSupportedExtensions(); std::vector nameFilters; for (const auto& extension : supportedExtensions) { nameFilters.push_back(L"." + extension.toLower().toStdWString()); } nameFilters.push_back(QString(UNFINISHED).toLower().toStdWString()); QDir const dir(QDir::fromNativeSeparators(m_OutputDirectory)); // find orphaned meta files and delete them (sounds cruel but it's better for // everyone) QStringList orphans; QStringList const metaFiles = dir.entryList(QStringList() << "*.meta"); foreach (const QString& metaFile, metaFiles) { QString const baseFile = metaFile.left(metaFile.length() - 5); if (!QFile::exists(dir.absoluteFilePath(baseFile))) { orphans.append(dir.absoluteFilePath(metaFile)); } } if (!orphans.empty()) { log::debug("{} orphaned meta files will be deleted", orphans.size()); shellDelete(orphans, true); } std::set seen; struct Context { DownloadManager& self; std::set& seen; std::vector& extensions; }; Context cx = {.self=*this, .seen=seen, .extensions=nameFilters}; for (auto&& d : m_ActiveDownloads) { cx.seen.insert(d->m_FileName.toLower().toStdWString()); cx.seen.insert( QFileInfo(d->m_Output.fileName()).fileName().toLower().toStdWString()); } env::forEachEntry( QDir::toNativeSeparators(m_OutputDirectory).toStdWString(), &cx, nullptr, nullptr, [](void* data, std::wstring_view f, FILETIME, uint64_t size) { auto& cx = *static_cast(data); std::wstring lc = MOShared::ToLowerCopy(f); bool interestingExt = false; for (auto&& ext : cx.extensions) { if (lc.ends_with(ext)) { interestingExt = true; break; } } if (!interestingExt) { return; } if (cx.seen.contains(lc)) { return; } QString const fileName = QDir::fromNativeSeparators(cx.self.m_OutputDirectory) + "/" + QString::fromWCharArray(f.data(), f.size()); DownloadInfo* info = DownloadInfo::createFromMeta( fileName, cx.self.m_ShowHidden, cx.self.m_OutputDirectory, size); if (info == nullptr) { return; } cx.self.m_ActiveDownloads.push_front(info); cx.self.m_ByID.insert(info->m_DownloadID, info); cx.seen.insert(std::move(lc)); cx.seen.insert( QFileInfo(info->m_Output.fileName()).fileName().toLower().toStdWString()); }); log::debug("saw {} downloads", m_ActiveDownloads.size()); emit update(-1); } catch (const std::bad_alloc&) { reportError(tr("Memory allocation error (in refreshing directory).")); } } void DownloadManager::queryDownloadListInfo() { int incompleteCount = 0; for (size_t i = 0; std::cmp_less(i , m_ActiveDownloads.size()); i++) { if (isInfoIncomplete(i)) { incompleteCount++; } } if (incompleteCount <= 5 || QMessageBox::question( m_ParentWidget, tr("Query Metadata"), tr("There are %1 downloads with incomplete metadata.\n\n" "Do you want to fetch all incomplete metadata?\n" "API requests will be consumed, and Mod Organizer may stutter.") .arg(incompleteCount), QMessageBox::Yes | QMessageBox::No) == QMessageBox::Yes) { TimeThis const tt("DownloadManager::queryDownloadListInfo()"); log::info("Querying metadata for every download with incomplete info..."); startDisableDirWatcher(); for (size_t i = 0; std::cmp_less(i , m_ActiveDownloads.size()); i++) { if (isInfoIncomplete(i)) { queryInfoMd5(i, false); } } endDisableDirWatcher(); log::info("Metadata has been retrieved successfully!"); } } bool DownloadManager::addDownload(const QStringList& URLs, QString gameName, int modID, int fileID, const ModRepositoryFileInfo* fileInfo, std::optional reservedID) { // Parse the URL properly instead of feeding it to QFileInfo — QFileInfo // doesn't understand URLs, so its fileName() can pull in query-string junk // on Linux and we end up saving downloads as UUIDs from the S3 object key. const QUrl parsedUrl = QUrl::fromEncoded(URLs.first().toLocal8Bit()); QString fileName; // Nexus S3-signed download URLs carry the desired filename in the // `response-content-disposition` query parameter, e.g. // response-content-disposition=attachment;filename="JohnnyGuitar NVSE - 5.06.7z" // (percent-encoded on the wire). QUrlQuery decodes values for us. This // must be checked first — the URL path itself is just the CDN object key, // which looks like "9183bb...b-fdcf" and is useless as a filename. const QUrlQuery query(parsedUrl); const QString kDispoKey = QStringLiteral("response-content-disposition"); if (query.hasQueryItem(kDispoKey)) { const QString disposition = query.queryItemValue(kDispoKey, QUrl::FullyDecoded); // RFC 6266: filename="quoted" or filename=unquoted-token or filename*=ext-value. static const QRegularExpression rxQuoted( QStringLiteral("filename=\"([^\"]+)\"")); static const QRegularExpression rxUnquoted( QStringLiteral("filename=([^;\\s]+)")); static const QRegularExpression rxExtValue( QStringLiteral("filename\\*=[^']*'[^']*'([^;\\s]+)")); QRegularExpressionMatch m = rxQuoted.match(disposition); if (!m.hasMatch()) m = rxExtValue.match(disposition); if (!m.hasMatch()) m = rxUnquoted.match(disposition); if (m.hasMatch()) { fileName = QUrl::fromPercentEncoding(m.captured(1).toUtf8()); } } // Fall back to the last path segment of the URL if the disposition trick // didn't produce anything. if (fileName.isEmpty()) { fileName = parsedUrl.fileName(); if (!fileName.isEmpty()) { fileName = QUrl::fromPercentEncoding(fileName.toUtf8()); } } if (fileName.isEmpty()) { fileName = "unknown"; } QUrl const preferredUrl = QUrl::fromEncoded(URLs.first().toLocal8Bit()); log::debug("selected download url: {}", preferredUrl.toString()); QHttp2Configuration h2Conf; h2Conf.setSessionReceiveWindowSize( 16777215); // 16 MiB, based on Chrome and Firefox values h2Conf.setStreamReceiveWindowSize(16777215); QNetworkRequest request(preferredUrl); request.setHeader(QNetworkRequest::UserAgentHeader, m_NexusInterface->getAccessManager()->userAgent()); request.setAttribute(QNetworkRequest::CacheSaveControlAttribute, false); request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); request.setHttp2Configuration(h2Conf); return addDownload(m_NexusInterface->getAccessManager()->get(request), URLs, fileName, gameName, modID, fileID, fileInfo, reservedID); } bool DownloadManager::addDownload(QNetworkReply* reply, const ModRepositoryFileInfo* fileInfo) { QString fileName = getFileNameFromNetworkReply(reply); if (fileName.isEmpty()) { fileName = "unknown"; } return addDownload(reply, QStringList(reply->url().toString()), fileName, fileInfo->gameName, fileInfo->modID, fileInfo->fileID, fileInfo); } bool DownloadManager::addDownload(QNetworkReply* reply, const QStringList& URLs, const QString& fileName, QString gameName, int modID, int fileID, const ModRepositoryFileInfo* fileInfo, std::optional reservedID) { // download invoked from an already open network reply (i.e. download link in the // browser) DownloadInfo* newDownload = DownloadInfo::createNew(fileInfo, URLs, reservedID); QString baseName = fileName; if (!fileInfo->fileName.isEmpty()) { baseName = fileInfo->fileName; } else { QString const dispoName = getFileNameFromNetworkReply(reply); if (!dispoName.isEmpty()) { baseName = dispoName; } } // baseName could be a URL at this point so strip out the URL query int const queryIndex = baseName.indexOf("?"); if (queryIndex >= 0) { baseName.truncate(queryIndex); } startDisableDirWatcher(); newDownload->setName(getDownloadFileName(baseName), false); endDisableDirWatcher(); startDownload(reply, newDownload, false); // emit update(-1); return true; } void DownloadManager::removePending(QString gameName, int modID, int fileID) { QString gameShortName = gameName; QStringList games(m_ManagedGame->validShortNames()); games += m_ManagedGame->gameShortName(); for (const auto& game : games) { MOBase::IPluginGame* gamePlugin = m_OrganizerCore->getGame(game); if (gamePlugin != nullptr && gamePlugin->gameNexusName().compare(gameName, Qt::CaseInsensitive) == 0) { gameShortName = gamePlugin->gameShortName(); break; } } emit aboutToUpdate(); for (auto iter : m_PendingDownloads) { if (gameShortName.compare(std::get<0>(iter), Qt::CaseInsensitive) == 0 && (std::get<1>(iter) == modID) && (std::get<2>(iter) == fileID)) { m_PendingDownloads.removeAt(m_PendingDownloads.indexOf(iter)); break; } } emit update(-1); } void DownloadManager::startDownload(QNetworkReply* reply, DownloadInfo* newDownload, bool resume) { reply->setReadBufferSize( 1024 * 1024); // don't read more than 1MB at once to avoid memory troubles newDownload->m_Reply = reply; setState(newDownload, STATE_DOWNLOADING); if (newDownload->m_Urls.count() == 0) { newDownload->m_Urls = QStringList(reply->url().toString()); } QIODevice::OpenMode mode = QIODevice::WriteOnly; if (resume) { mode |= QIODevice::Append; } newDownload->m_StartTime.start(); createMetaFile(newDownload); if (!newDownload->m_Output.open(mode)) { reportError(tr("failed to download %1: could not open output file: %2") .arg(reply->url().toString()) .arg(newDownload->m_Output.fileName())); return; } connect(newDownload->m_Reply, SIGNAL(downloadProgress(qint64, qint64)), this, SLOT(downloadProgress(qint64, qint64))); connect(newDownload->m_Reply, SIGNAL(errorOccurred(QNetworkReply::NetworkError)), this, SLOT(downloadError(QNetworkReply::NetworkError))); connect(newDownload->m_Reply, SIGNAL(readyRead()), this, SLOT(downloadReadyRead())); connect(newDownload->m_Reply, SIGNAL(metaDataChanged()), this, SLOT(metaDataChanged())); if (!resume) { newDownload->m_PreResumeSize = newDownload->m_Output.size(); removePending(newDownload->m_FileInfo->gameName, newDownload->m_FileInfo->modID, newDownload->m_FileInfo->fileID); emit aboutToUpdate(); m_ActiveDownloads.append(newDownload); m_ByID.insert(newDownload->m_DownloadID, newDownload); emit update(-1); emit downloadAdded(); if (QFile::exists(m_OutputDirectory + "/" + newDownload->m_FileName)) { setState(newDownload, STATE_PAUSING); QCoreApplication::processEvents(); if (QMessageBox::question( m_ParentWidget, tr("Download again?"), tr("A file with the same name \"%1\" has already been downloaded. " "Do you want to download it again? The new file will receive a " "different name.") .arg(newDownload->m_FileName), QMessageBox::Yes | QMessageBox::No) == QMessageBox::No) { if (reply->isFinished()) setState(newDownload, STATE_CANCELED); else setState(newDownload, STATE_CANCELING); } else { startDisableDirWatcher(); newDownload->setName(getDownloadFileName(newDownload->m_FileName, true), true); endDisableDirWatcher(); if (newDownload->m_State == STATE_PAUSED) resumeDownload(indexByInfo(newDownload)); else setState(newDownload, STATE_DOWNLOADING); } } else connect(newDownload->m_Reply, SIGNAL(finished()), this, SLOT(downloadFinished())); QCoreApplication::processEvents(); if (newDownload->m_State != STATE_DOWNLOADING && newDownload->m_State != STATE_READY && newDownload->m_State != STATE_FETCHINGMODINFO && reply->isFinished()) { int const index = indexByInfo(newDownload); if (index >= 0) { downloadFinished(index); } return; } } else connect(newDownload->m_Reply, SIGNAL(finished()), this, SLOT(downloadFinished())); } void DownloadManager::addNXMDownload(const QString& url) { NXMUrl const nxmInfo(url); QStringList validGames; const MOBase::IPluginGame* foundGame = nullptr; validGames.append(m_ManagedGame->gameShortName()); validGames.append(m_ManagedGame->validShortNames()); for (auto game : validGames) { MOBase::IPluginGame* gamePlugin = m_OrganizerCore->getGame(game); // some game plugins give names in validShortNames() that may refer to other // plugins, like ttw returning "FalloutNV" and "Fallout3"; if these plugins // are not loaded, getGame() might return null if (!gamePlugin) { // log an error, it's most probably not normal log::error("no plugin for game '{}', an antivirus might have deleted it", game); continue; } if (nxmInfo.game().compare(gamePlugin->gameShortName(), Qt::CaseInsensitive) == 0 || nxmInfo.game().compare(gamePlugin->gameNexusName(), Qt::CaseInsensitive) == 0) { foundGame = gamePlugin; break; } } log::debug("add nxm download: {}", url); // The Nexus API game name to use for requests (may differ from foundGame's short name // for special domains like "site" that have no matching game plugin). QString apiGameName; if (foundGame == nullptr) { // "site" is Nexus Mods' game-agnostic domain for tools/utilities (e.g. BethINI Pie). // Allow these downloads using the current managed game's API context. if (nxmInfo.game().compare("site", Qt::CaseInsensitive) == 0) { log::debug("NXM link is for nexusmods.com/site (game-agnostic tool), allowing download"); apiGameName = nxmInfo.game(); // keep "site" for correct API URLs } else { log::debug("download requested for wrong game (game: {}, url: {})", m_ManagedGame->gameShortName(), nxmInfo.game()); QMessageBox::information( m_ParentWidget, tr("Wrong Game"), tr("The download link is for a mod for \"%1\" but this instance of MO " "has been set up for \"%2\".") .arg(nxmInfo.game()) .arg(m_ManagedGame->gameShortName()), QMessageBox::Ok); return; } } else { apiGameName = foundGame->gameShortName(); } for (auto tuple : m_PendingDownloads) { if (std::get<0>(tuple).compare(apiGameName, Qt::CaseInsensitive) == 0 && std::get<1>(tuple) == nxmInfo.modId() && std::get<2>(tuple) == nxmInfo.fileId()) { const auto infoStr = tr("There is already a download queued for this file.\n\nMod %1\nFile %2") .arg(nxmInfo.modId()) .arg(nxmInfo.fileId()); log::debug("download requested is already queued (mod: {}, file: {})", nxmInfo.modId(), nxmInfo.fileId()); QMessageBox::information(m_ParentWidget, tr("Already Queued"), infoStr, QMessageBox::Ok); return; } } for (DownloadInfo* download : m_ActiveDownloads) { if (download->m_FileInfo->modID == nxmInfo.modId() && download->m_FileInfo->fileID == nxmInfo.fileId()) { if (download->m_State == STATE_DOWNLOADING || download->m_State == STATE_PAUSED || download->m_State == STATE_STARTED) { QString debugStr( "download requested is already started (mod %1: %2, file %3: %4)"); QString infoStr(tr("There is already a download started for this file.\n\nMod " "%1:\t%2\nFile %3:\t%4")); // %1 debugStr = debugStr.arg(download->m_FileInfo->modID); infoStr = infoStr.arg(download->m_FileInfo->modID); // %2 if (!download->m_FileInfo->name.isEmpty()) { debugStr = debugStr.arg(download->m_FileInfo->name); infoStr = infoStr.arg(download->m_FileInfo->name); } else if (!download->m_FileInfo->modName.isEmpty()) { debugStr = debugStr.arg(download->m_FileInfo->modName); infoStr = infoStr.arg(download->m_FileInfo->modName); } else { debugStr = debugStr.arg(QStringLiteral("")); infoStr = infoStr.arg(QStringLiteral("")); } // %3 debugStr = debugStr.arg(download->m_FileInfo->fileID); infoStr = infoStr.arg(download->m_FileInfo->fileID); // %4 if (!download->m_FileInfo->fileName.isEmpty()) { debugStr = debugStr.arg(download->m_FileInfo->fileName); infoStr = infoStr.arg(download->m_FileInfo->fileName); } else if (!download->m_FileName.isEmpty()) { debugStr = debugStr.arg(download->m_FileName); infoStr = infoStr.arg(download->m_FileName); } else { debugStr = debugStr.arg(QStringLiteral("")); infoStr = infoStr.arg(QStringLiteral("")); } log::debug("{}", debugStr); QMessageBox::information(m_ParentWidget, tr("Already Started"), infoStr, QMessageBox::Ok); return; } } } emit aboutToUpdate(); m_PendingDownloads.append( std::make_tuple(apiGameName, nxmInfo.modId(), nxmInfo.fileId())); emit update(-1); emit downloadAdded(); ModRepositoryFileInfo* info = new ModRepositoryFileInfo(); info->nexusKey = nxmInfo.key(); info->nexusExpires = nxmInfo.expires(); info->nexusDownloadUser = nxmInfo.userId(); QObject* test = info; m_RequestIDs.insert(m_NexusInterface->requestFileInfo( apiGameName, nxmInfo.modId(), nxmInfo.fileId(), this, QVariant::fromValue(test), "")); } void DownloadManager::removeFile(int index, bool deleteFile) { // Avoid triggering refreshes from DirWatcher ScopedDisableDirWatcher const scopedDirWatcher(this); if (index >= m_ActiveDownloads.size()) { throw MyException(tr("remove: invalid download index %1").arg(index)); } DownloadInfo* download = m_ActiveDownloads.at(index); QString filePath = m_OutputDirectory + "/" + download->m_FileName; if ((download->m_State == STATE_STARTED) || (download->m_State == STATE_DOWNLOADING)) { // shouldn't have been possible log::error("tried to remove active download"); return; } if ((download->m_State == STATE_PAUSED) || (download->m_State == STATE_ERROR)) { filePath = download->m_Output.fileName(); } if (deleteFile) { if (!shellDelete(QStringList(filePath), true)) { reportError(tr("failed to delete %1").arg(filePath)); return; } QFile const metaFile(filePath.append(".meta")); if (metaFile.exists() && !shellDelete(QStringList(filePath), true)) { reportError(tr("failed to delete meta file for %1").arg(filePath)); } } else { QSettings metaSettings(filePath.append(".meta"), QSettings::IniFormat); if (!download->m_Hidden) metaSettings.setValue("removed", true); } m_DownloadRemoved(index); } class LessThanWrapper { public: LessThanWrapper(DownloadManager* manager) : m_Manager(manager) {} bool operator()(int LHS, int RHS) { return m_Manager->getFileName(LHS).compare(m_Manager->getFileName(RHS), Qt::CaseInsensitive) < 0; } private: DownloadManager* m_Manager; }; bool DownloadManager::ByName(int LHS, int RHS) { return m_ActiveDownloads[LHS]->m_FileName < m_ActiveDownloads[RHS]->m_FileName; } void DownloadManager::refreshAlphabeticalTranslation() { m_AlphabeticalTranslation.clear(); int pos = 0; for (QVector::iterator iter = m_ActiveDownloads.begin(); iter != m_ActiveDownloads.end(); ++iter, ++pos) { m_AlphabeticalTranslation.push_back(pos); } std::sort(m_AlphabeticalTranslation.begin(), m_AlphabeticalTranslation.end(), LessThanWrapper(this)); } void DownloadManager::restoreDownload(int index) { if (index < 0) { DownloadState const minState = STATE_READY; index = 0; for (QVector::const_iterator iter = m_ActiveDownloads.begin(); iter != m_ActiveDownloads.end(); ++iter) { if ((*iter)->m_State >= minState) { restoreDownload(index); } index++; } } else { if (index >= m_ActiveDownloads.size()) { throw MyException(tr("restore: invalid download index: %1").arg(index)); } DownloadInfo* download = m_ActiveDownloads.at(index); if (download->m_Hidden) { download->m_Hidden = false; QString filePath = m_OutputDirectory + "/" + download->m_FileName; // avoid dirWatcher triggering refreshes startDisableDirWatcher(); QSettings metaSettings(filePath.append(".meta"), QSettings::IniFormat); metaSettings.setValue("removed", false); endDisableDirWatcher(); } } } void DownloadManager::removeDownload(int index, bool deleteFile) { try { // avoid dirWatcher triggering refreshes ScopedDisableDirWatcher const scopedDirWatcher(this); emit aboutToUpdate(); if (index < 0) { bool const removeAll = (index == -1); DownloadState const removeState = (index == -2 ? STATE_INSTALLED : STATE_UNINSTALLED); index = 0; for (QVector::iterator iter = m_ActiveDownloads.begin(); iter != m_ActiveDownloads.end();) { DownloadState const downloadState = (*iter)->m_State; if ((removeAll && (downloadState >= STATE_READY)) || (removeState == downloadState)) { removeFile(index, deleteFile); m_ByID.remove((*iter)->m_DownloadID); delete *iter; iter = m_ActiveDownloads.erase(iter); } else { ++iter; ++index; } } } else { if (index >= m_ActiveDownloads.size()) { reportError(tr("remove: invalid download index %1").arg(index)); // emit update(-1); return; } removeFile(index, deleteFile); m_ByID.remove(m_ActiveDownloads.at(index)->m_DownloadID); delete m_ActiveDownloads.at(index); m_ActiveDownloads.erase(m_ActiveDownloads.begin() + index); } emit update(-1); } catch (const std::exception& e) { log::error("failed to remove download: {}", e.what()); } refreshList(); } void DownloadManager::cancelDownload(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("cancel: invalid download index %1").arg(index)); return; } DownloadInfo* info = m_ActiveDownloads.at(index); if (info->m_State == STATE_DOWNLOADING || info->m_State == STATE_STARTED || info->m_State == STATE_PAUSING) { setState(info, STATE_CANCELING); } } void DownloadManager::pauseDownload(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("pause: invalid download index %1").arg(index)); return; } DownloadInfo* info = m_ActiveDownloads.at(index); if (info->m_State == STATE_DOWNLOADING) { if ((info->m_Reply != nullptr) && (info->m_Reply->isRunning())) { setState(info, STATE_PAUSING); } else { setState(info, STATE_PAUSED); } } else if ((info->m_State == STATE_FETCHINGMODINFO) || (info->m_State == STATE_FETCHINGFILEINFO) || (info->m_State == STATE_FETCHINGMODINFO_MD5)) { setState(info, STATE_READY); } } void DownloadManager::resumeDownload(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("resume: invalid download index %1").arg(index)); return; } DownloadInfo* info = m_ActiveDownloads[index]; info->m_Tries = AUTOMATIC_RETRIES; resumeDownloadInt(index); } void DownloadManager::resumeDownloadInt(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("resume (int): invalid download index %1").arg(index)); return; } DownloadInfo* info = m_ActiveDownloads[index]; // Check for finished download; if (info->m_TotalSize <= info->m_Output.size() && info->m_Reply != nullptr && info->m_Reply->isFinished() && info->m_State != STATE_ERROR) { setState(info, STATE_DOWNLOADING); downloadFinished(index); return; } if (info->isPausedState() || info->m_State == STATE_PAUSING) { if (info->m_State == STATE_PAUSING) { if (info->m_Output.isOpen()) { writeData(info); if (info->m_State == STATE_PAUSING) { setState(info, STATE_PAUSED); } } } if ((info->m_Urls.empty()) || ((info->m_Urls.size() == 1) && (info->m_Urls[0].size() == 0))) { emit showMessage( tr("No known download urls. Sorry, this download can't be resumed.")); return; } if (info->m_State == STATE_ERROR) { info->m_CurrentUrl = (info->m_CurrentUrl + 1) % info->m_Urls.count(); } log::debug("request resume from url {}", info->currentURL()); QNetworkRequest request(QUrl::fromEncoded(info->currentURL().toLocal8Bit())); request.setHeader(QNetworkRequest::UserAgentHeader, m_NexusInterface->getAccessManager()->userAgent()); if (info->m_State != STATE_ERROR) { info->m_ResumePos = info->m_Output.size(); QByteArray const rangeHeader = "bytes=" + QByteArray::number(info->m_ResumePos) + "-"; request.setRawHeader("Range", rangeHeader); } info->m_DownloadLast = 0; info->m_DownloadTimeLast = 0; info->m_DownloadAcc = accumulator_set>( tag::rolling_window::window_size = 200); info->m_DownloadTimeAcc = accumulator_set>( tag::rolling_window::window_size = 200); log::debug("resume at {} bytes", info->m_ResumePos); startDownload(m_NexusInterface->getAccessManager()->get(request), info, true); } emit update(index); } DownloadManager::DownloadInfo* DownloadManager::downloadInfoByID(unsigned int id) { // O(1) via m_ByID; the prior linear scan over m_ActiveDownloads couldn't // distinguish "never existed" from "valid but waiting on a callback that // already erased it," so late callbacks would dereference freed memory. // Callers treat nullptr as the canonical "stale ID" signal. return m_ByID.value(id, nullptr); } void DownloadManager::queryInfo(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("query: invalid download index %1").arg(index)); return; } DownloadInfo* info = m_ActiveDownloads[index]; if (info->m_FileInfo->repository != "Nexus") { log::warn("re-querying file info is currently only possible with Nexus"); return; } if (info->m_State < DownloadManager::STATE_READY) { // UI shouldn't allow this return; } if (info->m_FileInfo->modID <= 0) { QString const fileName = getFileName(index); QString ignore; NexusInterface::interpretNexusFileName(fileName, ignore, info->m_FileInfo->modID, info->m_AskIfNotFound); if (info->m_FileInfo->modID < 0) { bool ok = false; int const modId = QInputDialog::getInt(nullptr, tr("Please enter the Nexus mod ID"), tr("Mod ID:"), 1, 1, std::numeric_limits::max(), 1, &ok); // careful now: while the dialog was displayed, events were processed. // the download list might have changed and our info-ptr invalidated. if (ok) m_ActiveDownloads[index]->m_FileInfo->modID = modId; return; } } if (info->m_FileInfo->gameName.isEmpty()) { if (!info->m_AskIfNotFound) { // in case the info couldn't be retrieved from Nexus due to connection issues, // don't create a meta file so you can query again after the issues are resolved. return; } SelectionDialog selection( tr("Please select the source game code for %1").arg(getFileName(index))); std::vector> choices = m_NexusInterface->getGameChoices(m_ManagedGame); if (choices.size() == 1) { info->m_FileInfo->gameName = choices[0].first; } else { for (const auto& choice : choices) { selection.addChoice(choice.first, choice.second, choice.first); } if (selection.exec() == QDialog::Accepted) { info->m_FileInfo->gameName = selection.getChoiceData().toString(); } else { info->m_FileInfo->gameName = m_ManagedGame->gameShortName(); } } } info->m_ReQueried = true; setState(info, STATE_FETCHINGMODINFO); } void DownloadManager::queryInfoMd5(int index, bool askIfNotFound) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("query: invalid download index %1").arg(index)); return; } DownloadInfo* info = m_ActiveDownloads[index]; if (info->m_FileInfo->repository != "Nexus") { log::warn("re-querying file info is currently only possible with Nexus"); return; } if (info->m_State < DownloadManager::STATE_READY) { // UI shouldn't allow this return; } info->m_GamesToQuery << m_ManagedGame->gameShortName(); info->m_GamesToQuery << m_ManagedGame->validShortNames(); QFile downloadFile(info->m_FileName); if (!downloadFile.exists()) { downloadFile.setFileName(m_OrganizerCore->downloadsPath() + "/" + info->m_FileName); } if (!downloadFile.exists()) { log::error("Can't find download file '{}'", info->m_FileName); return; } if (!downloadFile.open(QIODevice::ReadOnly)) { log::error("Can't open download file '{}'", info->m_FileName); return; } QCryptographicHash hash(QCryptographicHash::Md5); const qint64 progressStep = 10 * 1024 * 1024; QProgressDialog progress(tr("Hashing download file '%1'").arg(info->m_FileName), tr("Cancel"), 0, downloadFile.size() / progressStep); progress.setWindowModality(Qt::WindowModal); progress.setMinimumDuration(1000); for (qint64 i = 0; i < downloadFile.size(); i += progressStep) { progress.setValue(progress.value() + 1); if (progress.wasCanceled()) { break; } hash.addData(downloadFile.read(progressStep)); } if (progress.wasCanceled()) { downloadFile.close(); return; } progress.close(); downloadFile.close(); info->m_Hash = hash.result(); info->m_ReQueried = true; info->m_AskIfNotFound = askIfNotFound; setState(info, STATE_FETCHINGMODINFO_MD5); } void DownloadManager::visitOnNexus(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("VisitNexus: invalid download index %1").arg(index)); return; } DownloadInfo* info = m_ActiveDownloads[index]; if (info->m_FileInfo->repository != "Nexus") { log::warn("Visiting mod page is currently only possible with Nexus"); return; } if (info->m_State < DownloadManager::STATE_READY) { // UI shouldn't allow this return; } int const modID = info->m_FileInfo->modID; QString const gameName = info->m_FileInfo->gameName; if (modID > 0) { shell::Open(QUrl(m_NexusInterface->getModURL(modID, gameName))); } else { emit showMessage(tr("Nexus ID for this Mod is unknown")); } } void DownloadManager::visitUploaderProfile(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("VisitUploaderProfile: invalid download index %1").arg(index)); return; } DownloadInfo* info = m_ActiveDownloads[index]; if (info->m_State < DownloadManager::STATE_READY) { // UI shouldn't allow this return; } const auto& uploaderUrl = info->m_FileInfo->uploaderUrl; if (!uploaderUrl.isEmpty()) { shell::Open(QUrl(uploaderUrl)); } else { emit showMessage(tr("Uploader for this Mod is unknown")); } } void DownloadManager::openFile(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("OpenFile: invalid download index %1").arg(index)); return; } QDir const path = QDir(m_OutputDirectory); if (path.exists(getFileName(index))) { shell::Open(getFilePath(index)); return; } shell::Explore(m_OutputDirectory); return; } void DownloadManager::openMetaFile(int index) { if (index < 0 || index >= m_ActiveDownloads.size()) { log::error("OpenMetaFile: invalid download index {}", index); return; } const auto path = QDir(m_OutputDirectory); const auto filePath = getFilePath(index); const auto metaPath = filePath + ".meta"; if (path.exists(metaPath)) { shell::Open(metaPath); return; } else { QSettings metaFile(metaPath, QSettings::IniFormat); metaFile.setValue("removed", false); } if (path.exists(metaPath)) { shell::Open(metaPath); return; } shell::Explore(filePath); } void DownloadManager::openInDownloadsFolder(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { reportError(tr("OpenFileInDownloadsFolder: invalid download index %1").arg(index)); return; } const auto path = getFilePath(index); if (QFile::exists(path)) { shell::Explore(path); return; } else { const auto unfinished = path + ".unfinished"; if (QFile::exists(unfinished)) { shell::Explore(unfinished); return; } } shell::Explore(m_OutputDirectory); } int DownloadManager::numTotalDownloads() const { return m_ActiveDownloads.size(); } int DownloadManager::numPendingDownloads() const { return m_PendingDownloads.size(); } std::tuple DownloadManager::getPendingDownload(int index) { if ((index < 0) || (index >= m_PendingDownloads.size())) { throw MyException(tr("get pending: invalid download index %1").arg(index)); } return m_PendingDownloads.at(index); } QString DownloadManager::getFilePath(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("get path: invalid download index %1").arg(index)); } return m_OutputDirectory + "/" + m_ActiveDownloads.at(index)->m_FileName; } QString DownloadManager::getFileTypeString(int fileType) { switch (fileType) { case 1: return tr("Main"); case 2: return tr("Update"); case 3: return tr("Optional"); case 4: return tr("Old"); case 5: return tr("Miscellaneous"); case 6: return tr("Deleted"); case 7: return tr("Archived"); default: return tr("Unknown"); } } QString DownloadManager::getDisplayName(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("display name: invalid download index %1").arg(index)); } return getDisplayNameForInfo(m_ActiveDownloads.at(index)); } QString DownloadManager::getDisplayNameForInfo(const DownloadInfo* info) const { QTextDocument doc; if (!info->m_FileInfo->name.isEmpty()) { doc.setHtml(info->m_FileInfo->name); return QString("%1 (%2, v%3)") .arg(doc.toPlainText()) .arg(getFileTypeString(info->m_FileInfo->fileCategory)) .arg(info->m_FileInfo->version.displayString()); } else { doc.setHtml(info->m_FileName); return doc.toPlainText(); } } QString DownloadManager::getFileName(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("file name: invalid download index %1").arg(index)); } return m_ActiveDownloads.at(index)->m_FileName; } QDateTime DownloadManager::getFileTime(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("file time: invalid download index %1").arg(index)); } DownloadInfo* info = m_ActiveDownloads.at(index); if (!info->m_Created.isValid()) { QFileInfo const fileInfo(info->m_Output); info->m_Created = fileInfo.birthTime(); if (!info->m_Created.isValid()) info->m_Created = fileInfo.metadataChangeTime(); if (!info->m_Created.isValid()) info->m_Created = fileInfo.lastModified(); } return info->m_Created; } qint64 DownloadManager::getFileSize(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("file size: invalid download index %1").arg(index)); } return m_ActiveDownloads.at(index)->m_TotalSize; } std::pair DownloadManager::getProgress(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("progress: invalid download index %1").arg(index)); } return m_ActiveDownloads.at(index)->m_Progress; } DownloadManager::DownloadState DownloadManager::getState(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("state: invalid download index %1").arg(index)); } return m_ActiveDownloads.at(index)->m_State; } bool DownloadManager::isInfoIncomplete(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("infocomplete: invalid download index %1").arg(index)); } DownloadInfo* info = m_ActiveDownloads.at(index); if (info->m_FileInfo->repository != "Nexus") { // other repositories currently don't support re-querying info anyway return false; } return (info->m_FileInfo->fileID == 0) || (info->m_FileInfo->modID == 0); } int DownloadManager::getModID(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("mod id: invalid download index %1").arg(index)); } return m_ActiveDownloads.at(index)->m_FileInfo->modID; } QString DownloadManager::getDisplayGameName(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("mod id: invalid download index %1").arg(index)); } QString gameName = m_ActiveDownloads.at(index)->m_FileInfo->gameName; IPluginGame* gamePlugin = m_OrganizerCore->getGame(gameName); if (gamePlugin) { gameName = gamePlugin->gameName(); } return gameName; } QString DownloadManager::getGameName(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("mod id: invalid download index %1").arg(index)); } return m_ActiveDownloads.at(index)->m_FileInfo->gameName; } bool DownloadManager::isHidden(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("ishidden: invalid download index %1").arg(index)); } return m_ActiveDownloads.at(index)->m_Hidden; } const ModRepositoryFileInfo* DownloadManager::getFileInfo(int index) const { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("file info: invalid download index %1").arg(index)); } return m_ActiveDownloads.at(index)->m_FileInfo; } void DownloadManager::markInstalled(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("mark installed: invalid download index %1").arg(index)); } // Avoid triggering refreshes from DirWatcher ScopedDisableDirWatcher const scopedDirWatcher(this); DownloadInfo* info = m_ActiveDownloads.at(index); QSettings metaFile(info->m_Output.fileName() + ".meta", QSettings::IniFormat); metaFile.setValue("installed", true); metaFile.setValue("uninstalled", false); setState(m_ActiveDownloads.at(index), STATE_INSTALLED); } void DownloadManager::markInstalled(QString fileName) { int const index = indexByName(fileName); if (index >= 0) { markInstalled(index); } else { DownloadInfo* info = getDownloadInfo(fileName); if (info != nullptr) { // Avoid triggering refreshes from DirWatcher ScopedDisableDirWatcher const scopedDirWatcher(this); QSettings metaFile(info->m_Output.fileName() + ".meta", QSettings::IniFormat); metaFile.setValue("installed", true); metaFile.setValue("uninstalled", false); delete info; } } } DownloadManager::DownloadInfo* DownloadManager::getDownloadInfo(QString fileName) { return DownloadInfo::createFromMeta(fileName, true, m_OutputDirectory); } void DownloadManager::markUninstalled(int index) { if ((index < 0) || (index >= m_ActiveDownloads.size())) { throw MyException(tr("mark uninstalled: invalid download index %1").arg(index)); } // Avoid triggering refreshes from DirWatcher ScopedDisableDirWatcher const scopedDirWatcher(this); DownloadInfo* info = m_ActiveDownloads.at(index); QSettings metaFile(info->m_Output.fileName() + ".meta", QSettings::IniFormat); metaFile.setValue("uninstalled", true); setState(m_ActiveDownloads.at(index), STATE_UNINSTALLED); } void DownloadManager::markUninstalled(QString fileName) { int const index = indexByName(fileName); if (index >= 0) { markUninstalled(index); } else { QString const filePath = QDir::fromNativeSeparators(m_OutputDirectory) + "/" + fileName; DownloadInfo* info = getDownloadInfo(filePath); if (info != nullptr) { // Avoid triggering refreshes from DirWatcher ScopedDisableDirWatcher const scopedDirWatcher(this); QSettings metaFile(info->m_Output.fileName() + ".meta", QSettings::IniFormat); metaFile.setValue("uninstalled", true); delete info; } } } QString DownloadManager::getDownloadFileName(const QString& baseName, bool rename) const { QString fullPath = m_OutputDirectory + "/" + MOBase::sanitizeFileName(baseName); if (QFile::exists(fullPath) && rename) { int i = 1; while (QFile::exists( QString("%1/%2_%3").arg(m_OutputDirectory).arg(i).arg(baseName))) { ++i; } fullPath = QString("%1/%2_%3").arg(m_OutputDirectory).arg(i).arg(baseName); } return fullPath; } QString DownloadManager::getFileNameFromNetworkReply(QNetworkReply* reply) { if (reply->hasRawHeader("Content-Disposition")) { std::regex const exp("filename=\"(.+)\""); std::cmatch result; if (std::regex_search(reply->rawHeader("Content-Disposition").constData(), result, exp)) { return MOBase::sanitizeFileName(QString::fromUtf8(result.str(1).c_str())); } } return {}; } void DownloadManager::setState(DownloadManager::DownloadInfo* info, DownloadManager::DownloadState state) { int row = 0; for (int i = 0; i < m_ActiveDownloads.size(); ++i) { if (m_ActiveDownloads[i] == info) { row = i; break; } } info->m_State = state; switch (state) { case STATE_CANCELING: { // Force termination so the download transitions through finished(). if (info->m_Reply != nullptr && info->m_Reply->isRunning()) { info->m_Reply->abort(); } } break; case STATE_PAUSED: { if (info->m_Reply != nullptr) { info->m_Reply->abort(); } info->m_Output.close(); m_DownloadPaused(row); } break; case STATE_ERROR: { if (info->m_Reply != nullptr) { info->m_Reply->abort(); } info->m_Output.close(); m_DownloadFailed(row); } break; case STATE_CANCELED: { if (info->m_Reply != nullptr) { info->m_Reply->abort(); } m_DownloadFailed(row); } break; case STATE_FETCHINGMODINFO: { m_RequestIDs.insert(m_NexusInterface->requestDescription( info->m_FileInfo->gameName, info->m_FileInfo->modID, this, info->m_DownloadID, QString())); } break; case STATE_FETCHINGFILEINFO: { m_RequestIDs.insert(m_NexusInterface->requestFiles(info->m_FileInfo->gameName, info->m_FileInfo->modID, this, info->m_DownloadID, QString())); } break; case STATE_FETCHINGMODINFO_MD5: { log::debug("Searching {} for MD5 of {}", info->m_GamesToQuery[0], QString(info->m_Hash.toHex())); m_RequestIDs.insert(m_NexusInterface->requestInfoFromMd5( info->m_GamesToQuery[0], info->m_Hash, this, info->m_DownloadID, QString())); } break; case STATE_READY: { createMetaFile(info); m_DownloadComplete(row); } break; default: /* NOP */ break; } emit stateChanged(row, state); } DownloadManager::DownloadInfo* DownloadManager::findDownload(QObject* reply, int* index) const { // reverse search as newer, thus more relevant, downloads are at the end for (int i = m_ActiveDownloads.size() - 1; i >= 0; --i) { if (m_ActiveDownloads[i]->m_Reply == reply) { if (index != nullptr) { *index = i; } return m_ActiveDownloads[i]; } } return nullptr; } void DownloadManager::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) { if (bytesTotal == 0) { return; } int index = 0; try { DownloadInfo* info = findDownload(this->sender(), &index); if (info != nullptr) { info->m_HasData = true; if (info->m_State == STATE_CANCELING) { setState(info, STATE_CANCELED); } else if (info->m_State == STATE_PAUSING) { setState(info, STATE_PAUSED); } else { if (bytesTotal > info->m_TotalSize) { info->m_TotalSize = bytesTotal; } info->m_Progress.first = ((info->m_ResumePos + bytesReceived) * 100) / (info->m_ResumePos + bytesTotal); qint64 const elapsed = info->m_StartTime.elapsed(); info->m_DownloadAcc(bytesReceived - info->m_DownloadLast); info->m_DownloadLast = bytesReceived; info->m_DownloadTimeAcc(elapsed - info->m_DownloadTimeLast); info->m_DownloadTimeLast = elapsed; // calculate the download speed const double speed = rolling_mean(info->m_DownloadAcc) / (rolling_mean(info->m_DownloadTimeAcc) / 1000.0); ; const qint64 remaining = (bytesTotal - bytesReceived) / speed * 1000; info->m_Progress.second = tr("%1% - %2 - ~%3") .arg(info->m_Progress.first) .arg(MOBase::localizedByteSpeed(speed)) .arg(MOBase::localizedTimeRemaining(remaining)); TaskProgressManager::instance().updateProgress(info->m_TaskProgressId, bytesReceived, bytesTotal); emit update(index); } } } catch (const std::bad_alloc&) { reportError(tr("Memory allocation error (in processing progress event).")); } } void DownloadManager::downloadReadyRead() { try { writeData(findDownload(this->sender())); } catch (const std::bad_alloc&) { reportError(tr("Memory allocation error (in processing downloaded data).")); } } void DownloadManager::createMetaFile(DownloadInfo* info) { // Avoid triggering refreshes from DirWatcher ScopedDisableDirWatcher const scopedDirWatcher(this); QSettings metaFile(QString("%1.meta").arg(info->m_Output.fileName()), QSettings::IniFormat); metaFile.setValue("gameName", info->m_FileInfo->gameName); metaFile.setValue("modID", info->m_FileInfo->modID); metaFile.setValue("fileID", info->m_FileInfo->fileID); metaFile.setValue("url", info->m_Urls.join(";")); metaFile.setValue("name", info->m_FileInfo->name); metaFile.setValue("description", info->m_FileInfo->description); metaFile.setValue("modName", info->m_FileInfo->modName); metaFile.setValue("version", info->m_FileInfo->version.canonicalString()); metaFile.setValue("newestVersion", info->m_FileInfo->newestVersion.canonicalString()); metaFile.setValue("fileTime", info->m_FileInfo->fileTime); metaFile.setValue("fileCategory", info->m_FileInfo->fileCategory); metaFile.setValue("category", info->m_FileInfo->categoryID); metaFile.setValue("repository", info->m_FileInfo->repository); metaFile.setValue("userData", info->m_FileInfo->userData); metaFile.setValue("author", info->m_FileInfo->author); metaFile.setValue("uploader", info->m_FileInfo->uploader); metaFile.setValue("uploaderUrl", info->m_FileInfo->uploaderUrl); metaFile.setValue("installed", info->m_State == DownloadManager::STATE_INSTALLED); metaFile.setValue("uninstalled", info->m_State == DownloadManager::STATE_UNINSTALLED); metaFile.setValue("paused", (info->m_State == DownloadManager::STATE_PAUSED) || (info->m_State == DownloadManager::STATE_ERROR)); metaFile.setValue("removed", info->m_Hidden); // slightly hackish... for (int i = 0; i < m_ActiveDownloads.size(); ++i) { if (m_ActiveDownloads[i] == info) { emit update(i); } } } void DownloadManager::nxmDescriptionAvailable(QString, int, QVariant userData, QVariant resultData, int requestID) { std::set::iterator const idIter = m_RequestIDs.find(requestID); if (idIter == m_RequestIDs.end()) { return; } else { m_RequestIDs.erase(idIter); } QVariantMap result = resultData.toMap(); DownloadInfo* info = downloadInfoByID(userData.toInt()); if (info == nullptr) return; info->m_FileInfo->categoryID = result["category_id"].toInt(); info->m_FileInfo->author = result["author"].toString(); info->m_FileInfo->uploader = result["uploaded_by"].toString(); info->m_FileInfo->uploaderUrl = result["uploaded_users_profile_url"].toString(); QTextDocument doc; doc.setHtml(result["name"].toString().trimmed()); info->m_FileInfo->modName = doc.toPlainText(); if (info->m_FileInfo->fileID != 0) { setState(info, STATE_READY); if (m_OrganizerCore->settings().interface().showDownloadNotifications()) { m_OrganizerCore->showNotification( tr("Download complete"), tr("%1 is ready to be installed.").arg(getDisplayNameForInfo(info))); } } else { setState(info, STATE_FETCHINGFILEINFO); } } void DownloadManager::nxmFilesAvailable(QString, int, QVariant userData, QVariant resultData, int requestID) { std::set::iterator const idIter = m_RequestIDs.find(requestID); if (idIter == m_RequestIDs.end()) { return; } else { m_RequestIDs.erase(idIter); } DownloadInfo* info = downloadInfoByID(userData.toInt()); if (info == nullptr) return; QVariantMap result = resultData.toMap(); QVariantList files = result["files"].toList(); // MO sometimes prepends _ to the filename in case of duplicate downloads. // this may muck up the file name comparison QString alternativeLocalName = info->m_FileName; QRegularExpression const expression("^\\d_(.*)$"); auto match = expression.match(alternativeLocalName); if (match.hasMatch()) { alternativeLocalName = match.captured(1); } // Copy the matched Nexus file metadata onto info. Used by both passes // below so the fileID-priority path and the filename-fallback path stay // consistent. auto applyMatch = [&info](const QVariantMap& fileInfo) { info->m_FileInfo->name = fileInfo["name"].toString(); info->m_FileInfo->version.parse(fileInfo["version"].toString()); if (!info->m_FileInfo->version.isValid()) { info->m_FileInfo->version = info->m_FileInfo->newestVersion; } info->m_FileInfo->fileCategory = fileInfo["category_id"].toInt(); info->m_FileInfo->fileTime = QDateTime::fromMSecsSinceEpoch(fileInfo["uploaded_timestamp"].toLongLong()); info->m_FileInfo->fileID = fileInfo["file_id"].toInt(); info->m_FileInfo->fileName = fileInfo["file_name"].toString(); info->m_FileInfo->description = BBCode::convertToHTML(fileInfo["description"].toString()); info->m_FileInfo->author = fileInfo["author"].toString(); info->m_FileInfo->uploader = fileInfo["uploaded_by"].toString(); info->m_FileInfo->uploaderUrl = fileInfo["uploaded_users_profile_url"].toString(); }; bool found = false; // Pass 1: prefer matching by Nexus file_id when we already have one. // file_id is canonical; filename can disagree after a local rename or // a Nexus-side filename change. if (info->m_FileInfo->fileID != 0) { for (const QVariant& file : files) { QVariantMap const fileInfo = file.toMap(); if (fileInfo["file_id"].toInt() == info->m_FileInfo->fileID) { applyMatch(fileInfo); found = true; break; } } } // Pass 2: fall back to filename matching (also the only path when we // don't have a usable fileID yet). if (!found) { for (const QVariant& file : files) { QVariantMap fileInfo = file.toMap(); QString const fileName = fileInfo["file_name"].toString(); QString const fileNameVariant = fileName.mid(0).replace(' ', '_'); if ((fileName == info->m_RemoteFileName) || (fileNameVariant == info->m_RemoteFileName) || (fileName == info->m_FileName) || (fileNameVariant == info->m_FileName) || (fileName == alternativeLocalName) || (fileNameVariant == alternativeLocalName)) { applyMatch(fileInfo); found = true; break; } } } if (info->m_ReQueried) { if (found) { emit showMessage(tr("Information updated")); } else if (result.count() == 0) { emit showMessage(tr("No matching file found on Nexus! Maybe this file is no " "longer available or it was renamed?")); } else { SelectionDialog selection(tr("No file on Nexus matches the selected file by " "name. Please manually choose the correct one.")); std::sort(files.begin(), files.end(), [](const QVariant& lhs, const QVariant& rhs) { return lhs.toMap()["uploaded_timestamp"].toInt() > rhs.toMap()["uploaded_timestamp"].toInt(); }); for (const QVariant& file : files) { QVariantMap fileInfo = file.toMap(); if (fileInfo["category_id"].toInt() != NexusInterface::FileStatus::REMOVED && fileInfo["category_id"].toInt() != NexusInterface::FileStatus::ARCHIVED) selection.addChoice(fileInfo["file_name"].toString(), "", file); } if (selection.exec() == QDialog::Accepted) { QVariantMap fileInfo = selection.getChoiceData().toMap(); info->m_FileInfo->name = fileInfo["name"].toString(); info->m_FileInfo->version.parse(fileInfo["version"].toString()); info->m_FileInfo->fileCategory = fileInfo["category_id"].toInt(); info->m_FileInfo->fileID = fileInfo["file_id"].toInt(); } else { emit showMessage(tr("No matching file found on Nexus! Maybe this file is no " "longer available or it was renamed?")); } } } else { if (info->m_FileInfo->fileID == 0) { log::warn("could not determine file id for {} (state {})", info->m_FileName, info->m_State); } } setState(info, STATE_READY); } void DownloadManager::nxmFileInfoAvailable(QString gameName, int modID, int fileID, QVariant userData, QVariant resultData, int requestID) { std::set::iterator const idIter = m_RequestIDs.find(requestID); if (idIter == m_RequestIDs.end()) { return; } else { m_RequestIDs.erase(idIter); } ModRepositoryFileInfo* info = qobject_cast(qvariant_cast(userData)); QVariantMap result = resultData.toMap(); info->name = result["name"].toString(); info->version.parse(result["version"].toString()); if (!info->version.isValid()) { info->version = info->newestVersion; } info->fileName = result["file_name"].toString(); info->fileCategory = result["category_id"].toInt(); info->fileTime = QDateTime::fromMSecsSinceEpoch(result["uploaded_timestamp"].toLongLong()); info->description = BBCode::convertToHTML(result["description"].toString()); info->repository = "Nexus"; // Default to the raw game name; overridden below if a matching plugin is found. // This preserves special domains like "site" that have no game plugin. info->gameName = gameName; QStringList games(m_ManagedGame->validShortNames()); games += m_ManagedGame->gameShortName(); for (const auto& game : games) { MOBase::IPluginGame* gamePlugin = m_OrganizerCore->getGame(game); if (gamePlugin != nullptr && gamePlugin->gameNexusName().compare(gameName, Qt::CaseInsensitive) == 0) { info->gameName = gamePlugin->gameShortName(); break; } } info->modID = modID; info->fileID = fileID; QObject* test = info; m_RequestIDs.insert( m_NexusInterface->requestDownloadURL(info->gameName, info->modID, info->fileID, this, QVariant::fromValue(test), QString())); } static int evaluateFileInfoMap(const QVariantMap& map, const ServerList::container& preferredServers) { int preference = 0; bool found = false; const auto name = map["short_name"].toString(); for (const auto& server : preferredServers) { if (server.name() == name) { preference = server.preferred(); found = true; break; } } if (!found) { return 0; } return 100 + preference * 20; } // sort function to sort by best download server // bool ServerByPreference(const ServerList::container& preferredServers, const QVariant& LHS, const QVariant& RHS) { const auto a = evaluateFileInfoMap(LHS.toMap(), preferredServers); const auto b = evaluateFileInfoMap(RHS.toMap(), preferredServers); return (a > b); } int DownloadManager::startDownloadURLs(const QStringList& urls) { // Reserve the download ID up front so the caller (plugin via IOrganizer) // gets the canonical ID rather than a stale m_ActiveDownloads index. const unsigned int reservedID = DownloadInfo::newDownloadID(); ModRepositoryFileInfo const info; if (!addDownload(urls, "", -1, -1, &info, reservedID)) { return 0; } return static_cast(reservedID); } int DownloadManager::startDownloadURLWithMeta(const QString& url, const QString& name, const QString& modName, const QString& version, const QString& source) { const unsigned int reservedID = DownloadInfo::newDownloadID(); ModRepositoryFileInfo info; info.name = name; info.modName = modName; info.version = version; info.repository = source; if (!addDownload(QStringList() << url, "", -1, -1, &info, reservedID)) { return 0; } return static_cast(reservedID); } int DownloadManager::startDownloadNexusFile(const QString& gameName, int modID, int fileID) { int const newID = m_ActiveDownloads.size(); addNXMDownload( QString("nxm://%1/mods/%2/files/%3").arg(gameName).arg(modID).arg(fileID)); return newID; } QString DownloadManager::downloadPath(int id) const { return getFilePath(id); } boost::signals2::connection DownloadManager::onDownloadComplete(const std::function& callback) { return m_DownloadComplete.connect(callback); } boost::signals2::connection DownloadManager::onDownloadPaused(const std::function& callback) { return m_DownloadPaused.connect(callback); } boost::signals2::connection DownloadManager::onDownloadFailed(const std::function& callback) { return m_DownloadFailed.connect(callback); } boost::signals2::connection DownloadManager::onDownloadRemoved(const std::function& callback) { return m_DownloadRemoved.connect(callback); } int DownloadManager::indexByName(const QString& fileName) const { for (int i = 0; i < m_ActiveDownloads.size(); ++i) { if (m_ActiveDownloads[i]->m_FileName == fileName) { return i; } } return -1; } int DownloadManager::indexByInfo(const DownloadInfo* info) const { for (int i = 0; i < m_ActiveDownloads.size(); ++i) { if (m_ActiveDownloads[i] == info) { return i; } } return -1; } void DownloadManager::nxmDownloadURLsAvailable(QString gameName, int modID, int fileID, QVariant userData, QVariant resultData, int requestID) { using namespace boost::placeholders; std::set::iterator const idIter = m_RequestIDs.find(requestID); if (idIter == m_RequestIDs.end()) { return; } else { m_RequestIDs.erase(idIter); } ModRepositoryFileInfo* info = qobject_cast(qvariant_cast(userData)); QVariantList resultList = resultData.toList(); if (resultList.empty()) { removePending(gameName, modID, fileID); emit showMessage(tr("No download server available. Please try again later.")); return; } const auto servers = m_OrganizerCore->settings().network().servers(); std::sort(resultList.begin(), resultList.end(), boost::bind(&ServerByPreference, servers.getPreferred(), _1, _2)); info->userData["downloadMap"] = resultList; QStringList URLs; foreach (const QVariant& server, resultList) { URLs.append(server.toMap()["URI"].toString()); } addDownload(URLs, gameName, modID, fileID, info); } void DownloadManager::nxmFileInfoFromMd5Available(QString gameName, QVariant userData, QVariant resultData, int requestID) { std::set::iterator const idIter = m_RequestIDs.find(requestID); if (idIter == m_RequestIDs.end()) { return; } else { m_RequestIDs.erase(idIter); } DownloadInfo* info = downloadInfoByID(userData.toInt()); // This can come back with multiple results with the same file was uploaded multiple // times (for whatever reason) auto resultlist = resultData.toList(); int chosenIdx = resultlist.count() > 1 ? -1 : 0; // Prefer matching by Nexus file_id when we already have one. file_id is // canonical; the filename pass below can mis-pick after a local rename // or a Nexus-side filename change. if (chosenIdx < 0 && info->m_FileInfo->fileID != 0) { for (int i = 0; i < resultlist.count(); i++) { auto results = resultlist[i].toMap(); auto fileDetails = results["file_details"].toMap(); if (fileDetails["file_id"].toInt() == info->m_FileInfo->fileID) { chosenIdx = i; break; } } } // Look for the exact file name if (chosenIdx < 0) { for (int i = 0; i < resultlist.count(); i++) { auto results = resultlist[i].toMap(); auto fileDetails = results["file_details"].toMap(); if (fileDetails["file_name"].toString().compare(info->m_FileName, Qt::CaseInsensitive) == 0) { if (chosenIdx < 0) { chosenIdx = i; // intentional to not break in order to check other results } else { log::debug("Multiple files with same name found during MD5 search."); chosenIdx = -1; break; } } } } // Look for the only active one if (chosenIdx < 0) { for (int i = 0; i < resultlist.count(); i++) { auto results = resultlist[i].toMap(); auto fileDetails = results["file_details"].toMap(); if (fileDetails["category_id"].toInt() != NexusInterface::FileStatus::REMOVED && fileDetails["category_id"].toInt() != NexusInterface::FileStatus::ARCHIVED) { if (chosenIdx < 0) { chosenIdx = i; // intentional to not break in order to check other results } else { log::debug("Multiple active files found during MD5 search."); chosenIdx = -1; break; } } } } // Unable to determine the correct mod / file. Revert to the old method if (chosenIdx < 0) { // don't use the normal state set function as we don't want to create a meta file info->m_State = DownloadManager::STATE_READY; queryInfo(m_ActiveDownloads.indexOf(info)); return; } auto results = resultlist[chosenIdx].toMap(); auto fileDetails = results["file_details"].toMap(); auto modDetails = results["mod"].toMap(); info->m_FileInfo->name = fileDetails["name"].toString(); info->m_FileInfo->fileID = fileDetails["file_id"].toInt(); info->m_FileInfo->description = BBCode::convertToHTML(fileDetails["description"].toString()); info->m_FileInfo->version.parse(fileDetails["version"].toString()); if (!info->m_FileInfo->version.isValid()) info->m_FileInfo->version.parse(fileDetails["mod_version"].toString()); info->m_FileInfo->fileCategory = fileDetails["category_id"].toInt(); info->m_FileInfo->modID = modDetails["mod_id"].toInt(); info->m_FileInfo->modName = modDetails["name"].toString(); info->m_FileInfo->categoryID = modDetails["category_id"].toInt(); info->m_FileInfo->author = modDetails["author"].toString(); info->m_FileInfo->uploader = modDetails["uploaded_by"].toString(); info->m_FileInfo->uploaderUrl = modDetails["uploaded_users_profile_url"].toString(); QString gameShortName = gameName; QStringList games(m_ManagedGame->validShortNames()); games += m_ManagedGame->gameShortName(); for (const auto& game : games) { MOBase::IPluginGame* gamePlugin = m_OrganizerCore->getGame(game); if (gamePlugin != nullptr && gamePlugin->gameNexusName().compare(gameName, Qt::CaseInsensitive) == 0) { gameShortName = gamePlugin->gameShortName(); break; } } info->m_FileInfo->gameName = gameShortName; // If the file description is not present, send another query to get it if (!fileDetails["description"].isValid()) { info->m_RemoteFileName = fileDetails["file_name"].toString(); setState(info, STATE_FETCHINGFILEINFO); } else { setState(info, STATE_READY); } } void DownloadManager::nxmRequestFailed(QString gameName, int modID, int fileID, QVariant userData, int requestID, int errorCode, const QString& errorString) { std::set::iterator const idIter = m_RequestIDs.find(requestID); if (idIter == m_RequestIDs.end()) { return; } else { m_RequestIDs.erase(idIter); } DownloadInfo* userDataInfo = downloadInfoByID(userData.toInt()); int index = 0; for (QVector::iterator iter = m_ActiveDownloads.begin(); iter != m_ActiveDownloads.end(); ++iter, ++index) { DownloadInfo* info = *iter; if (info == nullptr) continue; if (info != userDataInfo) continue; // MD5 searches continue until all possible games are done if (info->m_State == STATE_FETCHINGMODINFO_MD5) { if (info->m_GamesToQuery.count() >= 2) { info->m_GamesToQuery.pop_front(); setState(info, STATE_FETCHINGMODINFO_MD5); return; } else if (errorCode == 404 && !info->m_AskIfNotFound) { // prevent re-querying (only possible with repository = "Nexus") info->m_FileInfo->repository = ""; setState(info, STATE_READY); return; } else { info->m_State = STATE_READY; queryInfo(index); emit update(index); return; } } if (info->m_FileInfo->modID == modID) { if (info->m_State < STATE_FETCHINGMODINFO) { m_ByID.remove(info->m_DownloadID); m_ActiveDownloads.erase(iter); delete info; } else { setState(info, STATE_READY); } emit update(index); break; } } removePending(gameName, modID, fileID); emit showMessage(tr("Failed to request file info from nexus: %1").arg(errorString)); } void DownloadManager::downloadFinished(int index) { DownloadInfo* info; if (index > 0) info = m_ActiveDownloads[index]; else { info = findDownload(this->sender(), &index); if (info == nullptr && index == 0) { info = m_ActiveDownloads[index]; } } if (info != nullptr) { QNetworkReply* reply = info->m_Reply; QByteArray data; if (reply->isOpen() && info->m_HasData) { data = reply->readAll(); info->m_Output.write(data); } info->m_Output.close(); TaskProgressManager::instance().forgetMe(info->m_TaskProgressId); bool error = false; if ((info->m_State != STATE_CANCELING) && (info->m_State != STATE_PAUSING)) { bool const textData = reply->header(QNetworkRequest::ContentTypeHeader) .toString() .startsWith("text", Qt::CaseInsensitive); if (textData) emit showMessage( tr("Warning: Content type is: %1") .arg(reply->header(QNetworkRequest::ContentTypeHeader).toString())); if ((info->m_Output.size() == 0) || ((reply->error() != QNetworkReply::NoError) && (reply->error() != QNetworkReply::OperationCanceledError))) { if (reply->error() == QNetworkReply::UnknownContentError) emit showMessage( tr("Download header content length: %1 downloaded file size: %2") .arg(reply->header(QNetworkRequest::ContentLengthHeader).toLongLong()) .arg(info->m_Output.size())); if (info->m_Tries == 0) { emit showMessage(tr("Download failed: %1 (%2)") .arg(reply->errorString()) .arg(reply->error())); if (m_OrganizerCore->settings().interface().showDownloadNotifications()) { m_OrganizerCore->showNotification( tr("Download failed"), tr("%1 failed to download.").arg(getDisplayNameForInfo(info)), QSystemTrayIcon::MessageIcon::Critical); } } error = true; setState(info, STATE_ERROR); } } if (info->m_State == STATE_CANCELING) { setState(info, STATE_CANCELED); } else if (info->m_State == STATE_PAUSING) { if (info->m_Output.isOpen() && info->m_HasData && info->m_Reply != nullptr) { info->m_Output.write(info->m_Reply->readAll()); } setState(info, STATE_PAUSED); } if (info->m_State == STATE_CANCELED || (info->m_Tries == 0 && error)) { emit aboutToUpdate(); info->m_Output.remove(); m_ByID.remove(info->m_DownloadID); delete info; m_ActiveDownloads.erase(m_ActiveDownloads.begin() + index); if (error) emit showMessage( tr("We were unable to download the file due to errors after four retries. " "There may be an issue with the Nexus servers.")); emit update(-1); return; } else if (info->isPausedState() || info->m_State == STATE_PAUSING) { info->m_Output.close(); createMetaFile(info); emit update(index); } else { QString const url = info->m_Urls[info->m_CurrentUrl]; if (info->m_FileInfo->userData.contains("downloadMap")) { foreach (const QVariant& server, info->m_FileInfo->userData["downloadMap"].toList()) { QVariantMap serverMap = server.toMap(); if (serverMap["URI"].toString() == url) { int const deltaTime = info->m_StartTime.elapsed() / 1000; if (deltaTime > 5) { emit downloadSpeed(serverMap["short_name"].toString(), (info->m_TotalSize - info->m_PreResumeSize) / deltaTime); } // no division by zero please! Also, if the download is shorter than a // few seconds, the result is way to inprecise break; } } } bool const isNexus = info->m_FileInfo->repository == "Nexus"; // need to change state before changing the file name, otherwise .unfinished is // appended if (isNexus) { setState(info, STATE_FETCHINGMODINFO); } else { setState(info, STATE_NOFETCH); } QString const newName = getFileNameFromNetworkReply(reply); QString const oldName = QFileInfo(info->m_Output).fileName(); startDisableDirWatcher(); // Rename to Content-Disposition if either we have no name yet, or the // name we seeded from the API looks like a CDN object key. if (!newName.isEmpty() && (oldName.isEmpty() || looksLikeCdnObjectKey(info->m_FileName))) { info->setName(getDownloadFileName(newName), true); } else { info->setName(m_OutputDirectory + "/" + info->m_FileName, true); // don't rename but remove the ".unfinished" extension } endDisableDirWatcher(); if (!isNexus) { setState(info, STATE_READY); } emit update(index); } reply->close(); reply->deleteLater(); if ((info->m_Tries > 0) && error) { --info->m_Tries; resumeDownloadInt(index); } } else { log::warn("no download index {}", index); } } void DownloadManager::downloadError(QNetworkReply::NetworkError error) { if (error != QNetworkReply::OperationCanceledError) { QNetworkReply* reply = qobject_cast(sender()); log::warn("{} ({})", reply != nullptr ? reply->errorString() : "Download error occured", error); } } void DownloadManager::metaDataChanged() { int index = 0; DownloadInfo* info = findDownload(this->sender(), &index); if (info != nullptr) { const QString newName = getFileNameFromNetworkReply(info->m_Reply); // Prefer Content-Disposition over whatever we seeded from the API when // the seeded name looks like a CDN object key. Otherwise only take it // if we have no name at all yet. const bool shouldReplace = !newName.isEmpty() && (info->m_FileName.isEmpty() || looksLikeCdnObjectKey(info->m_FileName)); if (shouldReplace) { log::info("metaDataChanged: replacing '{}' with Content-Disposition '{}'", info->m_FileName, newName); startDisableDirWatcher(); info->setName(getDownloadFileName(newName), true); endDisableDirWatcher(); refreshAlphabeticalTranslation(); if (!info->m_Output.isOpen() && !info->m_Output.open(QIODevice::WriteOnly | QIODevice::Append)) { reportError(tr("failed to re-open %1").arg(info->m_FileName)); setState(info, STATE_CANCELING); } } } else { log::warn("meta data event for unknown download"); } } void DownloadManager::directoryChanged(const QString&) { if (DownloadManager::m_DirWatcherDisabler == 0) refreshList(); } void DownloadManager::managedGameChanged(MOBase::IPluginGame const* managedGame) { m_ManagedGame = managedGame; } void DownloadManager::checkDownloadTimeout() { for (int i = 0; i < m_ActiveDownloads.size(); ++i) { DownloadInfo* info = m_ActiveDownloads[i]; if (info->m_State != STATE_DOWNLOADING || info->m_Reply == nullptr || !info->m_Reply->isOpen()) { continue; } const bool stalled = (info->m_StartTime.elapsed() - info->m_DownloadTimeLast) > (5 * 1000); if (!stalled) { continue; } pauseDownload(i); downloadFinished(i); // downloadFinished() can remove the download entry, so find it again. const int index = indexByInfo(info); if (index < 0) { continue; } if (info->m_Tries <= 0) { emit showMessage(tr("Download stalled repeatedly and was paused. " "Please retry manually.")); continue; } --info->m_Tries; log::warn("download '{}' stalled, retrying ({} retries left)", info->m_FileName, info->m_Tries); resumeDownloadInt(index); emit update(index); } } void DownloadManager::writeData(DownloadInfo* info) { if (info != nullptr && info->m_Reply != nullptr) { qint64 ret = info->m_Output.write(info->m_Reply->readAll()); if (ret < info->m_Reply->size()) { QString const fileName = info->m_FileName; // m_FileName may be destroyed after setState setState(info, DownloadState::STATE_CANCELED); log::error("Unable to write download \"{}\" to drive (return {})", info->m_FileName, ret); reportError(tr("Unable to write download to drive (return %1).\n" "Check the drive's available storage.\n\n" "Canceling download \"%2\"...") .arg(ret) .arg(fileName)); } } }