From 7ee008e150bc5bcf76082d726f719ee0fdfda982 Mon Sep 17 00:00:00 2001 From: SulfurNitride Date: Wed, 11 Feb 2026 02:37:39 -0600 Subject: Fluorine Manager: full Linux port of Mod Organizer 2 Complete native Linux port with FUSE-based virtual filesystem, Proton/umu-run integration, and Flatpak packaging. Key features: - FUSE VFS replacing Windows USVFS (in-process + standalone helper for Flatpak) - Proton/GE-Proton/umu-run launcher with env var forwarding - Flatpak support (sandbox-aware VFS, NXM handler, umu-run) - Wine prefix management UI - Case-insensitive path resolution for Linux filesystems - QSettings-safe INI handling (avoids Bethesda INI corruption) - Portable instance support with auto-generated launcher scripts Co-Authored-By: Claude Opus 4.6 --- libs/basic_games/games/baldursgate3/__init__.py | 0 .../games/baldursgate3/bg3_data_checker.py | 58 ++++ .../games/baldursgate3/bg3_data_content.py | 54 ++++ .../games/baldursgate3/bg3_file_mapper.py | 131 +++++++++ libs/basic_games/games/baldursgate3/bg3_utils.py | 274 +++++++++++++++++++ .../games/baldursgate3/lslib_retriever.py | 162 +++++++++++ libs/basic_games/games/baldursgate3/pak_parser.py | 295 +++++++++++++++++++++ .../games/baldursgate3/plugins/__init__.py | 13 + .../games/baldursgate3/plugins/bg3_tool_plugin.py | 49 ++++ .../plugins/check_for_lslib_updates_plugin.py | 15 ++ .../plugins/convert_jsons_to_yaml_plugin.py | 58 ++++ .../games/baldursgate3/plugins/icons.py | 42 +++ .../plugins/reparse_pak_metadata_plugin.py | 17 ++ 13 files changed, 1168 insertions(+) create mode 100644 libs/basic_games/games/baldursgate3/__init__.py create mode 100644 libs/basic_games/games/baldursgate3/bg3_data_checker.py create mode 100644 libs/basic_games/games/baldursgate3/bg3_data_content.py create mode 100644 libs/basic_games/games/baldursgate3/bg3_file_mapper.py create mode 100644 libs/basic_games/games/baldursgate3/bg3_utils.py create mode 100644 libs/basic_games/games/baldursgate3/lslib_retriever.py create mode 100644 libs/basic_games/games/baldursgate3/pak_parser.py create mode 100644 libs/basic_games/games/baldursgate3/plugins/__init__.py create mode 100644 libs/basic_games/games/baldursgate3/plugins/bg3_tool_plugin.py create mode 100644 libs/basic_games/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py create mode 100644 libs/basic_games/games/baldursgate3/plugins/convert_jsons_to_yaml_plugin.py create mode 100644 libs/basic_games/games/baldursgate3/plugins/icons.py create mode 100644 libs/basic_games/games/baldursgate3/plugins/reparse_pak_metadata_plugin.py (limited to 'libs/basic_games/games/baldursgate3') diff --git a/libs/basic_games/games/baldursgate3/__init__.py b/libs/basic_games/games/baldursgate3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libs/basic_games/games/baldursgate3/bg3_data_checker.py b/libs/basic_games/games/baldursgate3/bg3_data_checker.py new file mode 100644 index 0000000..695d21c --- /dev/null +++ b/libs/basic_games/games/baldursgate3/bg3_data_checker.py @@ -0,0 +1,58 @@ +from pathlib import Path + +import mobase + +from ...basic_features import BasicModDataChecker, GlobPatterns, utils +from . import bg3_utils + + +class BG3ModDataChecker(BasicModDataChecker): + def __init__(self): + super().__init__( + GlobPatterns( + valid=[ + "*.pak", + str(Path("Mods") / "*.pak"), # standard mods + "bin", # native mods / Script Extender + "Script Extender", # mods which are configured via jsons in this folder + "Data", # loose file mods + ] + + [str(Path("*") / f) for f in bg3_utils.loose_file_folders], + move={ + "Root/": "", # root builder not needed + "*.dll": "bin/", + "ScriptExtenderSettings.json": "bin/", + } + | {f: "Data/" for f in bg3_utils.loose_file_folders}, + delete=["info.json", "*.txt"], + ) + ) + + def dataLooksValid( + self, filetree: mobase.IFileTree + ) -> mobase.ModDataChecker.CheckReturn: + status = mobase.ModDataChecker.INVALID + rp = self._regex_patterns + for entry in filetree: + name = entry.name().casefold() + if rp.unfold.match(name): + if utils.is_directory(entry): + status = self.dataLooksValid(entry) + else: + status = mobase.ModDataChecker.INVALID + break + elif rp.valid.match(name): + if status is mobase.ModDataChecker.INVALID: + status = mobase.ModDataChecker.VALID + elif isinstance(entry, mobase.IFileTree): + status = ( + mobase.ModDataChecker.VALID + if all(rp.valid.match(e.pathFrom(filetree)) for e in entry) + else mobase.ModDataChecker.INVALID + ) + elif rp.delete.match(name) or rp.move_match(name) is not None: + status = mobase.ModDataChecker.FIXABLE + else: + status = mobase.ModDataChecker.INVALID + break + return status diff --git a/libs/basic_games/games/baldursgate3/bg3_data_content.py b/libs/basic_games/games/baldursgate3/bg3_data_content.py new file mode 100644 index 0000000..c378520 --- /dev/null +++ b/libs/basic_games/games/baldursgate3/bg3_data_content.py @@ -0,0 +1,54 @@ +from enum import IntEnum, auto + +import mobase + +from . import bg3_utils + + +class Content(IntEnum): + PAK = auto() + WORKSPACE = auto() + NATIVE = auto() + LOOSE_FILES = auto() + SE_FILES = auto() + + +class BG3DataContent(mobase.ModDataContent): + BG3_CONTENTS: list[tuple[Content, str, str, bool] | tuple[Content, str, str]] = [ + (Content.WORKSPACE, "Mod workspace", ":/MO/gui/content/script"), + (Content.PAK, "Pak", ":/MO/gui/content/bsa"), + (Content.LOOSE_FILES, "Loose file override mod", ":/MO/gui/content/texture"), + (Content.SE_FILES, "Script Extender Files", ":/MO/gui/content/inifile"), + (Content.NATIVE, "Native DLL mod", ":/MO/gui/content/plugin"), + ] + + def getAllContents(self) -> list[mobase.ModDataContent.Content]: + return [ + mobase.ModDataContent.Content(id, name, icon, *filter_only) + for id, name, icon, *filter_only in self.BG3_CONTENTS + ] + + def getContentsFor(self, filetree: mobase.IFileTree) -> list[int]: + contents: set[int] = set() + for entry in filetree: + if isinstance(entry, mobase.IFileTree): + match entry.name(): + case "Script Extender": + contents.add(Content.SE_FILES) + case "Data": + contents.add(Content.LOOSE_FILES) + case "Mods": + for e in entry: + if e.name().endswith(".pak"): + contents.add(Content.PAK) + break + case "bin": + contents.add(Content.NATIVE) + case _: + for e in entry: + if e.name() in bg3_utils.loose_file_folders: + contents.add(Content.WORKSPACE) + break + elif entry.name().endswith(".pak"): + contents.add(Content.PAK) + return list(contents) diff --git a/libs/basic_games/games/baldursgate3/bg3_file_mapper.py b/libs/basic_games/games/baldursgate3/bg3_file_mapper.py new file mode 100644 index 0000000..0ae6ef8 --- /dev/null +++ b/libs/basic_games/games/baldursgate3/bg3_file_mapper.py @@ -0,0 +1,131 @@ +import functools +import os +from pathlib import Path +from typing import Callable, Optional + +from PyQt6.QtCore import QDir, QLoggingCategory, qDebug, qInfo, qWarning +from PyQt6.QtWidgets import QApplication + +import mobase + +from . import bg3_utils + + +class BG3FileMapper(mobase.IPluginFileMapper): + current_mappings: list[mobase.Mapping] = [] + + def __init__(self, utils: bg3_utils.BG3Utils, doc_dir: Callable[[], QDir]): + super().__init__() + self._utils = utils + self.doc_dir = doc_dir + + @functools.cached_property + def doc_path(self): + return Path(self.doc_dir().path()) + + def mappings(self) -> list[mobase.Mapping]: + qInfo("creating custom bg3 mappings") + self.current_mappings.clear() + active_mods = self._utils.active_mods() + if not active_mods: + return [] + progress = self._utils.create_progress_window( + "Mapping files to documents folder", len(active_mods) + 1 + ) + docs_path_mods = self.doc_path / "Mods" + docs_path_se = self.doc_path / "Script Extender" + for mod in active_mods: + modpath = Path(mod.absolutePath()) + self.map_files(modpath, dest=docs_path_mods, pattern="*.pak", rel=False) + self.map_files(modpath / "Script Extender", dest=docs_path_se) + if self._utils.convert_yamls_to_json: + self.map_files(modpath / "bin", only_convert=True) + progress.setValue(progress.value() + 1) + QApplication.processEvents() + if progress.wasCanceled(): + qWarning("mapping canceled by user") + return self.current_mappings + (self._utils.overwrite_path / "Script Extender").mkdir( + parents=True, exist_ok=True + ) + (self._utils.overwrite_path / "Stats").mkdir(parents=True, exist_ok=True) + (self._utils.overwrite_path / "Temp").mkdir(parents=True, exist_ok=True) + (self._utils.overwrite_path / "LevelCache").mkdir(parents=True, exist_ok=True) + (self._utils.overwrite_path / "Stats").mkdir(parents=True, exist_ok=True) + (self._utils.overwrite_path / "Mods").mkdir(parents=True, exist_ok=True) + (self._utils.overwrite_path / "GMCampaigns").mkdir(parents=True, exist_ok=True) + self.map_files(self._utils.overwrite_path) + self.create_mapping( + self._utils.modsettings_path, + self.doc_path + / "PlayerProfiles" + / "Public" + / self._utils.modsettings_path.name, + ) + progress.setValue(len(active_mods) + 1) + QApplication.processEvents() + progress.close() + cat = QLoggingCategory.defaultCategory() + if cat is not None and cat.isDebugEnabled(): + qDebug( + f"resolved mappings: { {m.source: m.destination for m in self.current_mappings} }" + ) + return self.current_mappings + + def map_files( + self, + path: Path, + dest: Optional[Path] = None, + pattern: str = "*", + rel: bool = True, + only_convert: bool = False, + ): + dest = dest if dest else self.doc_path + dest_func: Callable[[Path], str] = ( + (lambda f: os.path.relpath(f, path)) if rel else lambda f: f.name + ) + found_jsons: set[Path] = set() + for file in list(path.rglob(pattern)): + if self._utils.convert_yamls_to_json and ( + file.name.endswith(".yaml") or file.name.endswith(".yml") + ): + converted_path = file.parent / file.name.replace( + ".yaml", ".json" + ).replace(".yml", ".json") + try: + if not converted_path.exists() or os.path.getmtime( + file + ) > os.path.getmtime(converted_path): + import json + + import yaml + + with open(file, "r") as yaml_file: + with open(converted_path, "w") as json_file: + json.dump( + yaml.safe_load(yaml_file), json_file, indent=2 + ) + qDebug(f"Converted {file} to JSON") + found_jsons.add(converted_path) + except OSError as e: + qWarning(f"Error accessing file {converted_path}: {e}") + elif file.name.endswith(".json"): + found_jsons.add(file) + elif not only_convert: + self.create_mapping(file, dest / dest_func(file)) + if only_convert: + return + for file in found_jsons: + self.create_mapping(file, dest / dest_func(file)) + + def create_mapping(self, file: Path, dest: Path): + bg3_utils.create_dir_if_needed(dest) + + self.current_mappings.append( + mobase.Mapping( + source=str(file), + destination=str(dest), + is_directory=file.is_dir(), + create_target=True, + ) + ) diff --git a/libs/basic_games/games/baldursgate3/bg3_utils.py b/libs/basic_games/games/baldursgate3/bg3_utils.py new file mode 100644 index 0000000..958d173 --- /dev/null +++ b/libs/basic_games/games/baldursgate3/bg3_utils.py @@ -0,0 +1,274 @@ +import functools +import shutil +import typing +from pathlib import Path +from time import sleep + +from PyQt6.QtCore import ( + QCoreApplication, + QDir, + QEventLoop, + QRunnable, + Qt, + QThread, + QThreadPool, + qInfo, + qWarning, +) +from PyQt6.QtWidgets import QApplication, QMainWindow, QProgressDialog + +import mobase + +loose_file_folders = { + "Public", + "Mods", + "Generated", + "Localization", + "ScriptExtender", +} + + +def get_node_string( + folder: str = "", + md5: str = "", + name: str = "", + publish_handle: str = "0", + uuid: str = "", + version64: str = "0", +) -> str: + return f""" + + + + + + + + """ + + +class BG3Utils: + _mod_settings_xml_start = """\ + + + + + + + + """ + get_node_string( + folder="GustavX", + name="GustavX", + uuid="cb555efe-2d9e-131f-8195-a89329d218ea", + version64="36028797018963968", + ) + _mod_settings_xml_end = """ + + + + + +""" + + def __init__(self, name: str): + self.main_window = None + self._name = name + from . import lslib_retriever, pak_parser + + self.lslib_retriever = lslib_retriever.LSLibRetriever(self) + self._pak_parser = pak_parser.BG3PakParser(self) + + def init(self, organizer: mobase.IOrganizer): + self._organizer = organizer + + @functools.cached_property + def autobuild_paks(self): + return bool(self.get_setting("autobuild_paks")) + + @functools.cached_property + def extract_full_package(self): + return bool(self.get_setting("extract_full_package")) + + @functools.cached_property + def remove_extracted_metadata(self): + return bool(self.get_setting("remove_extracted_metadata")) + + @functools.cached_property + def force_load_dlls(self): + return bool(self.get_setting("force_load_dlls")) + + @functools.cached_property + def log_diff(self): + return bool(self.get_setting("log_diff")) + + @functools.cached_property + def convert_yamls_to_json(self): + return bool(self.get_setting("convert_yamls_to_json")) + + @functools.cached_property + def log_dir(self): + return create_dir_if_needed(Path(self._organizer.basePath()) / "logs") + + @functools.cached_property + def modsettings_backup(self): + return create_dir_if_needed(self.plugin_data_path / "temp" / "modsettings.lsx") + + @functools.cached_property + def modsettings_path(self): + return create_dir_if_needed( + Path(self._organizer.profilePath()) / "modsettings.lsx" + ) + + @functools.cached_property + def plugin_data_path(self) -> Path: + """Gets the path to the data folder for the current plugin.""" + return create_dir_if_needed( + Path(self._organizer.pluginDataPath(), self._name).absolute() + ) + + @functools.cached_property + def tools_dir(self): + return create_dir_if_needed(self.plugin_data_path / "tools") + + @functools.cached_property + def overwrite_path(self): + return create_dir_if_needed(Path(self._organizer.overwritePath())) + + def active_mods(self) -> list[mobase.IModInterface]: + modlist = self._organizer.modList() + return [ + modlist.getMod(mod_name) + for mod_name in filter( + lambda mod: modlist.state(mod) & mobase.ModState.ACTIVE, + modlist.allModsByProfilePriority(), + ) + ] + + def _set_setting(self, key: str, value: mobase.MoVariant): + self._organizer.setPluginSetting(self._name, key, value) + + def get_setting(self, key: str) -> mobase.MoVariant: + return self._organizer.pluginSetting(self._name, key) + + def tr(self, trstr: str) -> str: + return QCoreApplication.translate(self._name, trstr) + + def create_progress_window( + self, title: str, max_progress: int, msg: str = "", cancelable: bool = True + ) -> QProgressDialog: + progress = QProgressDialog( + self.tr(msg if msg else title), + self.tr("Cancel") if cancelable else None, + 0, + max_progress, + self.main_window, + ) + progress.setWindowTitle(self.tr(f"BG3 Plugin: {title}")) + progress.setWindowModality(Qt.WindowModality.ApplicationModal) + progress.show() + return progress + + def on_user_interface_initialized(self, window: QMainWindow) -> None: + self.main_window = window + + def on_settings_changed( + self, + plugin_name: str, + setting: str, + old: mobase.MoVariant, + new: mobase.MoVariant, + ) -> None: + if self._name != plugin_name: + return + if setting in { + "extract_full_package", + "autobuild_paks", + "remove_extracted_metadata", + "force_load_dlls", + "log_diff", + "convert_yamls_to_json", + } and hasattr(self, setting): + delattr(self, setting) + + def construct_modsettings_xml( + self, + exec_path: str = "", + working_dir: typing.Optional[QDir] = None, + args: str = "", + force_reparse_metadata: bool = False, + ) -> bool: + if ( + "bin/bg3" not in exec_path + or not self.lslib_retriever.download_lslib_if_missing() + ): + return True + active_mods = self.active_mods() + progress = self.create_progress_window( + "Generating modsettings.xml", len(active_mods) + ) + threadpool = QThreadPool.globalInstance() + if threadpool is None: + return False + metadata: dict[str, str] = {} + + def retrieve_mod_metadata_in_new_thread(mod: mobase.IModInterface): + return lambda: metadata.update( + self._pak_parser.get_metadata_for_files_in_mod( + mod, force_reparse_metadata + ) + ) + + for mod in active_mods: + if progress.wasCanceled(): + qWarning("processing canceled by user") + return False + threadpool.start(QRunnable.create(retrieve_mod_metadata_in_new_thread(mod))) + count = 0 + num_active_mods = len(active_mods) + total_intervals_to_wait = (num_active_mods * 2) + 20 + while len(metadata.keys()) < num_active_mods: + progress.setValue(len(metadata.keys())) + QApplication.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, 100) + count += 1 + if count == total_intervals_to_wait or progress.wasCanceled(): + remaining_mods = {mod.name() for mod in active_mods} - metadata.keys() + qWarning(f"processing did not finish in time for: {remaining_mods}") + progress.close() + break + QThread.msleep(100) + progress.setValue(num_active_mods) + QApplication.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, 100) + progress.close() + qInfo(f"writing mod load order to {self.modsettings_path}") + self.modsettings_path.parent.mkdir(parents=True, exist_ok=True) + self.modsettings_path.write_text( + ( + self._mod_settings_xml_start + + "".join( + metadata[mod.name()] + for mod in active_mods + if mod.name() in metadata + ) + + self._mod_settings_xml_end + ) + ) + qInfo( + f"backing up generated file {self.modsettings_path} to {self.modsettings_backup}, " + f"check the backup after the executable runs for differences with the file used by the game if you encounter issues" + ) + self.modsettings_backup.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(self.modsettings_path, self.modsettings_backup) + sleep(0.5) + return True + + def on_mod_installed(self, mod: mobase.IModInterface) -> None: + if self.lslib_retriever.download_lslib_if_missing(): + self._pak_parser.get_metadata_for_files_in_mod(mod, True) + + +def create_dir_if_needed(path: Path) -> Path: + if "." not in path.name[1:]: + path.mkdir(parents=True, exist_ok=True) + else: + path.parent.mkdir(parents=True, exist_ok=True) + return path diff --git a/libs/basic_games/games/baldursgate3/lslib_retriever.py b/libs/basic_games/games/baldursgate3/lslib_retriever.py new file mode 100644 index 0000000..ee1122c --- /dev/null +++ b/libs/basic_games/games/baldursgate3/lslib_retriever.py @@ -0,0 +1,162 @@ +import json +import shutil +import traceback +import urllib.request +import zipfile +from functools import cached_property + +from PyQt6.QtCore import qDebug, qWarning +from PyQt6.QtWidgets import QApplication, QMessageBox + +from . import bg3_utils + + +class LSLibRetriever: + def __init__(self, utils: bg3_utils.BG3Utils): + self._utils = utils + + @cached_property + def _needed_lslib_files(self): + return { + self._utils.tools_dir / x + for x in { + "CommandLineArgumentsParser.dll", + "Divine.dll", + "Divine.dll.config", + "Divine.exe", + "Divine.runtimeconfig.json", + "K4os.Compression.LZ4.dll", + "K4os.Compression.LZ4.Streams.dll", + "LSLib.dll", + "LSLibNative.dll", + "LZ4.dll", + "Newtonsoft.Json.dll", + "System.IO.Hashing.dll", + "ZstdSharp.dll", + } + } + + def download_lslib_if_missing(self, force: bool = False) -> bool: + if not force and all(x.exists() for x in self._needed_lslib_files): + return True + try: + self._utils.tools_dir.mkdir(exist_ok=True, parents=True) + downloaded = False + + def reporthook(block_num: int, block_size: int, total_size: int) -> None: + if total_size > 0: + progress.setValue( + min(int(block_num * block_size * 100 / total_size), 100) + ) + QApplication.processEvents() + + with urllib.request.urlopen( + "https://api.github.com/repos/Norbyte/lslib/releases/latest" + ) as response: + assets = json.loads(response.read().decode("utf-8"))["assets"][0] + zip_path = self._utils.tools_dir / assets["name"] + if not zip_path.exists(): + old_archives = list(self._utils.tools_dir.glob("*.zip")) + msg_box = QMessageBox(self._utils.main_window) + msg_box.setWindowTitle( + self._utils.tr("Baldur's Gate 3 Plugin - Missing dependencies") + ) + if old_archives: + msg_box.setText(self._utils.tr("LSLib update available.")) + else: + msg_box.setText( + self._utils.tr( + "LSLib tools are missing.\nThese are necessary for the plugin to create the load order file for BG3." + ) + ) + msg_box.addButton( + self._utils.tr("Download"), + QMessageBox.ButtonRole.DestructiveRole, + ) + exit_btn = msg_box.addButton( + self._utils.tr("Exit"), QMessageBox.ButtonRole.ActionRole + ) + msg_box.setIcon(QMessageBox.Icon.Warning) + msg_box.exec() + + if msg_box.clickedButton() == exit_btn: + if not old_archives: + err = QMessageBox(self._utils.main_window) + err.setIcon(QMessageBox.Icon.Critical) + err.setText( + "LSLib tools are required for the proper generation of the modsettings.xml file, file will not be generated" + ) + err.exec() + return False + else: + progress = self._utils.create_progress_window( + "Downloading LSLib", 100, cancelable=False + ) + urllib.request.urlretrieve( + assets["browser_download_url"], str(zip_path), reporthook + ) + progress.close() + downloaded = True + for archive in old_archives: + archive.unlink() + old_archives = [] + else: + old_archives = [] + new_msg = QMessageBox(self._utils.main_window) + new_msg.setIcon(QMessageBox.Icon.Information) + new_msg.setText( + self._utils.tr("Latest version of LSLib already downloaded!") + ) + new_msg.exec() + + except Exception as e: + qDebug(f"Download failed: {e}") + err = QMessageBox(self._utils.main_window) + err.setIcon(QMessageBox.Icon.Critical) + err.setText( + self._utils.tr( + f"Failed to download LSLib tools:\n{traceback.format_exc()}" + ) + ) + err.exec() + return False + try: + if old_archives: + zip_path = sorted(old_archives)[-1] + if old_archives or not downloaded: + dialog_message = "Ensuring all necessary LSLib files have been extracted from archive..." + win_title = "Verifying LSLib files" + else: + dialog_message = "Extracting/Updating LSLib files..." + win_title = "Extracting LSLib" + x_progress = self._utils.create_progress_window( + win_title, len(self._needed_lslib_files), msg=dialog_message + ) + with zipfile.ZipFile(zip_path, "r") as zip_ref: + for file in self._needed_lslib_files: + if downloaded or not file.exists(): + shutil.move( + zip_ref.extract( + f"Packed/Tools/{file.name}", self._utils.tools_dir + ), + file, + ) + x_progress.setValue(x_progress.value() + 1) + QApplication.processEvents() + if x_progress.wasCanceled(): + qWarning("processing canceled by user") + return False + x_progress.close() + shutil.rmtree(self._utils.tools_dir / "Packed", ignore_errors=True) + except Exception as e: + qDebug(f"Extraction failed: {e}") + err = QMessageBox(self._utils.main_window) + err.setIcon(QMessageBox.Icon.Critical) + err.setText( + self._utils.tr( + f"Failed to extract LSLib tools:\n{traceback.format_exc()}" + ) + ) + err.exec() + return False + return True diff --git a/libs/basic_games/games/baldursgate3/pak_parser.py b/libs/basic_games/games/baldursgate3/pak_parser.py new file mode 100644 index 0000000..18598a9 --- /dev/null +++ b/libs/basic_games/games/baldursgate3/pak_parser.py @@ -0,0 +1,295 @@ +import configparser +import hashlib +import os +import re +import shutil +import subprocess +import traceback +from functools import cached_property +from pathlib import Path +from typing import Callable +from xml.etree import ElementTree +from xml.etree.ElementTree import Element + +from PyQt6.QtCore import ( + qDebug, + qInfo, + qWarning, +) + +import mobase + +from . import bg3_utils + + +class BG3PakParser: + def __init__(self, utils: bg3_utils.BG3Utils): + self._utils = utils + + _mod_cache: dict[Path, bool] = {} + _types = { + "Folder": "", + "MD5": "", + "Name": "", + "PublishHandle": "0", + "UUID": "", + "Version64": "0", + } + + @cached_property + def _divine_command(self): + return f"{self._utils.tools_dir / 'Divine.exe'} -g bg3 -l info" + + @cached_property + def _folder_pattern(self): + return re.compile("Data|Script Extender|bin|Mods") + + def get_metadata_for_files_in_mod( + self, mod: mobase.IModInterface, force_reparse_metadata: bool + ): + return { + mod.name(): "".join( + [ + self._get_metadata_for_file(mod, file, force_reparse_metadata) + for file in sorted( + list(Path(mod.absolutePath()).rglob("*.pak")) + + ( + [ + f + for f in Path(mod.absolutePath()).glob("*") + if f.is_dir() + ] + if self._utils.autobuild_paks + else [] + ) + ) + ] + ) + } + + def _get_metadata_for_file( + self, + mod: mobase.IModInterface, + file: Path, + force_reparse_metadata: bool, + ) -> str: + meta_ini = Path(mod.absolutePath()) / "meta.ini" + config = configparser.ConfigParser(interpolation=None) + config.read(meta_ini, encoding="utf-8") + try: + if file.name.endswith("pak"): + meta_file = ( + self._utils.plugin_data_path + / "temp" + / "extracted_metadata" + / f"{file.name[: int(len(file.name) / 2)]}-{hashlib.md5(str(file).encode(), usedforsecurity=False).hexdigest()[:5]}.lsx" + ) + try: + if ( + not force_reparse_metadata + and config.has_section(file.name) + and ( + "override" in config[file.name].keys() + or "Folder" in config[file.name].keys() + ) + ): + return get_module_short_desc(config, file) + meta_file.parent.mkdir(parents=True, exist_ok=True) + meta_file.unlink(missing_ok=True) + out_dir = ( + str(meta_file)[:-4] if self._utils.extract_full_package else "" + ) + can_continue = True + if self.run_divine( + f'{"extract-package" if self._utils.extract_full_package else "extract-single-file -f meta.lsx"} -d "{meta_file if not self._utils.extract_full_package else out_dir}"', + file, + ).returncode: + can_continue = False + if can_continue and self._utils.extract_full_package: + qDebug(f"archive {file} extracted to {out_dir}") + if self.run_divine( + f'convert-resources -d "{out_dir}" -i lsf -o lsx -x "*.lsf"', + out_dir, + ).returncode: + qDebug( + f"failed to convert lsf files in {out_dir} to readable lsx" + ) + extracted_meta_files = list(Path(out_dir).rglob("meta.lsx")) + if len(extracted_meta_files) == 0: + qInfo( + f"No meta.lsx files found in {file.name}, {file.name} determined to be an override mod" + ) + can_continue = False + else: + shutil.copyfile( + extracted_meta_files[0], + meta_file, + ) + elif can_continue and not meta_file.exists(): + qInfo( + f"No meta.lsx files found in {file.name}, {file.name} determined to be an override mod" + ) + can_continue = False + return self.metadata_to_ini( + config, file, mod, meta_ini, can_continue, lambda: meta_file + ) + finally: + if self._utils.remove_extracted_metadata: + meta_file.unlink(missing_ok=True) + if self._utils.extract_full_package: + Path(str(meta_file)[:-4]).unlink(missing_ok=True) + elif file.is_dir(): + if self._folder_pattern.search(file.name): + return "" + for folder in bg3_utils.loose_file_folders: + if next(file.glob(f"{folder}/*"), False): + break + else: + return "" + qInfo(f"packable dir: {file}") + if (file.parent / f"{file.name}.pak").exists() or ( + file.parent / "Mods" / f"{file.name}.pak" + ).exists(): + qInfo( + f"pak with same name as packable dir exists in mod directory. not packing dir {file}" + ) + return "" + parent_mod_name = file.parent.name.replace(" ", "_") + pak_path = ( + self._utils.overwrite_path + / f"Mods/{parent_mod_name}_{file.name}.pak" + ) + build_pak = True + if pak_path.exists(): + try: + pak_creation_time = os.path.getmtime(pak_path) + for root, _, files in file.walk(): + for f in files: + file_path = root.joinpath(f) + try: + if os.path.getmtime(file_path) > pak_creation_time: + break + except OSError as e: + qDebug(f"Error accessing file {file_path}: {e}") + break + else: + build_pak = False + except OSError as e: + qDebug(f"Error accessing file {pak_path}: {e}") + build_pak = False + if build_pak: + pak_path.unlink(missing_ok=True) + if self.run_divine( + f'create-package -d "{pak_path}"', file + ).returncode: + return "" + meta_files = list(file.glob("Mods/*/meta.lsx")) + return self.metadata_to_ini( + config, + file, + mod, + meta_ini, + len(meta_files) > 0, + lambda: meta_files[0], + ) + else: + return "" + except Exception: + qWarning(traceback.format_exc()) + return "" + + def run_divine( + self, action: str, source: Path | str + ) -> subprocess.CompletedProcess[str]: + command = f'{self._divine_command} -a {action} -s "{source}"' + result = subprocess.run( + command, + creationflags=subprocess.CREATE_NO_WINDOW, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode: + qWarning( + f"{command.replace(str(Path.home()), '~', 1).replace(str(Path.home()), '$HOME')}" + f" returned stdout: {result.stdout}, stderr: {result.stderr}, code {result.returncode}" + ) + return result + + def get_attr_value(self, root: Element, attr_id: str) -> str: + default_val = self._types.get(attr_id) or "" + attr = root.find(f".//attribute[@id='{attr_id}']") + return default_val if attr is None else attr.get("value", default_val) + + def metadata_to_ini( + self, + config: configparser.ConfigParser, + file: Path, + mod: mobase.IModInterface, + meta_ini: Path, + condition: bool, + to_parse: Callable[[], Path], + ): + config[file.name] = {} + if condition: + root = ( + ElementTree.parse(to_parse()) + .getroot() + .find(".//node[@id='ModuleInfo']") + ) + if root is None: + qInfo(f"No ModuleInfo node found in meta.lsx for {mod.name()} ") + else: + section = config[file.name] + folder_name = self.get_attr_value(root, "Folder") + if file.is_dir(): + self._mod_cache[file] = ( + len(list(file.glob(f"*/{folder_name}/**"))) > 1 + or len( + list(file.glob("Public/Engine/Timeline/MaterialGroups/*")) + ) + > 0 + ) + elif file not in self._mod_cache: + # a mod which has a meta.lsx and is not an override mod meets at least one of three conditions: + # 1. it has files in Public/Engine/Timeline/MaterialGroups, or + # 2. it has files in Mods// other than the meta.lsx file, or + # 3. it has files in Public/ + result = self.run_divine( + f'list-package --use-regex -x "(/{re.escape(folder_name)}/(?!meta\\.lsx))|(Public/Engine/Timeline/MaterialGroups)"', + file, + ) + self._mod_cache[file] = ( + result.returncode == 0 and result.stdout.strip() != "" + ) + if self._mod_cache[file]: + for key in self._types: + section[key] = self.get_attr_value(root, key) + else: + qInfo(f"pak {file.name} determined to be an override mod") + section["override"] = "True" + section["Folder"] = folder_name + else: + config[file.name]["override"] = "True" + with open(meta_ini, "w+", encoding="utf-8") as f: + config.write(f) + return get_module_short_desc(config, file) + + +def get_module_short_desc(config: configparser.ConfigParser, file: Path) -> str: + if not config.has_section(file.name): + return "" + section: configparser.SectionProxy = config[file.name] + return ( + "" + if "override" in section.keys() or "Name" not in section.keys() + else bg3_utils.get_node_string( + folder=section["Folder"], + md5=section["MD5"], + name=section["Name"], + publish_handle=section["PublishHandle"], + uuid=section["UUID"], + version64=section["Version64"], + ) + ) diff --git a/libs/basic_games/games/baldursgate3/plugins/__init__.py b/libs/basic_games/games/baldursgate3/plugins/__init__.py new file mode 100644 index 0000000..1b69faf --- /dev/null +++ b/libs/basic_games/games/baldursgate3/plugins/__init__.py @@ -0,0 +1,13 @@ +import mobase + +from .check_for_lslib_updates_plugin import BG3ToolCheckForLsLibUpdates +from .convert_jsons_to_yaml_plugin import BG3ToolConvertJsonsToYaml +from .reparse_pak_metadata_plugin import BG3ToolReparsePakMetadata + + +def createPlugins() -> list[mobase.IPluginTool]: + return [ + BG3ToolCheckForLsLibUpdates(), + BG3ToolReparsePakMetadata(), + BG3ToolConvertJsonsToYaml(), + ] diff --git a/libs/basic_games/games/baldursgate3/plugins/bg3_tool_plugin.py b/libs/basic_games/games/baldursgate3/plugins/bg3_tool_plugin.py new file mode 100644 index 0000000..fc258a6 --- /dev/null +++ b/libs/basic_games/games/baldursgate3/plugins/bg3_tool_plugin.py @@ -0,0 +1,49 @@ +from PyQt6.QtCore import QCoreApplication +from PyQt6.QtGui import QIcon, QPixmap + +import mobase + + +class BG3ToolPlugin(mobase.IPluginTool, mobase.IPlugin): + desc = sub_name = "" + icon_bytes: bytes + + def __init__(self): + mobase.IPluginTool.__init__(self) + mobase.IPlugin.__init__(self) + self._pluginName = self._displayName = "BG3 Tools" + self._pluginVersion = mobase.VersionInfo(1, 0, 0) + pixmap = QPixmap() + pixmap.loadFromData(self.icon_bytes, "SVG") + self.qicon = QIcon(pixmap) + + def init(self, organizer: mobase.IOrganizer) -> bool: + self._organizer = organizer + return True + + def version(self): + return self._pluginVersion + + def author(self): + return "daescha" + + def name(self): + return f"{self._pluginName}: {self.sub_name}" + + def displayName(self): + return f"{self._displayName}/{self.sub_name}" + + def tooltip(self): + return self.description() + + def enabledByDefault(self): + return self._organizer.managedGame().name() == "Baldur's Gate 3 Plugin" + + def settings(self) -> list[mobase.PluginSetting]: + return [] + + def icon(self) -> QIcon: + return self.qicon + + def description(self) -> str: + return QCoreApplication.translate(self._pluginName, self.desc) diff --git a/libs/basic_games/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py b/libs/basic_games/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py new file mode 100644 index 0000000..b534e8a --- /dev/null +++ b/libs/basic_games/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py @@ -0,0 +1,15 @@ +from .bg3_tool_plugin import BG3ToolPlugin +from .icons import download + + +class BG3ToolCheckForLsLibUpdates(BG3ToolPlugin): + icon_bytes = download + sub_name = "Check For LsLib Updates" + desc = "Check to see if there has been a new release of LSLib and create download dialog if so." + + def display(self): + from ...game_baldursgate3 import BG3Game + + game_plugin = self._organizer.managedGame() + if isinstance(game_plugin, BG3Game): + game_plugin.utils.lslib_retriever.download_lslib_if_missing(True) diff --git a/libs/basic_games/games/baldursgate3/plugins/convert_jsons_to_yaml_plugin.py b/libs/basic_games/games/baldursgate3/plugins/convert_jsons_to_yaml_plugin.py new file mode 100644 index 0000000..fde3431 --- /dev/null +++ b/libs/basic_games/games/baldursgate3/plugins/convert_jsons_to_yaml_plugin.py @@ -0,0 +1,58 @@ +import json +import os +from pathlib import Path + +from PyQt6.QtCore import qInfo, qWarning +from PyQt6.QtWidgets import QApplication + +from .bg3_tool_plugin import BG3ToolPlugin +from .icons import exchange + + +class BG3ToolConvertJsonsToYaml(BG3ToolPlugin): + icon_bytes = exchange + sub_name = "Convert JSONS to YAML" + desc = "Convert all jsons in active mods to yaml immediately." + + def display(self): + from ...game_baldursgate3 import BG3Game + + game_plugin = self._organizer.managedGame() + if not isinstance(game_plugin, BG3Game): + return + utils = game_plugin.utils + qInfo("converting all json files to yaml") + active_mods = utils.active_mods() + progress = utils.create_progress_window( + "Converting all json files to yaml", len(active_mods) + 1 + ) + for mod in active_mods: + _convert_jsons_in_dir_to_yaml(Path(mod.absolutePath())) + progress.setValue(progress.value() + 1) + QApplication.processEvents() + if progress.wasCanceled(): + qWarning("conversion canceled by user") + return + _convert_jsons_in_dir_to_yaml(utils.overwrite_path) + progress.setValue(len(active_mods) + 1) + QApplication.processEvents() + progress.close() + + +def _convert_jsons_in_dir_to_yaml(path: Path): + for file in list(path.rglob("*.json")): + converted_path = file.parent / file.name.replace(".json", ".yaml") + try: + if not converted_path.exists() or os.path.getmtime(file) > os.path.getmtime( + converted_path + ): + import yaml + + with open(file, "r") as json_file: + with open(converted_path, "w") as yaml_file: + yaml.dump( + json.load(json_file), yaml_file, indent=2, sort_keys=False + ) + qInfo(f"Converted {file} to YAML") + except OSError as e: + qWarning(f"Error accessing file {converted_path}: {e}") diff --git a/libs/basic_games/games/baldursgate3/plugins/icons.py b/libs/basic_games/games/baldursgate3/plugins/icons.py new file mode 100644 index 0000000..adc1023 --- /dev/null +++ b/libs/basic_games/games/baldursgate3/plugins/icons.py @@ -0,0 +1,42 @@ +refresh = b""" + + + + + + + +""" + +exchange = b""" + + + + + + + + + + +""" + +download = b""" + + + + + + + + + + +""" diff --git a/libs/basic_games/games/baldursgate3/plugins/reparse_pak_metadata_plugin.py b/libs/basic_games/games/baldursgate3/plugins/reparse_pak_metadata_plugin.py new file mode 100644 index 0000000..f32a8a0 --- /dev/null +++ b/libs/basic_games/games/baldursgate3/plugins/reparse_pak_metadata_plugin.py @@ -0,0 +1,17 @@ +from .bg3_tool_plugin import BG3ToolPlugin +from .icons import refresh + + +class BG3ToolReparsePakMetadata(BG3ToolPlugin): + icon_bytes = refresh + sub_name = "Reparse Pak Metadata" + desc = "Force reparsing mod metadata immediately." + + def display(self): + from ...game_baldursgate3 import BG3Game + + game_plugin = self._organizer.managedGame() + if isinstance(game_plugin, BG3Game): + game_plugin.utils.construct_modsettings_xml( + exec_path="bin/bg3", force_reparse_metadata=True + ) -- cgit v1.3.1