Replace LsLib via proton with native larian-formats Python package

9 files changed, 25 insertions, 426 deletions

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 13a67d5..309d09b 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -71,7 +71,7 @@ ENV PATH="/opt/python-bundled/bin:${PATH}"
 # ── Build-time Python packages ──
 # pybind11/sip/etc. for compiling mobase.so; PyQt6 is staged into the distribution.
 RUN /opt/python-bundled/bin/pip3 install --no-cache-dir \
-      pybind11==2.13.6 sip psutil vdf "PyQt6>=6.10,<6.11"
+      pybind11==2.13.6 sip psutil larian-formats==0.2.0 vdf "PyQt6>=6.10,<6.11"
 
 # ── Rust toolchain ──
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
diff --git a/docker/build-inner.sh b/docker/build-inner.sh
index cd6c3e1..865e0b4 100755
--- a/docker/build-inner.sh
+++ b/docker/build-inner.sh
@@ -289,7 +289,7 @@ rm -f  "${PYTHON_OUT}/lib/python3.12/turtle.py"
 rm -rf "${PYTHON_OUT}/lib/python3.12/site-packages"
 mkdir -p "${PYTHON_OUT}/lib/python3.12/site-packages"
 # Copy runtime-required packages back in
-for pkg in psutil vdf; do
+for pkg in psutil vdf larian_formats; do
     pkg_dir="$("${PBS_SRC}/bin/python3" -c "import importlib.util; s=importlib.util.find_spec('${pkg}'); print(s.submodule_search_locations[0] if s and s.submodule_search_locations else (s.origin if s else ''))" 2>/dev/null || true)"
     if [ -d "${pkg_dir}" ]; then
         cp -a "${pkg_dir}" "${PYTHON_OUT}/lib/python3.12/site-packages/"
diff --git a/libs/basic_games/games/baldursgate3/bg3_utils.py b/libs/basic_games/games/baldursgate3/bg3_utils.py
index 958d173..bbe5cda 100644
--- a/libs/basic_games/games/baldursgate3/bg3_utils.py
+++ b/libs/basic_games/games/baldursgate3/bg3_utils.py
@@ -73,9 +73,8 @@ class BG3Utils:
     def __init__(self, name: str):
         self.main_window = None
         self._name = name
-        from . import lslib_retriever, pak_parser
+        from . import pak_parser
 
-        self.lslib_retriever = lslib_retriever.LSLibRetriever(self)
         self._pak_parser = pak_parser.BG3PakParser(self)
 
     def init(self, organizer: mobase.IOrganizer):
@@ -197,10 +196,7 @@ class BG3Utils:
         args: str = "",
         force_reparse_metadata: bool = False,
     ) -> bool:
-        if (
-            "bin/bg3" not in exec_path
-            or not self.lslib_retriever.download_lslib_if_missing()
-        ):
+        if "bin/bg3" not in exec_path:
             return True
         active_mods = self.active_mods()
         progress = self.create_progress_window(
@@ -262,8 +258,7 @@ class BG3Utils:
         return True
 
     def on_mod_installed(self, mod: mobase.IModInterface) -> None:
-        if self.lslib_retriever.download_lslib_if_missing():
-            self._pak_parser.get_metadata_for_files_in_mod(mod, True)
+        self._pak_parser.get_metadata_for_files_in_mod(mod, True)
 
 
 def create_dir_if_needed(path: Path) -> Path:
diff --git a/libs/basic_games/games/baldursgate3/lslib_retriever.py b/libs/basic_games/games/baldursgate3/lslib_retriever.py
deleted file mode 100644
index 8b0e1dc..0000000
--- a/libs/basic_games/games/baldursgate3/lslib_retriever.py
+++ /dev/null
@@ -1,193 +0,0 @@
-import json
-import shutil
-import traceback
-import urllib.request
-import zipfile
-from functools import cached_property
-
-from PyQt6.QtCore import qDebug, qWarning
-from PyQt6.QtWidgets import QApplication, QMessageBox
-
-from . import bg3_utils
-
-
-class LSLibRetriever:
-    def __init__(self, utils: bg3_utils.BG3Utils):
-        self._utils = utils
-
-    # Files that MUST be present for Divine to work.
-    _REQUIRED_FILES = {
-        "Divine.dll",
-        "Divine.exe",
-        "LSLib.dll",
-    }
-
-    # Additional files that may or may not be in a given LSLib release.
-    _OPTIONAL_FILES = {
-        "CommandLineArgumentsParser.dll",
-        "Divine.dll.config",
-        "Divine.runtimeconfig.json",
-        "K4os.Compression.LZ4.dll",
-        "K4os.Compression.LZ4.Streams.dll",
-        "LSLibNative.dll",
-        "LZ4.dll",
-        "Newtonsoft.Json.dll",
-        "System.IO.Hashing.dll",
-        "ZstdSharp.dll",
-    }
-
-    @cached_property
-    def _needed_lslib_files(self):
-        return {
-            self._utils.tools_dir / x
-            for x in self._REQUIRED_FILES | self._OPTIONAL_FILES
-        }
-
-    def download_lslib_if_missing(self, force: bool = False) -> bool:
-        # Only check required files to avoid re-downloading when optional
-        # files were not present in the archive.
-        required_paths = {
-            self._utils.tools_dir / x for x in self._REQUIRED_FILES
-        }
-        if not force and all(x.exists() for x in required_paths):
-            return True
-        try:
-            self._utils.tools_dir.mkdir(exist_ok=True, parents=True)
-            downloaded = False
-
-            def reporthook(block_num: int, block_size: int, total_size: int) -> None:
-                if total_size > 0:
-                    progress.setValue(
-                        min(int(block_num * block_size * 100 / total_size), 100)
-                    )
-                    QApplication.processEvents()
-
-            with urllib.request.urlopen(
-                "https://api.github.com/repos/Norbyte/lslib/releases/latest"
-            ) as response:
-                assets = json.loads(response.read().decode("utf-8"))["assets"][0]
-                zip_path = self._utils.tools_dir / assets["name"]
-                if not zip_path.exists():
-                    old_archives = list(self._utils.tools_dir.glob("*.zip"))
-                    msg_box = QMessageBox(self._utils.main_window)
-                    msg_box.setWindowTitle(
-                        self._utils.tr("Baldur's Gate 3 Plugin - Missing dependencies")
-                    )
-                    if old_archives:
-                        msg_box.setText(self._utils.tr("LSLib update available."))
-                    else:
-                        msg_box.setText(
-                            self._utils.tr(
-                                "LSLib tools are missing.\nThese are necessary for the plugin to create the load order file for BG3."
-                            )
-                        )
-                    msg_box.addButton(
-                        self._utils.tr("Download"),
-                        QMessageBox.ButtonRole.DestructiveRole,
-                    )
-                    exit_btn = msg_box.addButton(
-                        self._utils.tr("Exit"), QMessageBox.ButtonRole.ActionRole
-                    )
-                    msg_box.setIcon(QMessageBox.Icon.Warning)
-                    msg_box.exec()
-
-                    if msg_box.clickedButton() == exit_btn:
-                        if not old_archives:
-                            err = QMessageBox(self._utils.main_window)
-                            err.setIcon(QMessageBox.Icon.Critical)
-                            err.setText(
-                                "LSLib tools are required for the proper generation of the modsettings.xml file, file will not be generated"
-                            )
-                            err.exec()
-                            return False
-                    else:
-                        progress = self._utils.create_progress_window(
-                            "Downloading LSLib", 100, cancelable=False
-                        )
-                        urllib.request.urlretrieve(
-                            assets["browser_download_url"], str(zip_path), reporthook
-                        )
-                        progress.close()
-                        downloaded = True
-                        for archive in old_archives:
-                            archive.unlink()
-                        old_archives = []
-                else:
-                    old_archives = []
-                    new_msg = QMessageBox(self._utils.main_window)
-                    new_msg.setIcon(QMessageBox.Icon.Information)
-                    new_msg.setText(
-                        self._utils.tr("Latest version of LSLib already downloaded!")
-                    )
-                    new_msg.exec()
-
-        except Exception as e:
-            qDebug(f"Download failed: {e}")
-            err = QMessageBox(self._utils.main_window)
-            err.setIcon(QMessageBox.Icon.Critical)
-            err.setText(
-                self._utils.tr(
-                    f"Failed to download LSLib tools:\n{traceback.format_exc()}"
-                )
-            )
-            err.exec()
-            return False
-        try:
-            if old_archives:
-                zip_path = sorted(old_archives)[-1]
-            if old_archives or not downloaded:
-                dialog_message = "Ensuring all necessary LSLib files have been extracted from archive..."
-                win_title = "Verifying LSLib files"
-            else:
-                dialog_message = "Extracting/Updating LSLib files..."
-                win_title = "Extracting LSLib"
-            x_progress = self._utils.create_progress_window(
-                win_title, len(self._needed_lslib_files), msg=dialog_message
-            )
-            with zipfile.ZipFile(zip_path, "r") as zip_ref:
-                # Build a lookup from basename → zip entry so we can handle
-                # any zip directory layout (Packed/Tools/, Tools/, flat, etc.)
-                zip_names = zip_ref.namelist()
-                entry_map: dict[str, str] = {}
-                for zn in zip_names:
-                    base = zn.rsplit("/", 1)[-1] if "/" in zn else zn
-                    if base:
-                        entry_map[base] = zn
-
-                for file in self._needed_lslib_files:
-                    if downloaded or not file.exists():
-                        zip_entry = entry_map.get(file.name)
-                        if zip_entry is None:
-                            if file.name in self._REQUIRED_FILES:
-                                raise KeyError(
-                                    f"Required file '{file.name}' not found in archive"
-                                )
-                            qDebug(f"LSLib: optional file '{file.name}' not in archive, skipping")
-                            continue
-                        extracted = zip_ref.extract(zip_entry, self._utils.tools_dir)
-                        # Move from nested zip path to flat tools_dir
-                        if extracted != str(file):
-                            shutil.move(extracted, file)
-                    x_progress.setValue(x_progress.value() + 1)
-                    QApplication.processEvents()
-                    if x_progress.wasCanceled():
-                        qWarning("processing canceled by user")
-                        return False
-            x_progress.close()
-            # Clean up any nested directories left from extraction
-            for subdir in ("Packed", "Tools"):
-                nested = self._utils.tools_dir / subdir
-                if nested.is_dir():
-                    shutil.rmtree(nested, ignore_errors=True)
-        except Exception as e:
-            qDebug(f"Extraction failed: {e}")
-            err = QMessageBox(self._utils.main_window)
-            err.setIcon(QMessageBox.Icon.Critical)
-            err.setText(
-                self._utils.tr(
-                    f"Failed to extract LSLib tools:\n{traceback.format_exc()}"
-                )
-            )
-            err.exec()
-            return False
-        return True
diff --git a/libs/basic_games/games/baldursgate3/pak_parser.py b/libs/basic_games/games/baldursgate3/pak_parser.py
index 533c72e..fab4c89 100644
--- a/libs/basic_games/games/baldursgate3/pak_parser.py
+++ b/libs/basic_games/games/baldursgate3/pak_parser.py
@@ -14,6 +14,7 @@ from typing import Callable
 from xml.etree import ElementTree
 from xml.etree.ElementTree import Element
 
+import larian_formats
 from PyQt6.QtCore import (
     qDebug,
     qInfo,
@@ -40,42 +41,6 @@ class BG3PakParser:
     }
 
     @cached_property
-    def _divine_command(self):
-        divine_exe = self._utils.tools_dir / "Divine.exe"
-        if platform.system() != "Windows":
-            wine = self._find_proton_wine()
-            if wine:
-                return f'"{wine}" "{divine_exe}" -g bg3 -l info'
-            qWarning(
-                "BG3: could not find Proton/Wine to run Divine.exe. "
-                "Ensure a Proton version is configured in Settings > Proton."
-            )
-        return f'"{divine_exe}" -g bg3 -l info'
-
-    def _find_proton_wine(self) -> str | None:
-        """Locate the wine binary from the configured Proton installation."""
-        try:
-            config_dir = os.environ.get(
-                "XDG_CONFIG_HOME", os.path.join(Path.home(), ".config")
-            )
-            cfg_path = os.path.join(config_dir, "fluorine", "config.json")
-            if not os.path.isfile(cfg_path):
-                return None
-            import json as _json
-            with open(cfg_path, "r") as f:
-                cfg = _json.load(f)
-            proton_path = cfg.get("proton_path", "")
-            if not proton_path:
-                return None
-            for subdir in ("files/bin/wine", "dist/bin/wine"):
-                candidate = os.path.join(proton_path, subdir)
-                if os.path.isfile(candidate):
-                    return candidate
-        except Exception as e:
-            qDebug(f"BG3: failed to read Fluorine config for Proton wine: {e}")
-        return None
-
-    @cached_property
     def _folder_pattern(self):
         return re.compile("Data|Script Extender|bin|Mods")
 
@@ -113,66 +78,17 @@ class BG3PakParser:
         config.read(meta_ini, encoding="utf-8")
         try:
             if file.name.endswith("pak"):
-                meta_file = (
-                    self._utils.plugin_data_path
-                    / "temp"
-                    / "extracted_metadata"
-                    / f"{file.name[: int(len(file.name) / 2)]}-{hashlib.md5(str(file).encode(), usedforsecurity=False).hexdigest()[:5]}.lsx"
-                )
-                try:
-                    if (
-                        not force_reparse_metadata
-                        and config.has_section(file.name)
-                        and (
-                            "override" in config[file.name].keys()
-                            or "Folder" in config[file.name].keys()
-                        )
-                    ):
-                        return get_module_short_desc(config, file)
-                    meta_file.parent.mkdir(parents=True, exist_ok=True)
-                    meta_file.unlink(missing_ok=True)
-                    out_dir = (
-                        str(meta_file)[:-4] if self._utils.extract_full_package else ""
+                if (
+                    not force_reparse_metadata
+                    and config.has_section(file.name)
+                    and (
+                        "override" in config[file.name].keys()
+                        or "Folder" in config[file.name].keys()
                     )
-                    can_continue = True
-                    if self.run_divine(
-                        f'{"extract-package" if self._utils.extract_full_package else "extract-single-file -f meta.lsx"} -d "{meta_file if not self._utils.extract_full_package else out_dir}"',
-                        file,
-                    ).returncode:
-                        can_continue = False
-                    if can_continue and self._utils.extract_full_package:
-                        qDebug(f"archive {file} extracted to {out_dir}")
-                        if self.run_divine(
-                            f'convert-resources -d "{out_dir}" -i lsf -o lsx -x "*.lsf"',
-                            out_dir,
-                        ).returncode:
-                            qDebug(
-                                f"failed to convert lsf files in {out_dir} to readable lsx"
-                            )
-                        extracted_meta_files = list(Path(out_dir).rglob("meta.lsx"))
-                        if len(extracted_meta_files) == 0:
-                            qInfo(
-                                f"No meta.lsx files found in {file.name}, {file.name} determined to be an override mod"
-                            )
-                            can_continue = False
-                        else:
-                            shutil.copyfile(
-                                extracted_meta_files[0],
-                                meta_file,
-                            )
-                    elif can_continue and not meta_file.exists():
-                        qInfo(
-                            f"No meta.lsx files found in {file.name}, {file.name} determined to be an override mod"
-                        )
-                        can_continue = False
-                    return self.metadata_to_ini(
-                        config, file, mod, meta_ini, can_continue, lambda: meta_file
-                    )
-                finally:
-                    if self._utils.remove_extracted_metadata:
-                        meta_file.unlink(missing_ok=True)
-                        if self._utils.extract_full_package:
-                            Path(str(meta_file)[:-4]).unlink(missing_ok=True)
+                ):
+                    return get_module_short_desc(config, file)
+
+                return self.metadata_to_ini(config, file, mod, meta_ini)
             elif file.is_dir():
                 if self._folder_pattern.search(file.name):
                     return ""
@@ -214,18 +130,13 @@ class BG3PakParser:
                         build_pak = False
                 if build_pak:
                     pak_path.unlink(missing_ok=True)
-                    if self.run_divine(
-                        f'create-package -d "{pak_path}"', file
-                    ).returncode:
-                        return ""
-                meta_files = list(file.glob("Mods/*/meta.lsx"))
+                    larian_formats.pack_loose_files(file, pak_path)
+
                 return self.metadata_to_ini(
                     config,
-                    file,
+                    pak_path,
                     mod,
                     meta_ini,
-                    len(meta_files) > 0,
-                    lambda: meta_files[0],
                 )
             else:
                 return ""
@@ -233,68 +144,6 @@ class BG3PakParser:
             qWarning(traceback.format_exc())
             return ""
 
-    @staticmethod
-    def _to_wine_path(path: str) -> str:
-        """Convert a Linux absolute path to a Wine Z: drive path."""
-        if path.startswith("/"):
-            return "Z:" + path.replace("/", "\\")
-        return path
-
-    def run_divine(
-        self, action: str, source: Path | str
-    ) -> subprocess.CompletedProcess[str]:
-        if platform.system() != "Windows":
-            # Divine.exe is a .NET Windows app — it needs Windows-style paths.
-            # Wine maps Z:\ to the Linux root, so /home/... becomes Z:\home\...
-            wine_source = self._to_wine_path(str(source))
-            wine_action = re.sub(
-                r'"(/[^"]*)"',
-                lambda m: '"' + self._to_wine_path(m.group(1)) + '"',
-                action,
-            )
-            command = f'{self._divine_command} -a {wine_action} -s "{wine_source}"'
-        else:
-            command = f'{self._divine_command} -a {action} -s "{source}"'
-        kwargs: dict = dict(
-            check=False,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
-        )
-        if platform.system() == "Windows":
-            kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
-            kwargs["shell"] = True
-            result = subprocess.run(command, **kwargs)
-        else:
-            # Set WINEPREFIX so Proton's wine uses the game's prefix.
-            wine_prefix = ""
-            try:
-                config_dir = os.environ.get(
-                    "XDG_CONFIG_HOME", os.path.join(Path.home(), ".config")
-                )
-                cfg_path = os.path.join(config_dir, "fluorine", "config.json")
-                if os.path.isfile(cfg_path):
-                    import json as _json
-                    with open(cfg_path, "r") as f:
-                        cfg = _json.load(f)
-                    wine_prefix = cfg.get("prefix_path", "")
-            except Exception:
-                pass
-
-            env = os.environ.copy()
-            if wine_prefix:
-                env["WINEPREFIX"] = wine_prefix
-            kwargs["env"] = env
-            kwargs["shell"] = True
-            result = subprocess.run(command, **kwargs)
-
-        if result.returncode:
-            qWarning(
-                f"{command.replace(str(Path.home()), '~', 1).replace(str(Path.home()), '$HOME')}"
-                f" returned stdout: {result.stdout}, stderr: {result.stderr}, code {result.returncode}"
-            )
-        return result
-
     def get_attr_value(self, root: Element, attr_id: str) -> str:
         default_val = self._types.get(attr_id) or ""
         attr = root.find(f".//attribute[@id='{attr_id}']")
@@ -306,49 +155,12 @@ class BG3PakParser:
         file: Path,
         mod: mobase.IModInterface,
         meta_ini: Path,
-        condition: bool,
-        to_parse: Callable[[], Path],
     ):
         config[file.name] = {}
-        if condition:
-            root = (
-                ElementTree.parse(to_parse())
-                .getroot()
-                .find(".//node[@id='ModuleInfo']")
-            )
-            if root is None:
-                qInfo(f"No ModuleInfo node found in meta.lsx for {mod.name()} ")
-            else:
-                section = config[file.name]
-                folder_name = self.get_attr_value(root, "Folder")
-                if file.is_dir():
-                    self._mod_cache[file] = (
-                        len(list(file.glob(f"*/{folder_name}/**"))) > 1
-                        or len(
-                            list(file.glob("Public/Engine/Timeline/MaterialGroups/*"))
-                        )
-                        > 0
-                    )
-                elif file not in self._mod_cache:
-                    # a mod which has a meta.lsx and is not an override mod meets at least one of three conditions:
-                    # 1. it has files in Public/Engine/Timeline/MaterialGroups, or
-                    # 2. it has files in Mods/<folder_name>/ other than the meta.lsx file, or
-                    # 3. it has files in Public/<folder_name>
-                    result = self.run_divine(
-                        f'list-package --use-regex -x "(/{re.escape(folder_name)}/(?!meta\\.lsx))|(Public/Engine/Timeline/MaterialGroups)"',
-                        file,
-                    )
-                    self._mod_cache[file] = (
-                        result.returncode == 0 and result.stdout.strip() != ""
-                    )
-                if self._mod_cache[file]:
-                    for key in self._types:
-                        section[key] = self.get_attr_value(root, key)
-                else:
-                    qInfo(f"pak {file.name} determined to be an override mod")
-                    section["override"] = "True"
-                    section["Folder"] = folder_name
-        else:
+        metadata = larian_formats.get_metadata_for_file(file)
+        config[file.name].update({k: str(v) for k, v in metadata.items()})
+
+        if larian_formats.is_override(file):
             config[file.name]["override"] = "True"
         with open(meta_ini, "w+", encoding="utf-8") as f:
             config.write(f)
diff --git a/libs/basic_games/games/baldursgate3/plugins/__init__.py b/libs/basic_games/games/baldursgate3/plugins/__init__.py
index 1b69faf..f0b4b54 100644
--- a/libs/basic_games/games/baldursgate3/plugins/__init__.py
+++ b/libs/basic_games/games/baldursgate3/plugins/__init__.py
@@ -1,13 +1,11 @@
 import mobase
 
-from .check_for_lslib_updates_plugin import BG3ToolCheckForLsLibUpdates
 from .convert_jsons_to_yaml_plugin import BG3ToolConvertJsonsToYaml
 from .reparse_pak_metadata_plugin import BG3ToolReparsePakMetadata
 
 
 def createPlugins() -> list[mobase.IPluginTool]:
     return [
-        BG3ToolCheckForLsLibUpdates(),
         BG3ToolReparsePakMetadata(),
         BG3ToolConvertJsonsToYaml(),
     ]
diff --git a/libs/basic_games/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py b/libs/basic_games/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py
deleted file mode 100644
index b534e8a..0000000
--- a/libs/basic_games/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from .bg3_tool_plugin import BG3ToolPlugin
-from .icons import download
-
-
-class BG3ToolCheckForLsLibUpdates(BG3ToolPlugin):
-    icon_bytes = download
-    sub_name = "Check For LsLib Updates"
-    desc = "Check to see if there has been a new release of LSLib and create download dialog if so."
-
-    def display(self):
-        from ...game_baldursgate3 import BG3Game
-
-        game_plugin = self._organizer.managedGame()
-        if isinstance(game_plugin, BG3Game):
-            game_plugin.utils.lslib_retriever.download_lslib_if_missing(True)
diff --git a/libs/basic_games/plugin-requirements.txt b/libs/basic_games/plugin-requirements.txt
index f945efd..c31bc1a 100644
--- a/libs/basic_games/plugin-requirements.txt
+++ b/libs/basic_games/plugin-requirements.txt
@@ -2,3 +2,4 @@ psutil==5.8.0
 vdf==3.4
 lzokay==1.1.5
 pyyaml==6.0.2
+larian-formats==0.2.0
diff --git a/libs/basic_games/pyproject.toml b/libs/basic_games/pyproject.toml
index 89778ca..35c5274 100644
--- a/libs/basic_games/pyproject.toml
+++ b/libs/basic_games/pyproject.toml
@@ -16,6 +16,7 @@ package-mode = false
 [tool.poetry.dependencies]
 psutil = "^5.8.0"
 vdf = "3.4"
+larian-formats = "0.2.0"
 lzokay = "1.1.5"
 pyqt6 = "6.7.0"
 pyyaml = "^6.0.2"