diff options
Diffstat (limited to 'libs/bsatk')
28 files changed, 3131 insertions, 0 deletions
diff --git a/libs/bsatk/.clang-format b/libs/bsatk/.clang-format new file mode 100644 index 0000000..6098e1f --- /dev/null +++ b/libs/bsatk/.clang-format @@ -0,0 +1,41 @@ +--- +# We'll use defaults from the LLVM style, but with 4 columns indentation. +BasedOnStyle: LLVM +IndentWidth: 2 +--- +Language: Cpp +DeriveLineEnding: false +UseCRLF: true +DerivePointerAlignment: false +PointerAlignment: Left +AlignConsecutiveAssignments: true +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: Empty +AlwaysBreakTemplateDeclarations: Yes +AccessModifierOffset: -2 +AlignTrailingComments: true +SpacesBeforeTrailingComments: 2 +NamespaceIndentation: Inner +MaxEmptyLinesToKeep: 1 +BreakBeforeBraces: Custom +BraceWrapping: + AfterCaseLabel: false + AfterClass: true + AfterControlStatement: false + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: true + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: true +ColumnLimit: 88 +ForEachMacros: ['Q_FOREACH', 'foreach'] diff --git a/libs/bsatk/.gitattributes b/libs/bsatk/.gitattributes new file mode 100644 index 0000000..f869712 --- /dev/null +++ b/libs/bsatk/.gitattributes @@ -0,0 +1,7 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Explicitly declare text files you want to always be normalized and converted +# to native line endings on checkout. +*.cpp text eol=crlf +*.h text eol=crlf diff --git a/libs/bsatk/.github/workflows/build.yml b/libs/bsatk/.github/workflows/build.yml new file mode 100644 index 0000000..fdad926 --- /dev/null +++ b/libs/bsatk/.github/workflows/build.yml @@ -0,0 +1,40 @@ +name: Build BSATK + +on: + push: + branches: [master] + pull_request: + types: [opened, synchronize, reopened] + +env: + VCPKG_BINARY_SOURCES: ${{ vars.AZ_BLOB_VCPKG_URL != '' && format('clear;x-azblob,{0},{1},readwrite', vars.AZ_BLOB_VCPKG_URL, secrets.AZ_BLOB_SAS) || '' }} + +jobs: + build: + runs-on: windows-2022 + steps: + # set VCPKG Root + - name: "Set environmental variables" + shell: bash + run: | + echo "VCPKG_ROOT=$VCPKG_INSTALLATION_ROOT" >> $GITHUB_ENV + + # checkout + - uses: actions/checkout@v4 + + - name: Configure BSATK build + shell: pwsh + run: | + cmake --preset vs2022-windows "-DCMAKE_INSTALL_PREFIX=install" + + - name: Build BSATK + run: cmake --build vsbuild --config RelWithDebInfo + + - name: Install BSATK + run: cmake --install vsbuild --config RelWithDebInfo + + - name: Upload BSATK artifact + uses: actions/upload-artifact@master + with: + name: BSATK + path: ./install diff --git a/libs/bsatk/.github/workflows/linting.yml b/libs/bsatk/.github/workflows/linting.yml new file mode 100644 index 0000000..6f01385 --- /dev/null +++ b/libs/bsatk/.github/workflows/linting.yml @@ -0,0 +1,16 @@ +name: Lint BSATK + +on: + push: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Check format + uses: ModOrganizer2/check-formatting-action@master + with: + check-path: "." diff --git a/libs/bsatk/.gitignore b/libs/bsatk/.gitignore new file mode 100644 index 0000000..cf71be7 --- /dev/null +++ b/libs/bsatk/.gitignore @@ -0,0 +1,5 @@ +edit +CMakeLists.txt.user +/msbuild.log +/*std*.log +/*build diff --git a/libs/bsatk/.pre-commit-config.yaml b/libs/bsatk/.pre-commit-config.yaml new file mode 100644 index 0000000..eb8969c --- /dev/null +++ b/libs/bsatk/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-merge-conflict + - id: check-case-conflict + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v21.1.8 + hooks: + - id: clang-format + 'types_or': [c++, c] + +ci: + autofix_commit_msg: "[pre-commit.ci] Auto fixes from pre-commit.com hooks." + autofix_prs: true + autoupdate_commit_msg: "[pre-commit.ci] Pre-commit autoupdate." + autoupdate_schedule: quarterly + submodules: false diff --git a/libs/bsatk/CMakeFiles/CMakeSystem.cmake b/libs/bsatk/CMakeFiles/CMakeSystem.cmake new file mode 100644 index 0000000..c9c4e81 --- /dev/null +++ b/libs/bsatk/CMakeFiles/CMakeSystem.cmake @@ -0,0 +1,15 @@ +set(CMAKE_HOST_SYSTEM "Linux-6.18.9-2-cachyos") +set(CMAKE_HOST_SYSTEM_NAME "Linux") +set(CMAKE_HOST_SYSTEM_VERSION "6.18.9-2-cachyos") +set(CMAKE_HOST_SYSTEM_PROCESSOR "x86_64") + + + +set(CMAKE_SYSTEM "Linux-6.18.9-2-cachyos") +set(CMAKE_SYSTEM_NAME "Linux") +set(CMAKE_SYSTEM_VERSION "6.18.9-2-cachyos") +set(CMAKE_SYSTEM_PROCESSOR "x86_64") + +set(CMAKE_CROSSCOMPILING "FALSE") + +set(CMAKE_SYSTEM_LOADED 1) diff --git a/libs/bsatk/CMakeLists.txt b/libs/bsatk/CMakeLists.txt new file mode 100644 index 0000000..571ce1f --- /dev/null +++ b/libs/bsatk/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 3.16) + +include(CMakePackageConfigHelpers) + +project(bsatk) + +add_subdirectory(src) + +configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/config.cmake.in + "${CMAKE_CURRENT_BINARY_DIR}/mo2-bsatk-config.cmake" + INSTALL_DESTINATION "lib/cmake/mo2-bsatk" + NO_SET_AND_CHECK_MACRO + NO_CHECK_REQUIRED_COMPONENTS_MACRO +) + +include(${CMAKE_CURRENT_SOURCE_DIR}/Version.cmake) +set(BSATK_VERSION + ${BSATK_MAJOR_VERSION}.${BSATK_MINOR_VERSION}.${BSATK_PATCH_VERSION}) + +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/mo2-bsatk-config-version.cmake" + VERSION "${BSATK_VERSION}" + COMPATIBILITY AnyNewerVersion +) + +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/mo2-bsatk-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/mo2-bsatk-config-version.cmake + DESTINATION lib/cmake/mo2-bsatk +) diff --git a/libs/bsatk/CMakePresets.json b/libs/bsatk/CMakePresets.json new file mode 100644 index 0000000..cc009b3 --- /dev/null +++ b/libs/bsatk/CMakePresets.json @@ -0,0 +1,57 @@ +{ + "configurePresets": [ + { + "errors": { + "deprecated": true + }, + "hidden": true, + "name": "cmake-dev", + "warnings": { + "deprecated": true, + "dev": true + } + }, + { + "cacheVariables": { + "VCPKG_MANIFEST_NO_DEFAULT_FEATURES": { + "type": "BOOL", + "value": "ON" + } + }, + "toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake", + "hidden": true, + "name": "vcpkg" + }, + { + "hidden": true, + "inherits": ["vcpkg"], + "name": "vcpkg-dev" + }, + { + "binaryDir": "${sourceDir}/vsbuild", + "architecture": { + "strategy": "set", + "value": "x64" + }, + "cacheVariables": { + "CMAKE_CXX_FLAGS": "/EHsc /MP /W4", + "VCPKG_TARGET_TRIPLET": { + "type": "STRING", + "value": "x64-windows-static-md" + } + }, + "generator": "Visual Studio 17 2022", + "inherits": ["cmake-dev", "vcpkg-dev"], + "name": "vs2022-windows", + "toolset": "v143" + } + ], + "buildPresets": [ + { + "name": "vs2022-windows", + "resolvePackageReferences": "on", + "configurePreset": "vs2022-windows" + } + ], + "version": 4 +} diff --git a/libs/bsatk/README.md b/libs/bsatk/README.md new file mode 100644 index 0000000..c5fe1e2 --- /dev/null +++ b/libs/bsatk/README.md @@ -0,0 +1,3 @@ +[](https://ci.appveyor.com/project/Modorganizer2/modorganizer-bsatk) + +# modorganizer-bsatk diff --git a/libs/bsatk/Version.cmake b/libs/bsatk/Version.cmake new file mode 100644 index 0000000..e38ed2b --- /dev/null +++ b/libs/bsatk/Version.cmake @@ -0,0 +1,3 @@ +set(BSATK_MAJOR_VERSION 1) +set(BSATK_MINOR_VERSION 4) +set(BSATK_PATCH_VERSION 1) diff --git a/libs/bsatk/cmake/config.cmake.in b/libs/bsatk/cmake/config.cmake.in new file mode 100644 index 0000000..ae33254 --- /dev/null +++ b/libs/bsatk/cmake/config.cmake.in @@ -0,0 +1,10 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) + +find_dependency(Boost CONFIG COMPONENTS thread interprocess) +find_dependency(ZLIB) +find_dependency(lz4 CONFIG) +find_dependency(mo2-dds-header CONFIG) + +include ( "${CMAKE_CURRENT_LIST_DIR}/mo2-bsatk-targets.cmake" ) diff --git a/libs/bsatk/include/bsatk/bsaarchive.h b/libs/bsatk/include/bsatk/bsaarchive.h new file mode 100644 index 0000000..cd15f64 --- /dev/null +++ b/libs/bsatk/include/bsatk/bsaarchive.h @@ -0,0 +1,244 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef BSA_ARCHIVE_H +#define BSA_ARCHIVE_H + +#include <functional> +#include <memory> +#include <queue> +#include <vector> + +#include <dxgiformat.h> + +#include <DDS.h> + +#include "bsafolder.h" +#include "bsatypes.h" +#include "errorcodes.h" +namespace boost +{ +class mutex; +namespace interprocess +{ + class interprocess_semaphore; +} +} // namespace boost + +namespace BSA +{ + +class File; + +/** + * @brief top level structure to represent a bsa file + */ +class Archive +{ +public: + using DataBuffer = std::pair<std::shared_ptr<unsigned char[]>, BSAULong>; + +private: + static const unsigned int FLAG_HASDIRNAMES = 0x00000001; + static const unsigned int FLAG_HASFILENAMES = 0x00000002; + static const unsigned int FLAG_DEFAULTCOMPRESSED = 0x00000004; + static const unsigned int FLAG_NAMEPREFIXED = + 0x00000100; // if set, the full file name is prefixed before a data block + + /* Record flags */ +#define OB_BSAFILE_FLAG_COMPRESS \ + 0xC0000000 //!< Bit mask with OBBSAFileInfo::sizeFlags to get the compression status + +public: + /** + * constructor + */ + Archive(); + ~Archive(); + /** + * read the archive from file + * @param fileName name of the file to read from + * @param testHashes if true, the hashes of file names will be checked to ensure the + * file is valid. This can be skipped for performance reasons + * @return ERROR_NONE on success or an error code + */ + EErrorCode read(const char* fileName, bool testHashes); + /** + * write the archive to disc + * @param fileName name of the file to write to + * @return ERROR_NONE on success or an error code + */ + EErrorCode write(const char* fileName); + /** + * @brief close the archive + */ + void close(); + /** + * change the archive type + * @param type new archive type + */ + void setType(ArchiveType type) { m_Type = type; } + /** + * @return type of the archive (supported game) + */ + ArchiveType getType() const { return m_Type; } + /** + * retrieve top-level folder + * @return descriptor of the root folder + */ + Folder::Ptr getRoot() { return m_RootFolder; } + /** + * extract a file from the archive + * @param file descriptor of the file to extract + * @param outputDirectory name of the directory to extract to. + * may be absolute or relative + * @return ERROR_NONE on success or an error code + */ + EErrorCode extract(File::Ptr file, const char* outputDirectory) const; + /** + * @return archive flags + */ + BSAULong getFlags() const { return m_ArchiveFlags; } + + /** + * extract all files. this is potentially faster than iterating over all files and + * extracting each + * @param outputDirectory name of the directory to extract to. + * may be absolute or relative + * @param progress callback function called on progress + * @param overwrite if true (default) files are overwritten if they exist + * @return ERROR_NONE on success or an error code + */ + EErrorCode + extractAll(const char* outputDirectory, + const std::function<bool(int value, std::string fileName)>& progress, + bool overwrite = true); + + /** + * @param file the file to check + * @return true if the file is compressed, false otherwise + */ + bool compressed(const File::Ptr& file) const; + /** + * create a new file to be placed in this archive. The new file is NOT + * added to a folder, use BSA::Folder::addFile for that + * @param name name of the file to be used inside the archive + * @param sourceName filename path to the file to add + * @param compressed true if the file should be compressed (not supported yet!) + * @return pointer to the new file + */ + File::Ptr createFile(const std::string& name, const std::string& sourceName, + bool compressed); + +private: + struct Header + { + uint32_t fileIdentifier; + char archType[5]; + ArchiveType type; + BSAUInt offset; + BSAUInt archiveFlags; + BSAUInt folderCount; + BSAUInt fileCount; + BSAUInt folderNameLength; + BSAUInt fileNameLength; + BSAUInt fileFlags; + BSAHash nameTableOffset; + }; + + struct FileInfo + { + File::Ptr file; + DataBuffer data; + }; + +private: + static Header readHeader(std::fstream& infile); + + static ArchiveType typeFromID(BSAULong typeID); + + static std::shared_ptr<unsigned char[]> decompress(unsigned char* inBuffer, + BSAULong inSize, + EErrorCode& result, + BSAULong& outSize); + + BSAULong typeToID(ArchiveType type); + + Folder readFolderRecord(std::fstream& file); + + // EErrorCode extractDirect(const File &fileInfo, std::ofstream &outFile); + // EErrorCode extractCompressed(const File &fileInfo, std::ofstream &outFile); + + bool defaultCompressed() const { return m_ArchiveFlags & FLAG_DEFAULTCOMPRESSED; } + // starting with FO3 the bsa may prefix the file name to the file blob if archive flag + // 0x100 is set + bool namePrefixed() const + { + return (m_Type != TYPE_OBLIVION) && ((m_ArchiveFlags & FLAG_NAMEPREFIXED) != 0); + } + + BSAULong countFiles() const; + + std::vector<std::string> collectFolderNames() const; + std::vector<std::string> collectFileNames() const; + + BSAULong countCharacters(const std::vector<std::string>& list) const; + BSAULong determineFileFlags(const std::vector<std::string>& fileList) const; + + void writeHeader(std::fstream& outfile, BSAULong fileFlags, BSAULong numFolders, + BSAULong folderNamesLength, BSAULong fileNamesLength); + + DirectX::DDS_HEADER getDDSHeader(File::Ptr file, + DirectX::DDS_HEADER_DXT10& DX10Header, + bool& isDX10) const; + void getDX10Header(DirectX::DDS_HEADER_DXT10& DX10Header, File::Ptr file, + DirectX::DDS_HEADER DDSHeader) const; + + EErrorCode extractDirect(File::Ptr file, std::ofstream& outFile) const; + EErrorCode extractCompressed(File::Ptr file, std::ofstream& outFile) const; + + void createFolders(const std::string& targetDirectory, Folder::Ptr folder); + + void readFiles(std::queue<FileInfo>& queue, boost::mutex& mutex, + boost::interprocess::interprocess_semaphore& bufferCount, + boost::interprocess::interprocess_semaphore& queueFree, + std::vector<File::Ptr>::iterator begin, + std::vector<File::Ptr>::iterator end); + + void extractFiles(const std::string& targetDirectory, std::queue<FileInfo>& queue, + boost::mutex& mutex, + boost::interprocess::interprocess_semaphore& bufferCount, + boost::interprocess::interprocess_semaphore& queueFree, + int totalFiles, bool overwrite, int& filesDone); + + void cleanFolder(Folder::Ptr folder); + +private: + mutable std::fstream m_File; + + Folder::Ptr m_RootFolder; + + BSAULong m_ArchiveFlags; + ArchiveType m_Type; +}; + +} // namespace BSA + +#endif // BSA_ARCHIVE_H diff --git a/libs/bsatk/include/bsatk/bsaexception.h b/libs/bsatk/include/bsatk/bsaexception.h new file mode 100644 index 0000000..1624f4a --- /dev/null +++ b/libs/bsatk/include/bsatk/bsaexception.h @@ -0,0 +1,51 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef BSAEXCEPTION_H +#define BSAEXCEPTION_H + +#include <string> + +/** + * construct a string from a printf-style format + * @param format printf-style format to create the string from + * @param ... variable parameter list + * @return the constructed string + */ +std::string makeString(const char* format, ...); + +/** + * custom exception to be thrown when invalid data is encountered + */ +class data_invalid_exception : public std::exception +{ + +public: + explicit data_invalid_exception(const std::string& message); + + virtual ~data_invalid_exception() noexcept {} + + virtual const char* what() const noexcept { return m_Message.c_str(); } + +private: + std::string m_Message; +}; + +#endif // BSAEXCEPTION_H diff --git a/libs/bsatk/include/bsatk/bsafile.h b/libs/bsatk/include/bsatk/bsafile.h new file mode 100644 index 0000000..64fbf85 --- /dev/null +++ b/libs/bsatk/include/bsatk/bsafile.h @@ -0,0 +1,145 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef BSAFILE_H +#define BSAFILE_H + +#include <fstream> +#include <memory> +#include <vector> + +#include "errorcodes.h" +#include "filehash.h" + +namespace BSA +{ + +class Folder; + +class File +{ + + friend class Folder; + friend class Archive; + +public: + typedef std::shared_ptr<File> Ptr; + friend bool ByOffset(const File::Ptr& LHS, const File::Ptr& RHS); + +private: + static const unsigned int SIZEMASK = 0x3fffffff; + static const unsigned int COMPRESSMASK = 0xC0000000; + +public: + /** + * @return the name of the file + */ + const std::string& getName() const { return m_Name; } + /** + * @return full path of this file within the archive + */ + std::string getFilePath() const; + /** + * @return size of the file. If the source is an archive and the file is + * compressed, this returns the compressed size! + */ + BSAULong getFileSize() const { return m_FileSize; } + + BSAULong getUncompressedFileSize() const { return m_UncompressedFileSize; } + +private: + // copy constructor not implemented + File(const File& reference); + + // assignment operator not implemented + File& operator=(const File& reference); + + /** + * construct file from source archive + * @param file a file read from + * @param folder the folder to add the file to + * @param type the archive type we're parsing + */ + File(std::fstream& file, Folder* folder); + + /** + * construct file from morrowind BSA or BA2 + * @param name of the base file from source archive + * @param folder the folder to add the file to + * @param fileSize the file size of the file in the archive + * @param dataOffset the offset of the file data in the archive + * @param toggleCompressed the detected compression mode of the archive + */ + File(const std::string& name, Folder* folder, BSAULong fileSizee, BSAHash dataOffset, + BSAULong uncompressedFileSize, FO4TextureHeader header, + std::vector<FO4TextureChunk>& texChunks); + + /** + * construct from loose file + * @param name the base name of the file inside the archive + * @param sourceFile the file to read from + * @param folder the folder to add the file to + * @param toggleCompressed if true, the default compression mode of the + * archive is overwritten + */ + File(const std::string& name, const std::string& sourceFile, Folder* folder, + bool toggleCompressed); + + /** + * @return true if its compression mode for this file differs from the archive default + */ + bool compressToggled() const { return m_ToggleCompressed; } + + /** + * @return offset to the file data. Only valid if the source of the file is + * an archive + */ + BSAHash getDataOffset() const { return m_DataOffset; } + void writeHeader(std::fstream& file) const; + EErrorCode writeData(std::fstream& sourceArchive, std::fstream& targetArchive) const; + + void setFileSize(BSAULong fileSize) { m_FileSize = fileSize; } + + void readFileName(std::fstream& file, bool testHashes); + +private: + Folder* m_Folder; + bool m_New; + + BSAHash m_NameHash; + std::string m_Name; + mutable BSAULong m_FileSize; + mutable BSAULong m_UncompressedFileSize; + BSAHash m_DataOffset; + bool m_ToggleCompressed; + + FO4TextureHeader m_TextureHeader = {}; + std::vector<FO4TextureChunk> m_TextureChunks; + + std::string m_SourceFile; + bool m_ToggleCompressedWrite; + mutable BSAULong m_DataOffsetWrite; +}; + +extern bool ByOffset(const File::Ptr& LHS, const File::Ptr& RHS); + +} // namespace BSA + +#endif // BSAFILE_H diff --git a/libs/bsatk/include/bsatk/bsafolder.h b/libs/bsatk/include/bsatk/bsafolder.h new file mode 100644 index 0000000..f1afa29 --- /dev/null +++ b/libs/bsatk/include/bsatk/bsafolder.h @@ -0,0 +1,170 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef BSAFOLDER_H +#define BSAFOLDER_H + +#include "bsafile.h" +#include "bsatypes.h" +#include "errorcodes.h" +#include <memory> +#include <string> +#include <unordered_map> +#include <vector> + +namespace BSA +{ + +class Folder +{ + + friend class Archive; + +public: + typedef std::shared_ptr<Folder> Ptr; + +public: + /** + * @return name of this folder + */ + const std::string& getName() const { return m_Name; } + /** + * @return full path to this folder + */ + std::string getFullPath() const; + /** + * @return the number of subfolders within this folder + */ + unsigned int getNumSubFolders() const + { + return static_cast<unsigned int>(m_SubFolders.size()); + } + /** + * @param index index of a subfolder within this folder + * @return a descriptor for the subfolder + * @throw out_of_range this will throw an exception if the index is invalid + */ + const Folder::Ptr getSubFolder(unsigned int index) const; + /** + * @return the number of files in this folder + */ + unsigned int getNumFiles() const { return static_cast<unsigned int>(m_Files.size()); } + /** + * @return the number of files in this folder and subfolder + */ + unsigned int countFiles() const; + /** + * @param index index of a file in this folder + * @return a descriptor for the file + * @throw out_of_range this will throw an exception if the index is invalid + */ + const File::Ptr getFile(unsigned int index) const; + /** + * adds a new file to the folder + * @param file the new file to add + */ + void addFile(const File::Ptr& file) { m_Files.push_back(file); } + /** + * add an empty folder as a subfolder to this one. + * @param folderName name of the new folder + * @return pointer to the new folder + * @note this folder will not be written to the bsa if it has no content + */ + Folder::Ptr addFolder(const std::string& folderName); + +private: + /** + * construct a new, empty, folder + */ + Folder(); + + // copy constructor not implemented + Folder(const Folder& reference); + + // assignment operator - not implemented + Folder& operator=(const Folder& reference); + + /** + * factory function to read a folder object from disc. This also reads part of the + * information about the files within + * @param file file stream to read from, already placed at the correct position + * @param fileNamesLength length of the file names list. This is required to correctly + * calculate offsets + * @param endPos position inside file where the last folder header ends. This is + * updated by the constructor so that it is the correct value after all + * folders are read + * @return the new Folder object + */ + Folder::Ptr readFolder(std::fstream& file, BSAUInt fileNamesLength, BSAUInt& endPos); + + Folder::Ptr readFolderSE(std::fstream& file, BSAUInt fileNamesLength, + BSAUInt& endPos); + + /** + * recursive function to determine the correct subfolder to place the new + * folder in + */ + void addFolderInt(Folder::Ptr folder); + + /** + * recursive function that returns an existing folder match or generates the structure + * for a new folder + * @param folder folder to find or create + * @return the final determined / generated Folder + */ + Folder::Ptr addOrFindFolderInt(Folder* folder); + + /** + * Add a new folder to the structure. + * It will automatically be added to the correct sub-folder if applicable. + */ + Folder::Ptr addFolder(std::fstream& file, BSAUInt fileNamesLength, BSAUInt& endPos, + ArchiveType type); + + Folder::Ptr addFolderFromFile(std::string filePath, BSAUInt size, BSAHash offset, + BSAUInt uncompressedSize, FO4TextureHeader header, + std::vector<FO4TextureChunk>& texChunks); + + bool resolveFileNames(std::fstream& file, bool testHashes); + + void writeHeader(std::fstream& file) const; + void writeData(std::fstream& file, BSAULong fileNamesLength) const; + EErrorCode writeFileData(std::fstream& sourceFile, std::fstream& targetFile) const; + void collectFolders(std::vector<Folder::Ptr>& folderList) const; + void collectFiles(std::vector<File::Ptr>& fileList) const; + void collectFileNames(std::vector<std::string>& nameList) const; + void collectFolderNames(std::vector<std::string>& nameList) const; + +private: + Folder* m_Parent; + BSAHash m_NameHash; + std::string m_Name; + BSAULong m_FileCount; + BSAHash m_Offset; + std::vector<Folder::Ptr> m_SubFolders; + std::unordered_map<std::string, Folder::Ptr> m_SubFoldersByName; + std::vector<File::Ptr> m_Files; + + mutable BSAULong m_OffsetWrite; +}; + +} // namespace BSA + +#endif /* BSAFOLDER_H */ diff --git a/libs/bsatk/include/bsatk/bsatk.h b/libs/bsatk/include/bsatk/bsatk.h new file mode 100644 index 0000000..330a240 --- /dev/null +++ b/libs/bsatk/include/bsatk/bsatk.h @@ -0,0 +1,15 @@ +/* + * File: bsatk.h + * Author: tannin + * + * Created on March 16, 2012, 10:35 AM + */ + +#ifndef BSATK_H +#define BSATK_H + +#include "bsaarchive.h" +#include "bsafile.h" +#include "bsafolder.h" + +#endif /* BSATK_H */ diff --git a/libs/bsatk/include/bsatk/bsatypes.h b/libs/bsatk/include/bsatk/bsatypes.h new file mode 100644 index 0000000..27dbdf5 --- /dev/null +++ b/libs/bsatk/include/bsatk/bsatypes.h @@ -0,0 +1,133 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef BSATYPES_H +#define BSATYPES_H + +#include <fstream> +#include <string> + +#include <dxgiformat.h> + +#include <DDS.h> + +#include "bsaexception.h" + +#ifdef WIN32 +#include <Windows.h> + +typedef unsigned char BSAUChar; +typedef unsigned short BSAUShort; +typedef unsigned int BSAUInt; +typedef unsigned long BSAULong; +typedef unsigned long long BSAHash; + +#else // WIN32 +#include <stdint.h> + +typedef unsigned char BSAUChar; +typedef unsigned short BSAUShort; +typedef unsigned int BSAUInt; +typedef unsigned long BSAULong; +typedef unsigned long long BSAHash; + +#endif // WIN32 + +enum ArchiveType +{ + TYPE_MORROWIND, + TYPE_OBLIVION, + TYPE_FALLOUT3, + TYPE_FALLOUTNV = TYPE_FALLOUT3, + TYPE_SKYRIM = TYPE_FALLOUT3, + TYPE_SKYRIMSE, + TYPE_FALLOUT4, + TYPE_STARFIELD, + TYPE_STARFIELD_LZ4_TEXTURE, + TYPE_FALLOUT4NG_7, + TYPE_FALLOUT4NG_8 +}; + +struct MorrowindFileOffset +{ + BSAUInt size; + BSAUInt offset; +}; + +struct FO4TextureHeader +{ + BSAUInt nameHash; + char extension[4]; + BSAUInt dirHash; + BSAUChar unknown1; + BSAUChar chunkNumber; + BSAUShort chunkHeaderSize; + BSAUShort height; + BSAUShort width; + BSAUChar mipCount; + DXGI_FORMAT format; + bool isCubemap; + BSAUChar unknown2; +}; + +struct FO4TextureChunk +{ + BSAHash offset; + BSAUInt packedSize; + BSAUInt unpackedSize; + BSAUShort startMip; + BSAUShort endMip; + BSAUInt unknown; +}; + +template <typename T> +static T readType(std::fstream& file) +{ + union + { + char buffer[sizeof(T)]; + T value; + }; + if (!file.read(buffer, sizeof(T))) { + throw data_invalid_exception("can't read from bsa"); + } + return value; +} + +template <typename T> +static void writeType(std::fstream& file, const T& value) +{ + union + { + char buffer[sizeof(T)]; + T valueTemp; + }; + valueTemp = value; + + file.write(buffer, sizeof(T)); +} + +std::string readBString(std::fstream& file); +void writeBString(std::fstream& file, const std::string& string); + +std::string readZString(std::fstream& file); +void writeZString(std::fstream& file, const std::string& string); + +#endif // BSATYPES_H diff --git a/libs/bsatk/include/bsatk/errorcodes.h b/libs/bsatk/include/bsatk/errorcodes.h new file mode 100644 index 0000000..f2780db --- /dev/null +++ b/libs/bsatk/include/bsatk/errorcodes.h @@ -0,0 +1,41 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef ERRORCODES_H +#define ERRORCODES_H + +namespace BSA +{ + +enum EErrorCode +{ + ERROR_NONE, + ERROR_INVALIDHASHES, + ERROR_FILENOTFOUND, + ERROR_INVALIDDATA, + ERROR_ACCESSFAILED, + ERROR_ZLIBINITFAILED, + ERROR_SOURCEFILEMISSING, + ERROR_CANCELED +}; + +}; + +#endif // ERRORCODES_H diff --git a/libs/bsatk/include/bsatk/filehash.h b/libs/bsatk/include/bsatk/filehash.h new file mode 100644 index 0000000..f5bfd6b --- /dev/null +++ b/libs/bsatk/include/bsatk/filehash.h @@ -0,0 +1,30 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef FILEHASH_H +#define FILEHASH_H + +#include <string> + +#include "bsatypes.h" + +BSAHash calculateBSAHash(const std::string& fileName); + +#endif // FILEHASH_H diff --git a/libs/bsatk/src/CMakeLists.txt b/libs/bsatk/src/CMakeLists.txt new file mode 100644 index 0000000..99a0f26 --- /dev/null +++ b/libs/bsatk/src/CMakeLists.txt @@ -0,0 +1,83 @@ +cmake_minimum_required(VERSION 3.16) + +find_package(Boost REQUIRED COMPONENTS thread) +find_package(ZLIB REQUIRED) +find_package(mo2-dds-header CONFIG REQUIRED) + +# lz4: try CONFIG first, fall back to pkg-config +find_package(lz4 CONFIG QUIET) +if (NOT lz4_FOUND) + find_package(PkgConfig REQUIRED) + pkg_check_modules(LZ4 REQUIRED IMPORTED_TARGET liblz4) +endif() +# lz4 1.10+ exports LZ4::lz4, older versions export lz4::lz4 +if(TARGET LZ4::lz4 AND NOT TARGET lz4::lz4) + add_library(lz4::lz4 ALIAS LZ4::lz4) +endif() + +add_library(bsatk STATIC) +target_link_libraries(bsatk + PUBLIC mo2::dds-header + PRIVATE ZLIB::ZLIB Boost::thread +) + +# Link lz4 based on how it was found +if (lz4_FOUND) + target_link_libraries(bsatk PRIVATE lz4::lz4) +else() + target_link_libraries(bsatk PRIVATE PkgConfig::LZ4) +endif() + +target_sources(bsatk + PRIVATE + bsaarchive.cpp + bsaexception.cpp + bsafile.cpp + bsafolder.cpp + bsatypes.cpp + filehash.cpp + PUBLIC + FILE_SET HEADERS + BASE_DIRS ${CMAKE_CURRENT_LIST_DIR}/../include + FILES + ${CMAKE_CURRENT_LIST_DIR}/../include/bsatk/bsaarchive.h + ${CMAKE_CURRENT_LIST_DIR}/../include/bsatk/bsaexception.h + ${CMAKE_CURRENT_LIST_DIR}/../include/bsatk/bsafile.h + ${CMAKE_CURRENT_LIST_DIR}/../include/bsatk/bsafolder.h + ${CMAKE_CURRENT_LIST_DIR}/../include/bsatk/bsatk.h + ${CMAKE_CURRENT_LIST_DIR}/../include/bsatk/bsatypes.h + ${CMAKE_CURRENT_LIST_DIR}/../include/bsatk/errorcodes.h + ${CMAKE_CURRENT_LIST_DIR}/../include/bsatk/filehash.h + +) + +set_target_properties(bsatk PROPERTIES CXX_STANDARD 20) + +# for building +target_include_directories(bsatk PRIVATE ${CMAKE_CURRENT_LIST_DIR}/../include/bsatk) + +if (MSVC) + target_compile_options(bsatk + PRIVATE + "/MP" + "/W4" + "/external:anglebrackets" + "/external:W0" + ) + target_link_options(bsatk + PRIVATE + $<$<CONFIG:RelWithDebInfo>:/LTCG /INCREMENTAL:NO /OPT:REF /OPT:ICF> + ) + + set_target_properties(bsatk PROPERTIES VS_STARTUP_PROJECT bsatk) +endif() + +add_library(mo2::bsatk ALIAS bsatk) + +# install +install(TARGETS bsatk EXPORT bsatkTargets FILE_SET HEADERS) +install(EXPORT bsatkTargets + FILE mo2-bsatk-targets.cmake + NAMESPACE mo2:: + DESTINATION lib/cmake/mo2-bsatk +) diff --git a/libs/bsatk/src/bsaarchive.cpp b/libs/bsatk/src/bsaarchive.cpp new file mode 100644 index 0000000..e1655c1 --- /dev/null +++ b/libs/bsatk/src/bsaarchive.cpp @@ -0,0 +1,1237 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "bsaarchive.h" +#include "bsaexception.h" +#include "bsafile.h" +#include "bsafolder.h" +#include <algorithm> +#include <boost/interprocess/sync/interprocess_semaphore.hpp> +#include <boost/interprocess/sync/scoped_lock.hpp> +#include <boost/shared_array.hpp> +#include <boost/thread.hpp> +#include <cstring> +#include <fstream> +#include <iostream> +#include <lz4.h> +#include <lz4frame.h> +#include <memory> +#include <queue> +#include <sys/stat.h> +#include <zlib.h> +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include <Windows.h> +#endif + +using std::fstream; + +using namespace boost::posix_time; + +namespace BSA +{ + +Archive::Archive() + : m_RootFolder(new Folder), m_ArchiveFlags(FLAG_HASDIRNAMES | FLAG_HASFILENAMES), + m_Type(TYPE_SKYRIM) +{} + +Archive::~Archive() +{ + if (m_File.is_open()) { + m_File.close(); + } + std::vector<Folder::Ptr> folders; + m_RootFolder->collectFolders(folders); + cleanFolder(m_RootFolder); + m_RootFolder.reset(); +} + +ArchiveType Archive::typeFromID(BSAULong typeID) +{ + switch (typeID) { + case 0x100: + return TYPE_MORROWIND; + case 0x67: + return TYPE_OBLIVION; + case 0x68: + return TYPE_FALLOUT3; + case 0x69: + return TYPE_SKYRIMSE; + case 0x01: + return TYPE_FALLOUT4; + case 0x02: + return TYPE_STARFIELD; + case 0x03: + return TYPE_STARFIELD_LZ4_TEXTURE; + case 0x07: + return TYPE_FALLOUT4NG_7; + case 0x08: + return TYPE_FALLOUT4NG_8; + default: + throw data_invalid_exception(makeString("invalid type %d", typeID)); + } +} + +BSAULong Archive::typeToID(ArchiveType type) +{ + switch (type) { + case TYPE_MORROWIND: + return 0x100; + case TYPE_OBLIVION: + return 0x67; + case TYPE_FALLOUT3: + return 0x68; + case TYPE_SKYRIMSE: + return 0x69; + case TYPE_FALLOUT4: + return 0x01; + case TYPE_STARFIELD: + return 0x02; + case TYPE_STARFIELD_LZ4_TEXTURE: + return 0x03; + case TYPE_FALLOUT4NG_7: + return 0x07; + case TYPE_FALLOUT4NG_8: + return 0x08; + default: + throw data_invalid_exception(makeString("invalid type %d", type)); + } +} + +Archive::Header Archive::readHeader(std::fstream& infile) +{ + Header result; + + result.fileIdentifier = readType<uint32_t>(infile); + if (result.fileIdentifier != 0x00415342 && result.fileIdentifier != 0x58445442 && + result.fileIdentifier != 0x00000100) { + throw data_invalid_exception(makeString("not a bsa or ba2 file")); + } + + if (result.fileIdentifier != 0x00000100) { + ArchiveType type = typeFromID(readType<BSAUInt>(infile)); + if (type == TYPE_FALLOUT4 || type == TYPE_STARFIELD || + type == TYPE_STARFIELD_LZ4_TEXTURE || type == TYPE_FALLOUT4NG_7 || + type == TYPE_FALLOUT4NG_8) { + result.type = type; + infile.read(result.archType, 4); + result.archType[4] = '\0'; + result.fileCount = readType<BSAUInt>(infile); + result.nameTableOffset = readType<BSAHash>(infile); + result.archiveFlags = FLAG_HASDIRNAMES | FLAG_HASFILENAMES; + } else { + result.type = type; + result.offset = readType<BSAUInt>(infile); + result.archiveFlags = readType<BSAUInt>(infile); + result.folderCount = readType<BSAUInt>(infile); + result.fileCount = readType<BSAUInt>(infile); + result.folderNameLength = readType<BSAUInt>(infile); + result.fileNameLength = readType<BSAUInt>(infile); + result.fileFlags = readType<BSAUInt>(infile); + } + } else { + result.type = TYPE_MORROWIND; + result.offset = readType<BSAUInt>(infile); + result.fileCount = readType<BSAUInt>(infile); + result.archiveFlags = FLAG_HASDIRNAMES | FLAG_HASFILENAMES; + } + + return result; +} + +EErrorCode Archive::read(const char* fileName, bool testHashes) +{ + m_File.open(fileName, fstream::in | fstream::binary); + if (!m_File.is_open()) { + return ERROR_FILENOTFOUND; + } + m_File.exceptions(std::ios_base::badbit); + try { + Header header; + try { + header = readHeader(m_File); + } catch (const data_invalid_exception& e) { + throw data_invalid_exception(makeString("%s (filename: %s)", e.what(), fileName)); + } + m_ArchiveFlags = header.archiveFlags; + m_Type = header.type; + if (m_Type == TYPE_FALLOUT4 || m_Type == TYPE_STARFIELD || + m_Type == TYPE_STARFIELD_LZ4_TEXTURE || m_Type == TYPE_FALLOUT4NG_7 || + m_Type == TYPE_FALLOUT4NG_8) { + + m_File.seekg(header.nameTableOffset); + + std::vector<std::string> fileNames; + for (unsigned int i = 0; i < header.fileCount; ++i) { + BSAUShort length = readType<BSAUShort>(m_File); + + char* file = new char[length + 1]; + m_File.read(file, length); + file[length] = '\0'; + + fileNames.push_back(file); + delete[] file; + } + std::streamoff offset; + switch (m_Type) { + case TYPE_STARFIELD: + offset = 32; + break; + case TYPE_STARFIELD_LZ4_TEXTURE: + offset = 36; + break; + default: + offset = 24; + } + if (strcmp(header.archType, "GNRL") == 0) { + m_File.seekg(offset, std::ios::beg); + for (unsigned int i = 0; i < header.fileCount; ++i) { + BSAUInt nameHash = readType<BSAUInt>(m_File); + char* extension = new char[4]; + m_File.read(extension, 4); + BSAUInt dirHash = readType<BSAUInt>(m_File); + m_File.seekg(4, std::ios::cur); + BSAHash offset = readType<BSAHash>(m_File); + BSAUInt packedSize = readType<BSAUInt>(m_File); + BSAUInt unpackedSize = readType<BSAUInt>(m_File); + m_File.seekg(4, std::ios::cur); + std::vector<FO4TextureChunk> dummy; + Folder::Ptr newDir = m_RootFolder->addFolderFromFile( + fileNames[i], packedSize, offset, unpackedSize, {}, dummy); + delete[] extension; + } + } else if (strcmp(header.archType, "DX10") == 0) { + m_File.seekg(offset, std::ios::beg); + for (unsigned int i = 0; i < header.fileCount; ++i) { + FO4TextureHeader texHeader; + texHeader.nameHash = readType<BSAUInt>(m_File); + m_File.read(texHeader.extension, 4); + texHeader.dirHash = readType<BSAUInt>(m_File); + texHeader.unknown1 = readType<BSAUChar>(m_File); + texHeader.chunkNumber = readType<BSAUChar>(m_File); + texHeader.chunkHeaderSize = readType<BSAUShort>(m_File); + texHeader.height = readType<BSAUShort>(m_File); + texHeader.width = readType<BSAUShort>(m_File); + texHeader.mipCount = readType<BSAUChar>(m_File); + texHeader.format = static_cast<DXGI_FORMAT>(readType<BSAUChar>(m_File)); + texHeader.isCubemap = readType<bool>(m_File); + texHeader.unknown2 = readType<BSAUChar>(m_File); + std::vector<FO4TextureChunk> chunks; + for (unsigned int j = 0; j < texHeader.chunkNumber; ++j) { + FO4TextureChunk chunk; + chunk.offset = readType<BSAHash>(m_File); + chunk.packedSize = readType<BSAUInt>(m_File); + chunk.unpackedSize = readType<BSAUInt>(m_File); + chunk.startMip = readType<BSAUShort>(m_File); + chunk.endMip = readType<BSAUShort>(m_File); + chunk.unknown = readType<BSAUInt>(m_File); + chunks.push_back(chunk); + } + Folder::Ptr newDir = m_RootFolder->addFolderFromFile( + fileNames[i], chunks[0].packedSize, chunks[0].offset, + chunks[0].unpackedSize, texHeader, chunks); + } + } + + return ERROR_NONE; + } else if (m_Type == TYPE_MORROWIND) { + BSAUInt dataOffset = 12 + header.offset + header.fileCount * 8; + + std::vector<MorrowindFileOffset> fileSizeOffset(header.fileCount); + m_File.read((char*)fileSizeOffset.data(), + header.fileCount * sizeof(MorrowindFileOffset)); + std::vector<BSAUInt> fileNameOffset(header.fileCount); + m_File.read((char*)fileNameOffset.data(), header.fileCount * sizeof(BSAUInt)); + BSAUInt last = header.offset - 12 * header.fileCount; + for (uint32_t i = 0; i < header.fileCount; ++i) { + uint32_t index = 0; + if (i + 1 == header.fileCount) + index = last; + else + index = fileNameOffset[i + 1] - fileNameOffset[i]; + char* filePath = new char[index + 1]; + m_File.read(filePath, index); + filePath[index] = '\0'; + + std::vector<FO4TextureChunk> dummy; + Folder::Ptr newDir = m_RootFolder->addFolderFromFile( + filePath, fileSizeOffset[i].size, dataOffset + fileSizeOffset[i].offset, 0, + {}, dummy); + delete[] filePath; + } + return ERROR_NONE; + } else { + // flat list of folders as they were stored in the archive + std::vector<Folder::Ptr> folders; + + for (unsigned long i = 0; i < header.folderCount; ++i) { + folders.push_back(m_RootFolder->addFolder(m_File, header.fileNameLength, + header.offset, header.type)); + } + + m_File.seekg(header.offset); + + bool hashesValid = true; + for (std::vector<Folder::Ptr>::iterator iter = folders.begin(); + iter != folders.end(); ++iter) { + if (!(*iter)->resolveFileNames(m_File, testHashes)) { + hashesValid = false; + } + } + return hashesValid ? ERROR_NONE : ERROR_INVALIDHASHES; + } + } catch (std::ios_base::failure&) { + return ERROR_INVALIDDATA; + } +} + +void Archive::close() +{ + m_File.close(); +} + +BSAULong Archive::countFiles() const +{ + return m_RootFolder->countFiles(); +} + +std::vector<std::string> Archive::collectFolderNames() const +{ + std::vector<std::string> result; + m_RootFolder->collectFolderNames(result); + return result; +} + +std::vector<std::string> Archive::collectFileNames() const +{ + std::vector<std::string> result; + m_RootFolder->collectFileNames(result); + return result; +} + +BSAULong Archive::countCharacters(const std::vector<std::string>& list) const +{ + size_t sum = 0; + for (std::vector<std::string>::const_iterator iter = list.begin(); iter != list.end(); + ++iter) { + sum += iter->length() + 1; + } + return static_cast<BSAULong>(sum); +} + +#ifndef WIN32 +#define _stricmp strcasecmp +#endif // WIN32 + +static bool endsWith(const std::string& fileName, const char* extension) +{ + size_t endLength = strlen(extension); + if (fileName.length() < endLength) { + return false; + } + return _stricmp(&fileName[fileName.length() - endLength], extension) == 0; +} + +BSAULong Archive::determineFileFlags(const std::vector<std::string>& fileList) const +{ + BSAULong result = 0; + + bool hasNIF = false; + bool hasDDS = false; + bool hasXML = false; + bool hasWAV = false; + bool hasMP3 = false; + bool hasTXT = false; + bool hasSPT = false; + bool hasTEX = false; + bool hasCTL = false; + + for (std::vector<std::string>::const_iterator iter = fileList.begin(); + iter != fileList.end(); ++iter) { + if (!hasNIF && endsWith(*iter, ".nif")) { + hasNIF = true; + result |= 1 << 0; + } else if (!hasDDS && endsWith(*iter, ".dds")) { + hasDDS = true; + result |= 1 << 1; + } else if (!hasXML && endsWith(*iter, ".xml")) { + hasXML = true; + result |= 1 << 2; + } else if (!hasWAV && endsWith(*iter, ".wav")) { + hasWAV = true; + result |= 1 << 3; + } else if (!hasMP3 && endsWith(*iter, ".mp3")) { + hasMP3 = true; + result |= 1 << 4; + } else if (!hasTXT && endsWith(*iter, ".txt")) { + hasTXT = true; + result |= 1 << 5; + } else if (!hasSPT && endsWith(*iter, ".spt")) { + hasSPT = true; + result |= 1 << 6; + } else if (!hasTEX && endsWith(*iter, ".tex")) { + hasTEX = true; + result |= 1 << 7; + } else if (!hasCTL && endsWith(*iter, ".ctl")) { + hasCTL = true; + result |= 1 << 8; + } + } + return result; +} + +void Archive::writeHeader(std::fstream& outfile, BSAULong fileFlags, + BSAULong numFolders, BSAULong folderNamesLength, + BSAULong fileNamesLength) +{ + outfile.write("BSA\0", 4); + writeType<BSAULong>(outfile, typeToID(m_Type)); + writeType<BSAULong>(outfile, 0x24); // header size is static + writeType<BSAULong>(outfile, m_ArchiveFlags); + writeType<BSAULong>(outfile, numFolders); + writeType<BSAULong>(outfile, countFiles()); + writeType<BSAULong>(outfile, folderNamesLength); + writeType<BSAULong>(outfile, fileNamesLength); + writeType<BSAULong>(outfile, fileFlags); +} + +EErrorCode Archive::write(const char* fileName) +{ + std::fstream outfile; + outfile.open(fileName, fstream::out | fstream::binary); + if (!outfile.is_open()) { + return ERROR_ACCESSFAILED; + } + outfile.exceptions(std::ios_base::badbit); + + std::vector<Folder::Ptr> folders; + m_RootFolder->collectFolders(folders); + + std::vector<std::string> folderNames; + std::vector<std::string> fileNames; + BSAULong folderNamesLength = 0; + BSAULong fileNamesLength = 0; + for (std::vector<Folder::Ptr>::const_iterator folderIter = folders.begin(); + folderIter != folders.end(); ++folderIter) { + std::string fullPath = (*folderIter)->getFullPath(); + folderNames.push_back(fullPath); + folderNamesLength += static_cast<BSAULong>(fullPath.length()); + for (std::vector<File::Ptr>::const_iterator fileIter = + (*folderIter)->m_Files.begin(); + fileIter != (*folderIter)->m_Files.end(); ++fileIter) { + fileNames.push_back((*fileIter)->m_Name); + fileNamesLength += static_cast<BSAULong>((*fileIter)->m_Name.length()); + } + } + + try { + writeHeader(outfile, determineFileFlags(fileNames), + static_cast<BSAULong>(folderNames.size()), folderNamesLength, + fileNamesLength); +#pragma message("folders (and files?) need to be sorted by hash!") + // dummy-pass: before we can store the actual folder data + + // prepare folder and file headers +#pragma message("it's unnecessary to write actual data, placeholders are sufficient") + for (std::vector<Folder::Ptr>::const_iterator folderIter = folders.begin(); + folderIter != folders.end(); ++folderIter) { + (*folderIter)->writeHeader(outfile); + } + + for (std::vector<Folder::Ptr>::const_iterator folderIter = folders.begin(); + folderIter != folders.end(); ++folderIter) { + (*folderIter)->writeData(outfile, fileNamesLength); + } + + // write file names + for (std::vector<std::string>::const_iterator folderIter = fileNames.begin(); + folderIter != fileNames.end(); ++folderIter) { + writeZString(outfile, *folderIter); + } + + // write file data + for (std::vector<Folder::Ptr>::iterator folderIter = folders.begin(); + folderIter != folders.end(); ++folderIter) { + (*folderIter)->writeFileData(m_File, outfile); + } + + outfile.seekp(0x24, fstream::beg); + + // re-write folder and file structure, this time with the correct + // offsets + for (std::vector<Folder::Ptr>::const_iterator folderIter = folders.begin(); + folderIter != folders.end(); ++folderIter) { + (*folderIter)->writeHeader(outfile); + } + + for (std::vector<Folder::Ptr>::const_iterator folderIter = folders.begin(); + folderIter != folders.end(); ++folderIter) { + (*folderIter)->writeData(outfile, fileNamesLength); + } + + outfile.close(); + return ERROR_NONE; + } catch (std::ios_base::failure&) { + outfile.close(); + return ERROR_INVALIDDATA; + } +} + +DirectX::DDS_HEADER Archive::getDDSHeader(File::Ptr file, + DirectX::DDS_HEADER_DXT10& DX10Header, + bool& isDX10) const +{ + DirectX::DDS_HEADER DDSHeaderData = {}; + DDSHeaderData.size = sizeof(DDSHeaderData); + DDSHeaderData.flags = + DDS_HEADER_FLAGS_TEXTURE | DDS_HEADER_FLAGS_LINEARSIZE | DDS_HEADER_FLAGS_MIPMAP; + DDSHeaderData.height = file->m_TextureHeader.height; + DDSHeaderData.width = file->m_TextureHeader.width; + DDSHeaderData.mipMapCount = file->m_TextureHeader.mipCount; + DDSHeaderData.ddspf.size = sizeof(DirectX::DDS_PIXELFORMAT); + DDSHeaderData.caps = DDS_SURFACE_FLAGS_TEXTURE | DDS_SURFACE_FLAGS_MIPMAP; + + if (file->m_TextureHeader.isCubemap) + DDSHeaderData.caps2 = DDS_CUBEMAP_ALLFACES; + + bool supported = true; + + switch (file->m_TextureHeader.format) { + case DXGI_FORMAT_BC1_UNORM: + case DXGI_FORMAT_BC1_UNORM_SRGB: + DDSHeaderData.ddspf = DirectX::DDSPF_DXT1; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height / 2; + break; + + case DXGI_FORMAT_BC2_UNORM: + case DXGI_FORMAT_BC2_UNORM_SRGB: + DDSHeaderData.ddspf = DirectX::DDSPF_DXT3; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height; + break; + + case DXGI_FORMAT_BC3_UNORM: + case DXGI_FORMAT_BC3_UNORM_SRGB: + DDSHeaderData.ddspf = DirectX::DDSPF_DXT5; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height; + break; + + case DXGI_FORMAT_BC4_UNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_BC4_UNORM; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height; + break; + + case DXGI_FORMAT_BC4_SNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_BC4_SNORM; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height; + break; + + case DXGI_FORMAT_BC5_UNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_BC5_UNORM; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height; + break; + + case DXGI_FORMAT_BC5_SNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_BC5_SNORM; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height; + break; + + case DXGI_FORMAT_BC7_UNORM: + case DXGI_FORMAT_BC7_UNORM_SRGB: + DDSHeaderData.ddspf = DirectX::DDSPF_DX10; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height; + + isDX10 = true; + DX10Header.dxgiFormat = file->m_TextureHeader.format; + break; + + case DXGI_FORMAT_R8G8B8A8_UNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_A8B8G8R8; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height * 4; // 32bpp + break; + + case DXGI_FORMAT_B8G8R8A8_UNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_A8R8G8B8; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height * 4; // 32bpp + break; + + case DXGI_FORMAT_B8G8R8X8_UNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_X8R8G8B8; + break; + + case DXGI_FORMAT_R8_UNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_L8; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height; // 8bpp + break; + + case DXGI_FORMAT_R16_UNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_L16; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height * 2; // 16bpp + break; + + case DXGI_FORMAT_R8G8_UNORM: + DDSHeaderData.ddspf = DirectX::DDSPF_A8L8; + DDSHeaderData.pitchOrLinearSize = + file->m_TextureHeader.width * file->m_TextureHeader.height * 2; // 16bpp + break; + + default: + return {}; + break; + } + + return DDSHeaderData; +} + +void Archive::getDX10Header(DirectX::DDS_HEADER_DXT10& DX10Header, File::Ptr file, + DirectX::DDS_HEADER DDSHeader) const +{ + DX10Header.resourceDimension = DirectX::DDS_DIMENSION_TEXTURE2D; + DX10Header.miscFlag = 0; + DX10Header.arraySize = 1; + DX10Header.miscFlags2 = 0; +} + +static const unsigned long CHUNK_SIZE = 128 * 1024; + +EErrorCode Archive::extractDirect(File::Ptr file, std::ofstream& outFile) const +{ + EErrorCode result = ERROR_NONE; + if (file->m_FileSize == 0) { + // don't try to read empty file + return result; + } + + m_File.clear(); + m_File.seekg(static_cast<std::ifstream::pos_type>(file->m_DataOffset), std::ios::beg); + + if (m_Type == TYPE_FALLOUT4 || m_Type == TYPE_STARFIELD || + m_Type == TYPE_STARFIELD_LZ4_TEXTURE || m_Type == TYPE_FALLOUT4NG_7 || + m_Type == TYPE_FALLOUT4NG_8) { + if (!file->m_TextureChunks.size()) { + BSAULong size = file->m_UncompressedFileSize; + std::unique_ptr<char[]> buffer(new char[size]); + m_File.read(buffer.get(), size); + outFile.write(buffer.get(), size); + } else { + bool isDX10 = false; + DirectX::DDS_HEADER_DXT10 DX10HeaderData = {}; + DirectX::DDS_HEADER DDSHeaderData = getDDSHeader(file, DX10HeaderData, isDX10); + + outFile.write("DDS ", 4); + char* DDSHeader = new char[sizeof(DDSHeaderData)]; + memcpy(DDSHeader, &DDSHeaderData, sizeof(DDSHeaderData)); + outFile.write(DDSHeader, sizeof(DDSHeaderData)); + delete[] DDSHeader; + + if (isDX10) { + getDX10Header(DX10HeaderData, file, DDSHeaderData); + char* DX10Header = new char[sizeof(DX10HeaderData)]; + memcpy(DX10Header, &DX10HeaderData, sizeof(DX10HeaderData)); + outFile.write(DX10Header, sizeof(DX10HeaderData)); + delete[] DX10Header; + } + + for (BSAUInt i = 0; i < file->m_TextureChunks.size(); ++i) { + BSAULong length = file->m_TextureChunks[i].unpackedSize; + std::unique_ptr<char[]> chunk(new char[length]); + m_File.read(chunk.get(), length); + outFile.write(chunk.get(), length); + } + } + } else { + BSAULong size = file->m_FileSize; + if (namePrefixed()) { + std::string fullName = readBString(m_File); + if (size <= fullName.length()) { +#pragma message("report error!") + return result; + } + size -= fullName.length() + 1; + } + std::unique_ptr<unsigned char[]> buffer(new unsigned char[size]); + m_File.read(reinterpret_cast<char*>(buffer.get()), size); + if (result == ERROR_NONE) + outFile.write(reinterpret_cast<char*>(buffer.get()), size); + } + std::unique_ptr<char[]> inBuffer(new char[CHUNK_SIZE]); + + try { + unsigned long sizeLeft = file->m_FileSize; + + while (sizeLeft > 0) { + int chunkSize = (std::min)(sizeLeft, CHUNK_SIZE); + m_File.read(inBuffer.get(), chunkSize); + outFile.write(inBuffer.get(), chunkSize); + sizeLeft -= chunkSize; + } + } catch (const std::exception&) { + result = ERROR_INVALIDDATA; + } + return result; +} + +std::shared_ptr<unsigned char[]> Archive::decompress(unsigned char* inBuffer, + BSAULong inSize, + EErrorCode& result, + BSAULong& outSize) +{ + if (outSize == 0) { + memcpy(&outSize, inBuffer, sizeof(BSAULong)); + inBuffer += sizeof(BSAULong); + inSize -= sizeof(BSAULong); + } + + if ((inSize == 0) || (outSize == 0)) { + return std::shared_ptr<unsigned char[]>(); + } + + std::shared_ptr<unsigned char[]> outBuffer(new unsigned char[outSize]); + + z_stream stream; + try { + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.opaque = Z_NULL; + stream.avail_in = inSize; + stream.next_in = static_cast<Bytef*>(inBuffer); + int zlibRet = inflateInit2(&stream, 15 + 32); + if (zlibRet != Z_OK) { + result = ERROR_ZLIBINITFAILED; + return std::shared_ptr<unsigned char[]>(); + } + + do { + stream.avail_out = outSize; + stream.next_out = reinterpret_cast<Bytef*>(outBuffer.get()); + zlibRet = inflate(&stream, Z_NO_FLUSH); + if ((zlibRet != Z_OK) && (zlibRet != Z_STREAM_END) && (zlibRet != Z_BUF_ERROR)) { +#pragma message("pass result code to caller") + throw std::runtime_error("invalid data"); + } + } while (stream.avail_out == 0); + inflateEnd(&stream); + return outBuffer; + } catch (const std::exception&) { + result = ERROR_INVALIDDATA; + inflateEnd(&stream); + return std::shared_ptr<unsigned char[]>(); + } +} + +EErrorCode Archive::extractCompressed(File::Ptr file, std::ofstream& outFile) const +{ + EErrorCode result = ERROR_NONE; + if (file->m_FileSize == 0) { + // don't try to read empty file + return result; + } + + m_File.clear(); + m_File.seekg(static_cast<std::ifstream::pos_type>(file->m_DataOffset), std::ios::beg); + + if (m_Type == TYPE_FALLOUT4 || m_Type == TYPE_STARFIELD || + m_Type == TYPE_STARFIELD_LZ4_TEXTURE || m_Type == TYPE_FALLOUT4NG_7 || + m_Type == TYPE_FALLOUT4NG_8) { + if (!file->m_TextureChunks.size()) { + BSAULong inSize = file->m_FileSize; + std::unique_ptr<unsigned char[]> inBuffer(new unsigned char[inSize]); + m_File.read(reinterpret_cast<char*>(inBuffer.get()), inSize); + BSAULong length = file->m_UncompressedFileSize; + std::shared_ptr<unsigned char[]> buffer = + decompress(inBuffer.get(), inSize, result, length); + if (result == ERROR_NONE) { + outFile.write(reinterpret_cast<char*>(buffer.get()), length); + } + } else { + bool isDX10 = false; + DirectX::DDS_HEADER_DXT10 DX10HeaderData = {}; + DirectX::DDS_HEADER DDSHeaderData = getDDSHeader(file, DX10HeaderData, isDX10); + + outFile.write("DDS ", 4); + char* DDSHeader = new char[sizeof(DDSHeaderData)]; + memcpy(DDSHeader, &DDSHeaderData, sizeof(DDSHeaderData)); + outFile.write(DDSHeader, sizeof(DDSHeaderData)); + delete[] DDSHeader; + + if (isDX10) { + getDX10Header(DX10HeaderData, file, DDSHeaderData); + char* DX10Header = new char[sizeof(DX10HeaderData)]; + memcpy(DX10Header, &DX10HeaderData, sizeof(DX10HeaderData)); + outFile.write(DX10Header, sizeof(DX10HeaderData)); + delete[] DX10Header; + } + + for (BSAUInt i = 0; i < file->m_TextureChunks.size(); ++i) { + BSAULong length = file->m_TextureChunks[i].unpackedSize; + if (file->m_TextureChunks[i].packedSize > 0) { + unsigned char* chunk = new unsigned char[file->m_TextureChunks[i].packedSize]; + m_File.read(reinterpret_cast<char*>(chunk), + file->m_TextureChunks[i].packedSize); + if (m_Type == TYPE_STARFIELD_LZ4_TEXTURE) { + char* unpackedChunk = new char[length]; + LZ4_decompress_safe(reinterpret_cast<char*>(chunk), unpackedChunk, + file->m_TextureChunks[i].packedSize, length); + outFile.write(unpackedChunk, length); + delete[] unpackedChunk; + } else { + std::shared_ptr<unsigned char[]> unpackedChunk = + decompress(chunk, file->m_TextureChunks[i].packedSize, result, length); + if (result == ERROR_NONE) { + outFile.write(reinterpret_cast<char*>(unpackedChunk.get()), length); + } + } + delete[] chunk; + } else { + char* chunk = new char[length]; + m_File.read(chunk, length); + outFile.write(chunk, length); + delete[] chunk; + } + } + } + } else if (m_Type == TYPE_SKYRIMSE) { + BSAULong inSize = file->m_FileSize; + if (namePrefixed()) { + std::string fullName = readBString(m_File); + if (inSize <= fullName.length()) { +#pragma message("report error!") + return result; + } + inSize -= fullName.length() + 1; + } + BSAULong outSize = readType<BSAULong>(m_File); + inSize -= sizeof(BSAULong); + std::unique_ptr<unsigned char[]> inBuffer(new unsigned char[inSize]); + m_File.read(reinterpret_cast<char*>(inBuffer.get()), inSize); + + LZ4F_decompressionContext_t dcContext = nullptr; + LZ4F_decompressOptions_t options = {}; + LZ4F_createDecompressionContext(&dcContext, LZ4F_VERSION); + size_t lzOutSize = outSize; + size_t lzInSize = inSize; + + std::unique_ptr<unsigned char[]> outBuffer(new unsigned char[outSize]); + LZ4F_decompress(dcContext, outBuffer.get(), &lzOutSize, inBuffer.get(), &lzInSize, + &options); + + outFile.write(reinterpret_cast<char*>(outBuffer.get()), outSize); + } else { + BSAULong inSize = file->m_FileSize; + if (namePrefixed()) { + std::string fullName = readBString(m_File); + if (inSize <= fullName.length()) { +#pragma message("report error!") + return result; + } + inSize -= fullName.length() + 1; + } + std::unique_ptr<unsigned char[]> inBuffer(new unsigned char[inSize]); + m_File.read(reinterpret_cast<char*>(inBuffer.get()), inSize); + BSAULong length = 0UL; + std::shared_ptr<unsigned char[]> buffer = + decompress(inBuffer.get(), inSize, result, length); + if (result == ERROR_NONE) { + outFile.write(reinterpret_cast<char*>(buffer.get()), length); + } + } + + return result; +} + +EErrorCode Archive::extract(File::Ptr file, const char* outputDirectory) const +{ + std::string fileName = makeString("%s/%s", outputDirectory, file->getName().c_str()); + std::ofstream outputFile(fileName.c_str(), + fstream::out | fstream::binary | fstream::trunc); + if (!outputFile.is_open()) { + return ERROR_ACCESSFAILED; + } + + EErrorCode result = ERROR_NONE; + if (compressed(file)) { + result = extractCompressed(file, outputFile); + } else { + result = extractDirect(file, outputFile); + } + outputFile.close(); + return result; +} + +void Archive::readFiles(std::queue<FileInfo>& queue, boost::mutex& mutex, + boost::interprocess::interprocess_semaphore& bufferCount, + boost::interprocess::interprocess_semaphore& queueFree, + std::vector<File::Ptr>::iterator begin, + std::vector<File::Ptr>::iterator end) +{ + for (; begin != end && !boost::this_thread::interruption_requested(); ++begin) { + queueFree.wait(); + + FileInfo fileInfo; + fileInfo.file = *begin; + size_t size = static_cast<size_t>(fileInfo.file->m_FileSize); + + m_File.seekg(fileInfo.file->m_DataOffset); + if (m_Type != TYPE_FALLOUT4 && m_Type != TYPE_STARFIELD && + m_Type != TYPE_STARFIELD_LZ4_TEXTURE && m_Type != TYPE_FALLOUT4NG_7 && + m_Type != TYPE_FALLOUT4NG_8) { + if (namePrefixed()) { + std::string fullName = readBString(m_File); + if (size <= fullName.length()) { +#pragma message("report error!") + continue; + } + size -= fullName.length() + 1; + } + if (m_Type == TYPE_SKYRIMSE && compressed(fileInfo.file)) { + fileInfo.file->m_UncompressedFileSize = readType<BSAULong>(m_File); + size -= 4; + } + if (!fileInfo.file->m_TextureChunks.size()) { + fileInfo.data = + std::make_pair(std::shared_ptr<unsigned char[]>(new unsigned char[size]), + static_cast<BSAULong>(size)); + m_File.read(reinterpret_cast<char*>(fileInfo.data.first.get()), size); + } + } else { + if (!fileInfo.file->m_TextureChunks.size()) { + if (size == 0) + size = fileInfo.file->m_UncompressedFileSize; + fileInfo.data = + std::make_pair(std::shared_ptr<unsigned char[]>(new unsigned char[size]), + static_cast<BSAULong>(size)); + m_File.read(reinterpret_cast<char*>(fileInfo.data.first.get()), size); + } else { + fileInfo.file->m_UncompressedFileSize = 0L; + BSAULong totalSize = 0U; + for (BSAUInt i = 0; i < fileInfo.file->m_TextureChunks.size(); ++i) { + totalSize += fileInfo.file->m_TextureChunks[i].unpackedSize; + } + char* chunkData = new char[totalSize]; + BSAULong currentPos = 0U; + for (BSAUInt i = 0; i < fileInfo.file->m_TextureChunks.size(); ++i) { + BSAULong length = fileInfo.file->m_TextureChunks[i].unpackedSize; + if (fileInfo.file->m_TextureChunks[i].packedSize > 0) { + char* chunk = new char[fileInfo.file->m_TextureChunks[i].packedSize]; + m_File.read(chunk, fileInfo.file->m_TextureChunks[i].packedSize); + if (m_Type == TYPE_FALLOUT4 || m_Type == TYPE_STARFIELD || + m_Type == TYPE_FALLOUT4NG_7 || m_Type == TYPE_FALLOUT4NG_8) { + EErrorCode result = ERROR_NONE; + try { + std::shared_ptr<unsigned char[]> unpackedChunk = decompress( + reinterpret_cast<unsigned char*>(chunk), + fileInfo.file->m_TextureChunks[i].packedSize, result, length); + memcpy(chunkData + currentPos, + reinterpret_cast<char*>(unpackedChunk.get()), length); + unpackedChunk.reset(); + } catch (const std::exception&) { +#pragma message("report error!") + continue; + } + } else { + char* unpackedChunk = new char[length]; + LZ4_decompress_safe(chunk, unpackedChunk, + fileInfo.file->m_TextureChunks[i].packedSize, length); + memcpy(chunkData + currentPos, unpackedChunk, length); + delete[] unpackedChunk; + } + delete[] chunk; + fileInfo.file->m_UncompressedFileSize += length; + } else { + char* chunk = new char[length]; + m_File.read(chunk, length); + memcpy(chunkData + currentPos, chunk, length); + delete[] chunk; + } + currentPos += length; + } + fileInfo.file->m_FileSize = 0; + fileInfo.data = std::make_pair(std::shared_ptr<unsigned char[]>( + reinterpret_cast<unsigned char*>(chunkData)), + static_cast<BSAULong>(totalSize)); + } + } + + { + boost::interprocess::scoped_lock<boost::mutex> lock(mutex); + queue.push(fileInfo); + } + bufferCount.post(); + } +} + +inline bool fileExists(const std::string& name) +{ + struct stat buffer; + return stat(name.c_str(), &buffer) != -1; +} + +void Archive::extractFiles(const std::string& targetDirectory, + std::queue<FileInfo>& queue, boost::mutex& mutex, + boost::interprocess::interprocess_semaphore& bufferCount, + boost::interprocess::interprocess_semaphore& queueFree, + int totalFiles, bool overwrite, int& filesDone) +{ + for (int i = 0; i < totalFiles; ++i) { + bufferCount.wait(); + if (boost::this_thread::interruption_requested()) { + break; + } + + FileInfo fileInfo; + + { + boost::interprocess::scoped_lock<boost::mutex> lock(mutex); + fileInfo = queue.front(); + ++filesDone; + queue.pop(); + } + queueFree.post(); + + DataBuffer dataBuffer = fileInfo.data; + + std::string fileName = makeString("%s\\%s", targetDirectory.c_str(), + fileInfo.file->getFilePath().c_str()); + if (!overwrite && fileExists(fileName)) { + continue; + } + + std::ofstream outputFile(fileName.c_str(), + fstream::out | fstream::binary | fstream::trunc); + + if (!outputFile.is_open()) { +#pragma message("report error!") + continue; + // return ERROR_ACCESSFAILED; + } + + if (m_Type != TYPE_FALLOUT4 && m_Type != TYPE_STARFIELD && + m_Type != TYPE_STARFIELD_LZ4_TEXTURE && m_Type != TYPE_FALLOUT4NG_7 && + m_Type != TYPE_FALLOUT4NG_8) { + // BSA extraction + if (compressed(fileInfo.file)) { + // Decompress data + if (m_Type != TYPE_SKYRIMSE) { + // Oblivion - Skyrim LE use gzip compression + EErrorCode result = ERROR_NONE; + try { + BSAULong length = 0UL; + std::shared_ptr<unsigned char[]> buffer = + decompress(dataBuffer.first.get(), dataBuffer.second, result, length); + if (buffer.get() != nullptr) { + outputFile.write(reinterpret_cast<char*>(buffer.get()), length); + buffer.reset(); + } + } catch (const std::exception&) { +#pragma message("report error!") + dataBuffer.first.reset(); + fileInfo.data.first.reset(); + continue; + } + } else { + // Skyrim SE uses LZ4 Frame compression + if (!fileInfo.file->m_TextureChunks.size()) { + char* outBuffer = new char[fileInfo.file->m_UncompressedFileSize]; + + LZ4F_decompressionContext_t dcContext = nullptr; + LZ4F_decompressOptions_t options = {}; + LZ4F_createDecompressionContext(&dcContext, LZ4F_VERSION); + size_t outSize = fileInfo.file->m_UncompressedFileSize; + size_t inSize = dataBuffer.second; + + LZ4F_decompress(dcContext, outBuffer, &outSize, dataBuffer.first.get(), + &inSize, &options); + outputFile.write(outBuffer, fileInfo.file->m_UncompressedFileSize); + LZ4F_freeDecompressionContext(dcContext); + delete[] outBuffer; + } + } + } else { + // No compression - just write the data. + outputFile.write(reinterpret_cast<char*>(dataBuffer.first.get()), + dataBuffer.second); + } + } else { + // BA2 format + if (fileInfo.file->m_TextureChunks.size()) { + // Texture stream format - requires building the header data for the DDS file + bool isDX10 = false; + DirectX::DDS_HEADER_DXT10 DX10HeaderData = {}; + DirectX::DDS_HEADER DDSHeaderData = + getDDSHeader(fileInfo.file, DX10HeaderData, isDX10); + + outputFile.write("DDS ", 4); + char* DDSHeader = new char[sizeof(DDSHeaderData)]; + memcpy(DDSHeader, &DDSHeaderData, sizeof(DDSHeaderData)); + outputFile.write(DDSHeader, sizeof(DDSHeaderData)); + delete[] DDSHeader; + + if (isDX10) { + // This format requires DX10 header info + getDX10Header(DX10HeaderData, fileInfo.file, DDSHeaderData); + + char* DX10Header = new char[sizeof(DX10HeaderData)]; + memcpy(DX10Header, &DX10HeaderData, sizeof(DX10HeaderData)); + outputFile.write(DX10Header, sizeof(DX10HeaderData)); + delete[] DX10Header; + } + } + + EErrorCode result = ERROR_NONE; + try { + if (fileInfo.file->m_FileSize > 0 && !fileInfo.file->m_TextureChunks.size()) { + BSAULong length = fileInfo.file->m_UncompressedFileSize; + std::shared_ptr<unsigned char[]> buffer = + decompress(dataBuffer.first.get(), dataBuffer.second, result, length); + if (buffer.get() != nullptr) { + outputFile.write(reinterpret_cast<char*>(buffer.get()), length); + buffer.reset(); + } + } else { + outputFile.write(reinterpret_cast<char*>(dataBuffer.first.get()), + dataBuffer.second); + } + } catch (const std::exception&) { +#pragma message("report error!") + dataBuffer.first.reset(); + fileInfo.data.first.reset(); + continue; + } + } + dataBuffer.first.reset(); + fileInfo.data.first.reset(); + } +} + +void Archive::createFolders(const std::string& targetDirectory, Folder::Ptr folder) +{ + for (std::vector<Folder::Ptr>::iterator iter = folder->m_SubFolders.begin(); + iter != folder->m_SubFolders.end(); ++iter) { + std::string subDirName = targetDirectory + "/" + (*iter)->getName(); +#ifdef _WIN32 + ::CreateDirectoryA(subDirName.c_str(), nullptr); +#else + mkdir(subDirName.c_str(), 0755); +#endif + createFolders(subDirName, *iter); + } +} + +EErrorCode Archive::extractAll( + const char* outputDirectory, + const std::function<bool(int value, std::string fileName)>& progress, + bool overwrite) +{ +#pragma message("report errors") + createFolders(outputDirectory, m_RootFolder); + + std::vector<File::Ptr> fileList; + m_RootFolder->collectFiles(fileList); + std::sort(fileList.begin(), fileList.end(), ByOffset); + m_File.seekg((*(fileList.begin()))->m_DataOffset); + + std::queue<FileInfo> buffers; + boost::mutex queueMutex; + int filesDone = 0; + boost::interprocess::interprocess_semaphore bufferCount(0); + boost::interprocess::interprocess_semaphore queueFree(100); + + boost::thread readerThread(boost::bind(&Archive::readFiles, this, boost::ref(buffers), + boost::ref(queueMutex), + boost::ref(bufferCount), boost::ref(queueFree), + fileList.begin(), fileList.end())); + + boost::thread extractThread(boost::bind( + &Archive::extractFiles, this, outputDirectory, boost::ref(buffers), + boost::ref(queueMutex), boost::ref(bufferCount), boost::ref(queueFree), + static_cast<int>(fileList.size()), overwrite, boost::ref(filesDone))); + + bool readerDone = false; + bool extractDone = false; + bool canceled = false; + while (!readerDone || !extractDone) { + if (!readerDone) { + readerDone = readerThread.timed_join(boost::posix_time::millisec(100)); + } + if (readerDone) { + extractDone = extractThread.timed_join(boost::posix_time::millisec(100)); + // don't cancel extractor before reader is done or else reader may be stuck trying + // to write to a queue + if (canceled) { + // ensure the extract thread wakes up. + extractThread.interrupt(); + bufferCount.post(); + } + } + size_t index = (std::min)(static_cast<size_t>(filesDone), fileList.size() - 1); + if (!progress((filesDone * 100) / static_cast<int>(fileList.size()), + fileList[index]->getName()) && + !canceled) { + readerThread.interrupt(); + canceled = true; // don't interrupt repeatedly + } + } + + return ERROR_NONE; +} + +bool Archive::compressed(const File::Ptr& file) const +{ + if (m_Type != TYPE_FALLOUT4 && m_Type != TYPE_FALLOUT4NG_7 && + m_Type != TYPE_FALLOUT4NG_8) + return file->compressToggled() ^ defaultCompressed(); + return (file->m_FileSize > 0); +} + +File::Ptr Archive::createFile(const std::string& name, const std::string& sourceName, + bool compressed) +{ + return File::Ptr( + new File(name, sourceName, nullptr, defaultCompressed() != compressed)); +} + +void Archive::cleanFolder(Folder::Ptr folder) +{ + std::vector<Folder::Ptr> folders; + folder->collectFolders(folders); + for (Folder::Ptr subFolder : folders) { + cleanFolder(subFolder); + } + std::vector<File::Ptr> files; + folder->collectFiles(files); + for (File::Ptr file : files) { + file.reset(); + } + folder.reset(); +} + +} // namespace BSA diff --git a/libs/bsatk/src/bsaexception.cpp b/libs/bsatk/src/bsaexception.cpp new file mode 100644 index 0000000..9422848 --- /dev/null +++ b/libs/bsatk/src/bsaexception.cpp @@ -0,0 +1,39 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "bsaexception.h" + +#include <cstdarg> +#include <cstdio> + +#pragma warning(disable : 4996) + +std::string makeString(const char* format, ...) +{ + va_list argList; + va_start(argList, format); + char buffer[1024]; + vsnprintf(buffer, 1024, format, argList); + return std::string(buffer); +} + +data_invalid_exception::data_invalid_exception(const std::string& message) + : m_Message(message) +{} diff --git a/libs/bsatk/src/bsafile.cpp b/libs/bsatk/src/bsafile.cpp new file mode 100644 index 0000000..db4e4e6 --- /dev/null +++ b/libs/bsatk/src/bsafile.cpp @@ -0,0 +1,154 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "bsafile.h" + +#include <algorithm> +#include <climits> +#include <cstring> +#include <memory> +#include <stdexcept> + +#include "bsaexception.h" +#include "bsafolder.h" +#include "filehash.h" + +using std::fstream; +using std::ifstream; +using std::ofstream; + +namespace BSA +{ + +bool ByOffset(const File::Ptr& LHS, const File::Ptr& RHS) +{ + return LHS->getDataOffset() < RHS->getDataOffset(); +} + +static const unsigned long CHUNK_SIZE = 128 * 1024; + +File::File(std::fstream& file, Folder* folder) + : m_Folder(folder), m_New(false), m_FileSize(0), m_UncompressedFileSize(0), + m_ToggleCompressedWrite(false), m_DataOffsetWrite(0) +{ + m_NameHash = readType<BSAHash>(file); + m_FileSize = readType<BSAULong>(file); + m_DataOffset = readType<BSAULong>(file); + m_ToggleCompressed = m_FileSize & COMPRESSMASK; + m_FileSize = m_FileSize & SIZEMASK; +} + +File::File(const std::string& name, Folder* folder, BSAULong fileSize, + BSAHash dataOffset, BSAULong uncompressedFileSize, FO4TextureHeader header, + std::vector<FO4TextureChunk>& texChunks) + : m_Folder(folder), m_New(false), m_Name(name), m_FileSize(fileSize), + m_UncompressedFileSize(uncompressedFileSize), m_DataOffset(dataOffset), + m_TextureHeader(header), m_ToggleCompressedWrite(false), + m_TextureChunks(texChunks), m_DataOffsetWrite(0) +{ + m_NameHash = calculateBSAHash(name); + m_ToggleCompressed = false; + if (m_FileSize > 0 && m_UncompressedFileSize > 0) + m_ToggleCompressed = true; +} + +File::File(const std::string& name, const std::string& sourceFile, Folder* folder, + bool toggleCompressed) + : m_Folder(folder), m_New(true), m_Name(name), m_FileSize(0), + m_UncompressedFileSize(0), m_DataOffset(0), m_ToggleCompressed(toggleCompressed), + m_SourceFile(sourceFile), m_ToggleCompressedWrite(toggleCompressed), + m_DataOffsetWrite(0) +{ + m_NameHash = calculateBSAHash(name); +} + +std::string File::getFilePath() const +{ + return m_Folder->getFullPath() + "/" + m_Name; +} + +void File::writeHeader(fstream& file) const +{ + writeType<BSAHash>(file, m_NameHash); + BSAULong size = m_FileSize; + if (m_ToggleCompressed) { + size |= (1 << 30); + } + writeType<BSAULong>(file, size); + writeType<BSAULong>(file, m_DataOffsetWrite); +} + +EErrorCode File::writeData(fstream& sourceArchive, fstream& targetArchive) const +{ + m_DataOffsetWrite = static_cast<BSAULong>(targetArchive.tellp()); + EErrorCode result = ERROR_NONE; + + std::unique_ptr<char[]> inBuffer(new char[CHUNK_SIZE]); + + if (m_SourceFile.length() == 0) { + // copy from source archive +#pragma message("we may have to compress/decompress!") + sourceArchive.seekg(m_DataOffset, fstream::beg); + + try { + unsigned long sizeLeft = m_FileSize; + while (sizeLeft > 0) { + int chunkSize = (std::min)(sizeLeft, CHUNK_SIZE); + sourceArchive.read(inBuffer.get(), chunkSize); + targetArchive.write(inBuffer.get(), chunkSize); + sizeLeft -= chunkSize; + } + } catch (const std::exception&) { + result = ERROR_INVALIDDATA; + } + } else { + // copy from file on disc + fstream sourceFile; + sourceFile.open(m_SourceFile.c_str()); + if (!sourceFile.is_open()) { + return ERROR_SOURCEFILEMISSING; + } + sourceFile.seekg(0, fstream::end); + m_FileSize = static_cast<BSAULong>(sourceFile.tellg()); + unsigned long sizeLeft = m_FileSize; + sourceFile.seekg(0, fstream::beg); + while (sizeLeft > 0) { + int chunkSize = (std::min)(sizeLeft, CHUNK_SIZE); + sourceFile.read(inBuffer.get(), chunkSize); + targetArchive.write(inBuffer.get(), chunkSize); + sizeLeft -= chunkSize; + } + } + return result; +} + +void File::readFileName(fstream& file, bool testHashes) +{ + m_Name = readZString(file); + if (testHashes) { + if (calculateBSAHash(m_Name) != m_NameHash) { + throw data_invalid_exception( + makeString("invalid name hash for \"%s\" (%lx vs %lx)", m_Name.c_str(), + calculateBSAHash(m_Name), m_NameHash)); + } + } +} + +} // namespace BSA diff --git a/libs/bsatk/src/bsafolder.cpp b/libs/bsatk/src/bsafolder.cpp new file mode 100644 index 0000000..21e2611 --- /dev/null +++ b/libs/bsatk/src/bsafolder.cpp @@ -0,0 +1,342 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include <filesystem> +#include <limits.h> + +#include "bsaarchive.h" +#include "bsaexception.h" +#include "bsafile.h" +#include "bsafolder.h" + +using std::fstream; + +namespace BSA +{ + +Folder::Folder() : m_Parent(nullptr), m_Name() +{ + m_NameHash = calculateBSAHash(m_Name); + m_FileCount = 0; + m_Offset = ULONG_MAX; +} + +Folder::Ptr Folder::readFolder(std::fstream& file, BSAUInt fileNamesLength, + BSAUInt& endPos) +{ + Folder::Ptr result(new Folder()); + result->m_NameHash = readType<BSAHash>(file); + result->m_FileCount = readType<unsigned long>(file); + result->m_Offset = readType<unsigned long>(file); + std::streamoff pos = file.tellg(); + + file.seekg(result->m_Offset - fileNamesLength, fstream::beg); + + result->m_Name = readBString(file); + + for (unsigned long i = 0UL; i < result->m_FileCount; ++i) { + result->m_Files.push_back(File::Ptr(new File(file, result.get()))); + } + + if (static_cast<unsigned long>(file.tellg()) > endPos) { + endPos = static_cast<BSAULong>(file.tellg()); + } + + file.seekg(pos); + + return result; +} + +Folder::Ptr Folder::readFolderSE(std::fstream& file, BSAUInt fileNamesLength, + BSAUInt& endPos) +{ + Folder::Ptr result(new Folder()); + result->m_NameHash = readType<BSAHash>(file); + result->m_FileCount = readType<BSAUInt>(file); + readType<BSAUInt>(file); + result->m_Offset = readType<BSAHash>(file); + std::streamoff pos = file.tellg(); + + file.seekg(result->m_Offset - fileNamesLength, fstream::beg); + + result->m_Name = readBString(file); + + for (unsigned long i = 0UL; i < result->m_FileCount; ++i) { + result->m_Files.push_back(File::Ptr(new File(file, result.get()))); + } + + if (static_cast<unsigned long>(file.tellg()) > endPos) { + endPos = static_cast<BSAULong>(file.tellg()); + } + + file.seekg(pos); + + return result; +} + +void Folder::writeHeader(std::fstream& file) const +{ + writeType<BSAHash>(file, m_NameHash); + writeType<BSAULong>(file, static_cast<BSAULong>(m_Files.size())); + writeType<BSAULong>(file, m_OffsetWrite); +} + +void Folder::writeData(std::fstream& file, BSAULong fileNamesLength) const +{ + m_OffsetWrite = static_cast<BSAULong>(file.tellp()) + fileNamesLength; + writeBString(file, getFullPath()); + for (std::vector<File::Ptr>::const_iterator iter = m_Files.begin(); + iter != m_Files.end(); ++iter) { + (*iter)->writeHeader(file); + } +} + +EErrorCode Folder::writeFileData(std::fstream& sourceFile, + std::fstream& targetFile) const +{ + for (std::vector<File::Ptr>::const_iterator iter = m_Files.begin(); + iter != m_Files.end(); ++iter) { + EErrorCode error = (*iter)->writeData(sourceFile, targetFile); + if (error != ERROR_NONE) { + return error; + } + } + return ERROR_NONE; +} + +std::string Folder::getFullPath() const +{ + if (m_Parent != nullptr) { + std::string temp = m_Parent->getFullPath(); + if (temp.length() != 0) { + return temp.append("\\").append(m_Name); + } else { + return m_Name; + } + } else { + // root folder shouldn't have a name + return std::string(); + } +} + +void Folder::addFolderInt(Folder::Ptr folder) +{ + std::filesystem::path path(folder->m_Name); + auto it = path.begin(); + std::string firstStr = it->string(); + std::filesystem::path remaining; + for (++it; it != path.end(); ++it) { + remaining /= *it; + } + + if (m_SubFoldersByName.contains(firstStr)) { + // remove the matched part of the path and recurse + folder->m_Name = remaining.string(); + m_SubFoldersByName.at(firstStr)->addFolderInt(folder); + return; + } + + // no subfolder matches, create one + if (remaining.empty()) { + // no more path components, add the new folder right here + folder->m_Parent = this; + m_SubFolders.push_back(folder); + m_SubFoldersByName[firstStr] = folder; + } else { + // add dummy folder for the next path component + Folder::Ptr dummy(new Folder); + dummy->m_Parent = this; + dummy->m_Name = firstStr; + folder->m_Name = remaining.string(); + dummy->addFolderInt(folder); + m_SubFolders.push_back(dummy); + m_SubFoldersByName[firstStr] = dummy; + } +} + +Folder::Ptr Folder::addOrFindFolderInt(Folder* folder) +{ + std::filesystem::path path(folder->m_Name); + auto it = path.begin(); + std::string firstStr = it->string(); + std::filesystem::path remaining; + for (++it; it != path.end(); ++it) { + remaining /= *it; + } + + if (m_SubFoldersByName.contains(firstStr)) { + if (!remaining.empty()) { + // remove the matched part of the path and recurse + folder->m_Name = remaining.string(); + return m_SubFoldersByName.at(firstStr)->addOrFindFolderInt(folder); + } else { + return m_SubFoldersByName.at(firstStr); + } + } + + // no subfolder matches, create one + if (remaining.empty()) { + // no more path components, add the new folder right here + folder->m_Parent = this; + m_SubFolders.push_back(Folder::Ptr(folder)); + m_SubFoldersByName[firstStr] = m_SubFolders.back(); + return m_SubFolders.back(); + } else { + // add dummy folder for the next path component + Folder::Ptr dummy(new Folder); + dummy->m_Parent = this; + dummy->m_Name = firstStr; + folder->m_Name = remaining.string(); + Folder::Ptr result = dummy->addOrFindFolderInt(folder); + m_SubFolders.push_back(dummy); + m_SubFoldersByName[firstStr] = dummy; + return result; + } +} + +Folder::Ptr Folder::addFolder(std::fstream& file, BSAUInt fileNamesLength, + BSAUInt& endPos, ArchiveType type) +{ + Folder::Ptr temp; + if (type == ArchiveType::TYPE_SKYRIMSE) + temp = readFolderSE(file, fileNamesLength, endPos); + else + temp = readFolder(file, fileNamesLength, endPos); + addFolderInt(temp); + + return temp; +} + +Folder::Ptr Folder::addFolderFromFile(std::string filePath, BSAUInt size, + BSAHash offset, BSAUInt uncompressedSize, + FO4TextureHeader header, + std::vector<FO4TextureChunk>& texChunks) +{ + std::filesystem::path file(filePath); + + Folder* tempFolder = new Folder(); + + tempFolder->m_NameHash = calculateBSAHash(filePath); + tempFolder->m_Name = file.parent_path().string(); + Folder::Ptr result = addOrFindFolderInt(tempFolder); + + if (result.get() != tempFolder) { + delete tempFolder; + } + + std::string fileName = file.filename().string(); + + result->m_FileCount++; + result->m_Files.push_back(File::Ptr(new File(fileName, result.get(), size, offset, + uncompressedSize, header, texChunks))); + + return result; +} + +bool Folder::resolveFileNames(std::fstream& file, bool testHashes) +{ + bool hashesValid = true; + for (std::vector<File::Ptr>::iterator iter = m_Files.begin(); iter != m_Files.end(); + ++iter) { + try { + (*iter)->readFileName(file, testHashes); + } catch (const std::exception&) { + hashesValid = false; + } + } + return hashesValid; +} + +const Folder::Ptr Folder::getSubFolder(unsigned int index) const +{ + return m_SubFolders.at(index); +} + +unsigned int Folder::countFiles() const +{ + unsigned int result = 0; + for (std::vector<Folder::Ptr>::const_iterator iter = m_SubFolders.begin(); + iter != m_SubFolders.end(); ++iter) { + result += (*iter)->countFiles(); + } + return result + static_cast<unsigned int>(m_Files.size()); +} + +const File::Ptr Folder::getFile(unsigned int index) const +{ + return m_Files.at(index); +} + +Folder::Ptr Folder::addFolder(const std::string& folderName) +{ + Folder::Ptr newFolder(new Folder); + newFolder->m_Name = folderName; + newFolder->m_Parent = this; + m_SubFolders.push_back(newFolder); + return newFolder; +} + +void Folder::collectFolders(std::vector<Folder::Ptr>& folderList) const +{ + for (std::vector<Folder::Ptr>::const_iterator iter = m_SubFolders.begin(); + iter != m_SubFolders.end(); ++iter) { + if ((*iter)->m_Files.size() != 0) { + folderList.push_back(*iter); + } + (*iter)->collectFolders(folderList); + } +} + +void Folder::collectFiles(std::vector<File::Ptr>& fileList) const +{ + for (std::vector<File::Ptr>::const_iterator fileIter = m_Files.begin(); + fileIter != m_Files.end(); ++fileIter) { + fileList.push_back(*fileIter); + } + for (std::vector<Folder::Ptr>::const_iterator folderIter = m_SubFolders.begin(); + folderIter != m_SubFolders.end(); ++folderIter) { + (*folderIter)->collectFiles(fileList); + } +} + +void Folder::collectFileNames(std::vector<std::string>& nameList) const +{ + for (std::vector<File::Ptr>::const_iterator iter = m_Files.begin(); + iter != m_Files.end(); ++iter) { + nameList.push_back((*iter)->getName()); + } + for (std::vector<Folder::Ptr>::const_iterator iter = m_SubFolders.begin(); + iter != m_SubFolders.end(); ++iter) { + (*iter)->collectFileNames(nameList); + } +} + +void Folder::collectFolderNames(std::vector<std::string>& nameList) const +{ + if (m_Files.size() != 0) { + nameList.push_back(getFullPath()); + } + for (std::vector<Folder::Ptr>::const_iterator iter = m_SubFolders.begin(); + iter != m_SubFolders.end(); ++iter) { + (*iter)->collectFolderNames(nameList); + } +} + +} // namespace BSA diff --git a/libs/bsatk/src/bsatypes.cpp b/libs/bsatk/src/bsatypes.cpp new file mode 100644 index 0000000..eb33af3 --- /dev/null +++ b/libs/bsatk/src/bsatypes.cpp @@ -0,0 +1,70 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "bsatypes.h" + +#include <algorithm> +#include <climits> +#include <cstring> +#include <fstream> +#include <string> + +using std::fstream; + +/*#ifndef MAX_PATH +#define MAX_PATH PATH_MAX +#endif // MAX_PATH +*/ + +std::string readBString(fstream& file) +{ + unsigned char length = readType<unsigned char>(file); + char buffer[256]; + if (length > 0) { + if (!file.read(buffer, length)) { + throw data_invalid_exception("can't read from bsa"); + } + } + buffer[length] = '\0'; + return std::string(buffer); +} + +void writeBString(fstream& file, const std::string& string) +{ + unsigned int length = + std::min<unsigned int>(static_cast<unsigned int>(string.length()), 255); + writeType<unsigned char>(file, length + 1); + file.write(string.c_str(), length + 1); +} + +std::string readZString(fstream& file) +{ + char buffer[FILENAME_MAX]; + memset(buffer, '\0', FILENAME_MAX); + if (!file.getline(buffer, FILENAME_MAX, '\0')) { + throw data_invalid_exception("can't read from bsa"); + } + return std::string(buffer); +} + +void writeZString(fstream& file, const std::string& string) +{ + file.write(string.c_str(), string.length() + 1); +} diff --git a/libs/bsatk/src/filehash.cpp b/libs/bsatk/src/filehash.cpp new file mode 100644 index 0000000..717a02c --- /dev/null +++ b/libs/bsatk/src/filehash.cpp @@ -0,0 +1,100 @@ +/* +Mod Organizer BSA handling + +Copyright (C) 2012 Sebastian Herbord. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "filehash.h" + +#include <algorithm> +#include <climits> +#include <cstdlib> +#include <cstring> + +#ifndef MAX_PATH +#define MAX_PATH PATH_MAX +#endif // MAX_PATH + +static unsigned long genHashInt(const unsigned char* pos, const unsigned char* end) +{ + unsigned long hash = 0; + for (; pos < end; ++pos) { + hash *= 0x1003f; + hash += *pos; + } + return hash; +} + +/** + * @brief calculateBSAHash + * @param fileName + * @return + * @note the hash calculated for folders seem to be wrong + */ +BSAHash calculateBSAHash(const std::string& fileName) +{ + char fileNameLower[FILENAME_MAX + 1]; + int i = 0; + for (; i < FILENAME_MAX && fileName[i] != '\0'; ++i) { + fileNameLower[i] = tolower(fileName[i]); + if (fileNameLower[i] == '/') { + fileNameLower[i] = '\\'; + } + } + fileNameLower[i] = '\0'; + + unsigned char* fileNameLowerU = reinterpret_cast<unsigned char*>(fileNameLower); + + size_t length = strlen(fileNameLower); + + char* ext = strrchr(fileNameLower, '.'); + if (ext == nullptr) { + ext = fileNameLower + length; + } + + size_t extLen = strlen(ext); + length -= extLen; + + unsigned char* extU = reinterpret_cast<unsigned char*>(ext); + + BSAHash hash1 = 0ULL; + + if (length > 0) { + hash1 = static_cast<BSAHash>(fileNameLowerU[length - 1] | + ((length > 2 ? fileNameLowerU[length - 2] : 0) << 8) | + (length << 16) | (fileNameLowerU[0] << 24)); + } + + if (extLen > 0) { + if (strcmp(ext + 1, "kf") == 0) { + hash1 |= 0x80; + } else if (strcmp(ext + 1, "nif") == 0) { + hash1 |= 0x8000; + } else if (strcmp(ext + 1, "dds") == 0) { + hash1 |= 0x8080; + } else if (strcmp(ext + 1, "wav") == 0) { + hash1 |= 0x80000000; + } + + BSAHash hash2 = static_cast<BSAHash>(genHashInt(fileNameLowerU + 1, extU - 2)) + + static_cast<BSAHash>(genHashInt(extU, extU + strlen(ext))); + + hash1 |= (hash2 & 0xFFFFFFFF) << 32; + } + + return hash1; +} diff --git a/libs/bsatk/vcpkg.json b/libs/bsatk/vcpkg.json new file mode 100644 index 0000000..7a82d05 --- /dev/null +++ b/libs/bsatk/vcpkg.json @@ -0,0 +1,30 @@ +{ + "dependencies": [ + "mo2-dds-header", + "zlib", + "lz4", + "boost-thread", + "boost-interprocess" + ], + "vcpkg-configuration": { + "default-registry": { + "kind": "git", + "repository": "https://github.com/Microsoft/vcpkg", + "baseline": "294f76666c3000630d828703e675814c05a4fd43" + }, + "registries": [ + { + "kind": "git", + "repository": "https://github.com/Microsoft/vcpkg", + "baseline": "294f76666c3000630d828703e675814c05a4fd43", + "packages": ["boost*", "boost-*"] + }, + { + "kind": "git", + "repository": "https://github.com/ModOrganizer2/vcpkg-registry", + "baseline": "228cda39fe9d1eeed789c0ef64fd1235dab3b11e", + "packages": ["mo2-dds-header"] + } + ] + } +} |