/* Mod Organizer BSA handling Copyright (C) 2012 Sebastian Herbord. All rights reserved. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "bsaarchive.h" #include "bsaexception.h" #include "bsafile.h" #include "bsafolder.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include #endif using std::fstream; using namespace boost::posix_time; namespace BSA { Archive::Archive() : m_RootFolder(new Folder), m_ArchiveFlags(FLAG_HASDIRNAMES | FLAG_HASFILENAMES), m_Type(TYPE_SKYRIM) {} Archive::~Archive() { if (m_File.is_open()) { m_File.close(); } std::vector folders; m_RootFolder->collectFolders(folders); cleanFolder(m_RootFolder); m_RootFolder.reset(); } ArchiveType Archive::typeFromID(BSAULong typeID) { switch (typeID) { case 0x100: return TYPE_MORROWIND; case 0x67: return TYPE_OBLIVION; case 0x68: return TYPE_FALLOUT3; case 0x69: return TYPE_SKYRIMSE; case 0x01: return TYPE_FALLOUT4; case 0x02: return TYPE_STARFIELD; case 0x03: return TYPE_STARFIELD_LZ4_TEXTURE; case 0x07: return TYPE_FALLOUT4NG_7; case 0x08: return TYPE_FALLOUT4NG_8; default: throw data_invalid_exception(makeString("invalid type %d", typeID)); } } BSAULong Archive::typeToID(ArchiveType type) { switch (type) { case TYPE_MORROWIND: return 0x100; case TYPE_OBLIVION: return 0x67; case TYPE_FALLOUT3: return 0x68; case TYPE_SKYRIMSE: return 0x69; case TYPE_FALLOUT4: return 0x01; case TYPE_STARFIELD: return 0x02; case TYPE_STARFIELD_LZ4_TEXTURE: return 0x03; case TYPE_FALLOUT4NG_7: return 0x07; case TYPE_FALLOUT4NG_8: return 0x08; default: throw data_invalid_exception(makeString("invalid type %d", type)); } } Archive::Header Archive::readHeader(std::fstream& infile) { Header result; result.fileIdentifier = readType(infile); if (result.fileIdentifier != 0x00415342 && result.fileIdentifier != 0x58445442 && result.fileIdentifier != 0x00000100) { throw data_invalid_exception(makeString("not a bsa or ba2 file")); } if (result.fileIdentifier != 0x00000100) { ArchiveType type = typeFromID(readType(infile)); if (type == TYPE_FALLOUT4 || type == TYPE_STARFIELD || type == TYPE_STARFIELD_LZ4_TEXTURE || type == TYPE_FALLOUT4NG_7 || type == TYPE_FALLOUT4NG_8) { result.type = type; infile.read(result.archType, 4); result.archType[4] = '\0'; result.fileCount = readType(infile); result.nameTableOffset = readType(infile); result.archiveFlags = FLAG_HASDIRNAMES | FLAG_HASFILENAMES; } else { result.type = type; result.offset = readType(infile); result.archiveFlags = readType(infile); result.folderCount = readType(infile); result.fileCount = readType(infile); result.folderNameLength = readType(infile); result.fileNameLength = readType(infile); result.fileFlags = readType(infile); } } else { result.type = TYPE_MORROWIND; result.offset = readType(infile); result.fileCount = readType(infile); result.archiveFlags = FLAG_HASDIRNAMES | FLAG_HASFILENAMES; } return result; } EErrorCode Archive::read(const char* fileName, bool testHashes) { m_File.open(fileName, fstream::in | fstream::binary); if (!m_File.is_open()) { return ERROR_FILENOTFOUND; } m_File.exceptions(std::ios_base::badbit); try { Header header; try { header = readHeader(m_File); } catch (const data_invalid_exception& e) { throw data_invalid_exception(makeString("%s (filename: %s)", e.what(), fileName)); } m_ArchiveFlags = header.archiveFlags; m_Type = header.type; if (m_Type == TYPE_FALLOUT4 || m_Type == TYPE_STARFIELD || m_Type == TYPE_STARFIELD_LZ4_TEXTURE || m_Type == TYPE_FALLOUT4NG_7 || m_Type == TYPE_FALLOUT4NG_8) { m_File.seekg(header.nameTableOffset); std::vector fileNames; for (unsigned int i = 0; i < header.fileCount; ++i) { BSAUShort length = readType(m_File); char* file = new char[length + 1]; m_File.read(file, length); file[length] = '\0'; fileNames.push_back(file); delete[] file; } std::streamoff offset; switch (m_Type) { case TYPE_STARFIELD: offset = 32; break; case TYPE_STARFIELD_LZ4_TEXTURE: offset = 36; break; default: offset = 24; } if (strcmp(header.archType, "GNRL") == 0) { m_File.seekg(offset, std::ios::beg); for (unsigned int i = 0; i < header.fileCount; ++i) { BSAUInt nameHash = readType(m_File); char* extension = new char[4]; m_File.read(extension, 4); BSAUInt dirHash = readType(m_File); m_File.seekg(4, std::ios::cur); BSAHash offset = readType(m_File); BSAUInt packedSize = readType(m_File); BSAUInt unpackedSize = readType(m_File); m_File.seekg(4, std::ios::cur); std::vector dummy; Folder::Ptr newDir = m_RootFolder->addFolderFromFile( fileNames[i], packedSize, offset, unpackedSize, {}, dummy); delete[] extension; } } else if (strcmp(header.archType, "DX10") == 0) { m_File.seekg(offset, std::ios::beg); for (unsigned int i = 0; i < header.fileCount; ++i) { FO4TextureHeader texHeader; texHeader.nameHash = readType(m_File); m_File.read(texHeader.extension, 4); texHeader.dirHash = readType(m_File); texHeader.unknown1 = readType(m_File); texHeader.chunkNumber = readType(m_File); texHeader.chunkHeaderSize = readType(m_File); texHeader.height = readType(m_File); texHeader.width = readType(m_File); texHeader.mipCount = readType(m_File); texHeader.format = static_cast(readType(m_File)); texHeader.isCubemap = readType(m_File); texHeader.unknown2 = readType(m_File); std::vector chunks; for (unsigned int j = 0; j < texHeader.chunkNumber; ++j) { FO4TextureChunk chunk; chunk.offset = readType(m_File); chunk.packedSize = readType(m_File); chunk.unpackedSize = readType(m_File); chunk.startMip = readType(m_File); chunk.endMip = readType(m_File); chunk.unknown = readType(m_File); chunks.push_back(chunk); } Folder::Ptr newDir = m_RootFolder->addFolderFromFile( fileNames[i], chunks[0].packedSize, chunks[0].offset, chunks[0].unpackedSize, texHeader, chunks); } } return ERROR_NONE; } else if (m_Type == TYPE_MORROWIND) { BSAUInt dataOffset = 12 + header.offset + header.fileCount * 8; std::vector fileSizeOffset(header.fileCount); m_File.read((char*)fileSizeOffset.data(), header.fileCount * sizeof(MorrowindFileOffset)); std::vector fileNameOffset(header.fileCount); m_File.read((char*)fileNameOffset.data(), header.fileCount * sizeof(BSAUInt)); BSAUInt last = header.offset - 12 * header.fileCount; for (uint32_t i = 0; i < header.fileCount; ++i) { uint32_t index = 0; if (i + 1 == header.fileCount) index = last; else index = fileNameOffset[i + 1] - fileNameOffset[i]; char* filePath = new char[index + 1]; m_File.read(filePath, index); filePath[index] = '\0'; std::vector dummy; Folder::Ptr newDir = m_RootFolder->addFolderFromFile( filePath, fileSizeOffset[i].size, dataOffset + fileSizeOffset[i].offset, 0, {}, dummy); delete[] filePath; } return ERROR_NONE; } else { // flat list of folders as they were stored in the archive std::vector folders; for (BSAUInt i = 0; i < header.folderCount; ++i) { folders.push_back(m_RootFolder->addFolder(m_File, header.fileNameLength, header.offset, header.type)); } m_File.seekg(header.offset); bool hashesValid = true; for (std::vector::iterator iter = folders.begin(); iter != folders.end(); ++iter) { if (!(*iter)->resolveFileNames(m_File, testHashes)) { hashesValid = false; } } return hashesValid ? ERROR_NONE : ERROR_INVALIDHASHES; } } catch (std::ios_base::failure&) { return ERROR_INVALIDDATA; } } void Archive::close() { m_File.close(); } BSAULong Archive::countFiles() const { return m_RootFolder->countFiles(); } std::vector Archive::collectFolderNames() const { std::vector result; m_RootFolder->collectFolderNames(result); return result; } std::vector Archive::collectFileNames() const { std::vector result; m_RootFolder->collectFileNames(result); return result; } BSAULong Archive::countCharacters(const std::vector& list) const { size_t sum = 0; for (std::vector::const_iterator iter = list.begin(); iter != list.end(); ++iter) { sum += iter->length() + 1; } return static_cast(sum); } #ifndef WIN32 #define _stricmp strcasecmp #endif // WIN32 static bool endsWith(const std::string& fileName, const char* extension) { size_t endLength = strlen(extension); if (fileName.length() < endLength) { return false; } return _stricmp(&fileName[fileName.length() - endLength], extension) == 0; } BSAULong Archive::determineFileFlags(const std::vector& fileList) const { BSAULong result = 0; bool hasNIF = false; bool hasDDS = false; bool hasXML = false; bool hasWAV = false; bool hasMP3 = false; bool hasTXT = false; bool hasSPT = false; bool hasTEX = false; bool hasCTL = false; for (std::vector::const_iterator iter = fileList.begin(); iter != fileList.end(); ++iter) { if (!hasNIF && endsWith(*iter, ".nif")) { hasNIF = true; result |= 1 << 0; } else if (!hasDDS && endsWith(*iter, ".dds")) { hasDDS = true; result |= 1 << 1; } else if (!hasXML && endsWith(*iter, ".xml")) { hasXML = true; result |= 1 << 2; } else if (!hasWAV && endsWith(*iter, ".wav")) { hasWAV = true; result |= 1 << 3; } else if (!hasMP3 && endsWith(*iter, ".mp3")) { hasMP3 = true; result |= 1 << 4; } else if (!hasTXT && endsWith(*iter, ".txt")) { hasTXT = true; result |= 1 << 5; } else if (!hasSPT && endsWith(*iter, ".spt")) { hasSPT = true; result |= 1 << 6; } else if (!hasTEX && endsWith(*iter, ".tex")) { hasTEX = true; result |= 1 << 7; } else if (!hasCTL && endsWith(*iter, ".ctl")) { hasCTL = true; result |= 1 << 8; } } return result; } void Archive::writeHeader(std::fstream& outfile, BSAULong fileFlags, BSAULong numFolders, BSAULong folderNamesLength, BSAULong fileNamesLength) { outfile.write("BSA\0", 4); writeType(outfile, typeToID(m_Type)); writeType(outfile, 0x24); // header size is static writeType(outfile, m_ArchiveFlags); writeType(outfile, numFolders); writeType(outfile, countFiles()); writeType(outfile, folderNamesLength); writeType(outfile, fileNamesLength); writeType(outfile, fileFlags); } EErrorCode Archive::write(const char* fileName) { std::fstream outfile; outfile.open(fileName, fstream::out | fstream::binary); if (!outfile.is_open()) { return ERROR_ACCESSFAILED; } outfile.exceptions(std::ios_base::badbit); std::vector folders; m_RootFolder->collectFolders(folders); std::vector folderNames; std::vector fileNames; BSAULong folderNamesLength = 0; BSAULong fileNamesLength = 0; for (std::vector::const_iterator folderIter = folders.begin(); folderIter != folders.end(); ++folderIter) { std::string fullPath = (*folderIter)->getFullPath(); folderNames.push_back(fullPath); folderNamesLength += static_cast(fullPath.length()); for (std::vector::const_iterator fileIter = (*folderIter)->m_Files.begin(); fileIter != (*folderIter)->m_Files.end(); ++fileIter) { fileNames.push_back((*fileIter)->m_Name); fileNamesLength += static_cast((*fileIter)->m_Name.length()); } } try { writeHeader(outfile, determineFileFlags(fileNames), static_cast(folderNames.size()), folderNamesLength, fileNamesLength); // dummy-pass: before we can store the actual folder data // prepare folder and file headers for (std::vector::const_iterator folderIter = folders.begin(); folderIter != folders.end(); ++folderIter) { (*folderIter)->writeHeader(outfile); } for (std::vector::const_iterator folderIter = folders.begin(); folderIter != folders.end(); ++folderIter) { (*folderIter)->writeData(outfile, fileNamesLength); } // write file names for (std::vector::const_iterator folderIter = fileNames.begin(); folderIter != fileNames.end(); ++folderIter) { writeZString(outfile, *folderIter); } // write file data for (std::vector::iterator folderIter = folders.begin(); folderIter != folders.end(); ++folderIter) { (*folderIter)->writeFileData(m_File, outfile); } outfile.seekp(0x24, fstream::beg); // re-write folder and file structure, this time with the correct // offsets for (std::vector::const_iterator folderIter = folders.begin(); folderIter != folders.end(); ++folderIter) { (*folderIter)->writeHeader(outfile); } for (std::vector::const_iterator folderIter = folders.begin(); folderIter != folders.end(); ++folderIter) { (*folderIter)->writeData(outfile, fileNamesLength); } outfile.close(); return ERROR_NONE; } catch (std::ios_base::failure&) { outfile.close(); return ERROR_INVALIDDATA; } } DirectX::DDS_HEADER Archive::getDDSHeader(File::Ptr file, DirectX::DDS_HEADER_DXT10& DX10Header, bool& isDX10) const { DirectX::DDS_HEADER DDSHeaderData = {}; DDSHeaderData.size = sizeof(DDSHeaderData); DDSHeaderData.flags = DDS_HEADER_FLAGS_TEXTURE | DDS_HEADER_FLAGS_LINEARSIZE | DDS_HEADER_FLAGS_MIPMAP; DDSHeaderData.height = file->m_TextureHeader.height; DDSHeaderData.width = file->m_TextureHeader.width; DDSHeaderData.mipMapCount = file->m_TextureHeader.mipCount; DDSHeaderData.ddspf.size = sizeof(DirectX::DDS_PIXELFORMAT); DDSHeaderData.caps = DDS_SURFACE_FLAGS_TEXTURE | DDS_SURFACE_FLAGS_MIPMAP; if (file->m_TextureHeader.isCubemap) DDSHeaderData.caps2 = DDS_CUBEMAP_ALLFACES; bool supported = true; switch (file->m_TextureHeader.format) { case DXGI_FORMAT_BC1_UNORM: case DXGI_FORMAT_BC1_UNORM_SRGB: DDSHeaderData.ddspf = DirectX::DDSPF_DXT1; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height / 2; break; case DXGI_FORMAT_BC2_UNORM: case DXGI_FORMAT_BC2_UNORM_SRGB: DDSHeaderData.ddspf = DirectX::DDSPF_DXT3; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height; break; case DXGI_FORMAT_BC3_UNORM: case DXGI_FORMAT_BC3_UNORM_SRGB: DDSHeaderData.ddspf = DirectX::DDSPF_DXT5; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height; break; case DXGI_FORMAT_BC4_UNORM: DDSHeaderData.ddspf = DirectX::DDSPF_BC4_UNORM; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height; break; case DXGI_FORMAT_BC4_SNORM: DDSHeaderData.ddspf = DirectX::DDSPF_BC4_SNORM; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height; break; case DXGI_FORMAT_BC5_UNORM: DDSHeaderData.ddspf = DirectX::DDSPF_BC5_UNORM; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height; break; case DXGI_FORMAT_BC5_SNORM: DDSHeaderData.ddspf = DirectX::DDSPF_BC5_SNORM; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height; break; case DXGI_FORMAT_BC7_UNORM: case DXGI_FORMAT_BC7_UNORM_SRGB: DDSHeaderData.ddspf = DirectX::DDSPF_DX10; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height; isDX10 = true; DX10Header.dxgiFormat = file->m_TextureHeader.format; break; case DXGI_FORMAT_R8G8B8A8_UNORM: DDSHeaderData.ddspf = DirectX::DDSPF_A8B8G8R8; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height * 4; // 32bpp break; case DXGI_FORMAT_B8G8R8A8_UNORM: DDSHeaderData.ddspf = DirectX::DDSPF_A8R8G8B8; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height * 4; // 32bpp break; case DXGI_FORMAT_B8G8R8X8_UNORM: DDSHeaderData.ddspf = DirectX::DDSPF_X8R8G8B8; break; case DXGI_FORMAT_R8_UNORM: DDSHeaderData.ddspf = DirectX::DDSPF_L8; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height; // 8bpp break; case DXGI_FORMAT_R16_UNORM: DDSHeaderData.ddspf = DirectX::DDSPF_L16; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height * 2; // 16bpp break; case DXGI_FORMAT_R8G8_UNORM: DDSHeaderData.ddspf = DirectX::DDSPF_A8L8; DDSHeaderData.pitchOrLinearSize = file->m_TextureHeader.width * file->m_TextureHeader.height * 2; // 16bpp break; default: return {}; break; } return DDSHeaderData; } void Archive::getDX10Header(DirectX::DDS_HEADER_DXT10& DX10Header, File::Ptr file, DirectX::DDS_HEADER DDSHeader) const { DX10Header.resourceDimension = DirectX::DDS_DIMENSION_TEXTURE2D; DX10Header.miscFlag = 0; DX10Header.arraySize = 1; DX10Header.miscFlags2 = 0; } static const BSAULong CHUNK_SIZE = 128 * 1024; EErrorCode Archive::extractDirect(File::Ptr file, std::ostream& outFile) const { EErrorCode result = ERROR_NONE; if (file->m_FileSize == 0) { // don't try to read empty file return result; } m_File.clear(); m_File.seekg(static_cast(file->m_DataOffset), std::ios::beg); if (m_Type == TYPE_FALLOUT4 || m_Type == TYPE_STARFIELD || m_Type == TYPE_STARFIELD_LZ4_TEXTURE || m_Type == TYPE_FALLOUT4NG_7 || m_Type == TYPE_FALLOUT4NG_8) { if (!file->m_TextureChunks.size()) { BSAULong size = file->m_UncompressedFileSize; std::unique_ptr buffer(new char[size]); m_File.read(buffer.get(), size); outFile.write(buffer.get(), size); } else { bool isDX10 = false; DirectX::DDS_HEADER_DXT10 DX10HeaderData = {}; DirectX::DDS_HEADER DDSHeaderData = getDDSHeader(file, DX10HeaderData, isDX10); outFile.write("DDS ", 4); char* DDSHeader = new char[sizeof(DDSHeaderData)]; memcpy(DDSHeader, &DDSHeaderData, sizeof(DDSHeaderData)); outFile.write(DDSHeader, sizeof(DDSHeaderData)); delete[] DDSHeader; if (isDX10) { getDX10Header(DX10HeaderData, file, DDSHeaderData); char* DX10Header = new char[sizeof(DX10HeaderData)]; memcpy(DX10Header, &DX10HeaderData, sizeof(DX10HeaderData)); outFile.write(DX10Header, sizeof(DX10HeaderData)); delete[] DX10Header; } for (BSAUInt i = 0; i < file->m_TextureChunks.size(); ++i) { BSAULong length = file->m_TextureChunks[i].unpackedSize; std::unique_ptr chunk(new char[length]); m_File.read(chunk.get(), length); outFile.write(chunk.get(), length); } } } else { BSAULong size = file->m_FileSize; if (namePrefixed()) { std::string fullName = readBString(m_File); if (size <= fullName.length()) { return result; } size -= fullName.length() + 1; } std::unique_ptr buffer(new unsigned char[size]); m_File.read(reinterpret_cast(buffer.get()), size); if (result == ERROR_NONE) outFile.write(reinterpret_cast(buffer.get()), size); } std::unique_ptr inBuffer(new char[CHUNK_SIZE]); try { BSAULong sizeLeft = file->m_FileSize; while (sizeLeft > 0) { int chunkSize = (std::min)(sizeLeft, CHUNK_SIZE); m_File.read(inBuffer.get(), chunkSize); outFile.write(inBuffer.get(), chunkSize); sizeLeft -= chunkSize; } } catch (const std::exception&) { result = ERROR_INVALIDDATA; } return result; } std::shared_ptr Archive::decompress(unsigned char* inBuffer, BSAULong inSize, EErrorCode& result, BSAULong& outSize) { if (outSize == 0) { memcpy(&outSize, inBuffer, sizeof(BSAULong)); inBuffer += sizeof(BSAULong); inSize -= sizeof(BSAULong); } if ((inSize == 0) || (outSize == 0)) { return std::shared_ptr(); } std::shared_ptr outBuffer(new unsigned char[outSize]); z_stream stream; try { stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; stream.avail_in = inSize; stream.next_in = static_cast(inBuffer); int zlibRet = inflateInit2(&stream, 15 + 32); if (zlibRet != Z_OK) { result = ERROR_ZLIBINITFAILED; return std::shared_ptr(); } do { stream.avail_out = outSize; stream.next_out = reinterpret_cast(outBuffer.get()); zlibRet = inflate(&stream, Z_NO_FLUSH); if ((zlibRet != Z_OK) && (zlibRet != Z_STREAM_END) && (zlibRet != Z_BUF_ERROR)) { throw std::runtime_error("invalid data"); } } while (stream.avail_out == 0); inflateEnd(&stream); return outBuffer; } catch (const std::exception&) { result = ERROR_INVALIDDATA; inflateEnd(&stream); return std::shared_ptr(); } } EErrorCode Archive::extractCompressed(File::Ptr file, std::ostream& outFile) const { EErrorCode result = ERROR_NONE; if (file->m_FileSize == 0) { // don't try to read empty file return result; } m_File.clear(); m_File.seekg(static_cast(file->m_DataOffset), std::ios::beg); if (m_Type == TYPE_FALLOUT4 || m_Type == TYPE_STARFIELD || m_Type == TYPE_STARFIELD_LZ4_TEXTURE || m_Type == TYPE_FALLOUT4NG_7 || m_Type == TYPE_FALLOUT4NG_8) { if (!file->m_TextureChunks.size()) { BSAULong inSize = file->m_FileSize; std::unique_ptr inBuffer(new unsigned char[inSize]); m_File.read(reinterpret_cast(inBuffer.get()), inSize); BSAULong length = file->m_UncompressedFileSize; std::shared_ptr buffer = decompress(inBuffer.get(), inSize, result, length); if (result == ERROR_NONE) { outFile.write(reinterpret_cast(buffer.get()), length); } } else { bool isDX10 = false; DirectX::DDS_HEADER_DXT10 DX10HeaderData = {}; DirectX::DDS_HEADER DDSHeaderData = getDDSHeader(file, DX10HeaderData, isDX10); outFile.write("DDS ", 4); char* DDSHeader = new char[sizeof(DDSHeaderData)]; memcpy(DDSHeader, &DDSHeaderData, sizeof(DDSHeaderData)); outFile.write(DDSHeader, sizeof(DDSHeaderData)); delete[] DDSHeader; if (isDX10) { getDX10Header(DX10HeaderData, file, DDSHeaderData); char* DX10Header = new char[sizeof(DX10HeaderData)]; memcpy(DX10Header, &DX10HeaderData, sizeof(DX10HeaderData)); outFile.write(DX10Header, sizeof(DX10HeaderData)); delete[] DX10Header; } for (BSAUInt i = 0; i < file->m_TextureChunks.size(); ++i) { BSAULong length = file->m_TextureChunks[i].unpackedSize; if (file->m_TextureChunks[i].packedSize > 0) { unsigned char* chunk = new unsigned char[file->m_TextureChunks[i].packedSize]; m_File.read(reinterpret_cast(chunk), file->m_TextureChunks[i].packedSize); if (m_Type == TYPE_STARFIELD_LZ4_TEXTURE) { char* unpackedChunk = new char[length]; LZ4_decompress_safe(reinterpret_cast(chunk), unpackedChunk, file->m_TextureChunks[i].packedSize, length); outFile.write(unpackedChunk, length); delete[] unpackedChunk; } else { std::shared_ptr unpackedChunk = decompress(chunk, file->m_TextureChunks[i].packedSize, result, length); if (result == ERROR_NONE) { outFile.write(reinterpret_cast(unpackedChunk.get()), length); } } delete[] chunk; } else { char* chunk = new char[length]; m_File.read(chunk, length); outFile.write(chunk, length); delete[] chunk; } } } } else if (m_Type == TYPE_SKYRIMSE) { BSAULong inSize = file->m_FileSize; if (namePrefixed()) { std::string fullName = readBString(m_File); if (inSize <= fullName.length()) { return result; } inSize -= fullName.length() + 1; } BSAULong outSize = readType(m_File); inSize -= sizeof(BSAULong); std::unique_ptr inBuffer(new unsigned char[inSize]); m_File.read(reinterpret_cast(inBuffer.get()), inSize); LZ4F_decompressionContext_t dcContext = nullptr; LZ4F_decompressOptions_t options = {}; LZ4F_createDecompressionContext(&dcContext, LZ4F_VERSION); size_t lzOutSize = outSize; size_t lzInSize = inSize; std::unique_ptr outBuffer(new unsigned char[outSize]); LZ4F_decompress(dcContext, outBuffer.get(), &lzOutSize, inBuffer.get(), &lzInSize, &options); outFile.write(reinterpret_cast(outBuffer.get()), outSize); } else { BSAULong inSize = file->m_FileSize; if (namePrefixed()) { std::string fullName = readBString(m_File); if (inSize <= fullName.length()) { return result; } inSize -= fullName.length() + 1; } std::unique_ptr inBuffer(new unsigned char[inSize]); m_File.read(reinterpret_cast(inBuffer.get()), inSize); BSAULong length = 0UL; std::shared_ptr buffer = decompress(inBuffer.get(), inSize, result, length); if (result == ERROR_NONE) { outFile.write(reinterpret_cast(buffer.get()), length); } } return result; } EErrorCode Archive::extract(File::Ptr file, const char* outputDirectory) const { std::string fileName = makeString("%s/%s", outputDirectory, file->getName().c_str()); std::ofstream outputFile(fileName.c_str(), fstream::out | fstream::binary | fstream::trunc); if (!outputFile.is_open()) { return ERROR_ACCESSFAILED; } EErrorCode result = ERROR_NONE; if (compressed(file)) { result = extractCompressed(file, outputFile); } else { result = extractDirect(file, outputFile); } outputFile.close(); return result; } EErrorCode Archive::extractToMemory(File::Ptr file, std::vector& result) const { result.clear(); if (!file) { return ERROR_INVALIDDATA; } std::ostringstream stream(std::ios::out | std::ios::binary); EErrorCode rc = ERROR_NONE; if (compressed(file)) { rc = extractCompressed(file, stream); } else { rc = extractDirect(file, stream); } if (rc != ERROR_NONE) { return rc; } const std::string& data = stream.str(); result.assign(reinterpret_cast(data.data()), reinterpret_cast(data.data() + data.size())); return ERROR_NONE; } static bool iequals_bsa(const std::string& lhs, const std::string& rhs) { if (lhs.size() != rhs.size()) { return false; } for (std::size_t i = 0; i < lhs.size(); ++i) { if (std::tolower(static_cast(lhs[i])) != std::tolower(static_cast(rhs[i]))) { return false; } } return true; } // Normalize a path for case-insensitive, separator-agnostic comparison: // all forward slashes, all lowercase. Used by findFile only — doesn't mutate // stored names. static std::string normalize_bsa_path(const std::string& in) { std::string out; out.reserve(in.size()); for (char ch : in) { if (ch == '\\') { ch = '/'; } out.push_back(static_cast( std::tolower(static_cast(ch)))); } // strip any leading separator std::size_t start = 0; while (start < out.size() && out[start] == '/') { ++start; } return out.substr(start); } static void collect_files_recursive(Folder::Ptr folder, std::vector& out) { for (unsigned int i = 0; i < folder->getNumFiles(); ++i) { out.push_back(folder->getFile(i)); } for (unsigned int i = 0; i < folder->getNumSubFolders(); ++i) { collect_files_recursive(folder->getSubFolder(i), out); } } File::Ptr Archive::findFile(const std::string& path) const { if (!m_RootFolder || path.empty()) { return nullptr; } const std::string want = normalize_bsa_path(path); // BSAs store flat folder names like "meshes\\actors\\character". On Linux // std::filesystem::path doesn't treat '\\' as a separator, so bsatk builds // a flat folder tree there. Rather than trying to reverse-engineer the // structure, iterate every File in the archive and compare its full path // (normalized) against the caller-supplied one. std::vector allFiles; collect_files_recursive(m_RootFolder, allFiles); for (const File::Ptr& entry : allFiles) { if (normalize_bsa_path(entry->getFilePath()) == want) { return entry; } } return nullptr; } void Archive::readFiles(std::queue& queue, boost::mutex& mutex, boost::interprocess::interprocess_semaphore& bufferCount, boost::interprocess::interprocess_semaphore& queueFree, std::vector::iterator begin, std::vector::iterator end) { for (; begin != end && !boost::this_thread::interruption_requested(); ++begin) { queueFree.wait(); FileInfo fileInfo; fileInfo.file = *begin; size_t size = static_cast(fileInfo.file->m_FileSize); m_File.seekg(fileInfo.file->m_DataOffset); if (m_Type != TYPE_FALLOUT4 && m_Type != TYPE_STARFIELD && m_Type != TYPE_STARFIELD_LZ4_TEXTURE && m_Type != TYPE_FALLOUT4NG_7 && m_Type != TYPE_FALLOUT4NG_8) { if (namePrefixed()) { std::string fullName = readBString(m_File); if (size <= fullName.length()) { continue; } size -= fullName.length() + 1; } if (m_Type == TYPE_SKYRIMSE && compressed(fileInfo.file)) { fileInfo.file->m_UncompressedFileSize = readType(m_File); size -= 4; } if (!fileInfo.file->m_TextureChunks.size()) { fileInfo.data = std::make_pair(std::shared_ptr(new unsigned char[size]), static_cast(size)); m_File.read(reinterpret_cast(fileInfo.data.first.get()), size); } } else { if (!fileInfo.file->m_TextureChunks.size()) { if (size == 0) size = fileInfo.file->m_UncompressedFileSize; fileInfo.data = std::make_pair(std::shared_ptr(new unsigned char[size]), static_cast(size)); m_File.read(reinterpret_cast(fileInfo.data.first.get()), size); } else { fileInfo.file->m_UncompressedFileSize = 0L; BSAULong totalSize = 0U; for (BSAUInt i = 0; i < fileInfo.file->m_TextureChunks.size(); ++i) { totalSize += fileInfo.file->m_TextureChunks[i].unpackedSize; } char* chunkData = new char[totalSize]; BSAULong currentPos = 0U; for (BSAUInt i = 0; i < fileInfo.file->m_TextureChunks.size(); ++i) { BSAULong length = fileInfo.file->m_TextureChunks[i].unpackedSize; if (fileInfo.file->m_TextureChunks[i].packedSize > 0) { char* chunk = new char[fileInfo.file->m_TextureChunks[i].packedSize]; m_File.read(chunk, fileInfo.file->m_TextureChunks[i].packedSize); if (m_Type == TYPE_FALLOUT4 || m_Type == TYPE_STARFIELD || m_Type == TYPE_FALLOUT4NG_7 || m_Type == TYPE_FALLOUT4NG_8) { EErrorCode result = ERROR_NONE; try { std::shared_ptr unpackedChunk = decompress( reinterpret_cast(chunk), fileInfo.file->m_TextureChunks[i].packedSize, result, length); memcpy(chunkData + currentPos, reinterpret_cast(unpackedChunk.get()), length); unpackedChunk.reset(); } catch (const std::exception&) { continue; } } else { char* unpackedChunk = new char[length]; LZ4_decompress_safe(chunk, unpackedChunk, fileInfo.file->m_TextureChunks[i].packedSize, length); memcpy(chunkData + currentPos, unpackedChunk, length); delete[] unpackedChunk; } delete[] chunk; fileInfo.file->m_UncompressedFileSize += length; } else { char* chunk = new char[length]; m_File.read(chunk, length); memcpy(chunkData + currentPos, chunk, length); delete[] chunk; } currentPos += length; } fileInfo.file->m_FileSize = 0; fileInfo.data = std::make_pair(std::shared_ptr( reinterpret_cast(chunkData)), static_cast(totalSize)); } } { boost::interprocess::scoped_lock lock(mutex); queue.push(fileInfo); } bufferCount.post(); } } inline bool fileExists(const std::string& name) { struct stat buffer; return stat(name.c_str(), &buffer) != -1; } void Archive::extractFiles(const std::string& targetDirectory, std::queue& queue, boost::mutex& mutex, boost::interprocess::interprocess_semaphore& bufferCount, boost::interprocess::interprocess_semaphore& queueFree, int totalFiles, bool overwrite, int& filesDone) { for (int i = 0; i < totalFiles; ++i) { bufferCount.wait(); if (boost::this_thread::interruption_requested()) { break; } FileInfo fileInfo; { boost::interprocess::scoped_lock lock(mutex); fileInfo = queue.front(); ++filesDone; queue.pop(); } queueFree.post(); DataBuffer dataBuffer = fileInfo.data; std::string fileName = makeString("%s\\%s", targetDirectory.c_str(), fileInfo.file->getFilePath().c_str()); if (!overwrite && fileExists(fileName)) { continue; } std::ofstream outputFile(fileName.c_str(), fstream::out | fstream::binary | fstream::trunc); if (!outputFile.is_open()) { continue; // return ERROR_ACCESSFAILED; } if (m_Type != TYPE_FALLOUT4 && m_Type != TYPE_STARFIELD && m_Type != TYPE_STARFIELD_LZ4_TEXTURE && m_Type != TYPE_FALLOUT4NG_7 && m_Type != TYPE_FALLOUT4NG_8) { // BSA extraction if (compressed(fileInfo.file)) { // Decompress data if (m_Type != TYPE_SKYRIMSE) { // Oblivion - Skyrim LE use gzip compression EErrorCode result = ERROR_NONE; try { BSAULong length = 0UL; std::shared_ptr buffer = decompress(dataBuffer.first.get(), dataBuffer.second, result, length); if (buffer.get() != nullptr) { outputFile.write(reinterpret_cast(buffer.get()), length); buffer.reset(); } } catch (const std::exception&) { dataBuffer.first.reset(); fileInfo.data.first.reset(); continue; } } else { // Skyrim SE uses LZ4 Frame compression if (!fileInfo.file->m_TextureChunks.size()) { char* outBuffer = new char[fileInfo.file->m_UncompressedFileSize]; LZ4F_decompressionContext_t dcContext = nullptr; LZ4F_decompressOptions_t options = {}; LZ4F_createDecompressionContext(&dcContext, LZ4F_VERSION); size_t outSize = fileInfo.file->m_UncompressedFileSize; size_t inSize = dataBuffer.second; LZ4F_decompress(dcContext, outBuffer, &outSize, dataBuffer.first.get(), &inSize, &options); outputFile.write(outBuffer, fileInfo.file->m_UncompressedFileSize); LZ4F_freeDecompressionContext(dcContext); delete[] outBuffer; } } } else { // No compression - just write the data. outputFile.write(reinterpret_cast(dataBuffer.first.get()), dataBuffer.second); } } else { // BA2 format if (fileInfo.file->m_TextureChunks.size()) { // Texture stream format - requires building the header data for the DDS file bool isDX10 = false; DirectX::DDS_HEADER_DXT10 DX10HeaderData = {}; DirectX::DDS_HEADER DDSHeaderData = getDDSHeader(fileInfo.file, DX10HeaderData, isDX10); outputFile.write("DDS ", 4); char* DDSHeader = new char[sizeof(DDSHeaderData)]; memcpy(DDSHeader, &DDSHeaderData, sizeof(DDSHeaderData)); outputFile.write(DDSHeader, sizeof(DDSHeaderData)); delete[] DDSHeader; if (isDX10) { // This format requires DX10 header info getDX10Header(DX10HeaderData, fileInfo.file, DDSHeaderData); char* DX10Header = new char[sizeof(DX10HeaderData)]; memcpy(DX10Header, &DX10HeaderData, sizeof(DX10HeaderData)); outputFile.write(DX10Header, sizeof(DX10HeaderData)); delete[] DX10Header; } } EErrorCode result = ERROR_NONE; try { if (fileInfo.file->m_FileSize > 0 && !fileInfo.file->m_TextureChunks.size()) { BSAULong length = fileInfo.file->m_UncompressedFileSize; std::shared_ptr buffer = decompress(dataBuffer.first.get(), dataBuffer.second, result, length); if (buffer.get() != nullptr) { outputFile.write(reinterpret_cast(buffer.get()), length); buffer.reset(); } } else { outputFile.write(reinterpret_cast(dataBuffer.first.get()), dataBuffer.second); } } catch (const std::exception&) { dataBuffer.first.reset(); fileInfo.data.first.reset(); continue; } } dataBuffer.first.reset(); fileInfo.data.first.reset(); } } void Archive::createFolders(const std::string& targetDirectory, Folder::Ptr folder) { for (std::vector::iterator iter = folder->m_SubFolders.begin(); iter != folder->m_SubFolders.end(); ++iter) { std::string subDirName = targetDirectory + "/" + (*iter)->getName(); #ifdef _WIN32 ::CreateDirectoryA(subDirName.c_str(), nullptr); #else mkdir(subDirName.c_str(), 0755); #endif createFolders(subDirName, *iter); } } EErrorCode Archive::extractAll( const char* outputDirectory, const std::function& progress, bool overwrite) { createFolders(outputDirectory, m_RootFolder); std::vector fileList; m_RootFolder->collectFiles(fileList); std::sort(fileList.begin(), fileList.end(), ByOffset); m_File.seekg((*(fileList.begin()))->m_DataOffset); std::queue buffers; boost::mutex queueMutex; int filesDone = 0; boost::interprocess::interprocess_semaphore bufferCount(0); boost::interprocess::interprocess_semaphore queueFree(100); boost::thread readerThread(boost::bind(&Archive::readFiles, this, boost::ref(buffers), boost::ref(queueMutex), boost::ref(bufferCount), boost::ref(queueFree), fileList.begin(), fileList.end())); boost::thread extractThread(boost::bind( &Archive::extractFiles, this, outputDirectory, boost::ref(buffers), boost::ref(queueMutex), boost::ref(bufferCount), boost::ref(queueFree), static_cast(fileList.size()), overwrite, boost::ref(filesDone))); bool readerDone = false; bool extractDone = false; bool canceled = false; while (!readerDone || !extractDone) { if (!readerDone) { readerDone = readerThread.timed_join(boost::posix_time::millisec(100)); } if (readerDone) { extractDone = extractThread.timed_join(boost::posix_time::millisec(100)); // don't cancel extractor before reader is done or else reader may be stuck trying // to write to a queue if (canceled) { // ensure the extract thread wakes up. extractThread.interrupt(); bufferCount.post(); } } size_t index = (std::min)(static_cast(filesDone), fileList.size() - 1); if (!progress((filesDone * 100) / static_cast(fileList.size()), fileList[index]->getName()) && !canceled) { readerThread.interrupt(); canceled = true; // don't interrupt repeatedly } } return ERROR_NONE; } bool Archive::compressed(const File::Ptr& file) const { if (m_Type != TYPE_FALLOUT4 && m_Type != TYPE_FALLOUT4NG_7 && m_Type != TYPE_FALLOUT4NG_8) return file->compressToggled() ^ defaultCompressed(); return (file->m_FileSize > 0); } File::Ptr Archive::createFile(const std::string& name, const std::string& sourceName, bool compressed) { return File::Ptr( new File(name, sourceName, nullptr, defaultCompressed() != compressed)); } void Archive::cleanFolder(Folder::Ptr folder) { std::vector folders; folder->collectFolders(folders); for (Folder::Ptr subFolder : folders) { cleanFolder(subFolder); } std::vector files; folder->collectFiles(files); for (File::Ptr file : files) { file.reset(); } folder.reset(); } } // namespace BSA