Skip to content

Commit

Permalink
Fix deflating sqpack data entry generator
Browse files Browse the repository at this point in the history
  • Loading branch information
@Soreepeong
Soreepeong committed Jan 7, 2022
1 parent 8b0371e commit cb83287
Show file tree
Hide file tree
Showing 16 changed files with 744 additions and 293 deletions.
7 changes: 6 additions & 1 deletion ScratchProject/ScratchProject.vcxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,12 +224,17 @@
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Create</PrecompiledHeader>
</ClCompile>
<ClCompile Include="Test.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="Test_TtmpMeta.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="Test_TtmpMeta.cpp" />
<ClCompile Include="Test_Sound.cpp">
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Use</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Use</PrecompiledHeader>
Expand Down
210 changes: 187 additions & 23 deletions ScratchProject/Test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <XivAlexanderCommon/Sqex_Sqpack_Creator.h>
#include <XivAlexanderCommon/Sqex_Texture_ModifiableTextureStream.h>
#include <XivAlexanderCommon/Utils_Win32_ThreadPool.h>
#include <XivAlexanderCommon/XaZlib.h>

std::shared_ptr<Sqex::RandomAccessStream> StripSecondaryMipmaps(std::shared_ptr<Sqex::RandomAccessStream> src) {
return src;
Expand All @@ -19,36 +20,33 @@ std::shared_ptr<Sqex::RandomAccessStream> StripSecondaryMipmaps(std::shared_ptr<
std::shared_ptr<Sqex::RandomAccessStream> StripLodModels(std::shared_ptr<Sqex::RandomAccessStream> src) {
return src;

if (src->ReadStream<Sqex::Model::Header>(0).LodCount == 1)
return src;

auto data = src->ReadStreamIntoVector<uint8_t>(0);
auto& header = *reinterpret_cast<Sqex::Model::Header*>(&data[0]);
header.VertexOffset[1] = header.VertexOffset[2] = header.IndexOffset[1] = header.IndexOffset[2] = header.IndexOffset[0] + header.IndexSize[0];
header.VertexOffset[1] = header.VertexOffset[2] = header.IndexOffset[1] = header.IndexOffset[2] = 0;
header.VertexSize[1] = header.VertexSize[2] = header.IndexSize[1] = header.IndexSize[2] = 0;
header.LodCount = 1;
data.resize(header.VertexOffset[1]);
data.resize(header.IndexOffset[0] + header.IndexSize[0]);
src = std::make_shared<Sqex::MemoryRandomAccessStream>(data);

/*auto raw = std::make_shared<Sqex::Sqpack::EntryRawStream>(std::make_shared<Sqex::Sqpack::OnTheFlyModelEntryProvider>(Sqex::Sqpack::EntryPathSpec{}, src));
auto data2 = raw->ReadStreamIntoVector<uint8_t>(0);
std::cout << memcmp(data.data(), data2.data(), data.size());*/
return src;
}

std::shared_ptr<Sqex::Sqpack::EntryProvider> ToDecompressedEntryProvider(std::shared_ptr<Sqex::Sqpack::EntryProvider> src) {
auto pathSpec = src->PathSpec();
auto rawStream = std::make_shared<Sqex::Sqpack::EntryRawStream>(std::move(src));
switch (rawStream->EntryType()) {
case Sqex::Sqpack::SqData::FileEntryType::Empty:
return std::make_shared<Sqex::Sqpack::EmptyEntryProvider>(std::move(pathSpec));
const auto& pathSpec = src->PathSpec();
switch (src->EntryType()) {
case Sqex::Sqpack::SqData::FileEntryType::EmptyOrObfuscated:
if (src->ReadStream<Sqex::Sqpack::SqData::FileEntryHeader>(0).DecompressedSize == 0)
return std::make_shared<Sqex::Sqpack::EmptyOrObfuscatedEntryProvider>(pathSpec);
else
return src; // obfuscated; not applicable

case Sqex::Sqpack::SqData::FileEntryType::Binary:
return std::make_shared<Sqex::Sqpack::OnTheFlyBinaryEntryProvider>(std::move(pathSpec), std::move(rawStream));
return std::make_shared<Sqex::Sqpack::MemoryBinaryEntryProvider>(pathSpec, std::make_shared<Sqex::Sqpack::EntryRawStream>(std::move(src)), Z_NO_COMPRESSION);

case Sqex::Sqpack::SqData::FileEntryType::Model:
return std::make_shared<Sqex::Sqpack::OnTheFlyModelEntryProvider>(std::move(pathSpec), StripLodModels(std::move(rawStream)));
return std::make_shared<Sqex::Sqpack::MemoryModelEntryProvider>(pathSpec, StripLodModels(std::make_shared<Sqex::Sqpack::EntryRawStream>(std::move(src))), Z_NO_COMPRESSION);

case Sqex::Sqpack::SqData::FileEntryType::Texture:
return std::make_shared<Sqex::Sqpack::OnTheFlyTextureEntryProvider>(std::move(pathSpec), StripSecondaryMipmaps(std::move(rawStream)));
// return std::make_shared<Sqex::Sqpack::MemoryTextureEntryProvider>(std::move(pathSpec), StripSecondaryMipmaps(std::move(rawStream)));
return std::make_shared<Sqex::Sqpack::MemoryTextureEntryProvider>(pathSpec, StripSecondaryMipmaps(std::make_shared<Sqex::Sqpack::EntryRawStream>(std::move(src))), Z_NO_COMPRESSION);
}
throw std::runtime_error("Unknown entry type");
}
Expand Down Expand Up @@ -100,15 +98,110 @@ void UnpackTtmp(const std::filesystem::path srcDir, const std::filesystem::path&
tout << out;
}

std::string DumpSqtexInfo(std::span<uint8_t> buf) {
using namespace Sqex::Sqpack::SqData;
std::stringstream ss;

const auto& entryHeader = *reinterpret_cast<FileEntryHeader*>(&buf[0]);
ss << std::format("HeaderSize={} Type={} Decompressed={} AllocUnit={} OccupiedUnit={} Blocks={}\n",
entryHeader.HeaderSize.Value(), static_cast<int>(entryHeader.Type.Value()), entryHeader.DecompressedSize.Value(),
entryHeader.AllocatedSpaceUnitCount.Value(), entryHeader.OccupiedSpaceUnitCount.Value(), entryHeader.BlockCountOrVersion.Value());
const auto locators = std::span(reinterpret_cast<TextureBlockHeaderLocator*>(&buf[sizeof entryHeader]), entryHeader.BlockCountOrVersion.Value());
const auto subBlocks = std::span(reinterpret_cast<uint16_t*>(&buf[sizeof entryHeader + locators.size_bytes()]), locators.back().FirstSubBlockIndex.Value() + locators.back().SubBlockCount.Value());
const auto& texHeader = *reinterpret_cast<Sqex::Texture::Header*>(&buf[entryHeader.HeaderSize]);
ss << std::format("Unk1={} Header={} Type={} Width={} Height={} Layers={} Mipmaps={}\n",
texHeader.Unknown1.Value(), texHeader.HeaderSize.Value(), static_cast<uint32_t>(texHeader.Type.Value()), texHeader.Width.Value(), texHeader.Height.Value(), texHeader.Layers.Value(), texHeader.MipmapCount.Value());
const auto mipmapOffsets = std::span(reinterpret_cast<uint32_t*>(&buf[entryHeader.HeaderSize + sizeof texHeader]), texHeader.MipmapCount);

ss << "Locators:\n";
size_t lastOffset = 0;
for (size_t i = 0; i < locators.size(); ++i) {
const auto& l = locators[i];
ss << std::format("\tOffset={} Size={} Decompressed={} Subblocks={}:{}\n",
l.FirstBlockOffset.Value(), l.TotalSize.Value(), l.DecompressedSize.Value(), l.FirstSubBlockIndex.Value(), l.FirstSubBlockIndex.Value() + l.SubBlockCount.Value());

uint32_t baseRequestOffset = 0;
if (i < mipmapOffsets.size()) {
lastOffset = mipmapOffsets[i];
ss << std::format("\tRequest={} (mipmapOffset)\n", lastOffset);
} else if (i > 0) {
ss << std::format("\tRequest={} (calc)\n", lastOffset);
} else {
lastOffset = 0;
ss << std::format("\tRequest={} (first)\n", lastOffset);
}
auto pos = entryHeader.HeaderSize + l.FirstBlockOffset;
for (auto j = l.FirstSubBlockIndex.Value(); j < l.FirstSubBlockIndex.Value() + l.SubBlockCount.Value(); ++j) {
const auto& blockHeader = *reinterpret_cast<BlockHeader*>(&buf[pos]);
ss << std::format("\t\tSub: Size={} HeaderSize={} Version={} Compressed={} Decompressed={}\n",
subBlocks[j],
blockHeader.HeaderSize.Value(), blockHeader.Version.Value(), blockHeader.CompressedSize.Value(), blockHeader.DecompressedSize.Value());
pos += subBlocks[ j];
}
lastOffset += l.DecompressedSize;
}

return ss.str();
}

int main() {
//const auto reader = Sqex::Sqpack::Reader(LR"(C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack\ffxiv\020000.win32.index)");

//std::vector<uint8_t> buf, buf2, buf3;
//std::map<Sqex::Sqpack::SqIndex::LEDataLocator, Sqex::Sqpack::SqIndex::LEDataLocator> locatorMap;
//for (size_t i = 0; i < reader.EntryInfo.size(); ++i) {
// // if (i != 22883) continue;
// if (i != 22889) continue;
// //if (i != 164870) continue;
// //if (i != 174182) continue;
// const auto& [locator, entry] = reader.EntryInfo[i];
// if (!locator.Value)
// continue;

// std::cout << std::format("{:0>6}/{:0>6} Locator={:08x} FullPathHash={:08x} PathHash={:08x} NameHash={:08x}\n",
// i, reader.EntryInfo.size(),
// locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash);

// buf.resize(entry.Allocation);
// reader.Data[locator.DatFileIndex].Stream->ReadStream(locator.DatFileOffset(), std::span(buf));
// auto& eh = *reinterpret_cast<Sqex::Sqpack::SqData::FileEntryHeader*>(&buf[0]);
// eh.AllocatedSpaceUnitCount = eh.OccupiedSpaceUnitCount;
// buf.resize(eh.GetTotalEntrySize());

// const auto provider = std::make_shared<Sqex::Sqpack::RandomAccessStreamAsEntryProviderView>(entry.PathSpec, std::make_shared<Sqex::MemoryRandomAccessStream>(std::span(buf)));
// if (provider->EntryType() != Sqex::Sqpack::SqData::FileEntryType::Texture)
// continue;

// const auto rawStream = std::make_shared<Sqex::Sqpack::EntryRawStream>(provider.get());
// buf2.resize(rawStream->StreamSize());
// rawStream->ReadStream(0, std::span(buf2));

// const auto decompressedEntryProvider = std::make_shared<Sqex::Sqpack::MemoryTextureEntryProvider>(entry.PathSpec, rawStream);
// buf3.resize(decompressedEntryProvider->StreamSize());
// decompressedEntryProvider->ReadStreamPartial(0, &buf3[0], buf3.size());

// if (memcmp(&buf[0], &buf3[0], buf3.size()) == 0)
// continue;

// DumpSqtexInfo(std::span(buf));
// DumpSqtexInfo(std::span(buf3));

// std::ofstream(std::format(L"Z:/sqpacktest/{}_{:0>6}_{:08x}_{:08x}_{:08x}_{:08x}.sqd", L"000000", i, locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash), std::ios::binary)
// .write(reinterpret_cast<const char*>(buf.data()), buf.size());
// std::ofstream(std::format(L"Z:/sqpacktest/{}_{:0>6}_{:08x}_{:08x}_{:08x}_{:08x}.tex", L"000000", i, locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash), std::ios::binary)
// .write(reinterpret_cast<const char*>(buf2.data()), buf2.size());
// std::ofstream(std::format(L"Z:/sqpacktest/{}_{:0>6}_{:08x}_{:08x}_{:08x}_{:08x}.rep", L"000000", i, locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash), std::ios::binary)
// .write(reinterpret_cast<const char*>(buf3.data()), buf3.size());
//}

Utils::Win32::TpEnvironment tpenv;
for (const auto& index : std::filesystem::recursive_directory_iterator(std::filesystem::path(LR"(C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack.orig\)"))) {
for (const auto& index : std::filesystem::recursive_directory_iterator(std::filesystem::path(LR"(C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack\)"))) {
if (index.path().extension() != ".index")
continue;

tpenv.SubmitWork([path = index.path()]() {
const auto expac = path.parent_path().filename().string();
const auto dir = std::filesystem::path(LR"(C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack\)") / expac;
const auto dir = std::filesystem::path(LR"(Z:\sqpacktest\game\sqpack)") / expac;
std::filesystem::create_directories(dir);

if (std::filesystem::exists(dir / path.filename()))
Expand All @@ -118,13 +211,84 @@ int main() {
creator.AddEntriesFromSqPack(path, true, true);

const auto reader = Sqex::Sqpack::Reader(path);

std::vector<uint8_t> buf, buf2, buf3;
std::map<Sqex::Sqpack::SqIndex::LEDataLocator, Sqex::Sqpack::SqIndex::LEDataLocator> locatorMap;
for (size_t i = 0; i < reader.EntryInfo.size(); ++i) {
const auto& [srcLoc, entry] = reader.EntryInfo[i];
if (!srcLoc.Value)
const auto& [locator, entry] = reader.EntryInfo[i];
if (!locator.Value)
continue;

if (i % 64 == 0)
std::cout << std::format("{:0>6}/{:0>6}_{}_{:08x}_{:08x}_{:08x}_{:08x}\n", i, reader.EntryInfo.size(), creator.DatName, locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash);

buf.resize(entry.Allocation);
reader.Data[locator.DatFileIndex].Stream->ReadStream(locator.DatFileOffset(), std::span(buf));
auto& eh = *reinterpret_cast<Sqex::Sqpack::SqData::FileEntryHeader*>(&buf[0]);
eh.AllocatedSpaceUnitCount = eh.OccupiedSpaceUnitCount;
buf.resize(eh.GetTotalEntrySize());

auto ep = reader.GetEntryProvider(entry.PathSpec, locator, entry.Allocation);
if (ep->EntryType() == Sqex::Sqpack::SqData::FileEntryType::EmptyOrObfuscated) {
creator.AddEntry(ep);
continue;
auto ep = reader.GetEntryProvider(entry.PathSpec, srcLoc, entry.Allocation);
}
creator.AddEntry(ToDecompressedEntryProvider(ep));

/*const auto provider = std::make_shared<Sqex::Sqpack::RandomAccessStreamAsEntryProviderView>(entry.PathSpec, std::make_shared<Sqex::MemoryRandomAccessStream>(std::span(buf)));
auto rawStream = std::make_shared<Sqex::Sqpack::EntryRawStream>(provider.get());
buf2.resize(rawStream->StreamSize());
rawStream->ReadStream(0, std::span(buf2));
const wchar_t* fileType;
std::shared_ptr<Sqex::Sqpack::EntryProvider> decompressedEntryProvider;
switch (rawStream->EntryType()) {
case Sqex::Sqpack::SqData::FileEntryType::Binary:
fileType = L"bin";
decompressedEntryProvider = std::make_shared<Sqex::Sqpack::MemoryBinaryEntryProvider>(entry.PathSpec, std::move(rawStream));
break;
case Sqex::Sqpack::SqData::FileEntryType::Model:
fileType = L"mdl";
decompressedEntryProvider = std::make_shared<Sqex::Sqpack::MemoryModelEntryProvider>(entry.PathSpec, std::move(rawStream));
break;
case Sqex::Sqpack::SqData::FileEntryType::Texture:
fileType = L"tex";
decompressedEntryProvider = std::make_shared<Sqex::Sqpack::MemoryTextureEntryProvider>(entry.PathSpec, std::move(rawStream));
break;
default:
continue;
}
buf3.resize(decompressedEntryProvider->StreamSize());
decompressedEntryProvider->ReadStreamPartial(0, &buf3[0], buf3.size());
auto ep = reader.GetEntryProvider(entry.PathSpec, locator, entry.Allocation);
if (memcmp(&buf[0], &buf3[0], buf3.size()) == 0)
creator.AddEntry(ToDecompressedEntryProvider(ep));
else {
creator.AddEntry(ep);
if (provider->EntryType() == Sqex::Sqpack::SqData::FileEntryType::Texture) {
auto s = DumpSqtexInfo(buf);
std::ofstream(std::format(L"Z:/sqpacktest/{}_{:0>6}_{:08x}_{:08x}_{:08x}_{:08x}.sqd.txt", creator.DatName, i, locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash, fileType), std::ios::binary)
.write(s.data(), s.size());
}
std::ofstream(std::format(L"Z:/sqpacktest/{}_{:0>6}_{:08x}_{:08x}_{:08x}_{:08x}.sqd", creator.DatName, i, locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash), std::ios::binary)
.write(reinterpret_cast<const char*>(buf.data()), buf.size());
std::ofstream(std::format(L"Z:/sqpacktest/{}_{:0>6}_{:08x}_{:08x}_{:08x}_{:08x}.{}", creator.DatName, i, locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash, fileType), std::ios::binary)
.write(reinterpret_cast<const char*>(buf2.data()), buf2.size());
if (provider->EntryType() == Sqex::Sqpack::SqData::FileEntryType::Texture) {
auto s = DumpSqtexInfo(buf3);
std::ofstream(std::format(L"Z:/sqpacktest/{}_{:0>6}_{:08x}_{:08x}_{:08x}_{:08x}.rep.txt", creator.DatName, i, locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash, fileType), std::ios::binary)
.write(s.data(), s.size());
}
std::ofstream(std::format(L"Z:/sqpacktest/{}_{:0>6}_{:08x}_{:08x}_{:08x}_{:08x}.rep", creator.DatName, i, locator.Value, entry.PathSpec.FullPathHash, entry.PathSpec.PathHash, entry.PathSpec.NameHash), std::ios::binary)
.write(reinterpret_cast<const char*>(buf3.data()), buf3.size());
}*/
}

const auto views = creator.AsViews(false);
Expand Down
3 changes: 3 additions & 0 deletions ScratchProject/pch.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
#define NOMINMAX
#include <Windows.h>

#include <cryptopp/base64.h>
#include <cryptopp/blowfish.h>
#include <cryptopp/modes.h>
#include <cryptopp/sha.h>
#include <zlib.h>
#include <ogg/ogg.h>
Expand Down
4 changes: 2 additions & 2 deletions XivAlexanderCommon/Sqex_Sqpack_Creator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ Sqex::Sqpack::Creator::AddEntryResult Sqex::Sqpack::Creator::AddEntryFromFile(En
auto extensionLower = path.extension().wstring();
CharLowerW(&extensionLower[0]);
if (file_size(path) == 0) {
provider = std::make_shared<EmptyEntryProvider>(std::move(pathSpec));
provider = std::make_shared<EmptyOrObfuscatedEntryProvider>(std::move(pathSpec));
} else if (extensionLower == L".tex") {
provider = std::make_shared<OnTheFlyTextureEntryProvider>(std::move(pathSpec), path);
} else if (extensionLower == L".mdl") {
Expand Down Expand Up @@ -250,7 +250,7 @@ void Sqex::Sqpack::Creator::ReserveSwappableSpace(EntryPathSpec pathSpec, uint32
} else if (const auto it = m_pImpl->m_fullEntries.find(pathSpec); it != m_pImpl->m_fullEntries.end()) {
it->second->EntryReservedSize = std::max(it->second->EntryReservedSize, size);
} else {
auto entry = std::make_unique<Entry>(0, 0, 0, SqIndex::LEDataLocator{ 0, 0 }, size, 0, std::make_shared<EmptyEntryProvider>(std::move(pathSpec)));
auto entry = std::make_unique<Entry>(0, 0, 0, SqIndex::LEDataLocator{ 0, 0 }, size, 0, std::make_shared<EmptyOrObfuscatedEntryProvider>(std::move(pathSpec)));
if (entry->Provider->PathSpec().HasOriginal())
m_pImpl->m_fullEntries.emplace(entry->Provider->PathSpec(), std::move(entry));
else
Expand Down
Loading

0 comments on commit cb83287

Please sign in to comment.