Skip to content

Commit

Permalink
Updated python script to use new "extract directory" mode.
Browse files Browse the repository at this point in the history
Additionally fixed oversight with audio files and bug in OTRGui.
  • Loading branch information
NEstelami committed Apr 6, 2022
1 parent a593176 commit c1eb71f
Show file tree
Hide file tree
Showing 7 changed files with 167 additions and 109 deletions.
4 changes: 4 additions & 0 deletions OTRExporter/OTRExporter/Main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,10 @@ static void ExporterProgramEnd()
auto fileData = File::ReadAllBytes(item);
otrArchive->AddFile(StringHelper::Split(item, "Extract\\")[1], (uintptr_t)fileData.data(), fileData.size());
}

otrArchive->AddFile("Audiobank", (uintptr_t)Globals::Instance->GetBaseromFile("Audiobank").data(), Globals::Instance->GetBaseromFile("Audiobank").size());
otrArchive->AddFile("Audioseq", (uintptr_t)Globals::Instance->GetBaseromFile("Audioseq").data(), Globals::Instance->GetBaseromFile("Audioseq").size());
otrArchive->AddFile("Audiotable", (uintptr_t)Globals::Instance->GetBaseromFile("Audiotable").data(), Globals::Instance->GetBaseromFile("Audiotable").size());
}
}

Expand Down
99 changes: 7 additions & 92 deletions OTRExporter/extract_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,11 @@
from multiprocessing import Pool, cpu_count, Event, Manager, ProcessError
import shutil

def SignalHandler(sig, frame):
print(f'Signal {sig} received. Aborting...')
mainAbort.set()
# Don't exit immediately to update the extracted assets file.

def BuildOTR():
shutil.copyfile("baserom/Audiobank", "Extract/Audiobank")
shutil.copyfile("baserom/Audioseq", "Extract/Audioseq")
shutil.copyfile("baserom/Audiotable", "Extract/Audiotable")

def BuildOTR(xmlPath):
shutil.copytree("assets", "Extract/assets")

execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"

execStr += " botr -se OTR"
execStr += " ed -i %s -b baserom.z64 -fl CFG\\filelists -o placeholder -osf placeholder -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath)

print(execStr)
exitValue = os.system(execStr)
Expand All @@ -28,98 +18,23 @@ def BuildOTR():
print("Aborting...", file=os.sys.stderr)
print("\n")

def ExtractFile(xmlPath, outputPath, outputSourcePath):
execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"
execStr += " e -eh -i %s -b baserom/ -o %s -osf %s -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath, outputPath, outputSourcePath)

if "overlays" in xmlPath:
execStr += " --static"

print(execStr)
exitValue = os.system(execStr)
#exitValue = 0
if exitValue != 0:
print("\n")
print("Error when extracting from file " + xmlPath, file=os.sys.stderr)
print("Aborting...", file=os.sys.stderr)
print("\n")

def ExtractFunc(fullPath):
*pathList, xmlName = fullPath.split(os.sep)
objectName = os.path.splitext(xmlName)[0]

outPath = os.path.join("..\\soh\\assets\\", *pathList[5:], objectName)
os.makedirs(outPath, exist_ok=True)
outSourcePath = outPath

ExtractFile(fullPath, outPath, outSourcePath)

def initializeWorker(abort, test):
global globalAbort
globalAbort = abort


def main():
parser = argparse.ArgumentParser(description="baserom asset extractor")
parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep")
parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true")
parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true")
parser.add_argument("-v", "--version", help="Sets game version.")
args = parser.parse_args()

global mainAbort
mainAbort = Event()
manager = Manager()
signal.signal(signal.SIGINT, SignalHandler)

extractedAssetsTracker = manager.dict()


# TODO: Read from makerom file to automatically determine game version
xmlVer = "GC_NMQ_D"

if (args.version == "gc_pal_nmpq"):
xmlVer = "GC_NMQ_PAL_F"
elif (args.version == "dbg_mq"):
xmlVer = "GC_MQ_D"

asset_path = args.single
if asset_path is not None:
fullPath = os.path.join("..\\soh\\assets", "xml", asset_path + ".xml")
if not os.path.exists(fullPath):
print(f"Error. File {fullPath} doesn't exists.", file=os.sys.stderr)
exit(1)

ExtractFunc(fullPath)
else:
extract_text_path = "assets/text/message_data.h"
if os.path.isfile(extract_text_path):
extract_text_path = None
extract_staff_text_path = "assets/text/message_data_staff.h"
if os.path.isfile(extract_staff_text_path):
extract_staff_text_path = None

xmlFiles = []
for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets\\xml\\", xmlVer)):
for file in files:
fullPath = os.path.join(currentPath, file)
if file.endswith(".xml"):
xmlFiles.append(fullPath)

try:
numCores = 2
print("Extracting assets with " + str(numCores) + " CPU cores.")
with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, 0)) as p:
p.map(ExtractFunc, xmlFiles)
except Exception as e:
print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr)
print("Disabling mutliprocessing.", file=os.sys.stderr)

initializeWorker(mainAbort, 0)
for singlePath in xmlFiles:
ExtractFunc(singlePath)


BuildOTR()
if (os.path.exists("Extract")):
shutil.rmtree("Extract")

BuildOTR("..\\soh\\assets\\xml\\" + xmlVer + "\\")

if __name__ == "__main__":
main()
125 changes: 125 additions & 0 deletions OTRExporter/extract_assets_old.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
#!/usr/bin/env python3

import argparse, json, os, signal, time, sys, shutil
from multiprocessing import Pool, cpu_count, Event, Manager, ProcessError
import shutil

def SignalHandler(sig, frame):
print(f'Signal {sig} received. Aborting...')
mainAbort.set()
# Don't exit immediately to update the extracted assets file.

def BuildOTR():
shutil.copyfile("baserom/Audiobank", "Extract/Audiobank")
shutil.copyfile("baserom/Audioseq", "Extract/Audioseq")
shutil.copyfile("baserom/Audiotable", "Extract/Audiotable")

shutil.copytree("assets", "Extract/assets")

execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"

execStr += " botr -se OTR"

print(execStr)
exitValue = os.system(execStr)
if exitValue != 0:
print("\n")
print("Error when building the OTR file...", file=os.sys.stderr)
print("Aborting...", file=os.sys.stderr)
print("\n")

def ExtractFile(xmlPath, outputPath, outputSourcePath):
execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"
execStr += " e -eh -i %s -b baserom/ -o %s -osf %s -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath, outputPath, outputSourcePath)

if "overlays" in xmlPath:
execStr += " --static"

print(execStr)
exitValue = os.system(execStr)
#exitValue = 0
if exitValue != 0:
print("\n")
print("Error when extracting from file " + xmlPath, file=os.sys.stderr)
print("Aborting...", file=os.sys.stderr)
print("\n")

def ExtractFunc(fullPath):
*pathList, xmlName = fullPath.split(os.sep)
objectName = os.path.splitext(xmlName)[0]

outPath = os.path.join("..\\soh\\assets\\", *pathList[5:], objectName)
os.makedirs(outPath, exist_ok=True)
outSourcePath = outPath

ExtractFile(fullPath, outPath, outSourcePath)

def initializeWorker(abort, test):
global globalAbort
globalAbort = abort


def main():
parser = argparse.ArgumentParser(description="baserom asset extractor")
parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep")
parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true")
parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true")
parser.add_argument("-v", "--version", help="Sets game version.")
args = parser.parse_args()

global mainAbort
mainAbort = Event()
manager = Manager()
signal.signal(signal.SIGINT, SignalHandler)

extractedAssetsTracker = manager.dict()

xmlVer = "GC_NMQ_D"

if (args.version == "gc_pal_nmpq"):
xmlVer = "GC_NMQ_PAL_F"
elif (args.version == "dbg_mq"):
xmlVer = "GC_MQ_D"

asset_path = args.single
if asset_path is not None:
fullPath = os.path.join("..\\soh\\assets", "xml", asset_path + ".xml")
if not os.path.exists(fullPath):
print(f"Error. File {fullPath} doesn't exists.", file=os.sys.stderr)
exit(1)

ExtractFunc(fullPath)
else:
extract_text_path = "assets/text/message_data.h"
if os.path.isfile(extract_text_path):
extract_text_path = None
extract_staff_text_path = "assets/text/message_data_staff.h"
if os.path.isfile(extract_staff_text_path):
extract_staff_text_path = None

xmlFiles = []
for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets\\xml\\", xmlVer)):
for file in files:
fullPath = os.path.join(currentPath, file)
if file.endswith(".xml"):
xmlFiles.append(fullPath)

try:
numCores = 2
print("Extracting assets with " + str(numCores) + " CPU cores.")
with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, 0)) as p:
p.map(ExtractFunc, xmlFiles)
except Exception as e:
print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr)
print("Disabling mutliprocessing.", file=os.sys.stderr)

initializeWorker(mainAbort, 0)
for singlePath in xmlFiles:
ExtractFunc(singlePath)


BuildOTR()
shutil.rmtree("Extract")

if __name__ == "__main__":
main()
9 changes: 5 additions & 4 deletions OTRGui/src/game/game.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,11 @@ void ExtractRom()
if (MoonUtils::exists("Extract")) MoonUtils::rm("Extract");

MoonUtils::mkdir("Extract");
MoonUtils::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
MoonUtils::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
MoonUtils::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
MoonUtils::copy("tmp/baserom/version", "Extract/version");
//MoonUtils::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
//MoonUtils::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
//MoonUtils::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
//MoonUtils::copy("tmp/baserom/version", "Extract/version");
MoonUtils::write("Extract/version", (char*)&version.crc, sizeof(version.crc));

MoonUtils::copy("assets/game/", "Extract/assets/");

Expand Down
1 change: 0 additions & 1 deletion OTRGui/src/impl/extractor/extractor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ void startWorker(RomVersion version) {

Util::write("tmp/baserom/version", (char*)&version.crc, sizeof(version.crc));


if (oldExtractMode)
{
std::vector<std::string> files;
Expand Down
12 changes: 0 additions & 12 deletions ZAPDTR/ZAPD/Main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -387,18 +387,6 @@ int main(int argc, char* argv[])
{
BuildAssetBlob(Globals::Instance->inputPath, Globals::Instance->outputPath);
}
/*
else if (fileMode == ZFileMode::BuildOverlay)
{
ZOverlay* overlay =
ZOverlay::FromBuild(Path::GetDirectoryName(Globals::Instance->inputPath),
Path::GetDirectoryName(Globals::Instance->cfgPath));
if (overlay != nullptr)
File::WriteAllText(Globals::Instance->outputPath.string(),
overlay->GetSourceOutputCode(""));
}
*/

if (exporterSet != nullptr && exporterSet->endProgramFunc != nullptr)
exporterSet->endProgramFunc();
Expand Down
26 changes: 26 additions & 0 deletions ZAPDTR/ZAPD/ZFile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -823,6 +823,32 @@ void ZFile::GenerateSourceHeaderFiles()

if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllText(headerFilename, formatter.GetOutput());
else if (Globals::Instance->sourceOutputPath != "")
{
std::string xmlPath = xmlFilePath.string();
xmlPath = StringHelper::Replace(xmlPath, "\\", "/");
auto pathList = StringHelper::Split(xmlPath, "/");
std::string outPath = "";

for (int i = 0; i < 3; i++)
outPath += pathList[i] + "/";

for (int i = 5; i < pathList.size(); i++)
{
if (i == pathList.size() - 1)
{
outPath += Path::GetFileNameWithoutExtension(pathList[i]) + "/";
outPath += outName.string() + ".h";
}
else
outPath += pathList[i];

if (i < pathList.size() - 1)
outPath += "/";
}

File::WriteAllText(outPath, formatter.GetOutput());
}
}

std::string ZFile::GetHeaderInclude() const
Expand Down

0 comments on commit c1eb71f

Please sign in to comment.