Skip to content

Commit

Permalink
[VFS] Check hash of STFS hash tables (& cache them) when reading, fal…
Browse files Browse the repository at this point in the history 
…lback if invalid.

This'll try salvaging any corrupt packages loaded in: normally we find the block numbers belonging to a file by reading them from the hash table.
Seems there's some packages out there (eg. Mass Effect 2 demo) that have corrupt hash tables though, so using the block numbers from there just results in a crash.
By verifying the hash of each hash table we can detect if this is the case, and if so we can try just using current_block_number + 1 instead of use any invalid block number.
(we also let the user know about the corrupt table in the log file)

In LIVE/PIRS packages this should hopefully let us get the correct data, since files are usually stored inside consecutive blocks in those package types.
It's doubtful that it'd help much with CON ones though, since those are pretty much a living filesystem.
The older & more used a CON package is, the more likely blocks will be fragmented throughout the file...
Reading from the hash table is the only way to properly read data from these, using current_block + 1 likely won't help much (we'd be going wxPirs-mode, in a way :P)

CON packages do have something that might help with this though: redundant hash blocks, where each hash table is actually made up of two blocks.
Maybe in future we can find a way to automatically use the secondary block whenever the primary one is invalid.
  • Loading branch information
@emoose @illusion0001
emoose authored and illusion0001 committed Apr 7, 2020
1 parent 1f31d53 commit c89a3eb
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 29 deletions.
103 changes: 76 additions & 27 deletions src/xenia/vfs/devices/stfs_container_device.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <queue>
#include <vector>

#include "third_party/crypto/TinySHA1.hpp"
#include "xenia/base/logging.h"
#include "xenia/base/math.h"
#include "xenia/vfs/devices/stfs_container_entry.h"
Expand Down Expand Up @@ -566,6 +567,10 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
table_block_index = block_hash.level0_next_block();
}

// At this point we've read in all the data we need from the hash tables
// Let's free some mem by clearing the cache we made.
cached_tables_.clear();

if (all_entries.size() > 0) {
return Error::kSuccess;
}
Expand Down Expand Up @@ -640,7 +645,7 @@ size_t StfsContainerDevice::STFSDataBlockToBackingHashBlockOffset(

StfsHashEntry StfsContainerDevice::STFSGetLevelNHashEntry(
const uint8_t* map_ptr, uint32_t block_index, uint32_t level,
bool secondary_block) {
uint8_t* hash_in_out, bool secondary_block) {
uint32_t record = block_index;
for (uint32_t i = 0; i < level; i++) {
record = record / kSTFSDataBlocksPerHashLevel[0];
Expand All @@ -654,10 +659,48 @@ StfsHashEntry StfsContainerDevice::STFSGetLevelNHashEntry(
hash_offset += bytes_per_sector(); // read from this tables secondary block
}

const uint8_t* hash_data = map_ptr + hash_offset;
bool invalid_table = std::find(invalid_tables_.begin(), invalid_tables_.end(),
hash_offset) != invalid_tables_.end();

if (!cached_tables_.count(hash_offset)) {
// Cache the table in memory, since it's likely to be needed again
auto hash_data = (const StfsHashTable*)(map_ptr + hash_offset);
cached_tables_[hash_offset] = *hash_data;

// If hash is provided we'll try comparing it to the hash of this table
if (hash_in_out && !invalid_table) {
sha1::SHA1 sha;
sha.processBytes(hash_data, 0x1000);

uint8_t digest[0x14];
sha.finalize(digest);
if (memcmp(digest, hash_in_out, 0x14)) {
XELOGW(
"STFSGetLevelNHashEntry: level %d hash table at 0x%llX "
"is corrupt (hash mismatch)!",
level, hash_offset);
invalid_table = true;
invalid_tables_.push_back(hash_offset);
}
}
}

if (invalid_table) {
// If table is corrupt there's no use reading invalid data, lets try
// salvaging things by providing next block as block + 1, should work fine
// for LIVE/PIRS packages hopefully.
StfsHashEntry entry = {0};
entry.level0_next_block(block_index + 1);
return entry;
}

StfsHashTable& hash_table = cached_tables_[hash_offset];

auto* entry = (StfsHashEntry*)(hash_data + record * 0x18);
return *entry;
auto& entry = hash_table.entries[record];
if (hash_in_out) {
memcpy(hash_in_out, entry.sha1, 0x14);
}
return entry;
}

StfsHashEntry StfsContainerDevice::STFSGetLevel0HashEntry(
Expand All @@ -668,36 +711,42 @@ StfsHashEntry StfsContainerDevice::STFSGetLevel0HashEntry(
use_secondary_block = true;
}

// Copy our top hash table hash into the buffer...
uint8_t hash[0x14];
memcpy(hash, header_.metadata.stfs_volume_descriptor.root_hash, 0x14);

// Check upper hash table levels to find which table (primary/secondary) to
// use.
// We should be able to skip this if it's a read-only package, since the hash
// tables in those only use one block
if (!header_.metadata.stfs_volume_descriptor.flags.read_only_format) {
auto num_blocks =
header_.metadata.stfs_volume_descriptor.allocated_block_count;

if (num_blocks >= kSTFSDataBlocksPerHashLevel[1]) {
// Get the L2 entry for the block
auto l2_entry =
STFSGetLevelNHashEntry(map_ptr, block_index, 2, use_secondary_block);
use_secondary_block = false;
if (l2_entry.levelN_activeindex()) {
use_secondary_block = true;
}

// We used to always skip this if package is read-only, but it seems there's
// a lot of LIVE/PIRS packages with corrupt hash tables out there.
// Checking the hash table hashes is the only way to detect (and then
// possibly salvage) these.
auto num_blocks =
header_.metadata.stfs_volume_descriptor.allocated_block_count;

if (num_blocks >= kSTFSDataBlocksPerHashLevel[1]) {
// Get the L2 entry for the block
auto l2_entry = STFSGetLevelNHashEntry(map_ptr, block_index, 2, hash,
use_secondary_block);
use_secondary_block = false;
if (l2_entry.levelN_activeindex()) {
use_secondary_block = true;
}
}

if (num_blocks >= kSTFSDataBlocksPerHashLevel[0]) {
// Get the L1 entry for this block
auto l1_entry =
STFSGetLevelNHashEntry(map_ptr, block_index, 1, use_secondary_block);
use_secondary_block = false;
if (l1_entry.levelN_activeindex()) {
use_secondary_block = true;
}
if (num_blocks >= kSTFSDataBlocksPerHashLevel[0]) {
// Get the L1 entry for this block
auto l1_entry = STFSGetLevelNHashEntry(map_ptr, block_index, 1, hash,
use_secondary_block);
use_secondary_block = false;
if (l1_entry.levelN_activeindex()) {
use_secondary_block = true;
}
}

return STFSGetLevelNHashEntry(map_ptr, block_index, 0, use_secondary_block);
return STFSGetLevelNHashEntry(map_ptr, block_index, 0, hash,
use_secondary_block);
}

uint32_t StfsContainerDevice::ReadMagic(const std::wstring& path) {
Expand Down
25 changes: 23 additions & 2 deletions src/xenia/vfs/devices/stfs_container_device.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,11 @@ struct StfsVolumeDescriptor {
uint8_t version;
union {
struct {
uint8_t read_only_format : 1;
uint8_t root_active_index : 1;
uint8_t read_only_format : 1; // if set, only uses a single backing-block
// per hash table (no resiliency),
// otherwise uses two
uint8_t root_active_index : 1; // if set, uses secondary backing-block
// for the highest-level hash table
uint8_t directory_overallocated : 1;
uint8_t directory_index_bounds_valid : 1;
};
Expand Down Expand Up @@ -70,6 +73,11 @@ struct StfsHashEntry {

// If this is a level0 entry, this points to the next block in the chain
uint32_t level0_next_block() { return info3 | (info2 << 8) | (info1 << 16); }
void level0_next_block(uint32_t value) {
info3 = static_cast<uint8_t>(value & 0xFF);
info2 = static_cast<uint8_t>((value >> 8) & 0xFF);
info1 = static_cast<uint8_t>((value >> 16) & 0xFF);
}

// If this is level 1 or 2, this says whether the hash table this entry refers
// to is using the secondary block or not
Expand All @@ -79,6 +87,13 @@ struct StfsHashEntry {
};
static_assert_size(StfsHashEntry, 0x18);

struct StfsHashTable {
StfsHashEntry entries[0xAA];
xe::be<uint32_t> num_blocks; // "committed" blocks
uint8_t padding[12];
};
static_assert_size(StfsHashTable, 0x1000);

/* SVOD */
struct SvodDeviceDescriptor {
uint8_t descriptor_length;
Expand Down Expand Up @@ -489,6 +504,7 @@ class StfsContainerDevice : public Device {

StfsHashEntry STFSGetLevelNHashEntry(const uint8_t* map_ptr,
uint32_t block_index, uint32_t level,
uint8_t* hash_in_out = nullptr,
bool secondary_block = false);

StfsHashEntry STFSGetLevel0HashEntry(const uint8_t* map_ptr,
Expand All @@ -503,6 +519,11 @@ class StfsContainerDevice : public Device {
uint32_t blocks_per_hash_table_ = 1;
uint32_t block_step_[2] = {0xAB, 0x718F};

// Any STFS hash tables that we read from will be cached here, since it's
// likely that they'll be needed again
std::map<size_t, StfsHashTable> cached_tables_;
std::vector<size_t> invalid_tables_;

size_t base_offset_;
size_t magic_offset_;
std::unique_ptr<Entry> root_entry_;
Expand Down

0 comments on commit c89a3eb

Please sign in to comment.