From f3ce29bf2d415ccd5c783b30867a2548bb545145 Mon Sep 17 00:00:00 2001 From: offtkp Date: Fri, 28 Jul 2023 19:54:18 +0300 Subject: [PATCH 1/7] Add IVFC parser --- CMakeLists.txt | 4 ++- include/fs/ivfc.hpp | 22 ++++++++++++ include/fs/romfs.hpp | 13 ++++++++ src/core/fs/ivfc.cpp | 78 +++++++++++++++++++++++++++++++++++++++++++ src/core/fs/romfs.cpp | 21 ++++++++++++ 5 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 include/fs/ivfc.hpp create mode 100644 include/fs/romfs.hpp create mode 100644 src/core/fs/ivfc.cpp create mode 100644 src/core/fs/romfs.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 618b08d3..c5725ff0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,7 +126,8 @@ set(PICA_SOURCE_FILES src/core/PICA/gpu.cpp src/core/PICA/regs.cpp src/core/PICA set(LOADER_SOURCE_FILES src/core/loader/elf.cpp src/core/loader/ncsd.cpp src/core/loader/ncch.cpp src/core/loader/lz77.cpp) set(FS_SOURCE_FILES src/core/fs/archive_self_ncch.cpp src/core/fs/archive_save_data.cpp src/core/fs/archive_sdmc.cpp - src/core/fs/archive_ext_save_data.cpp src/core/fs/archive_ncch.cpp + src/core/fs/archive_ext_save_data.cpp src/core/fs/archive_ncch.cpp src/core/fs/romfs.cpp + src/core/fs/ivfc.cpp ) set(RENDERER_SW_SOURCE_FILES src/core/renderer_sw/renderer_sw.cpp) @@ -156,6 +157,7 @@ set(HEADER_FILES include/emulator.hpp include/helpers.hpp include/termcolor.hpp include/crypto/aes_engine.hpp include/metaprogramming.hpp include/PICA/pica_vertex.hpp include/config.hpp include/services/ir_user.hpp include/http_server.hpp include/cheats.hpp include/action_replay.hpp include/renderer_sw/renderer_sw.hpp include/compiler_builtins.hpp + include/fs/romfs.hpp include/fs/ivfc.hpp ) set(THIRD_PARTY_SOURCE_FILES third_party/imgui/imgui.cpp diff --git a/include/fs/ivfc.hpp b/include/fs/ivfc.hpp new file mode 100644 index 00000000..b8d72380 --- /dev/null +++ b/include/fs/ivfc.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include "helpers.hpp" + +namespace IVFC { + + struct IVFCLevel { + u64 logicalOffset; + u64 size; + u64 blockSize; + }; + + struct IVFC { + u64 masterHashSize; + std::vector levels; + }; + + size_t parseIVFC(uintptr_t ivfcStart, IVFC& ivfc); + +} // namespace IVFC \ No newline at end of file diff --git a/include/fs/romfs.hpp b/include/fs/romfs.hpp new file mode 100644 index 00000000..4ce26c62 --- /dev/null +++ b/include/fs/romfs.hpp @@ -0,0 +1,13 @@ +#pragma once +#include "helpers.hpp" +#include + +namespace RomFS { + + struct RomFSNode { + std::vector children; + }; + + RomFSNode parseRomFSTree(uintptr_t romFS, u64 romFSSize); + +} // namespace RomFS \ No newline at end of file diff --git a/src/core/fs/ivfc.cpp b/src/core/fs/ivfc.cpp new file mode 100644 index 00000000..db633550 --- /dev/null +++ b/src/core/fs/ivfc.cpp @@ -0,0 +1,78 @@ +#include "fs/ivfc.hpp" + +namespace IVFC { + + size_t parseIVFC(uintptr_t ivfcStart, IVFC& ivfc) { + uintptr_t ivfcPointer = ivfcStart; + std::string magicIVFC((char*)ivfcPointer, 4); + ivfcPointer += 4; + + if (magicIVFC != "IVFC") { + printf("Invalid IVFC magic: %s\n", magicIVFC.c_str()); + return 0; + } + + uint32_t magicIdentifier = *(u32*)ivfcPointer; + ivfcPointer += 4; + + // RomFS IVFC uses 0x10000, DISA/DIFF IVFC uses 0x20000 here + if (magicIdentifier != 0x10000 && magicIdentifier != 0x20000) { + printf("Invalid IVFC magic identifier: %08X\n", magicIdentifier); + return 0; + } + + if (magicIdentifier == 0x10000) { + ivfc.masterHashSize = *(u32*)ivfcPointer; + ivfcPointer += 4; + // RomFS IVFC uses 3 levels + ivfc.levels.resize(3); + } else { + ivfc.masterHashSize = *(u64*)ivfcPointer; + ivfcPointer += 8; + // DISA/DIFF IVFC uses 4 levels + ivfc.levels.resize(4); + } + + for (size_t i = 0; i < ivfc.levels.size(); i++) { + IVFCLevel level; + + level.logicalOffset = *(u64*)ivfcPointer; + ivfcPointer += 8; + + level.size = *(u64*)ivfcPointer; + ivfcPointer += 8; + + // This field is in log2 + level.blockSize = 1 << *(u32*)ivfcPointer; + ivfcPointer += 4; + + // Skip 4 reserved bytes + ivfcPointer += 4; + + ivfc.levels[i] = level; + } + + u64 ivfcDescriptorSize = *(u64*)ivfcPointer; + ivfcPointer += 8; + + uintptr_t ivfcActualSize = ivfcPointer - ivfcStart; + + // According to 3DBrew, this is usually the case but not guaranteed + if (ivfcActualSize != ivfcDescriptorSize) { + printf("IVFC descriptor size mismatch: %lx != %lx\n", ivfcActualSize, ivfcDescriptorSize); + } + + if (magicIdentifier == 0x10000 && ivfcActualSize != 0x5C) { + // This is always 0x5C bytes long + printf("Invalid IVFC size: %08x\n", (u32)ivfcActualSize); + return 0; + } else if (magicIdentifier == 0x20000 && ivfcActualSize != 0x78) { + // This is always 0x78 bytes long + printf("Invalid IVFC size: %08x\n", (u32)ivfcActualSize); + return 0; + } + + return ivfcActualSize; + } + +} // namespace IVFC \ No newline at end of file diff --git a/src/core/fs/romfs.cpp b/src/core/fs/romfs.cpp new file mode 100644 index 00000000..95c34264 --- /dev/null +++ b/src/core/fs/romfs.cpp @@ -0,0 +1,21 @@ +#include "fs/romfs.hpp" +#include "fs/ivfc.hpp" +#include + +namespace RomFS { + + RomFSNode parseRomFSTree(uintptr_t romFS, u64 romFSSize) { + RomFSNode root; + + IVFC::IVFC ivfc; + size_t ivfcSize = IVFC::parseIVFC((uintptr_t)romFS, ivfc); + + if (ivfcSize == 0) { + printf("Failed to parse IVFC\n"); + return {}; + } + + return root; + } + +} // namespace RomFS \ No newline at end of file From 2de35bd39d95c4fc7a4ce9c810ca29da653e11dd Mon Sep 17 00:00:00 2001 From: offtkp Date: Sat, 29 Jul 2023 04:02:13 +0300 Subject: [PATCH 2/7] Add RomFS directory traversal stuff --- include/fs/romfs.hpp | 13 ++++- src/core/fs/romfs.cpp | 119 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 4 deletions(-) diff --git a/include/fs/romfs.hpp b/include/fs/romfs.hpp index 4ce26c62..b42e873c 100644 --- a/include/fs/romfs.hpp +++ b/include/fs/romfs.hpp @@ -1,13 +1,22 @@ #pragma once #include "helpers.hpp" +#include #include +#include namespace RomFS { struct RomFSNode { - std::vector children; + std::u16string name {}; + // The file/directory offset relative to the start of the RomFS + u64 offset { 0 }; + u64 size { 0 }; + bool isDirectory { false }; + + std::vector> directories {}; + std::vector> files {}; }; - RomFSNode parseRomFSTree(uintptr_t romFS, u64 romFSSize); + std::unique_ptr parseRomFSTree(uintptr_t romFS, u64 romFSSize); } // namespace RomFS \ No newline at end of file diff --git a/src/core/fs/romfs.cpp b/src/core/fs/romfs.cpp index 95c34264..281c8b4a 100644 --- a/src/core/fs/romfs.cpp +++ b/src/core/fs/romfs.cpp @@ -1,12 +1,93 @@ #include "fs/romfs.hpp" #include "fs/ivfc.hpp" +#include "helpers.hpp" #include +#include +#include +#include namespace RomFS { - RomFSNode parseRomFSTree(uintptr_t romFS, u64 romFSSize) { - RomFSNode root; + constexpr u32 metadataInvalidEntry = 0xFFFFFFFF; + struct Level3Header { + uint32_t headerSize; + uint32_t directoryHashTableOffset; + uint32_t directoryHashTableSize; + uint32_t directoryMetadataOffset; + uint32_t directoryMetadataSize; + uint32_t fileHashTableOffset; + uint32_t fileHashTableSize; + uint32_t fileMetadataOffset; + uint32_t fileMetadataSize; + uint32_t fileDataOffset; + }; + + inline uintptr_t align(uintptr_t value, uintptr_t alignment) { + if (value % alignment == 0) + return value; + + return value + (alignment - (value % alignment)); + } + + inline void printNode(const RomFSNode& node, int indentation, std::string path) { + for (int i = 0; i < indentation; i++) { + printf(" "); + } + printf("%s%s\n", path.c_str(), std::string(node.name.begin(), node.name.end()).c_str()); + path += std::string(node.name.begin(), node.name.end()) + "/"; + indentation++; + for (auto& directory : node.directories) { + printNode(*directory, indentation, path); + } + indentation--; + } + + std::vector> parseDirectory(const uintptr_t metadataBase, const uintptr_t metadataOffset) { + std::vector> directories {}; + + // Get offset of first child directory + u32* metadataPtr = (u32*)(metadataBase + metadataOffset); + metadataPtr += 2; + u32 currentDirectoryOffset = *metadataPtr; + + // Loop over all the sibling directories of the first child to get all the children directories + while (currentDirectoryOffset != metadataInvalidEntry) { + metadataPtr = (u32*)(metadataBase + currentDirectoryOffset); + metadataPtr++; // Skip the parent offset + u32 siblingDirectoryOffset = *metadataPtr++; + metadataPtr++; // Skip the child offset + metadataPtr++; // Skip the first file offset + metadataPtr++; // Skip the next directory in hash table offset + u32 nameLength = (*metadataPtr++) / 2; + + // Arbitrary limit + if (nameLength > 128) { + printf("Invalid directory name length: %08X\n", nameLength); + return {}; + } + + char16_t* namePtr = (char16_t*)metadataPtr; + std::u16string name(namePtr, nameLength); + + std::unique_ptr directory = std::make_unique(); + directory->isDirectory = true; + directory->name = name; + directory->offset = currentDirectoryOffset; + directories.push_back(std::move(directory)); + + currentDirectoryOffset = siblingDirectoryOffset; + } + + // Loop over all the children directories to get their children + for (auto& directory : directories) { + directory->directories = parseDirectory(metadataBase, directory->offset); + } + + return directories; + } + + std::unique_ptr parseRomFSTree(uintptr_t romFS, u64 romFSSize) { IVFC::IVFC ivfc; size_t ivfcSize = IVFC::parseIVFC((uintptr_t)romFS, ivfc); @@ -15,6 +96,40 @@ namespace RomFS { return {}; } + uintptr_t masterHashOffset = RomFS::align(ivfcSize, 0x10); + // For a weird reason, the level 3 offset is not the one in the IVFC, instead it's + // the first block after the master hash + // TODO: Find out why and explain in the comment + uintptr_t level3Offset = RomFS::align(masterHashOffset + ivfc.masterHashSize, ivfc.levels[2].blockSize); + uintptr_t const level3Base = (uintptr_t)romFS + level3Offset; + u32* level3Ptr = (u32*)level3Base; + + Level3Header header; + header.headerSize = *level3Ptr++; + header.directoryHashTableOffset = *level3Ptr++; + header.directoryHashTableSize = *level3Ptr++; + header.directoryMetadataOffset = *level3Ptr++; + header.directoryMetadataSize = *level3Ptr++; + header.fileHashTableOffset = *level3Ptr++; + header.fileHashTableSize = *level3Ptr++; + header.fileMetadataOffset = *level3Ptr++; + header.fileMetadataSize = *level3Ptr++; + header.fileDataOffset = *level3Ptr; + + if (header.headerSize != 0x28) { + printf("Invalid level 3 header size: %08X\n", header.headerSize); + return {}; + } + + std::unique_ptr root = std::make_unique(); + root->isDirectory = true; + root->name = u""; + root->offset = 0; + root->directories = parseDirectory(level3Base + header.directoryMetadataOffset, 0); + + // If you want to print the tree, uncomment this + // printNode(*root, 0, ""); + return root; } From fb6ec3aa5fd1f2a65f9ae55ebf505931f133d4c7 Mon Sep 17 00:00:00 2001 From: offtkp Date: Sat, 29 Jul 2023 17:01:44 +0300 Subject: [PATCH 3/7] Remove recursion from RomFS parse, clang-format --- include/fs/ivfc.hpp | 26 ++--- include/fs/romfs.hpp | 29 +++--- src/core/fs/ivfc.cpp | 118 +++++++++++----------- src/core/fs/romfs.cpp | 223 ++++++++++++++++++++++-------------------- 4 files changed, 202 insertions(+), 194 deletions(-) diff --git a/include/fs/ivfc.hpp b/include/fs/ivfc.hpp index b8d72380..7c3b8c0e 100644 --- a/include/fs/ivfc.hpp +++ b/include/fs/ivfc.hpp @@ -1,22 +1,22 @@ #pragma once -#include -#include +#include + #include "helpers.hpp" namespace IVFC { - struct IVFCLevel { - u64 logicalOffset; - u64 size; - u64 blockSize; - }; + struct IVFCLevel { + u64 logicalOffset; + u64 size; + u64 blockSize; + }; - struct IVFC { - u64 masterHashSize; - std::vector levels; - }; + struct IVFC { + u64 masterHashSize; + std::vector levels; + }; - size_t parseIVFC(uintptr_t ivfcStart, IVFC& ivfc); + size_t parseIVFC(uintptr_t ivfcStart, IVFC& ivfc); -} // namespace IVFC \ No newline at end of file +} // namespace IVFC \ No newline at end of file diff --git a/include/fs/romfs.hpp b/include/fs/romfs.hpp index b42e873c..cdc7dcdc 100644 --- a/include/fs/romfs.hpp +++ b/include/fs/romfs.hpp @@ -1,22 +1,23 @@ #pragma once -#include "helpers.hpp" +#include #include #include -#include + +#include "helpers.hpp" namespace RomFS { - struct RomFSNode { - std::u16string name {}; - // The file/directory offset relative to the start of the RomFS - u64 offset { 0 }; - u64 size { 0 }; - bool isDirectory { false }; + struct RomFSNode { + std::u16string name; + // The file/directory offset relative to the start of the RomFS + u64 offset = 0; + u64 size = 0; + bool isDirectory = false; - std::vector> directories {}; - std::vector> files {}; - }; + std::vector> directories; + std::vector> files; + }; - std::unique_ptr parseRomFSTree(uintptr_t romFS, u64 romFSSize); - -} // namespace RomFS \ No newline at end of file + std::unique_ptr parseRomFSTree(uintptr_t romFS, u64 romFSSize); + +} // namespace RomFS \ No newline at end of file diff --git a/src/core/fs/ivfc.cpp b/src/core/fs/ivfc.cpp index db633550..65cc2e61 100644 --- a/src/core/fs/ivfc.cpp +++ b/src/core/fs/ivfc.cpp @@ -2,77 +2,77 @@ namespace IVFC { - size_t parseIVFC(uintptr_t ivfcStart, IVFC& ivfc) { - uintptr_t ivfcPointer = ivfcStart; - std::string magicIVFC((char*)ivfcPointer, 4); - ivfcPointer += 4; + size_t parseIVFC(uintptr_t ivfcStart, IVFC& ivfc) { + uintptr_t ivfcPointer = ivfcStart; - if (magicIVFC != "IVFC") { - printf("Invalid IVFC magic: %s\n", magicIVFC.c_str()); - return 0; - } + char* ivfcCharPtr = (char*)ivfcPointer; + if (ivfcCharPtr[0] != 'I' || ivfcCharPtr[1] != 'V' || ivfcCharPtr[2] != 'F' || ivfcCharPtr[3] != 'C') { + printf("Invalid header on IVFC\n"); + return 0; + } + ivfcPointer += 4; - uint32_t magicIdentifier = *(u32*)ivfcPointer; - ivfcPointer += 4; + u32 magicIdentifier = *(u32*)ivfcPointer; + ivfcPointer += 4; - // RomFS IVFC uses 0x10000, DISA/DIFF IVFC uses 0x20000 here - if (magicIdentifier != 0x10000 && magicIdentifier != 0x20000) { - printf("Invalid IVFC magic identifier: %08X\n", magicIdentifier); - return 0; - } + // RomFS IVFC uses 0x10000, DISA/DIFF IVFC uses 0x20000 here + if (magicIdentifier != 0x10000 && magicIdentifier != 0x20000) { + printf("Invalid IVFC magic identifier: %08X\n", magicIdentifier); + return 0; + } - if (magicIdentifier == 0x10000) { - ivfc.masterHashSize = *(u32*)ivfcPointer; - ivfcPointer += 4; - // RomFS IVFC uses 3 levels - ivfc.levels.resize(3); - } else { - ivfc.masterHashSize = *(u64*)ivfcPointer; - ivfcPointer += 8; - // DISA/DIFF IVFC uses 4 levels - ivfc.levels.resize(4); - } + if (magicIdentifier == 0x10000) { + ivfc.masterHashSize = *(u32*)ivfcPointer; + ivfcPointer += 4; + // RomFS IVFC uses 3 levels + ivfc.levels.resize(3); + } else { + ivfc.masterHashSize = *(u64*)ivfcPointer; + ivfcPointer += 8; + // DISA/DIFF IVFC uses 4 levels + ivfc.levels.resize(4); + } - for (size_t i = 0; i < ivfc.levels.size(); i++) { - IVFCLevel level; - - level.logicalOffset = *(u64*)ivfcPointer; - ivfcPointer += 8; + for (size_t i = 0; i < ivfc.levels.size(); i++) { + IVFCLevel level; - level.size = *(u64*)ivfcPointer; - ivfcPointer += 8; + level.logicalOffset = *(u64*)ivfcPointer; + ivfcPointer += 8; - // This field is in log2 - level.blockSize = 1 << *(u32*)ivfcPointer; - ivfcPointer += 4; + level.size = *(u64*)ivfcPointer; + ivfcPointer += 8; - // Skip 4 reserved bytes - ivfcPointer += 4; + // This field is in log2 + level.blockSize = 1 << *(u32*)ivfcPointer; + ivfcPointer += 4; - ivfc.levels[i] = level; - } + // Skip 4 reserved bytes + ivfcPointer += 4; - u64 ivfcDescriptorSize = *(u64*)ivfcPointer; - ivfcPointer += 8; + ivfc.levels[i] = level; + } - uintptr_t ivfcActualSize = ivfcPointer - ivfcStart; + u64 ivfcDescriptorSize = *(u64*)ivfcPointer; + ivfcPointer += 8; - // According to 3DBrew, this is usually the case but not guaranteed - if (ivfcActualSize != ivfcDescriptorSize) { - printf("IVFC descriptor size mismatch: %lx != %lx\n", ivfcActualSize, ivfcDescriptorSize); - } + uintptr_t ivfcActualSize = ivfcPointer - ivfcStart; - if (magicIdentifier == 0x10000 && ivfcActualSize != 0x5C) { - // This is always 0x5C bytes long - printf("Invalid IVFC size: %08x\n", (u32)ivfcActualSize); - return 0; - } else if (magicIdentifier == 0x20000 && ivfcActualSize != 0x78) { - // This is always 0x78 bytes long - printf("Invalid IVFC size: %08x\n", (u32)ivfcActualSize); - return 0; - } + // According to 3DBrew, this is usually the case but not guaranteed + if (ivfcActualSize != ivfcDescriptorSize) { + printf("IVFC descriptor size mismatch: %lx != %lx\n", ivfcActualSize, ivfcDescriptorSize); + } - return ivfcActualSize; - } + if (magicIdentifier == 0x10000 && ivfcActualSize != 0x5C) { + // This is always 0x5C bytes long + printf("Invalid IVFC size: %08x\n", (u32)ivfcActualSize); + return 0; + } else if (magicIdentifier == 0x20000 && ivfcActualSize != 0x78) { + // This is always 0x78 bytes long + printf("Invalid IVFC size: %08x\n", (u32)ivfcActualSize); + return 0; + } -} // namespace IVFC \ No newline at end of file + return ivfcActualSize; + } + +} // namespace IVFC \ No newline at end of file diff --git a/src/core/fs/romfs.cpp b/src/core/fs/romfs.cpp index 281c8b4a..9bd14682 100644 --- a/src/core/fs/romfs.cpp +++ b/src/core/fs/romfs.cpp @@ -1,136 +1,143 @@ #include "fs/romfs.hpp" + +#include +#include +#include + #include "fs/ivfc.hpp" #include "helpers.hpp" -#include -#include -#include -#include namespace RomFS { - constexpr u32 metadataInvalidEntry = 0xFFFFFFFF; + constexpr u32 metadataInvalidEntry = 0xFFFFFFFF; - struct Level3Header { - uint32_t headerSize; - uint32_t directoryHashTableOffset; - uint32_t directoryHashTableSize; - uint32_t directoryMetadataOffset; - uint32_t directoryMetadataSize; - uint32_t fileHashTableOffset; - uint32_t fileHashTableSize; - uint32_t fileMetadataOffset; - uint32_t fileMetadataSize; - uint32_t fileDataOffset; - }; + struct Level3Header { + u32 headerSize; + u32 directoryHashTableOffset; + u32 directoryHashTableSize; + u32 directoryMetadataOffset; + u32 directoryMetadataSize; + u32 fileHashTableOffset; + u32 fileHashTableSize; + u32 fileMetadataOffset; + u32 fileMetadataSize; + u32 fileDataOffset; + }; - inline uintptr_t align(uintptr_t value, uintptr_t alignment) { - if (value % alignment == 0) - return value; + inline uintptr_t align(uintptr_t value, uintptr_t alignment) { + if (value % alignment == 0) return value; - return value + (alignment - (value % alignment)); - } + return value + (alignment - (value % alignment)); + } - inline void printNode(const RomFSNode& node, int indentation, std::string path) { - for (int i = 0; i < indentation; i++) { - printf(" "); - } - printf("%s%s\n", path.c_str(), std::string(node.name.begin(), node.name.end()).c_str()); - path += std::string(node.name.begin(), node.name.end()) + "/"; - indentation++; - for (auto& directory : node.directories) { - printNode(*directory, indentation, path); - } - indentation--; - } + inline void printNode(const RomFSNode& node, int indentation, std::string path) { + for (int i = 0; i < indentation; i++) { + printf(" "); + } + printf("%s%s\n", path.c_str(), std::string(node.name.begin(), node.name.end()).c_str()); + path += std::string(node.name.begin(), node.name.end()) + "/"; + indentation++; + for (auto& directory : node.directories) { + printNode(*directory, indentation, path); + } + indentation--; + } - std::vector> parseDirectory(const uintptr_t metadataBase, const uintptr_t metadataOffset) { - std::vector> directories {}; + std::unique_ptr parseRootDirectory(uintptr_t metadataBase) { + std::unique_ptr rootDirectory = std::make_unique(); + rootDirectory->isDirectory = true; + rootDirectory->name = u"romfs:"; + rootDirectory->offset = 0; - // Get offset of first child directory - u32* metadataPtr = (u32*)(metadataBase + metadataOffset); - metadataPtr += 2; - u32 currentDirectoryOffset = *metadataPtr; + std::queue directoryOffsets; + directoryOffsets.push(rootDirectory.get()); - // Loop over all the sibling directories of the first child to get all the children directories - while (currentDirectoryOffset != metadataInvalidEntry) { - metadataPtr = (u32*)(metadataBase + currentDirectoryOffset); - metadataPtr++; // Skip the parent offset - u32 siblingDirectoryOffset = *metadataPtr++; - metadataPtr++; // Skip the child offset - metadataPtr++; // Skip the first file offset - metadataPtr++; // Skip the next directory in hash table offset - u32 nameLength = (*metadataPtr++) / 2; + while (!directoryOffsets.empty()) { + RomFSNode* currentNode = directoryOffsets.front(); + directoryOffsets.pop(); - // Arbitrary limit - if (nameLength > 128) { - printf("Invalid directory name length: %08X\n", nameLength); - return {}; - } + u32* metadataPtr = (u32*)(metadataBase + currentNode->offset); + metadataPtr += 2; - char16_t* namePtr = (char16_t*)metadataPtr; - std::u16string name(namePtr, nameLength); + // Offset of first child directory + u32 currentDirectoryOffset = *metadataPtr; - std::unique_ptr directory = std::make_unique(); - directory->isDirectory = true; - directory->name = name; - directory->offset = currentDirectoryOffset; - directories.push_back(std::move(directory)); + // Loop over all the sibling directories of the first child to get all the children directories + // of the current directory + while (currentDirectoryOffset != metadataInvalidEntry) { + metadataPtr = (u32*)(metadataBase + currentDirectoryOffset); + metadataPtr++; // Skip the parent offset + u32 siblingDirectoryOffset = *metadataPtr; + // Skip the rest of the fields + metadataPtr += 4; + u32 nameLength = *metadataPtr++ / 2; - currentDirectoryOffset = siblingDirectoryOffset; - } + // Arbitrary limit + if (nameLength > 128) { + printf("Invalid directory name length: %08X\n", nameLength); + return {}; + } - // Loop over all the children directories to get their children - for (auto& directory : directories) { - directory->directories = parseDirectory(metadataBase, directory->offset); - } + char16_t* namePtr = (char16_t*)metadataPtr; + std::u16string name(namePtr, nameLength); - return directories; - } + std::unique_ptr directory = std::make_unique(); + directory->isDirectory = true; + directory->name = name; + directory->offset = currentDirectoryOffset; + currentNode->directories.push_back(std::move(directory)); - std::unique_ptr parseRomFSTree(uintptr_t romFS, u64 romFSSize) { - IVFC::IVFC ivfc; - size_t ivfcSize = IVFC::parseIVFC((uintptr_t)romFS, ivfc); + currentDirectoryOffset = siblingDirectoryOffset; + } - if (ivfcSize == 0) { - printf("Failed to parse IVFC\n"); - return {}; - } + for (auto& directory : currentNode->directories) { + directoryOffsets.push(directory.get()); + } + } - uintptr_t masterHashOffset = RomFS::align(ivfcSize, 0x10); - // For a weird reason, the level 3 offset is not the one in the IVFC, instead it's - // the first block after the master hash - // TODO: Find out why and explain in the comment - uintptr_t level3Offset = RomFS::align(masterHashOffset + ivfc.masterHashSize, ivfc.levels[2].blockSize); - uintptr_t const level3Base = (uintptr_t)romFS + level3Offset; - u32* level3Ptr = (u32*)level3Base; + return rootDirectory; + } - Level3Header header; - header.headerSize = *level3Ptr++; - header.directoryHashTableOffset = *level3Ptr++; - header.directoryHashTableSize = *level3Ptr++; - header.directoryMetadataOffset = *level3Ptr++; - header.directoryMetadataSize = *level3Ptr++; - header.fileHashTableOffset = *level3Ptr++; - header.fileHashTableSize = *level3Ptr++; - header.fileMetadataOffset = *level3Ptr++; - header.fileMetadataSize = *level3Ptr++; - header.fileDataOffset = *level3Ptr; + std::unique_ptr parseRomFSTree(uintptr_t romFS, u64 romFSSize) { + IVFC::IVFC ivfc; + size_t ivfcSize = IVFC::parseIVFC((uintptr_t)romFS, ivfc); - if (header.headerSize != 0x28) { - printf("Invalid level 3 header size: %08X\n", header.headerSize); - return {}; - } + if (ivfcSize == 0) { + printf("Failed to parse IVFC\n"); + return {}; + } - std::unique_ptr root = std::make_unique(); - root->isDirectory = true; - root->name = u""; - root->offset = 0; - root->directories = parseDirectory(level3Base + header.directoryMetadataOffset, 0); + uintptr_t masterHashOffset = RomFS::align(ivfcSize, 0x10); + // For a weird reason, the level 3 offset is not the one in the IVFC, instead it's + // the first block after the master hash + // TODO: Find out why and explain in the comment + uintptr_t level3Offset = RomFS::align(masterHashOffset + ivfc.masterHashSize, ivfc.levels[2].blockSize); + uintptr_t level3Base = (uintptr_t)romFS + level3Offset; + u32* level3Ptr = (u32*)level3Base; - // If you want to print the tree, uncomment this - // printNode(*root, 0, ""); + Level3Header header; + header.headerSize = *level3Ptr++; + header.directoryHashTableOffset = *level3Ptr++; + header.directoryHashTableSize = *level3Ptr++; + header.directoryMetadataOffset = *level3Ptr++; + header.directoryMetadataSize = *level3Ptr++; + header.fileHashTableOffset = *level3Ptr++; + header.fileHashTableSize = *level3Ptr++; + header.fileMetadataOffset = *level3Ptr++; + header.fileMetadataSize = *level3Ptr++; + header.fileDataOffset = *level3Ptr; - return root; - } + if (header.headerSize != 0x28) { + printf("Invalid level 3 header size: %08X\n", header.headerSize); + return {}; + } -} // namespace RomFS \ No newline at end of file + std::unique_ptr root = parseRootDirectory(level3Base + header.directoryMetadataOffset); + + // If you want to print the tree, uncomment this + // printNode(*root, 0, ""); + + return root; + } + +} // namespace RomFS \ No newline at end of file From 85bbd713b09aa77073207cb6e3bafeb3da4a673a Mon Sep 17 00:00:00 2001 From: offtkp Date: Sat, 29 Jul 2023 18:25:31 +0300 Subject: [PATCH 4/7] Parse RomFS files in RomFS parser --- include/fs/romfs.hpp | 5 +-- src/core/fs/romfs.cpp | 82 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/include/fs/romfs.hpp b/include/fs/romfs.hpp index cdc7dcdc..a87f964d 100644 --- a/include/fs/romfs.hpp +++ b/include/fs/romfs.hpp @@ -10,8 +10,9 @@ namespace RomFS { struct RomFSNode { std::u16string name; // The file/directory offset relative to the start of the RomFS - u64 offset = 0; - u64 size = 0; + u64 metadataOffset = 0; + u64 dataOffset = 0; + u64 dataSize = 0; bool isDirectory = false; std::vector> directories; diff --git a/src/core/fs/romfs.cpp b/src/core/fs/romfs.cpp index 9bd14682..cf302cc1 100644 --- a/src/core/fs/romfs.cpp +++ b/src/core/fs/romfs.cpp @@ -30,24 +30,74 @@ namespace RomFS { return value + (alignment - (value % alignment)); } - inline void printNode(const RomFSNode& node, int indentation, std::string path) { + inline void printNode(const RomFSNode& node, int indentation) { for (int i = 0; i < indentation; i++) { printf(" "); } - printf("%s%s\n", path.c_str(), std::string(node.name.begin(), node.name.end()).c_str()); - path += std::string(node.name.begin(), node.name.end()) + "/"; + printf("%s/\n", std::string(node.name.begin(), node.name.end()).c_str()); + + for (auto& file : node.files) { + for (int i = 0; i <= indentation; i++) { + printf(" "); + } + printf("%s\n", std::string(file->name.begin(), file->name.end()).c_str()); + } + indentation++; for (auto& directory : node.directories) { - printNode(*directory, indentation, path); + printNode(*directory, indentation); } indentation--; } - std::unique_ptr parseRootDirectory(uintptr_t metadataBase) { + std::vector> getFiles(uintptr_t fileMetadataBase, u32 currentFileOffset) { + std::vector> files; + + while (currentFileOffset != metadataInvalidEntry) { + u32* metadataPtr = (u32*)(fileMetadataBase + currentFileOffset); + metadataPtr++; // Skip the containing directory + u32 nextFileOffset = *metadataPtr++; + u64 fileDataOffset = *(u64*)metadataPtr; + metadataPtr += 2; + u64 fileSize = *(u64*)metadataPtr; + metadataPtr += 2; + metadataPtr++; // Skip the offset of the next file in the same hash table bucket + u32 nameLength = *metadataPtr++ / 2; + + // Arbitrary limit + if (nameLength > 128) { + printf("Invalid file name length: %08X\n", nameLength); + return {}; + } + + char16_t* namePtr = (char16_t*)metadataPtr; + std::u16string name(namePtr, nameLength); + + std::unique_ptr file = std::make_unique(); + file->isDirectory = false; + file->name = name; + file->metadataOffset = currentFileOffset; + file->dataOffset = fileDataOffset; + file->dataSize = fileSize; + + files.push_back(std::move(file)); + + currentFileOffset = nextFileOffset; + } + + return files; + } + + std::unique_ptr parseRootDirectory(uintptr_t directoryMetadataBase, uintptr_t fileMetadataBase) { std::unique_ptr rootDirectory = std::make_unique(); rootDirectory->isDirectory = true; rootDirectory->name = u"romfs:"; - rootDirectory->offset = 0; + rootDirectory->metadataOffset = 0; + + u32 rootFilesOffset = *((u32*)(directoryMetadataBase) + 3); + if (rootFilesOffset != metadataInvalidEntry) { + rootDirectory->files = getFiles(fileMetadataBase, rootFilesOffset); + } std::queue directoryOffsets; directoryOffsets.push(rootDirectory.get()); @@ -56,7 +106,7 @@ namespace RomFS { RomFSNode* currentNode = directoryOffsets.front(); directoryOffsets.pop(); - u32* metadataPtr = (u32*)(metadataBase + currentNode->offset); + u32* metadataPtr = (u32*)(directoryMetadataBase + currentNode->metadataOffset); metadataPtr += 2; // Offset of first child directory @@ -65,11 +115,12 @@ namespace RomFS { // Loop over all the sibling directories of the first child to get all the children directories // of the current directory while (currentDirectoryOffset != metadataInvalidEntry) { - metadataPtr = (u32*)(metadataBase + currentDirectoryOffset); + metadataPtr = (u32*)(directoryMetadataBase + currentDirectoryOffset); metadataPtr++; // Skip the parent offset - u32 siblingDirectoryOffset = *metadataPtr; - // Skip the rest of the fields - metadataPtr += 4; + u32 siblingDirectoryOffset = *metadataPtr++; + metadataPtr++; // Skip offset of first child directory + u32 currentFileOffset = *metadataPtr++; + metadataPtr++; // Skip offset of next directory in the same hash table bucket u32 nameLength = *metadataPtr++ / 2; // Arbitrary limit @@ -84,9 +135,10 @@ namespace RomFS { std::unique_ptr directory = std::make_unique(); directory->isDirectory = true; directory->name = name; - directory->offset = currentDirectoryOffset; - currentNode->directories.push_back(std::move(directory)); + directory->metadataOffset = currentDirectoryOffset; + directory->files = getFiles(fileMetadataBase, currentFileOffset); + currentNode->directories.push_back(std::move(directory)); currentDirectoryOffset = siblingDirectoryOffset; } @@ -132,10 +184,10 @@ namespace RomFS { return {}; } - std::unique_ptr root = parseRootDirectory(level3Base + header.directoryMetadataOffset); + std::unique_ptr root = parseRootDirectory(level3Base + header.directoryMetadataOffset, level3Base + header.fileMetadataOffset); // If you want to print the tree, uncomment this - // printNode(*root, 0, ""); + // printNode(*root, 0); return root; } From d610cfa8268f963ae5f10a91d6d2fd17d5e59978 Mon Sep 17 00:00:00 2001 From: offtkp Date: Sat, 29 Jul 2023 18:50:51 +0300 Subject: [PATCH 5/7] Change some function specifiers --- src/core/fs/romfs.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/fs/romfs.cpp b/src/core/fs/romfs.cpp index cf302cc1..1f081e9a 100644 --- a/src/core/fs/romfs.cpp +++ b/src/core/fs/romfs.cpp @@ -24,13 +24,13 @@ namespace RomFS { u32 fileDataOffset; }; - inline uintptr_t align(uintptr_t value, uintptr_t alignment) { + inline constexpr uintptr_t alignUp(uintptr_t value, uintptr_t alignment) { if (value % alignment == 0) return value; return value + (alignment - (value % alignment)); } - inline void printNode(const RomFSNode& node, int indentation) { + void printNode(const RomFSNode& node, int indentation) { for (int i = 0; i < indentation; i++) { printf(" "); } @@ -159,11 +159,11 @@ namespace RomFS { return {}; } - uintptr_t masterHashOffset = RomFS::align(ivfcSize, 0x10); + uintptr_t masterHashOffset = RomFS::alignUp(ivfcSize, 0x10); // For a weird reason, the level 3 offset is not the one in the IVFC, instead it's // the first block after the master hash // TODO: Find out why and explain in the comment - uintptr_t level3Offset = RomFS::align(masterHashOffset + ivfc.masterHashSize, ivfc.levels[2].blockSize); + uintptr_t level3Offset = RomFS::alignUp(masterHashOffset + ivfc.masterHashSize, ivfc.levels[2].blockSize); uintptr_t level3Base = (uintptr_t)romFS + level3Offset; u32* level3Ptr = (u32*)level3Base; From e6098ba7ae3d4bb8775f8df25a3a7f0f07862a4d Mon Sep 17 00:00:00 2001 From: offtkp Date: Sat, 29 Jul 2023 23:04:32 +0300 Subject: [PATCH 6/7] Remove the TODO in romfs.cpp --- src/core/fs/romfs.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/fs/romfs.cpp b/src/core/fs/romfs.cpp index 1f081e9a..620227c4 100644 --- a/src/core/fs/romfs.cpp +++ b/src/core/fs/romfs.cpp @@ -160,9 +160,10 @@ namespace RomFS { } uintptr_t masterHashOffset = RomFS::alignUp(ivfcSize, 0x10); - // For a weird reason, the level 3 offset is not the one in the IVFC, instead it's - // the first block after the master hash - // TODO: Find out why and explain in the comment + // From GBATEK: + // The "Logical Offsets" are completely unrelated to the physical offsets in the RomFS partition. + // Instead, the "Logical Offsets" might be something about where to map the Level 1-3 sections in + // virtual memory (with the physical Level 3,1,2 ordering being re-ordered to Level 1,2,3)? uintptr_t level3Offset = RomFS::alignUp(masterHashOffset + ivfc.masterHashSize, ivfc.levels[2].blockSize); uintptr_t level3Base = (uintptr_t)romFS + level3Offset; u32* level3Ptr = (u32*)level3Base; From 2e12eea8064fc287058dd2b3b7ff7822b1ef5204 Mon Sep 17 00:00:00 2001 From: wheremyfoodat <44909372+wheremyfoodat@users.noreply.github.com> Date: Sun, 30 Jul 2023 01:01:39 +0300 Subject: [PATCH 7/7] Tone down spacing --- include/fs/ivfc.hpp | 2 -- include/fs/romfs.hpp | 2 -- src/core/fs/ivfc.cpp | 2 -- src/core/fs/romfs.cpp | 2 -- 4 files changed, 8 deletions(-) diff --git a/include/fs/ivfc.hpp b/include/fs/ivfc.hpp index 7c3b8c0e..bb5724be 100644 --- a/include/fs/ivfc.hpp +++ b/include/fs/ivfc.hpp @@ -5,7 +5,6 @@ #include "helpers.hpp" namespace IVFC { - struct IVFCLevel { u64 logicalOffset; u64 size; @@ -18,5 +17,4 @@ namespace IVFC { }; size_t parseIVFC(uintptr_t ivfcStart, IVFC& ivfc); - } // namespace IVFC \ No newline at end of file diff --git a/include/fs/romfs.hpp b/include/fs/romfs.hpp index a87f964d..20213761 100644 --- a/include/fs/romfs.hpp +++ b/include/fs/romfs.hpp @@ -6,7 +6,6 @@ #include "helpers.hpp" namespace RomFS { - struct RomFSNode { std::u16string name; // The file/directory offset relative to the start of the RomFS @@ -20,5 +19,4 @@ namespace RomFS { }; std::unique_ptr parseRomFSTree(uintptr_t romFS, u64 romFSSize); - } // namespace RomFS \ No newline at end of file diff --git a/src/core/fs/ivfc.cpp b/src/core/fs/ivfc.cpp index 65cc2e61..78e50f29 100644 --- a/src/core/fs/ivfc.cpp +++ b/src/core/fs/ivfc.cpp @@ -1,7 +1,6 @@ #include "fs/ivfc.hpp" namespace IVFC { - size_t parseIVFC(uintptr_t ivfcStart, IVFC& ivfc) { uintptr_t ivfcPointer = ivfcStart; @@ -74,5 +73,4 @@ namespace IVFC { return ivfcActualSize; } - } // namespace IVFC \ No newline at end of file diff --git a/src/core/fs/romfs.cpp b/src/core/fs/romfs.cpp index 620227c4..1c826b19 100644 --- a/src/core/fs/romfs.cpp +++ b/src/core/fs/romfs.cpp @@ -8,7 +8,6 @@ #include "helpers.hpp" namespace RomFS { - constexpr u32 metadataInvalidEntry = 0xFFFFFFFF; struct Level3Header { @@ -192,5 +191,4 @@ namespace RomFS { return root; } - } // namespace RomFS \ No newline at end of file