From 42c86ac541f71ccac4bec618bd83a869dd6c2152 Mon Sep 17 00:00:00 2001 From: wheremyfoodat <44909372+wheremyfoodat@users.noreply.github.com> Date: Wed, 12 Jul 2023 21:33:13 +0300 Subject: [PATCH 1/7] Better FS::ReadDirectory --- include/fs/archive_base.hpp | 22 ++++++++- src/core/kernel/directory_operations.cpp | 63 +++++++++++++++++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/include/fs/archive_base.hpp b/include/fs/archive_base.hpp index e1b4caa0..4bd877f0 100644 --- a/include/fs/archive_base.hpp +++ b/include/fs/archive_base.hpp @@ -116,15 +116,35 @@ struct ArchiveSession { ArchiveSession(ArchiveBase* archive, const FSPath& filePath, bool isOpen = true) : archive(archive), path(filePath), isOpen(isOpen) {} }; +struct DirectoryEntry { + std::filesystem::path path; + bool isDirectory; +}; + struct DirectorySession { ArchiveBase* archive = nullptr; // For directories which are mirrored to a specific path on the disk, this contains that path // Otherwise this is a nullopt std::optional pathOnDisk; + + // Iterators for traversing the directory in Directory::Read + std::vector entries; + size_t currentEntry; + bool isOpen; DirectorySession(ArchiveBase* archive, std::filesystem::path path, bool isOpen = true) : archive(archive), pathOnDisk(path), - isOpen(isOpen) {} + isOpen(isOpen) { + currentEntry = 0; // Start from entry 0 + + // Read all directory entries, cache them + for (auto& e : std::filesystem::directory_iterator(path)) { + DirectoryEntry entry; + entry.path = e.path(); + entry.isDirectory = std::filesystem::is_directory(e); + entries.push_back(entry); + } + } }; // Represents a file descriptor obtained from OpenFile. If the optional is nullopt, opening the file failed. diff --git a/src/core/kernel/directory_operations.cpp b/src/core/kernel/directory_operations.cpp index fe4f58f4..a1204724 100644 --- a/src/core/kernel/directory_operations.cpp +++ b/src/core/kernel/directory_operations.cpp @@ -1,3 +1,5 @@ +#include + #include "kernel.hpp" namespace DirectoryOps { @@ -21,7 +23,7 @@ void Kernel::closeDirectory(u32 messagePointer, Handle directory) { const auto p = getObject(directory, KernelObjectType::Directory); if (p == nullptr) [[unlikely]] { - Helpers::panic("Called CloseFile on non-existent file"); + Helpers::panic("Called CloseDirectory on non-existent directory"); } p->getData()->isOpen = false; @@ -34,7 +36,64 @@ void Kernel::readDirectory(u32 messagePointer, Handle directory) { const u32 outPointer = mem.read32(messagePointer + 12); logFileIO("Directory::Read (handle = %X, entry count = %d, out pointer = %08X)\n", directory, entryCount, outPointer); Helpers::panicDev("Unimplemented FsDir::Read"); + + const auto p = getObject(directory, KernelObjectType::Directory); + if (p == nullptr) [[unlikely]] { + Helpers::panic("Called ReadDirectory on non-existent directory"); + } + + DirectorySession* session = p->getData(); + if (!session->pathOnDisk.has_value()) [[unlikely]] { + Helpers::panic("Called ReadDirectory on directory that doesn't have a path on disk"); + } + + std::filesystem::path dirPath = session->pathOnDisk.value(); + + int count = 0; + while (count < entryCount && session->currentEntry < session->entries.size()) { + const auto& entry = session->entries[session->currentEntry]; + std::filesystem::path path = entry.path; + std::filesystem::path extension = path.extension(); + std::filesystem::path relative = path.lexically_relative(dirPath); + bool isDirectory = std::filesystem::is_directory(relative); + std::cout << "Relative path: " << relative << "\nIs directory: " << isDirectory << "\n"; + + std::u16string nameU16 = relative.u16string(); + std::string nameString = relative.string(); + std::string extensionString = extension.string(); + + const u32 entryPointer = outPointer + (count * 0x228); // 0x228 is the size of a single entry + u32 utfPointer = entryPointer; + u32 namePointer = entryPointer + 0x20C; + u32 extensionPointer = entryPointer + 0x216; + u32 attributePointer = entryPointer + 0x21C; + u32 sizePointer = entryPointer + 0x220; + + for (auto c : nameU16) { + mem.write16(utfPointer, u16(c)); + utfPointer += sizeof(u16); + } + mem.write16(utfPointer, 0); // Null terminate the UTF16 name + + for (auto c : nameString) { + mem.write8(namePointer, u8(c)); + namePointer += sizeof(u8); + } + mem.write8(namePointer, 0); // Null terminate 8.3 name + + for (auto c : extensionString) { + mem.write8(extensionPointer, u8(c)); + extensionPointer += sizeof(u8); + } + mem.write8(extensionPointer, 0); // Null terminate 8.3 extension + mem.write8(outPointer + 0x21A, 1); // Always 1 according to 3DBrew + + mem.write8(attributePointer, entry.isDirectory ? 1 : 0); // "Is directory" attribute + + count++; // Increment number of read directories + session->currentEntry++; // Increment index of the entry currently being read + } mem.write32(messagePointer + 4, Result::Success); - mem.write32(messagePointer + 8, 0); + mem.write32(messagePointer + 8, count); } \ No newline at end of file From f64c662ab9053720a426b6aa8ecb013989df3148 Mon Sep 17 00:00:00 2001 From: wheremyfoodat <44909372+wheremyfoodat@users.noreply.github.com> Date: Wed, 12 Jul 2023 21:40:21 +0300 Subject: [PATCH 2/7] Stop lying to people --- include/fs/archive_base.hpp | 27 ++++++++++++------------ src/core/kernel/directory_operations.cpp | 4 ++-- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/include/fs/archive_base.hpp b/include/fs/archive_base.hpp index 4bd877f0..0b0f65a1 100644 --- a/include/fs/archive_base.hpp +++ b/include/fs/archive_base.hpp @@ -122,29 +122,28 @@ struct DirectoryEntry { }; struct DirectorySession { - ArchiveBase* archive = nullptr; - // For directories which are mirrored to a specific path on the disk, this contains that path - // Otherwise this is a nullopt - std::optional pathOnDisk; + ArchiveBase* archive = nullptr; + // For directories which are mirrored to a specific path on the disk, this contains that path + // Otherwise this is a nullopt + std::optional pathOnDisk; - // Iterators for traversing the directory in Directory::Read + // The list of directory entries + the index of the entry we're currently inspecting std::vector entries; size_t currentEntry; - bool isOpen; + bool isOpen; - DirectorySession(ArchiveBase* archive, std::filesystem::path path, bool isOpen = true) : archive(archive), pathOnDisk(path), - isOpen(isOpen) { - currentEntry = 0; // Start from entry 0 - - // Read all directory entries, cache them - for (auto& e : std::filesystem::directory_iterator(path)) { + DirectorySession(ArchiveBase* archive, std::filesystem::path path, bool isOpen = true) : archive(archive), pathOnDisk(path), isOpen(isOpen) { + currentEntry = 0; // Start from entry 0 + + // Read all directory entries, cache them + for (auto& e : std::filesystem::directory_iterator(path)) { DirectoryEntry entry; entry.path = e.path(); entry.isDirectory = std::filesystem::is_directory(e); entries.push_back(entry); - } - } + } + } }; // Represents a file descriptor obtained from OpenFile. If the optional is nullopt, opening the file failed. diff --git a/src/core/kernel/directory_operations.cpp b/src/core/kernel/directory_operations.cpp index fca1e108..462b7e75 100644 --- a/src/core/kernel/directory_operations.cpp +++ b/src/core/kernel/directory_operations.cpp @@ -35,7 +35,6 @@ void Kernel::readDirectory(u32 messagePointer, Handle directory) { const u32 entryCount = mem.read32(messagePointer + 4); const u32 outPointer = mem.read32(messagePointer + 12); logFileIO("Directory::Read (handle = %X, entry count = %d, out pointer = %08X)\n", directory, entryCount, outPointer); - Helpers::panicDev("Unimplemented FsDir::Read"); const auto p = getObject(directory, KernelObjectType::Directory); if (p == nullptr) [[unlikely]] { @@ -56,7 +55,6 @@ void Kernel::readDirectory(u32 messagePointer, Handle directory) { std::filesystem::path extension = path.extension(); std::filesystem::path relative = path.lexically_relative(dirPath); bool isDirectory = std::filesystem::is_directory(relative); - std::cout << "Relative path: " << relative << "\nIs directory: " << isDirectory << "\n"; std::u16string nameU16 = relative.u16string(); std::string nameString = relative.string(); @@ -76,6 +74,8 @@ void Kernel::readDirectory(u32 messagePointer, Handle directory) { mem.write16(utfPointer, 0); // Null terminate the UTF16 name for (auto c : nameString) { + //if (c == '.') continue; // Ignore initial dot + mem.write8(namePointer, u8(c)); namePointer += sizeof(u8); } From eb90151f218025a7e09d20c939d6af0c49beeffe Mon Sep 17 00:00:00 2001 From: wheremyfoodat <44909372+wheremyfoodat@users.noreply.github.com> Date: Thu, 13 Jul 2023 14:22:03 +0300 Subject: [PATCH 3/7] [IR] Implement shmem header Revert "[IR] Implement shmem header" This reverts commit d6c2470591ffd93978e4d55a45b20bda0dd6070c. From 50742f7bb1e35c688a382efdb640676259eef003 Mon Sep 17 00:00:00 2001 From: wheremyfoodat <44909372+wheremyfoodat@users.noreply.github.com> Date: Mon, 17 Jul 2023 02:54:05 +0300 Subject: [PATCH 4/7] [FS] Proper 8.3 filename conversion hopefully --- src/core/kernel/directory_operations.cpp | 97 +++++++++++++++++++++--- 1 file changed, 86 insertions(+), 11 deletions(-) diff --git a/src/core/kernel/directory_operations.cpp b/src/core/kernel/directory_operations.cpp index 462b7e75..81671754 100644 --- a/src/core/kernel/directory_operations.cpp +++ b/src/core/kernel/directory_operations.cpp @@ -1,4 +1,8 @@ +#include +#include #include +#include +#include #include "kernel.hpp" @@ -9,6 +13,79 @@ namespace DirectoryOps { }; } +// Helper to convert std::string to an 8.3 filename to mimic how Directory::Read works +using ShortFilename = std::array; +using ShortExtension = std::array; +using Filename83 = std::pair; + +// The input string should be the stem and extension together, not separately +// Eg something like "boop.png", "panda.txt", etc +Filename83 convertTo83(const std::string& path) { + ShortFilename filename; + ShortExtension extension; + + // Convert a character to add it to the 8.3 name + // "Characters such as + are changed to the underscore _, and letters are put in uppercase" + // For now we put letters in uppercase until we find out what is supposed to be converted to _ and so on + auto convertCharacter = [](char c) { return (char) std::toupper(c); }; + + // List of forbidden character for 8.3 filenames, from Citra + // TODO: Use constexpr when C++20 support is solid + const std::string forbiddenChars = ".\"/\\[]:;=, "; + + // By default space-initialize the whole name, append null terminator in the end for both the filename and extension + filename.fill(' '); + extension.fill(' '); + filename[filename.size() - 1] = '\0'; + extension[extension.size() - 1] = '\0'; + + // Find the position of the dot in the string + auto dotPos = path.rfind('.'); + // Wikipedia: If a file name has no extension, a trailing . has no effect + // Thus check if the last character is a dot and ignore it, prefering the previous dot if it exists + if (dotPos == path.size() - 1) { + dotPos = path.rfind('.', dotPos); // Get previous dot + } + + // If pointPos is not npos we have a valid dot character, and as such an extension + bool haveExtension = dotPos != std::string::npos; + int validCharacterCount = 0; + bool filenameTooBig = false; + + // Parse characters until we're done OR until we reach 9 characters, in which case according to Wikipedia we must truncate to 6 letters + // And append ~1 in the end + for (auto c : path.substr(0, dotPos)) { + // Character is forbidden, we must ignore it + if (forbiddenChars.find(c) != std::string::npos) { + continue; + } + + // We already have capped the amount of characters, thus our filename is too big + if (validCharacterCount == filename.size()) { + filenameTooBig = true; + break; + } + filename[validCharacterCount++] = convertCharacter(c); // Append character to filename + } + + // Truncate name to 6 characters and denote that it is too big + // TODO: Wikipedia says we should also do this if the filename contains an invalid character, including spaces. Must test + if (filenameTooBig) { + filename[6] = '~'; + filename[7] = '1'; + } + + if (haveExtension) { + int extensionLen = 0; + // Copy up to 3 characters from the dot onwards to the extension + for (auto c : path.substr(dotPos + 1, 3)) { + extension[extensionLen++] = convertCharacter(c); + } + } + + return {filename, extension}; +} + void Kernel::handleDirectoryOperation(u32 messagePointer, Handle directory) { const u32 cmd = mem.read32(messagePointer); switch (cmd) { @@ -30,7 +107,6 @@ void Kernel::closeDirectory(u32 messagePointer, Handle directory) { mem.write32(messagePointer + 4, Result::Success); } - void Kernel::readDirectory(u32 messagePointer, Handle directory) { const u32 entryCount = mem.read32(messagePointer + 4); const u32 outPointer = mem.read32(messagePointer + 12); @@ -52,13 +128,12 @@ void Kernel::readDirectory(u32 messagePointer, Handle directory) { while (count < entryCount && session->currentEntry < session->entries.size()) { const auto& entry = session->entries[session->currentEntry]; std::filesystem::path path = entry.path; - std::filesystem::path extension = path.extension(); + std::filesystem::path filename = path.filename(); + std::filesystem::path relative = path.lexically_relative(dirPath); bool isDirectory = std::filesystem::is_directory(relative); std::u16string nameU16 = relative.u16string(); - std::string nameString = relative.string(); - std::string extensionString = extension.string(); const u32 entryPointer = outPointer + (count * 0x228); // 0x228 is the size of a single entry u32 utfPointer = entryPointer; @@ -67,27 +142,27 @@ void Kernel::readDirectory(u32 messagePointer, Handle directory) { u32 attributePointer = entryPointer + 0x21C; u32 sizePointer = entryPointer + 0x220; + std::string filenameString = filename.string(); + auto [shortFilename, shortExtension] = convertTo83(filenameString); + for (auto c : nameU16) { mem.write16(utfPointer, u16(c)); utfPointer += sizeof(u16); } mem.write16(utfPointer, 0); // Null terminate the UTF16 name - for (auto c : nameString) { - //if (c == '.') continue; // Ignore initial dot - + // Write 8.3 filename-extension + for (auto c : shortFilename) { mem.write8(namePointer, u8(c)); namePointer += sizeof(u8); } - mem.write8(namePointer, 0); // Null terminate 8.3 name - for (auto c : extensionString) { + for (auto c : shortExtension) { mem.write8(extensionPointer, u8(c)); extensionPointer += sizeof(u8); } - mem.write8(extensionPointer, 0); // Null terminate 8.3 extension - mem.write8(outPointer + 0x21A, 1); // Always 1 according to 3DBrew + mem.write8(outPointer + 0x21A, 1); // Always 1 according to 3DBrew mem.write8(attributePointer, entry.isDirectory ? 1 : 0); // "Is directory" attribute count++; // Increment number of read directories From f08fe1086215eef489beda40c870b19caa1abb8a Mon Sep 17 00:00:00 2001 From: wheremyfoodat <44909372+wheremyfoodat@users.noreply.github.com> Date: Mon, 17 Jul 2023 02:58:55 +0300 Subject: [PATCH 5/7] [FS::Dir] Fix command response headers --- src/core/kernel/directory_operations.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/kernel/directory_operations.cpp b/src/core/kernel/directory_operations.cpp index 81671754..6d4ac963 100644 --- a/src/core/kernel/directory_operations.cpp +++ b/src/core/kernel/directory_operations.cpp @@ -4,6 +4,7 @@ #include #include +#include "ipc.hpp" #include "kernel.hpp" namespace DirectoryOps { @@ -104,6 +105,7 @@ void Kernel::closeDirectory(u32 messagePointer, Handle directory) { } p->getData()->isOpen = false; + mem.write32(messagePointer, IPC::responseHeader(0x802, 1, 0)); mem.write32(messagePointer + 4, Result::Success); } @@ -169,6 +171,7 @@ void Kernel::readDirectory(u32 messagePointer, Handle directory) { session->currentEntry++; // Increment index of the entry currently being read } + mem.write32(messagePointer, IPC::responseHeader(0x801, 2, 2)); mem.write32(messagePointer + 4, Result::Success); mem.write32(messagePointer + 8, count); } From 22bfb092b880ddfb67ef268497964eac895ab9b2 Mon Sep 17 00:00:00 2001 From: wheremyfoodat <44909372+wheremyfoodat@users.noreply.github.com> Date: Mon, 17 Jul 2023 03:11:21 +0300 Subject: [PATCH 6/7] Fix 8.3 conversion derp --- src/core/kernel/directory_operations.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/kernel/directory_operations.cpp b/src/core/kernel/directory_operations.cpp index 6d4ac963..bf1241f1 100644 --- a/src/core/kernel/directory_operations.cpp +++ b/src/core/kernel/directory_operations.cpp @@ -62,7 +62,7 @@ Filename83 convertTo83(const std::string& path) { } // We already have capped the amount of characters, thus our filename is too big - if (validCharacterCount == filename.size()) { + if (validCharacterCount == 8) { filenameTooBig = true; break; } From cdf01aaf4304cbd05e73aaf173a1c4b5dc0b6843 Mon Sep 17 00:00:00 2001 From: wheremyfoodat <44909372+wheremyfoodat@users.noreply.github.com> Date: Mon, 17 Jul 2023 03:16:26 +0300 Subject: [PATCH 7/7] [FS::Directory] Better attribute handling --- src/core/kernel/directory_operations.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/kernel/directory_operations.cpp b/src/core/kernel/directory_operations.cpp index bf1241f1..d4cac064 100644 --- a/src/core/kernel/directory_operations.cpp +++ b/src/core/kernel/directory_operations.cpp @@ -136,6 +136,7 @@ void Kernel::readDirectory(u32 messagePointer, Handle directory) { bool isDirectory = std::filesystem::is_directory(relative); std::u16string nameU16 = relative.u16string(); + bool isHidden = nameU16[0] == u'.'; // If the first character is a dot then this is a hidden file/folder const u32 entryPointer = outPointer + (count * 0x228); // 0x228 is the size of a single entry u32 utfPointer = entryPointer; @@ -164,8 +165,11 @@ void Kernel::readDirectory(u32 messagePointer, Handle directory) { extensionPointer += sizeof(u8); } - mem.write8(outPointer + 0x21A, 1); // Always 1 according to 3DBrew - mem.write8(attributePointer, entry.isDirectory ? 1 : 0); // "Is directory" attribute + mem.write8(outPointer + 0x21A, 1); // Always 1 according to 3DBrew + mem.write8(attributePointer, entry.isDirectory ? 1 : 0); // "Is directory" attribute + mem.write8(attributePointer + 1, isHidden ? 1 : 0); // "Is hidden" attribute + mem.write8(attributePointer + 2, entry.isDirectory ? 0 : 1); // "Is archive" attribute + mem.write8(attributePointer + 3, 0); // "Is read-only" attribute count++; // Increment number of read directories session->currentEntry++; // Increment index of the entry currently being read