From bd8db3f7f84a050a3837c1ae05832d2da63cfbb4 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 21:53:04 -0400 Subject: [PATCH 01/14] common_paths: Add Load and Dump dirs --- src/common/common_paths.h | 2 ++ src/common/file_util.cpp | 2 ++ src/common/file_util.h | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/common/common_paths.h b/src/common/common_paths.h index df2ce80b1..4f88de768 100644 --- a/src/common/common_paths.h +++ b/src/common/common_paths.h @@ -33,6 +33,8 @@ #define NAND_DIR "nand" #define SYSDATA_DIR "sysdata" #define KEYS_DIR "keys" +#define LOAD_DIR "load" +#define DUMP_DIR "dump" #define LOG_DIR "log" // Filenames diff --git a/src/common/file_util.cpp b/src/common/file_util.cpp index 21a0b9738..548463787 100644 --- a/src/common/file_util.cpp +++ b/src/common/file_util.cpp @@ -705,6 +705,8 @@ const std::string& GetUserPath(UserPath path, const std::string& new_path) { #endif paths.emplace(UserPath::SDMCDir, user_path + SDMC_DIR DIR_SEP); paths.emplace(UserPath::NANDDir, user_path + NAND_DIR DIR_SEP); + paths.emplace(UserPath::LoadDir, user_path + LOAD_DIR DIR_SEP); + paths.emplace(UserPath::DumpDir, user_path + DUMP_DIR DIR_SEP); paths.emplace(UserPath::SysDataDir, user_path + SYSDATA_DIR DIR_SEP); paths.emplace(UserPath::KeysDir, user_path + KEYS_DIR DIR_SEP); // TODO: Put the logs in a better location for each OS diff --git a/src/common/file_util.h b/src/common/file_util.h index 24c1e413c..3d8fe6264 100644 --- a/src/common/file_util.h +++ b/src/common/file_util.h @@ -29,6 +29,8 @@ enum class UserPath { NANDDir, RootDir, SDMCDir, + LoadDir, + DumpDir, SysDataDir, UserDir, }; From d6cbb3a3e0d89508e53accac1c2b823dae5e8cc2 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 21:54:14 -0400 Subject: [PATCH 02/14] vfs: Add GetEntries method Maps name string to directory or file. --- src/core/file_sys/vfs.cpp | 9 +++++++++ src/core/file_sys/vfs.h | 5 +++++ src/core/file_sys/vfs_real.cpp | 17 +++++++++++++++++ src/core/file_sys/vfs_real.h | 1 + 4 files changed, 32 insertions(+) diff --git a/src/core/file_sys/vfs.cpp b/src/core/file_sys/vfs.cpp index d7b52abfd..1ddfb7600 100644 --- a/src/core/file_sys/vfs.cpp +++ b/src/core/file_sys/vfs.cpp @@ -399,6 +399,15 @@ bool VfsDirectory::Copy(std::string_view src, std::string_view dest) { return f2->WriteBytes(f1->ReadAllBytes()) == f1->GetSize(); } +std::map VfsDirectory::GetEntries() const { + std::map out; + for (const auto& dir : GetSubdirectories()) + out.emplace(dir->GetName(), VfsEntryType::Directory); + for (const auto& file : GetFiles()) + out.emplace(file->GetName(), VfsEntryType::File); + return out; +} + std::string VfsDirectory::GetFullPath() const { if (IsRoot()) return GetName(); diff --git a/src/core/file_sys/vfs.h b/src/core/file_sys/vfs.h index 74489b452..828e87f38 100644 --- a/src/core/file_sys/vfs.h +++ b/src/core/file_sys/vfs.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include #include @@ -265,6 +266,10 @@ public: // dest. virtual bool Copy(std::string_view src, std::string_view dest); + // Gets all of the entries directly in the directory (files and dirs), returning a map between + // item name -> type. + virtual std::map GetEntries() const; + // Interprets the file with name file instead as a directory of type directory. // The directory must have a constructor that takes a single argument of type // std::shared_ptr. Allows to reinterpret container files (i.e NCA, zip, XCI, etc) as a diff --git a/src/core/file_sys/vfs_real.cpp b/src/core/file_sys/vfs_real.cpp index 5e242e20f..a58a02de7 100644 --- a/src/core/file_sys/vfs_real.cpp +++ b/src/core/file_sys/vfs_real.cpp @@ -413,6 +413,23 @@ std::string RealVfsDirectory::GetFullPath() const { return out; } +std::map RealVfsDirectory::GetEntries() const { + if (perms == Mode::Append) + return {}; + + std::map out; + FileUtil::ForeachDirectoryEntry( + nullptr, path, + [&out](u64* entries_out, const std::string& directory, const std::string& filename) { + const std::string full_path = directory + DIR_SEP + filename; + out.emplace(filename, FileUtil::IsDirectory(full_path) ? VfsEntryType::Directory + : VfsEntryType::File); + return true; + }); + + return out; +} + bool RealVfsDirectory::ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) { return false; } diff --git a/src/core/file_sys/vfs_real.h b/src/core/file_sys/vfs_real.h index 681c43e82..3af2a6961 100644 --- a/src/core/file_sys/vfs_real.h +++ b/src/core/file_sys/vfs_real.h @@ -98,6 +98,7 @@ public: bool DeleteFile(std::string_view name) override; bool Rename(std::string_view name) override; std::string GetFullPath() const override; + std::map GetEntries() const override; protected: bool ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) override; From f68e324672ba93cf932e64a05cbdad871cb6e235 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 21:55:47 -0400 Subject: [PATCH 03/14] vfs: Add and rewite VfsRawCopy functions --- src/core/file_sys/vfs.cpp | 36 ++++++++++++++++++++++++++++++++---- src/core/file_sys/vfs.h | 6 ++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/core/file_sys/vfs.cpp b/src/core/file_sys/vfs.cpp index 1ddfb7600..218cfde66 100644 --- a/src/core/file_sys/vfs.cpp +++ b/src/core/file_sys/vfs.cpp @@ -463,13 +463,41 @@ bool DeepEquals(const VirtualFile& file1, const VirtualFile& file2, std::size_t return true; } -bool VfsRawCopy(VirtualFile src, VirtualFile dest) { - if (src == nullptr || dest == nullptr) +bool VfsRawCopy(const VirtualFile& src, const VirtualFile& dest, size_t block_size) { + if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) return false; if (!dest->Resize(src->GetSize())) return false; - std::vector data = src->ReadAllBytes(); - return dest->WriteBytes(data, 0) == data.size(); + + std::vector temp(std::min(block_size, src->GetSize())); + for (size_t i = 0; i < src->GetSize(); i += block_size) { + const auto read = std::min(block_size, src->GetSize() - i); + const auto block = src->Read(temp.data(), read, i); + + if (dest->Write(temp.data(), read, i) != read) + return false; + } + + return true; +} + +bool VfsRawCopyD(const VirtualDir& src, const VirtualDir& dest, size_t block_size) { + if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) + return false; + + for (const auto& file : src->GetFiles()) { + const auto out = dest->CreateFile(file->GetName()); + if (!VfsRawCopy(file, out, block_size)) + return false; + } + + for (const auto& dir : src->GetSubdirectories()) { + const auto out = dest->CreateSubdirectory(dir->GetName()); + if (!VfsRawCopyD(dir, out, block_size)) + return false; + } + + return true; } VirtualDir GetOrCreateDirectoryRelative(const VirtualDir& rel, std::string_view path) { diff --git a/src/core/file_sys/vfs.h b/src/core/file_sys/vfs.h index 828e87f38..6aec4c164 100644 --- a/src/core/file_sys/vfs.h +++ b/src/core/file_sys/vfs.h @@ -316,12 +316,14 @@ public: }; // Compare the two files, byte-for-byte, in increments specificed by block_size -bool DeepEquals(const VirtualFile& file1, const VirtualFile& file2, std::size_t block_size = 0x200); +bool DeepEquals(const VirtualFile& file1, const VirtualFile& file2, size_t block_size = 0x1000); // A method that copies the raw data between two different implementations of VirtualFile. If you // are using the same implementation, it is probably better to use the Copy method in the parent // directory of src/dest. -bool VfsRawCopy(VirtualFile src, VirtualFile dest); +bool VfsRawCopy(const VirtualFile& src, const VirtualFile& dest, size_t block_size = 0x1000); + +bool VfsRawCopyD(const VirtualDir& src, const VirtualDir& dest, size_t block_size = 0x1000); // Checks if the directory at path relative to rel exists. If it does, returns that. If it does not // it attempts to create it and returns the new dir or nullptr on failure. From c65d4d119fceac4b08530e372d0a6e4ca24f121a Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 21:57:39 -0400 Subject: [PATCH 04/14] vfs_static: Add StaticVfsFile Always returns the template argument byte for all reads. Doesn't support writes. --- src/core/CMakeLists.txt | 1 + src/core/file_sys/vfs_static.h | 77 ++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 src/core/file_sys/vfs_static.h diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 26f727d96..67d1f9615 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -63,6 +63,7 @@ add_library(core STATIC file_sys/vfs_offset.h file_sys/vfs_real.cpp file_sys/vfs_real.h + file_sys/vfs_static.h file_sys/vfs_vector.cpp file_sys/vfs_vector.h file_sys/xts_archive.cpp diff --git a/src/core/file_sys/vfs_static.h b/src/core/file_sys/vfs_static.h new file mode 100644 index 000000000..5bc8ca52e --- /dev/null +++ b/src/core/file_sys/vfs_static.h @@ -0,0 +1,77 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include + +#include "core/file_sys/vfs.h" + +namespace FileSys { + +template +class StaticVfsFile : public VfsFile { +public: + explicit StaticVfsFile(size_t size = 0, std::string name = "", VirtualDir parent = nullptr) + : size(size), name(name), parent(parent) {} + + std::string GetName() const override { + return name; + } + + size_t GetSize() const override { + return size; + } + + bool Resize(size_t new_size) override { + size = new_size; + return true; + } + + std::shared_ptr GetContainingDirectory() const override { + return parent; + } + + bool IsWritable() const override { + return false; + } + + bool IsReadable() const override { + return true; + } + + size_t Read(u8* data, size_t length, size_t offset) const override { + const auto read = std::min(length, size - offset); + std::fill(data, data + read, value); + return read; + } + + size_t Write(const u8* data, size_t length, size_t offset) override { + return 0; + } + + boost::optional ReadByte(size_t offset) const override { + if (offset < size) + return value; + return boost::none; + } + + std::vector ReadBytes(size_t length, size_t offset) const override { + const auto read = std::min(length, size - offset); + return std::vector(read, value); + } + + bool Rename(std::string_view new_name) override { + name = new_name; + return true; + } + +private: + size_t size; + std::string name; + VirtualDir parent; +}; + +} // namespace FileSys From b52343a428b76ef0fa74c6a939aef7eab33feedb Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 21:58:33 -0400 Subject: [PATCH 05/14] vfs_vector: Add VectorVfsFile Maps a vector into the VFS interface. --- src/core/file_sys/vfs_vector.cpp | 51 ++++++++++++++++++++++++++++++++ src/core/file_sys/vfs_vector.h | 24 +++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/src/core/file_sys/vfs_vector.cpp b/src/core/file_sys/vfs_vector.cpp index ec7f735b5..efca3d4ad 100644 --- a/src/core/file_sys/vfs_vector.cpp +++ b/src/core/file_sys/vfs_vector.cpp @@ -7,6 +7,57 @@ #include "core/file_sys/vfs_vector.h" namespace FileSys { +VectorVfsFile::VectorVfsFile(std::vector initial_data, std::string name, VirtualDir parent) + : data(std::move(initial_data)), name(std::move(name)), parent(std::move(parent)) {} + +std::string VectorVfsFile::GetName() const { + return name; +} + +size_t VectorVfsFile::GetSize() const { + return data.size(); +} + +bool VectorVfsFile::Resize(size_t new_size) { + data.resize(new_size); + return data.size() == new_size; +} + +std::shared_ptr VectorVfsFile::GetContainingDirectory() const { + return parent; +} + +bool VectorVfsFile::IsWritable() const { + return true; +} + +bool VectorVfsFile::IsReadable() const { + return true; +} + +size_t VectorVfsFile::Read(u8* data_, size_t length, size_t offset) const { + const auto read = std::min(length, data.size() - offset); + std::memcpy(data_, data.data() + offset, read); + return read; +} + +size_t VectorVfsFile::Write(const u8* data_, size_t length, size_t offset) { + if (offset + length > data.size()) + data.resize(offset + length); + const auto write = std::min(length, data.size() - offset); + std::memcpy(data.data(), data_, write); + return write; +} + +bool VectorVfsFile::Rename(std::string_view name_) { + name = name_; + return true; +} + +void VectorVfsFile::Assign(std::vector new_data) { + data = new_data; +} + VectorVfsDirectory::VectorVfsDirectory(std::vector files_, std::vector dirs_, std::string name_, VirtualDir parent_) diff --git a/src/core/file_sys/vfs_vector.h b/src/core/file_sys/vfs_vector.h index cba44a7a6..c84e137a9 100644 --- a/src/core/file_sys/vfs_vector.h +++ b/src/core/file_sys/vfs_vector.h @@ -8,6 +8,30 @@ namespace FileSys { +// An implementation of VfsFile that is backed by a vector optionally supplied upon construction +class VectorVfsFile : public VfsFile { +public: + explicit VectorVfsFile(std::vector initial_data = {}, std::string name = "", + VirtualDir parent = nullptr); + + std::string GetName() const override; + size_t GetSize() const override; + bool Resize(size_t new_size) override; + std::shared_ptr GetContainingDirectory() const override; + bool IsWritable() const override; + bool IsReadable() const override; + size_t Read(u8* data, size_t length, size_t offset) const override; + size_t Write(const u8* data, size_t length, size_t offset) override; + bool Rename(std::string_view name) override; + + virtual void Assign(std::vector new_data); + +private: + std::vector data; + VirtualDir parent; + std::string name; +}; + // An implementation of VfsDirectory that maintains two vectors for subdirectories and files. // Vector data is supplied upon construction. class VectorVfsDirectory : public VfsDirectory { From 3e5c3d0f166b432b1e56e6e3a286feb21281bc7a Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 21:59:44 -0400 Subject: [PATCH 06/14] vfs_layered: Add LayeredVfsDirectory Reads multiple dirs through as if a waterfall. --- src/core/file_sys/vfs_layered.cpp | 128 ++++++++++++++++++++++++++++++ src/core/file_sys/vfs_layered.h | 50 ++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 src/core/file_sys/vfs_layered.cpp create mode 100644 src/core/file_sys/vfs_layered.h diff --git a/src/core/file_sys/vfs_layered.cpp b/src/core/file_sys/vfs_layered.cpp new file mode 100644 index 000000000..802f49384 --- /dev/null +++ b/src/core/file_sys/vfs_layered.cpp @@ -0,0 +1,128 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include +#include "core/file_sys/vfs_layered.h" + +namespace FileSys { + +VirtualDir LayerDirectories(std::vector dirs, std::string name) { + if (dirs.empty()) + return nullptr; + if (dirs.size() == 1) + return dirs[0]; + + return std::shared_ptr(new LayeredVfsDirectory(std::move(dirs), std::move(name))); +} + +LayeredVfsDirectory::LayeredVfsDirectory(std::vector dirs, std::string name) + : dirs(std::move(dirs)), name(std::move(name)) {} + +std::shared_ptr LayeredVfsDirectory::GetFileRelative(std::string_view path) const { + VirtualFile file; + for (const auto& layer : dirs) { + file = layer->GetFileRelative(path); + if (file != nullptr) + return file; + } + + return nullptr; +} + +std::shared_ptr LayeredVfsDirectory::GetDirectoryRelative( + std::string_view path) const { + std::vector out; + for (const auto& layer : dirs) { + const auto dir = layer->GetDirectoryRelative(path); + if (dir != nullptr) + out.push_back(dir); + } + + return LayerDirectories(out); +} + +std::shared_ptr LayeredVfsDirectory::GetFile(std::string_view name) const { + return GetFileRelative(name); +} + +std::shared_ptr LayeredVfsDirectory::GetSubdirectory(std::string_view name) const { + return GetDirectoryRelative(name); +} + +std::string LayeredVfsDirectory::GetFullPath() const { + return dirs[0]->GetFullPath(); +} + +std::vector> LayeredVfsDirectory::GetFiles() const { + std::vector out; + for (const auto& layer : dirs) { + for (const auto& file : layer->GetFiles()) { + if (std::find_if(out.begin(), out.end(), [&file](const VirtualFile& comp) { + return comp->GetName() == file->GetName(); + }) == out.end()) + out.push_back(file); + } + } + + return out; +} + +std::vector> LayeredVfsDirectory::GetSubdirectories() const { + std::vector names; + for (const auto& layer : dirs) { + for (const auto& sd : layer->GetSubdirectories()) { + if (std::find(names.begin(), names.end(), sd->GetName()) == names.end()) + names.push_back(sd->GetName()); + } + } + + std::vector out; + for (const auto& subdir : names) + out.push_back(GetSubdirectory(subdir)); + + return out; +} + +bool LayeredVfsDirectory::IsWritable() const { + return false; +} + +bool LayeredVfsDirectory::IsReadable() const { + return true; +} + +std::string LayeredVfsDirectory::GetName() const { + return name.empty() ? dirs[0]->GetName() : name; +} + +std::shared_ptr LayeredVfsDirectory::GetParentDirectory() const { + return dirs[0]->GetParentDirectory(); +} + +std::shared_ptr LayeredVfsDirectory::CreateSubdirectory(std::string_view name) { + return nullptr; +} + +std::shared_ptr LayeredVfsDirectory::CreateFile(std::string_view name) { + return nullptr; +} + +bool LayeredVfsDirectory::DeleteSubdirectory(std::string_view name) { + return false; +} + +bool LayeredVfsDirectory::DeleteFile(std::string_view name) { + return false; +} + +bool LayeredVfsDirectory::Rename(std::string_view name_) { + name = name_; + return true; +} + +bool LayeredVfsDirectory::ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) { + return false; +} +} // namespace FileSys diff --git a/src/core/file_sys/vfs_layered.h b/src/core/file_sys/vfs_layered.h new file mode 100644 index 000000000..f345c2fb6 --- /dev/null +++ b/src/core/file_sys/vfs_layered.h @@ -0,0 +1,50 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include "core/file_sys/vfs.h" + +namespace FileSys { + +// Wrapper function to allow for more efficient handling of dirs.size() == 0, 1 cases. +VirtualDir LayerDirectories(std::vector dirs, std::string name = ""); + +// Class that stacks multiple VfsDirectories on top of each other, attempting to read from the first +// one and falling back to the one after. The highest priority directory (overwrites all others) +// should be element 0 in the dirs vector. +class LayeredVfsDirectory : public VfsDirectory { + friend VirtualDir LayerDirectories(std::vector dirs, std::string name); + + LayeredVfsDirectory(std::vector dirs, std::string name); + +public: + std::shared_ptr GetFileRelative(std::string_view path) const override; + std::shared_ptr GetDirectoryRelative(std::string_view path) const override; + std::shared_ptr GetFile(std::string_view name) const override; + std::shared_ptr GetSubdirectory(std::string_view name) const override; + std::string GetFullPath() const override; + + std::vector> GetFiles() const override; + std::vector> GetSubdirectories() const override; + bool IsWritable() const override; + bool IsReadable() const override; + std::string GetName() const override; + std::shared_ptr GetParentDirectory() const override; + std::shared_ptr CreateSubdirectory(std::string_view name) override; + std::shared_ptr CreateFile(std::string_view name) override; + bool DeleteSubdirectory(std::string_view name) override; + bool DeleteFile(std::string_view name) override; + bool Rename(std::string_view name) override; + +protected: + bool ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) override; + +private: + std::vector dirs; + std::string name; +}; + +} // namespace FileSys From 44fdac334cef098c46b6e26e2f65b09756574e60 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 22:00:44 -0400 Subject: [PATCH 07/14] vfs_concat: Rewrite and fix ConcatenatedVfsFile --- src/core/file_sys/vfs_concat.cpp | 39 +++++++++++++++++++++----------- src/core/file_sys/vfs_concat.h | 34 +++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/src/core/file_sys/vfs_concat.cpp b/src/core/file_sys/vfs_concat.cpp index dc7a279a9..0c07e162e 100644 --- a/src/core/file_sys/vfs_concat.cpp +++ b/src/core/file_sys/vfs_concat.cpp @@ -5,10 +5,22 @@ #include #include +#include "common/assert.h" #include "core/file_sys/vfs_concat.h" namespace FileSys { +bool VerifyConcatenationMap(std::map map) { + for (auto iter = map.begin(); iter != --map.end();) { + const auto old = iter++; + if (old->first + old->second->GetSize() != iter->first) { + return false; + } + } + + return map.begin()->first == 0; +} + VirtualFile ConcatenateFiles(std::vector files, std::string name) { if (files.empty()) return nullptr; @@ -27,7 +39,10 @@ ConcatenatedVfsFile::ConcatenatedVfsFile(std::vector files_, std::s } } -ConcatenatedVfsFile::~ConcatenatedVfsFile() = default; +ConcatenatedVfsFile::ConcatenatedVfsFile(std::map files_, std::string name) + : files(std::move(files_)), name(std::move(name)) { + ASSERT(VerifyConcatenationMap(files)); +} std::string ConcatenatedVfsFile::GetName() const { if (files.empty()) @@ -62,28 +77,25 @@ bool ConcatenatedVfsFile::IsReadable() const { } std::size_t ConcatenatedVfsFile::Read(u8* data, std::size_t length, std::size_t offset) const { - auto entry = files.end(); + std::pair entry = *files.rbegin(); for (auto iter = files.begin(); iter != files.end(); ++iter) { if (iter->first > offset) { - entry = --iter; + entry = *--iter; break; } } - // Check if the entry should be the last one. The loop above will make it end(). - if (entry == files.end() && offset < files.rbegin()->first + files.rbegin()->second->GetSize()) - --entry; - - if (entry == files.end()) + if (entry.first + entry.second->GetSize() <= offset) return 0; - const auto remaining = entry->second->GetSize() + offset - entry->first; - if (length > remaining) { - return entry->second->Read(data, remaining, offset - entry->first) + - Read(data + remaining, length - remaining, offset + remaining); + const auto read_in = + std::min(entry.first + entry.second->GetSize() - offset, entry.second->GetSize()); + if (length > read_in) { + return entry.second->Read(data, read_in, offset - entry.first) + + Read(data + read_in, length - read_in, offset + read_in); } - return entry->second->Read(data, length, offset - entry->first); + return entry.second->Read(data, std::min(read_in, length), offset - entry.first); } std::size_t ConcatenatedVfsFile::Write(const u8* data, std::size_t length, std::size_t offset) { @@ -93,4 +105,5 @@ std::size_t ConcatenatedVfsFile::Write(const u8* data, std::size_t length, std:: bool ConcatenatedVfsFile::Rename(std::string_view name) { return false; } + } // namespace FileSys diff --git a/src/core/file_sys/vfs_concat.h b/src/core/file_sys/vfs_concat.h index 717d04bdc..c65c20d15 100644 --- a/src/core/file_sys/vfs_concat.h +++ b/src/core/file_sys/vfs_concat.h @@ -4,22 +4,54 @@ #pragma once +#include #include #include #include #include "core/file_sys/vfs.h" +#include "core/file_sys/vfs_static.h" namespace FileSys { +class ConcatenatedVfsFile; + // Wrapper function to allow for more efficient handling of files.size() == 0, 1 cases. VirtualFile ConcatenateFiles(std::vector files, std::string name = ""); +// Convenience function that turns a map of offsets to files into a concatenated file, filling gaps +// with template parameter. +template +VirtualFile ConcatenateFiles(std::map files, std::string name = "") { + if (files.empty()) + return nullptr; + if (files.size() == 1) + return files.begin()->second; + + for (auto iter = files.begin(); iter != --files.end();) { + const auto old = iter++; + if (old->first + old->second->GetSize() != iter->first) { + files.emplace(old->first + old->second->GetSize(), + std::make_shared>(iter->first - old->first - + old->second->GetSize())); + } + } + + if (files.begin()->first != 0) + files.emplace(0, std::make_shared>(files.begin()->first)); + + return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); +} + // Class that wraps multiple vfs files and concatenates them, making reads seamless. Currently // read-only. class ConcatenatedVfsFile : public VfsFile { friend VirtualFile ConcatenateFiles(std::vector files, std::string name); + template + friend VirtualFile ConcatenateFiles(std::map files, std::string name); + ConcatenatedVfsFile(std::vector files, std::string name); + ConcatenatedVfsFile(std::map files, std::string name); public: ~ConcatenatedVfsFile() override; @@ -36,7 +68,7 @@ public: private: // Maps starting offset to file -- more efficient. - boost::container::flat_map files; + std::map files; std::string name; }; From 16188acb500cb238281dff28d944008ea56eb81b Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 22:02:44 -0400 Subject: [PATCH 08/14] patch_manager: Add LayeredFS mods support --- src/core/file_sys/patch_manager.cpp | 43 ++++++++++++++++++++++++++++- src/core/file_sys/patch_manager.h | 2 ++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index aebc69d52..74a0acf1a 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -11,6 +11,7 @@ #include "core/file_sys/patch_manager.h" #include "core/file_sys/registered_cache.h" #include "core/file_sys/romfs.h" +#include "core/file_sys/vfs_layered.h" #include "core/hle/service/filesystem/filesystem.h" #include "core/loader/loader.h" @@ -31,8 +32,9 @@ std::string FormatTitleVersion(u32 version, TitleVersionFormat format) { return fmt::format("v{}.{}.{}", bytes[3], bytes[2], bytes[1]); } -constexpr std::array PATCH_TYPE_NAMES{ +constexpr std::array PATCH_TYPE_NAMES{ "Update", + "LayeredFS", }; std::string FormatPatchTypeName(PatchType type) { @@ -89,6 +91,41 @@ VirtualFile PatchManager::PatchRomFS(VirtualFile romfs, u64 ivfc_offset, } } + // LayeredFS + const auto load_dir = Service::FileSystem::GetModificationLoadRoot(title_id); + if (type == ContentRecordType::Program && load_dir != nullptr && load_dir->GetSize() > 0) { + const auto extracted = ExtractRomFS(romfs); + + if (extracted != nullptr) { + auto patch_dirs = load_dir->GetSubdirectories(); + std::sort(patch_dirs.begin(), patch_dirs.end(), + [](const VirtualDir& l, const VirtualDir& r) { + return l->GetName() < r->GetName(); + }); + + std::vector layers; + layers.reserve(patch_dirs.size() + 1); + for (const auto& subdir : patch_dirs) { + const auto romfs_dir = subdir->GetSubdirectory("romfs"); + if (romfs_dir != nullptr) + layers.push_back(romfs_dir); + } + + layers.push_back(extracted); + + const auto layered = LayerDirectories(layers); + + if (layered != nullptr) { + const auto packed = CreateRomFS(layered); + + if (packed != nullptr) { + LOG_INFO(Loader, " RomFS: LayeredFS patches applied successfully"); + romfs = packed; + } + } + } + } + return romfs; } @@ -114,6 +151,10 @@ std::map PatchManager::GetPatchVersionNames() const { } } + const auto lfs_dir = Service::FileSystem::GetModificationLoadRoot(title_id); + if (lfs_dir != nullptr && lfs_dir->GetSize() > 0) + out[PatchType::LayeredFS] = ""; + return out; } diff --git a/src/core/file_sys/patch_manager.h b/src/core/file_sys/patch_manager.h index 209cab1dc..464f17515 100644 --- a/src/core/file_sys/patch_manager.h +++ b/src/core/file_sys/patch_manager.h @@ -26,6 +26,7 @@ std::string FormatTitleVersion(u32 version, enum class PatchType { Update, + LayeredFS, }; std::string FormatPatchTypeName(PatchType type); @@ -42,6 +43,7 @@ public: // Currently tracked RomFS patches: // - Game Updates + // - LayeredFS VirtualFile PatchRomFS(VirtualFile base, u64 ivfc_offset, ContentRecordType type = ContentRecordType::Program) const; From 50a470eab8a409901250d2d3cca5399e9c243f59 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 22:03:36 -0400 Subject: [PATCH 09/14] bis_factory: Add mod directory VFS getter --- src/core/CMakeLists.txt | 4 ++++ src/core/file_sys/bis_factory.cpp | 12 ++++++++++-- src/core/file_sys/bis_factory.h | 5 ++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 67d1f9615..ead86fd85 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -32,6 +32,8 @@ add_library(core STATIC file_sys/control_metadata.h file_sys/directory.h file_sys/errors.h + file_sys/fsmitm_romfsbuild.cpp + file_sys/fsmitm_romfsbuild.hpp file_sys/mode.h file_sys/nca_metadata.cpp file_sys/nca_metadata.h @@ -59,6 +61,8 @@ add_library(core STATIC file_sys/vfs.h file_sys/vfs_concat.cpp file_sys/vfs_concat.h + file_sys/vfs_layered.cpp + file_sys/vfs_layered.h file_sys/vfs_offset.cpp file_sys/vfs_offset.h file_sys/vfs_real.cpp diff --git a/src/core/file_sys/bis_factory.cpp b/src/core/file_sys/bis_factory.cpp index 205492897..012e08e7d 100644 --- a/src/core/file_sys/bis_factory.cpp +++ b/src/core/file_sys/bis_factory.cpp @@ -4,11 +4,12 @@ #include "core/file_sys/bis_factory.h" #include "core/file_sys/registered_cache.h" +#include "fmt/format.h" namespace FileSys { -BISFactory::BISFactory(VirtualDir nand_root_) - : nand_root(std::move(nand_root_)), +BISFactory::BISFactory(VirtualDir nand_root_, VirtualDir load_root_) + : nand_root(std::move(nand_root_)), load_root(std::move(load_root_)), sysnand_cache(std::make_shared( GetOrCreateDirectoryRelative(nand_root, "/system/Contents/registered"))), usrnand_cache(std::make_shared( @@ -24,4 +25,11 @@ std::shared_ptr BISFactory::GetUserNANDContents() const { return usrnand_cache; } +VirtualDir BISFactory::GetModificationLoadRoot(u64 title_id) const { + // LayeredFS doesn't work on updates and title id-less homebrew + if (title_id == 0 || (title_id & 0x800) > 0) + return nullptr; + return GetOrCreateDirectoryRelative(load_root, fmt::format("/{:016X}", title_id)); +} + } // namespace FileSys diff --git a/src/core/file_sys/bis_factory.h b/src/core/file_sys/bis_factory.h index 9523dd864..0d81967cc 100644 --- a/src/core/file_sys/bis_factory.h +++ b/src/core/file_sys/bis_factory.h @@ -17,14 +17,17 @@ class RegisteredCache; /// registered caches. class BISFactory { public: - explicit BISFactory(VirtualDir nand_root); + BISFactory(VirtualDir nand_root, VirtualDir load_root); ~BISFactory(); std::shared_ptr GetSystemNANDContents() const; std::shared_ptr GetUserNANDContents() const; + VirtualDir GetModificationLoadRoot(u64 title_id) const; + private: VirtualDir nand_root; + VirtualDir load_root; std::shared_ptr sysnand_cache; std::shared_ptr usrnand_cache; From 940a711caf12f15b9b30d0119fa6466c81091479 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 22:04:15 -0400 Subject: [PATCH 10/14] filesystem: Add LayeredFS VFS directory getter --- src/core/hle/service/filesystem/filesystem.cpp | 13 ++++++++++++- src/core/hle/service/filesystem/filesystem.h | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/core/hle/service/filesystem/filesystem.cpp b/src/core/hle/service/filesystem/filesystem.cpp index d349ee686..aed2abb71 100644 --- a/src/core/hle/service/filesystem/filesystem.cpp +++ b/src/core/hle/service/filesystem/filesystem.cpp @@ -343,6 +343,15 @@ std::shared_ptr GetSDMCContents() { return sdmc_factory->GetSDMCContents(); } +FileSys::VirtualDir GetModificationLoadRoot(u64 title_id) { + LOG_TRACE(Service_FS, "Opening mod load root for tid={:016X}", title_id); + + if (bis_factory == nullptr) + return nullptr; + + return bis_factory->GetModificationLoadRoot(title_id); +} + void CreateFactories(const FileSys::VirtualFilesystem& vfs, bool overwrite) { if (overwrite) { bis_factory = nullptr; @@ -354,9 +363,11 @@ void CreateFactories(const FileSys::VirtualFilesystem& vfs, bool overwrite) { FileSys::Mode::ReadWrite); auto sd_directory = vfs->OpenDirectory(FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir), FileSys::Mode::ReadWrite); + auto load_directory = vfs->OpenDirectory(FileUtil::GetUserPath(FileUtil::UserPath::LoadDir), + FileSys::Mode::ReadWrite); if (bis_factory == nullptr) - bis_factory = std::make_unique(nand_directory); + bis_factory = std::make_unique(nand_directory, load_directory); if (save_data_factory == nullptr) save_data_factory = std::make_unique(std::move(nand_directory)); if (sdmc_factory == nullptr) diff --git a/src/core/hle/service/filesystem/filesystem.h b/src/core/hle/service/filesystem/filesystem.h index aab65a2b8..7039a2247 100644 --- a/src/core/hle/service/filesystem/filesystem.h +++ b/src/core/hle/service/filesystem/filesystem.h @@ -52,6 +52,8 @@ std::shared_ptr GetSystemNANDContents(); std::shared_ptr GetUserNANDContents(); std::shared_ptr GetSDMCContents(); +FileSys::VirtualDir GetModificationLoadRoot(u64 title_id); + // Creates the SaveData, SDMC, and BIS Factories. Should be called once and before any function // above is called. void CreateFactories(const FileSys::VirtualFilesystem& vfs, bool overwrite = true); From 6eada3c57d30087ee3e058ae18ab037b8e438f2c Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 22:04:38 -0400 Subject: [PATCH 11/14] file_sys: Port Atmosphere-NX fs_mitm implementation --- src/core/file_sys/fsmitm_romfsbuild.cpp | 368 ++++++++++++++++++++++++ src/core/file_sys/fsmitm_romfsbuild.hpp | 106 +++++++ 2 files changed, 474 insertions(+) create mode 100644 src/core/file_sys/fsmitm_romfsbuild.cpp create mode 100644 src/core/file_sys/fsmitm_romfsbuild.hpp diff --git a/src/core/file_sys/fsmitm_romfsbuild.cpp b/src/core/file_sys/fsmitm_romfsbuild.cpp new file mode 100644 index 000000000..a6fe6719d --- /dev/null +++ b/src/core/file_sys/fsmitm_romfsbuild.cpp @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2018 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * Adapted by DarkLordZach for use/interaction with yuzu + * + * Modifications Copyright 2018 yuzu emulator team + * Licensed under GPLv2 or any later version + * Refer to the license.txt file included. + */ + +#include +#include "common/assert.h" +#include "fsmitm_romfsbuild.hpp" +#include "vfs.h" +#include "vfs_vector.h" + +namespace FileSys { + +constexpr u64 FS_MAX_PATH = 0x301; + +constexpr u32 ROMFS_ENTRY_EMPTY = 0xFFFFFFFF; +constexpr u32 ROMFS_FILEPARTITION_OFS = 0x200; + +/* Types for building a RomFS. */ +struct RomFSHeader { + u64 header_size; + u64 dir_hash_table_ofs; + u64 dir_hash_table_size; + u64 dir_table_ofs; + u64 dir_table_size; + u64 file_hash_table_ofs; + u64 file_hash_table_size; + u64 file_table_ofs; + u64 file_table_size; + u64 file_partition_ofs; +}; +static_assert(sizeof(RomFSHeader) == 0x50, "RomFSHeader has incorrect size."); + +struct RomFSDirectoryEntry { + u32 parent; + u32 sibling; + u32 child; + u32 file; + u32 hash; + u32 name_size; + char name[]; +}; +static_assert(sizeof(RomFSDirectoryEntry) == 0x18, "RomFSDirectoryEntry has incorrect size."); + +struct RomFSFileEntry { + u32 parent; + u32 sibling; + u64 offset; + u64 size; + u32 hash; + u32 name_size; + char name[]; +}; +static_assert(sizeof(RomFSFileEntry) == 0x20, "RomFSFileEntry has incorrect size."); + +struct RomFSBuildFileContext; + +struct RomFSBuildDirectoryContext { + char* path; + u32 cur_path_ofs; + u32 path_len; + u32 entry_offset = 0; + RomFSBuildDirectoryContext* parent = nullptr; + RomFSBuildDirectoryContext* child = nullptr; + RomFSBuildDirectoryContext* sibling = nullptr; + RomFSBuildFileContext* file = nullptr; +}; + +struct RomFSBuildFileContext { + char* path; + u32 cur_path_ofs; + u32 path_len; + u32 entry_offset = 0; + u64 offset = 0; + u64 size = 0; + RomFSBuildDirectoryContext* parent = nullptr; + RomFSBuildFileContext* sibling = nullptr; + VirtualFile source = nullptr; +}; + +void RomFSBuildContext::VisitDirectory(VirtualDir root_romfs, RomFSBuildDirectoryContext* parent) { + std::vector child_dirs; + + VirtualDir dir; + + if (parent->path_len == 0) + dir = root_romfs; + else + dir = root_romfs->GetDirectoryRelative(parent->path); + + const auto entries = dir->GetEntries(); + + for (const auto& kv : entries) { + if (kv.second == VfsEntryType::Directory) { + RomFSBuildDirectoryContext* child = new RomFSBuildDirectoryContext({0}); + /* Set child's path. */ + child->cur_path_ofs = parent->path_len + 1; + child->path_len = child->cur_path_ofs + kv.first.size(); + child->path = new char[child->path_len + 1]; + strcpy(child->path, parent->path); + ASSERT(child->path_len < FS_MAX_PATH); + strcat(child->path + parent->path_len, "/"); + strcat(child->path + parent->path_len, kv.first.c_str()); + + if (!this->AddDirectory(parent, child, nullptr)) { + delete child->path; + delete child; + } else { + child_dirs.push_back(child); + } + } else { + RomFSBuildFileContext* child = new RomFSBuildFileContext({0}); + /* Set child's path. */ + child->cur_path_ofs = parent->path_len + 1; + child->path_len = child->cur_path_ofs + kv.first.size(); + child->path = new char[child->path_len + 1]; + strcpy(child->path, parent->path); + ASSERT(child->path_len < FS_MAX_PATH); + strcat(child->path + parent->path_len, "/"); + strcat(child->path + parent->path_len, kv.first.c_str()); + + child->source = root_romfs->GetFileRelative(child->path); + + child->size = child->source->GetSize(); + + if (!this->AddFile(parent, child)) { + delete child->path; + delete child; + } + } + } + + for (auto& child : child_dirs) { + this->VisitDirectory(root_romfs, child); + } +} + +bool RomFSBuildContext::AddDirectory(RomFSBuildDirectoryContext* parent_dir_ctx, + RomFSBuildDirectoryContext* dir_ctx, + RomFSBuildDirectoryContext** out_dir_ctx) { + /* Check whether it's already in the known directories. */ + auto existing = this->directories.find(dir_ctx->path); + if (existing != this->directories.end()) { + if (out_dir_ctx) { + *out_dir_ctx = existing->second; + } + return false; + } + + /* Add a new directory. */ + this->num_dirs++; + this->dir_table_size += + sizeof(RomFSDirectoryEntry) + ((dir_ctx->path_len - dir_ctx->cur_path_ofs + 3) & ~3); + dir_ctx->parent = parent_dir_ctx; + this->directories.insert({dir_ctx->path, dir_ctx}); + + if (out_dir_ctx) { + *out_dir_ctx = dir_ctx; + } + return true; +} + +bool RomFSBuildContext::AddFile(RomFSBuildDirectoryContext* parent_dir_ctx, + RomFSBuildFileContext* file_ctx) { + /* Check whether it's already in the known files. */ + auto existing = this->files.find(file_ctx->path); + if (existing != this->files.end()) { + return false; + } + + /* Add a new file. */ + this->num_files++; + this->file_table_size += + sizeof(RomFSFileEntry) + ((file_ctx->path_len - file_ctx->cur_path_ofs + 3) & ~3); + file_ctx->parent = parent_dir_ctx; + this->files.insert({file_ctx->path, file_ctx}); + + return true; +} + +RomFSBuildContext::RomFSBuildContext(VirtualDir base_) : base(std::move(base_)) { + this->root = new RomFSBuildDirectoryContext({0}); + this->root->path = new char[1]; + this->root->path[0] = '\x00'; + this->directories.insert({this->root->path, this->root}); + this->num_dirs = 1; + this->dir_table_size = 0x18; + + VisitDirectory(base, this->root); +} + +std::map RomFSBuildContext::Build() { + std::map out; + RomFSBuildFileContext* cur_file; + RomFSBuildDirectoryContext* cur_dir; + + const auto dir_hash_table_entry_count = romfs_get_hash_table_count(this->num_dirs); + const auto file_hash_table_entry_count = romfs_get_hash_table_count(this->num_files); + this->dir_hash_table_size = 4 * dir_hash_table_entry_count; + this->file_hash_table_size = 4 * file_hash_table_entry_count; + + /* Assign metadata pointers */ + RomFSHeader* header = new RomFSHeader({0}); + auto metadata = new u8[this->dir_hash_table_size + this->dir_table_size + + this->file_hash_table_size + this->file_table_size]; + auto dir_hash_table = reinterpret_cast(metadata); + const auto dir_table = reinterpret_cast( + reinterpret_cast(dir_hash_table) + this->dir_hash_table_size); + auto file_hash_table = + reinterpret_cast(reinterpret_cast(dir_table) + this->dir_table_size); + const auto file_table = reinterpret_cast( + reinterpret_cast(file_hash_table) + this->file_hash_table_size); + + /* Clear out hash tables. */ + for (u32 i = 0; i < dir_hash_table_entry_count; i++) + dir_hash_table[i] = ROMFS_ENTRY_EMPTY; + for (u32 i = 0; i < file_hash_table_entry_count; i++) + file_hash_table[i] = ROMFS_ENTRY_EMPTY; + + out.clear(); + + /* Determine file offsets. */ + u32 entry_offset = 0; + RomFSBuildFileContext* prev_file = nullptr; + for (const auto& it : this->files) { + cur_file = it.second; + this->file_partition_size = (this->file_partition_size + 0xFULL) & ~0xFULL; + cur_file->offset = this->file_partition_size; + this->file_partition_size += cur_file->size; + cur_file->entry_offset = entry_offset; + entry_offset += + sizeof(RomFSFileEntry) + ((cur_file->path_len - cur_file->cur_path_ofs + 3) & ~3); + prev_file = cur_file; + } + /* Assign deferred parent/sibling ownership. */ + for (auto it = this->files.rbegin(); it != this->files.rend(); it++) { + cur_file = it->second; + cur_file->sibling = cur_file->parent->file; + cur_file->parent->file = cur_file; + } + + /* Determine directory offsets. */ + entry_offset = 0; + for (const auto& it : this->directories) { + cur_dir = it.second; + cur_dir->entry_offset = entry_offset; + entry_offset += + sizeof(RomFSDirectoryEntry) + ((cur_dir->path_len - cur_dir->cur_path_ofs + 3) & ~3); + } + /* Assign deferred parent/sibling ownership. */ + for (auto it = this->directories.rbegin(); it->second != this->root; it++) { + cur_dir = it->second; + cur_dir->sibling = cur_dir->parent->child; + cur_dir->parent->child = cur_dir; + } + + /* Populate file tables. */ + for (const auto& it : this->files) { + cur_file = it.second; + RomFSFileEntry* cur_entry = romfs_get_fentry(file_table, cur_file->entry_offset); + + cur_entry->parent = cur_file->parent->entry_offset; + cur_entry->sibling = + (cur_file->sibling == nullptr) ? ROMFS_ENTRY_EMPTY : cur_file->sibling->entry_offset; + cur_entry->offset = cur_file->offset; + cur_entry->size = cur_file->size; + + const auto name_size = cur_file->path_len - cur_file->cur_path_ofs; + const auto hash = romfs_calc_path_hash(cur_file->parent->entry_offset, + reinterpret_cast(cur_file->path) + + cur_file->cur_path_ofs, + 0, name_size); + cur_entry->hash = file_hash_table[hash % file_hash_table_entry_count]; + file_hash_table[hash % file_hash_table_entry_count] = cur_file->entry_offset; + + cur_entry->name_size = name_size; + memset(cur_entry->name, 0, (cur_entry->name_size + 3) & ~3); + memcpy(cur_entry->name, cur_file->path + cur_file->cur_path_ofs, name_size); + + out.emplace(cur_file->offset + ROMFS_FILEPARTITION_OFS, cur_file->source); + } + + /* Populate dir tables. */ + for (const auto& it : this->directories) { + cur_dir = it.second; + RomFSDirectoryEntry* cur_entry = romfs_get_direntry(dir_table, cur_dir->entry_offset); + cur_entry->parent = cur_dir == this->root ? 0 : cur_dir->parent->entry_offset; + cur_entry->sibling = + (cur_dir->sibling == nullptr) ? ROMFS_ENTRY_EMPTY : cur_dir->sibling->entry_offset; + cur_entry->child = + (cur_dir->child == nullptr) ? ROMFS_ENTRY_EMPTY : cur_dir->child->entry_offset; + cur_entry->file = + (cur_dir->file == nullptr) ? ROMFS_ENTRY_EMPTY : cur_dir->file->entry_offset; + + u32 name_size = cur_dir->path_len - cur_dir->cur_path_ofs; + u32 hash = romfs_calc_path_hash( + cur_dir == this->root ? 0 : cur_dir->parent->entry_offset, + reinterpret_cast(cur_dir->path) + cur_dir->cur_path_ofs, 0, name_size); + cur_entry->hash = dir_hash_table[hash % dir_hash_table_entry_count]; + dir_hash_table[hash % dir_hash_table_entry_count] = cur_dir->entry_offset; + + cur_entry->name_size = name_size; + memset(cur_entry->name, 0, (cur_entry->name_size + 3) & ~3); + memcpy(cur_entry->name, cur_dir->path + cur_dir->cur_path_ofs, name_size); + } + + /* Delete directories. */ + for (const auto& it : this->directories) { + cur_dir = it.second; + delete cur_dir->path; + delete cur_dir; + } + this->root = nullptr; + this->directories.clear(); + + /* Delete files. */ + for (const auto& it : this->files) { + cur_file = it.second; + delete cur_file->path; + delete cur_file; + } + this->files.clear(); + + /* Set header fields. */ + header->header_size = sizeof(*header); + header->file_hash_table_size = this->file_hash_table_size; + header->file_table_size = this->file_table_size; + header->dir_hash_table_size = this->dir_hash_table_size; + header->dir_table_size = this->dir_table_size; + header->file_partition_ofs = ROMFS_FILEPARTITION_OFS; + header->dir_hash_table_ofs = + (header->file_partition_ofs + this->file_partition_size + 3ULL) & ~3ULL; + header->dir_table_ofs = header->dir_hash_table_ofs + header->dir_hash_table_size; + header->file_hash_table_ofs = header->dir_table_ofs + header->dir_table_size; + header->file_table_ofs = header->file_hash_table_ofs + header->file_hash_table_size; + + std::vector header_data(sizeof(RomFSHeader)); + std::memcpy(header_data.data(), header, header_data.size()); + out.emplace(0, std::make_shared(header_data)); + + std::vector meta_out(this->file_hash_table_size + this->file_table_size + + this->dir_hash_table_size + this->dir_table_size); + std::memcpy(meta_out.data(), metadata, meta_out.size()); + out.emplace(header->dir_hash_table_ofs, std::make_shared(meta_out)); + + return out; +} + +} // namespace FileSys diff --git a/src/core/file_sys/fsmitm_romfsbuild.hpp b/src/core/file_sys/fsmitm_romfsbuild.hpp new file mode 100644 index 000000000..b897aab21 --- /dev/null +++ b/src/core/file_sys/fsmitm_romfsbuild.hpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2018 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * Adapted by DarkLordZach for use/interaction with yuzu + * + * Modifications Copyright 2018 yuzu emulator team + * Licensed under GPLv2 or any later version + * Refer to the license.txt file included. + */ + +#pragma once +#include +#include +#include "common/common_types.h" +#include "vfs.h" + +namespace FileSys { + +/* Used as comparator for std::map */ +struct build_ctx_cmp { + bool operator()(const char* a, const char* b) const { + return strcmp(a, b) < 0; + } +}; + +struct RomFSDirectoryEntry; +struct RomFSFileEntry; +struct RomFSBuildDirectoryContext; +struct RomFSBuildFileContext; + +class RomFSBuildContext { +private: + VirtualDir base; + RomFSBuildDirectoryContext* root; + std::map directories; + std::map files; + u64 num_dirs = 0; + u64 num_files = 0; + u64 dir_table_size = 0; + u64 file_table_size = 0; + u64 dir_hash_table_size = 0; + u64 file_hash_table_size = 0; + u64 file_partition_size = 0; + + void VisitDirectory(VirtualDir filesys, RomFSBuildDirectoryContext* parent); + + bool AddDirectory(RomFSBuildDirectoryContext* parent_dir_ctx, + RomFSBuildDirectoryContext* dir_ctx, + RomFSBuildDirectoryContext** out_dir_ctx); + bool AddFile(RomFSBuildDirectoryContext* parent_dir_ctx, RomFSBuildFileContext* file_ctx); + +public: + explicit RomFSBuildContext(VirtualDir base); + + /* This finalizes the context. */ + std::map Build(); +}; + +static inline RomFSDirectoryEntry* romfs_get_direntry(void* directories, uint32_t offset) { + return (RomFSDirectoryEntry*)((uintptr_t)directories + offset); +} + +static inline RomFSFileEntry* romfs_get_fentry(void* files, uint32_t offset) { + return (RomFSFileEntry*)((uintptr_t)files + offset); +} + +static inline uint32_t romfs_calc_path_hash(uint32_t parent, const unsigned char* path, + uint32_t start, size_t path_len) { + uint32_t hash = parent ^ 123456789; + for (uint32_t i = 0; i < path_len; i++) { + hash = (hash >> 5) | (hash << 27); + hash ^= path[start + i]; + } + + return hash; +} + +static inline uint32_t romfs_get_hash_table_count(uint32_t num_entries) { + if (num_entries < 3) { + return 3; + } else if (num_entries < 19) { + return num_entries | 1; + } + uint32_t count = num_entries; + while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 || + count % 11 == 0 || count % 13 == 0 || count % 17 == 0) { + count++; + } + return count; +} + +} // namespace FileSys From 050547b801ccc7cfea65c142658e8d2a987472d2 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 22:04:51 -0400 Subject: [PATCH 12/14] romfs: Implement CreateRomFS --- src/core/file_sys/romfs.cpp | 20 +++++++++++++++++--- src/core/file_sys/romfs.h | 9 ++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/core/file_sys/romfs.cpp b/src/core/file_sys/romfs.cpp index 9f6e41cdf..71e4e0e2f 100644 --- a/src/core/file_sys/romfs.cpp +++ b/src/core/file_sys/romfs.cpp @@ -4,8 +4,10 @@ #include "common/common_types.h" #include "common/swap.h" +#include "core/file_sys/fsmitm_romfsbuild.hpp" #include "core/file_sys/romfs.h" #include "core/file_sys/vfs.h" +#include "core/file_sys/vfs_concat.h" #include "core/file_sys/vfs_offset.h" #include "core/file_sys/vfs_vector.h" @@ -98,7 +100,7 @@ void ProcessDirectory(VirtualFile file, std::size_t dir_offset, std::size_t file } } -VirtualDir ExtractRomFS(VirtualFile file) { +VirtualDir ExtractRomFS(VirtualFile file, bool traverse_into_data) { RomFSHeader header{}; if (file->ReadObject(&header) != sizeof(RomFSHeader)) return nullptr; @@ -117,9 +119,21 @@ VirtualDir ExtractRomFS(VirtualFile file) { VirtualDir out = std::move(root); - while (out->GetSubdirectory("") != nullptr) - out = out->GetSubdirectory(""); + while (out->GetSubdirectories().size() == 1 && out->GetFiles().size() == 0) { + if (out->GetSubdirectories().front()->GetName() == "data" && !traverse_into_data) + break; + out = out->GetSubdirectories().front(); + } return out; } + +VirtualFile CreateRomFS(VirtualDir dir) { + if (dir == nullptr) + return nullptr; + + RomFSBuildContext ctx{dir}; + return ConcatenateFiles<0>(ctx.Build(), dir->GetName()); +} + } // namespace FileSys diff --git a/src/core/file_sys/romfs.h b/src/core/file_sys/romfs.h index e54a7d7a9..8e82585a0 100644 --- a/src/core/file_sys/romfs.h +++ b/src/core/file_sys/romfs.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include "common/common_funcs.h" #include "common/common_types.h" #include "common/swap.h" @@ -12,6 +13,8 @@ namespace FileSys { +struct RomFSHeader; + struct IVFCLevel { u64_le offset; u64_le size; @@ -31,6 +34,10 @@ static_assert(sizeof(IVFCHeader) == 0xE0, "IVFCHeader has incorrect size."); // Converts a RomFS binary blob to VFS Filesystem // Returns nullptr on failure -VirtualDir ExtractRomFS(VirtualFile file); +VirtualDir ExtractRomFS(VirtualFile file, bool traverse_into_data = true); + +// Converts a VFS filesystem into a RomFS binary +// Returns nullptr on failure +VirtualFile CreateRomFS(VirtualDir dir); } // namespace FileSys From ba0873d33cf6a08c6f52df45b45a14dcf91f4cf0 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Wed, 19 Sep 2018 22:09:23 -0400 Subject: [PATCH 13/14] qt: Add UI elements for LayeredFS and related tools --- src/core/file_sys/registered_cache.cpp | 2 +- src/core/file_sys/registered_cache.h | 2 +- src/yuzu/game_list.cpp | 9 ++ src/yuzu/game_list.h | 7 +- src/yuzu/main.cpp | 145 ++++++++++++++++++++++++- src/yuzu/main.h | 2 + 6 files changed, 162 insertions(+), 5 deletions(-) diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index dad7ae10b..0dda0b861 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -480,7 +480,7 @@ InstallResult RegisteredCache::RawInstallNCA(std::shared_ptr nca, const Vfs auto out = dir->CreateFileRelative(path); if (out == nullptr) return InstallResult::ErrorCopyFailed; - return copy(in, out) ? InstallResult::Success : InstallResult::ErrorCopyFailed; + return copy(in, out, 0x400000) ? InstallResult::Success : InstallResult::ErrorCopyFailed; } bool RegisteredCache::RawInstallYuzuMeta(const CNMT& cnmt) { diff --git a/src/core/file_sys/registered_cache.h b/src/core/file_sys/registered_cache.h index f487b0cf0..c0cd59fc5 100644 --- a/src/core/file_sys/registered_cache.h +++ b/src/core/file_sys/registered_cache.h @@ -27,7 +27,7 @@ struct ContentRecord; using NcaID = std::array; using RegisteredCacheParsingFunction = std::function; -using VfsCopyFunction = std::function; +using VfsCopyFunction = std::function; enum class InstallResult { Success, diff --git a/src/yuzu/game_list.cpp b/src/yuzu/game_list.cpp index e8b2f720a..991ae10cd 100644 --- a/src/yuzu/game_list.cpp +++ b/src/yuzu/game_list.cpp @@ -318,9 +318,14 @@ void GameList::PopupContextMenu(const QPoint& menu_location) { int row = item_model->itemFromIndex(item)->row(); QStandardItem* child_file = item_model->invisibleRootItem()->child(row, COLUMN_NAME); u64 program_id = child_file->data(GameListItemPath::ProgramIdRole).toULongLong(); + std::string path = child_file->data(GameListItemPath::FullPathRole).toString().toStdString(); QMenu context_menu; QAction* open_save_location = context_menu.addAction(tr("Open Save Data Location")); + QAction* open_lfs_location = context_menu.addAction(tr("Open Mod Data Location")); + context_menu.addSeparator(); + QAction* dump_romfs = context_menu.addAction(tr("Dump RomFS")); + QAction* copy_tid = context_menu.addAction(tr("Copy Title ID to Clipboard")); QAction* navigate_to_gamedb_entry = context_menu.addAction(tr("Navigate to GameDB entry")); open_save_location->setEnabled(program_id != 0); @@ -329,6 +334,10 @@ void GameList::PopupContextMenu(const QPoint& menu_location) { connect(open_save_location, &QAction::triggered, [&]() { emit OpenFolderRequested(program_id, GameListOpenTarget::SaveData); }); + connect(open_lfs_location, &QAction::triggered, + [&]() { emit OpenFolderRequested(program_id, GameListOpenTarget::ModData); }); + connect(dump_romfs, &QAction::triggered, [&]() { emit DumpRomFSRequested(program_id, path); }); + connect(copy_tid, &QAction::triggered, [&]() { emit CopyTIDRequested(program_id); }); connect(navigate_to_gamedb_entry, &QAction::triggered, [&]() { emit NavigateToGamedbEntryRequested(program_id, compatibility_list); }); diff --git a/src/yuzu/game_list.h b/src/yuzu/game_list.h index 2713e7b54..3bf51870e 100644 --- a/src/yuzu/game_list.h +++ b/src/yuzu/game_list.h @@ -28,7 +28,10 @@ namespace FileSys { class VfsFilesystem; } -enum class GameListOpenTarget { SaveData }; +enum class GameListOpenTarget { + SaveData, + ModData, +}; class GameList : public QWidget { Q_OBJECT @@ -89,6 +92,8 @@ signals: void GameChosen(QString game_path); void ShouldCancelWorker(); void OpenFolderRequested(u64 program_id, GameListOpenTarget target); + void DumpRomFSRequested(u64 program_id, const std::string& game_path); + void CopyTIDRequested(u64 program_id); void NavigateToGamedbEntryRequested(u64 program_id, const CompatibilityList& compatibility_list); diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index 45bb1d1d1..7cfe8a32f 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -7,6 +7,22 @@ #include #include +// VFS includes must be before glad as they will conflict with Windows file api, which uses defines. +#include "core/file_sys/vfs.h" +#include "core/file_sys/vfs_real.h" + +// These are wrappers to avoid the calls to CreateDirectory and CreateFile becuase of the Windows +// defines. +static FileSys::VirtualDir VfsFilesystemCreateDirectoryWrapper( + const FileSys::VirtualFilesystem& vfs, const std::string& path, FileSys::Mode mode) { + return vfs->CreateDirectory(path, mode); +} + +static FileSys::VirtualFile VfsDirectoryCreateFileWrapper(const FileSys::VirtualDir& dir, + const std::string& path) { + return dir->CreateFile(path); +} + #include #include @@ -30,16 +46,18 @@ #include "common/telemetry.h" #include "core/core.h" #include "core/crypto/key_manager.h" +#include "core/file_sys/bis_factory.h" #include "core/file_sys/card_image.h" #include "core/file_sys/content_archive.h" #include "core/file_sys/control_metadata.h" #include "core/file_sys/patch_manager.h" #include "core/file_sys/registered_cache.h" +#include "core/file_sys/romfs.h" #include "core/file_sys/savedata_factory.h" #include "core/file_sys/submission_package.h" -#include "core/file_sys/vfs_real.h" #include "core/hle/kernel/process.h" #include "core/hle/service/filesystem/filesystem.h" +#include "core/hle/service/filesystem/fsp_ldr.h" #include "core/loader/loader.h" #include "core/perf_stats.h" #include "core/settings.h" @@ -362,6 +380,8 @@ void GMainWindow::RestoreUIState() { void GMainWindow::ConnectWidgetEvents() { connect(game_list, &GameList::GameChosen, this, &GMainWindow::OnGameListLoadFile); connect(game_list, &GameList::OpenFolderRequested, this, &GMainWindow::OnGameListOpenFolder); + connect(game_list, &GameList::DumpRomFSRequested, this, &GMainWindow::OnGameListDumpRomFS); + connect(game_list, &GameList::CopyTIDRequested, this, &GMainWindow::OnGameListCopyTID); connect(game_list, &GameList::NavigateToGamedbEntryRequested, this, &GMainWindow::OnGameListNavigateToGamedbEntry); @@ -713,6 +733,12 @@ void GMainWindow::OnGameListOpenFolder(u64 program_id, GameListOpenTarget target program_id, user_id, 0); break; } + case GameListOpenTarget::ModData: { + open_target = "Mod Data"; + const auto load_dir = FileUtil::GetUserPath(FileUtil::UserPath::LoadDir); + path = fmt::format("{}{:016X}", load_dir, program_id); + break; + } default: UNIMPLEMENTED(); } @@ -730,6 +756,120 @@ void GMainWindow::OnGameListOpenFolder(u64 program_id, GameListOpenTarget target QDesktopServices::openUrl(QUrl::fromLocalFile(qpath)); } +void GMainWindow::OnGameListDumpRomFS(u64 program_id, const std::string& game_path) { + const auto path = fmt::format("{}{:016X}/romfs", + FileUtil::GetUserPath(FileUtil::UserPath::DumpDir), program_id); + + auto failed = [this, &path]() { + QMessageBox::warning(this, tr("RomFS Extraction Failed!"), + tr("There was an error copying the RomFS files or the user " + "cancelled the operation.")); + vfs->DeleteDirectory(path); + }; + + const auto loader = Loader::GetLoader(vfs->OpenFile(game_path, FileSys::Mode::Read)); + if (loader == nullptr) { + failed(); + return; + } + + FileSys::VirtualFile file; + if (loader->ReadRomFS(file) != Loader::ResultStatus::Success) { + failed(); + return; + } + + const auto romfs = + loader->IsRomFSUpdatable() + ? FileSys::PatchManager(program_id).PatchRomFS(file, loader->ReadRomFSIVFCOffset()) + : file; + + const auto extracted = FileSys::ExtractRomFS(romfs, false); + if (extracted == nullptr) { + failed(); + return; + } + + const auto out = VfsFilesystemCreateDirectoryWrapper(vfs, path, FileSys::Mode::ReadWrite); + + if (out == nullptr) { + failed(); + return; + } + + bool ok; + const auto res = QInputDialog::getItem( + this, tr("Select RomFS Dump Mode"), + tr("Please select the how you would like the RomFS dumped.
Full will copy all of the " + "files into the new directory while
skeleton will only create the directory " + "structure."), + {"Full", "Skeleton"}, 0, false, &ok); + if (!ok) + failed(); + + const auto full = res == "Full"; + + const static std::function calculate_entry_size = + [](const FileSys::VirtualDir& dir, bool full) { + size_t out = 0; + for (const auto& subdir : dir->GetSubdirectories()) + out += 1 + calculate_entry_size(subdir, full); + return out + full ? dir->GetFiles().size() : 0; + }; + const auto entry_size = calculate_entry_size(extracted, full); + + QProgressDialog progress(tr("Extracting RomFS..."), tr("Cancel"), 0, entry_size, this); + progress.setWindowModality(Qt::WindowModal); + progress.setMinimumDuration(100); + + const static std::function + qt_raw_copy = [](QProgressDialog& dialog, const FileSys::VirtualDir& src, + const FileSys::VirtualDir& dest, size_t block_size, bool full) { + if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) + return false; + if (dialog.wasCanceled()) + return false; + + if (full) { + for (const auto& file : src->GetFiles()) { + const auto out = VfsDirectoryCreateFileWrapper(dest, file->GetName()); + if (!FileSys::VfsRawCopy(file, out, block_size)) + return false; + dialog.setValue(dialog.value() + 1); + if (dialog.wasCanceled()) + return false; + } + } + + for (const auto& dir : src->GetSubdirectories()) { + const auto out = dest->CreateSubdirectory(dir->GetName()); + if (!qt_raw_copy(dialog, dir, out, block_size, full)) + return false; + dialog.setValue(dialog.value() + 1); + if (dialog.wasCanceled()) + return false; + } + + return true; + }; + + if (qt_raw_copy(progress, extracted, out, 0x400000, full)) { + progress.close(); + QMessageBox::information(this, tr("RomFS Extraction Succeeded!"), + tr("The operation completed successfully.")); + QDesktopServices::openUrl(QUrl::fromLocalFile(QString::fromStdString(path))); + } else { + progress.close(); + failed(); + } +} + +void GMainWindow::OnGameListCopyTID(u64 program_id) { + QClipboard* clipboard = QGuiApplication::clipboard(); + clipboard->setText(QString::fromStdString(fmt::format("{:016X}", program_id))); +} + void GMainWindow::OnGameListNavigateToGamedbEntry(u64 program_id, const CompatibilityList& compatibility_list) { const auto it = FindMatchingCompatibilityEntry(compatibility_list, program_id); @@ -790,7 +930,8 @@ void GMainWindow::OnMenuInstallToNAND() { return; } - const auto qt_raw_copy = [this](FileSys::VirtualFile src, FileSys::VirtualFile dest) { + const auto qt_raw_copy = [this](const FileSys::VirtualFile& src, + const FileSys::VirtualFile& dest, size_t block_size) { if (src == nullptr || dest == nullptr) return false; if (!dest->Resize(src->GetSize())) diff --git a/src/yuzu/main.h b/src/yuzu/main.h index 552e3e61c..8ee9242b1 100644 --- a/src/yuzu/main.h +++ b/src/yuzu/main.h @@ -138,6 +138,8 @@ private slots: /// Called whenever a user selects a game in the game list widget. void OnGameListLoadFile(QString game_path); void OnGameListOpenFolder(u64 program_id, GameListOpenTarget target); + void OnGameListDumpRomFS(u64 program_id, const std::string& game_path); + void OnGameListCopyTID(u64 program_id); void OnGameListNavigateToGamedbEntry(u64 program_id, const CompatibilityList& compatibility_list); void OnMenuLoadFile(); From b3c2ec362bbbdd89da9c0aa84b425717f5e3d351 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sun, 23 Sep 2018 21:50:16 -0400 Subject: [PATCH 14/14] fsmitm: Cleanup and modernize fsmitm port --- src/core/CMakeLists.txt | 2 +- src/core/file_sys/bis_factory.cpp | 2 +- src/core/file_sys/bis_factory.h | 2 +- src/core/file_sys/fsmitm_romfsbuild.cpp | 354 ++++++++++++------------ src/core/file_sys/fsmitm_romfsbuild.h | 70 +++++ src/core/file_sys/fsmitm_romfsbuild.hpp | 106 ------- src/core/file_sys/patch_manager.cpp | 72 ++--- src/core/file_sys/registered_cache.cpp | 9 +- src/core/file_sys/romfs.cpp | 9 +- src/core/file_sys/romfs.h | 8 +- src/core/file_sys/vfs.cpp | 4 +- src/core/file_sys/vfs.h | 5 +- src/core/file_sys/vfs_concat.cpp | 21 +- src/core/file_sys/vfs_concat.h | 58 ++-- src/core/file_sys/vfs_layered.cpp | 15 +- src/core/file_sys/vfs_layered.h | 2 + src/core/file_sys/vfs_real.cpp | 4 +- src/core/file_sys/vfs_real.h | 2 +- src/core/file_sys/vfs_static.h | 3 +- src/core/file_sys/vfs_vector.cpp | 7 +- src/core/file_sys/vfs_vector.h | 1 + src/yuzu/main.cpp | 2 +- 22 files changed, 379 insertions(+), 379 deletions(-) create mode 100644 src/core/file_sys/fsmitm_romfsbuild.h delete mode 100644 src/core/file_sys/fsmitm_romfsbuild.hpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index ead86fd85..23fd6e920 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -33,7 +33,7 @@ add_library(core STATIC file_sys/directory.h file_sys/errors.h file_sys/fsmitm_romfsbuild.cpp - file_sys/fsmitm_romfsbuild.hpp + file_sys/fsmitm_romfsbuild.h file_sys/mode.h file_sys/nca_metadata.cpp file_sys/nca_metadata.h diff --git a/src/core/file_sys/bis_factory.cpp b/src/core/file_sys/bis_factory.cpp index 012e08e7d..6102ef476 100644 --- a/src/core/file_sys/bis_factory.cpp +++ b/src/core/file_sys/bis_factory.cpp @@ -2,9 +2,9 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include "core/file_sys/bis_factory.h" #include "core/file_sys/registered_cache.h" -#include "fmt/format.h" namespace FileSys { diff --git a/src/core/file_sys/bis_factory.h b/src/core/file_sys/bis_factory.h index 0d81967cc..c352e0925 100644 --- a/src/core/file_sys/bis_factory.h +++ b/src/core/file_sys/bis_factory.h @@ -17,7 +17,7 @@ class RegisteredCache; /// registered caches. class BISFactory { public: - BISFactory(VirtualDir nand_root, VirtualDir load_root); + explicit BISFactory(VirtualDir nand_root, VirtualDir load_root); ~BISFactory(); std::shared_ptr GetSystemNANDContents() const; diff --git a/src/core/file_sys/fsmitm_romfsbuild.cpp b/src/core/file_sys/fsmitm_romfsbuild.cpp index a6fe6719d..21fc3d796 100644 --- a/src/core/file_sys/fsmitm_romfsbuild.cpp +++ b/src/core/file_sys/fsmitm_romfsbuild.cpp @@ -24,9 +24,9 @@ #include #include "common/assert.h" -#include "fsmitm_romfsbuild.hpp" -#include "vfs.h" -#include "vfs_vector.h" +#include "core/file_sys/fsmitm_romfsbuild.h" +#include "core/file_sys/vfs.h" +#include "core/file_sys/vfs_vector.h" namespace FileSys { @@ -35,7 +35,7 @@ constexpr u64 FS_MAX_PATH = 0x301; constexpr u32 ROMFS_ENTRY_EMPTY = 0xFFFFFFFF; constexpr u32 ROMFS_FILEPARTITION_OFS = 0x200; -/* Types for building a RomFS. */ +// Types for building a RomFS. struct RomFSHeader { u64 header_size; u64 dir_hash_table_ofs; @@ -57,7 +57,6 @@ struct RomFSDirectoryEntry { u32 file; u32 hash; u32 name_size; - char name[]; }; static_assert(sizeof(RomFSDirectoryEntry) == 0x18, "RomFSDirectoryEntry has incorrect size."); @@ -68,37 +67,63 @@ struct RomFSFileEntry { u64 size; u32 hash; u32 name_size; - char name[]; }; static_assert(sizeof(RomFSFileEntry) == 0x20, "RomFSFileEntry has incorrect size."); struct RomFSBuildFileContext; struct RomFSBuildDirectoryContext { - char* path; - u32 cur_path_ofs; - u32 path_len; + std::string path = ""; + u32 cur_path_ofs = 0; + u32 path_len = 0; u32 entry_offset = 0; - RomFSBuildDirectoryContext* parent = nullptr; - RomFSBuildDirectoryContext* child = nullptr; - RomFSBuildDirectoryContext* sibling = nullptr; - RomFSBuildFileContext* file = nullptr; + std::shared_ptr parent; + std::shared_ptr child; + std::shared_ptr sibling; + std::shared_ptr file; }; struct RomFSBuildFileContext { - char* path; - u32 cur_path_ofs; - u32 path_len; + std::string path = ""; + u32 cur_path_ofs = 0; + u32 path_len = 0; u32 entry_offset = 0; u64 offset = 0; u64 size = 0; - RomFSBuildDirectoryContext* parent = nullptr; - RomFSBuildFileContext* sibling = nullptr; + std::shared_ptr parent; + std::shared_ptr sibling; VirtualFile source = nullptr; + + RomFSBuildFileContext() : path(""), cur_path_ofs(0), path_len(0) {} }; -void RomFSBuildContext::VisitDirectory(VirtualDir root_romfs, RomFSBuildDirectoryContext* parent) { - std::vector child_dirs; +static u32 romfs_calc_path_hash(u32 parent, std::string path, u32 start, size_t path_len) { + u32 hash = parent ^ 123456789; + for (u32 i = 0; i < path_len; i++) { + hash = (hash >> 5) | (hash << 27); + hash ^= path[start + i]; + } + + return hash; +} + +static u32 romfs_get_hash_table_count(u32 num_entries) { + if (num_entries < 3) { + return 3; + } else if (num_entries < 19) { + return num_entries | 1; + } + u32 count = num_entries; + while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 || + count % 11 == 0 || count % 13 == 0 || count % 17 == 0) { + count++; + } + return count; +} + +void RomFSBuildContext::VisitDirectory(VirtualDir root_romfs, + std::shared_ptr parent) { + std::vector> child_dirs; VirtualDir dir; @@ -111,41 +136,33 @@ void RomFSBuildContext::VisitDirectory(VirtualDir root_romfs, RomFSBuildDirector for (const auto& kv : entries) { if (kv.second == VfsEntryType::Directory) { - RomFSBuildDirectoryContext* child = new RomFSBuildDirectoryContext({0}); - /* Set child's path. */ + const auto child = std::make_shared(); + // Set child's path. child->cur_path_ofs = parent->path_len + 1; child->path_len = child->cur_path_ofs + kv.first.size(); - child->path = new char[child->path_len + 1]; - strcpy(child->path, parent->path); - ASSERT(child->path_len < FS_MAX_PATH); - strcat(child->path + parent->path_len, "/"); - strcat(child->path + parent->path_len, kv.first.c_str()); + child->path = parent->path + "/" + kv.first; - if (!this->AddDirectory(parent, child, nullptr)) { - delete child->path; - delete child; - } else { + // Sanity check on path_len + ASSERT(child->path_len < FS_MAX_PATH); + + if (AddDirectory(parent, child)) { child_dirs.push_back(child); } } else { - RomFSBuildFileContext* child = new RomFSBuildFileContext({0}); - /* Set child's path. */ + const auto child = std::make_shared(); + // Set child's path. child->cur_path_ofs = parent->path_len + 1; child->path_len = child->cur_path_ofs + kv.first.size(); - child->path = new char[child->path_len + 1]; - strcpy(child->path, parent->path); + child->path = parent->path + "/" + kv.first; + + // Sanity check on path_len ASSERT(child->path_len < FS_MAX_PATH); - strcat(child->path + parent->path_len, "/"); - strcat(child->path + parent->path_len, kv.first.c_str()); child->source = root_romfs->GetFileRelative(child->path); child->size = child->source->GetSize(); - if (!this->AddFile(parent, child)) { - delete child->path; - delete child; - } + AddFile(parent, child); } } @@ -154,213 +171,200 @@ void RomFSBuildContext::VisitDirectory(VirtualDir root_romfs, RomFSBuildDirector } } -bool RomFSBuildContext::AddDirectory(RomFSBuildDirectoryContext* parent_dir_ctx, - RomFSBuildDirectoryContext* dir_ctx, - RomFSBuildDirectoryContext** out_dir_ctx) { - /* Check whether it's already in the known directories. */ - auto existing = this->directories.find(dir_ctx->path); - if (existing != this->directories.end()) { - if (out_dir_ctx) { - *out_dir_ctx = existing->second; - } +bool RomFSBuildContext::AddDirectory(std::shared_ptr parent_dir_ctx, + std::shared_ptr dir_ctx) { + // Check whether it's already in the known directories. + const auto existing = directories.find(dir_ctx->path); + if (existing != directories.end()) return false; - } - /* Add a new directory. */ - this->num_dirs++; - this->dir_table_size += + // Add a new directory. + num_dirs++; + dir_table_size += sizeof(RomFSDirectoryEntry) + ((dir_ctx->path_len - dir_ctx->cur_path_ofs + 3) & ~3); dir_ctx->parent = parent_dir_ctx; - this->directories.insert({dir_ctx->path, dir_ctx}); + directories.emplace(dir_ctx->path, dir_ctx); - if (out_dir_ctx) { - *out_dir_ctx = dir_ctx; - } return true; } -bool RomFSBuildContext::AddFile(RomFSBuildDirectoryContext* parent_dir_ctx, - RomFSBuildFileContext* file_ctx) { - /* Check whether it's already in the known files. */ - auto existing = this->files.find(file_ctx->path); - if (existing != this->files.end()) { +bool RomFSBuildContext::AddFile(std::shared_ptr parent_dir_ctx, + std::shared_ptr file_ctx) { + // Check whether it's already in the known files. + const auto existing = files.find(file_ctx->path); + if (existing != files.end()) { return false; } - /* Add a new file. */ - this->num_files++; - this->file_table_size += + // Add a new file. + num_files++; + file_table_size += sizeof(RomFSFileEntry) + ((file_ctx->path_len - file_ctx->cur_path_ofs + 3) & ~3); file_ctx->parent = parent_dir_ctx; - this->files.insert({file_ctx->path, file_ctx}); + files.emplace(file_ctx->path, file_ctx); return true; } RomFSBuildContext::RomFSBuildContext(VirtualDir base_) : base(std::move(base_)) { - this->root = new RomFSBuildDirectoryContext({0}); - this->root->path = new char[1]; - this->root->path[0] = '\x00'; - this->directories.insert({this->root->path, this->root}); - this->num_dirs = 1; - this->dir_table_size = 0x18; + root = std::make_shared(); + root->path = "\0"; + directories.emplace(root->path, root); + num_dirs = 1; + dir_table_size = 0x18; - VisitDirectory(base, this->root); + VisitDirectory(base, root); } +RomFSBuildContext::~RomFSBuildContext() = default; + std::map RomFSBuildContext::Build() { - std::map out; - RomFSBuildFileContext* cur_file; - RomFSBuildDirectoryContext* cur_dir; + const auto dir_hash_table_entry_count = romfs_get_hash_table_count(num_dirs); + const auto file_hash_table_entry_count = romfs_get_hash_table_count(num_files); + dir_hash_table_size = 4 * dir_hash_table_entry_count; + file_hash_table_size = 4 * file_hash_table_entry_count; - const auto dir_hash_table_entry_count = romfs_get_hash_table_count(this->num_dirs); - const auto file_hash_table_entry_count = romfs_get_hash_table_count(this->num_files); - this->dir_hash_table_size = 4 * dir_hash_table_entry_count; - this->file_hash_table_size = 4 * file_hash_table_entry_count; + // Assign metadata pointers + RomFSHeader header{}; - /* Assign metadata pointers */ - RomFSHeader* header = new RomFSHeader({0}); - auto metadata = new u8[this->dir_hash_table_size + this->dir_table_size + - this->file_hash_table_size + this->file_table_size]; - auto dir_hash_table = reinterpret_cast(metadata); - const auto dir_table = reinterpret_cast( - reinterpret_cast(dir_hash_table) + this->dir_hash_table_size); - auto file_hash_table = - reinterpret_cast(reinterpret_cast(dir_table) + this->dir_table_size); - const auto file_table = reinterpret_cast( - reinterpret_cast(file_hash_table) + this->file_hash_table_size); + std::vector dir_hash_table(dir_hash_table_entry_count, ROMFS_ENTRY_EMPTY); + std::vector file_hash_table(file_hash_table_entry_count, ROMFS_ENTRY_EMPTY); - /* Clear out hash tables. */ + std::vector dir_table(dir_table_size); + std::vector file_table(file_table_size); + + // Clear out hash tables. for (u32 i = 0; i < dir_hash_table_entry_count; i++) dir_hash_table[i] = ROMFS_ENTRY_EMPTY; for (u32 i = 0; i < file_hash_table_entry_count; i++) file_hash_table[i] = ROMFS_ENTRY_EMPTY; - out.clear(); + std::shared_ptr cur_file; - /* Determine file offsets. */ + // Determine file offsets. u32 entry_offset = 0; - RomFSBuildFileContext* prev_file = nullptr; - for (const auto& it : this->files) { + std::shared_ptr prev_file = nullptr; + for (const auto& it : files) { cur_file = it.second; - this->file_partition_size = (this->file_partition_size + 0xFULL) & ~0xFULL; - cur_file->offset = this->file_partition_size; - this->file_partition_size += cur_file->size; + file_partition_size = (file_partition_size + 0xFULL) & ~0xFULL; + cur_file->offset = file_partition_size; + file_partition_size += cur_file->size; cur_file->entry_offset = entry_offset; entry_offset += sizeof(RomFSFileEntry) + ((cur_file->path_len - cur_file->cur_path_ofs + 3) & ~3); prev_file = cur_file; } - /* Assign deferred parent/sibling ownership. */ - for (auto it = this->files.rbegin(); it != this->files.rend(); it++) { + // Assign deferred parent/sibling ownership. + for (auto it = files.rbegin(); it != files.rend(); ++it) { cur_file = it->second; cur_file->sibling = cur_file->parent->file; cur_file->parent->file = cur_file; } - /* Determine directory offsets. */ + std::shared_ptr cur_dir; + + // Determine directory offsets. entry_offset = 0; - for (const auto& it : this->directories) { + for (const auto& it : directories) { cur_dir = it.second; cur_dir->entry_offset = entry_offset; entry_offset += sizeof(RomFSDirectoryEntry) + ((cur_dir->path_len - cur_dir->cur_path_ofs + 3) & ~3); } - /* Assign deferred parent/sibling ownership. */ - for (auto it = this->directories.rbegin(); it->second != this->root; it++) { + // Assign deferred parent/sibling ownership. + for (auto it = directories.rbegin(); it->second != root; ++it) { cur_dir = it->second; cur_dir->sibling = cur_dir->parent->child; cur_dir->parent->child = cur_dir; } - /* Populate file tables. */ - for (const auto& it : this->files) { - cur_file = it.second; - RomFSFileEntry* cur_entry = romfs_get_fentry(file_table, cur_file->entry_offset); + std::map out; - cur_entry->parent = cur_file->parent->entry_offset; - cur_entry->sibling = - (cur_file->sibling == nullptr) ? ROMFS_ENTRY_EMPTY : cur_file->sibling->entry_offset; - cur_entry->offset = cur_file->offset; - cur_entry->size = cur_file->size; + // Populate file tables. + for (const auto& it : files) { + cur_file = it.second; + RomFSFileEntry cur_entry{}; + + cur_entry.parent = cur_file->parent->entry_offset; + cur_entry.sibling = + cur_file->sibling == nullptr ? ROMFS_ENTRY_EMPTY : cur_file->sibling->entry_offset; + cur_entry.offset = cur_file->offset; + cur_entry.size = cur_file->size; const auto name_size = cur_file->path_len - cur_file->cur_path_ofs; - const auto hash = romfs_calc_path_hash(cur_file->parent->entry_offset, - reinterpret_cast(cur_file->path) + - cur_file->cur_path_ofs, - 0, name_size); - cur_entry->hash = file_hash_table[hash % file_hash_table_entry_count]; + const auto hash = romfs_calc_path_hash(cur_file->parent->entry_offset, cur_file->path, + cur_file->cur_path_ofs, name_size); + cur_entry.hash = file_hash_table[hash % file_hash_table_entry_count]; file_hash_table[hash % file_hash_table_entry_count] = cur_file->entry_offset; - cur_entry->name_size = name_size; - memset(cur_entry->name, 0, (cur_entry->name_size + 3) & ~3); - memcpy(cur_entry->name, cur_file->path + cur_file->cur_path_ofs, name_size); + cur_entry.name_size = name_size; out.emplace(cur_file->offset + ROMFS_FILEPARTITION_OFS, cur_file->source); + std::memcpy(file_table.data() + cur_file->entry_offset, &cur_entry, sizeof(RomFSFileEntry)); + std::memset(file_table.data() + cur_file->entry_offset + sizeof(RomFSFileEntry), 0, + (cur_entry.name_size + 3) & ~3); + std::memcpy(file_table.data() + cur_file->entry_offset + sizeof(RomFSFileEntry), + cur_file->path.data() + cur_file->cur_path_ofs, name_size); } - /* Populate dir tables. */ - for (const auto& it : this->directories) { + // Populate dir tables. + for (const auto& it : directories) { cur_dir = it.second; - RomFSDirectoryEntry* cur_entry = romfs_get_direntry(dir_table, cur_dir->entry_offset); - cur_entry->parent = cur_dir == this->root ? 0 : cur_dir->parent->entry_offset; - cur_entry->sibling = - (cur_dir->sibling == nullptr) ? ROMFS_ENTRY_EMPTY : cur_dir->sibling->entry_offset; - cur_entry->child = - (cur_dir->child == nullptr) ? ROMFS_ENTRY_EMPTY : cur_dir->child->entry_offset; - cur_entry->file = - (cur_dir->file == nullptr) ? ROMFS_ENTRY_EMPTY : cur_dir->file->entry_offset; + RomFSDirectoryEntry cur_entry{}; - u32 name_size = cur_dir->path_len - cur_dir->cur_path_ofs; - u32 hash = romfs_calc_path_hash( - cur_dir == this->root ? 0 : cur_dir->parent->entry_offset, - reinterpret_cast(cur_dir->path) + cur_dir->cur_path_ofs, 0, name_size); - cur_entry->hash = dir_hash_table[hash % dir_hash_table_entry_count]; + cur_entry.parent = cur_dir == root ? 0 : cur_dir->parent->entry_offset; + cur_entry.sibling = + cur_dir->sibling == nullptr ? ROMFS_ENTRY_EMPTY : cur_dir->sibling->entry_offset; + cur_entry.child = + cur_dir->child == nullptr ? ROMFS_ENTRY_EMPTY : cur_dir->child->entry_offset; + cur_entry.file = cur_dir->file == nullptr ? ROMFS_ENTRY_EMPTY : cur_dir->file->entry_offset; + + const auto name_size = cur_dir->path_len - cur_dir->cur_path_ofs; + const auto hash = romfs_calc_path_hash(cur_dir == root ? 0 : cur_dir->parent->entry_offset, + cur_dir->path, cur_dir->cur_path_ofs, name_size); + cur_entry.hash = dir_hash_table[hash % dir_hash_table_entry_count]; dir_hash_table[hash % dir_hash_table_entry_count] = cur_dir->entry_offset; - cur_entry->name_size = name_size; - memset(cur_entry->name, 0, (cur_entry->name_size + 3) & ~3); - memcpy(cur_entry->name, cur_dir->path + cur_dir->cur_path_ofs, name_size); + cur_entry.name_size = name_size; + + std::memcpy(dir_table.data() + cur_dir->entry_offset, &cur_entry, + sizeof(RomFSDirectoryEntry)); + std::memcpy(dir_table.data() + cur_dir->entry_offset, &cur_entry, + sizeof(RomFSDirectoryEntry)); + std::memset(dir_table.data() + cur_dir->entry_offset + sizeof(RomFSDirectoryEntry), 0, + (cur_entry.name_size + 3) & ~3); + std::memcpy(dir_table.data() + cur_dir->entry_offset + sizeof(RomFSDirectoryEntry), + cur_dir->path.data() + cur_dir->cur_path_ofs, name_size); } - /* Delete directories. */ - for (const auto& it : this->directories) { - cur_dir = it.second; - delete cur_dir->path; - delete cur_dir; - } - this->root = nullptr; - this->directories.clear(); - - /* Delete files. */ - for (const auto& it : this->files) { - cur_file = it.second; - delete cur_file->path; - delete cur_file; - } - this->files.clear(); - - /* Set header fields. */ - header->header_size = sizeof(*header); - header->file_hash_table_size = this->file_hash_table_size; - header->file_table_size = this->file_table_size; - header->dir_hash_table_size = this->dir_hash_table_size; - header->dir_table_size = this->dir_table_size; - header->file_partition_ofs = ROMFS_FILEPARTITION_OFS; - header->dir_hash_table_ofs = - (header->file_partition_ofs + this->file_partition_size + 3ULL) & ~3ULL; - header->dir_table_ofs = header->dir_hash_table_ofs + header->dir_hash_table_size; - header->file_hash_table_ofs = header->dir_table_ofs + header->dir_table_size; - header->file_table_ofs = header->file_hash_table_ofs + header->file_hash_table_size; + // Set header fields. + header.header_size = sizeof(RomFSHeader); + header.file_hash_table_size = file_hash_table_size; + header.file_table_size = file_table_size; + header.dir_hash_table_size = dir_hash_table_size; + header.dir_table_size = dir_table_size; + header.file_partition_ofs = ROMFS_FILEPARTITION_OFS; + header.dir_hash_table_ofs = (header.file_partition_ofs + file_partition_size + 3ULL) & ~3ULL; + header.dir_table_ofs = header.dir_hash_table_ofs + header.dir_hash_table_size; + header.file_hash_table_ofs = header.dir_table_ofs + header.dir_table_size; + header.file_table_ofs = header.file_hash_table_ofs + header.file_hash_table_size; std::vector header_data(sizeof(RomFSHeader)); - std::memcpy(header_data.data(), header, header_data.size()); + std::memcpy(header_data.data(), &header, header_data.size()); out.emplace(0, std::make_shared(header_data)); - std::vector meta_out(this->file_hash_table_size + this->file_table_size + - this->dir_hash_table_size + this->dir_table_size); - std::memcpy(meta_out.data(), metadata, meta_out.size()); - out.emplace(header->dir_hash_table_ofs, std::make_shared(meta_out)); + std::vector metadata(file_hash_table_size + file_table_size + dir_hash_table_size + + dir_table_size); + auto index = 0; + std::memcpy(metadata.data(), dir_hash_table.data(), dir_hash_table.size() * sizeof(u32)); + index += dir_hash_table.size() * sizeof(u32); + std::memcpy(metadata.data() + index, dir_table.data(), dir_table.size()); + index += dir_table.size(); + std::memcpy(metadata.data() + index, file_hash_table.data(), + file_hash_table.size() * sizeof(u32)); + index += file_hash_table.size() * sizeof(u32); + std::memcpy(metadata.data() + index, file_table.data(), file_table.size()); + out.emplace(header.dir_hash_table_ofs, std::make_shared(metadata)); return out; } diff --git a/src/core/file_sys/fsmitm_romfsbuild.h b/src/core/file_sys/fsmitm_romfsbuild.h new file mode 100644 index 000000000..b0c3c123b --- /dev/null +++ b/src/core/file_sys/fsmitm_romfsbuild.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * Adapted by DarkLordZach for use/interaction with yuzu + * + * Modifications Copyright 2018 yuzu emulator team + * Licensed under GPLv2 or any later version + * Refer to the license.txt file included. + */ + +#pragma once + +#include +#include +#include +#include +#include "common/common_types.h" +#include "core/file_sys/vfs.h" + +namespace FileSys { + +struct RomFSBuildDirectoryContext; +struct RomFSBuildFileContext; +struct RomFSDirectoryEntry; +struct RomFSFileEntry; + +class RomFSBuildContext { +public: + explicit RomFSBuildContext(VirtualDir base); + ~RomFSBuildContext(); + + // This finalizes the context. + std::map Build(); + +private: + VirtualDir base; + std::shared_ptr root; + std::map, std::less<>> directories; + std::map, std::less<>> files; + u64 num_dirs = 0; + u64 num_files = 0; + u64 dir_table_size = 0; + u64 file_table_size = 0; + u64 dir_hash_table_size = 0; + u64 file_hash_table_size = 0; + u64 file_partition_size = 0; + + void VisitDirectory(VirtualDir filesys, std::shared_ptr parent); + + bool AddDirectory(std::shared_ptr parent_dir_ctx, + std::shared_ptr dir_ctx); + bool AddFile(std::shared_ptr parent_dir_ctx, + std::shared_ptr file_ctx); +}; + +} // namespace FileSys diff --git a/src/core/file_sys/fsmitm_romfsbuild.hpp b/src/core/file_sys/fsmitm_romfsbuild.hpp deleted file mode 100644 index b897aab21..000000000 --- a/src/core/file_sys/fsmitm_romfsbuild.hpp +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2018 Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -/* - * Adapted by DarkLordZach for use/interaction with yuzu - * - * Modifications Copyright 2018 yuzu emulator team - * Licensed under GPLv2 or any later version - * Refer to the license.txt file included. - */ - -#pragma once -#include -#include -#include "common/common_types.h" -#include "vfs.h" - -namespace FileSys { - -/* Used as comparator for std::map */ -struct build_ctx_cmp { - bool operator()(const char* a, const char* b) const { - return strcmp(a, b) < 0; - } -}; - -struct RomFSDirectoryEntry; -struct RomFSFileEntry; -struct RomFSBuildDirectoryContext; -struct RomFSBuildFileContext; - -class RomFSBuildContext { -private: - VirtualDir base; - RomFSBuildDirectoryContext* root; - std::map directories; - std::map files; - u64 num_dirs = 0; - u64 num_files = 0; - u64 dir_table_size = 0; - u64 file_table_size = 0; - u64 dir_hash_table_size = 0; - u64 file_hash_table_size = 0; - u64 file_partition_size = 0; - - void VisitDirectory(VirtualDir filesys, RomFSBuildDirectoryContext* parent); - - bool AddDirectory(RomFSBuildDirectoryContext* parent_dir_ctx, - RomFSBuildDirectoryContext* dir_ctx, - RomFSBuildDirectoryContext** out_dir_ctx); - bool AddFile(RomFSBuildDirectoryContext* parent_dir_ctx, RomFSBuildFileContext* file_ctx); - -public: - explicit RomFSBuildContext(VirtualDir base); - - /* This finalizes the context. */ - std::map Build(); -}; - -static inline RomFSDirectoryEntry* romfs_get_direntry(void* directories, uint32_t offset) { - return (RomFSDirectoryEntry*)((uintptr_t)directories + offset); -} - -static inline RomFSFileEntry* romfs_get_fentry(void* files, uint32_t offset) { - return (RomFSFileEntry*)((uintptr_t)files + offset); -} - -static inline uint32_t romfs_calc_path_hash(uint32_t parent, const unsigned char* path, - uint32_t start, size_t path_len) { - uint32_t hash = parent ^ 123456789; - for (uint32_t i = 0; i < path_len; i++) { - hash = (hash >> 5) | (hash << 27); - hash ^= path[start + i]; - } - - return hash; -} - -static inline uint32_t romfs_get_hash_table_count(uint32_t num_entries) { - if (num_entries < 3) { - return 3; - } else if (num_entries < 19) { - return num_entries | 1; - } - uint32_t count = num_entries; - while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 || - count % 11 == 0 || count % 13 == 0 || count % 17 == 0) { - count++; - } - return count; -} - -} // namespace FileSys diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index 74a0acf1a..af3f9a78f 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -68,6 +68,42 @@ VirtualDir PatchManager::PatchExeFS(VirtualDir exefs) const { return exefs; } +static void ApplyLayeredFS(VirtualFile& romfs, u64 title_id, ContentRecordType type) { + const auto load_dir = Service::FileSystem::GetModificationLoadRoot(title_id); + if (type == ContentRecordType::Program && load_dir != nullptr && load_dir->GetSize() > 0) { + auto extracted = ExtractRomFS(romfs); + + if (extracted != nullptr) { + auto patch_dirs = load_dir->GetSubdirectories(); + std::sort(patch_dirs.begin(), patch_dirs.end(), + [](const VirtualDir& l, const VirtualDir& r) { + return l->GetName() < r->GetName(); + }); + + std::vector layers; + layers.reserve(patch_dirs.size() + 1); + for (const auto& subdir : patch_dirs) { + auto romfs_dir = subdir->GetSubdirectory("romfs"); + if (romfs_dir != nullptr) + layers.push_back(std::move(romfs_dir)); + } + + layers.push_back(std::move(extracted)); + + const auto layered = LayerDirectories(layers); + + if (layered != nullptr) { + auto packed = CreateRomFS(layered); + + if (packed != nullptr) { + LOG_INFO(Loader, " RomFS: LayeredFS patches applied successfully"); + romfs = std::move(packed); + } + } + } + } +} + VirtualFile PatchManager::PatchRomFS(VirtualFile romfs, u64 ivfc_offset, ContentRecordType type) const { LOG_INFO(Loader, "Patching RomFS for title_id={:016X}, type={:02X}", title_id, @@ -92,39 +128,7 @@ VirtualFile PatchManager::PatchRomFS(VirtualFile romfs, u64 ivfc_offset, } // LayeredFS - const auto load_dir = Service::FileSystem::GetModificationLoadRoot(title_id); - if (type == ContentRecordType::Program && load_dir != nullptr && load_dir->GetSize() > 0) { - const auto extracted = ExtractRomFS(romfs); - - if (extracted != nullptr) { - auto patch_dirs = load_dir->GetSubdirectories(); - std::sort(patch_dirs.begin(), patch_dirs.end(), - [](const VirtualDir& l, const VirtualDir& r) { - return l->GetName() < r->GetName(); - }); - - std::vector layers; - layers.reserve(patch_dirs.size() + 1); - for (const auto& subdir : patch_dirs) { - const auto romfs_dir = subdir->GetSubdirectory("romfs"); - if (romfs_dir != nullptr) - layers.push_back(romfs_dir); - } - - layers.push_back(extracted); - - const auto layered = LayerDirectories(layers); - - if (layered != nullptr) { - const auto packed = CreateRomFS(layered); - - if (packed != nullptr) { - LOG_INFO(Loader, " RomFS: LayeredFS patches applied successfully"); - romfs = packed; - } - } - } - } + ApplyLayeredFS(romfs, title_id, type); return romfs; } @@ -153,7 +157,7 @@ std::map PatchManager::GetPatchVersionNames() const { const auto lfs_dir = Service::FileSystem::GetModificationLoadRoot(title_id); if (lfs_dir != nullptr && lfs_dir->GetSize() > 0) - out[PatchType::LayeredFS] = ""; + out.insert_or_assign(PatchType::LayeredFS, ""); return out; } diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index 0dda0b861..653ef2e7b 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -18,6 +18,10 @@ #include "core/loader/loader.h" namespace FileSys { + +// The size of blocks to use when vfs raw copying into nand. +constexpr size_t VFS_RC_LARGE_COPY_BLOCK = 0x400000; + std::string RegisteredCacheEntry::DebugInfo() const { return fmt::format("title_id={:016X}, content_type={:02X}", title_id, static_cast(type)); } @@ -121,7 +125,7 @@ VirtualFile RegisteredCache::OpenFileOrDirectoryConcat(const VirtualDir& dir, if (concat.empty()) return nullptr; - file = FileSys::ConcatenateFiles(concat); + file = FileSys::ConcatenateFiles(concat, concat.front()->GetName()); } return file; @@ -480,7 +484,8 @@ InstallResult RegisteredCache::RawInstallNCA(std::shared_ptr nca, const Vfs auto out = dir->CreateFileRelative(path); if (out == nullptr) return InstallResult::ErrorCopyFailed; - return copy(in, out, 0x400000) ? InstallResult::Success : InstallResult::ErrorCopyFailed; + return copy(in, out, VFS_RC_LARGE_COPY_BLOCK) ? InstallResult::Success + : InstallResult::ErrorCopyFailed; } bool RegisteredCache::RawInstallYuzuMeta(const CNMT& cnmt) { diff --git a/src/core/file_sys/romfs.cpp b/src/core/file_sys/romfs.cpp index 71e4e0e2f..205284a4d 100644 --- a/src/core/file_sys/romfs.cpp +++ b/src/core/file_sys/romfs.cpp @@ -4,7 +4,7 @@ #include "common/common_types.h" #include "common/swap.h" -#include "core/file_sys/fsmitm_romfsbuild.hpp" +#include "core/file_sys/fsmitm_romfsbuild.h" #include "core/file_sys/romfs.h" #include "core/file_sys/vfs.h" #include "core/file_sys/vfs_concat.h" @@ -100,7 +100,7 @@ void ProcessDirectory(VirtualFile file, std::size_t dir_offset, std::size_t file } } -VirtualDir ExtractRomFS(VirtualFile file, bool traverse_into_data) { +VirtualDir ExtractRomFS(VirtualFile file, RomFSExtractionType type) { RomFSHeader header{}; if (file->ReadObject(&header) != sizeof(RomFSHeader)) return nullptr; @@ -119,8 +119,9 @@ VirtualDir ExtractRomFS(VirtualFile file, bool traverse_into_data) { VirtualDir out = std::move(root); - while (out->GetSubdirectories().size() == 1 && out->GetFiles().size() == 0) { - if (out->GetSubdirectories().front()->GetName() == "data" && !traverse_into_data) + while (out->GetSubdirectories().size() == 1 && out->GetFiles().empty()) { + if (out->GetSubdirectories().front()->GetName() == "data" && + type == RomFSExtractionType::Truncated) break; out = out->GetSubdirectories().front(); } diff --git a/src/core/file_sys/romfs.h b/src/core/file_sys/romfs.h index 8e82585a0..ecd1eb725 100644 --- a/src/core/file_sys/romfs.h +++ b/src/core/file_sys/romfs.h @@ -32,9 +32,15 @@ struct IVFCHeader { }; static_assert(sizeof(IVFCHeader) == 0xE0, "IVFCHeader has incorrect size."); +enum class RomFSExtractionType { + Full, // Includes data directory + Truncated, // Traverses into data directory +}; + // Converts a RomFS binary blob to VFS Filesystem // Returns nullptr on failure -VirtualDir ExtractRomFS(VirtualFile file, bool traverse_into_data = true); +VirtualDir ExtractRomFS(VirtualFile file, + RomFSExtractionType type = RomFSExtractionType::Truncated); // Converts a VFS filesystem into a RomFS binary // Returns nullptr on failure diff --git a/src/core/file_sys/vfs.cpp b/src/core/file_sys/vfs.cpp index 218cfde66..5fbea1739 100644 --- a/src/core/file_sys/vfs.cpp +++ b/src/core/file_sys/vfs.cpp @@ -399,8 +399,8 @@ bool VfsDirectory::Copy(std::string_view src, std::string_view dest) { return f2->WriteBytes(f1->ReadAllBytes()) == f1->GetSize(); } -std::map VfsDirectory::GetEntries() const { - std::map out; +std::map> VfsDirectory::GetEntries() const { + std::map> out; for (const auto& dir : GetSubdirectories()) out.emplace(dir->GetName(), VfsEntryType::Directory); for (const auto& file : GetFiles()) diff --git a/src/core/file_sys/vfs.h b/src/core/file_sys/vfs.h index 6aec4c164..cea4aa8b8 100644 --- a/src/core/file_sys/vfs.h +++ b/src/core/file_sys/vfs.h @@ -268,7 +268,7 @@ public: // Gets all of the entries directly in the directory (files and dirs), returning a map between // item name -> type. - virtual std::map GetEntries() const; + virtual std::map> GetEntries() const; // Interprets the file with name file instead as a directory of type directory. // The directory must have a constructor that takes a single argument of type @@ -323,6 +323,9 @@ bool DeepEquals(const VirtualFile& file1, const VirtualFile& file2, size_t block // directory of src/dest. bool VfsRawCopy(const VirtualFile& src, const VirtualFile& dest, size_t block_size = 0x1000); +// A method that performs a similar function to VfsRawCopy above, but instead copies entire +// directories. It suffers the same performance penalties as above and an implementation-specific +// Copy should always be preferred. bool VfsRawCopyD(const VirtualDir& src, const VirtualDir& dest, size_t block_size = 0x1000); // Checks if the directory at path relative to rel exists. If it does, returns that. If it does not diff --git a/src/core/file_sys/vfs_concat.cpp b/src/core/file_sys/vfs_concat.cpp index 0c07e162e..d9f9911da 100644 --- a/src/core/file_sys/vfs_concat.cpp +++ b/src/core/file_sys/vfs_concat.cpp @@ -10,8 +10,9 @@ namespace FileSys { -bool VerifyConcatenationMap(std::map map) { - for (auto iter = map.begin(); iter != --map.end();) { +static bool VerifyConcatenationMapContinuity(const std::map& map) { + const auto last_valid = --map.end(); + for (auto iter = map.begin(); iter != last_valid;) { const auto old = iter++; if (old->first + old->second->GetSize() != iter->first) { return false; @@ -41,9 +42,11 @@ ConcatenatedVfsFile::ConcatenatedVfsFile(std::vector files_, std::s ConcatenatedVfsFile::ConcatenatedVfsFile(std::map files_, std::string name) : files(std::move(files_)), name(std::move(name)) { - ASSERT(VerifyConcatenationMap(files)); + ASSERT(VerifyConcatenationMapContinuity(files)); } +ConcatenatedVfsFile::~ConcatenatedVfsFile() = default; + std::string ConcatenatedVfsFile::GetName() const { if (files.empty()) return ""; @@ -77,25 +80,25 @@ bool ConcatenatedVfsFile::IsReadable() const { } std::size_t ConcatenatedVfsFile::Read(u8* data, std::size_t length, std::size_t offset) const { - std::pair entry = *files.rbegin(); + auto entry = --files.end(); for (auto iter = files.begin(); iter != files.end(); ++iter) { if (iter->first > offset) { - entry = *--iter; + entry = --iter; break; } } - if (entry.first + entry.second->GetSize() <= offset) + if (entry->first + entry->second->GetSize() <= offset) return 0; const auto read_in = - std::min(entry.first + entry.second->GetSize() - offset, entry.second->GetSize()); + std::min(entry->first + entry->second->GetSize() - offset, entry->second->GetSize()); if (length > read_in) { - return entry.second->Read(data, read_in, offset - entry.first) + + return entry->second->Read(data, read_in, offset - entry->first) + Read(data + read_in, length - read_in, offset + read_in); } - return entry.second->Read(data, std::min(read_in, length), offset - entry.first); + return entry->second->Read(data, std::min(read_in, length), offset - entry->first); } std::size_t ConcatenatedVfsFile::Write(const u8* data, std::size_t length, std::size_t offset) { diff --git a/src/core/file_sys/vfs_concat.h b/src/core/file_sys/vfs_concat.h index c65c20d15..76211d38a 100644 --- a/src/core/file_sys/vfs_concat.h +++ b/src/core/file_sys/vfs_concat.h @@ -13,35 +13,6 @@ namespace FileSys { -class ConcatenatedVfsFile; - -// Wrapper function to allow for more efficient handling of files.size() == 0, 1 cases. -VirtualFile ConcatenateFiles(std::vector files, std::string name = ""); - -// Convenience function that turns a map of offsets to files into a concatenated file, filling gaps -// with template parameter. -template -VirtualFile ConcatenateFiles(std::map files, std::string name = "") { - if (files.empty()) - return nullptr; - if (files.size() == 1) - return files.begin()->second; - - for (auto iter = files.begin(); iter != --files.end();) { - const auto old = iter++; - if (old->first + old->second->GetSize() != iter->first) { - files.emplace(old->first + old->second->GetSize(), - std::make_shared>(iter->first - old->first - - old->second->GetSize())); - } - } - - if (files.begin()->first != 0) - files.emplace(0, std::make_shared>(files.begin()->first)); - - return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); -} - // Class that wraps multiple vfs files and concatenates them, making reads seamless. Currently // read-only. class ConcatenatedVfsFile : public VfsFile { @@ -72,4 +43,33 @@ private: std::string name; }; +// Wrapper function to allow for more efficient handling of files.size() == 0, 1 cases. +VirtualFile ConcatenateFiles(std::vector files, std::string name); + +// Convenience function that turns a map of offsets to files into a concatenated file, filling gaps +// with template parameter. +template +VirtualFile ConcatenateFiles(std::map files, std::string name) { + if (files.empty()) + return nullptr; + if (files.size() == 1) + return files.begin()->second; + + const auto last_valid = --files.end(); + for (auto iter = files.begin(); iter != last_valid;) { + const auto old = iter++; + if (old->first + old->second->GetSize() != iter->first) { + files.emplace(old->first + old->second->GetSize(), + std::make_shared>(iter->first - old->first - + old->second->GetSize())); + } + } + + // Ensure the map starts at offset 0 (start of file), otherwise pad to fill. + if (files.begin()->first != 0) + files.emplace(0, std::make_shared>(files.begin()->first)); + + return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); +} + } // namespace FileSys diff --git a/src/core/file_sys/vfs_layered.cpp b/src/core/file_sys/vfs_layered.cpp index 802f49384..45563d7ae 100644 --- a/src/core/file_sys/vfs_layered.cpp +++ b/src/core/file_sys/vfs_layered.cpp @@ -20,10 +20,11 @@ VirtualDir LayerDirectories(std::vector dirs, std::string name) { LayeredVfsDirectory::LayeredVfsDirectory(std::vector dirs, std::string name) : dirs(std::move(dirs)), name(std::move(name)) {} +LayeredVfsDirectory::~LayeredVfsDirectory() = default; + std::shared_ptr LayeredVfsDirectory::GetFileRelative(std::string_view path) const { - VirtualFile file; for (const auto& layer : dirs) { - file = layer->GetFileRelative(path); + const auto file = layer->GetFileRelative(path); if (file != nullptr) return file; } @@ -35,12 +36,12 @@ std::shared_ptr LayeredVfsDirectory::GetDirectoryRelative( std::string_view path) const { std::vector out; for (const auto& layer : dirs) { - const auto dir = layer->GetDirectoryRelative(path); + auto dir = layer->GetDirectoryRelative(path); if (dir != nullptr) - out.push_back(dir); + out.push_back(std::move(dir)); } - return LayerDirectories(out); + return LayerDirectories(std::move(out)); } std::shared_ptr LayeredVfsDirectory::GetFile(std::string_view name) const { @@ -61,8 +62,9 @@ std::vector> LayeredVfsDirectory::GetFiles() const { for (const auto& file : layer->GetFiles()) { if (std::find_if(out.begin(), out.end(), [&file](const VirtualFile& comp) { return comp->GetName() == file->GetName(); - }) == out.end()) + }) == out.end()) { out.push_back(file); + } } } @@ -79,6 +81,7 @@ std::vector> LayeredVfsDirectory::GetSubdirectorie } std::vector out; + out.reserve(names.size()); for (const auto& subdir : names) out.push_back(GetSubdirectory(subdir)); diff --git a/src/core/file_sys/vfs_layered.h b/src/core/file_sys/vfs_layered.h index f345c2fb6..4f6e341ab 100644 --- a/src/core/file_sys/vfs_layered.h +++ b/src/core/file_sys/vfs_layered.h @@ -21,6 +21,8 @@ class LayeredVfsDirectory : public VfsDirectory { LayeredVfsDirectory(std::vector dirs, std::string name); public: + ~LayeredVfsDirectory() override; + std::shared_ptr GetFileRelative(std::string_view path) const override; std::shared_ptr GetDirectoryRelative(std::string_view path) const override; std::shared_ptr GetFile(std::string_view name) const override; diff --git a/src/core/file_sys/vfs_real.cpp b/src/core/file_sys/vfs_real.cpp index a58a02de7..9defad04c 100644 --- a/src/core/file_sys/vfs_real.cpp +++ b/src/core/file_sys/vfs_real.cpp @@ -413,11 +413,11 @@ std::string RealVfsDirectory::GetFullPath() const { return out; } -std::map RealVfsDirectory::GetEntries() const { +std::map> RealVfsDirectory::GetEntries() const { if (perms == Mode::Append) return {}; - std::map out; + std::map> out; FileUtil::ForeachDirectoryEntry( nullptr, path, [&out](u64* entries_out, const std::string& directory, const std::string& filename) { diff --git a/src/core/file_sys/vfs_real.h b/src/core/file_sys/vfs_real.h index 3af2a6961..5b61db90d 100644 --- a/src/core/file_sys/vfs_real.h +++ b/src/core/file_sys/vfs_real.h @@ -98,7 +98,7 @@ public: bool DeleteFile(std::string_view name) override; bool Rename(std::string_view name) override; std::string GetFullPath() const override; - std::map GetEntries() const override; + std::map> GetEntries() const override; protected: bool ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) override; diff --git a/src/core/file_sys/vfs_static.h b/src/core/file_sys/vfs_static.h index 5bc8ca52e..4dd47ffcc 100644 --- a/src/core/file_sys/vfs_static.h +++ b/src/core/file_sys/vfs_static.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include @@ -15,7 +16,7 @@ template class StaticVfsFile : public VfsFile { public: explicit StaticVfsFile(size_t size = 0, std::string name = "", VirtualDir parent = nullptr) - : size(size), name(name), parent(parent) {} + : size(size), name(std::move(name)), parent(std::move(parent)) {} std::string GetName() const override { return name; diff --git a/src/core/file_sys/vfs_vector.cpp b/src/core/file_sys/vfs_vector.cpp index efca3d4ad..7033e2c88 100644 --- a/src/core/file_sys/vfs_vector.cpp +++ b/src/core/file_sys/vfs_vector.cpp @@ -3,6 +3,7 @@ // Refer to the license.txt file included. #include +#include #include #include "core/file_sys/vfs_vector.h" @@ -10,6 +11,8 @@ namespace FileSys { VectorVfsFile::VectorVfsFile(std::vector initial_data, std::string name, VirtualDir parent) : data(std::move(initial_data)), name(std::move(name)), parent(std::move(parent)) {} +VectorVfsFile::~VectorVfsFile() = default; + std::string VectorVfsFile::GetName() const { return name; } @@ -20,7 +23,7 @@ size_t VectorVfsFile::GetSize() const { bool VectorVfsFile::Resize(size_t new_size) { data.resize(new_size); - return data.size() == new_size; + return true; } std::shared_ptr VectorVfsFile::GetContainingDirectory() const { @@ -55,7 +58,7 @@ bool VectorVfsFile::Rename(std::string_view name_) { } void VectorVfsFile::Assign(std::vector new_data) { - data = new_data; + data = std::move(new_data); } VectorVfsDirectory::VectorVfsDirectory(std::vector files_, diff --git a/src/core/file_sys/vfs_vector.h b/src/core/file_sys/vfs_vector.h index c84e137a9..115c3ae95 100644 --- a/src/core/file_sys/vfs_vector.h +++ b/src/core/file_sys/vfs_vector.h @@ -13,6 +13,7 @@ class VectorVfsFile : public VfsFile { public: explicit VectorVfsFile(std::vector initial_data = {}, std::string name = "", VirtualDir parent = nullptr); + ~VectorVfsFile() override; std::string GetName() const override; size_t GetSize() const override; diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index 7cfe8a32f..dc8b5407d 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -784,7 +784,7 @@ void GMainWindow::OnGameListDumpRomFS(u64 program_id, const std::string& game_pa ? FileSys::PatchManager(program_id).PatchRomFS(file, loader->ReadRomFSIVFCOffset()) : file; - const auto extracted = FileSys::ExtractRomFS(romfs, false); + const auto extracted = FileSys::ExtractRomFS(romfs, FileSys::RomFSExtractionType::Full); if (extracted == nullptr) { failed(); return;