diff --git a/CMakeLists.txt b/CMakeLists.txt index e0e07c7..379bb0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -359,6 +359,7 @@ if(PSPRECOMP_BUILD_TESTS) "${PORTABLEKIT_ROOT}/tests/hle_extension_tests.cpp" "${PORTABLEKIT_ROOT}/tests/hle_extension_test_host.cpp" "${PORTABLEKIT_ROOT}/host/hle/hle_extensions.cpp" + "${PORTABLEKIT_ROOT}/host/extension_keys.cpp" "${PORTABLEKIT_ROOT}/examples/hle_extension/example_extension.cpp") target_include_directories(portablekit_hle_extension_tests PRIVATE "${PORTABLEKIT_ROOT}/host") target_link_libraries(portablekit_hle_extension_tests PRIVATE psprecomp_core) @@ -407,3 +408,24 @@ if(PSPRECOMP_BUILD_TESTS) endif() set_target_properties(portablekit_host_check PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") endif() + +# A read filter on an open file, through the real IoFileMgrForUser handlers: +# the system layer the check above builds, with a test's main. +if(PSPRECOMP_BUILD_TESTS) + set(_io_test_sources ${_check_sources}) + list(REMOVE_ITEM _io_test_sources "${PORTABLEKIT_ROOT}/host/main.cpp") + add_executable(portablekit_io_filter_tests "${PORTABLEKIT_ROOT}/tests/io_filter_tests.cpp" ${_io_test_sources}) + target_compile_features(portablekit_io_filter_tests PRIVATE cxx_std_20) + target_compile_definitions(portablekit_io_filter_tests PRIVATE + PORTABLEKIT_NIDS_CSV="${PORTABLEKIT_ROOT}/configs/nids.csv") + target_include_directories(portablekit_io_filter_tests PRIVATE + "${PORTABLEKIT_ROOT}/host" "${PORTABLEKIT_ROOT}/third_party" + "${CMAKE_CURRENT_BINARY_DIR}/host_check_nids") + target_link_libraries(portablekit_io_filter_tests PRIVATE psprecomp_core Threads::Threads ${CMAKE_DL_LIBS}) + _portablekit_link_hle_extensions(portablekit_io_filter_tests) + if(WIN32) + target_link_libraries(portablekit_io_filter_tests PRIVATE ws2_32 iphlpapi) + endif() + set_target_properties(portablekit_io_filter_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") + add_test(NAME portablekit_io_filter_tests COMMAND portablekit_io_filter_tests) +endif() diff --git a/apps/portablekit/src/app_main.cpp b/apps/portablekit/src/app_main.cpp index 8ee451b..962312e 100644 --- a/apps/portablekit/src/app_main.cpp +++ b/apps/portablekit/src/app_main.cpp @@ -424,13 +424,16 @@ int command_keys(const Arguments &args) { if (g_json) { std::vector problems; for (const std::string &problem : report.problems) problems.push_back(Json().field("problem", problem)); + std::vector warnings; + for (const std::string &warning : report.warnings) warnings.push_back(Json().field("warning", warning)); std::cout << Json().field("ok", ok).field("path", path_text(keys_file_path())) .field("executables", report.can_decrypt_executables) .field("saves", report.can_encrypt_saves) .field("tags", static_cast(report.tag_count)) - .array("problems", problems).str() << "\n"; + .array("problems", problems).array("warnings", warnings).str() << "\n"; } else { for (const std::string &problem : report.problems) std::cerr << " " << problem << "\n"; + for (const std::string &warning : report.warnings) std::cerr << " warning: " << warning << "\n"; if (ok) std::cout << "Keys saved to " << path_text(keys_file_path()) << "\n"; } return ok ? 0 : kKeysRejected; @@ -440,11 +443,17 @@ int command_keys(const Arguments &args) { if (g_json) { std::vector problems; for (const std::string &problem : report.problems) problems.push_back(Json().field("problem", problem)); + std::vector warnings; + for (const std::string &warning : report.warnings) warnings.push_back(Json().field("warning", warning)); + std::vector declared; + for (const KeysReport::DeclaredStatus &key : report.declared) + declared.push_back(Json().field("name", key.name).field("module", key.module).field("present", key.present)); std::cout << Json().field("ok", true).field("path", path_text(report.path)).field("found", report.file_found) .field("executables", report.can_decrypt_executables) .field("saves", report.can_encrypt_saves) .field("tags", static_cast(report.tag_count)) - .array("problems", problems).str() << "\n"; + .array("problems", problems).array("warnings", warnings) + .array("extension_keys", declared).str() << "\n"; return 0; } std::cout << "Keys file: " << path_text(report.path) << (report.file_found ? "" : " (none)") << "\n" @@ -452,6 +461,16 @@ int command_keys(const Arguments &args) { << " encrypt saves: " << (report.can_encrypt_saves ? "yes" : "no (saves are kept unencrypted)") << "\n" << " tag keys: " << report.tag_count << "\n"; for (const std::string &problem : report.problems) std::cout << " problem: " << problem << "\n"; + for (const std::string &warning : report.warnings) std::cout << " warning: " << warning << "\n"; + // The keys HLE extension modules read, by module. + std::string module; + for (const KeysReport::DeclaredStatus &key : report.declared) { + if (key.module != module) { + module = key.module; + std::cout << " keys " << module << " reads:\n"; + } + std::cout << " " << key.name << ": " << (key.present ? "present" : "missing") << "\n"; + } return 0; } diff --git a/apps/portablekit/src/keys_file.cpp b/apps/portablekit/src/keys_file.cpp index fbfaa44..4f48340 100644 --- a/apps/portablekit/src/keys_file.cpp +++ b/apps/portablekit/src/keys_file.cpp @@ -3,10 +3,13 @@ #include "app_home.hpp" #include "text_format.hpp" +#include "extension_keys.hpp" #include "psprecomp/sha256.hpp" #include #include +#include +#include namespace portablekit::app { namespace { @@ -51,14 +54,55 @@ Key16 to_key(const std::vector &bytes) { return key; } +// What the reader says of a name it does not use, and what that becomes once +// HLE extension modules had their say: nothing for a name one declares, a +// warning for the rest (another program's keys may share the file). +constexpr std::string_view kNotUsed = " is not a key this program uses."; + +void sort_out_unused_names(KeysReport &report, const std::set &declared_found) { + std::vector kept; + for (std::string &problem : report.problems) { + if (!problem.ends_with(kNotUsed)) { + kept.push_back(std::move(problem)); + continue; + } + const std::string written = problem.substr(0, problem.size() - kNotUsed.size()); + if (declared_found.contains(lower(written))) continue; + report.warnings.push_back(written + " is not a key this program or its extension modules use; it is ignored."); + } + report.problems = std::move(kept); +} + +// The keys HLE extension modules declare, and whether the file gave them. +void list_declared(KeysReport &report) { + for (const DeclaredKey &key : declared_extension_keys()) + report.declared.push_back({key.name, key.module, key_value(&report.keys, key.name).has_value()}); +} + +// Copies a checked keys file to /keys.txt. +bool copy_to_home(const std::filesystem::path &source, KeysReport &report) { + std::error_code ec; + std::filesystem::create_directories(keys_file_path().parent_path(), ec); + std::filesystem::copy_file(source, keys_file_path(), std::filesystem::copy_options::overwrite_existing, ec); + if (ec) { + report.problems.push_back("Cannot copy it to " + path_text(keys_file_path()) + ": " + ec.message()); + return false; + } + return true; +} + } // namespace KeysReport read_keys_file(const std::filesystem::path &path) { KeysReport report; report.path = path; std::error_code ec; - if (!std::filesystem::is_regular_file(path, ec)) return report; + if (!std::filesystem::is_regular_file(path, ec)) { + list_declared(report); + return report; + } report.file_found = true; + std::set declared_found; // names a module declares, kept as its keys const KeyValues values = read_key_values(path); for (const auto &[raw_name, value] : values) { const std::string name = lower(raw_name); @@ -86,6 +130,25 @@ KeysReport read_keys_file(const std::filesystem::path &path) { report.keys.tags[static_cast(tag)] = std::move(entry); continue; } + // A key an HLE extension module declares (extension_keys.hpp), unless + // it is one this program checks itself: that check stays. + if (const DeclaredKey *declared = find_declared_key(declared_extension_keys(), name); + declared != nullptr && expected_fingerprint(name) == nullptr) { + if (std::string problem = check_declared_key(*declared, *bytes, raw_name); !problem.empty()) { + report.problems.push_back(std::move(problem)); + continue; + } + store_named_key(report.keys, name, *bytes); + declared_found.insert(name); + // A 16-byte one goes on, so a name this program also reads in its + // own way is kept that way as well. + if (bytes->size() != 16u) continue; + } else if (bytes->size() != 16u && expected_fingerprint(name) == nullptr && !name.starts_with("kirk.") && + !name.starts_with("savedata.")) { + // Not a name anything here reads: a warning, whatever its length. + report.warnings.push_back(raw_name + " is not a key this program or its extension modules use; it is ignored."); + continue; + } if (bytes->size() != 16u) { report.problems.push_back(raw_name + " must be 16 bytes."); continue; @@ -108,6 +171,8 @@ KeysReport read_keys_file(const std::filesystem::path &path) { report.keys.savedata[std::atoi(name.substr(9).c_str())] = to_key(*bytes); } } + sort_out_unused_names(report, declared_found); + list_declared(report); report.tag_count = report.keys.tags.size(); report.can_decrypt_executables = report.keys.kirk(0x5Du) != nullptr && report.keys.kirk_cmd1.has_value(); report.can_encrypt_saves = true; @@ -135,6 +200,7 @@ bool import_keys_file(const std::filesystem::path &source, KeysReport &report) { return false; } if (!report.problems.empty()) return false; + if (!report.keys.named.empty()) return copy_to_home(source, report); // keys only modules use if (report.keys.kirk_aes.empty() && !report.keys.kirk_cmd1 && report.keys.savedata.empty() && report.keys.tags.empty()) { report.problems.push_back("The file holds no keys."); @@ -157,10 +223,13 @@ namespace portablekit { // The framework asks here for every key it uses (host/crypto_keys.hpp). const CryptoKeys *crypto_keys() { const app::KeysReport &report = app::active_keys(); + if (!report.keys.named.empty()) return &report.keys; if (report.keys.kirk_aes.empty() && !report.keys.kirk_cmd1 && report.keys.savedata.empty() && report.keys.tags.empty()) return nullptr; return &report.keys; } +bool keys_come_from_keys_file() { return true; } + } // namespace portablekit diff --git a/apps/portablekit/src/keys_file.hpp b/apps/portablekit/src/keys_file.hpp index 51d6290..c1d5a26 100644 --- a/apps/portablekit/src/keys_file.hpp +++ b/apps/portablekit/src/keys_file.hpp @@ -12,6 +12,11 @@ // tag.C0CB167C = <16 or 0x90 bytes> what an executable's tag selects // tag.C0CB167C.slot = 5D (optional) the KIRK slot of that tag // +// HLE extension modules may declare more names (Registry::declare_key), each +// with its length and optionally a fingerprint; they are read and checked the +// same way. A name nothing declares is a warning and is left out, but does not +// refuse the file. +// // The fixed keys are checked against SHA-256 fingerprints compiled into the // program, so a mistyped key is named instead of producing garbage; a // fingerprint does not reveal the key. Tag keys have no fingerprint: an @@ -30,6 +35,16 @@ struct KeysReport { std::filesystem::path path; CryptoKeys keys; std::vector problems; // one sentence each + // Names nothing reads (not this program, not an extension module): kept + // out, but they do not refuse the file. + std::vector warnings; + // The keys HLE extension modules declare, and whether the file has them. + struct DeclaredStatus { + std::string name; + std::string module; + bool present{}; + }; + std::vector declared; // What the keys that passed allow. bool can_decrypt_executables{}; // kirk.aes.5D and kirk.cmd1 bool can_encrypt_saves{}; // every save-data key diff --git a/cmake/PortableKit.cmake b/cmake/PortableKit.cmake index 15206c4..0e9dc36 100644 --- a/cmake/PortableKit.cmake +++ b/cmake/PortableKit.cmake @@ -65,6 +65,7 @@ set(PORTABLEKIT_HOST_SOURCES host/hle/hle_adhoc.cpp host/hle/hle_extensions.cpp host/hle/hle_extension_host.cpp + host/extension_keys.cpp host/adhoc/client.cpp host/adhoc/discovery.cpp host/adhoc/host.cpp diff --git a/docs/DESKTOP_APP.md b/docs/DESKTOP_APP.md index e08ddb7..e1e9e14 100644 --- a/docs/DESKTOP_APP.md +++ b/docs/DESKTOP_APP.md @@ -165,6 +165,7 @@ tag.C0CB167C.slot = 5D # optional: the KIRK slot the tag's header uses - `portablekit keys import ` checks the file and copies it into the data folder. Every fixed key is checked against a SHA-256 fingerprint compiled into the program, so the player is told "`savedata.4` is not the right key" instead of getting garbage later. Tag keys have no fingerprint: an executable that decrypts to an ELF the runtime can load is the check. - What each group enables: `kirk.aes.5D` + `kirk.cmd1` + the game's `tag.*` decrypt that game's `EBOOT.BIN`; the nine save slots and `savedata.2`–`7` encrypt and decrypt saves. `portablekit keys status` says which of these the file provides. - Without keys, a game can still be added when its disc carries an unencrypted `BOOT.BIN` (LocoRoco 2 does), or when the player gives an executable they decrypted (`--executable`). The program never names a source of keys. +- A name the program does not use is a warning ("... is not a key this program or its extension modules use; it is ignored.") and is left out; it does not refuse the file. HLE extension modules the build links may declare more names, which are read and checked like the program's own ([HLE_EXTENSIONS.md](HLE_EXTENSIONS.md#keys)); `keys status` lists them by module. Messages when something is missing, as the prototype prints them: diff --git a/docs/HLE_EXTENSIONS.md b/docs/HLE_EXTENSIONS.md index 7a1f394..9d6f7fb 100644 --- a/docs/HLE_EXTENSIONS.md +++ b/docs/HLE_EXTENSIONS.md @@ -71,6 +71,10 @@ Everything a module may rely on is in `include/portablekit/hle_extension.hpp`. ` | `now_us()` | Emulated time since boot. | | `log_once(key, message)` | Log a line the first time `key` is seen. | | `env(name)` | The port's variable `_`. | +| `Registry::declare_key(name, sha256, size)` | Declares a key the module reads (see [Keys](#keys)). Since interface version 3. | +| `key(name)`, `key_bytes(name)`, `kirk_key(slot)` | The program's keys by their keys-file name: PortableKit's own and declared ones; empty when missing. | +| `read_open_file(fd, offset, bytes)` | The raw bytes of an open guest file, without moving its position. | +| `set_file_filter(fd, filter)` | What the game reads from an open file instead of its bytes (see [Filtered files](#filtered-files)). | Handlers run on the emulation thread, one at a time. A module may include PortableKit's `host/` headers to reach further than this, but those change without notice; a need that comes up twice belongs in the header instead. @@ -114,6 +118,25 @@ PORTABLEKIT_HLE_EXTENSION(my_module) { `previous` must be called with the caller's registers as the handler found them, and as its last act, in place of `finish()`. +## Keys + +A module that needs keys the console's crypto uses declares each name it reads, at registration: + +```cpp +registry.declare_key("vendor.table", "", 0x90); // fingerprint and size optional +``` + +and reads it at run time with `ext::key_bytes("vendor.table")` (`ext::key` for 16-byte keys, `ext::kirk_key(slot)` for `kirk.aes.`). The names are the keys file's, not case-sensitive, and a lookup also finds PortableKit's own keys (`kirk.aes.5D`, `kirk.cmd1`, `savedata.2`, `tag.`). + +- **Desktop app.** The player's keys file accepts a declared name like one of PortableKit's own: it checks the length and, when one is declared, the fingerprint, and a wrong value is a problem that refuses the file on import. A declared `kirk.aes.` is also a KIRK key for PortableKit. A name nothing declares is a warning and is left out, but no longer refuses the file. `portablekit keys status` lists the declared keys by module, present or missing (`--json`: `extension_keys`). A name PortableKit checks itself keeps PortableKit's check. +- **Ports.** PortableKit's own keys are compiled in. Declared keys come from an optional keys file in the same format: `_KEYS`, else `keys.txt` in the data folder (`_DATA_DIR`, else the port's own); names nothing declares are left alone there. Startup says `HLE extension keys: N of M declared keys present ()`, with a line for each wrong one. + +An entry function may be called more than once (the keys are collected before the game starts); it only adds to its registry. + +## Filtered files + +`set_file_filter(fd, filter)` puts a `FileFilter` (its `size()` and `read(offset, destination, length)`) between the game and an open file, such as the plain text of an encrypted container. While it is set, PortableKit's own `sceIoRead`, `sceIoReadAsync`, `sceIoLseek`, `sceIoLseek32` (and their Async forms) and the disc seek ioctl `0x01010005` work on what the filter gives: positions and sizes count its bytes, and asynchronous results come with `sceIoWaitAsync`/`sceIoPollAsync` as usual. The position is left where it was when the filter is set. The filter reads the file's own bytes with `read_open_file`, which a filter does not change. A null filter restores the file; closing it (`sceIoClose`, `sceIoCloseAsync`) drops the filter. The raw disc device (`umd0:` and `sce_lbn` sector files, which count sectors) and directories take no filter. Writing to a filtered file writes the file's own bytes. + ## What the log says ```text @@ -134,7 +157,7 @@ HLE extensions: 1 module linked, ignored (PORTABLEKIT_NO_HLE_EXTENSIONS) ## Tests -`portablekit_hle_extension_tests` builds the example module into a runtime with no kernel and checks a fill-in, an override, a fill-in left to the built-in, two modules adding one function, and the log lines. +`portablekit_io_filter_tests` runs PortableKit's own IoFileMgrForUser handlers on a memory stick folder with a filter: synchronous and asynchronous reads and seeks, the raw bytes, removing it and closing. `portablekit_hle_extension_tests` builds the example module into a runtime with no kernel and checks a fill-in, an override, a fill-in left to the built-in, two modules adding one function, chained overrides, the log lines, and declared keys: collecting them, checking a value, looking keys up and reading a port's keys file. ## Licensing diff --git a/examples/hle_extension/example_extension.cpp b/examples/hle_extension/example_extension.cpp index 8381ddf..66711f1 100644 --- a/examples/hle_extension/example_extension.cpp +++ b/examples/hle_extension/example_extension.cpp @@ -1,5 +1,5 @@ -// An example HLE extension module: one function filled in, one overridden and -// one wrapped (a chained override that passes every call on). +// An example HLE extension module: one function filled in, one overridden, one +// wrapped (a chained override that passes every call on) and one key declared. // Both behave as PortableKit already does, so building it into a program // changes nothing a game can see; it shows the mechanism and is what // tests/hle_extension_tests.cpp checks it with. @@ -32,6 +32,9 @@ void watch_writeback_invalidate(ext::Runtime &rt, ext::AllegrexContext &ctx, con } // namespace PORTABLEKIT_HLE_EXTENSION(example) { + // A key the module would read with ext::key("example.demo"). Its value is + // not secret, it only shows the mechanism: 00112233445566778899AABBCCDDEEFF. + registry.declare_key("example.demo", "a8faed6abbf35c12a4b26e40f6feb19d736d90045c83b9f9a31f638d323e6811"); registry.add("sceHttp", 0x87797BDDu, "sceHttpsLoadDefaultCert", load_default_cert); registry.override_builtin("UtilsForUser", 0x79D1C3FAu, "sceKernelDcacheWritebackAll", writeback_data_cache); registry.override_chained("UtilsForUser", 0xB435DEC5u, "sceKernelDcacheWritebackInvalidateAll", diff --git a/host/crypto_keys.hpp b/host/crypto_keys.hpp index eefca0d..d69bbe7 100644 --- a/host/crypto_keys.hpp +++ b/host/crypto_keys.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include namespace portablekit { @@ -29,6 +30,9 @@ struct CryptoKeys { // The AES key of KIRK command 1 (signed and encrypted blocks), which wraps // an executable's payload key. std::optional kirk_cmd1; + // Keys HLE extension modules declare (extension_keys.hpp), by their + // keys-file name in lower case, of any length. + std::map> named; // Save-data keys 2 to 7, in the order they are usually published. std::map savedata; // What an executable's "~PSP" tag selects: a 16-byte key for the newer @@ -54,6 +58,11 @@ struct CryptoKeys { // by crypto_keys_builtin.cpp, or by a program that loads them from a file. [[nodiscard]] const CryptoKeys *crypto_keys(); +// True when the keys come from the player's keys file (the desktop app) +// rather than being compiled in (a port, which then reads keys HLE extension +// modules declare from a keys file of its own). Defined next to crypto_keys(). +[[nodiscard]] bool keys_come_from_keys_file(); + // True when every key save-data encryption needs is present. [[nodiscard]] bool savedata_keys_available(); diff --git a/host/crypto_keys_builtin.cpp b/host/crypto_keys_builtin.cpp index 140cd41..275700b 100644 --- a/host/crypto_keys_builtin.cpp +++ b/host/crypto_keys_builtin.cpp @@ -33,4 +33,6 @@ const CryptoKeys *crypto_keys() { return &keys; } +bool keys_come_from_keys_file() { return false; } + } // namespace portablekit diff --git a/host/extension_keys.cpp b/host/extension_keys.cpp new file mode 100644 index 0000000..4179f27 --- /dev/null +++ b/host/extension_keys.cpp @@ -0,0 +1,168 @@ +#include "extension_keys.hpp" + +#include "psprecomp/sha256.hpp" + +#include +#include +#include +#include + +namespace portablekit { +namespace { + +std::string lower(std::string_view text) { + std::string out(text); + for (char &c : out) c = static_cast(std::tolower(static_cast(c))); + return out; +} + +std::string trim(std::string_view text) { + const auto first = text.find_first_not_of(" \t\r\n"); + if (first == std::string_view::npos) return {}; + const auto last = text.find_last_not_of(" \t\r\n"); + return std::string(text.substr(first, last - first + 1)); +} + +// Hex digits, spaces allowed between them; empty on anything else. +std::optional> parse_hex(std::string_view text) { + std::string digits; + for (const char c : text) { + if (c == ' ' || c == '\t') continue; + if (!std::isxdigit(static_cast(c))) return std::nullopt; + digits.push_back(c); + } + if (digits.empty() || digits.size() % 2u != 0u) return std::nullopt; + std::vector bytes; + for (std::size_t i = 0; i < digits.size(); i += 2) + bytes.push_back(static_cast(std::strtoul(digits.substr(i, 2).c_str(), nullptr, 16))); + return bytes; +} + +// "kirk.aes." -> slot. +std::optional kirk_slot(std::string_view name) { + constexpr std::string_view kPrefix = "kirk.aes."; + if (!name.starts_with(kPrefix) || name.size() == kPrefix.size() || name.size() > kPrefix.size() + 2u) + return std::nullopt; + const std::string digits(name.substr(kPrefix.size())); + char *end = nullptr; + const unsigned long slot = std::strtoul(digits.c_str(), &end, 16); + if (*end != '\0') return std::nullopt; + return static_cast(slot); +} + +} // namespace + +std::vector collect_declared_keys(std::span modules, + std::vector ¬es) { + std::vector keys; + for (const HleExtensionModule &module : modules) { + hle_extension::Registry registry; + module.entry(registry); + for (const hle_extension::KeyDeclaration &declaration : registry.keys()) { + DeclaredKey key{lower(trim(declaration.name)), lower(trim(declaration.sha256)), declaration.size, + module.title}; + if (key.name.empty() || key.size == 0u || key.name.starts_with("tag.")) { + notes.push_back(std::string(module.title) + " declares a key \"" + declaration.name + + "\" that cannot be one; it is ignored."); + continue; + } + if (const DeclaredKey *earlier = find_declared_key(keys, key.name)) { + if (earlier->sha256 != key.sha256 || earlier->size != key.size) + notes.push_back(std::string(module.title) + " declares " + key.name + " differently from " + + earlier->module + "; " + earlier->module + "'s declaration is used."); + continue; + } + keys.push_back(std::move(key)); + } + } + return keys; +} + +const DeclaredKey *find_declared_key(std::span keys, std::string_view name) { + const std::string wanted = lower(name); + for (const DeclaredKey &key : keys) + if (key.name == wanted) return &key; + return nullptr; +} + +std::string check_declared_key(const DeclaredKey &key, std::span value, + std::string_view written_name) { + if (value.size() != key.size) + return std::string(written_name) + " must be " + std::to_string(key.size) + " bytes (" + key.module + + " declares it)."; + if (!key.sha256.empty() && psprecomp::sha256_bytes(value) != key.sha256) + return std::string(written_name) + " is not the right key: its fingerprint does not match (" + key.module + + " declares it)."; + return {}; +} + +void store_named_key(CryptoKeys &keys, const std::string &name, std::vector value) { + if (const auto slot = kirk_slot(name); slot && value.size() == 16u) { + Key16 key{}; + std::copy(value.begin(), value.end(), key.begin()); + keys.kirk_aes[*slot] = key; + } + keys.named[name] = std::move(value); +} + +std::optional> key_value(const CryptoKeys *keys, std::string_view name) { + if (keys == nullptr) return std::nullopt; + const std::string wanted = lower(trim(name)); + const auto bytes = [](const Key16 &key) { return std::vector(key.begin(), key.end()); }; + if (const auto found = keys->named.find(wanted); found != keys->named.end()) return found->second; + if (const auto slot = kirk_slot(wanted)) { + if (const Key16 *key = keys->kirk(*slot)) return bytes(*key); + return std::nullopt; + } + if (wanted == "kirk.cmd1") { + if (keys->kirk_cmd1) return bytes(*keys->kirk_cmd1); + return std::nullopt; + } + if (wanted.starts_with("savedata.")) { + char *end = nullptr; + const long index = std::strtol(wanted.c_str() + 9, &end, 10); + if (*end == '\0') + if (const Key16 *key = keys->savedata_key(static_cast(index))) return bytes(*key); + return std::nullopt; + } + if (wanted.starts_with("tag.")) { + char *end = nullptr; + const unsigned long tag = std::strtoul(wanted.c_str() + 4, &end, 16); + if (*end != '\0') return std::nullopt; + const auto found = keys->tags.find(static_cast(tag)); + if (found == keys->tags.end()) return std::nullopt; + if (found->second.key) return bytes(*found->second.key); + if (!found->second.table.empty()) return found->second.table; + } + return std::nullopt; +} + +DeclaredKeysFile read_declared_keys_file(const std::filesystem::path &path, std::span declared) { + DeclaredKeysFile file; + file.path = path; + std::ifstream in(path); + if (!in) return file; + file.found = true; + std::string line; + while (std::getline(in, line)) { + if (const auto hash = line.find('#'); hash != std::string::npos) line.erase(hash); + const auto equals = line.find('='); + if (equals == std::string::npos) continue; + const std::string written = trim(std::string_view(line).substr(0, equals)); + const DeclaredKey *key = find_declared_key(declared, written); + if (key == nullptr) continue; + const auto value = parse_hex(trim(std::string_view(line).substr(equals + 1))); + if (!value) { + file.problems.push_back(written + " is not written in hex."); + continue; + } + if (std::string problem = check_declared_key(*key, *value, written); !problem.empty()) { + file.problems.push_back(std::move(problem)); + continue; + } + store_named_key(file.keys, key->name, *value); + } + return file; +} + +} // namespace portablekit diff --git a/host/extension_keys.hpp b/host/extension_keys.hpp new file mode 100644 index 0000000..5ee8503 --- /dev/null +++ b/host/extension_keys.hpp @@ -0,0 +1,62 @@ +#pragma once + +// Keys HLE extension modules declare (Registry::declare_key in +// include/portablekit/hle_extension.hpp): which names they are, checking a +// value against a declaration, and looking any key up by its keys-file name. +// The desktop app's keys file reader and a port's optional keys file both use +// this; see docs/HLE_EXTENSIONS.md, "Keys". + +#include "crypto_keys.hpp" +#include "hle/hle_extensions.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace portablekit { + +struct DeclaredKey { + std::string name; // lower case + std::string sha256; // lower case; empty: not checked + std::size_t size{}; + std::string module; // the module's title +}; + +// The keys `modules` declare, calling each module's entry into a registry of +// its own (an entry only adds to its registry, so calling it again is +// harmless). A name two modules declare differently keeps the first +// declaration, and `notes` says so. +[[nodiscard]] std::vector collect_declared_keys(std::span modules, + std::vector ¬es); +// Those of the modules this program links (linked_hle_extensions()), once. +[[nodiscard]] const std::vector &declared_extension_keys(); + +[[nodiscard]] const DeclaredKey *find_declared_key(std::span keys, std::string_view name); +// Empty when `value` is what `key` declares; otherwise one sentence about +// `written_name` (the name as the file spells it). +[[nodiscard]] std::string check_declared_key(const DeclaredKey &key, std::span value, + std::string_view written_name); +// Keeps `value` under `name` (lower case) in keys.named, and a 16-byte +// "kirk.aes." also in keys.kirk_aes. +void store_named_key(CryptoKeys &keys, const std::string &name, std::vector value); +// A key by its keys-file name, from what `keys` holds: kirk.aes., +// kirk.cmd1, savedata., tag. (its key or table) and keys.named. +[[nodiscard]] std::optional> key_value(const CryptoKeys *keys, std::string_view name); + +// A keys file read for declared keys only, as a port reads one: names nothing +// declares are left alone (PortableKit's own keys are compiled into a port). +struct DeclaredKeysFile { + std::filesystem::path path; + bool found{}; + CryptoKeys keys; // keys.named (and kirk_aes) only + std::vector problems; // one sentence each +}; +[[nodiscard]] DeclaredKeysFile read_declared_keys_file(const std::filesystem::path &path, + std::span declared); + +} // namespace portablekit diff --git a/host/hle/hle_common.hpp b/host/hle/hle_common.hpp index 818eaeb..d83d758 100644 --- a/host/hle/hle_common.hpp +++ b/host/hle/hle_common.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,13 @@ class HleRegistrar { // Reads from an open guest file descriptor without moving its position. // Returns the number of bytes read; 0 when the descriptor is unknown. std::size_t read_open_file(std::uint32_t fd, std::uint64_t offset, std::uint8_t *output, std::size_t size); +namespace hle_extension { +class FileFilter; +} +// What the game reads from an open file instead of its bytes (see +// hle_extension::set_file_filter); null restores them. False when `fd` is not +// a readable file of bytes. +bool set_open_file_filter(std::uint32_t fd, std::shared_ptr filter); // Opens a guest path for reading, reads from it and closes it again, without // the game seeing a descriptor. Returns the number of bytes read, or the // negative error sceIoOpen would have returned. diff --git a/host/hle/hle_extension_host.cpp b/host/hle/hle_extension_host.cpp index 3b8c6ee..4145555 100644 --- a/host/hle/hle_extension_host.cpp +++ b/host/hle/hle_extension_host.cpp @@ -3,8 +3,10 @@ // program was built with. #include "../profile.hpp" +#include "extension_keys.hpp" #include "hle_common.hpp" #include "hle_extensions.hpp" +#include "install/user_data.hpp" #include @@ -43,10 +45,92 @@ const char *env(const char *name) { return portablekit::env(name); } } // namespace hle_extension +namespace { + +// A port's keys file for declared keys: _KEYS, or keys.txt in the +// data folder (_DATA_DIR, else the port's own). Read once. +const DeclaredKeysFile &port_declared_keys() { + static const DeclaredKeysFile file = [] { + const std::vector &declared = declared_extension_keys(); + if (declared.empty()) return DeclaredKeysFile{}; + std::filesystem::path path; + if (const char *keys = env("KEYS"); keys != nullptr && *keys != '\0') path = install::path_from_utf8(keys); + else if (const char *data = env("DATA_DIR"); data != nullptr && *data != '\0') + path = install::path_from_utf8(data) / "keys.txt"; + else path = install::user_data_directory() / "keys.txt"; + return read_declared_keys_file(path, declared); + }(); + return file; +} + +// The program's keys: the player's keys file in the desktop app; compiled-in +// keys and the port's own keys file for declared ones in a port. +std::optional> find_key(std::string_view name) { + if (auto value = key_value(crypto_keys(), name)) return value; + if (keys_come_from_keys_file()) return std::nullopt; + return key_value(&port_declared_keys().keys, name); +} + +} // namespace + +const std::vector &declared_extension_keys() { + static const std::vector keys = [] { + std::vector notes; + std::vector collected = collect_declared_keys(linked_hle_extensions(), notes); + for (const std::string ¬e : notes) std::cerr << "[hle-extension] " << note << "\n"; + return collected; + }(); + return keys; +} + +namespace hle_extension { + +std::optional> key_bytes(std::string_view name) { return find_key(name); } + +std::optional key(std::string_view name) { + const auto value = find_key(name); + if (!value || value->size() != 16u) return std::nullopt; + Key16 out{}; + std::copy(value->begin(), value->end(), out.begin()); + return out; +} + +std::optional kirk_key(std::uint8_t slot) { + static constexpr char kDigits[] = "0123456789ABCDEF"; + const std::string name = std::string("kirk.aes.") + kDigits[slot >> 4u] + kDigits[slot & 0xFu]; + return key(name); +} + +std::size_t read_open_file(std::uint32_t fd, std::uint64_t offset, std::span output) { + return ::portablekit::read_open_file(fd, offset, output.data(), output.size()); +} + +bool set_file_filter(std::uint32_t fd, std::shared_ptr filter) { + return set_open_file_filter(fd, std::move(filter)); +} + +} // namespace hle_extension + HleExtensionReport install_linked_hle_extensions(Runtime &runtime, HleRegistrar &hle) { const std::span modules = linked_hle_extensions(); if (modules.empty()) return {}; print_hle_extension_modules(std::cout, modules); + // Declared keys: what a port found in its keys file (the desktop app says + // it with `portablekit keys status`). + if (const std::vector &declared = declared_extension_keys(); !declared.empty()) { + std::size_t present = 0u; + for (const DeclaredKey &key : declared) + if (find_key(key.name)) ++present; + std::cout << "HLE extension keys: " << present << " of " << declared.size() << " declared keys present"; + if (!keys_come_from_keys_file()) { + const DeclaredKeysFile &file = port_declared_keys(); + std::cout << " (" << install::path_to_utf8(file.path) << (file.found ? "" : ", not found") << ")"; + std::cout << "\n"; + for (const std::string &problem : file.problems) std::cout << " problem: " << problem << "\n"; + } else { + std::cout << " (portablekit keys status lists them)\n"; + } + } if (env_set("NO_HLE_EXTENSIONS")) { std::cout << "HLE extensions: " << modules.size() << (modules.size() == 1u ? " module" : " modules") << " linked, ignored (" << env_name("NO_HLE_EXTENSIONS") << ")\n"; diff --git a/host/hle/hle_io.cpp b/host/hle/hle_io.cpp index 6388bb7..60f3e03 100644 --- a/host/hle/hle_io.cpp +++ b/host/hle/hle_io.cpp @@ -8,6 +8,7 @@ #include "kernel/iso_image.hpp" #include "psprecomp/common.hpp" +#include "portablekit/hle_extension.hpp" #include #include @@ -78,6 +79,11 @@ struct OpenFile { // where the image is touched. bool sector_units{}; std::unique_ptr host; + // What the game reads instead of the file's bytes, if anything + // (hle_extension::set_file_filter): size and position then count its + // bytes, and the file's own size is kept here. + std::shared_ptr filter; + std::uint64_t unfiltered_size{}; // Asynchronous calls (sceIo*Async): the result of the one operation in // flight, kept until sceIoWaitAsync or sceIoPollAsync collects it. The // operation itself has already run: the files are the host's, and a @@ -291,8 +297,9 @@ std::size_t read_open_file(std::uint32_t fd, std::uint64_t offset, std::uint8_t OpenFile &file = found->second; const std::span out(output, size); if (file.kind == OpenFile::Kind::Disc) { - if (!io().disc || offset >= file.size) return 0u; - return io().disc->read(file.disc_offset + offset, out.first(std::min(size, file.size - offset))); + const std::uint64_t file_size = file.filter ? file.unfiltered_size : file.size; + if (!io().disc || offset >= file_size) return 0u; + return io().disc->read(file.disc_offset + offset, out.first(std::min(size, file_size - offset))); } if (file.kind == OpenFile::Kind::Host) { file.host->clear(); @@ -305,6 +312,47 @@ std::size_t read_open_file(std::uint32_t fd, std::uint64_t offset, std::uint8_t return 0u; } +bool set_open_file_filter(std::uint32_t fd, std::shared_ptr filter) { + const auto found = io().files.find(fd); + if (found == io().files.end()) return false; + OpenFile &file = found->second; + if ((file.kind != OpenFile::Kind::Disc && file.kind != OpenFile::Kind::Host) || file.sector_units) return false; + if (filter) { + if (!file.filter) file.unfiltered_size = file.size; + file.filter = std::move(filter); + file.size = file.filter->size(); + } else if (file.filter) { + file.filter.reset(); + file.size = file.unfiltered_size; + } + if (trace_io()) + std::cerr << "[io] filter fd=" << fd << " " << file.path << (file.filter ? " set, size " : " removed, size ") + << file.size << "\n"; + return true; +} + +// sceIoRead through a file's filter: `requested` bytes of what the filter +// gives at the position, which moves by as many. +static std::uint32_t read_filtered(Runtime &rt, std::uint32_t fd, OpenFile &file, std::uint32_t address, + std::uint32_t requested) { + const std::uint64_t available = file.position < file.size ? file.size - file.position : 0u; + std::vector buffer(static_cast(std::min(requested, available))); + const std::size_t count = buffer.empty() ? 0u : file.filter->read(file.position, buffer.data(), buffer.size()); + buffer.resize(std::min(count, buffer.size())); + rt.memory().copy_in(address, buffer); + if (file.kind == OpenFile::Kind::Disc) { + load_trace::note_disc_read(buffer.size()); + fast_loading::note_disc_read(); + } else { + load_trace::note_memory_stick_read(buffer.size()); + } + if (trace_io()) + std::cerr << "[io] read fd=" << fd << " " << file.path << " filtered offset=" << file.position + << " got=" << buffer.size() << " bytes -> " << psprecomp::hex32(address) << "\n"; + file.position += buffer.size(); + return static_cast(buffer.size()); +} + std::int64_t read_guest_file(const std::string &path, std::uint64_t offset, std::uint8_t *output, std::size_t size) { const std::int64_t fd = open_file(path, kOpenRead); if (fd < 0) return fd; @@ -323,6 +371,7 @@ std::uint32_t read_file_op(Runtime &rt, std::uint32_t fd, std::uint32_t address, (found->second.kind != OpenFile::Kind::Disc && found->second.kind != OpenFile::Kind::Host)) return io_error::kBadFileDescriptor; OpenFile &file = found->second; + if (file.filter) return read_filtered(rt, fd, file, address, requested); std::vector buffer; std::uint32_t result = 0u; if (file.kind == OpenFile::Kind::Disc) { @@ -539,6 +588,7 @@ void register_async_io(HleRegistrar &hle) { return; } file->host.reset(); + file->filter.reset(); file->kind = OpenFile::Kind::Failed; file->release_after_async = true; complete_async(*file, 0); @@ -652,6 +702,11 @@ void register_io(HleRegistrar &hle, const std::filesystem::path &disc_image, con hle.add("IoFileMgrForUser", "sceIoLseek", [](Runtime &, AllegrexContext &ctx) { kernel().finish64(ctx, static_cast(lseek_op(arg(ctx, 0), static_cast(arg64(ctx, 2)), arg(ctx, 4)))); }); + // sceIoLseek32(fd, offset, whence): a 32-bit offset and result. + hle.add("IoFileMgrForUser", "sceIoLseek32", [](Runtime &, AllegrexContext &ctx) { + const auto offset = static_cast(static_cast(arg(ctx, 1))); + kernel().finish(ctx, static_cast(lseek_op(arg(ctx, 0), offset, arg(ctx, 2)))); + }); register_async_io(hle); hle.add("IoFileMgrForUser", "sceIoGetstat", [](Runtime &rt, AllegrexContext &ctx) { const std::string path = read_cstring(rt.memory(), arg(ctx, 0), 256u); diff --git a/include/portablekit/hle_extension.hpp b/include/portablekit/hle_extension.hpp index 2f118b8..4b397d7 100644 --- a/include/portablekit/hle_extension.hpp +++ b/include/portablekit/hle_extension.hpp @@ -22,10 +22,12 @@ #include "psprecomp/guest_memory.hpp" #include "psprecomp/runtime.hpp" +#include #include #include #include #include +#include #include #include #include @@ -35,7 +37,8 @@ // Raised when this header changes, so a module can check what it is being // built against. 2: Registry::override_chained. -#define PORTABLEKIT_HLE_EXTENSION_INTERFACE 2 +// 3: Registry::declare_key, key lookup, read_open_file, set_file_filter. +#define PORTABLEKIT_HLE_EXTENSION_INTERFACE 3 namespace portablekit::hle_extension { @@ -78,6 +81,16 @@ struct Function { Chained chained; // set by override_chained; the mode is then Override }; +// A key a module reads from the player's keys file (see key() below), as the +// module declares it. The keys file accepts the name like one of PortableKit's +// own keys, checks its length and, when given, its fingerprint, and +// `portablekit keys status` lists it under the module. +struct KeyDeclaration { + std::string name; // as written in the keys file, e.g. "vendor.table" or "kirk.aes.38" + std::string sha256; // lower-case hex SHA-256 of the value; empty: not checked + std::size_t size = 16u; // bytes +}; + // What a module's entry function fills in. Adding a function does nothing yet; // the host applies what was added after the entry returns. class Registry { @@ -99,10 +112,18 @@ class Registry { Function{std::string(library), nid, std::string(name), Handler{}, Mode::Override, std::move(handler)}); } + // Declares a key the module reads with key() or key_bytes(). A name that + // PortableKit itself checks keeps PortableKit's check. + void declare_key(std::string_view name, std::string_view sha256 = {}, std::size_t size = 16u) { + keys_.push_back(KeyDeclaration{std::string(name), std::string(sha256), size}); + } + [[nodiscard]] const std::vector &functions() const noexcept { return functions_; } + [[nodiscard]] const std::vector &keys() const noexcept { return keys_; } private: std::vector functions_; + std::vector keys_; }; // --- What a handler needs -------------------------------------------------- @@ -197,6 +218,48 @@ void log_once(std::string_view key, std::string_view message); // PORTABLEKIT_ in the desktop app), or null when it is not set. [[nodiscard]] const char *env(const char *name); +// --- Keys -------------------------------------------------------------------- +// The program's keys, by the name the keys file gives them: PortableKit's own +// ("kirk.aes.5D", "kirk.cmd1", "savedata.2", ...) and those modules declare. +// In the desktop app they come from the player's keys file; in a port, +// PortableKit's own are compiled in and declared ones come from an optional +// keys file (_KEYS, or keys.txt in the port's data folder). Names are +// not case-sensitive. Empty when the program does not have the key. +using Key16 = std::array; +[[nodiscard]] std::optional key(std::string_view name); +// A key of any length, such as a table. +[[nodiscard]] std::optional> key_bytes(std::string_view name); +// The KIRK AES key of a slot: key("kirk.aes."). +[[nodiscard]] std::optional kirk_key(std::uint8_t slot); + +// --- Open files ---------------------------------------------------------------- +// Up to output.size() bytes of the open guest file `fd` at `offset`, as the +// file holds them (before any filter below), without moving its position. +// Returns how many were read: fewer at the end, 0 for an unknown descriptor. +std::size_t read_open_file(std::uint32_t fd, std::uint64_t offset, std::span output); + +// What a game reads from a file instead of its bytes, such as the plain text +// of an encrypted container. Called on the emulation thread. +class FileFilter { +public: + virtual ~FileFilter() = default; + // The size of what the game sees, in bytes. + [[nodiscard]] virtual std::uint64_t size() const = 0; + // Up to `length` bytes of it at `offset` into `destination`; returns how + // many, fewer at the end. The raw bytes are at hand through + // read_open_file(). + virtual std::size_t read(std::uint64_t offset, std::uint8_t *destination, std::size_t length) = 0; +}; +// From now on, PortableKit's sceIoRead, sceIoReadAsync, sceIoLseek, +// sceIoLseek32 (and their Async forms) and the disc seek ioctl 0x01010005 +// work on what `filter` gives for `fd`: positions and sizes count its bytes, +// and asynchronous results are delivered as usual. The position is left +// where it was. A null filter restores the file's own bytes; closing the file +// drops the filter. False when `fd` is not an open file PortableKit reads (an +// unknown descriptor, a directory, or the raw disc device, which counts +// sectors). +bool set_file_filter(std::uint32_t fd, std::shared_ptr filter); + } // namespace portablekit::hle_extension // Defines a module's entry function. `name` is the module's name as its diff --git a/tests/hle_extension_test_host.cpp b/tests/hle_extension_test_host.cpp index ed1b1bd..c7c0757 100644 --- a/tests/hle_extension_test_host.cpp +++ b/tests/hle_extension_test_host.cpp @@ -51,5 +51,11 @@ void queue_guest_call(std::uint32_t function, std::span arg call.argument_count = count; } std::int32_t current_thread_id() { return 0; } +// Keys and files are not what these tests exercise through the host. +std::optional> key_bytes(std::string_view) { return std::nullopt; } +std::optional key(std::string_view) { return std::nullopt; } +std::optional kirk_key(std::uint8_t) { return std::nullopt; } +std::size_t read_open_file(std::uint32_t, std::uint64_t, std::span) { return 0u; } +bool set_file_filter(std::uint32_t, std::shared_ptr) { return false; } } // namespace portablekit::hle_extension diff --git a/tests/hle_extension_tests.cpp b/tests/hle_extension_tests.cpp index c227fa0..4d85ace 100644 --- a/tests/hle_extension_tests.cpp +++ b/tests/hle_extension_tests.cpp @@ -3,11 +3,14 @@ // not, and the first module to add a function keeping it. Uses the example // module in examples/hle_extension/ and a runtime with no kernel or game. +#include "extension_keys.hpp" #include "hle/hle_extensions.hpp" #include "psprecomp/runtime.hpp" #include +#include +#include #include #include #include @@ -313,6 +316,72 @@ int main() { check(!report.active.empty() && !report.active[0].overrides, "and it is not counted as an override"); } + // Keys modules declare. + { + const auto declaring = [](ext::Registry ®istry) { + registry.declare_key("Vendor.Table", {}, 32u); + registry.declare_key("example.demo", "00"); // differs from the example's: the first stays + registry.declare_key("tag.1234"); // not a name a module can declare + }; + static std::function entry; + entry = declaring; + const HleExtensionModule modules[] = { + kExample, {"vendor", "Vendor module", "", "MIT", [](ext::Registry ®istry) { entry(registry); }}}; + std::vector notes; + const auto declared = portablekit::collect_declared_keys(modules, notes); + check(declared.size() == 2u && declared[0].name == "example.demo" && declared[0].module == "Example HLE extension" && + declared[1].name == "vendor.table" && declared[1].size == 32u, + "declared keys are collected in lower case with their module"); + check(notes.size() == 2u, "a conflicting and an impossible declaration are noted"); + + const std::vector demo{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF}; + std::vector wrong = demo; + wrong[0] ^= 1u; + check(portablekit::check_declared_key(declared[0], demo, "example.demo").empty(), + "a declared key with the right fingerprint passes"); + check(!portablekit::check_declared_key(declared[0], wrong, "example.demo").empty(), + "a wrong value of a fingerprinted key is refused"); + check(!portablekit::check_declared_key(declared[1], demo, "vendor.table").empty(), + "a value of the wrong length is refused"); + + portablekit::CryptoKeys keys; + keys.kirk_cmd1 = portablekit::Key16{1}; + keys.savedata[2] = portablekit::Key16{2}; + keys.tags[0xC0CB167Cu].table = std::vector(0x90u, 7u); + portablekit::store_named_key(keys, "kirk.aes.38", demo); + portablekit::store_named_key(keys, "vendor.table", std::vector(32u, 9u)); + check(keys.kirk(0x38u) != nullptr && (*keys.kirk(0x38u))[1] == 0x11u, + "a declared kirk.aes slot is also a KIRK key"); + check(portablekit::key_value(&keys, "KIRK.AES.38") == demo, "look-ups ignore case"); + check(portablekit::key_value(&keys, "kirk.cmd1").value_or(std::vector{}).at(0) == 1u && + portablekit::key_value(&keys, "savedata.2").value_or(std::vector{}).at(0) == 2u, + "PortableKit's own keys are found by their names"); + check(portablekit::key_value(&keys, "tag.C0CB167C").value_or(std::vector{}).size() == 0x90u && + portablekit::key_value(&keys, "vendor.table").value_or(std::vector{}).size() == 32u, + "tables and longer keys come back whole"); + check(!portablekit::key_value(&keys, "savedata.3") && !portablekit::key_value(&keys, "nothing.here") && + !portablekit::key_value(nullptr, "kirk.cmd1"), + "a key the program lacks is empty"); + + const auto path = std::filesystem::temp_directory_path() / "portablekit_declared_keys_test.txt"; + { + std::ofstream out(path); + out << "# a port's keys file\n" + << "kirk.aes.5D = 00000000000000000000000000000000 # PortableKit's own: left alone\n" + << "EXAMPLE.DEMO = 00112233 44556677 8899AABB CCDDEEFF\n" + << "vendor.table = 0011\n" + << "someone.else = 00\n"; + } + const auto file = portablekit::read_declared_keys_file(path, declared); + std::filesystem::remove(path); + check(file.found && portablekit::key_value(&file.keys, "example.demo") == demo, + "a port's keys file gives a declared key"); + check(file.problems.size() == 1u && !portablekit::key_value(&file.keys, "vendor.table") && + !portablekit::key_value(&file.keys, "kirk.aes.5D"), + "a wrong declared key is a problem, and names nothing declares are left alone"); + } + if (g_failures != 0) std::printf("%d failed\n", g_failures); return g_failures == 0 ? 0 : 1; } diff --git a/tests/io_filter_tests.cpp b/tests/io_filter_tests.cpp new file mode 100644 index 0000000..3021fa1 --- /dev/null +++ b/tests/io_filter_tests.cpp @@ -0,0 +1,179 @@ +// A read filter on an open file (hle_extension::set_file_filter): PortableKit's +// own sceIoRead, sceIoLseek, sceIoLseek32 and their asynchronous forms work on +// the filtered bytes, the raw bytes stay readable, and closing the file drops +// the filter. Runs the real IoFileMgrForUser handlers on a memory stick folder +// in the temporary directory, with no game and no running kernel. + +#include "hle/hle_common.hpp" +#include "portablekit/hle_extension.hpp" + +#include "psprecomp/runtime.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace ext = portablekit::hle_extension; + +int g_failures = 0; + +void check(bool condition, const char *what) { + std::printf("%s %s\n", condition ? "ok " : "FAIL", what); + if (!condition) ++g_failures; +} + +constexpr std::uint32_t kPath = 0x08800000u; +constexpr std::uint32_t kBuffer = 0x08810000u; +constexpr std::uint32_t kResult = 0x08820000u; +constexpr std::size_t kRawSize = 1000u; + +std::uint8_t raw_byte(std::size_t i) { return static_cast(i * 7u + 3u); } + +// The file's first half, each byte XORed with 0x5A. +class XorFilter : public ext::FileFilter { +public: + explicit XorFilter(std::uint32_t fd) : fd_(fd) {} + std::uint64_t size() const override { return kRawSize / 2u; } + std::size_t read(std::uint64_t offset, std::uint8_t *destination, std::size_t length) override { + if (offset >= size()) return 0u; + length = static_cast(std::min(length, size() - offset)); + const std::size_t count = ext::read_open_file(fd_, offset, std::span(destination, length)); + for (std::size_t i = 0; i < count; ++i) destination[i] ^= 0x5Au; + return count; + } + +private: + std::uint32_t fd_; +}; + +struct Io { + psprecomp::Runtime runtime; + std::uint32_t v1{}; + + std::uint32_t nid(const char *name) { + for (const auto &symbol : runtime.nids().all()) + if (symbol.library == "IoFileMgrForUser" && symbol.name == name) return symbol.nid; + std::printf("no NID for %s\n", name); + return 0u; + } + // Calls IoFileMgrForUser::name with a0, a1, ... (t0 onwards after a3). + std::uint32_t call(const char *name, std::initializer_list args) { + psprecomp::AllegrexContext ctx{}; + unsigned i = 0; + for (const std::uint32_t value : args) { + ctx.set_gpr(i < 4u ? 4u + i : 8u + (i - 4u), value); + ++i; + } + runtime.invoke_import("IoFileMgrForUser", nid(name), ctx); + v1 = ctx.gpr[3]; + return ctx.gpr[2]; + } + std::uint64_t result64() const { + return static_cast(runtime.memory().load32(kResult)) | + (static_cast(runtime.memory().load32(kResult + 4u)) << 32u); + } + bool guest_bytes_are(std::uint64_t filtered_offset, std::size_t count) { + for (std::size_t i = 0; i < count; ++i) + if (runtime.memory().load8(kBuffer + static_cast(i)) != + (raw_byte(static_cast(filtered_offset) + i) ^ 0x5Au)) + return false; + return true; + } +}; + +} // namespace + +int main(int, char **argv) { + (void)argv; + const auto stick = std::filesystem::temp_directory_path() / "portablekit_io_filter_test"; + std::filesystem::remove_all(stick); + std::filesystem::create_directories(stick / "DATA"); + { + std::ofstream out(stick / "DATA" / "FILE.BIN", std::ios::binary); + for (std::size_t i = 0; i < kRawSize; ++i) out.put(static_cast(raw_byte(i))); + } + + Io io; + io.runtime.nids().load_csv(PORTABLEKIT_NIDS_CSV); + portablekit::HleRegistrar hle(io.runtime); + portablekit::register_io(hle, {}, stick); + const std::string path = "ms0:/DATA/FILE.BIN"; + for (std::size_t i = 0; i <= path.size(); ++i) + io.runtime.memory().store8(kPath + static_cast(i), i < path.size() ? static_cast(path[i]) : 0u); + + const std::uint32_t fd = io.call("sceIoOpen", {kPath, 1u, 0u}); + check(static_cast(fd) > 0, "the file opens"); + io.call("sceIoLseek32", {fd, 4u, 0u}); + auto filter = std::make_shared(fd); + std::weak_ptr watch = filter; + check(ext::set_file_filter(fd, filter), "a filter is set on an open file"); + filter.reset(); + + // Synchronous seeks and reads. + check(io.call("sceIoLseek32", {fd, 0u, 1u}) == 4u, "the position is left where it was"); + check(io.call("sceIoLseek", {fd, 0u, 0u, 0u, 2u}) == kRawSize / 2u && io.v1 == 0u, + "sceIoLseek from the end counts the filtered size"); + check(io.call("sceIoLseek32", {fd, 10u, 0u}) == 10u, "sceIoLseek32 moves in filtered bytes"); + check(io.call("sceIoRead", {fd, kBuffer, 20u}) == 20u && io.guest_bytes_are(10u, 20u), + "sceIoRead gives the filtered bytes"); + check(io.call("sceIoLseek32", {fd, 0u, 1u}) == 30u, "and moves the position by as many"); + io.call("sceIoLseek32", {fd, static_cast(-5), 2u}); + check(io.call("sceIoRead", {fd, kBuffer, 20u}) == 5u && io.guest_bytes_are(kRawSize / 2u - 5u, 5u), + "a read at the end stops at the filtered size"); + check(io.call("sceIoRead", {fd, kBuffer, 20u}) == 0u, "and a read past it gives nothing"); + + // The raw bytes stay readable. + std::vector raw(8u); + bool raw_ok = ext::read_open_file(fd, 600u, raw) == raw.size(); + for (std::size_t i = 0; i < raw.size(); ++i) raw_ok = raw_ok && raw[i] == raw_byte(600u + i); + check(raw_ok, "read_open_file gives the file's own bytes, past the filtered size too"); + + // Asynchronous forms: the result comes with sceIoWaitAsync as usual. + check(io.call("sceIoLseekAsync", {fd, 0u, 3u, 0u, 0u}) == 0u && io.call("sceIoWaitAsync", {fd, kResult}) == 0u && + io.result64() == 3u, + "sceIoLseekAsync seeks in filtered bytes"); + check(io.call("sceIoReadAsync", {fd, kBuffer, 4u}) == 0u && io.call("sceIoWaitAsync", {fd, kResult}) == 0u && + io.result64() == 4u && io.guest_bytes_are(3u, 4u), + "sceIoReadAsync reads filtered bytes"); + check(io.call("sceIoLseek32Async", {fd, 0u, 2u}) == 0u && io.call("sceIoPollAsync", {fd, kResult}) == 0u && + io.result64() == kRawSize / 2u, + "sceIoLseek32Async from the end counts the filtered size"); + + // Removing the filter restores the file's own size. + check(ext::set_file_filter(fd, nullptr) && watch.expired(), + "a null filter removes it and lets it go"); + check(io.call("sceIoLseek", {fd, 0u, 0u, 0u, 2u}) == kRawSize, "and the file's own size is back"); + + // Closing drops the filter. + filter = std::make_shared(fd); + watch = filter; + ext::set_file_filter(fd, filter); + filter.reset(); + check(!watch.expired(), "the file keeps its filter"); + check(io.call("sceIoClose", {fd}) == 0u && watch.expired(), "sceIoClose drops it"); + check(!ext::set_file_filter(fd, std::make_shared(fd)), "a closed descriptor takes no filter"); + + const std::uint32_t second = io.call("sceIoOpen", {kPath, 1u, 0u}); + filter = std::make_shared(second); + watch = filter; + ext::set_file_filter(second, filter); + filter.reset(); + check(io.call("sceIoCloseAsync", {second}) == 0u && watch.expired(), "sceIoCloseAsync drops it at once"); + io.call("sceIoWaitAsync", {second, kResult}); + + for (std::size_t i = 0; i < 5u; ++i) io.runtime.memory().store8(kPath + static_cast(i), "ms0:"[i]); + io.runtime.memory().store8(kPath + 4u, 0u); + const std::uint32_t directory = io.call("sceIoDopen", {kPath}); + check(static_cast(directory) > 0 && !ext::set_file_filter(directory, std::make_shared(directory)), + "a directory takes no filter"); + + std::filesystem::remove_all(stick); + if (g_failures != 0) std::printf("%d failed\n", g_failures); + return g_failures == 0 ? 0 : 1; +}