From d6a4d082d4f7d2597392372f6daf279d4233e1a9 Mon Sep 17 00:00:00 2001 From: Rafie <107275895+Rafie-kun@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:54:49 +0600 Subject: [PATCH 1/2] Fix unlocked downloads resetting to 0% on pause/resume/restart - Make NetPacket replace buffers thread_local (cross-depot code contamination caused random 401s and manifest failures) - Fetch manifest codes in parallel (mutex only around Lua) and wait out the full 30s HTTP budget instead of 12s - Always bypass the 404 negative cache on real code requests so pause/resume re-checks the archive - Fall back to the request app_id for the owned-depot check when the depot map is not populated yet (restart ordering) - Wait for in-flight manifest fetches instead of reporting a miss - Apply manifest overrides to shared depots too; unique temp names; expire stale code futures --- .github/workflows/delete-version.yml | 2 +- .github/workflows/main.yml | 2 +- src/Hook/Hooks_Manifest.cpp | 34 +++++--- src/Hook/Hooks_NetPacket.cpp | 94 ++++++++++++++++------ src/Utils/SteamMetadata/ManifestCache.cpp | 27 ++++++- src/Utils/SteamMetadata/ManifestClient.cpp | 45 +++++++---- src/Utils/SteamMetadata/Mirror.cpp | 15 ++-- 7 files changed, 155 insertions(+), 64 deletions(-) diff --git a/.github/workflows/delete-version.yml b/.github/workflows/delete-version.yml index 1b9cbe52..826fa7b9 100644 --- a/.github/workflows/delete-version.yml +++ b/.github/workflows/delete-version.yml @@ -87,5 +87,5 @@ jobs: - name: Purge jsDelivr cache for pointer shell: bash run: | - curl -s "https://purge.jsdelivr.net/gh/madoiscool/BetterSteamTools@updates/opensteamtool/latest.toml" \ + curl -s "https://purge.jsdelivr.net/gh/${GITHUB_REPOSITORY}@updates/opensteamtool/latest.toml" \ >/dev/null || echo "jsDelivr purge failed (non-fatal)." diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee929a4d..67689a83 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,7 +70,7 @@ jobs: run: | try { Invoke-WebRequest -UseBasicParsing ` - "https://purge.jsdelivr.net/gh/madoiscool/BetterSteamTools@updates/opensteamtool/latest.toml" | Out-Null + "https://purge.jsdelivr.net/gh/${{ github.repository }}@updates/opensteamtool/latest.toml" | Out-Null } catch { Write-Host "jsDelivr purge failed (non-fatal): $_" } - name: Build Debug diff --git a/src/Hook/Hooks_Manifest.cpp b/src/Hook/Hooks_Manifest.cpp index d98c7fa0..ddaf2a5a 100644 --- a/src/Hook/Hooks_Manifest.cpp +++ b/src/Hook/Hooks_Manifest.cpp @@ -187,20 +187,28 @@ namespace { const auto& overrides = LuaConfig::GetManifestOverrides(); // Apply manifest overrides in place (only depots a lua explicitly pins). - if (!overrides.empty() && pDepotInfo && pDepotInfo->m_Size) { - for (uint32 i = 0; i < pDepotInfo->m_Size; ++i) { - DepotEntry& e = pDepotInfo->m_Memory.m_pMemory[i]; - auto it = overrides.find(e.DepotId); - if (it != overrides.end()) { - // if size=0 in the override, keep the original size(affects download display but not the actual download) - uint64_t newSize = it->second.size ? it->second.size : e.ManifestSize; - LOG_MANIFEST_INFO("BuildDepotDependency: patching depot {} gid={}->{} size={}->{}", - e.DepotId, e.ManifestGid, it->second.gid, - e.ManifestSize, newSize); - e.ManifestGid = it->second.gid; - e.ManifestSize = newSize; + // Both lists: a pinned depot can arrive via pSharedDepotInfo (shared + // installs), and preseeding below runs on both — patching only + // pDepotInfo left shared depots preseeding/requesting the wrong GID. + if (!overrides.empty()) { + auto patchVec = [&](CUtlVector* vec) { + if (!vec || !vec->m_Size) return; + for (uint32 i = 0; i < vec->m_Size; ++i) { + DepotEntry& e = vec->m_Memory.m_pMemory[i]; + auto it = overrides.find(e.DepotId); + if (it != overrides.end()) { + // if size=0 in the override, keep the original size(affects download display but not the actual download) + uint64_t newSize = it->second.size ? it->second.size : e.ManifestSize; + LOG_MANIFEST_INFO("BuildDepotDependency: patching depot {} gid={}->{} size={}->{}", + e.DepotId, e.ManifestGid, it->second.gid, + e.ManifestSize, newSize); + e.ManifestGid = it->second.gid; + e.ManifestSize = newSize; + } } - } + }; + patchVec(pDepotInfo); + patchVec(pSharedDepotInfo); } // Pre-seed \depotcache SYNCHRONOUSLY for every depot OST is diff --git a/src/Hook/Hooks_NetPacket.cpp b/src/Hook/Hooks_NetPacket.cpp index 4c79c881..f21c6b9f 100644 --- a/src/Hook/Hooks_NetPacket.cpp +++ b/src/Hook/Hooks_NetPacket.cpp @@ -39,24 +39,30 @@ namespace { constexpr int kPacketPoolSize = 8; // ── Incoming (RecvPkt) packet pool ───────────────────── - uint8 g_NewBody[kMaxBodySize]; - uint32 g_cbNewBody = 0; - uint8 g_NewHdr[kMaxHdrSize]; - uint32 g_cbNewHdr = 0; - bool g_NeedReplaceBody = false; - bool g_NeedReplaceHdr = false; - bool g_ResizedInPlace = false; - uint32 g_NewBodySize = 0; - uint8 g_RecvPacketPool[kPacketPoolSize][kMaxPacketSize]; - int g_RecvPacketPoolIdx = 0; + // NOTE: Steam dispatches network packets on more than one thread. These + // used to be plain globals, so two concurrent RecvPkt calls overwrote each + // other's replacement header/body and one depot's manifest code could land + // in another depot's reply (random 401s / downloads resetting to 0%). + // thread_local keeps each network thread on its own scratch space; the + // HandleRecv -> ReplaceRecvPacket use is always same-thread. + thread_local uint8 g_NewBody[kMaxBodySize]; + thread_local uint32 g_cbNewBody = 0; + thread_local uint8 g_NewHdr[kMaxHdrSize]; + thread_local uint32 g_cbNewHdr = 0; + thread_local bool g_NeedReplaceBody = false; + thread_local bool g_NeedReplaceHdr = false; + thread_local bool g_ResizedInPlace = false; + thread_local uint32 g_NewBodySize = 0; + thread_local uint8 g_RecvPacketPool[kPacketPoolSize][kMaxPacketSize]; + thread_local int g_RecvPacketPoolIdx = 0; // ── Outgoing (BBuildAndAsyncSendFrame) — same pattern ─────── - uint8 g_SendNewBody[kMaxBodySize]; - uint32 g_cbSendNewBody = 0; - bool g_NeedReplaceSend = false; - bool g_SuppressSend = false; // drop the outbound frame entirely (cloud RPC answered locally) - uint8 g_SendPacketPool[kPacketPoolSize][kMaxPacketSize]; - int g_SendPacketPoolIdx = 0; + thread_local uint8 g_SendNewBody[kMaxBodySize]; + thread_local uint32 g_cbSendNewBody = 0; + thread_local bool g_NeedReplaceSend = false; + thread_local bool g_SuppressSend = false; // drop the outbound frame entirely (cloud RPC answered locally) + thread_local uint8 g_SendPacketPool[kPacketPoolSize][kMaxPacketSize]; + thread_local int g_SendPacketPoolIdx = 0; // ── EMsg -> name lookup ───────────────────────── RESOLVE_FUNC(PchMsgNameFromEMsg, char*, EMsg eMsg); @@ -840,15 +846,26 @@ namespace Hooks_NetPacket_ManifestProbe { // Incoming: ContentServerDirectory.GetManifestRequestCode#1 (eMsg 147) // // Launches an async HTTP fetch on send; the recv handler waits up to -// 12 s for the result and patches both header (eresult=OK) and body -// (manifest_request_code). On timeout or failure the original +// kMaxWaitSeconds for the result and patches both header (eresult=OK) and +// body (manifest_request_code). On timeout or failure the original // response passes through unmodified. // ════════════════════════════════════════════════════════════════ namespace Hooks_NetPacket_Manifest { - std::unordered_map> g_CodeFutures; + struct CodeFetch { + std::shared_future future; + std::chrono::steady_clock::time_point created; + }; + std::unordered_map g_CodeFutures; std::mutex g_CodeMutex; - constexpr uint32 kMaxWaitSeconds = 12; + // HTTP fetch budget is resolve+connect+send+recv = 5+5+10+10 = 30 s worst + // case (plus Lua http_get). The old 12 s waiter timed out on any slow edge + // and the download failed (reset to 0%) while the fetch completed a moment + // later — the classic "random fail, retry works". Wait out the full budget. + constexpr uint32 kMaxWaitSeconds = 30; + // Replies that never arrive (batched inside k_EMsgMulti, which RecvJob + // skips, or a dropped job) must not leak map entries forever. + constexpr auto kFutureStaleAfter = std::chrono::seconds(120); // Passive capture: jobid_source -> (depot, gid) for every outgoing request, // so HandleRecv can harvest the genuine code Steam returns for depots this @@ -905,10 +922,23 @@ namespace Hooks_NetPacket_Manifest { // tests OwnedAppIdSet, which MarkOwned fills with *app* ids, against the // *depot* id passed here. BuildDepotDependency has already recorded the // depot -> app mapping (it runs seconds earlier), so bridge through that. + // Fallback: on restart / resume the code request can arrive BEFORE + // BuildDepotDependency runs this session, so the map is still empty. + // The request itself usually carries app_id — if that app is owned, + // leave Steam's code alone too. Without this, an owned depot gets a + // carrier-minted code injected, the CDN 401s it, and the download + // resets to 0%. AppId_t seenApp = 0; uint64_t seenGid = 0; + bool ownedDepot = false; if (Hooks_Manifest::LookupDepot(depotId, seenApp, seenGid) && seenApp && LuaConfig::IsOwned(seenApp)) { + ownedDepot = true; + } else if (appId && LuaConfig::IsOwned(appId)) { + seenApp = appId; + ownedDepot = true; + } + if (ownedDepot) { LOG_MANIFEST_INFO("GetManifestRequestCode: depot={} belongs to owned app {}, " "leaving Steam's own code alone", depotId, seenApp); return false; @@ -946,10 +976,19 @@ namespace Hooks_NetPacket_Manifest { // download). Use "is any app actively downloading": true during a real // user download, false at idle startup when Steam is only retrying its // scheduled-update queue. + // NOTE: the archive check below always bypasses the negative cache. + // A pause -> resume inside the 10 min negative window must re-GET: + // the donor may have supplied the manifest while paused, and the UI + // active-flag can lag the network thread on resume, so gating the + // bypass on `active` caused resume to ride a stale 404 and reset to + // 0%. Steady-state cost is nil — archived manifests hit the + // fs::exists short-circuit and never GET again; only genuinely + // missing manifests re-GET on each ~30 s retry. `active` still gates + // the "not ready" popup so idle retries stay silent. const bool active = Hooks_SteamUI::ActiveDownloadCount() > 0; OSTPlatform::Thread::StartDetached([a, d, g, active]() -> uint32_t { bool notArchived = false; - bool ok = ManifestCache::EnsureCached(a, d, g, 0, ¬Archived, /*bypassNeg=*/active); + bool ok = ManifestCache::EnsureCached(a, d, g, 0, ¬Archived, /*bypassNeg=*/true); if (active && !ok && notArchived) Hooks_Manifest::ReportMissingManifest(d, g); return 0; @@ -972,7 +1011,16 @@ namespace Hooks_NetPacket_Manifest { { std::lock_guard lock(g_CodeMutex); - g_CodeFutures[jobId] = task.share(); + // Sweep replies that never arrived (e.g. batched inside + // k_EMsgMulti, which RecvJob skips) so the map stays bounded. + const auto now = std::chrono::steady_clock::now(); + for (auto it = g_CodeFutures.begin(); it != g_CodeFutures.end(); ) { + if (now - it->second.created > kFutureStaleAfter) + it = g_CodeFutures.erase(it); + else + ++it; + } + g_CodeFutures[jobId] = CodeFetch{task.share(), now}; } return false; // Don't modify the outgoing request body @@ -1023,7 +1071,7 @@ namespace Hooks_NetPacket_Manifest { std::lock_guard lock(g_CodeMutex); auto it = g_CodeFutures.find(jobId); if (it == g_CodeFutures.end()) return; - future = it->second; + future = it->second.future; g_CodeFutures.erase(it); // Always clean up immediately } // Wait up to kMaxWaitSeconds seconds for the HTTP fetch to complete diff --git a/src/Utils/SteamMetadata/ManifestCache.cpp b/src/Utils/SteamMetadata/ManifestCache.cpp index a8ee1ccb..800357de 100644 --- a/src/Utils/SteamMetadata/ManifestCache.cpp +++ b/src/Utils/SteamMetadata/ManifestCache.cpp @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include @@ -96,9 +98,13 @@ namespace { std::error_code ec; fs::create_directories(dest.parent_path(), ec); // no-op if it exists - // Unique-ish temp beside the target so the rename stays on one volume. + // Unique temp beside the target so the rename stays on one volume. + // Thread id alone is not unique under re-entrancy (same thread fetching + // the same depot twice), so add a sequence number. + static std::atomic s_tmpSeq{0}; + const uint64_t seq = s_tmpSeq.fetch_add(1, std::memory_order_relaxed); const fs::path tmp = fs::path(dest).concat( - std::format(".{}.tmp", ::GetCurrentThreadId())); + std::format(".{}.{}.tmp", ::GetCurrentThreadId(), seq)); { std::ofstream ofs(tmp, std::ios::binary | std::ios::trunc); @@ -161,7 +167,22 @@ bool EnsureCached(AppId_t app, uint32_t depot, uint64_t gid, } InFlightGuard guard(key); - if (!guard.acquired) return false; // another worker is already on it + if (!guard.acquired) { + // Another worker is already fetching this exact depot:gid (preseed + // sweep racing a real code request, or two depots lists at once). The + // old code returned false immediately, so the loser reported a miss + // even though the manifest landed a moment later — first-press + // download failed and reset to 0%, retry worked. Wait briefly for the + // winner instead; outNotArchived stays false (no fresh 404 signal). + constexpr auto kJoinWait = std::chrono::seconds(10); + constexpr auto kJoinStep = std::chrono::milliseconds(100); + const auto joinDeadline = std::chrono::steady_clock::now() + kJoinWait; + do { + std::this_thread::sleep_for(kJoinStep); + if (fs::exists(dest, ec)) return true; + } while (std::chrono::steady_clock::now() < joinDeadline); + return false; + } // Re-check after taking the slot: the other worker may have just finished. if (fs::exists(dest, ec)) return true; diff --git a/src/Utils/SteamMetadata/ManifestClient.cpp b/src/Utils/SteamMetadata/ManifestClient.cpp index d2d4e2d0..8ac933a2 100644 --- a/src/Utils/SteamMetadata/ManifestClient.cpp +++ b/src/Utils/SteamMetadata/ManifestClient.cpp @@ -98,8 +98,7 @@ namespace ManifestClient { // ── fetch ───────────────────────────────────────────────────── - static bool FetchActive(uint64_t gid, uint64_t* outCode, AppId_t appId, AppId_t depotId) { - const Provider& p = *g_active; + static bool FetchActive(const Provider& p, uint64_t gid, uint64_t* outCode, AppId_t appId, AppId_t depotId) { const Config::ManifestTimeouts timeouts = Config::GetManifestTimeouts(); // app_id does not change the code — it only satisfies Steam's access @@ -138,24 +137,38 @@ namespace ManifestClient { bool FetchManifestRequestCode(uint64_t manifestGid, uint64_t* outRequestCode, AppId_t appId, AppId_t depotId) { - std::lock_guard lock(g_mutex); - - if (appId && depotId && LuaConfig::HasManifestCodeFuncEx()) { - if (LuaConfig::CallManifestFetchCodeEx(appId, depotId, manifestGid, outRequestCode)) { - LOG_MANIFEST_INFO("Manifest gid={} resolved via fetch_manifest_code_ex", manifestGid); - return true; + // The lua_State is single and not thread-safe, so the Lua phase stays + // serialized. But the old code held g_mutex across the HTTP fetch too, + // which serialized ALL depot fetches: with N depots the Nth job waited + // ~N x HTTP time while HandleRecv only waits kMaxWaitSeconds, so large + // games timed out randomly (download resets to 0%, retry works). Copy + // the active provider under the lock, run Lua under the lock, then do + // the network fetch with no lock held. + { + std::lock_guard lock(g_mutex); + + if (appId && depotId && LuaConfig::HasManifestCodeFuncEx()) { + if (LuaConfig::CallManifestFetchCodeEx(appId, depotId, manifestGid, outRequestCode)) { + LOG_MANIFEST_INFO("Manifest gid={} resolved via fetch_manifest_code_ex", manifestGid); + return true; + } + LOG_MANIFEST_WARN("Manifest gid={} fetch_manifest_code_ex returned nil, trying fetch_manifest_code", manifestGid); } - LOG_MANIFEST_WARN("Manifest gid={} fetch_manifest_code_ex returned nil, trying fetch_manifest_code", manifestGid); - } - if (LuaConfig::HasManifestCodeFunc()) { - if (LuaConfig::CallManifestFetchCode(manifestGid, outRequestCode)) { - LOG_MANIFEST_INFO("Manifest gid={} resolved via manifest.lua", manifestGid); - return true; + if (LuaConfig::HasManifestCodeFunc()) { + if (LuaConfig::CallManifestFetchCode(manifestGid, outRequestCode)) { + LOG_MANIFEST_INFO("Manifest gid={} resolved via manifest.lua", manifestGid); + return true; + } + LOG_MANIFEST_WARN("Manifest gid={} lua returned nil, falling back to config", manifestGid); } - LOG_MANIFEST_WARN("Manifest gid={} lua returned nil, falling back to config", manifestGid); } - return FetchActive(manifestGid, outRequestCode, appId, depotId); + Provider active; + { + std::lock_guard lock(g_mutex); + active = *g_active; + } + return FetchActive(active, manifestGid, outRequestCode, appId, depotId); } } diff --git a/src/Utils/SteamMetadata/Mirror.cpp b/src/Utils/SteamMetadata/Mirror.cpp index f2e44c4b..f568c116 100644 --- a/src/Utils/SteamMetadata/Mirror.cpp +++ b/src/Utils/SteamMetadata/Mirror.cpp @@ -9,14 +9,15 @@ namespace Mirror { namespace { - // Delivery mirror chain for the `updates` branch of madoiscool/BetterSteamTools. - // Independent hosts (not just CDN copies of one repo), tried in order until one - // returns HTTP 200. github-raw first for pointer freshness; jsDelivr and lua.tools - // cover regions where raw.githubusercontent is throttled/blocked. + // Delivery mirror chain for the `updates` branch of Rafie-kun/BetterSteamTools + // (fork). Independent hosts, tried in order until one returns HTTP 200. + // github-raw first for pointer freshness; jsDelivr covers regions where + // raw.githubusercontent is throttled/blocked. NOTE: no upstream fallback — + // the fork publishes its own latest.toml and falling back to upstream + // would offer upstream builds as "updates" over fork builds. constexpr const char* kBaseTemplates[] = { - "https://raw.githubusercontent.com/madoiscool/BetterSteamTools/updates/{path}", - "https://cdn.jsdelivr.net/gh/madoiscool/BetterSteamTools@updates/{path}", - "https://git.lua.tools/luatools/BetterSteamTools/raw/branch/updates/{path}", + "https://raw.githubusercontent.com/Rafie-kun/BetterSteamTools/updates/{path}", + "https://cdn.jsdelivr.net/gh/Rafie-kun/BetterSteamTools@updates/{path}", }; std::string Expand(std::string tmpl, std::string_view relPath) From 1fbbe0ccbafb97da6e2e0c01ea8ed493850f3ed7 Mon Sep 17 00:00:00 2001 From: Rafie <107275895+Rafie-kun@users.noreply.github.com> Date: Sat, 19 Sep 2026 02:33:28 +0600 Subject: [PATCH 2/2] Local-only achievements for unlocked games (toast + library, no online profile) - Journal stat stores (820/5466) per app under opensteamtool/stats instead of sending them to Valve - Synthesize 821 success so games see StoreStats succeed and the client toasts - Inject the local journal (+CR overlay) into 819 so overlay/library show progress - CloudRedirect NotifyStatsStored still fires so its sync keeps working - New [stats] local_only knob (default true); owned games always use the online path --- README.md | 1 + opensteamtool.example.toml | 6 + src/CMakeLists.txt | 1 + src/Hook/Hooks_NetPacket.cpp | 245 ++++++++++++++++++++++++++++++++- src/Utils/Config/Config.cpp | 10 ++ src/Utils/Config/Config.h | 6 + src/Utils/Stats/LocalStats.cpp | 141 +++++++++++++++++++ src/Utils/Stats/LocalStats.h | 31 +++++ src/proto/steam_messages.proto | 46 +++++++ 9 files changed, 481 insertions(+), 6 deletions(-) create mode 100644 src/Utils/Stats/LocalStats.cpp create mode 100644 src/Utils/Stats/LocalStats.h diff --git a/README.md b/README.md index 7dab56e1..27363752 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ The `extract_tickets` tool dumps the `AppTicket` and `ETicket` hex strings you n - Uses `setStat(appid, "steamid")` to configure which SteamID's achievement data to pull. - If no `setStat` is configured for an app, OpenSteamTool queries `https://stats.opensteamtool.com/{appid}` when `[stats] enable_api = true` (default). - Priority: `setStat` > stats API when enabled and valid > hardcoded preset SteamID `76561198028121353`. +- With `[stats] local_only = true` (default), achievement unlocks for added games stay local: stores are journaled under `/opensteamtool/stats/` and answered locally, so toasts and the library page work, CloudRedirect sync still runs, and nothing is written to your online profile. Set `local_only = false` to let stores reach Valve (unlocks will show on your profile). ### Online Fix - Add `-onlinefix` to the Steam launch parameters to enable 480-based online play in games that use lobby matchmaking. The current limitation is that only one such game can run at a time.To revert, simply remove -onlinefix from the launch parameters — online play returns to normal on the next launch. diff --git a/opensteamtool.example.toml b/opensteamtool.example.toml index 5694f6db..39c83cd5 100644 --- a/opensteamtool.example.toml +++ b/opensteamtool.example.toml @@ -82,6 +82,12 @@ enabled = true # SteamID when no Lua setStat(appid, "steamid") override exists. # Priority: setStat > stats API when enabled and valid > hardcoded preset SteamID. enable_api = true +# Keep achievement unlocks for Lua-unlocked games local: stat stores are +# journaled under /opensteamtool/stats and answered locally (toast + +# library still work, CloudRedirect sync still runs) instead of reaching +# Valve, so nothing lands on the online profile. Set to false to let stores +# flow to Valve as your account (unlocks WILL show on your profile). +local_only = true [update] # Check for a newer BetterSteamTools build on startup. When a newer build is diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 685bbc47..a9e56c9b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -121,6 +121,7 @@ add_library(OpenSteamTool SHARED Utils/SteamMetadata/RemoteToml.cpp Utils/SteamMetadata/StatsClient.cpp Utils/SteamMetadata/SteamDiagnostics.cpp + Utils/Stats/LocalStats.cpp Utils/Tokeer/TokeerBridge.cpp Utils/Tokeer/TokeerUri.cpp Utils/Update/AppUpdater.cpp diff --git a/src/Hook/Hooks_NetPacket.cpp b/src/Hook/Hooks_NetPacket.cpp index f21c6b9f..53e5ed51 100644 --- a/src/Hook/Hooks_NetPacket.cpp +++ b/src/Hook/Hooks_NetPacket.cpp @@ -15,6 +15,7 @@ #include "Utils/Tickets/EticketClient.h" #include "Utils/Support/FnvHash.h" #include "Utils/CloudRedirect/CloudRedirectHost.h" +#include "Utils/Stats/LocalStats.h" #include #include #include @@ -374,8 +375,191 @@ namespace Hooks_NetPacket_UserStats { return true; } + // ── Send: CMsgClientStoreUserStats (eMsg 820) / StoreUserStats2 (5466) ── + // + // Local-only stat stores. With [stats] local_only (default), stores for + // Lua-unlocked games never reach Valve: the (stat_id, stat_value) pairs + // are journaled to LocalStats (achievements travel as stat bits — there + // is no separate unlock list on the wire), CloudRedirect is still + // notified so its sync keeps working, the outbound frame is suppressed, + // and a synthesized 821 success is delivered so the game sees StoreStats + // succeed and the client shows the unlock toast. Nothing lands on the + // online profile. Owned games are untouched (normal online behavior). + std::mutex g_storeMutex; + std::deque> g_storePending; // ready-to-inject 821 frames + + void QueueStoreResponse(std::vector&& pkt) { + std::lock_guard lk(g_storeMutex); + if (g_storePending.size() < 64) + g_storePending.push_back(std::move(pkt)); + } + + // Deliver queued 821 frames by borrowing the carrier packet for one + // oRecvPkt call each (same trick as Cloud/LegacyKey). Runs on the network + // thread from inside the RecvPkt hook. + void DrainStoreResponses(void* pThis, CNetPacket* pCarrier, + bool (*invokeOriginal)(void*, CNetPacket*)) + { + for (;;) { + std::vector pkt; + { + std::lock_guard lk(g_storeMutex); + if (g_storePending.empty()) return; + pkt = std::move(g_storePending.front()); + g_storePending.pop_front(); + } + + uint8* origData = pCarrier->m_pubData; + uint32 origSize = pCarrier->m_cubData; + pCarrier->m_pubData = pkt.data(); + pCarrier->m_cubData = static_cast(pkt.size()); + invokeOriginal(pThis, pCarrier); + pCarrier->m_pubData = origData; + pCarrier->m_cubData = origSize; + LOG_ACHIEVEMENT_DEBUG("StoreUserStats: delivered {}-byte synthesized response", pkt.size()); + } + } + + // Build + queue an 821 success frame. jobId/steamId come from the request + // header when present (correlates the reply); protoHdr mirrors the + // request envelope (raw 820 in, raw 821 out). + void QueueStoreSuccess(uint64_t gameId, uint32_t crc, uint64_t jobId, + uint64_t steamId, bool protoHdr) { + CMsgClientStoreUserStatsResponse resp; + if (gameId) resp.set_game_id(gameId); + resp.set_eresult(static_cast(k_EResultOK)); + resp.set_crc_stats(crc); + + const uint32 cbBody = static_cast(resp.ByteSizeLong()); + + uint8 hdrBuf[kMaxHdrSize]; + uint32 cbHdr = 0; + if (protoHdr) { + CMsgProtoBufHeader respHdr; + if (jobId) respHdr.set_jobid_target(jobId); + if (steamId) respHdr.set_steamid(steamId); + respHdr.set_eresult(static_cast(k_EResultOK)); + cbHdr = static_cast(respHdr.ByteSizeLong()); + if (cbHdr > kMaxHdrSize || + !respHdr.SerializeToArray(hdrBuf, kMaxHdrSize)) + return; + } + + const uint32 total = sizeof(MsgHdr) + cbHdr + cbBody; + if (cbBody > kMaxBodySize || total > kMaxPacketSize) return; + + std::vector pkt(total); + auto* mhdr = reinterpret_cast(pkt.data()); + mhdr->eMsg = static_cast(static_cast(k_EMsgClientStoreUserStatsResponse) | + (protoHdr ? kMsgHdrProtoFlag : 0)); + mhdr->headerLength = cbHdr; + if (cbHdr) memcpy(pkt.data() + sizeof(MsgHdr), hdrBuf, cbHdr); + if (!resp.SerializeToArray(pkt.data() + sizeof(MsgHdr) + cbHdr, cbBody)) + return; + QueueStoreResponse(std::move(pkt)); + } + + // Shared tail: journal (or clear), notify CR, synthesize success. + // Returns true so the caller suppresses the outbound frame. + bool SuppressStore(AppId_t appId, uint64_t gameId, uint32_t crc, + const std::vector>& pairs, + bool explicitReset, + const CMsgProtoBufHeader* reqHdr, bool protoHdr) { + if (explicitReset) + LocalStats::Clear(appId); + else + LocalStats::RecordStore(appId, pairs); + // CloudRedirect sync keeps working off this notification. + CloudRedirectHost::NotifyStatsStored(appId); + + uint64_t jobId = 0, steamId = 0; + if (reqHdr) { + if (reqHdr->has_jobid_source()) jobId = reqHdr->jobid_source(); + if (reqHdr->has_steamid()) steamId = reqHdr->steamid(); + } + QueueStoreSuccess(gameId, crc, jobId, steamId, protoHdr); + LOG_ACHIEVEMENT_INFO("StoreUserStats: handled locally app={} ({} pair(s), reset={}), " + "suppressed from server", appId, pairs.size(), explicitReset); + return true; + } + + // Returns true when the frame must be suppressed (handled locally). + bool HandleSend_StoreUserStats(const uint8* pBody, uint32 cbBody, + const uint8* pHdr, uint32 cbHdr) + { + if (!Config::GetStatsLocalOnly()) return false; + + CMsgClientStoreUserStats req; + if (!req.ParseFromArray(pBody, cbBody)) return false; + if (!req.has_game_id()) return false; + const AppId_t appId = static_cast(req.game_id()); + if (!LuaConfig::HasDepot(appId)) return false; // owned: online path + + std::vector> pairs; + pairs.reserve(static_cast(req.stats_to_store_size())); + for (const auto& s : req.stats_to_store()) + pairs.emplace_back(s.stat_id(), s.stat_value()); + + CMsgProtoBufHeader hdr; + const CMsgProtoBufHeader* ph = + hdr.ParseFromArray(pHdr, cbHdr) ? &hdr : nullptr; + return SuppressStore(appId, req.game_id(), 0, pairs, + req.explicit_reset(), ph, /*protoHdr=*/true); + } + + // Non-proto fallback: if 820 ever arrives without the proto flag it never + // reaches SendJob (UnpackRaw bails). Caught straight off the send hook; + // true = answered locally, suppress the real send. + bool HandleSend_StoreUserStats_Raw(const uint8* pubData, uint32 cubData) { + if (!Config::GetStatsLocalOnly()) return false; + if (cubData <= sizeof(MsgHdr)) return false; + + CMsgClientStoreUserStats req; + if (!req.ParseFromArray(pubData + sizeof(MsgHdr), cubData - sizeof(MsgHdr))) + return false; + if (!req.has_game_id()) return false; + const AppId_t appId = static_cast(req.game_id()); + if (!LuaConfig::HasDepot(appId)) return false; // owned: online path + + std::vector> pairs; + pairs.reserve(static_cast(req.stats_to_store_size())); + for (const auto& s : req.stats_to_store()) + pairs.emplace_back(s.stat_id(), s.stat_value()); + + return SuppressStore(appId, req.game_id(), 0, pairs, + req.explicit_reset(), nullptr, /*protoHdr=*/false); + } + + // Returns true when the frame must be suppressed (handled locally). + bool HandleSend_StoreUserStats2(const uint8* pBody, uint32 cbBody, + const uint8* pHdr, uint32 cbHdr) + { + if (!Config::GetStatsLocalOnly()) return false; + + CMsgClientStoreUserStats2 req; + if (!req.ParseFromArray(pBody, cbBody)) return false; + if (!req.has_game_id()) return false; + const AppId_t appId = static_cast(req.game_id()); + if (!LuaConfig::HasDepot(appId)) return false; // owned: online path + + std::vector> pairs; + pairs.reserve(static_cast(req.stats_size())); + for (const auto& s : req.stats()) + pairs.emplace_back(s.stat_id(), s.stat_value()); + + CMsgProtoBufHeader hdr; + const CMsgProtoBufHeader* ph = + hdr.ParseFromArray(pHdr, cbHdr) ? &hdr : nullptr; + // NOTE: 5466 has no dedicated ack message in the protocol; a best- + // effort 821 (game_id-keyed) is synthesized so the client can toast. + // If some title ignores it, the journal + library display below still + // hold the unlock — check achievement.log for the path games use. + return SuppressStore(appId, req.game_id(), req.crc_stats(), pairs, + req.explicit_reset(), ph, /*protoHdr=*/true); + } + // ── Recv: CMsgClientGetUserStatsResponse (eMsg 819) ──────── - // Clear donor stats, overlay CR achievements, patch eresult->OK. + // Clear donor stats, overlay local journal + CR achievements, patch eresult->OK. bool HandleRecv_ClientGetUserStatsResponse(const uint8* pBody, uint32 cbBody) { CMsgClientGetUserStatsResponse resp; @@ -390,7 +574,10 @@ namespace Hooks_NetPacket_UserStats { resp.clear_achievement_blocks(); resp.set_eresult(1); // k_EResultOK - // Overlay CR's cloud-synced achievement state + // Donor data is gone. Overlay order: CloudRedirect's synced state + // first (it wins on conflict), then the local journal for everything + // CR does not cover — progress earned locally always shows in the + // overlay and on the library page, with or without CR. const auto appId = static_cast(resp.game_id()); CloudRedirectHost::AchievementBlock blocks[64]; uint32_t n = CloudRedirectHost::GetAchievements(appId, blocks, 64); @@ -409,8 +596,29 @@ namespace Hooks_NetPacket_UserStats { resp.set_crc_stats(crc); LOG_ACHIEVEMENT_DEBUG("ClientGetUserStats response: injected {} CR achievement blocks for app {}", n, appId); } else { - LOG_ACHIEVEMENT_DEBUG("ClientGetUserStats response: cleared stats/achievements (no CR data for app {})", appId); + LOG_ACHIEVEMENT_DEBUG("ClientGetUserStats response: no CR data for app {}", appId); + } + + // Local journal injection (capped so the message stays well under the + // replace pool). Stat values carry achievement bits, which the client + // maps through the schema for display. + constexpr size_t kMaxJournalInject = 512; + size_t injected = 0; + for (const auto& [statId, statValue] : LocalStats::GetStats(appId)) { + if (injected >= kMaxJournalInject) break; + bool covered = false; + for (uint32_t i = 0; i < n; ++i) { + if (blocks[i].statId == statId) { covered = true; break; } + } + if (covered) continue; + auto* s = resp.add_stats(); + s->set_stat_id(statId); + s->set_stat_value(statValue); + ++injected; } + if (injected) + LOG_ACHIEVEMENT_DEBUG("ClientGetUserStats response: injected {} local stat(s) for app {}", + injected, appId); auto newSize = resp.ByteSizeLong(); if (newSize > sizeof(g_NewBody)) { @@ -1812,10 +2020,19 @@ namespace { g_NeedReplaceSend = Hooks_NetPacket_UserStats::HandleSend_ClientGetUserStats(pBody, cbBody); return; + case k_EMsgClientStoreUserStats: { // 820 + // Local-only journal + synthesized ack; suppresses the real send + // so nothing reaches the online profile. + if (Hooks_NetPacket_UserStats::HandleSend_StoreUserStats(pBody, cbBody, pHdr, cbHdr)) + g_SuppressSend = true; + return; + } + case k_EMsgClientStoreUserStats2: { // 5466 - AppId_t appId = Hooks_NetPacket_RichPresence::g_PlayingAppId; - if (appId != 0) - CloudRedirectHost::NotifyStatsStored(appId); + // Same local-only path (game_id parsed from the body, which also + // fixes the old g_PlayingAppId approximation). + if (Hooks_NetPacket_UserStats::HandleSend_StoreUserStats2(pBody, cbBody, pHdr, cbHdr)) + g_SuppressSend = true; return; } @@ -1937,6 +2154,18 @@ namespace { } } + // 820 arriving WITHOUT the proto flag never reaches SendJob + // (UnpackRaw bails). Journal it locally all the same so the store + // cannot leak to the online profile through that path. + if (cubData >= sizeof(MsgHdr)) { + const uint32 rawEMsg = *reinterpret_cast(pubData); + if (!(rawEMsg & kMsgHdrProtoFlag) && + static_cast(rawEMsg) == k_EMsgClientStoreUserStats && + Hooks_NetPacket_UserStats::HandleSend_StoreUserStats_Raw(pubData, cubData)) { + return true; // journaled locally; report success so Steam treats it as sent + } + } + EMsg eMsg; const uint8 *pHdr, *pBody; uint32 cbHdr, cbBody; @@ -1981,6 +2210,10 @@ namespace { pThis, pPacket, [](void* pT, CNetPacket* pP) -> bool { return oRecvPkt(pT, pP) != nullptr; }); + Hooks_NetPacket_UserStats::DrainStoreResponses( + pThis, pPacket, + [](void* pT, CNetPacket* pP) -> bool { return oRecvPkt(pT, pP) != nullptr; }); + EMsg eMsg; const uint8 *pBody, *pHdr; uint32 cbBody, cbHdr; diff --git a/src/Utils/Config/Config.cpp b/src/Utils/Config/Config.cpp index fc84b696..8d28fea0 100644 --- a/src/Utils/Config/Config.cpp +++ b/src/Utils/Config/Config.cpp @@ -18,6 +18,7 @@ namespace { std::vector luaPaths; std::vector remoteUrlTemplates; bool statsEnableApi = true; + bool statsLocalOnly = true; bool updateEnabled = true; DonateSettings donate; std::vector injectDlls; @@ -55,6 +56,7 @@ namespace { luaPaths = snapshot.luaPaths; remoteUrlTemplates = snapshot.remoteUrlTemplates; statsEnableApi = snapshot.statsEnableApi; + statsLocalOnly = snapshot.statsLocalOnly; updateEnabled = snapshot.updateEnabled; donate = snapshot.donate; injectDlls = snapshot.injectDlls; @@ -154,6 +156,9 @@ namespace { if (auto val = (*stats)["enable_api"].value()) { snapshot.statsEnableApi = *val; } + if (auto val = (*stats)["local_only"].value()) { + snapshot.statsLocalOnly = *val; + } } // [update] @@ -289,6 +294,11 @@ namespace { return statsEnableApi; } + bool GetStatsLocalOnly() { + std::lock_guard lock(g_mutex); + return statsLocalOnly; + } + DonateSettings GetDonateSettings() { std::lock_guard lock(g_mutex); return donate; diff --git a/src/Utils/Config/Config.h b/src/Utils/Config/Config.h index ad6fbe03..8ad777fa 100644 --- a/src/Utils/Config/Config.h +++ b/src/Utils/Config/Config.h @@ -45,6 +45,7 @@ namespace Config { std::vector GetRemoteUrlTemplates(); CloudSettings GetCloudSettings(); bool GetStatsEnableApi(); + bool GetStatsLocalOnly(); bool GetUpdateEnabled(); // [donate] — contribute manifest request codes for depots this account owns. @@ -79,6 +80,11 @@ namespace Config { // [stats] inline bool statsEnableApi = true; + // [stats] local_only — keep achievement unlocks for Lua-unlocked games + // local: stat stores are journaled under /opensteamtool/stats and + // answered locally (toast + library still work) instead of reaching Valve, + // so nothing lands on the online profile. CloudRedirect sync still runs. + inline bool statsLocalOnly = true; // [update] - self-update check on startup (staged for next Steam launch). inline bool updateEnabled = true; diff --git a/src/Utils/Stats/LocalStats.cpp b/src/Utils/Stats/LocalStats.cpp new file mode 100644 index 00000000..b8ce9fa3 --- /dev/null +++ b/src/Utils/Stats/LocalStats.cpp @@ -0,0 +1,141 @@ +#include "LocalStats.h" + +#include "OSTPlatform/include/DynamicLibrary.h" +#include "Utils/Logging/Log.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace LocalStats { +namespace { + + constexpr uint32_t kFileMagic = 0x4C535441u; // 'LSTA' + constexpr uint32_t kMaxEntries = 4096; // sanity cap per app + constexpr uintmax_t kMaxFileBytes = 64u * 1024u; + + std::mutex g_mutex; + // appId -> (stat_id -> stat_value). Populated lazily from disk. + std::unordered_map> g_journal; + std::unordered_map g_loaded; + + fs::path StatsDir() { + const fs::path steamExe = OSTPlatform::DynamicLibrary::GetMainExecutablePath(); + if (steamExe.empty()) return {}; + return steamExe.parent_path() / "opensteamtool" / "stats"; + } + + fs::path StatsFile(AppId_t appId) { + return StatsDir() / (std::to_string(appId) + ".dat"); + } + + void LoadLocked(AppId_t appId) { + if (g_loaded.count(appId)) return; + g_loaded[appId] = true; + + std::error_code ec; + const fs::path path = StatsFile(appId); + const uintmax_t size = fs::file_size(path, ec); + if (ec || size < 8 || size > kMaxFileBytes) return; + + std::ifstream ifs(path, std::ios::binary); + if (!ifs) return; + uint32_t magic = 0, count = 0; + ifs.read(reinterpret_cast(&magic), 4); + ifs.read(reinterpret_cast(&count), 4); + if (!ifs || magic != kFileMagic || count > kMaxEntries) return; + + auto& slot = g_journal[appId]; + for (uint32_t i = 0; i < count; ++i) { + uint32_t id = 0, value = 0; + ifs.read(reinterpret_cast(&id), 4); + ifs.read(reinterpret_cast(&value), 4); + if (!ifs) break; + slot[id] = value; + } + LOG_ACHIEVEMENT_DEBUG("LocalStats: loaded {} stat(s) for app {}", slot.size(), appId); + } + + void SaveLocked(AppId_t appId) { + const auto it = g_journal.find(appId); + if (it == g_journal.end()) return; + + std::error_code ec; + fs::create_directories(StatsDir(), ec); + + // Write temp + rename so a crash never leaves a torn journal. + const fs::path dest = StatsFile(appId); + const fs::path tmp = fs::path(dest).concat(".tmp"); + { + std::ofstream ofs(tmp, std::ios::binary | std::ios::trunc); + if (!ofs) { + LOG_ACHIEVEMENT_WARN("LocalStats: cannot open {} for app {}", tmp.string(), appId); + return; + } + const uint32_t count = static_cast(it->second.size()); + ofs.write(reinterpret_cast(&kFileMagic), 4); + ofs.write(reinterpret_cast(&count), 4); + for (const auto& [id, value] : it->second) { + ofs.write(reinterpret_cast(&id), 4); + ofs.write(reinterpret_cast(&value), 4); + } + ofs.flush(); + if (!ofs) { + LOG_ACHIEVEMENT_WARN("LocalStats: write failed for app {}", appId); + fs::remove(tmp, ec); + return; + } + } + fs::rename(tmp, dest, ec); + if (ec) + LOG_ACHIEVEMENT_WARN("LocalStats: rename failed for app {} ({})", appId, ec.message()); + } + +} // namespace + +uint32_t RecordStore(AppId_t appId, + const std::vector>& pairs) { + if (!appId || pairs.empty()) return 0; + std::lock_guard lock(g_mutex); + LoadLocked(appId); + auto& slot = g_journal[appId]; + uint32_t merged = 0; + for (const auto& [id, value] : pairs) { + if (!slot.count(id) && slot.size() >= kMaxEntries) continue; + if (slot[id] != value) { + slot[id] = value; + ++merged; + } + } + if (merged) SaveLocked(appId); + LOG_ACHIEVEMENT_DEBUG("LocalStats: merged {} stat(s) for app {} ({} total)", + merged, appId, slot.size()); + return merged; +} + +void Clear(AppId_t appId) { + if (!appId) return; + std::lock_guard lock(g_mutex); + g_journal.erase(appId); + g_loaded[appId] = true; // do not resurrect from disk afterwards + std::error_code ec; + fs::remove(StatsFile(appId), ec); + LOG_ACHIEVEMENT_INFO("LocalStats: cleared journal for app {}", appId); +} + +std::vector> GetStats(AppId_t appId) { + std::lock_guard lock(g_mutex); + LoadLocked(appId); + const auto it = g_journal.find(appId); + if (it == g_journal.end()) return {}; + std::vector> out; + out.reserve(it->second.size()); + for (const auto& [id, value] : it->second) + out.emplace_back(id, value); + return out; +} + +} // namespace LocalStats diff --git a/src/Utils/Stats/LocalStats.h b/src/Utils/Stats/LocalStats.h new file mode 100644 index 00000000..f6e37c45 --- /dev/null +++ b/src/Utils/Stats/LocalStats.h @@ -0,0 +1,31 @@ +#pragma once + +// Local-only stat journal for unlocked games. +// +// Achievement unlocks travel as stat bits: the game ships (stat_id, +// stat_value) pairs in ClientStoreUserStats (820) / ClientStoreUserStats2 +// (5466). With [stats] local_only, those stores never reach Valve — they are +// journaled here, persisted under /opensteamtool/stats/.dat, +// and injected back into ClientGetUserStatsResponse (819) so the overlay and +// the library page show local progress. Nothing here ever leaves the machine +// except through CloudRedirect's own sync (which the user configures there). + +#include +#include +#include + +#include "Steam/Types.h" + +namespace LocalStats { + + // Merge store pairs into the journal (and persist). Returns merged count. + uint32_t RecordStore(AppId_t appId, + const std::vector>& pairs); + + // Drop all journaled stats for an app (explicit_reset from the game). + void Clear(AppId_t appId); + + // Snapshot of journaled (stat_id, stat_value) pairs. Empty when unknown. + std::vector> GetStats(AppId_t appId); + +} // namespace LocalStats diff --git a/src/proto/steam_messages.proto b/src/proto/steam_messages.proto index fd9a1f80..10564042 100644 --- a/src/proto/steam_messages.proto +++ b/src/proto/steam_messages.proto @@ -203,6 +203,52 @@ message CMsgClientGetUserStatsResponse { repeated .CMsgClientGetUserStatsResponse.Achievement_Blocks achievement_blocks = 6; } +// ============================================================ +// CMsgClientStoreUserStats (eMsg 820) / Response (eMsg 821) / +// CMsgClientStoreUserStats2 (eMsg 5466) +// Field layout mirrors SteamKit's steammessages_clientserver.proto. +// Achievements travel as stat bits: SetAchievement flips a bit in a stat +// and StoreStats ships (stat_id, stat_value) pairs — there is no separate +// per-achievement unlock list on the wire. +// ============================================================ +message CMsgClientStoreUserStats { + message Stats_To_Store { + optional uint32 stat_id = 1; + optional uint32 stat_value = 2; + } + + optional fixed64 game_id = 1; + optional bool explicit_reset = 2; + repeated .CMsgClientStoreUserStats.Stats_To_Store stats_to_store = 3; +} + +message CMsgClientStoreUserStatsResponse { + message Stats_Failed_Validation { + optional uint32 stat_id = 1; + optional uint32 reverted_stat_value = 2; + } + + optional fixed64 game_id = 1; + optional int32 eresult = 2 [default = 2]; + optional uint32 crc_stats = 3; + repeated .CMsgClientStoreUserStatsResponse.Stats_Failed_Validation stats_failed_validation = 4; + optional bool stats_out_of_date = 5; +} + +message CMsgClientStoreUserStats2 { + message Stats { + optional uint32 stat_id = 1; + optional uint32 stat_value = 2; + } + + optional fixed64 game_id = 1; + optional fixed64 settor_steam_id = 2; + optional fixed64 settee_steam_id = 3; + optional uint32 crc_stats = 4; + optional bool explicit_reset = 5; + repeated .CMsgClientStoreUserStats2.Stats stats = 6; +} + message CContentServerDirectory_GetManifestRequestCode_Request { optional uint32 app_id = 1; optional uint32 depot_id = 2;