From 862ad762f81a7aa2ae0a656da33539b948a1b4ed Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:22:28 +0200 Subject: [PATCH 001/131] A local no-replace rename falls back where the filesystem refuses it A local rename without replacing falls back where the filesystem refuses RENAME_NOREPLACE, RENAME_EXCL or hard links (FUSE, vfat, non-APFS volumes): a regular file moves by link and unlink, anything else by an existence check and a rename; Utility::moveIntoPlace chooses between replacing and not replacing --- utility/include/utility/rename.hpp | 36 +++++++++++- utility/source/utility/rename.cpp | 88 ++++++++++++++++++++++++------ 2 files changed, 104 insertions(+), 20 deletions(-) diff --git a/utility/include/utility/rename.hpp b/utility/include/utility/rename.hpp index d144470d..da278baf 100644 --- a/utility/include/utility/rename.hpp +++ b/utility/include/utility/rename.hpp @@ -10,8 +10,40 @@ namespace Utility * @brief Renames @p from to @p to and fails with std::errc::file_exists instead of replacing an existing @p to. * * Atomic where the platform offers it: renameat2 RENAME_NOREPLACE on Linux, renamex_np RENAME_EXCL on macOS, - * MoveFileExW without MOVEFILE_REPLACE_EXISTING on Windows. A Linux filesystem without RENAME_NOREPLACE falls - * back to link and unlink for files. + * MoveFileExW without MOVEFILE_REPLACE_EXISTING on Windows. Where the filesystem refuses those, a regular file + * moves by link and unlink on Linux, and anything else by @ref Detail::renameUnlessExists, which leaves a window + * between the check and the rename. */ std::expected renameNoReplace(std::filesystem::path const& from, std::filesystem::path const& to); + + /** + * @brief Moves a finished file from @p from to @p to. + * + * @param replace Whether an existing @p to is replaced. Without it, an existing @p to fails with + * std::errc::file_exists, see @ref renameNoReplace. + */ + std::expected + moveIntoPlace(std::filesystem::path const& from, std::filesystem::path const& to, bool replace); + + namespace Detail + { + /** + * @brief Whether @p error of an atomic no-replace rename or a hard link means the filesystem does not offer + * it, rather than that the move itself failed. + */ + bool isNoReplaceUnsupported(std::error_code error); + + /** + * @brief Moves the regular file @p from to @p to by a hard link and removing @p from. The link fails on an + * existing @p to. When @p from cannot be removed, the link is removed again. + */ + std::expected + renameByLink(std::filesystem::path const& from, std::filesystem::path const& to); + + /** + * @brief Renames @p from to @p to unless something, a dangling link included, exists at @p to. + */ + std::expected + renameUnlessExists(std::filesystem::path const& from, std::filesystem::path const& to); + } } diff --git a/utility/source/utility/rename.cpp b/utility/source/utility/rename.cpp index 184d5c59..c8b3b22b 100644 --- a/utility/source/utility/rename.cpp +++ b/utility/source/utility/rename.cpp @@ -13,9 +13,54 @@ namespace Utility { namespace { - std::unexpected lastErrno() +#ifndef _WIN32 + std::error_code lastErrno() { - return std::unexpected(std::error_code{errno, std::generic_category()}); + return std::error_code{errno, std::generic_category()}; + } +#endif + } + + namespace Detail + { + bool isNoReplaceUnsupported(std::error_code error) + { + return error == std::errc::invalid_argument || error == std::errc::function_not_supported || + error == std::errc::operation_not_supported || error == std::errc::not_supported || + error == std::errc::operation_not_permitted; + } + + std::expected + renameByLink(std::filesystem::path const& from, std::filesystem::path const& to) + { + std::error_code error{}; + std::filesystem::create_hard_link(from, to, error); + if (error) + return std::unexpected(error); + std::filesystem::remove(from, error); + if (error) + { + std::error_code undoError{}; + std::filesystem::remove(to, undoError); + return std::unexpected(error); + } + return {}; + } + + std::expected + renameUnlessExists(std::filesystem::path const& from, std::filesystem::path const& to) + { + std::error_code error{}; + const auto status = std::filesystem::symlink_status(to, error); + if (status.type() == std::filesystem::file_type::none) + return std::unexpected(error); + if (std::filesystem::exists(status)) + return std::unexpected(std::make_error_code(std::errc::file_exists)); + error.clear(); + std::filesystem::rename(from, to, error); + if (error) + return std::unexpected(error); + return {}; } } @@ -31,30 +76,37 @@ namespace Utility } return {}; #elif defined(__APPLE__) - if (::renamex_np(from.c_str(), to.c_str(), RENAME_EXCL) != 0) - return lastErrno(); - return {}; + if (::renamex_np(from.c_str(), to.c_str(), RENAME_EXCL) == 0) + return {}; + if (const auto error = lastErrno(); !Detail::isNoReplaceUnsupported(error)) + return std::unexpected(error); + return Detail::renameUnlessExists(from, to); #else if (::renameat2(AT_FDCWD, from.c_str(), AT_FDCWD, to.c_str(), RENAME_NOREPLACE) == 0) return {}; - if (errno != EINVAL && errno != ENOSYS) - return lastErrno(); + if (const auto error = lastErrno(); !Detail::isNoReplaceUnsupported(error)) + return std::unexpected(error); - // The filesystem cannot rename without replacing. A hard link fails on an existing target as well. std::error_code statusError{}; if (std::filesystem::is_regular_file(std::filesystem::symlink_status(from, statusError))) { - if (::link(from.c_str(), to.c_str()) != 0) - return lastErrno(); - if (::unlink(from.c_str()) != 0) - return lastErrno(); - return {}; + const auto linked = Detail::renameByLink(from, to); + if (linked || !Detail::isNoReplaceUnsupported(linked.error())) + return linked; } - if (std::filesystem::exists(std::filesystem::symlink_status(to, statusError))) - return std::unexpected(std::make_error_code(std::errc::file_exists)); - if (::rename(from.c_str(), to.c_str()) != 0) - return lastErrno(); - return {}; + return Detail::renameUnlessExists(from, to); #endif } + + std::expected + moveIntoPlace(std::filesystem::path const& from, std::filesystem::path const& to, bool replace) + { + if (!replace) + return renameNoReplace(from, to); + std::error_code error{}; + std::filesystem::rename(from, to, error); + if (error) + return std::unexpected(error); + return {}; + } } From 26ddd8eaae3b5f4f74394f641a44b311c49d6b05 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:26:22 +0200 Subject: [PATCH 002/131] A remote no-replace rename falls back when the server refuses hard links Remote renames without replacing fall back to a rename when the server refuses the hard link (vfat, CIFS, FUSE, protected_hardlinks), and a moved file whose source cannot be removed afterwards counts as moved; without posix-rename the server's own rename refuses an existing destination, so the destination is only checked after a failure. Replacing without posix-rename removes a leftover aside, puts back a destination that only exists aside, and names the aside when moving it back fails; moveIntoPlaceInStrand chooses between the two --- ssh/include/ssh/sftp_session.hpp | 52 ++++++++-- ssh/source/ssh/sftp_session.cpp | 169 ++++++++++++++++++++++--------- 2 files changed, 168 insertions(+), 53 deletions(-) diff --git a/ssh/include/ssh/sftp_session.hpp b/ssh/include/ssh/sftp_session.hpp index 49928968..0131e5c9 100644 --- a/ssh/include/ssh/sftp_session.hpp +++ b/ssh/include/ssh/sftp_session.hpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -271,16 +273,26 @@ namespace SecureShell renameInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination); /** - * @brief Renames without ever replacing @p destination. Fails with SSH_FX_FILE_ALREADY_EXISTS when it - * exists. Regular files move atomically through hardlink@openssh.com where the server has it. + * @brief Renames without replacing @p destination. Fails with SSH_FX_FILE_ALREADY_EXISTS when it exists. + * + * A regular file moves atomically through hardlink@openssh.com where the server has it and the link + * succeeds. Otherwise a server without posix-rename@openssh.com refuses an existing destination itself, + * and on one with it the destination is checked before the rename, which leaves a window. + * + * @param sourceType The type of @p source when the caller knows it, saves an lstat. */ - std::expected - renameNoReplaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination); + std::expected renameNoReplaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + std::optional sourceType = std::nullopt + ); /** - * @brief Renames over an existing @p destination without a moment where it is missing: atomically with - * posix-rename@openssh.com, otherwise by moving the destination aside to @p asideSuffix and - * back if the rename fails. + * @brief Renames over an existing @p destination. + * + * Atomic with posix-rename@openssh.com. Otherwise the destination is moved aside to @p asideSuffix first + * and moved back when the rename fails; if moving it back fails too, the error names where it is kept. + * A leftover aside is removed when the destination exists, and put back when it does not. */ std::expected renameReplaceInStrand( Paths::RemotePath const& source, @@ -288,6 +300,18 @@ namespace SecureShell std::string_view asideSuffix ); + /** + * @brief Moves a finished transfer from @p source to @p destination with @ref renameReplaceInStrand when + * @p replace is set, otherwise with @ref renameNoReplaceInStrand. + */ + std::expected moveIntoPlaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + bool replace, + std::string_view asideSuffix, + std::optional sourceType = std::nullopt + ); + /** * @brief Whether the server announced the SFTP extension @p name with @p version. */ @@ -409,6 +433,20 @@ namespace SecureShell */ static std::expected requireSet(Paths::RemotePath const& path); + static SftpError alreadyExists(Paths::RemotePath const& destination); + + /** + * @brief Moves the regular file @p source to @p destination by a hard link and removing @p source, or by a + * rename when the server refuses the link and @p destination does not exist. + */ + std::expected + hardlinkIntoPlaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination); + + /** + * @brief Renames @p destination to @p aside, removing a file left at @p aside first when the rename fails. + */ + std::expected moveAsideInStrand(Paths::RemotePath const& destination, Paths::RemotePath const& aside); + /** * @brief Reads a file into a string, stopping after maximumSize bytes. */ diff --git a/ssh/source/ssh/sftp_session.cpp b/ssh/source/ssh/sftp_session.cpp index c5968330..fd32b5e4 100644 --- a/ssh/source/ssh/sftp_session.cpp +++ b/ssh/source/ssh/sftp_session.cpp @@ -1,8 +1,6 @@ #include -#include #include #include -#include #include #include @@ -522,43 +520,73 @@ namespace SecureShell return sftp_extension_supported(session_, name, version) == 1; } - std::expected - SftpSession::renameNoReplaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination) + std::expected SftpSession::renameNoReplaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + std::optional sourceType + ) { assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(source); !valid) - return std::unexpected(valid.error()); - if (auto valid = requireSet(destination); !valid) - return std::unexpected(valid.error()); - const auto alreadyExists = [&destination]() + const auto type = sourceType.has_value() + ? std::expected{*sourceType} + : lstatInStrand(source).transform( + [](FileInformation const& information) + { + return information.type; + } + ); + if (!type.has_value()) + return std::unexpected(type.error()); + + if (*type == SharedData::FileType::Regular && supportsExtensionInStrand("hardlink@openssh.com")) + return hardlinkIntoPlaceInStrand(source, destination); + + // libssh sends posix-rename@openssh.com when the server offers it, which replaces, while a plain SFTP v3 + // rename refuses an existing destination. + if (!supportsExtensionInStrand("posix-rename@openssh.com")) { - return SftpError{ - .message = fmt::format("'{}' already exists", destination.value()), - .sftpError = SSH_FX_FILE_ALREADY_EXISTS, - }; - }; + const auto renamed = renameInStrand(source, destination); + if (!renamed.has_value() && lstatInStrand(destination).has_value()) + return std::unexpected(alreadyExists(destination)); + return renamed; + } + + // Leaves a window between the check and the rename. + if (lstatInStrand(destination).has_value()) + return std::unexpected(alreadyExists(destination)); + return renameInStrand(source, destination); + } - const auto sourceInfo = lstatInStrand(source); - if (!sourceInfo.has_value()) - return std::unexpected(sourceInfo.error()); + std::expected + SftpSession::hardlinkIntoPlaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination) + { + if (auto valid = requireSetInStrand(source, destination); !valid) + return valid; - // A hard link fails when the destination exists, so the move cannot replace anything. - if (sourceInfo->isRegularFile() && supportsExtensionInStrand("hardlink@openssh.com")) + if (sftp_hardlink(session_, source.c_str(), destination.c_str()) == SSH_OK) { - if (sftp_hardlink(session_, source.c_str(), destination.c_str()) != SSH_OK) + if (auto removed = removeFileInStrand(source); !removed.has_value()) { - if (lstatInStrand(destination).has_value()) - return std::unexpected(alreadyExists()); - return std::unexpected(lastError()); + Log::warn( + "SftpSession: Moved '{}' to '{}' but could not remove '{}': {}", + source.value(), + destination.value(), + source.value(), + removed.error().toString() + ); } - return removeFileInStrand(source); + return {}; } - // libssh sends posix-rename@openssh.com when the server offers it, which replaces. Plain SFTP v3 rename - // refuses an existing destination. So only a server with posix-rename but without hardlink leaves the - // window between this check and the rename; OpenSSH offers both. - if (lstatInStrand(destination).has_value()) - return std::unexpected(alreadyExists()); + const auto linkError = lastError(); + const auto existing = lstatInStrand(destination); + if (existing.has_value()) + return std::unexpected(alreadyExists(destination)); + if (existing.error().sftpError != SSH_FX_NO_SUCH_FILE) + return std::unexpected(linkError); + + // Filesystems without hard links, and Linux protected_hardlinks, refuse the link. + Log::debug("SftpSession: Cannot link '{}': {}, renaming instead.", source.value(), linkError.toString()); return renameInStrand(source, destination); } @@ -569,42 +597,57 @@ namespace SecureShell ) { assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(source); !valid) - return std::unexpected(valid.error()); - if (auto valid = requireSet(destination); !valid) - return std::unexpected(valid.error()); if (supportsExtensionInStrand("posix-rename@openssh.com")) return renameInStrand(source, destination); - // Plain SFTP rename refuses an existing destination: move it aside instead of deleting it first. - const auto destinationInfo = lstatInStrand(destination); - if (!destinationInfo.has_value()) - return renameInStrand(source, destination); - if (destinationInfo->isDirectory()) + // Plain SFTP rename refuses an existing destination, so it is moved aside instead of deleted first. + const auto aside = destination.withFilenameSuffix(asideSuffix); + if (!aside) { return std::unexpected( SftpError{ - .message = fmt::format("'{}' is a directory", destination.value()), - .sftpError = SSH_FX_FAILURE, + .message = fmt::format("Invalid suffix '{}' for '{}'", asideSuffix, destination.value()), + .wrapperError = WrapperErrors::InvalidPath, } ); } - const auto aside = destination.withFilenameSuffix(asideSuffix); - if (!aside) + const auto destinationInfo = lstatInStrand(destination); + if (!destinationInfo.has_value()) + { + if (destinationInfo.error().sftpError != SSH_FX_NO_SUCH_FILE || !lstatInStrand(*aside).has_value()) + return renameInStrand(source, destination); + // A destination that only exists aside, from a rollback that failed, is put back first. + if (auto restored = renameInStrand(*aside, destination); !restored.has_value()) + return restored; + } + else if (destinationInfo->isDirectory()) { return std::unexpected( SftpError{ - .message = fmt::format("Invalid suffix '{}' for '{}'", asideSuffix, destination.value()), - .wrapperError = WrapperErrors::InvalidPath, + .message = fmt::format("'{}' is a directory", destination.value()), + .sftpError = SSH_FX_FAILURE, } ); } - if (auto movedAside = renameInStrand(destination, *aside); !movedAside.has_value()) + + if (auto movedAside = moveAsideInStrand(destination, *aside); !movedAside.has_value()) return movedAside; if (auto renamed = renameInStrand(source, destination); !renamed.has_value()) { - std::ignore = renameInStrand(*aside, destination); + if (auto restored = renameInStrand(*aside, destination); !restored.has_value()) + { + Log::error( + "SftpSession: Could not restore '{}' from '{}': {}", + destination.value(), + aside->value(), + restored.error().toString() + ); + auto error = renamed.error(); + error.message = + fmt::format("{}. The previous '{}' is kept as '{}'", error.message, destination.value(), aside->value()); + return std::unexpected(std::move(error)); + } return renamed; } if (auto removed = removeFileInStrand(*aside); !removed.has_value()) @@ -612,6 +655,40 @@ namespace SecureShell return {}; } + std::expected + SftpSession::moveAsideInStrand(Paths::RemotePath const& destination, Paths::RemotePath const& aside) + { + const auto movedAside = renameInStrand(destination, aside); + if (movedAside.has_value() || !lstatInStrand(aside).has_value()) + return movedAside; + + // A leftover of an earlier replace holds the name. + if (auto removed = removeFileInStrand(aside); !removed.has_value()) + return removed; + return renameInStrand(destination, aside); + } + + std::expected SftpSession::moveIntoPlaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + bool replace, + std::string_view asideSuffix, + std::optional sourceType + ) + { + if (replace) + return renameReplaceInStrand(source, destination, asideSuffix); + return renameNoReplaceInStrand(source, destination, sourceType); + } + + SftpError SftpSession::alreadyExists(Paths::RemotePath const& destination) + { + return SftpError{ + .message = fmt::format("'{}' already exists", destination.value()), + .sftpError = SSH_FX_FILE_ALREADY_EXISTS, + }; + } + std::future> SftpSession::rename(Paths::RemotePath const& source, Paths::RemotePath const& destination) { From dd91bce7433f507cf16067dbab772c9dd2327e97 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:28:51 +0200 Subject: [PATCH 003/131] Transfers and renames move into place through one dispatcher per side Finished transfers and renames move into place through one dispatcher per side, Utility::moveIntoPlace and SftpSession::moveIntoPlaceInStrand, and the error mapping lives in Operation; a remote rename that replaces moves the old item aside under the configured upload temp suffix instead of a fixed .nui-sftp-old, and uploads pass the known file type so the no-replace move skips an lstat --- backend/include/backend/session.hpp | 2 + backend/include/backend/sftp/operation.hpp | 25 ++++++++++ backend/source/backend/rpc_filesystem.cpp | 21 ++++---- backend/source/backend/session.cpp | 19 ++++++-- .../sftp/archive_download_operation.cpp | 18 +++---- .../backend/sftp/archive_upload_operation.cpp | 18 +++---- .../backend/sftp/download_operation.cpp | 19 ++------ .../source/backend/sftp/upload_operation.cpp | 48 +++++++++---------- .../persistence/state/sftp_options.hpp | 12 +++++ 9 files changed, 107 insertions(+), 75 deletions(-) diff --git a/backend/include/backend/session.hpp b/backend/include/backend/session.hpp index 35d5571b..bc0d81d3 100644 --- a/backend/include/backend/session.hpp +++ b/backend/include/backend/session.hpp @@ -297,4 +297,6 @@ class Session BulkResumeRegistry* bulkResumeRegistry_{}; std::unordered_map, Ids::IdHash> channels_{}; std::unordered_map, Ids::IdHash> sftpChannels_{}; + /** @brief The temp file suffix of uploads, which names where a replaced item is kept during a rename. */ + std::string remoteTempFileSuffix_; }; \ No newline at end of file diff --git a/backend/include/backend/sftp/operation.hpp b/backend/include/backend/sftp/operation.hpp index 683d2f28..bcf30d1a 100644 --- a/backend/include/backend/sftp/operation.hpp +++ b/backend/include/backend/sftp/operation.hpp @@ -12,8 +12,11 @@ #include +#include + #include #include +#include class Operation { @@ -165,6 +168,28 @@ class Operation state_ = newState; } + /** + * @brief The error of a finished transfer that could not be moved into place on the server: FileExists when the + * destination exists and may not be replaced. + */ + static Error remoteMoveError(SecureShell::SftpError error) + { + if (error.sftpError == SSH_FX_FILE_ALREADY_EXISTS) + return Error{.type = ErrorType::FileExists}; + return Error{.type = ErrorType::SftpError, .sftpError = std::move(error)}; + } + + /** + * @brief The error of a finished transfer that could not be moved into place on this machine: FileExists when the + * destination exists and may not be replaced. + */ + static Error localMoveError(std::error_code error) + { + if (error == std::errc::file_exists) + return Error{.type = ErrorType::FileExists}; + return Error{.type = ErrorType::RenameFailure}; + } + /** * @brief The error of an async transfer that ended without completing. */ diff --git a/backend/source/backend/rpc_filesystem.cpp b/backend/source/backend/rpc_filesystem.cpp index bdf0f912..3a7060fc 100644 --- a/backend/source/backend/rpc_filesystem.cpp +++ b/backend/source/backend/rpc_filesystem.cpp @@ -376,18 +376,17 @@ void RpcFilesystem::registerRename() return reply.error("backend.rename.invalidName"); } - // Never replaces an existing item unless the user confirmed it. - std::error_code ec; - if (overwrite) - std::filesystem::rename(oldPath, newPath, ec); - else if (auto renamed = Utility::renameNoReplace(oldPath, newPath); !renamed.has_value()) - ec = renamed.error(); - if (ec == std::errc::file_exists) - return reply.error("backend.rename.targetExists"); - if (ec) + if (const auto moved = Utility::moveIntoPlace(oldPath, newPath, overwrite); !moved) { - Log::error("Failed to rename from '{}' to '{}': {}", oldPathUtf8, newPathUtf8, ec.message()); - return reply.error(ec.message()); + if (moved.error() == std::errc::file_exists) + return reply.error("backend.rename.targetExists"); + Log::error( + "Failed to rename from '{}' to '{}': {}", + oldPathUtf8, + newPathUtf8, + Utility::errorMessage(moved.error()) + ); + return reply.error(Utility::errorMessage(moved.error())); } Log::info("Successfully renamed from '{}' to '{}'", oldPathUtf8, newPathUtf8); diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index 38e4752e..6ad7eefd 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -95,6 +95,9 @@ Session::Session( , operationQueue_{std::make_shared< OperationQueue>(executor_, strand_, wnd, hub, sftpOptions, id_, sftpOptions.concurrency.value_or(Persistence::defaultSftpConcurrency))} , bulkResumeRegistry_{bulkResumeRegistry} + , remoteTempFileSuffix_{Persistence::effectiveTempFileSuffix( + sftpOptions.uploadOptions ? sftpOptions.uploadOptions->tempFileSuffix : std::nullopt + )} {} Session::~Session() @@ -1815,12 +1818,20 @@ void Session::registerRpcSftpRename() if (!oldPath || !newPath || !oldPath->parent() || oldPath->parent() != newPath->parent()) return reply.error("backend.rename.invalidName"); - // Never replaces an existing item unless the user confirmed it. auto fut = channel->performPromise( - [channel, oldPath = *oldPath, newPath = *newPath, overwrite]() + [session = std::weak_ptr{channel}, + oldPath = *oldPath, + newPath = *newPath, + overwrite, + asideSuffix = Persistence::asideFileSuffix(self.remoteTempFileSuffix_)]() + -> std::expected { - return overwrite ? channel->renameReplaceInStrand(oldPath, newPath, ".nui-sftp-old") - : channel->renameNoReplaceInStrand(oldPath, newPath); + const auto sftp = session.lock(); + if (!sftp) + return std::unexpected( + SecureShell::SftpError{.message = "The SFTP channel was closed"} + ); + return sftp->moveIntoPlaceInStrand(oldPath, newPath, overwrite, asideSuffix); } ); if (fut.wait_for(futureTimeout) != std::future_status::ready) diff --git a/backend/source/backend/sftp/archive_download_operation.cpp b/backend/source/backend/sftp/archive_download_operation.cpp index 53986882..02c5beed 100644 --- a/backend/source/backend/sftp/archive_download_operation.cpp +++ b/backend/source/backend/sftp/archive_download_operation.cpp @@ -665,18 +665,12 @@ std::expected ArchiveDownloadOperation::finaliseArchiveI Log::warn("ArchiveDownloadOperation: Cannot mark the archive as downloaded: {}", marked.message()); } - // Without permission to overwrite, the rename itself refuses an existing archive: no check-then-rename race. - std::error_code renameErr{}; - if (options_.mayOverwrite) - std::filesystem::rename(tempPath_, localArchiveNativePath_, renameErr); - else if (auto renamed = Utility::renameNoReplace(tempPath_, localArchiveNativePath_); !renamed.has_value()) - renameErr = renamed.error(); - if (renameErr == std::errc::file_exists) - return std::unexpected(Error{.type = ErrorType::FileExists}); - if (renameErr) - { - Log::error("ArchiveDownloadOperation: rename failed: {}", renameErr.message()); - return std::unexpected(Error{.type = ErrorType::RenameFailure}); + if (const auto moved = Utility::moveIntoPlace(tempPath_, localArchiveNativePath_, options_.mayOverwrite); !moved) + { + Log::error( + "ArchiveDownloadOperation: Failed to move the archive into place: {}", Utility::errorMessage(moved.error()) + ); + return std::unexpected(localMoveError(moved.error())); } return {}; } diff --git a/backend/source/backend/sftp/archive_upload_operation.cpp b/backend/source/backend/sftp/archive_upload_operation.cpp index c406cfce..fbfb1cce 100644 --- a/backend/source/backend/sftp/archive_upload_operation.cpp +++ b/backend/source/backend/sftp/archive_upload_operation.cpp @@ -646,19 +646,21 @@ std::expected ArchiveUploadOperation::finaliseArchiveInS stream->close(false); remoteStream_.reset(); - const auto renameResult = options_.mayOverwrite - ? sftp_->renameReplaceInStrand(tempPath_, remoteArchivePath_, options_.tempFileSuffix + ".old") - : sftp_->renameNoReplaceInStrand(tempPath_, remoteArchivePath_); - if (!renameResult.has_value()) + const auto moved = sftp.moveIntoPlaceInStrand( + tempPath_, + remoteArchivePath_, + options_.mayOverwrite, + Persistence::asideFileSuffix(options_.tempFileSuffix), + SharedData::FileType::Regular + ); + if (!moved.has_value()) { Log::error( "ArchiveUploadOperation: cannot move the archive to '{}': {}", options_.remoteArchivePath.value(), - renameResult.error().message + moved.error().message ); - if (renameResult.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) - return std::unexpected(Error{.type = ErrorType::FileExists}); - return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = renameResult.error()}); + return std::unexpected(remoteMoveError(moved.error())); } tempCreated_ = false; return {}; diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index 0302e2fd..99e07493 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -770,23 +770,14 @@ std::expected DownloadOperation::finalizeInStran Log::warn("DownloadOperation: Cannot mark '{}' as downloaded: {}", Paths::nativeToUtf8(tempPath), marked.message()); } - // Without permission to overwrite, the rename itself refuses an existing file: no check-then-rename race. - std::error_code ec{}; - if (options_.mayOverwrite) - std::filesystem::rename(tempPath, localNativePath_, ec); - else if (auto renamed = Utility::renameNoReplace(tempPath, localNativePath_); !renamed.has_value()) - ec = renamed.error(); - if (ec == std::errc::file_exists) + if (const auto moved = Utility::moveIntoPlace(tempPath, localNativePath_, options_.mayOverwrite); !moved) { Log::error( - "DownloadOperation: File '{}' already exists and may not be overwritten.", options_.localPath.toUtf8() + "DownloadOperation: Failed to move '{}' into place: {}", + options_.localPath.toUtf8(), + Utility::errorMessage(moved.error()) ); - return std::unexpected(Error{.type = ErrorType::FileExists}); - } - if (ec) - { - Log::error("DownloadOperation: Failed to rename file: {}", ec.message()); - return std::unexpected(Error{.type = ErrorType::RenameFailure}); + return std::unexpected(localMoveError(moved.error())); } if (options_.inheritPermissions) diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index c6fc4033..8d3769a0 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -329,20 +329,20 @@ std::expected UploadOperation::handleSymlinkInStra return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = result.error()}); } - const auto renameResult = options_.mayOverwrite - ? sftp_->renameReplaceInStrand(tempPath_, remotePath_, options_.tempFileSuffix + ".old") - : sftp_->renameNoReplaceInStrand(tempPath_, remotePath_); - if (!renameResult.has_value()) + const auto moved = sftp_->moveIntoPlaceInStrand( + tempPath_, + remotePath_, + options_.mayOverwrite, + Persistence::asideFileSuffix(options_.tempFileSuffix), + SharedData::FileType::Symlink + ); + if (!moved.has_value()) { std::ignore = sftp_->removeFileInStrand(tempPath_); - if (renameResult.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) - return std::unexpected(Error{.type = ErrorType::FileExists}); Log::error( - "UploadOperation: Failed to move symlink to '{}': {}.", - options_.remotePath.value(), - renameResult.error().message + "UploadOperation: Failed to move symlink to '{}': {}.", options_.remotePath.value(), moved.error().message ); - return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = renameResult.error()}); + return std::unexpected(remoteMoveError(moved.error())); } return {}; } @@ -775,23 +775,19 @@ std::expected UploadOperation::finalizeInStrand() stream->closeInStrand(); - // The destination is never deleted ahead of the rename: replacing is atomic or moves it aside, and - // without permission to overwrite the rename fails instead of replacing. - const auto renameResult = options_.mayOverwrite - ? sftp_->renameReplaceInStrand(tempPath_, remotePath_, options_.tempFileSuffix + ".old") - : sftp_->renameNoReplaceInStrand(tempPath_, remotePath_); - if (!renameResult.has_value()) + const auto moved = sftp_->moveIntoPlaceInStrand( + tempPath_, + remotePath_, + options_.mayOverwrite, + Persistence::asideFileSuffix(options_.tempFileSuffix), + SharedData::FileType::Regular + ); + if (!moved.has_value()) { - if (renameResult.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) - { - Log::warn( - "UploadOperation: Remote file '{}' already exists and may not be overwritten.", - options_.remotePath.value() - ); - return std::unexpected(Error{.type = ErrorType::FileExists}); - } - Log::error("Failed to rename remote sftp file: {}", renameResult.error().message); - return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = renameResult.error()}); + Log::error("UploadOperation: Failed to move '{}' into place: {}", options_.remotePath.value(), moved.error().message); + if (auto removed = sftp_->removeFileInStrand(tempPath_); !removed.has_value()) + Log::warn("UploadOperation: Cannot remove '{}': {}", tempPath_.value(), removed.error().message); + return std::unexpected(remoteMoveError(moved.error())); } // Preserve the local file's mtime on the remote file so subsequent syncs diff --git a/persistence/include/persistence/state/sftp_options.hpp b/persistence/include/persistence/state/sftp_options.hpp index 41a068fc..1c93f371 100644 --- a/persistence/include/persistence/state/sftp_options.hpp +++ b/persistence/include/persistence/state/sftp_options.hpp @@ -2,6 +2,8 @@ #include +#include + #include #include #include @@ -34,6 +36,16 @@ namespace Persistence return *configured; } + /** + * @brief The suffix a destination is moved aside to while a finished transfer replaces it on a server without + * posix-rename. + * @param tempFileSuffix The effective temp file suffix, see effectiveTempFileSuffix. + */ + inline std::string asideFileSuffix(std::string_view tempFileSuffix) + { + return fmt::format("{}.old", tempFileSuffix); + } + struct CommonTransferOptions : public DefaultMissingMember { std::optional tempFileSuffix{std::nullopt}; From e604a0dff632f292005d7dd6f5bd078dfe11de39 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:30:35 +0200 Subject: [PATCH 004/131] A timed-out queued rename keeps its paths; transfers await their tasks A queued remote rename that timed out runs later with its own copies of the paths instead of the freed operation, and transfers wait for their queued strand tasks before they are destroyed (archives and bulks too, not only while a stream is open), without throwing on a finalized strand --- backend/include/backend/sftp/operation.hpp | 10 ++++++++++ .../source/backend/sftp/archive_download_operation.cpp | 6 +----- .../source/backend/sftp/archive_upload_operation.cpp | 3 +-- .../source/backend/sftp/bulk_transfer_operation.cpp | 1 + backend/source/backend/sftp/download_operation.cpp | 8 +------- backend/source/backend/sftp/rename_operation.cpp | 9 ++++++--- backend/source/backend/sftp/upload_operation.cpp | 5 +---- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/backend/include/backend/sftp/operation.hpp b/backend/include/backend/sftp/operation.hpp index bcf30d1a..373aedb3 100644 --- a/backend/include/backend/sftp/operation.hpp +++ b/backend/include/backend/sftp/operation.hpp @@ -168,6 +168,16 @@ class Operation state_ = newState; } + /** + * @brief Waits until every task queued on @p strand so far has run, so none of them outlives the operation that + * queued it. Returns at once on the processing thread itself and on a finalized strand. + */ + static void waitForStrandTasks(SecureShell::ProcessingStrand* strand) + { + if (strand && !strand->withinProcessingThread()) + strand->pushPromiseTask([]() {}).wait(); + } + /** * @brief The error of a finished transfer that could not be moved into place on the server: FileExists when the * destination exists and may not be replaced. diff --git a/backend/source/backend/sftp/archive_download_operation.cpp b/backend/source/backend/sftp/archive_download_operation.cpp index 02c5beed..edb7553a 100644 --- a/backend/source/backend/sftp/archive_download_operation.cpp +++ b/backend/source/backend/sftp/archive_download_operation.cpp @@ -136,11 +136,7 @@ ArchiveDownloadOperation::ArchiveDownloadOperation(SecureShell::SftpSession& sft ArchiveDownloadOperation::~ArchiveDownloadOperation() { std::ignore = cancel(false); - - if (auto stream = currentStream_.lock(); stream) - { - stream->strand()->pushPromiseTask([]() {}).get(); - } + waitForStrandTasks(strand()); } std::expected ArchiveDownloadOperation::work() diff --git a/backend/source/backend/sftp/archive_upload_operation.cpp b/backend/source/backend/sftp/archive_upload_operation.cpp index fbfb1cce..12a7844e 100644 --- a/backend/source/backend/sftp/archive_upload_operation.cpp +++ b/backend/source/backend/sftp/archive_upload_operation.cpp @@ -130,8 +130,7 @@ ArchiveUploadOperation::ArchiveUploadOperation(SecureShell::SftpSession& sftp, O ArchiveUploadOperation::~ArchiveUploadOperation() { std::ignore = cancel(false); - if (auto stream = remoteStream_.lock(); stream) - stream->strand()->pushPromiseTask([]() {}).get(); + waitForStrandTasks(strand()); } std::expected ArchiveUploadOperation::work() diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 19fc4572..27386c2a 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -16,6 +16,7 @@ BulkTransferOperation::BulkTransferOperation(SecureShell::SftpSession& sftp, Com BulkTransferOperation::~BulkTransferOperation() { std::ignore = cancel(false); + waitForStrandTasks(strand()); } std::expected BulkTransferOperation::work() diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index 99e07493..acbffe12 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -28,13 +28,7 @@ DownloadOperation::DownloadOperation(SecureShell::SftpSession& sftp, DownloadOpe DownloadOperation::~DownloadOperation() { std::ignore = cancel(false); - - if (auto stream = fileStream_.lock(); stream && !stream->strand()->withinProcessingThread()) - { - // Wait for all tasks of the operation to finish. Skipped on the processing thread itself, - // where the wait could never be satisfied. - stream->strand()->pushPromiseTask([]() {}).get(); - } + waitForStrandTasks(strand()); } std::expected DownloadOperation::work() diff --git a/backend/source/backend/sftp/rename_operation.cpp b/backend/source/backend/sftp/rename_operation.cpp index 8fac293a..ac2e294e 100644 --- a/backend/source/backend/sftp/rename_operation.cpp +++ b/backend/source/backend/sftp/rename_operation.cpp @@ -31,10 +31,13 @@ std::expected RenameOperati case (Running): { // A queued rename mirrors a local move and must not replace what is at the destination. - auto fut = sftp_->performPromise( - [this]() + // The task outlives this operation when the wait times out. + auto fut = sftp->performPromise( + [session = sftp_, + source = sourcePath_, + destination = destinationPath_]() -> std::expected { - return sftp_->renameNoReplaceInStrand(sourcePath_, destinationPath_); + return sftp->renameNoReplaceInStrand(source, destination); } ); if (fut.wait_for(futureTimeout_) != std::future_status::ready) diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index 8d3769a0..4b572776 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -28,10 +28,7 @@ UploadOperation::UploadOperation(SecureShell::SftpSession& sftp, UploadOperation UploadOperation::~UploadOperation() { std::ignore = cancel(false); - // Wait for in-flight strand tasks unless we already run on the processing thread, where - // waiting would deadlock (tasks only get queued, never executed inline). - if (auto* stra = strand(); stra && !stra->withinProcessingThread()) - stra->pushPromiseTask([]() {}).get(); + waitForStrandTasks(strand()); } SecureShell::ProcessingStrand* UploadOperation::strand() const From a83e543dd62b0ca766edc2e1095ff9ab65665a88 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:33:11 +0200 Subject: [PATCH 005/131] A download with reserved space replaces only when every byte arrived A download with reserved space only replaces the destination when every byte arrived: the bytes written are counted for synchronous and asynchronous reads, a read ending early fails with SizeMismatch, and a resumed or already complete partial file reserves nothing; the temp path is built once and its cleanup no longer throws --- .../backend/sftp/download_operation.hpp | 9 +- .../backend/sftp/download_operation.cpp | 133 ++++++++++-------- 2 files changed, 86 insertions(+), 56 deletions(-) diff --git a/backend/include/backend/sftp/download_operation.hpp b/backend/include/backend/sftp/download_operation.hpp index 1f0296da..176c5d16 100644 --- a/backend/include/backend/sftp/download_operation.hpp +++ b/backend/include/backend/sftp/download_operation.hpp @@ -140,6 +140,11 @@ class DownloadOperation : public Operation std::expected readSymlinkInStrand(Paths::RemotePath const& remoteFullPath); + /** + * @brief The error of a file that ended before its listed size arrived. + */ + Error sizeMismatch() const; + void cleanup(); private: @@ -150,12 +155,14 @@ class DownloadOperation : public Operation Paths::RemotePath remotePath_{}; /** @brief options_.localPath for the filesystem calls, empty when none was given. */ std::filesystem::path localNativePath_{}; + /** @brief Where the download is written before it replaces localNativePath_, empty when no local path was given. */ + std::filesystem::path tempNativePath_{}; std::ofstream localFile_; SecureShell::BufferLease buffer_; std::shared_ptr asyncTransferContext_; /** @brief Bytes already present in an adopted partial file. */ std::uint64_t resumeOffset_{0u}; - /** @brief Bytes in the temp file so far, counted instead of tellp, which append mode misreports on Windows. */ + /** @brief Bytes of the content in the temp file so far, an adopted partial file included. */ std::uint64_t bytesInFile_{0u}; /** @brief An adopted partial file already holds the whole content. */ bool transferComplete_{false}; diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index acbffe12..003737e0 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -22,7 +22,11 @@ DownloadOperation::DownloadOperation(SecureShell::SftpSession& sftp, DownloadOpe options_.tempFileSuffix = Persistence::effectiveTempFileSuffix(options_.tempFileSuffix); remotePath_ = options_.remotePath; if (!options_.localPath.empty()) + { localNativePath_ = options_.localPath.native(); + tempNativePath_ = localNativePath_; + tempNativePath_ += options_.tempFileSuffix; + } } DownloadOperation::~DownloadOperation() @@ -340,21 +344,20 @@ std::expected DownloadOperation::readOnceInStran } const auto readAmount = result.value(); + const auto fileSize = options_.entry->size; if (readAmount == 0) { - return false; + if (bytesInFile_ >= fileSize) + return false; + Log::error("DownloadOperation: '{}' ended after {} of {} bytes.", options_.remotePath.value(), bytesInFile_, fileSize); + return enterErrorState(sizeMismatch()); } - std::uint64_t fileSize = 0; - bool good = true; - { - localFile_.write(buffer_.data(), static_cast(readAmount)); - bytesInFile_ += static_cast(readAmount); - fileSize = options_.entry->size; - good = localFile_.good(); - options_.progressCallback(0ull, fileSize, bytesInFile_, 0); - } + localFile_.write(buffer_.data(), static_cast(readAmount)); + bytesInFile_ += static_cast(readAmount); + const bool good = localFile_.good(); + options_.progressCallback(0ull, fileSize, bytesInFile_, 0); if (!good) { Log::error("DownloadOperation read cycle stopped: localFile_.good() == false"); @@ -363,12 +366,23 @@ std::expected DownloadOperation::readOnceInStran }); return false; } - return good && bytesInFile_ < fileSize; + return bytesInFile_ < fileSize; +} + +DownloadOperation::Error DownloadOperation::sizeMismatch() const +{ + return Error{ + .type = ErrorType::SizeMismatch, + .extraInfo = fmt::format( + "Received {} of {} bytes for {}", bytesInFile_, options_.entry->size, options_.remotePath.value() + ), + }; } bool DownloadOperation::commitBufferToFile(SecureShell::IFileStream::SignedSizeType bytesRead) { localFile_.write(buffer_.data(), static_cast(bytesRead)); + bytesInFile_ += static_cast(bytesRead); if (!localFile_.good()) { Log::error("DownloadOperation read cycle stopped: localFile_.good() == false"); @@ -410,8 +424,7 @@ DownloadOperation::refuseLinkedPath(std::filesystem::path const& path) const std::expected DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) { - auto tempPath = localNativePath_; - tempPath += options_.tempFileSuffix; + auto const& tempPath = tempNativePath_; // Opening for writing follows a link planted at the temp path or above it, out of the target directory. if (const auto refused = refuseLinkedPath(tempPath); !refused) @@ -425,48 +438,59 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) { Log::error( "DownloadOperation: Failed to create parent directories for '{}': {}.", - tempPath.generic_string(), + Paths::nativeToUtf8(tempPath), mkErr.message() ); return std::unexpected(Error{.type = ErrorType::CannotCreateDirectory}); } } - if (options_.tryContinue && std::filesystem::exists(tempPath)) + std::error_code existsError{}; + if (options_.tryContinue && std::filesystem::exists(tempPath, existsError)) { - // The size on disk, tellp right after opening in append mode reports 0 on Windows. std::error_code sizeError{}; const auto existingSize = std::filesystem::file_size(tempPath, sizeError); if (sizeError) { - Log::error("DownloadOperation: Cannot read the size of '{}': {}", tempPath.generic_string(), sizeError.message()); + Log::error( + "DownloadOperation: Cannot read the size of '{}': {}", + Paths::nativeToUtf8(tempPath), + Utility::errorMessage(sizeError) + ); return std::unexpected(Error{.type = ErrorType::FileStatFailed}); } // A leftover of another program or of an older version of the source must not be continued. - bool belongsToSource = existingSize > 0 && existingSize <= options_.entry->size; - if (belongsToSource) + const auto belongsToSource = [&]() -> std::expected { - const auto matches = ResumeCheck::tailsMatchInStrand(stream, tempPath, existingSize); - if (!matches) - return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = matches.error()}); - belongsToSource = *matches; - } + if (existingSize == 0 || existingSize > options_.entry->size) + return false; + return ResumeCheck::tailsMatchInStrand(stream, tempPath, existingSize) + .transform_error( + [](SecureShell::SftpError error) + { + return Error{.type = ErrorType::SftpError, .sftpError = std::move(error)}; + } + ); + }(); + if (!belongsToSource) + return std::unexpected(belongsToSource.error()); - if (belongsToSource && existingSize == options_.entry->size) + if (*belongsToSource && existingSize == options_.entry->size) { Log::info( "DownloadOperation: File '{}' already complete, will be renamed in finalize() step.", - tempPath.generic_string() + Paths::nativeToUtf8(tempPath) ); + bytesInFile_ = existingSize; transferComplete_ = true; return {}; } - else if (!belongsToSource) + else if (!*belongsToSource) { Log::info( "DownloadOperation: File '{}' is not a part of this source, discarding and starting over.", - tempPath.generic_string() + Paths::nativeToUtf8(tempPath) ); if (auto rewound = stream.seekInStrand(0); !rewound) return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = rewound.error()}); @@ -474,11 +498,11 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) } else { - Log::info("DownloadOperation: File '{}' is incomplete, continuing download.", tempPath.generic_string()); + Log::info("DownloadOperation: File '{}' is incomplete, continuing download.", Paths::nativeToUtf8(tempPath)); localFile_.open(tempPath, std::ios::binary | std::ios::app); if (!localFile_.is_open()) { - Log::error("DownloadOperation: Failed to open file for appending: {}", tempPath.generic_string()); + Log::error("DownloadOperation: Failed to open file for appending: {}", Paths::nativeToUtf8(tempPath)); return std::unexpected(Error{.type = ErrorType::OpenFailure}); } resumeOffset_ = existingSize; @@ -495,7 +519,7 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) if (!localFile_.is_open()) { - Log::error("DownloadOperation: Failed to open file: {}", tempPath.generic_string()); + Log::error("DownloadOperation: Failed to open file: {}", Paths::nativeToUtf8(tempPath)); return std::unexpected(Error{.type = ErrorType::OpenFailure}); } @@ -647,7 +671,8 @@ std::expected DownloadOperation::prepareInStrand ); } - if (options_.reserveSpace && options_.entry->size != 0) + // Appending to a partial file or finishing a complete one reserves nothing. + if (options_.reserveSpace && options_.entry->size != 0 && resumeOffset_ == 0 && !transferComplete_) { const auto pos = localFile_.tellp(); localFile_.seekp(options_.entry->size - 1); @@ -691,10 +716,17 @@ void DownloadOperation::cleanup() localFile_.close(); - auto tempPath = localNativePath_; - tempPath += options_.tempFileSuffix; - if (options_.doCleanup && std::filesystem::exists(tempPath)) - std::filesystem::remove(tempPath); + if (options_.doCleanup && !tempNativePath_.empty()) + { + std::error_code removeError{}; + std::filesystem::remove(tempNativePath_, removeError); + if (removeError) + Log::warn( + "DownloadOperation: Cannot remove '{}': {}", + Paths::nativeToUtf8(tempNativePath_), + Utility::errorMessage(removeError) + ); + } } void DownloadOperation::pause(bool doPause) @@ -729,31 +761,22 @@ std::expected DownloadOperation::finalizeInStran localFile_.close(); - auto tempPath = localNativePath_; - tempPath += options_.tempFileSuffix; + auto const& tempPath = tempNativePath_; - // A short file must never replace the destination. + // A short file must never replace the destination. A reserved file has its full size from the start, so the + // bytes written count. std::error_code sizeError{}; - const auto writtenSize = std::filesystem::file_size(tempPath, sizeError); - if (sizeError || writtenSize != options_.entry->size) + const auto sizeOnDisk = std::filesystem::file_size(tempPath, sizeError); + if (sizeError || sizeOnDisk != options_.entry->size || bytesInFile_ != options_.entry->size) { Log::error( - "DownloadOperation: '{}' holds {} bytes instead of {}.", - tempPath.generic_string(), - sizeError ? 0u : writtenSize, + "DownloadOperation: '{}' received {} bytes and holds {} instead of {}.", + Paths::nativeToUtf8(tempPath), + bytesInFile_, + sizeError ? 0u : sizeOnDisk, options_.entry->size ); - return std::unexpected( - Error{ - .type = ErrorType::SizeMismatch, - .extraInfo = fmt::format( - "Received {} of {} bytes for {}", - sizeError ? 0u : writtenSize, - options_.entry->size, - options_.remotePath.value() - ), - } - ); + return std::unexpected(sizeMismatch()); } // Marked before the rename, so the final file never exists without the mark. A filesystem without streams or From 34ad054776a5d9342e1aa40460193804459b1535 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:34:04 +0200 Subject: [PATCH 006/131] Add Utility::restrictToOwner and Utility::newFilePermissions Utility::restrictToOwner creates or limits a file to owner read and write without following a link, and Utility::newFilePermissions gives the mode the system would give a new file --- utility/include/utility/permissions.hpp | 14 +++++++ utility/source/utility/permissions.cpp | 55 +++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/utility/include/utility/permissions.hpp b/utility/include/utility/permissions.hpp index 56c661fd..6daae9a9 100644 --- a/utility/include/utility/permissions.hpp +++ b/utility/include/utility/permissions.hpp @@ -1,7 +1,9 @@ #pragma once +#include #include #include +#include namespace Utility { @@ -24,6 +26,18 @@ namespace Utility */ std::optional inheritablePermissions(std::filesystem::perms permissions); + /** + * @brief The permissions the system gives a new file: read and write for everyone, minus the process umask. + */ + std::filesystem::perms newFilePermissions(); + + /** + * @brief Creates @p path when it is missing and lets only its owner read and write it, so nothing written to it + * before its final permissions are set is readable by others. Never follows a link at @p path. Does + * nothing on Windows, where a new file takes the access rules of its directory. + */ + std::expected restrictToOwner(std::filesystem::path const& path); + /** * @brief The permissions of a local file as far as they mean anything. Unknown on Windows, where they are made up * from the read-only attribute and mark every file executable. diff --git a/utility/source/utility/permissions.cpp b/utility/source/utility/permissions.cpp index 956c6d12..dce93265 100644 --- a/utility/source/utility/permissions.cpp +++ b/utility/source/utility/permissions.cpp @@ -4,7 +4,10 @@ #include #ifndef _WIN32 +# include +# include # include +# include #endif namespace Utility @@ -31,6 +34,33 @@ namespace Utility return static_cast(mask); #endif } + +#ifndef _WIN32 + class UniqueDescriptor + { + public: + explicit UniqueDescriptor(int descriptor) + : descriptor_{descriptor} + {} + ~UniqueDescriptor() + { + if (descriptor_ >= 0) + ::close(descriptor_); + } + UniqueDescriptor(UniqueDescriptor const&) = delete; + UniqueDescriptor& operator=(UniqueDescriptor const&) = delete; + UniqueDescriptor(UniqueDescriptor&&) = delete; + UniqueDescriptor& operator=(UniqueDescriptor&&) = delete; + + int get() const + { + return descriptor_; + } + + private: + int descriptor_; + }; +#endif } std::optional sanitizePermissions(std::filesystem::perms permissions) @@ -54,6 +84,31 @@ namespace Utility return *sanitized & ~processUmask(); } + std::filesystem::perms newFilePermissions() + { + constexpr auto readWrite = std::filesystem::perms::owner_read | std::filesystem::perms::owner_write | + std::filesystem::perms::group_read | std::filesystem::perms::group_write | + std::filesystem::perms::others_read | std::filesystem::perms::others_write; + return readWrite & ~processUmask(); + } + + std::expected restrictToOwner(std::filesystem::path const& path) + { +#ifdef _WIN32 + static_cast(path); + return {}; +#else + const UniqueDescriptor descriptor{ + ::open(path.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC | O_NOFOLLOW, S_IRUSR | S_IWUSR) + }; + if (descriptor.get() < 0) + return std::unexpected(std::error_code{errno, std::generic_category()}); + if (::fchmod(descriptor.get(), S_IRUSR | S_IWUSR) != 0) + return std::unexpected(std::error_code{errno, std::generic_category()}); + return {}; +#endif + } + std::filesystem::perms localPermissions(std::filesystem::file_status const& status) { #ifdef _WIN32 From 7d6d65ef1057285bd23d375b9948f2436d66b47a Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:36:39 +0200 Subject: [PATCH 007/131] Transfers set times and permissions on the temp file before the move A download gets its modification time and permissions while it is still the temp file, which only its owner can read until then, and an upload sets the remote modification time on its temp file, so a failure there never reports failure after the destination was replaced; an upload whose finished temp file cannot be moved into place removes it --- .../backend/sftp/download_operation.hpp | 15 ++ .../include/backend/sftp/upload_operation.hpp | 5 + .../backend/sftp/download_operation.cpp | 128 ++++++++++-------- .../source/backend/sftp/upload_operation.cpp | 59 ++++---- 4 files changed, 115 insertions(+), 92 deletions(-) diff --git a/backend/include/backend/sftp/download_operation.hpp b/backend/include/backend/sftp/download_operation.hpp index 176c5d16..e0ecc8f1 100644 --- a/backend/include/backend/sftp/download_operation.hpp +++ b/backend/include/backend/sftp/download_operation.hpp @@ -145,6 +145,21 @@ class DownloadOperation : public Operation */ Error sizeMismatch() const; + /** + * @brief Creates the temp file, or limits an existing one, to its owner until the final permissions are set. + */ + std::expected restrictTempFile() const; + + /** + * @brief The permissions of the finished file: inherited, configured, or those of a new file. + */ + std::expected finalPermissionsInStrand(); + + /** + * @brief Gives @p path the modification time of the remote file. A failure is only logged. + */ + void applyModificationTime(std::filesystem::path const& path) const; + void cleanup(); private: diff --git a/backend/include/backend/sftp/upload_operation.hpp b/backend/include/backend/sftp/upload_operation.hpp index 618e5251..87ccee97 100644 --- a/backend/include/backend/sftp/upload_operation.hpp +++ b/backend/include/backend/sftp/upload_operation.hpp @@ -133,6 +133,11 @@ class UploadOperation : public Operation std::filesystem::perms determinePerms(std::filesystem::perms localPerms) const; + /** + * @brief Gives the remote temp file the modification time of the local file. A failure is only logged. + */ + void applyModificationTimeInStrand(SecureShell::SftpSession& sftp); + void cleanup(); private: diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index 003737e0..a6b8c944 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -494,11 +494,15 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) ); if (auto rewound = stream.seekInStrand(0); !rewound) return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = rewound.error()}); + if (const auto restricted = restrictTempFile(); !restricted) + return restricted; localFile_.open(tempPath, std::ios::binary | std::ios::trunc); } else { Log::info("DownloadOperation: File '{}' is incomplete, continuing download.", Paths::nativeToUtf8(tempPath)); + if (const auto restricted = restrictTempFile(); !restricted) + return restricted; localFile_.open(tempPath, std::ios::binary | std::ios::app); if (!localFile_.is_open()) { @@ -514,6 +518,8 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) } else { + if (const auto restricted = restrictTempFile(); !restricted) + return restricted; localFile_.open(tempPath, std::ios::binary | std::ios::trunc); } @@ -526,6 +532,20 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) return {}; } +std::expected DownloadOperation::restrictTempFile() const +{ + if (const auto restricted = Utility::restrictToOwner(tempNativePath_); !restricted) + { + Log::error( + "DownloadOperation: Cannot create '{}': {}", + Paths::nativeToUtf8(tempNativePath_), + Utility::errorMessage(restricted.error()) + ); + return std::unexpected(Error{.type = ErrorType::OpenFailure}); + } + return {}; +} + std::expected DownloadOperation::readSymlinkInStrand(Paths::RemotePath const& remoteFullPath) { @@ -787,73 +807,61 @@ std::expected DownloadOperation::finalizeInStran Log::warn("DownloadOperation: Cannot mark '{}' as downloaded: {}", Paths::nativeToUtf8(tempPath), marked.message()); } - if (const auto moved = Utility::moveIntoPlace(tempPath, localNativePath_, options_.mayOverwrite); !moved) + // Times and permissions go on the temp file, so nothing can fail once the destination is replaced. + applyModificationTime(tempPath); + const auto permissions = finalPermissionsInStrand(); + if (!permissions) + return std::unexpected(permissions.error()); + std::error_code permissionsError{}; + std::filesystem::permissions(tempPath, *permissions, permissionsError); + if (permissionsError) { - Log::error( - "DownloadOperation: Failed to move '{}' into place: {}", - options_.localPath.toUtf8(), - Utility::errorMessage(moved.error()) - ); - return std::unexpected(localMoveError(moved.error())); + Log::error("DownloadOperation: Failed to set permissions: {}", Utility::errorMessage(permissionsError)); + return std::unexpected(Error{.type = ErrorType::CannotSetFilePermissions}); } - if (options_.inheritPermissions) - { - auto stream = fileStream_.lock(); - if (!stream) - { - Log::error("DownloadOperation: File stream expired."); - return std::unexpected(Error{.type = ErrorType::FileStreamExpired}); - } + return ComparedStateCheck::moveLocalIntoPlace( + tempPath, localNativePath_, options_.mayOverwrite, options_.comparedDestination + ); +} - const auto fileInfo = stream->statInStrand(); - if (!fileInfo.has_value()) - { - Log::error("DownloadOperation: Failed to stat file."); - return std::unexpected(Error{.type = ErrorType::FileStatFailed, .sftpError = fileInfo.error()}); - } +std::expected DownloadOperation::finalPermissionsInStrand() +{ + if (!options_.inheritPermissions) + { + return options_.filePermissions.and_then(Utility::sanitizePermissions).value_or(Utility::newFilePermissions()); + } - // Special bits never travel and the umask applies, like for a file created locally. - std::error_code permissionsError{}; - if (const auto inherited = Utility::inheritablePermissions(fileInfo->permissions); inherited) - std::filesystem::permissions(localNativePath_, *inherited, permissionsError); - if (permissionsError) - { - Log::error("DownloadOperation: Failed to set permissions: {}", permissionsError.message()); - return std::unexpected(Error{.type = ErrorType::CannotSetFilePermissions}); - } + auto stream = fileStream_.lock(); + if (!stream) + { + Log::error("DownloadOperation: File stream expired."); + return std::unexpected(Error{.type = ErrorType::FileStreamExpired}); } - else if (options_.filePermissions) + const auto fileInfo = stream->statInStrand(); + if (!fileInfo.has_value()) { - std::error_code permissionsError{}; - if (const auto configured = Utility::sanitizePermissions(*options_.filePermissions); configured) - std::filesystem::permissions(localNativePath_, *configured, permissionsError); - if (permissionsError) - { - Log::error("DownloadOperation: Failed to set permissions: {}", permissionsError.message()); - return std::unexpected(Error{.type = ErrorType::CannotSetFilePermissions}); - } + Log::error("DownloadOperation: Failed to stat file."); + return std::unexpected(Error{.type = ErrorType::FileStatFailed, .sftpError = fileInfo.error()}); } - /* else keep default */ - - // Preserve the remote file's mtime on the downloaded local file so - // subsequent syncs don't see the file as modified (best-effort; a failure - // is only a warning). Mirrors UploadOperation's post-transfer setstat. - if (options_.entry) - { - using namespace std::chrono; - const auto sysTime = system_clock::time_point{duration_cast( - seconds{options_.entry->mtime} + nanoseconds{options_.entry->mtimeNsec} - )}; - const auto fileTime = std::chrono::file_clock::from_sys(sysTime); - std::error_code mtimeError{}; - std::filesystem::last_write_time(localNativePath_, fileTime, mtimeError); - if (mtimeError) - { - Log::warn( - "DownloadOperation: Failed to set mtime on '{}': {}.", options_.localPath.toUtf8(), mtimeError.message() - ); - } + // Special bits never travel and the umask applies, like for a file created locally. + return Utility::inheritablePermissions(fileInfo->permissions).value_or(Utility::newFilePermissions()); +} + +void DownloadOperation::applyModificationTime(std::filesystem::path const& path) const +{ + using namespace std::chrono; + const auto sysTime = system_clock::time_point{ + duration_cast(seconds{options_.entry->mtime} + nanoseconds{options_.entry->mtimeNsec}) + }; + std::error_code mtimeError{}; + std::filesystem::last_write_time(path, std::chrono::file_clock::from_sys(sysTime), mtimeError); + if (mtimeError) + { + Log::warn( + "DownloadOperation: Failed to set mtime on '{}': {}.", + Paths::nativeToUtf8(path), + Utility::errorMessage(mtimeError) + ); } - return {}; } \ No newline at end of file diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index 4b572776..e9173522 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -772,6 +772,9 @@ std::expected UploadOperation::finalizeInStrand() stream->closeInStrand(); + // The modification time goes on the temp file, so nothing can fail once the destination is replaced. + applyModificationTimeInStrand(); + const auto moved = sftp_->moveIntoPlaceInStrand( tempPath_, remotePath_, @@ -786,40 +789,32 @@ std::expected UploadOperation::finalizeInStrand() Log::warn("UploadOperation: Cannot remove '{}': {}", tempPath_.value(), removed.error().message); return std::unexpected(remoteMoveError(moved.error())); } + return {}; +} - // Preserve the local file's mtime on the remote file so subsequent syncs - // don't see the file as modified (best-effort; a failure is only a warning). - { - std::error_code errc{}; - const auto ftime = std::filesystem::last_write_time(localNativePath_, errc); - if (!errc) - { - const auto sctime = std::chrono::file_clock::to_sys(ftime); - const auto mtimeSecs = static_cast( - std::chrono::duration_cast(sctime.time_since_epoch()).count() - ); - sftp_attributes_struct attrs{}; - attrs.flags = SSH_FILEXFER_ATTR_ACMODTIME; - attrs.atime = mtimeSecs; - attrs.mtime = mtimeSecs; - const auto mtimeResult = sftp_->statInStrand(remotePath_, &attrs); - if (!mtimeResult.has_value()) - { - Log::warn( - "UploadOperation: Failed to set mtime on '{}': {}.", - options_.remotePath.value(), - mtimeResult.error().message - ); - } - } - else - { - Log::warn( - "UploadOperation: Could not read local mtime for '{}': {}.", options_.localPath.toUtf8(), errc.message() - ); - } +void UploadOperation::applyModificationTimeInStrand() +{ + std::error_code timeError{}; + const auto localTime = std::filesystem::last_write_time(localNativePath_, timeError); + if (timeError) + { + Log::warn( + "UploadOperation: Could not read local mtime for '{}': {}.", + options_.localPath.toUtf8(), + Utility::errorMessage(timeError) + ); + return; } - return {}; + const auto seconds = static_cast( + std::chrono::duration_cast(std::chrono::file_clock::to_sys(localTime).time_since_epoch()) + .count() + ); + sftp_attributes_struct attributes{}; + attributes.flags = SSH_FILEXFER_ATTR_ACMODTIME; + attributes.atime = seconds; + attributes.mtime = seconds; + if (const auto applied = sftp_->statInStrand(tempPath_, &attributes); !applied.has_value()) + Log::warn("UploadOperation: Failed to set mtime on '{}': {}.", tempPath_.value(), applied.error().message); } std::filesystem::perms UploadOperation::determinePerms(std::filesystem::perms localPerms) const From da88215747744284a4953e7e9be32e8404101912 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:37:49 +0200 Subject: [PATCH 008/131] A downloaded symlink is created under the temp name and moved into place A downloaded symlink is created under the temp name and moved into place, so it replaces an existing item when overwriting is allowed instead of failing on it; checking for an existing destination no longer throws --- .../backend/sftp/download_operation.cpp | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index a6b8c944..d7b996e8 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -74,7 +74,7 @@ std::expected DownloadO Log::error("DownloadOperation: Invalid local path."); return enterErrorState({.type = ErrorType::InvalidPath}); } - if (std::filesystem::exists(localNativePath_) && !options_.mayOverwrite) + if (std::error_code existsError{}; std::filesystem::exists(localNativePath_, existsError) && !options_.mayOverwrite) { Log::error( "DownloadOperation: File '{}' already exists and may not be overwritten.", @@ -276,19 +276,41 @@ std::expected DownloadOperation::handleSymlinkIn return std::unexpected(Error{.type = ErrorType::CannotCreateDirectory}); } } - // The copy keeps the link's own content, so a relative link still points into the downloaded tree. + // The link is created under the temp name and moved into place like a file. Its copy keeps the link's own + // content, so a relative link still points into the downloaded tree. + auto const& linkText = readResult.value().linkContent; + if (!Paths::isValidUtf8(linkText)) + { + Log::error("DownloadOperation: The link '{}' points to a name that is not UTF-8.", options_.remotePath.value()); + return std::unexpected( + Error{.type = ErrorType::InvalidName, .extraInfo = Paths::sanitizedUtf8(linkText)} + ); + } + std::filesystem::remove(tempNativePath_, ec); + ec.clear(); const auto linkContent = Paths::nativeFromUtf8(readResult.value().linkContent); if (linkEntry && linkEntry->isDirectory()) - std::filesystem::create_directory_symlink(linkContent, localNativePath_, ec); + std::filesystem::create_directory_symlink(linkContent, tempNativePath_, ec); else - std::filesystem::create_symlink(linkContent, localNativePath_, ec); + std::filesystem::create_symlink(linkContent, tempNativePath_, ec); if (ec) { Log::error( - "DownloadOperation: Failed to create symlink at '{}': {}.", options_.localPath.toUtf8(), ec.message() + "DownloadOperation: Failed to create symlink at '{}': {}.", + Paths::nativeToUtf8(tempNativePath_), + Utility::errorMessage(ec) ); return std::unexpected(Error{.type = ErrorType::CannotCreateSymlink}); } + + if (auto moved = ComparedStateCheck::moveLocalIntoPlace( + tempNativePath_, localNativePath_, options_.mayOverwrite, options_.comparedDestination + ); + !moved) + { + std::filesystem::remove(tempNativePath_, ec); + return std::unexpected(std::move(moved).error()); + } return {}; } @@ -576,8 +598,7 @@ std::expected DownloadOperation::prepare() return enterErrorState({.type = ErrorType::InvalidPath}); } - // Initial check. Check again later before rename. - if (std::filesystem::exists(localNativePath_)) + if (std::error_code existsError{}; std::filesystem::exists(localNativePath_, existsError)) { if (!options_.mayOverwrite) { From 1ae51fa81fb1b767a4640dfe24f91377e1da8375 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:39:23 +0200 Subject: [PATCH 009/131] A bulk download applies directory permissions once every entry is done A bulk download applies inherited or configured directory permissions once every entry is done, children before their parents, so a read-only directory still receives its files instead of failing them as skippable --- .../backend/sftp/bulk_download_operation.hpp | 10 +++- .../backend/sftp/bulk_transfer_operation.hpp | 8 +++ .../backend/sftp/bulk_download_operation.cpp | 58 ++++++++++++------- .../backend/sftp/bulk_transfer_operation.cpp | 2 + 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/backend/include/backend/sftp/bulk_download_operation.hpp b/backend/include/backend/sftp/bulk_download_operation.hpp index d782adc5..c9b76075 100644 --- a/backend/include/backend/sftp/bulk_download_operation.hpp +++ b/backend/include/backend/sftp/bulk_download_operation.hpp @@ -64,6 +64,7 @@ class BulkDownloadOperation : public BulkTransferOperation ) override; bool isSkippableError(Error const& error) const override; bool isSkippableDirectoryError(Error const& error) const override; + std::expected finishInStrand() override; bool sourceIsRemote() const noexcept override { @@ -78,9 +79,14 @@ class BulkDownloadOperation : public BulkTransferOperation private: std::expected createLocalDirectory(Paths::LocalPath const& path, SharedData::DirectoryEntry const& entry); - std::expected - applyPermsToDirectory(std::filesystem::path const& path, SharedData::DirectoryEntry const& entryToInheritFrom); + /** + * @brief Remembers the permissions @p path gets once the bulk is done, so a directory without write + * permission still receives its content. + */ + void deferDirectoryPermissions(std::filesystem::path const& path, SharedData::DirectoryEntry const& entryToInheritFrom); private: DownloadOperation::DownloadOperationOptions individualOptions_; + /** @brief Created directories with the permissions they get at the end, parents before their children. */ + std::vector> deferredDirectoryPermissions_{}; }; diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index a5a44c4e..f40265f0 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -244,6 +244,14 @@ class BulkTransferOperation : public Operation return false; } + /** + * @brief Runs once every entry is done, before the bulk completes. A failure fails the bulk. + */ + virtual std::expected finishInStrand() + { + return {}; + } + /** * @brief Log prefix, e.g. "BulkDownloadOperation". */ diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index d3d54d70..92839af2 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -5,6 +5,8 @@ #include #include +#include + namespace { BulkTransferOperation::CommonOptions toCommon(BulkDownloadOperation::BulkDownloadOperationOptions const& options) @@ -82,7 +84,8 @@ BulkDownloadOperation::createLocalDirectory(Paths::LocalPath const& localPath, S } ); } - return applyPermsToDirectory(path, entry); + deferDirectoryPermissions(path, entry); + return {}; } std::unique_ptr BulkDownloadOperation::makeChild( @@ -126,35 +129,46 @@ bool BulkDownloadOperation::isSkippableDirectoryError(Error const& error) const return error.type == ErrorType::LinkInLocalPath; } -std::expected BulkDownloadOperation::applyPermsToDirectory( +void BulkDownloadOperation::deferDirectoryPermissions( std::filesystem::path const& path, SharedData::DirectoryEntry const& entryToInheritFrom ) { // Unknown permissions (the scan root carries none) leave the directory as created. - std::optional toApplyPerms{std::nullopt}; - if (individualOptions_.inheritPermissions) - toApplyPerms = Utility::inheritablePermissions(entryToInheritFrom.permissions); - else if (individualOptions_.directoryPermissions) - toApplyPerms = Utility::sanitizePermissions(*individualOptions_.directoryPermissions); - - if (!toApplyPerms) - return {}; + const auto permissions = individualOptions_.inheritPermissions + ? Utility::inheritablePermissions(entryToInheritFrom.permissions) + : individualOptions_.directoryPermissions.and_then(Utility::sanitizePermissions); + if (permissions) + deferredDirectoryPermissions_.emplace_back(path, *permissions); +} - std::error_code ec; - std::filesystem::permissions(path, *toApplyPerms, ec); - if (ec) +std::expected BulkDownloadOperation::finishInStrand() +{ + // Children first, a parent that loses its search permission would stop them. + for (auto const& [path, permissions] : deferredDirectoryPermissions_ | std::views::reverse) { - Log::error( - "BulkDownloadOperation: Failed to set permissions on local directory: {}: {}", path.string(), ec.message() - ); - return std::unexpected( - Error{ - .type = ErrorType::CannotSetFilePermissions, - .extraInfo = fmt::format("Setting permissions on local directory: {}: {}", path.string(), ec.message()) - } - ); + std::error_code error{}; + std::filesystem::permissions(path, permissions, error); + if (error) + { + Log::error( + "BulkDownloadOperation: Failed to set permissions on local directory: {}: {}", + Paths::nativeToUtf8(path), + Utility::errorMessage(error) + ); + return std::unexpected( + Error{ + .type = ErrorType::CannotSetFilePermissions, + .extraInfo = fmt::format( + "Setting permissions on local directory: {}: {}", + Paths::nativeToUtf8(path), + Utility::errorMessage(error) + ), + } + ); + } } + deferredDirectoryPermissions_.clear(); return {}; } diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 27386c2a..82989cac 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -184,6 +184,8 @@ std::expected B if (slots_.empty() && deferred_.empty() && cursor_ == entries_.size()) { + if (const auto finished = finishInStrand(); !finished.has_value()) + return enterErrorState(finished.error()); emitProgress(true); Log::info("{}: Bulk transfer completed.", logName()); enterState(OperationState::Completed); From 196dc8de9430c01814a9044d4e3967bd5f40eb7a Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:41:39 +0200 Subject: [PATCH 010/131] The tar writer gives members with unknown permissions the default mode The tar writer gives a member with unknown permissions the mode of a new one (0644, 0755 for a directory, 0777 for a symlink) instead of 07777 --- tar-archive/include/tar_archive/header.hpp | 5 +++-- tar-archive/source/tar_archive/header.cpp | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tar-archive/include/tar_archive/header.hpp b/tar-archive/include/tar_archive/header.hpp index 7bb9eec1..8232a8da 100644 --- a/tar-archive/include/tar_archive/header.hpp +++ b/tar-archive/include/tar_archive/header.hpp @@ -93,8 +93,9 @@ namespace TarArchive }; /** - * @brief Build the on-disk tar records for a DirectoryEntry, emitting PAX extended - * headers when the entry's path, link target, or size exceed plain USTAR limits. + * @brief Build the on-disk tar records for a member, emitting PAX extended headers when its path, link + * target, or size exceed plain USTAR limits. Unknown permissions are written as 0755 for a directory, + * 0777 for a symlink and 0644 otherwise. */ std::expected buildRecords(MemberHeader const& meta); diff --git a/tar-archive/source/tar_archive/header.cpp b/tar-archive/source/tar_archive/header.cpp index 7189f707..49e9ce20 100644 --- a/tar-archive/source/tar_archive/header.cpp +++ b/tar-archive/source/tar_archive/header.cpp @@ -266,6 +266,21 @@ namespace TarArchive return payload; } + /** + * @brief The mode field: the permission bits, or what a newly created member of this type commonly gets + * when they are unknown. + */ + std::uint32_t modeOf(std::filesystem::perms permissions, TypeFlag typeflag) + { + if (permissions != std::filesystem::perms::unknown) + return static_cast(permissions) & 07777u; + if (typeflag == TypeFlag::Directory) + return 0755u; + if (typeflag == TypeFlag::SymbolicLink) + return 0777u; + return 0644u; + } + void populateUstarHeader( RawRecord& record, std::string const& name, @@ -279,7 +294,7 @@ namespace TarArchive record.fill(std::byte{0u}); writeString(record, fieldName, name); writeString(record, fieldPrefix, prefix); - writeNumeric(record, fieldMode, static_cast(meta.permissions) & 07777u); + writeNumeric(record, fieldMode, modeOf(meta.permissions, typeflag)); writeNumeric(record, fieldUid, meta.uid); writeNumeric(record, fieldGid, meta.gid); writeNumeric(record, fieldSize, payloadSize); From bc61be985a11e204a515bed2af217d95c1bebfae Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:41:39 +0200 Subject: [PATCH 011/131] Uploaded archives never carry setuid, setgid or sticky bits An uploaded archive never carries setuid, setgid or sticky bits, and members whose permissions are unknown, as on Windows, get the tar writer's defaults instead of 0777 --- .../backend/sftp/archive_upload_operation.cpp | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/backend/source/backend/sftp/archive_upload_operation.cpp b/backend/source/backend/sftp/archive_upload_operation.cpp index 12a7844e..8ff65196 100644 --- a/backend/source/backend/sftp/archive_upload_operation.cpp +++ b/backend/source/backend/sftp/archive_upload_operation.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -69,6 +70,17 @@ namespace }; } + /** + * @brief The tar header of @p entry under @p path. Special bits never travel, and unknown permissions, as on + * Windows, get the defaults of the tar writer. + */ + TarArchive::MemberHeader memberHeaderFor(std::string path, SharedData::DirectoryEntry const& entry) + { + auto header = TarArchive::memberHeaderOf(std::move(path), entry); + header.permissions = Utility::sanitizePermissions(header.permissions).value_or(std::filesystem::perms::unknown); + return header; + } + /** * @brief The metadata of @p localPath for its tar header. */ @@ -87,7 +99,7 @@ namespace } SharedData::DirectoryEntry entry{}; - entry.permissions = status.permissions(); + entry.permissions = Utility::localPermissions(status); entry.type = SharedData::fileTypeFromStdFilesystemType(status.type()); if (entry.type == SharedData::FileType::Regular) @@ -282,7 +294,7 @@ std::expected ArchiveUploadOperation::flattenRoots() totalPayloadBytes_ += rootEntry->size; resolvedEntries_.push_back( ResolvedEntry{ - .header = TarArchive::memberHeaderOf(archivePrefix.value(), *rootEntry), + .header = memberHeaderFor(archivePrefix.value(), *rootEntry), .source = localRoot, } ); @@ -291,7 +303,7 @@ std::expected ArchiveUploadOperation::flattenRoots() { rootEntry->size = 0u; resolvedEntries_.push_back( - ResolvedEntry{.header = TarArchive::memberHeaderOf(archivePrefix.value(), *rootEntry)} + ResolvedEntry{.header = memberHeaderFor(archivePrefix.value(), *rootEntry)} ); // Directory roots are expanded exclusively from a preceding @@ -354,14 +366,14 @@ void ArchiveUploadOperation::appendPrescannedFromScan( totalPayloadBytes_ += child.size; resolvedEntries_.push_back( ResolvedEntry{ - .header = TarArchive::memberHeaderOf(memberPath.value(), child), + .header = memberHeaderFor(memberPath.value(), child), .source = *std::move(source), } ); } else if (child.type == SharedData::FileType::Directory) { - ResolvedEntry dirEntry{.header = TarArchive::memberHeaderOf(memberPath.value(), child)}; + ResolvedEntry dirEntry{.header = memberHeaderFor(memberPath.value(), child)}; dirEntry.header.size = 0u; resolvedEntries_.push_back(std::move(dirEntry)); } From 8e24eb75281d97054fba52f0f0d7d08df3ed11b5 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:43:40 +0200 Subject: [PATCH 012/131] AsyncTransferContext answers hasEnded from its outcome AsyncTransferContext answers hasEnded from its outcome, the separate ended flag is gone --- ssh/include/ssh/async/transfer_context.hpp | 1 - ssh/source/ssh/async/transfer_context.cpp | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ssh/include/ssh/async/transfer_context.hpp b/ssh/include/ssh/async/transfer_context.hpp index 5d7580d8..774304df 100644 --- a/ssh/include/ssh/async/transfer_context.hpp +++ b/ssh/include/ssh/async/transfer_context.hpp @@ -68,7 +68,6 @@ namespace SecureShell private: std::atomic outcome_{Outcome::Running}; - std::atomic_bool ended_{false}; mutable std::mutex errorMutex_{}; std::optional error_{}; std::atomic_bool paused_{false}; diff --git a/ssh/source/ssh/async/transfer_context.cpp b/ssh/source/ssh/async/transfer_context.cpp index dc30247b..f3d759a5 100644 --- a/ssh/source/ssh/async/transfer_context.cpp +++ b/ssh/source/ssh/async/transfer_context.cpp @@ -6,7 +6,6 @@ namespace SecureShell { auto expected = Outcome::Running; outcome_.compare_exchange_strong(expected, outcome); - ended_.store(true); } void AsyncTransferContext::cancel() @@ -32,7 +31,7 @@ namespace SecureShell bool AsyncTransferContext::hasEnded() const { - return ended_.load(); + return outcome_.load() != Outcome::Running; } AsyncTransferContext::Outcome AsyncTransferContext::outcome() const From 52c520b67ef82e6da64c6962a4d6e30d102064b4 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:43:40 +0200 Subject: [PATCH 013/131] Archive downloads avoid the throwing filesystem overloads Archive downloads check for an existing archive and remove their temp file without the throwing filesystem overloads --- .../backend/sftp/archive_download_operation.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/source/backend/sftp/archive_download_operation.cpp b/backend/source/backend/sftp/archive_download_operation.cpp index edb7553a..248becdb 100644 --- a/backend/source/backend/sftp/archive_download_operation.cpp +++ b/backend/source/backend/sftp/archive_download_operation.cpp @@ -186,7 +186,7 @@ ArchiveDownloadOperation::workInStrand() return enterErrorState( {.type = ErrorType::ImplementationError, .extraInfo = "archive has no roots"} ); - if (std::filesystem::exists(localArchiveNativePath_) && !options_.mayOverwrite) + if (std::error_code existsError{}; std::filesystem::exists(localArchiveNativePath_, existsError) && !options_.mayOverwrite) return enterErrorState({.type = ErrorType::FileExists}); const auto prepareResult = prepareInStrand(); @@ -707,10 +707,16 @@ void ArchiveDownloadOperation::cleanup() currentStream_.reset(); writer_.reset(); - if (!tempPath_.empty() && std::filesystem::exists(tempPath_)) + if (!tempPath_.empty()) { - std::error_code removeErr{}; - std::filesystem::remove(tempPath_, removeErr); + std::error_code removeError{}; + std::filesystem::remove(tempPath_, removeError); + if (removeError) + Log::warn( + "ArchiveDownloadOperation: Cannot remove '{}': {}", + Paths::nativeToUtf8(tempPath_), + Utility::errorMessage(removeError) + ); } } From 129537218389f14bd968874339925de8e52eb6be Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:45:01 +0200 Subject: [PATCH 014/131] Read the process umask at the start of main The process umask is read at the start of main, before any thread creates files, since reading it outside Linux sets it for a moment --- backend/source/backend/main.cpp | 3 +++ utility/include/utility/permissions.hpp | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index 1e05f773..1a1977ac 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -443,6 +444,8 @@ void Main::startChildSignalTimer() int main(int const argc, char const* const* argv) { + // Reading the umask changes it for a moment on macOS, so it is read before any thread creates files. + static_cast(Utility::processUmask()); #ifdef _WIN32 // A GUI program has no console; help and --console write to the one it was started from. for (int index = 1; index < argc; ++index) diff --git a/utility/include/utility/permissions.hpp b/utility/include/utility/permissions.hpp index 6daae9a9..6f83dac5 100644 --- a/utility/include/utility/permissions.hpp +++ b/utility/include/utility/permissions.hpp @@ -17,7 +17,8 @@ namespace Utility std::optional sanitizePermissions(std::filesystem::perms permissions); /** - * @brief The file mode creation mask of this process, empty where there is none. + * @brief The file mode creation mask of this process, empty where there is none. Read once; outside Linux + * reading it sets it for a moment, so the first call has to happen before other threads create files. */ std::filesystem::perms processUmask(); From 0d75e4ec6de8d4754df48ba41e6f7d5e98d149e4 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:02:28 +0200 Subject: [PATCH 015/131] PathError::toString always yields UTF-8; one splitter for path segments PathError::toString writes control bytes and every byte outside a valid UTF-8 sequence as \xNN, so its text is always UTF-8; paths split segments with one splitter, build their text once, validate a segment only where it enters, and joining onto an unset RemotePath gives an unset path instead of undefined behaviour; a PathSegment reads from JSON --- paths/include/paths/path_error.hpp | 58 +++-- paths/include/paths/path_segment.hpp | 117 ++++----- paths/include/paths/paths.hpp | 25 +- paths/include/paths/relative_path.hpp | 58 ++--- paths/include/paths/remote_path.hpp | 68 +++--- paths/include/paths/utf8.hpp | 327 ++++++++++++++++++++++++++ paths/source/paths/CMakeLists.txt | 2 +- 7 files changed, 506 insertions(+), 149 deletions(-) create mode 100644 paths/include/paths/utf8.hpp diff --git a/paths/include/paths/path_error.hpp b/paths/include/paths/path_error.hpp index 2f7bd12b..2077c366 100644 --- a/paths/include/paths/path_error.hpp +++ b/paths/include/paths/path_error.hpp @@ -1,6 +1,11 @@ #pragma once +#include + +#include + #include +#include #include #include @@ -17,14 +22,30 @@ namespace Paths ContainsNul, InvalidUtf8, NotAbsolute, - /// A character the local platform cannot store in a name, like '\\' or ':' on Windows. + /** + * @brief A character the local platform cannot store in a name, like '\\' or ':' on Windows. + */ ReservedCharacter, - /// A Windows device name like CON or LPT1, with or without an extension. + /** + * @brief A Windows device name like CON or LPT1, with or without an extension. + */ ReservedDeviceName, - /// Windows drops a trailing dot or space, the name would refer to another file. + /** + * @brief Windows drops a trailing dot or space, the name would refer to another file. + */ TrailingDotOrSpace, - /// The path leaves the directory it is meant to stay in. + /** + * @brief The path leaves the directory it is meant to stay in. + */ EscapesRoot, + /** + * @brief A Windows device or namespace path like \\.\PhysicalDrive0 or \\?\Volume{...}, no file path. + */ + DevicePath, + /** + * @brief A name longer than the 255 UTF-16 code units Windows allows. + */ + NameTooLong, }; inline std::string_view toString(PathRejection rejection) @@ -51,6 +72,10 @@ namespace Paths return "ends with a dot or space"; case PathRejection::EscapesRoot: return "leaves its root directory"; + case PathRejection::DevicePath: + return "is a device path"; + case PathRejection::NameTooLong: + return "has a name that is too long"; } return "is invalid"; } @@ -63,24 +88,29 @@ namespace Paths PathRejection reason{PathRejection::Empty}; std::string input{}; + /** + * @brief The input in quotes followed by the reason. Control bytes and bytes that are not part of a valid + * UTF-8 sequence are written as \\xNN, so the text is always valid UTF-8. + */ std::string toString() const { std::string printable; printable.reserve(input.size()); - for (const auto character : input) + std::size_t index = 0; + while (index < input.size()) { - const auto byte = static_cast(character); - if (byte < 0x20u || byte == 0x7Fu) + const auto byte = static_cast(input[index]); + const auto length = Detail::utf8SequenceLength(input, index); + if (length == 0u || byte < 0x20u || byte == 0x7Fu) { - constexpr std::string_view digits = "0123456789ABCDEF"; - printable += "\\x"; - printable.push_back(digits[byte >> 4u]); - printable.push_back(digits[byte & 0x0Fu]); + fmt::format_to(std::back_inserter(printable), "\\x{:02X}", byte); + ++index; + continue; } - else - printable.push_back(character); + printable.append(input, index, length); + index += length; } - return "'" + printable + "' " + std::string{Paths::toString(reason)}; + return fmt::format("'{}' {}", printable, Paths::toString(reason)); } }; } diff --git a/paths/include/paths/path_segment.hpp b/paths/include/paths/path_segment.hpp index 92944454..aeb91a0d 100644 --- a/paths/include/paths/path_segment.hpp +++ b/paths/include/paths/path_segment.hpp @@ -1,66 +1,58 @@ #pragma once #include +#include #include #include #include +#include +#include #include #include #include namespace Paths { - /** - * @brief True when @p text is well-formed UTF-8 (no overlong forms, no surrogates). - */ - inline bool isValidUtf8(std::string_view text) + class RelativePath; + class RemotePath; + class LocalPath; + + namespace Detail { - std::size_t index = 0; - while (index < text.size()) + /** + * @brief The parts of @p text between '/' separators as string views, empty parts included. Empty text has + * no parts. + */ + inline auto splitAtSlashes(std::string_view text) { - const auto lead = static_cast(text[index]); - std::size_t length = 0; - std::uint32_t codePoint = 0; - if (lead < 0x80u) - { - ++index; - continue; - } - if ((lead & 0xE0u) == 0xC0u) - { - length = 2; - codePoint = lead & 0x1Fu; - } - else if ((lead & 0xF0u) == 0xE0u) - { - length = 3; - codePoint = lead & 0x0Fu; - } - else if ((lead & 0xF8u) == 0xF0u) - { - length = 4; - codePoint = lead & 0x07u; - } - else - return false; + return text | std::views::split('/') | + std::views::transform( + [](auto const& part) + { + return std::string_view{part.begin(), part.end()}; + } + ); + } + + /** + * @brief Appends @p tail to the path text @p value, with a '/' between them unless @p value ends in one. + */ + inline void appendSegment(std::string& value, std::string_view tail) + { + if (!value.empty() && value.back() != '/') + value.push_back('/'); + value.append(tail); + } - if (index + length > text.size()) - return false; - for (std::size_t offset = 1; offset < length; ++offset) - { - const auto continuation = static_cast(text[index + offset]); - if ((continuation & 0xC0u) != 0x80u) - return false; - codePoint = (codePoint << 6u) | (continuation & 0x3Fu); - } - constexpr std::uint32_t minimumForLength[] = {0u, 0u, 0x80u, 0x800u, 0x10000u}; - if (codePoint < minimumForLength[length] || codePoint > 0x10FFFFu || - (codePoint >= 0xD800u && codePoint <= 0xDFFFu)) - return false; - index += length; + /** + * @brief The text after the last '/' of @p value, all of it when there is none. + */ + inline std::string_view lastSegment(std::string_view value) + { + const auto slash = value.rfind('/'); + return slash == std::string_view::npos ? value : value.substr(slash + 1); } - return true; } /** @@ -74,21 +66,27 @@ namespace Paths public: static std::expected make(std::string_view utf8) { - const auto refuse = [utf8](PathRejection reason) - { - return std::unexpected(PathError{.reason = reason, .input = std::string{utf8}}); - }; + if (const auto rejection = rejectionOf(utf8); rejection) + return std::unexpected(PathError{.reason = *rejection, .input = std::string{utf8}}); + return PathSegment{std::string{utf8}}; + } + + /** + * @brief Why @p utf8 is no valid segment, nothing when it is one. + */ + static std::optional rejectionOf(std::string_view utf8) + { if (utf8.empty()) - return refuse(PathRejection::Empty); + return PathRejection::Empty; if (utf8 == "." || utf8 == "..") - return refuse(PathRejection::DotOrDotDot); + return PathRejection::DotOrDotDot; if (utf8.find('/') != std::string_view::npos) - return refuse(PathRejection::ContainsSeparator); + return PathRejection::ContainsSeparator; if (utf8.find('\0') != std::string_view::npos) - return refuse(PathRejection::ContainsNul); + return PathRejection::ContainsNul; if (!isValidUtf8(utf8)) - return refuse(PathRejection::InvalidUtf8); - return PathSegment{std::string{utf8}}; + return PathRejection::InvalidUtf8; + return std::nullopt; } std::string const& value() const noexcept @@ -104,6 +102,13 @@ namespace Paths auto operator<=>(PathSegment const&) const = default; private: + friend class RelativePath; + friend class RemotePath; + friend class LocalPath; + + /** + * @brief Takes @p value as it is, for text a path type already validated. + */ explicit PathSegment(std::string value) : value_{std::move(value)} {} diff --git a/paths/include/paths/paths.hpp b/paths/include/paths/paths.hpp index 0895c872..dc22b21e 100644 --- a/paths/include/paths/paths.hpp +++ b/paths/include/paths/paths.hpp @@ -31,15 +31,6 @@ namespace Paths // Paths travel as plain strings, invalid ones are refused when they are read. - inline void to_json(nlohmann::json& json, PathSegment const& segment) - { - json = segment.value(); - } - inline void from_json(nlohmann::json const& json, PathSegment& segment) - { - segment = Detail::orThrow(PathSegment::make(json.get())); - } - inline void to_json(nlohmann::json& json, RelativePath const& path) { json = path.value(); @@ -102,3 +93,19 @@ namespace Paths } #endif } + +/** + * @brief PathSegment has no unset state, so it is read through a serializer instead of from_json. + */ +template <> +struct nlohmann::adl_serializer +{ + static Paths::PathSegment from_json(nlohmann::json const& json) + { + return Paths::Detail::orThrow(Paths::PathSegment::make(json.get())); + } + static void to_json(nlohmann::json& json, Paths::PathSegment const& segment) + { + json = segment.value(); + } +}; diff --git a/paths/include/paths/relative_path.hpp b/paths/include/paths/relative_path.hpp index 4e5fa390..1f502dc6 100644 --- a/paths/include/paths/relative_path.hpp +++ b/paths/include/paths/relative_path.hpp @@ -25,43 +25,30 @@ namespace Paths static std::expected make(std::string_view utf8) { - RelativePath result; if (utf8.empty()) - return result; + return RelativePath{}; if (utf8.front() == '/') return std::unexpected(PathError{.reason = PathRejection::EscapesRoot, .input = std::string{utf8}}); - - std::size_t start = 0; - while (start <= utf8.size()) + for (auto const part : Detail::splitAtSlashes(utf8)) { - const auto end = std::min(utf8.find('/', start), utf8.size()); - auto segment = PathSegment::make(utf8.substr(start, end - start)); - if (!segment) - return std::unexpected(PathError{.reason = segment.error().reason, .input = std::string{utf8}}); - result = result / *segment; - start = end + 1; + if (const auto rejection = PathSegment::rejectionOf(part); rejection) + return std::unexpected(PathError{.reason = *rejection, .input = std::string{utf8}}); } - return result; + return RelativePath{std::string{utf8}}; } RelativePath operator/(PathSegment const& segment) const { RelativePath result{*this}; - if (!result.value_.empty()) - result.value_.push_back('/'); - result.value_ += segment.value(); + Detail::appendSegment(result.value_, segment.value()); return result; } RelativePath operator/(RelativePath const& other) const { - if (other.empty()) - return *this; - if (empty()) - return other; RelativePath result{*this}; - result.value_.push_back('/'); - result.value_ += other.value_; + if (!other.empty()) + Detail::appendSegment(result.value_, other.value_); return result; } @@ -70,16 +57,19 @@ namespace Paths return value_.empty(); } + /** + * @brief The names as views into this path, without copying or validating them again. + */ + auto segmentViews() const + { + return Detail::splitAtSlashes(value_); + } + std::vector segments() const { std::vector result; - std::size_t start = 0; - while (!value_.empty() && start <= value_.size()) - { - const auto end = std::min(value_.find('/', start), value_.size()); - result.push_back(*PathSegment::make(std::string_view{value_}.substr(start, end - start))); - start = end + 1; - } + for (auto const part : segmentViews()) + result.push_back(PathSegment{std::string{part}}); return result; } @@ -88,18 +78,14 @@ namespace Paths if (empty()) return std::nullopt; const auto slash = value_.rfind('/'); - RelativePath result; - if (slash != std::string::npos) - result.value_ = value_.substr(0, slash); - return result; + return RelativePath{slash == std::string::npos ? std::string{} : value_.substr(0, slash)}; } std::optional leaf() const { if (empty()) return std::nullopt; - const auto slash = value_.rfind('/'); - return *PathSegment::make(slash == std::string::npos ? value_ : value_.substr(slash + 1)); + return PathSegment{std::string{Detail::lastSegment(value_)}}; } /** @@ -126,6 +112,10 @@ namespace Paths auto operator<=>(RelativePath const&) const = default; private: + explicit RelativePath(std::string value) + : value_{std::move(value)} + {} + std::string value_{}; }; } diff --git a/paths/include/paths/remote_path.hpp b/paths/include/paths/remote_path.hpp index 74926388..bf574e9f 100644 --- a/paths/include/paths/remote_path.hpp +++ b/paths/include/paths/remote_path.hpp @@ -2,7 +2,9 @@ #include -#include +#include + +#include #include #include #include @@ -19,6 +21,7 @@ namespace Paths * platform. ".." is refused instead of resolved: the server decides what it means. * * A default constructed RemotePath is unset (@ref empty), every operation that touches the server must refuse it. + * Joining onto an unset path gives an unset path. */ class RemotePath { @@ -44,42 +47,37 @@ namespace Paths if (utf8.front() != '/') return refuse(PathRejection::NotAbsolute); - auto result = root(); - std::size_t start = 1; - while (start <= utf8.size()) + std::string value; + value.reserve(utf8.size()); + for (auto const name : Detail::splitAtSlashes(utf8)) { - const auto end = std::min(utf8.find('/', start), utf8.size()); - const auto name = utf8.substr(start, end - start); - start = end + 1; if (name.empty() || name == ".") continue; - auto segment = PathSegment::make(name); - if (!segment) - return refuse(segment.error().reason); - result = result / *segment; + if (const auto rejection = PathSegment::rejectionOf(name); rejection) + return refuse(*rejection); + value.push_back('/'); + value.append(name); } - return result; + if (value.empty()) + return root(); + return RemotePath{std::move(value)}; } RemotePath operator/(PathSegment const& segment) const { - assert(!empty()); + if (empty()) + return {}; RemotePath result{*this}; - if (result.value_.back() != '/') - result.value_.push_back('/'); - result.value_ += segment.value(); + Detail::appendSegment(result.value_, segment.value()); return result; } RemotePath operator/(RelativePath const& relative) const { - if (relative.empty()) + if (empty() || relative.empty()) return *this; - assert(!empty()); RemotePath result{*this}; - if (result.value_.back() != '/') - result.value_.push_back('/'); - result.value_ += relative.value(); + Detail::appendSegment(result.value_, relative.value()); return result; } @@ -91,7 +89,7 @@ namespace Paths const auto name = filename(); if (!name) return std::unexpected(PathError{.reason = PathRejection::Empty, .input = value_}); - return PathSegment::make(name->value() + std::string{suffix}) + return PathSegment::make(fmt::format("{}{}", name->view(), suffix)) .transform( [this](PathSegment const& segment) { @@ -117,26 +115,26 @@ namespace Paths if (empty()) return refuse(PathRejection::NotAbsolute); - auto result = linkContent.front() == '/' ? root() : parent().value_or(root()); - std::size_t start = 0; - while (start <= linkContent.size()) + std::string value = linkContent.front() == '/' ? std::string{} : parent().value_or(root()).value_; + if (value == "/") + value.clear(); + for (auto const name : Detail::splitAtSlashes(linkContent)) { - const auto end = std::min(linkContent.find('/', start), linkContent.size()); - const auto name = linkContent.substr(start, end - start); - start = end + 1; if (name.empty() || name == ".") continue; if (name == "..") { - result = result.parent().value_or(root()); + value.erase(std::min(value.rfind('/'), value.size())); continue; } - auto segment = PathSegment::make(name); - if (!segment) - return refuse(segment.error().reason); - result = result / *segment; + if (const auto rejection = PathSegment::rejectionOf(name); rejection) + return refuse(*rejection); + value.push_back('/'); + value.append(name); } - return result; + if (value.empty()) + return root(); + return RemotePath{std::move(value)}; } bool empty() const noexcept @@ -161,7 +159,7 @@ namespace Paths { if (empty() || isRoot()) return std::nullopt; - return *PathSegment::make(std::string_view{value_}.substr(value_.rfind('/') + 1)); + return PathSegment{std::string{Detail::lastSegment(value_)}}; } /** diff --git a/paths/include/paths/utf8.hpp b/paths/include/paths/utf8.hpp new file mode 100644 index 00000000..b79fe6da --- /dev/null +++ b/paths/include/paths/utf8.hpp @@ -0,0 +1,327 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace Paths +{ + /** + * @brief U+FFFD, written in place of every byte or code unit that is not part of a valid sequence. + */ + constexpr char32_t replacementCharacter = 0xFFFDu; + + /** + * @brief A 16 bit code unit type: char16_t, and wchar_t on Windows. + */ + template + concept Utf16CodeUnit = (std::same_as || std::same_as) && sizeof(CharT) == 2; + + namespace Detail + { + /** + * @brief Length of the well-formed UTF-8 sequence that starts at @p index of @p text, 0 when the bytes there + * are none (stray continuation byte, overlong form, surrogate, above U+10FFFF or truncated). + */ + constexpr std::size_t utf8SequenceLength(std::string_view text, std::size_t index) noexcept + { + const auto lead = static_cast(text[index]); + if (lead < 0x80u) + return 1u; + + const auto [length, initial] = [lead]() -> std::pair + { + if ((lead & 0xE0u) == 0xC0u) + return {2u, lead & 0x1Fu}; + if ((lead & 0xF0u) == 0xE0u) + return {3u, lead & 0x0Fu}; + if ((lead & 0xF8u) == 0xF0u) + return {4u, lead & 0x07u}; + return {0u, 0u}; + }(); + if (length == 0u || index + length > text.size()) + return 0u; + + std::uint32_t codePoint = initial; + for (std::size_t offset = 1; offset < length; ++offset) + { + const auto continuation = static_cast(text[index + offset]); + if ((continuation & 0xC0u) != 0x80u) + return 0u; + codePoint = (codePoint << 6u) | (continuation & 0x3Fu); + } + constexpr std::array minimumForLength{0u, 0u, 0x80u, 0x800u, 0x10000u}; + if (codePoint < minimumForLength[length] || codePoint > 0x10FFFFu || + (codePoint >= 0xD800u && codePoint <= 0xDFFFu)) + return 0u; + return length; + } + + /** + * @brief The code point of the valid sequence of @p length bytes at @p index of @p text, see + * @ref utf8SequenceLength. + */ + constexpr char32_t utf8CodePointAt(std::string_view text, std::size_t index, std::size_t length) noexcept + { + constexpr std::array leadMask{0u, 0x7Fu, 0x1Fu, 0x0Fu, 0x07u}; + std::uint32_t codePoint = static_cast(text[index]) & leadMask[length]; + for (std::size_t offset = 1; offset < length; ++offset) + codePoint = (codePoint << 6u) | (static_cast(text[index + offset]) & 0x3Fu); + return static_cast(codePoint); + } + + /** + * @brief True when @p codePoint is a Unicode scalar value: at most U+10FFFF and no surrogate. + */ + constexpr bool isScalarValue(char32_t codePoint) noexcept + { + return codePoint <= 0x10FFFFu && (codePoint < 0xD800u || codePoint > 0xDFFFu); + } + + /** + * @brief Appends @p codePoint to @p text as UTF-8, U+FFFD when it is no scalar value. + */ + constexpr void appendUtf8(std::string& text, char32_t codePoint) + { + if (!isScalarValue(codePoint)) + codePoint = replacementCharacter; + const auto byte = [](std::uint32_t value) + { + return static_cast(static_cast(value)); + }; + if (codePoint < 0x80u) + { + text.push_back(byte(codePoint)); + } + else if (codePoint < 0x800u) + { + text.push_back(byte(0xC0u | (codePoint >> 6u))); + text.push_back(byte(0x80u | (codePoint & 0x3Fu))); + } + else if (codePoint < 0x10000u) + { + text.push_back(byte(0xE0u | (codePoint >> 12u))); + text.push_back(byte(0x80u | ((codePoint >> 6u) & 0x3Fu))); + text.push_back(byte(0x80u | (codePoint & 0x3Fu))); + } + else + { + text.push_back(byte(0xF0u | (codePoint >> 18u))); + text.push_back(byte(0x80u | ((codePoint >> 12u) & 0x3Fu))); + text.push_back(byte(0x80u | ((codePoint >> 6u) & 0x3Fu))); + text.push_back(byte(0x80u | (codePoint & 0x3Fu))); + } + } + + /** + * @brief Calls @p onCodePoint with every code point of @p text, U+FFFD for each byte that is not part of a + * valid sequence. + */ + template + constexpr void forEachUtf8CodePoint(std::string_view text, FunctionT&& onCodePoint) + { + std::size_t index = 0; + while (index < text.size()) + { + const auto length = utf8SequenceLength(text, index); + if (length == 0u) + { + onCodePoint(replacementCharacter); + ++index; + continue; + } + onCodePoint(utf8CodePointAt(text, index, length)); + index += length; + } + } + + /** + * @brief Calls @p onCodePoint with every code point of the UTF-16 @p text, U+FFFD for each unpaired + * surrogate. + */ + template + constexpr void forEachUtf16CodePoint(std::basic_string_view text, FunctionT&& onCodePoint) + { + std::size_t index = 0; + while (index < text.size()) + { + const auto unit = static_cast(static_cast(text[index])); + const bool isHigh = unit >= 0xD800u && unit <= 0xDBFFu; + const auto next = index + 1 < text.size() + ? static_cast(static_cast(text[index + 1])) + : char32_t{0}; + if (isHigh && next >= 0xDC00u && next <= 0xDFFFu) + { + onCodePoint(0x10000u + ((unit - 0xD800u) << 10u) + (next - 0xDC00u)); + index += 2; + continue; + } + onCodePoint(unit >= 0xD800u && unit <= 0xDFFFu ? replacementCharacter : unit); + ++index; + } + } + } + + /** + * @brief True when @p text is well-formed UTF-8 (no overlong forms, no surrogates, nothing above U+10FFFF). + */ + constexpr bool isValidUtf8(std::string_view text) noexcept + { + std::size_t index = 0; + while (index < text.size()) + { + const auto length = Detail::utf8SequenceLength(text, index); + if (length == 0u) + return false; + index += length; + } + return true; + } + + /** + * @brief @p text with every byte that is not part of a valid UTF-8 sequence replaced by U+FFFD, so the result + * is always valid UTF-8. Valid text comes back unchanged. + */ + constexpr std::string sanitizedUtf8(std::string_view text) + { + if (isValidUtf8(text)) + return std::string{text}; + std::string result; + result.reserve(text.size() + 8u); + Detail::forEachUtf8CodePoint( + text, + [&result](char32_t codePoint) + { + Detail::appendUtf8(result, codePoint); + } + ); + return result; + } + + /** + * @brief The longest start of @p text with at most @p maximumBytes bytes that does not end inside a valid + * UTF-8 sequence. A byte outside any valid sequence counts as a sequence of its own. + */ + constexpr std::string_view truncatedUtf8(std::string_view text, std::size_t maximumBytes) noexcept + { + if (text.size() <= maximumBytes) + return text; + std::size_t end = 0; + while (end < text.size()) + { + const auto length = Detail::utf8SequenceLength(text, end); + const auto step = length == 0u ? 1u : length; + if (end + step > maximumBytes) + break; + end += step; + } + return text.substr(0, end); + } + + /** + * @brief The number of UTF-16 code units of the valid UTF-8 @p text, the unit Windows limits names in. + */ + constexpr std::size_t utf16Length(std::string_view text) noexcept + { + std::size_t units = 0; + std::size_t index = 0; + while (index < text.size()) + { + const auto length = Detail::utf8SequenceLength(text, index); + units += length == 4u ? 2u : 1u; + index += length == 0u ? 1u : length; + } + return units; + } + + /** + * @brief True when the UTF-16 @p text has no unpaired surrogate. + */ + template + constexpr bool isValidUtf16(std::basic_string_view text) noexcept + { + bool valid = true; + std::size_t index = 0; + while (index < text.size() && valid) + { + const auto unit = static_cast(text[index]); + if (unit >= 0xD800u && unit <= 0xDBFFu && index + 1 < text.size()) + { + const auto next = static_cast(text[index + 1]); + if (next >= 0xDC00u && next <= 0xDFFFu) + { + index += 2; + continue; + } + } + valid = unit < 0xD800u || unit > 0xDFFFu; + ++index; + } + return valid; + } + + /** + * @brief The UTF-8 @p text as UTF-16, every byte that is not part of a valid sequence becomes U+FFFD. + */ + template + constexpr std::basic_string utf8ToUtf16(std::string_view text) + { + std::basic_string result; + result.reserve(text.size()); + Detail::forEachUtf8CodePoint( + text, + [&result](char32_t codePoint) + { + if (codePoint < 0x10000u) + { + result.push_back(static_cast(codePoint)); + return; + } + const auto offset = codePoint - 0x10000u; + result.push_back(static_cast(0xD800u + (offset >> 10u))); + result.push_back(static_cast(0xDC00u + (offset & 0x3FFu))); + } + ); + return result; + } + + /** + * @brief The UTF-16 @p text as UTF-8, every unpaired surrogate becomes U+FFFD. + */ + template + constexpr std::string utf16ToUtf8(std::basic_string_view text) + { + std::string result; + result.reserve(text.size()); + Detail::forEachUtf16CodePoint( + text, + [&result](char32_t codePoint) + { + Detail::appendUtf8(result, codePoint); + } + ); + return result; + } + +#ifdef _WIN32 + /** + * @brief UTF-8 to the UTF-16 text of the wide Windows API, see @ref utf8ToUtf16. + */ + inline std::wstring utf8ToWide(std::string_view text) + { + return utf8ToUtf16(text); + } + + /** + * @brief Wide Windows API text to UTF-8, see @ref utf16ToUtf8. + */ + inline std::string wideToUtf8(std::wstring_view text) + { + return utf16ToUtf8(text); + } +#endif +} diff --git a/paths/source/paths/CMakeLists.txt b/paths/source/paths/CMakeLists.txt index 4cc13287..47391ec2 100644 --- a/paths/source/paths/CMakeLists.txt +++ b/paths/source/paths/CMakeLists.txt @@ -1,4 +1,4 @@ add_library(paths INTERFACE) target_include_directories(paths INTERFACE "${CMAKE_CURRENT_LIST_DIR}/../../include") -target_link_libraries(paths INTERFACE nlohmann_json::nlohmann_json) +target_link_libraries(paths INTERFACE nlohmann_json::nlohmann_json fmt::fmt) From 06e08a44baae012d0448261b8961e25d01437383 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:02:43 +0200 Subject: [PATCH 016/131] Windows device names and name length follow the Windows rules Windows device names include COM0, LPT0, COM and LPT with a superscript digit, CONIN$ and CONOUT$, also with spaces before the extension, and a Windows name may be at most 255 UTF-16 code units; device names are upper-cased as ASCII only --- paths/include/paths/local_path_segment.hpp | 84 ++++++++++++++++------ 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/paths/include/paths/local_path_segment.hpp b/paths/include/paths/local_path_segment.hpp index 5f345e28..1f71e691 100644 --- a/paths/include/paths/local_path_segment.hpp +++ b/paths/include/paths/local_path_segment.hpp @@ -1,10 +1,11 @@ #pragma once #include +#include #include #include -#include +#include #include #include #include @@ -34,7 +35,8 @@ namespace Paths } /** - * @brief The local platform paths default to. The frontend runs in WebAssembly and sets the backend's here. + * @brief The local platform paths default to. The frontend runs in WebAssembly and sets the backend's here + * before it reads any local path. */ inline LocalPlatform& defaultLocalPlatform() { @@ -46,8 +48,9 @@ namespace Paths * @brief A @ref PathSegment that can be created on the local filesystem of @p platform. * * Windows additionally refuses '\\', ':', '<', '>', '"', '|', '?', '*', control characters, device names - * like CON or LPT1 (also with an extension) and a trailing dot or space. A remote name has to pass here - * before it becomes part of a local path. + * like CON, CONIN$, COM0 or LPT¹ (also with an extension or spaces before it), a trailing dot or space and + * names longer than 255 UTF-16 code units. A remote name has to pass here before it becomes part of a local + * path. */ class LocalPathSegment { @@ -57,7 +60,7 @@ namespace Paths { if (platform == LocalPlatform::Windows) { - if (auto reason = windowsRejection(segment.view()); reason) + if (const auto reason = windowsRejection(segment.view()); reason) return std::unexpected(PathError{.reason = *reason, .input = segment.value()}); } return LocalPathSegment{segment}; @@ -74,6 +77,19 @@ namespace Paths ); } + /** + * @brief Why @p utf8 cannot be a local name on @p platform, nothing when it can. + */ + static std::optional + rejectionOf(std::string_view utf8, LocalPlatform platform = defaultLocalPlatform()) + { + if (const auto rejection = PathSegment::rejectionOf(utf8); rejection) + return rejection; + if (platform == LocalPlatform::Windows) + return windowsRejection(utf8); + return std::nullopt; + } + PathSegment const& segment() const noexcept { return segment_; @@ -91,36 +107,58 @@ namespace Paths : segment_{std::move(segment)} {} + /** + * @brief The Windows rules for a valid @ref PathSegment. + */ static std::optional windowsRejection(std::string_view name) { - for (const auto character : name) + constexpr std::size_t maximumNameLength = 255u; + const auto reserved = [](char character) { - const auto byte = static_cast(character); - if (byte < 0x20u || std::string_view{"\\:<>\"|?*"}.find(character) != std::string_view::npos) - return PathRejection::ReservedCharacter; - } + return static_cast(character) < 0x20u || + std::string_view{"\\:<>\"|?*"}.find(character) != std::string_view::npos; + }; + if (std::ranges::any_of(name, reserved)) + return PathRejection::ReservedCharacter; if (name.back() == '.' || name.back() == ' ') return PathRejection::TrailingDotOrSpace; + if (isDeviceName(name)) + return PathRejection::ReservedDeviceName; + if (utf16Length(name) > maximumNameLength) + return PathRejection::NameTooLong; + return std::nullopt; + } + + /** + * @brief True for a Windows device name. It counts with any extension and with spaces before it: + * "nul.txt" and "CON .log" are devices too. + */ + static bool isDeviceName(std::string_view name) + { + auto stem = name.substr(0, name.find('.')); + while (!stem.empty() && stem.back() == ' ') + stem.remove_suffix(1); - // Device names count with any extension: "nul.txt" is the NUL device too. - const auto stem = name.substr(0, name.find('.')); std::string upper{stem}; - std::transform( - upper.begin(), - upper.end(), + std::ranges::transform( + upper, upper.begin(), - [](unsigned char character) + [](char character) { - return static_cast(std::toupper(character)); + return character >= 'a' && character <= 'z' ? static_cast(character - 'a' + 'A') + : character; } ); - constexpr std::array devices{"CON", "PRN", "AUX", "NUL"}; + constexpr std::array devices{"CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$"}; if (std::ranges::find(devices, upper) != devices.end()) - return PathRejection::ReservedDeviceName; - if (upper.size() == 4 && (upper.starts_with("COM") || upper.starts_with("LPT")) && upper[3] >= '1' && - upper[3] <= '9') - return PathRejection::ReservedDeviceName; - return std::nullopt; + return true; + if (!upper.starts_with("COM") && !upper.starts_with("LPT")) + return false; + + const auto number = std::string_view{upper}.substr(3); + constexpr std::array superscripts{"\xC2\xB9", "\xC2\xB2", "\xC2\xB3"}; + return (number.size() == 1u && number[0] >= '0' && number[0] <= '9') || + std::ranges::find(superscripts, number) != superscripts.end(); } PathSegment segment_; From 477509262650db15c4a59c4b772fc54d6bd53218 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:02:53 +0200 Subject: [PATCH 017/131] A Windows LocalPath is normalized and device paths are refused A Windows LocalPath has an upper-case drive letter and no \\?\ prefix (\\?\UNC\server\share becomes //server/share), device and namespace paths like \\.\PhysicalDrive0 or a server named '.', '?' or '..' are refused, NUL is refused in the root too, and isWithinIgnoringCase compares ASCII letters without case for the delete guards; joining onto an unset LocalPath fails with Empty --- paths/include/paths/local_path.hpp | 229 +++++++++++++++++++---------- 1 file changed, 155 insertions(+), 74 deletions(-) diff --git a/paths/include/paths/local_path.hpp b/paths/include/paths/local_path.hpp index 9533d46b..49fffda0 100644 --- a/paths/include/paths/local_path.hpp +++ b/paths/include/paths/local_path.hpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include @@ -19,13 +19,104 @@ namespace Paths { + namespace Detail + { + /** + * @brief True when @p text starts with @p prefix, ASCII letters compared without case. + */ + inline bool startsWithIgnoringCase(std::string_view text, std::string_view prefix) + { + const auto folded = [](char character) + { + return character >= 'A' && character <= 'Z' ? static_cast(character - 'A' + 'a') : character; + }; + return text.size() >= prefix.size() && + std::ranges::equal(text.substr(0, prefix.size()), prefix, {}, folded, folded); + } + + /** + * @brief Removes a \\?\ prefix from @p generic, a '/' separated Windows path: "//?/C:/x" becomes "C:/x" + * and "//?/UNC/server/share" becomes "//server/share". DevicePath for the device namespace "//./", + * any other "//?/" path and a server named ".", "?" or "..". + */ + inline std::expected removeWin32Prefix(std::string& generic) + { + const auto isLetter = [](char character) + { + return (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'); + }; + if (generic.starts_with("//?/")) + { + const auto rest = std::string_view{generic}.substr(4); + if (startsWithIgnoringCase(rest, "unc/")) + generic.erase(2, 6); + else if (rest.size() >= 2 && isLetter(rest[0]) && rest[1] == ':') + generic.erase(0, 4); + else + return std::unexpected(PathRejection::DevicePath); + } + if (!generic.starts_with("//")) + return {}; + const auto server = std::string_view{generic}.substr(2, generic.find('/', 2) - 2); + if (server == "." || server == "?" || server == "..") + return std::unexpected(PathRejection::DevicePath); + return {}; + } + + /** + * @brief Length of the root prefix of @p generic ("/", "C:/", "C:" or "//server/share"), 0 when it is not + * absolute on @p platform. Expects '/' separators and no \\?\ prefix. + */ + inline std::size_t rootLengthOf(std::string_view generic, LocalPlatform platform) + { + if (platform == LocalPlatform::Posix) + return generic.starts_with('/') ? 1u : 0u; + + const auto isLetter = [](char character) + { + return (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'); + }; + if (generic.size() >= 3 && isLetter(generic[0]) && generic[1] == ':' && generic[2] == '/') + return 3u; + if (generic.size() == 2 && isLetter(generic[0]) && generic[1] == ':') + return 2u; + if (generic.starts_with("//")) + { + const auto serverEnd = generic.find('/', 2); + if (serverEnd == std::string_view::npos || serverEnd == 2) + return 0u; + const auto shareEnd = std::min(generic.find('/', serverEnd + 1), generic.size()); + if (shareEnd == serverEnd + 1) + return 0u; + return shareEnd; + } + return 0u; + } + + /** + * @brief True when @p path is @p base or lies below it, both generic path text with @p base ending in '/' + * only when it is a root. @p equal compares two characters. + */ + template + bool isWithinText(std::string_view path, std::string_view base, EqualT equal) + { + if (path.size() < base.size() || !std::ranges::equal(path.substr(0, base.size()), base, equal)) + return false; + return path.size() == base.size() || base.back() == '/' || path[base.size()] == '/'; + } + } + /** * @brief An absolute, normalized path on the local machine, kept as generic UTF-8 ('/' separated). * * The frontend runs in WebAssembly where std::filesystem would misread "C:/x", so the type does its own parsing * for both platforms. The backend converts to a native std::filesystem::path only through @ref native. * + * On Windows the drive letter is upper case, a \\?\ prefix is dropped (\\?\UNC\server\share becomes + * //server/share) and device paths like \\.\PhysicalDrive0 are refused. + * * A default constructed LocalPath is unset (@ref empty), every operation that touches the disk must refuse it. + * Joining onto an unset path gives an unset path. */ class LocalPath { @@ -41,51 +132,51 @@ namespace Paths }; if (utf8.empty()) return refuse(PathRejection::Empty); + if (utf8.find('\0') != std::string_view::npos) + return refuse(PathRejection::ContainsNul); if (!isValidUtf8(utf8)) return refuse(PathRejection::InvalidUtf8); std::string generic{utf8}; if (platform == LocalPlatform::Windows) + { std::ranges::replace(generic, '\\', '/'); + if (const auto removed = Detail::removeWin32Prefix(generic); !removed) + return refuse(removed.error()); + } - auto rootLength = rootLengthOf(generic, platform); + const auto rootLength = Detail::rootLengthOf(generic, platform); if (rootLength == 0) return refuse(PathRejection::NotAbsolute); - LocalPath result; - result.platform_ = platform; - result.value_ = generic.substr(0, rootLength); - if (result.value_.back() != '/') - result.value_.push_back('/'); - result.rootLength_ = result.value_.size(); - - std::vector segments; - std::string_view rest{generic}; - rest.remove_prefix(rootLength); - std::size_t start = 0; - while (start <= rest.size()) + std::vector names; + for (auto const name : Detail::splitAtSlashes(std::string_view{generic}.substr(rootLength))) { - const auto end = std::min(rest.find('/', start), rest.size()); - const auto name = rest.substr(start, end - start); - start = end + 1; if (name.empty() || name == ".") continue; if (name == "..") { - if (segments.empty()) + if (names.empty()) return refuse(PathRejection::EscapesRoot); - segments.pop_back(); + names.pop_back(); continue; } - segments.push_back(name); - } - for (auto const name : segments) - { - auto segment = LocalPathSegment::make(name, platform); - if (!segment) - return refuse(segment.error().reason); - result = result / *segment; + if (const auto rejection = LocalPathSegment::rejectionOf(name, platform); rejection) + return refuse(*rejection); + names.push_back(name); } + + LocalPath result; + result.platform_ = platform; + result.value_.reserve(generic.size() + 1); + result.value_.append(generic, 0, rootLength); + if (result.value_.back() != '/') + result.value_.push_back('/'); + if (platform == LocalPlatform::Windows && result.value_[0] >= 'a' && result.value_[0] <= 'z') + result.value_[0] = static_cast(result.value_[0] - 'a' + 'A'); + result.rootLength_ = result.value_.size(); + for (auto const name : names) + Detail::appendSegment(result.value_, name); return result; } @@ -109,27 +200,29 @@ namespace Paths LocalPath operator/(LocalPathSegment const& segment) const { - assert(!empty()); + if (empty()) + return {}; LocalPath result{*this}; - if (result.value_.back() != '/') - result.value_.push_back('/'); - result.value_ += segment.value(); + Detail::appendSegment(result.value_, segment.value()); return result; } /** - * @brief Joins @p relative, every segment has to be a valid name on this path's platform. + * @brief Joins @p relative, every segment has to be a valid name on this path's platform. Empty when this + * path is unset. */ std::expected join(RelativePath const& relative) const { - LocalPath result{*this}; - for (auto const& segment : relative.segments()) + if (empty()) + return std::unexpected(PathError{.reason = PathRejection::Empty, .input = relative.value()}); + for (auto const name : relative.segmentViews()) { - auto local = LocalPathSegment::make(segment, platform_); - if (!local) - return std::unexpected(local.error()); - result = result / *local; + if (const auto rejection = LocalPathSegment::rejectionOf(name, platform_); rejection) + return std::unexpected(PathError{.reason = *rejection, .input = std::string{name}}); } + LocalPath result{*this}; + if (!relative.empty()) + Detail::appendSegment(result.value_, relative.value()); return result; } @@ -157,21 +250,39 @@ namespace Paths { if (empty() || isRoot()) return std::nullopt; - return *PathSegment::make(std::string_view{value_}.substr(value_.rfind('/') + 1)); + return PathSegment{std::string{Detail::lastSegment(value_)}}; } /** - * @brief True when this is @p base or lies below it. + * @brief True when this is @p base or lies below it, compared exactly. */ bool isWithin(LocalPath const& base) const { if (empty() || base.empty() || platform_ != base.platform_) return false; - if (value_ == base.value_) - return true; - if (!value_.starts_with(base.value_)) + return Detail::isWithinText(value_, base.value_, std::ranges::equal_to{}); + } + + /** + * @brief @ref isWithin with ASCII letters compared without case. For checks that must not miss a path the + * filesystem treats as the same, like the delete guards on Windows and macOS, which fold case. + */ + bool isWithinIgnoringCase(LocalPath const& base) const + { + if (empty() || base.empty() || platform_ != base.platform_) return false; - return base.value_.back() == '/' || value_[base.value_.size()] == '/'; + const auto folded = [](char character) + { + return character >= 'A' && character <= 'Z' ? static_cast(character - 'A' + 'a') : character; + }; + return Detail::isWithinText( + value_, + base.value_, + [&folded](char left, char right) + { + return folded(left) == folded(right); + } + ); } /** @@ -205,36 +316,6 @@ namespace Paths auto operator<=>(LocalPath const&) const = default; private: - /** - * @brief Length of the root prefix ("/", "C:/" or "//server/share"), 0 when not absolute. - */ - static std::size_t rootLengthOf(std::string_view generic, LocalPlatform platform) - { - if (platform == LocalPlatform::Posix) - return generic.starts_with('/') ? 1u : 0u; - - const auto isLetter = [](char character) - { - return (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'); - }; - if (generic.size() >= 3 && isLetter(generic[0]) && generic[1] == ':' && generic[2] == '/') - return 3u; - if (generic.size() == 2 && isLetter(generic[0]) && generic[1] == ':') - return 2u; - if (generic.starts_with("//")) - { - // UNC: //server/share is the root. - const auto serverEnd = generic.find('/', 2); - if (serverEnd == std::string_view::npos || serverEnd == 2) - return 0u; - const auto shareEnd = std::min(generic.find('/', serverEnd + 1), generic.size()); - if (shareEnd == serverEnd + 1) - return 0u; - return shareEnd; - } - return 0u; - } - std::string value_{}; std::size_t rootLength_{0}; LocalPlatform platform_{LocalPlatform::Posix}; From 27173663305c498665425400feed717f71584526 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:05:28 +0200 Subject: [PATCH 018/131] The local delete guard folds case and compares homes by file identity The local delete guard also compares case-insensitively for Windows paths and on macOS, and refuses an existing directory that is a home or one of its ancestors by file identity (other case, 8.3 and NFD names, symlinked ancestors); it protects every home it finds (Windows: FOLDERID_Profile through SHGetKnownFolderPath, USERPROFILE, HOME; elsewhere HOME and the passwd entry), logs when there is none, never throws, splits on backslashes only for Windows paths and names the reason with PathError::toString --- .../include/backend/local_delete_guard.hpp | 60 ++---- backend/source/backend/local_delete_guard.cpp | 173 +++++++++++++++++- backend/source/backend/rpc_filesystem.cpp | 28 +-- .../source/backend/sftp/operation_queue.cpp | 8 +- 4 files changed, 193 insertions(+), 76 deletions(-) diff --git a/backend/include/backend/local_delete_guard.hpp b/backend/include/backend/local_delete_guard.hpp index 55120717..7f33e7a5 100644 --- a/backend/include/backend/local_delete_guard.hpp +++ b/backend/include/backend/local_delete_guard.hpp @@ -2,55 +2,33 @@ #include -#include - #include -#include -#include #include -#include +#include /** - * @brief Refuses a root directory and @p home or a directory containing it. + * @brief Refuses an unset path, a root directory, and each of @p homes or a directory containing one. + * + * Paths are compared as text, and without ASCII case for Windows paths and on macOS, whose filesystems fold case. + * A target that is an existing directory (not a link to one) is also compared by file identity with every home and + * each of its ancestors, which catches other spellings of the same directory: case beyond ASCII, 8.3 names, NFD + * names and ancestors reached through a symlink. */ -inline std::expected -refuseProtectedLocalDelete(Paths::LocalPath const& target, std::optional const& home) -{ - if (target.empty()) - return std::unexpected(std::string{"no path"}); - if (target.isRoot()) - return std::unexpected(fmt::format("'{}' is a root directory", target.toUtf8())); - if (home && home->isWithin(target)) - return std::unexpected(fmt::format("'{}' contains the home directory", target.toUtf8())); - return {}; -} +std::expected +refuseProtectedLocalDelete(Paths::LocalPath const& target, std::vector const& homes); /** - * @brief A local path that may be deleted, or the reason to refuse it: not absolute, containing '..', a root - * directory, or @p home or a directory containing it. + * @brief A local path that may be deleted, or the reason to refuse it: not a valid absolute path, containing '..' + * (also as '\\..' on Windows) or refused by @ref refuseProtectedLocalDelete. */ -inline std::expected -localDeleteTarget(std::string const& utf8, std::optional const& home) -{ - // LocalPath resolves ".." lexically, which goes elsewhere when a directory on the way is a symlink. - for (auto const segment : std::views::split(std::string_view{utf8}, '/')) - { - for (auto const part : std::views::split(segment, '\\')) - { - if (std::string_view{part.begin(), part.end()} == "..") - return std::unexpected(fmt::format("'{}' contains '..'", utf8)); - } - } - - auto target = Paths::LocalPath::fromUtf8(utf8); - if (!target) - return std::unexpected(fmt::format("'{}' {}", utf8, Paths::toString(target.error().reason))); - if (auto allowed = refuseProtectedLocalDelete(*target, home); !allowed) - return std::unexpected(std::move(allowed).error()); - return *std::move(target); -} +std::expected +localDeleteTarget(std::string const& utf8, std::vector const& homes); /** - * @brief The home directory of the user running the backend, empty when it cannot be determined. + * @brief The home directories of the user running the backend, which the delete guards protect, each once. + * + * On Windows the profile folder (FOLDERID_Profile) and the USERPROFILE and HOME variables, elsewhere the HOME + * variable and the home of the passwd entry. When none can be determined a warning is logged and only roots stay + * protected. Never throws. */ -std::optional localHomeDirectory(); +std::vector localHomeDirectories(); diff --git a/backend/source/backend/local_delete_guard.cpp b/backend/source/backend/local_delete_guard.cpp index cd786edc..3643013e 100644 --- a/backend/source/backend/local_delete_guard.cpp +++ b/backend/source/backend/local_delete_guard.cpp @@ -1,11 +1,172 @@ #include +#include -#include +#include -std::optional localHomeDirectory() +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# include +#else +# include +# include +#endif + +namespace +{ + bool foldsCase([[maybe_unused]] Paths::LocalPath const& path) + { +#ifdef __APPLE__ + return true; +#else + return path.platform() == Paths::LocalPlatform::Windows; +#endif + } + + bool containsAsText(Paths::LocalPath const& target, Paths::LocalPath const& home) + { + return home.isWithin(target) || (foldsCase(target) && home.isWithinIgnoringCase(target)); + } + + /** + * @brief True when @p target is an existing directory that is @p home or one of its ancestors on disk. + */ + bool isHomeOrAncestorOnDisk(Paths::LocalPath const& target, Paths::LocalPath const& home) + { + if (target.platform() != Paths::compiledLocalPlatform() || home.platform() != target.platform()) + return false; + const auto native = target.native(); + std::error_code error{}; + if (!std::filesystem::is_directory(std::filesystem::symlink_status(native, error))) + return false; + for (std::optional ancestor = home; ancestor; ancestor = ancestor->parent()) + { + if (std::filesystem::equivalent(ancestor->native(), native, error) && !error) + return true; + } + return false; + } + + void addHome(std::vector& homes, std::filesystem::path const& native) + { + if (native.empty()) + return; + try + { + auto home = Paths::LocalPath::make(native); + if (!home) + { + Log::warn("Ignoring the home directory candidate {}", home.error().toString()); + return; + } + if (std::ranges::find(homes, *home) == homes.end()) + homes.push_back(*std::move(home)); + } + catch (std::exception const& exception) + { + Log::warn( + "Ignoring a home directory candidate that is not representable: {}", Utility::errorMessage(exception) + ); + } + } + +#ifdef _WIN32 + struct CoTaskMemoryDeleter + { + void operator()(wchar_t* memory) const noexcept + { + CoTaskMemFree(memory); + } + }; + + void addEnvironmentHome(std::vector& homes, wchar_t const* name) + { + if (auto const* value = _wgetenv(name); value != nullptr && *value != L'\0') + addHome(homes, std::filesystem::path{value}); + } +#endif +} + +std::expected +refuseProtectedLocalDelete(Paths::LocalPath const& target, std::vector const& homes) { - auto home = Paths::LocalPath::make(Nui::resolvePath("%userprofile%")); - if (!home) - return std::nullopt; - return *std::move(home); + if (target.empty()) + return std::unexpected(std::string{"no path"}); + if (target.isRoot()) + return std::unexpected(fmt::format("'{}' is a root directory", target.toUtf8())); + const auto protectedHome = std::ranges::find_if( + homes, + [&target](Paths::LocalPath const& home) + { + return containsAsText(target, home) || isHomeOrAncestorOnDisk(target, home); + } + ); + if (protectedHome != homes.end()) + return std::unexpected( + fmt::format("'{}' is or contains the home directory '{}'", target.toUtf8(), protectedHome->toUtf8()) + ); + return {}; +} + +std::expected +localDeleteTarget(std::string const& utf8, std::vector const& homes) +{ + auto target = Paths::LocalPath::fromUtf8(utf8); + if (!target) + return std::unexpected(target.error().toString()); + + // LocalPath resolves ".." lexically, which goes elsewhere when a directory on the way is a symlink. + std::string generic{utf8}; + if (target->platform() == Paths::LocalPlatform::Windows) + std::ranges::replace(generic, '\\', '/'); + const auto isParentReference = [](std::string_view name) + { + return name == ".."; + }; + if (std::ranges::any_of(Paths::Detail::splitAtSlashes(generic), isParentReference)) + return std::unexpected(fmt::format("'{}' contains '..'", utf8)); + + if (auto allowed = refuseProtectedLocalDelete(*target, homes); !allowed) + return std::unexpected(std::move(allowed).error()); + return *std::move(target); +} + +std::vector localHomeDirectories() +{ + std::vector homes; +#ifdef _WIN32 + PWSTR rawProfile = nullptr; + const auto result = SHGetKnownFolderPath(FOLDERID_Profile, KF_FLAG_DEFAULT, nullptr, &rawProfile); + const std::unique_ptr profile{rawProfile}; + if (SUCCEEDED(result) && profile != nullptr) + addHome(homes, std::filesystem::path{profile.get()}); + addEnvironmentHome(homes, L"USERPROFILE"); + addEnvironmentHome(homes, L"HOME"); +#else + if (auto const* home = std::getenv("HOME"); home != nullptr && *home != '\0') + addHome(homes, std::filesystem::path{home}); + + const auto suggestedSize = sysconf(_SC_GETPW_R_SIZE_MAX); + std::vector buffer(suggestedSize > 0 ? static_cast(suggestedSize) : 16384u); + passwd entry{}; + passwd* found = nullptr; + if (getpwuid_r(getuid(), &entry, buffer.data(), buffer.size(), &found) == 0 && found != nullptr && + found->pw_dir != nullptr) + addHome(homes, std::filesystem::path{found->pw_dir}); +#endif + if (homes.empty()) + Log::warn("The home directory is unknown, local deletes only refuse root directories."); + return homes; } diff --git a/backend/source/backend/rpc_filesystem.cpp b/backend/source/backend/rpc_filesystem.cpp index 3a7060fc..f164bfc7 100644 --- a/backend/source/backend/rpc_filesystem.cpp +++ b/backend/source/backend/rpc_filesystem.cpp @@ -48,11 +48,6 @@ namespace ); } - std::expected deletableLocalPath(std::string const& utf8) - { - return localDeleteTarget(utf8, localHomeDirectory()); - } - #ifndef _WIN32 /** * @brief Per-call uid/gid → name cache, so a directory full of files owned by the same user @@ -223,7 +218,7 @@ void RpcFilesystem::registerRemove() const bool recursive = parameters.value("recursive", false); const auto pathUtf8 = parameters["path"].get(); - const auto target = deletableLocalPath(pathUtf8); + const auto target = localDeleteTarget(pathUtf8, localHomeDirectories()); if (!target) { Log::error("RpcFilesystem::remove refused: {}", target.error()); @@ -283,25 +278,8 @@ void RpcFilesystem::registerRemoveMultiple() const bool recursive = parameters.value("recursive", false); const auto pathsUtf8 = parameters["paths"].get>(); - - // Nothing is removed when one of the paths is refused. - std::vector targets; - targets.reserve(pathsUtf8.size()); - for (const auto& pathUtf8 : pathsUtf8) - { - auto target = deletableLocalPath(pathUtf8); - if (!target) - { - Log::error("RpcFilesystem::removeSome refused: {}", target.error()); - return reply.error(Utility::localizedMessage("backend.rpcFilesystem.removeRefused", pathUtf8)); - } - targets.push_back(*std::move(target)); - } - - // One item that cannot be removed does not keep the others, every failure is reported. - size_t removedCount = 0; - std::vector failures; - for (const auto& target : targets) + const auto removal = removeLocalItems(pathsUtf8, localHomeDirectories()); + if (!removal) { const auto path = target.native(); const auto& pathUtf8 = target.toUtf8(); diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index baac44e8..5c11260f 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1564,7 +1564,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( SharedData::BulkDownloadRequest downloads{.allowOverwrite = true, .mode = request.mode}; SharedData::BulkDeleteRequest remoteDeletes{.mode = request.mode, .listedContentOnly = true}; std::vector localDeletes; - const auto localHome = localHomeDirectory(); + const auto localHomes = localHomeDirectories(); for (auto& [entry, paths] : plan.entries) { @@ -1623,7 +1623,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( } case Action::DeleteLocal: { - if (auto allowed = refuseProtectedLocalDelete(paths.local, localHome); !allowed) + if (isBelowAny(refusedLocalDirectories, entry.relKey)) { Log::error("Sync: refusing to delete locally: {}", allowed.error()); result.failed.push_back(std::move(entry.relKey)); @@ -1695,10 +1695,10 @@ std::expected OperationQueue::addLocalDeleteOperation( // Assumed in strand if (targets.empty()) return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = "no targets"}); - const auto home = localHomeDirectory(); + const auto homes = localHomeDirectories(); for (auto const& target : targets) { - if (auto allowed = refuseProtectedLocalDelete(target, home); !allowed) + if (auto allowed = refuseProtectedLocalDelete(target, homes); !allowed) { Log::error("Refusing to delete locally: {}", allowed.error()); return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = target.toUtf8()}); From 95fe1a4fe4d76e97e61e5b009eb87af198a4e5e4 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:16:48 +0200 Subject: [PATCH 019/131] SftpSession canonicalizes paths and the start directory on the server SftpSession canonicalizes a path through the server (canonicalPathInStrand) and reports the start directory without any fallback (canonicalHomeDirectoryInStrand), which homeDirectoryInStrand builds on; the mock server's REALPATH resolves symlinks like realpath --- ssh/include/ssh/sftp_session.hpp | 18 ++++++++++++++ ssh/source/ssh/sftp_session.cpp | 42 ++++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/ssh/include/ssh/sftp_session.hpp b/ssh/include/ssh/sftp_session.hpp index 0131e5c9..3f4e846e 100644 --- a/ssh/include/ssh/sftp_session.hpp +++ b/ssh/include/ssh/sftp_session.hpp @@ -350,6 +350,19 @@ namespace SecureShell */ std::expected homeDirectoryInStrand(); + /** + * @brief The directory the server starts the user in, resolved by the server (realpath of "." on OpenSSH), + * without any fallback: an error when the server does not answer with an absolute path. Must be + * called from within the processing thread. + */ + std::expected canonicalHomeDirectoryInStrand(); + + /** + * @brief The server's canonical form of the existing @p path, with every symlink resolved (realpath on + * OpenSSH). Must be called from within the processing thread. + */ + std::expected canonicalPathInStrand(Paths::RemotePath const& path); + /** * @brief Reads the XDG user directories from /.config/user-dirs.dirs. A missing file yields no * directories instead of an error. @@ -433,6 +446,11 @@ namespace SecureShell */ static std::expected requireSet(Paths::RemotePath const& path); + /** + * @brief The absolute path the server canonicalizes @p path to. + */ + std::expected canonicalizeInStrand(char const* path); + static SftpError alreadyExists(Paths::RemotePath const& destination); /** diff --git a/ssh/source/ssh/sftp_session.cpp b/ssh/source/ssh/sftp_session.cpp index fd32b5e4..cbe5fd76 100644 --- a/ssh/source/ssh/sftp_session.cpp +++ b/ssh/source/ssh/sftp_session.cpp @@ -746,17 +746,45 @@ namespace SecureShell ); } - std::expected SftpSession::homeDirectoryInStrand() + std::expected SftpSession::canonicalizeInStrand(char const* path) { assert(strand_->withinProcessingThread()); const std::unique_ptr canonical{ - sftp_canonicalize_path(session_, "."), ssh_string_free_char + sftp_canonicalize_path(session_, path), ssh_string_free_char }; - if (canonical != nullptr && canonical.get()[0] == '/') - { - if (auto home = Paths::RemotePath::make(canonical.get()); home) - return *std::move(home); - } + if (canonical == nullptr) + return std::unexpected(lastError()); + return Paths::RemotePath::make(canonical.get()) + .transform_error( + [](Paths::PathError const& error) + { + return Error{ + .message = fmt::format("The server canonicalized to {}", error.toString()), + .sftpError = SSH_FX_FAILURE, + .wrapperError = WrapperErrors::InvalidPath, + }; + } + ); + } + + std::expected SftpSession::canonicalHomeDirectoryInStrand() + { + return canonicalizeInStrand("."); + } + + std::expected + SftpSession::canonicalPathInStrand(Paths::RemotePath const& path) + { + if (auto valid = requireSetInStrand(path); !valid) + return std::unexpected(valid.error()); + return canonicalizeInStrand(path.c_str()); + } + + std::expected SftpSession::homeDirectoryInStrand() + { + assert(strand_->withinProcessingThread()); + if (auto home = canonicalHomeDirectoryInStrand(); home) + return home; char* rawUser = nullptr; const auto optionResult = ssh_options_get(session_->session, SSH_OPTIONS_USER, &rawUser); From d27cb6aa409f78559c9a8eea0b1c038077e81a98 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:19:09 +0200 Subject: [PATCH 020/131] Remote deletes are checked by a RemoteDeleteGuard Remote deletes are checked by a RemoteDeleteGuard: it canonicalizes each target's parent on the server (once per parent and request, once per sync root) so a symlinked ancestor cannot hide the home, also refuses the home without regard to ASCII case and, on servers with Windows paths, names with a backslash or a trailing dot or space; an unknown home is looked up without the / fallback and logged. The three delete RPCs refuse through one refuseAnyProtected and answer with a localized reason that says nothing was deleted; the queue checks once more with the same guard --- .../backend/sftp/local_delete_operation.hpp | 6 +- .../include/backend/sftp/operation_queue.hpp | 55 ++-- .../backend/sftp/remote_delete_guard.hpp | 154 ++++++++++ backend/source/backend/CMakeLists.txt | 5 + backend/source/backend/session.cpp | 57 ++-- .../source/backend/sftp/operation_queue.cpp | 71 +++-- .../backend/sftp/remote_delete_guard.cpp | 267 ++++++++++++++++++ .../file_explorer/remote_side_model.cpp | 3 +- static/assets/languages/de_DE.yaml | 6 + static/assets/languages/en_US.yaml | 6 + 10 files changed, 548 insertions(+), 82 deletions(-) create mode 100644 backend/include/backend/sftp/remote_delete_guard.hpp create mode 100644 backend/source/backend/sftp/remote_delete_guard.cpp diff --git a/backend/include/backend/sftp/local_delete_operation.hpp b/backend/include/backend/sftp/local_delete_operation.hpp index a60c00d9..9f4f10d8 100644 --- a/backend/include/backend/sftp/local_delete_operation.hpp +++ b/backend/include/backend/sftp/local_delete_operation.hpp @@ -10,8 +10,10 @@ #include /** - * @brief Deletes local items through the queue, so they honour pause and show progress like remote deletes. - * Exactly the given items are removed: a directory only once empty, a link as the link itself. + * @brief Deletes local items in the LocalOperationQueue, so they can be held and canceled and show progress like + * remote deletes. Each work step deletes until its time budget is used. Links and junctions are removed + * themselves and never followed. The queue checks the items with the local delete guard before it adds the + * operation. */ class LocalDeleteOperation : public Operation { diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index 6af445b5..68480e09 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -151,8 +152,8 @@ class OperationQueue ); /** - * @brief Queues the delete of @p remotePath, refused for the root and for the home directory or any directory - * that contains it. + * @brief Queues the delete of @p remotePath, refused by a @ref remoteDeleteGuard for the root and for the home + * directory or any directory that contains it. */ std::expected addDeleteOperation( SecureShell::SftpSession& sftp, @@ -163,6 +164,20 @@ class OperationQueue SharedData::OperationMode mode = SharedData::OperationMode::Queued ); + /** + * @brief @ref addDeleteOperation checking with @p guard, so a caller that checked the target with it already + * costs no further round trip. + */ + std::expected addDeleteOperation( + RemoteDeleteGuard& guard, + SecureShell::SftpSession& sftp, + Ids::OperationId operationId, + Paths::RemotePath const& remotePath, + bool recursive, + bool insertRefresh, + SharedData::OperationMode mode = SharedData::OperationMode::Queued + ); + /** * @brief Bulk-add download operations from a pre-known entry list. * Files are taken at face value (no lstat — sizes come from the @@ -201,19 +216,28 @@ class OperationQueue ); /** - * @brief Bulk-delete: collapse N file entries (and any directory entries) - * into a single delete operation card on the frontend. File - * entries are pre-known so no SFTP stat is needed. Directory - * entries get the existing Scan+Delete pair so descendants are - * recursively removed. + * @brief Queues the remote items of @p request as one delete operation with one card, labelled with the + * directory they lie in. Entries a @ref remoteDeleteGuard refuses, and directories that could not be + * looked up in time, are failed entries of that card. * - * @param sftp SFTP session to use. - * @param request The remote items; directories are scanned and deleted with their content. - * @param bulkOperationId Single OperationId for the aggregate file-bulk - * card. Per-directory ids are generated locally. - * @return Number of entries successfully enqueued. + * @param sftp SFTP session to use. + * @param request The remote items; directories are scanned and deleted with their content, unless the request + * deletes listed content only. + * @param bulkOperationId Operation id of the one card. + * @return Number of entries queued for deletion; zero queues nothing, not even the card. + */ + std::size_t addBulkDeleteOperation( + SecureShell::SftpSession& sftp, + SharedData::BulkDeleteRequest const& request, + Ids::OperationId bulkOperationId + ); + + /** + * @brief @ref addBulkDeleteOperation checking with @p guard, so a caller that checked the targets with it + * already costs no further round trip. */ std::size_t addBulkDeleteOperation( + RemoteDeleteGuard& guard, SecureShell::SftpSession& sftp, SharedData::BulkDeleteRequest const& request, Ids::OperationId bulkOperationId @@ -242,10 +266,9 @@ class OperationQueue ); /** - * @brief Refuses an unset delete target, the root, and the home directory or any directory that contains it. + * @brief A guard for the remote deletes of one request, sharing the home directory this queue looked up. */ - std::expected - refuseProtectedDeleteTarget(SecureShell::SftpSession& sftp, Paths::RemotePath const& target); + RemoteDeleteGuard remoteDeleteGuard(SecureShell::SftpSession& sftp); std::expected addRenameOperation( SecureShell::SftpSession& sftp, @@ -380,6 +403,6 @@ class OperationQueue // the bulk's OperationId. Read by dumpBulkResumes() during teardown. std::unordered_map bulkResumeStash_{}; - /** @brief The server's home directory, asked once for the delete guard. */ + /** @brief The canonical home directory of the server, asked once for the delete guards. */ std::optional remoteHome_{}; }; diff --git a/backend/include/backend/sftp/remote_delete_guard.hpp b/backend/include/backend/sftp/remote_delete_guard.hpp new file mode 100644 index 00000000..31a3368b --- /dev/null +++ b/backend/include/backend/sftp/remote_delete_guard.hpp @@ -0,0 +1,154 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SecureShell +{ + class SftpSession; +} + +/** + * @brief Why a remote delete target is protected. + */ +struct RemoteDeleteRefusal +{ + enum class Reason : std::uint8_t + { + NoPath, + Root, + ContainsHome, + /** + * @brief On a server with Windows paths a backslash, or a trailing dot or space, can name another item. + */ + WindowsName, + /** + * @brief The server did not answer in time, so the target could not be compared with the home directory. + */ + NotVerified, + }; + + Reason reason{Reason::NoPath}; + std::string target{}; + std::string home{}; + + /** + * @brief English text for the log. + */ + std::string toString() const; + + /** + * @brief A message built with Utility::localizedMessage that also says nothing was deleted. + */ + std::string toLocalizedMessage() const; +}; + +namespace RemoteDeleteGuardDetail +{ + /** + * @brief True when @p path starts with a drive like "/C:", as OpenSSH for Windows reports paths. + */ + bool looksLikeWindowsPath(Paths::RemotePath const& path); + + /** + * @brief True when @p path looks like it lies on a server whose filesystem folds case: a Windows drive path or + * one below the /Users of macOS. + */ + bool looksCaseFolding(Paths::RemotePath const& path); + + /** + * @brief @ref Paths::RemotePath::isWithin with ASCII letters compared without case. + */ + bool isWithinIgnoringCase(Paths::RemotePath const& path, Paths::RemotePath const& base); + + /** + * @brief The checks that need no server: an unset path, the root, a name a Windows server could read as another + * one, and @p home within @p target, without ASCII case when the target or the home looks like it lies on a + * server that folds case (@ref looksCaseFolding). + */ + std::optional + refusalByText(Paths::RemotePath const& target, std::optional const& home); +} + +/** + * @brief Refuses remote delete targets that are the root, the home directory or a directory containing it, also when + * reached through a symlink: the parent of each target is canonicalized by the server, once per parent, and + * the target's name appended unresolved, because deleting a link removes the link. + * + * One guard serves one request. It runs on the session strand and waits for the SFTP strand. + */ +class RemoteDeleteGuard +{ + public: + /** + * @param knownHome The canonical home of the session. Looked up on the first check that needs it while unset, + * and stored there once known. While it is unknown the guard logs it and only refuses by text. + */ + RemoteDeleteGuard( + SecureShell::SftpSession& sftp, + std::optional& knownHome, + std::chrono::milliseconds timeout + ); + + std::expected check(Paths::RemotePath const& target); + + /** + * @brief The canonical home of the session, looked up like for a check. Nothing when it is unknown or the + * server did not answer in time. + */ + std::optional homeDirectory(); + + /** + * @brief Treats every directory below @p root as no symlink, so the canonical form of a path below it is the + * canonical @p root with the rest appended. For targets that come from an lstat scan of @p root, which + * does not descend into links. + */ + void trustScannedBelow(Paths::RemotePath root); + + private: + std::expected, RemoteDeleteRefusal> home(Paths::RemotePath const& target); + std::expected, RemoteDeleteRefusal> + canonicalDirectory(Paths::RemotePath const& directory, Paths::RemotePath const& target); + std::expected, RemoteDeleteRefusal> + lookUpCanonical(Paths::RemotePath const& directory, Paths::RemotePath const& target); + + SecureShell::SftpSession* sftp_; + std::optional* knownHome_; + std::chrono::milliseconds timeout_; + bool homeUnknown_{false}; + std::vector scannedRoots_{}; + std::unordered_map> canonicalDirectories_{}; +}; + +/** + * @brief The refusal of the first protected target in @p targets, nothing when every one may be deleted. + */ +template +std::optional +refuseAnyProtected(RemoteDeleteGuard& guard, RangeT&& targets, ProjectionT projection = {}) +{ + std::optional refusal{}; + const auto protectedTarget = std::ranges::find_if( + targets, + [&guard, &refusal](Paths::RemotePath const& target) + { + auto checked = guard.check(target); + if (!checked) + refusal = std::move(checked).error(); + return refusal.has_value(); + }, + projection + ); + if (protectedTarget == std::ranges::end(targets)) + return std::nullopt; + return refusal; +} diff --git a/backend/source/backend/CMakeLists.txt b/backend/source/backend/CMakeLists.txt index c2071970..7d7b8500 100644 --- a/backend/source/backend/CMakeLists.txt +++ b/backend/source/backend/CMakeLists.txt @@ -5,7 +5,10 @@ add_library( session.cpp rpc_filesystem.cpp local_delete_guard.cpp + local_removal.cpp + local_operation_queue.cpp existence_check.cpp + rpc_path_parameters.cpp rpc_system.cpp theme_finder.cpp dropped_entry.cpp @@ -31,8 +34,10 @@ add_library( sftp/bulk_download_operation.cpp sftp/bulk_upload_operation.cpp sftp/bulk_resume_registry.cpp + sftp/compared_state_check.cpp sftp/delete_operation.cpp sftp/local_delete_operation.cpp + sftp/remote_delete_guard.cpp sftp/rename_operation.cpp sftp/archive_download_operation.cpp sftp/archive_upload_operation.cpp diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index 6ad7eefd..aec08ab2 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -77,6 +77,15 @@ namespace } return result; } + + /** + * @brief Answers a delete request with the localized reason why nothing was deleted. + */ + void replyRefusal(RpcHelper::RpcOnce const& reply, RemoteDeleteRefusal const& refusal) + { + Log::error("Refusing to delete: {}", refusal.toString()); + reply.error(refusal.toLocalizedMessage()); + } } Session::Session( @@ -773,19 +782,16 @@ void Session::registerRpcQueuedRemoves() if (!self) return reply.error("Session no longer exists"); - // Nothing is queued when one of the targets is protected. - for (auto const& path : paths) - { - if (auto allowed = self->operationQueue_->refuseProtectedDeleteTarget(*channel, path); - !allowed) - return reply.error(fmt::format("Refusing to delete '{}'", path.value())); - } + auto guard = self.operationQueue_->remoteDeleteGuard(*channel); + if (const auto refusal = refuseAnyProtected(guard, paths); refusal) + return replyRefusal(reply, *refusal); std::vector operationIds; for (auto const& path : paths) { const auto newOperationId = Ids::generateOperationId(); const auto result = self->operationQueue_->addDeleteOperation( + guard, *channel, newOperationId, path, @@ -1766,12 +1772,9 @@ void Session::registerRpcSftpDeleteFiles() auto remotePaths = parseRemotePaths(paths); if (!remotePaths) return reply.error(remotePaths.error()); - for (auto const& path : *remotePaths) - { - if (auto allowed = self->operationQueue_->refuseProtectedDeleteTarget(*channel, path); - !allowed) - return reply.error(fmt::format("Refusing to delete '{}'", path.value())); - } + auto guard = self.operationQueue_->remoteDeleteGuard(*channel); + if (const auto refusal = refuseAnyProtected(guard, *remotePaths); refusal) + return replyRefusal(reply, *refusal); auto fut = channel->removeAll(*std::move(remotePaths)); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error("Failed to delete files: timeout"); @@ -1887,27 +1890,13 @@ void Session::registerRpcSftpAddBulkDeleteOperation() bulkOperationIdString, request](RpcHelper::RpcOnce&& reply, auto&& channel) { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - // Nothing is queued when one of the targets is protected. - for (auto const& entry : request.entries) - { - if (auto allowed = self->operationQueue_->refuseProtectedDeleteTarget(*channel, entry.target); - !allowed) - return reply.error(fmt::format("Refusing to delete '{}'", entry.target.value())); - } - - const auto enqueued = self->operationQueue_->addBulkDeleteOperation( - *channel, request, Ids::makeOperationId(bulkOperationIdString) - ); - - Log::info("addBulkDelete: queued {} entries", enqueued); - self->resetQueueThrottle(); - reply({{"success", true}, {"enqueued", enqueued}}); - }, - std::move(reply) + auto guard = queue.remoteDeleteGuard(channel); + if (const auto refusal = + refuseAnyProtected(guard, request.entries, &SharedData::BulkDeleteEntry::target); + refusal) + return std::unexpected(refusal->toLocalizedMessage()); + return enqueuedReply(queue.addBulkDeleteOperation(guard, channel, request, operationId)); + } ); } ); diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 5c11260f..64515623 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1565,6 +1565,8 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( SharedData::BulkDeleteRequest remoteDeletes{.mode = request.mode, .listedContentOnly = true}; std::vector localDeletes; const auto localHomes = localHomeDirectories(); + auto remoteGuard = remoteDeleteGuard(sftp); + remoteGuard.trustScannedBelow(session.remoteRoot()); for (auto& [entry, paths] : plan.entries) { @@ -1611,7 +1613,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( } case Action::DeleteRemote: { - if (!refuseProtectedDeleteTarget(sftp, paths.remote).has_value()) + if (isBelowAny(refusedRemoteDirectories, entry.relKey)) { result.failed.push_back(std::move(entry.relKey)); break; @@ -1661,7 +1663,8 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( request.downloadId ); } - if (!remoteDeletes.entries.empty() && addBulkDeleteOperation(sftp, remoteDeletes, request.remoteDeleteId) == 0) + if (!remoteDeletes.entries.empty() && + addBulkDeleteOperation(remoteGuard, sftp, remoteDeletes, request.remoteDeleteId) == 0) { std::ranges::move(result.remoteDeletes, std::back_inserter(result.failed)); result.remoteDeletes.clear(); @@ -1744,35 +1747,26 @@ std::expected OperationQueue::addLocalDeleteOperation( return {}; } -std::expected -OperationQueue::refuseProtectedDeleteTarget(SecureShell::SftpSession& sftp, Paths::RemotePath const& target) +RemoteDeleteGuard OperationQueue::remoteDeleteGuard(SecureShell::SftpSession& sftp) { - const auto refuse = [&target](std::string_view why) - { - Log::error("Refusing to delete '{}': {}", target.value(), why); - return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = target.value()}); - }; - if (target.empty()) - return refuse("no path"); - if (target.isRoot()) - return refuse("it is the root directory"); + return RemoteDeleteGuard{sftp, remoteHome_, sftpOpts_.operationTimeout.value_or(defaultFutureTimeout)}; +} - if (!remoteHome_) - { - auto homeFuture = sftp.homeDirectory(); - if (homeFuture.wait_for(sftpOpts_.operationTimeout.value_or(defaultFutureTimeout)) != std::future_status::ready) - return refuse("the home directory could not be determined in time"); - auto home = homeFuture.get(); - if (!home) - return refuse("the home directory could not be determined"); - remoteHome_ = *std::move(home); - } - if (remoteHome_->isWithin(target)) - return refuse(fmt::format("it contains the home directory '{}'", remoteHome_->value())); - return {}; +std::expected OperationQueue::addDeleteOperation( + SecureShell::SftpSession& sftp, + Ids::OperationId operationId, + Paths::RemotePath const& remotePath, + bool recursive, + bool insertRefresh, + SharedData::OperationMode mode +) +{ + auto guard = remoteDeleteGuard(sftp); + return addDeleteOperation(guard, sftp, std::move(operationId), remotePath, recursive, insertRefresh, mode); } std::expected OperationQueue::addDeleteOperation( + RemoteDeleteGuard& guard, SecureShell::SftpSession& sftp, Ids::OperationId operationId, Paths::RemotePath const& remotePath, @@ -1784,8 +1778,13 @@ std::expected OperationQueue::addDeleteOperation( // Assumed in strand auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - if (auto allowed = refuseProtectedDeleteTarget(sftp, remotePath); !allowed) - return allowed; + if (auto allowed = guard.check(remotePath); !allowed) + { + Log::error("Refusing to delete: {}", allowed.error().toString()); + return std::unexpected( + Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = allowed.error().toString()} + ); + } // lstat: a symlink to a directory is removed as a link, recursing would delete the target's content. auto statFut = sftp.lstat(remotePath); @@ -1861,6 +1860,17 @@ std::size_t OperationQueue::addBulkDeleteOperation( SharedData::BulkDeleteRequest const& request, Ids::OperationId bulkOperationId ) +{ + auto guard = remoteDeleteGuard(sftp); + return addBulkDeleteOperation(guard, sftp, request, std::move(bulkOperationId)); +} + +std::size_t OperationQueue::addBulkDeleteOperation( + RemoteDeleteGuard& guard, + SecureShell::SftpSession& sftp, + SharedData::BulkDeleteRequest const& request, + Ids::OperationId bulkOperationId +) { // Assumed in strand. One DeleteOperation, so one card and one completion for the whole request: files are // deleted as given, every real directory is scanned first and deleted with its content. @@ -1897,8 +1907,11 @@ std::size_t OperationQueue::addBulkDeleteOperation( std::size_t scannedDirectories = 0; for (auto const& entry : request.entries) { - if (!refuseProtectedDeleteTarget(sftp, entry.target).has_value()) + if (auto allowed = guard.check(entry.target); !allowed) + { + Log::error("Bulk delete: refusing to delete: {}", allowed.error().toString()); continue; + } if (entry.isDirectory && !request.listedContentOnly) { // lstat: a symlink to a directory is removed as a link, recursing would delete the target's content. diff --git a/backend/source/backend/sftp/remote_delete_guard.cpp b/backend/source/backend/sftp/remote_delete_guard.cpp new file mode 100644 index 00000000..a9ab9245 --- /dev/null +++ b/backend/source/backend/sftp/remote_delete_guard.cpp @@ -0,0 +1,267 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace +{ + char foldedCase(char character) + { + return character >= 'A' && character <= 'Z' ? static_cast(character - 'A' + 'a') : character; + } + + bool hasWindowsAmbiguousName(Paths::RemotePath const& path) + { + return std::ranges::any_of( + Paths::Detail::splitAtSlashes(path.view()), + [](std::string_view name) + { + return name.find('\\') != std::string_view::npos || + (!name.empty() && (name.back() == '.' || name.back() == ' ')); + } + ); + } + + bool containsHome(Paths::RemotePath const& target, Paths::RemotePath const& home) + { + if (home.isWithin(target)) + return true; + const auto foldsCase = + RemoteDeleteGuardDetail::looksCaseFolding(home) || RemoteDeleteGuardDetail::looksCaseFolding(target); + return foldsCase && RemoteDeleteGuardDetail::isWithinIgnoringCase(home, target); + } +} + +std::string RemoteDeleteRefusal::toString() const +{ + switch (reason) + { + case Reason::NoPath: + return "no path"; + case Reason::Root: + return fmt::format("'{}' is the root directory", target); + case Reason::ContainsHome: + return fmt::format("'{}' is or contains the home directory '{}'", target, home); + case Reason::WindowsName: + return fmt::format("'{}' could name another item on a server with Windows paths", target); + case Reason::NotVerified: + return fmt::format("'{}' could not be compared with the home directory in time", target); + } + return fmt::format("'{}' is protected", target); +} + +std::string RemoteDeleteRefusal::toLocalizedMessage() const +{ + switch (reason) + { + case Reason::NoPath: + return Utility::localizedMessage("backend.deleteGuard.noPath"); + case Reason::Root: + return Utility::localizedMessage("backend.deleteGuard.root", target); + case Reason::ContainsHome: + return Utility::localizedMessage("backend.deleteGuard.containsHome", target, home); + case Reason::WindowsName: + return Utility::localizedMessage("backend.deleteGuard.windowsName", target); + case Reason::NotVerified: + return Utility::localizedMessage("backend.deleteGuard.notVerified", target); + } + return Utility::localizedMessage("backend.deleteGuard.notVerified", target); +} + +namespace RemoteDeleteGuardDetail +{ + bool looksLikeWindowsPath(Paths::RemotePath const& path) + { + const auto text = path.view(); + const auto isLetter = [](char character) + { + return (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'); + }; + return text.size() >= 3 && text[0] == '/' && isLetter(text[1]) && text[2] == ':' && + (text.size() == 3 || text[3] == '/'); + } + + bool looksCaseFolding(Paths::RemotePath const& path) + { + const auto text = path.view(); + return looksLikeWindowsPath(path) || text == "/Users" || text.starts_with("/Users/"); + } + + bool isWithinIgnoringCase(Paths::RemotePath const& path, Paths::RemotePath const& base) + { + if (path.empty() || base.empty()) + return false; + if (base.isRoot()) + return true; + const auto text = path.view(); + const auto prefix = base.view(); + return text.size() >= prefix.size() && + std::ranges::equal(text.substr(0, prefix.size()), prefix, {}, foldedCase, foldedCase) && + (text.size() == prefix.size() || text[prefix.size()] == '/'); + } + + std::optional + refusalByText(Paths::RemotePath const& target, std::optional const& home) + { + if (target.empty()) + return RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::NoPath}; + if (target.isRoot()) + return RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::Root, .target = target.value()}; + const auto windowsServer = looksLikeWindowsPath(target) || (home && looksLikeWindowsPath(*home)); + if (windowsServer && hasWindowsAmbiguousName(target)) + return RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::WindowsName, .target = target.value()}; + if (home && containsHome(target, *home)) + { + return RemoteDeleteRefusal{ + .reason = RemoteDeleteRefusal::Reason::ContainsHome, + .target = target.value(), + .home = home->value(), + }; + } + return std::nullopt; + } +} + +RemoteDeleteGuard::RemoteDeleteGuard( + SecureShell::SftpSession& sftp, + std::optional& knownHome, + std::chrono::milliseconds timeout +) + : sftp_{&sftp} + , knownHome_{&knownHome} + , timeout_{timeout} +{} + +void RemoteDeleteGuard::trustScannedBelow(Paths::RemotePath root) +{ + scannedRoots_.push_back(std::move(root)); +} + +std::expected RemoteDeleteGuard::check(Paths::RemotePath const& target) +{ + if (const auto refusal = RemoteDeleteGuardDetail::refusalByText(target, std::nullopt); refusal) + return std::unexpected(*refusal); + + const auto refuseFor = [&target](std::optional const& home) + -> std::expected + { + if (const auto refusal = RemoteDeleteGuardDetail::refusalByText(target, home); refusal) + return std::unexpected(*refusal); + return {}; + }; + + return home(target).and_then( + [&](std::optional const& home) -> std::expected + { + if (auto refused = refuseFor(home); !refused || !home) + return refused; + return canonicalDirectory(*target.parent(), target) + .and_then( + [&](std::optional const& directory) -> std::expected + { + if (!directory || !containsHome(*directory / *target.filename(), *home)) + return {}; + return std::unexpected( + RemoteDeleteRefusal{ + .reason = RemoteDeleteRefusal::Reason::ContainsHome, + .target = target.value(), + .home = home->value(), + } + ); + } + ); + } + ); +} + +std::optional RemoteDeleteGuard::homeDirectory() +{ + return home(Paths::RemotePath{}).value_or(std::nullopt); +} + +std::expected, RemoteDeleteRefusal> +RemoteDeleteGuard::home(Paths::RemotePath const& target) +{ + if (*knownHome_ || homeUnknown_) + return *knownHome_; + + auto future = sftp_->performPromise( + [sftp = sftp_]() + { + return sftp->canonicalHomeDirectoryInStrand(); + } + ); + if (future.wait_for(timeout_) != std::future_status::ready) + return std::unexpected( + RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::NotVerified, .target = target.value()} + ); + auto home = future.get(); + if (!home) + { + Log::warn( + "The remote home directory is unknown ({}), remote deletes only refuse the root directory.", + home.error().toString() + ); + homeUnknown_ = true; + return std::nullopt; + } + *knownHome_ = *std::move(home); + return *knownHome_; +} + +std::expected, RemoteDeleteRefusal> +RemoteDeleteGuard::canonicalDirectory(Paths::RemotePath const& directory, Paths::RemotePath const& target) +{ + const auto scannedRoot = std::ranges::find_if( + scannedRoots_, + [&directory](Paths::RemotePath const& root) + { + return directory.isWithin(root) && directory != root; + } + ); + if (scannedRoot == scannedRoots_.end()) + return lookUpCanonical(directory, target); + return lookUpCanonical(*scannedRoot, target) + .transform( + [&directory, &root = *scannedRoot](std::optional const& canonicalRoot) + -> std::optional + { + if (!canonicalRoot) + return std::nullopt; + return *canonicalRoot / *directory.relativeTo(root); + } + ); +} + +std::expected, RemoteDeleteRefusal> +RemoteDeleteGuard::lookUpCanonical(Paths::RemotePath const& directory, Paths::RemotePath const& target) +{ + if (const auto cached = canonicalDirectories_.find(directory.value()); cached != canonicalDirectories_.end()) + return cached->second; + + auto future = sftp_->performPromise( + [sftp = sftp_, directory]() + { + return sftp->canonicalPathInStrand(directory); + } + ); + if (future.wait_for(timeout_) != std::future_status::ready) + return std::unexpected( + RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::NotVerified, .target = target.value()} + ); + auto canonical = future.get(); + if (!canonical) + { + // A directory that does not exist holds nothing to delete, the text checks still apply. + Log::debug("Delete guard: cannot canonicalize '{}': {}", directory.value(), canonical.error().toString()); + return canonicalDirectories_.emplace(directory.value(), std::nullopt).first->second; + } + return canonicalDirectories_.emplace(directory.value(), *std::move(canonical)).first->second; +} diff --git a/frontend/source/frontend/file_explorer/remote_side_model.cpp b/frontend/source/frontend/file_explorer/remote_side_model.cpp index 91a53bd9..d512bbaf 100644 --- a/frontend/source/frontend/file_explorer/remote_side_model.cpp +++ b/frontend/source/frontend/file_explorer/remote_side_model.cpp @@ -542,7 +542,8 @@ void RemoteSideModel::enqueueDeletes( .styleVariant = ScriptNuiComponents::StyleVariant::Danger, .headerText = language->get("remoteSideModel", "deleteFailed"), .text = fmt::format( - fmt::runtime(language->get("remoteSideModel", "failedToCreateDeleteOperation")), info + fmt::runtime(language->get("remoteSideModel", "failedToCreateDeleteOperation")), + language->translate(info) ), .buttons = ConfirmDialog::Button::Ok, }); diff --git a/static/assets/languages/de_DE.yaml b/static/assets/languages/de_DE.yaml index 82c752cf..fa6e84cc 100644 --- a/static/assets/languages/de_DE.yaml +++ b/static/assets/languages/de_DE.yaml @@ -1151,6 +1151,12 @@ terminalPanel: saveDialogTitle: "Terminalinhalt speichern" saveToFile: "Terminal in Datei speichern" backend: + deleteGuard: + noPath: "Es wurde nichts gelöscht. Ein Pfad fehlt." + root: "Es wurde nichts gelöscht. '{}' ist das Wurzelverzeichnis." + containsHome: "Es wurde nichts gelöscht. '{}' ist oder enthält das Heimatverzeichnis '{}'." + windowsName: "Es wurde nichts gelöscht. Der Server verwendet Windows-Pfade, unter denen '{}' ein anderes Element bezeichnen könnte." + notVerified: "Es wurde nichts gelöscht, weil der Server nicht rechtzeitig beantwortet hat, ob '{}' das Heimatverzeichnis enthält." rename: targetExists: "Ein Eintrag mit diesem Namen existiert bereits." invalidName: "Der neue Name muss ein einfacher Name im selben Verzeichnis sein." diff --git a/static/assets/languages/en_US.yaml b/static/assets/languages/en_US.yaml index bfa5ed6e..7929001d 100644 --- a/static/assets/languages/en_US.yaml +++ b/static/assets/languages/en_US.yaml @@ -1153,6 +1153,12 @@ terminalPanel: saveDialogTitle: "Save terminal contents" saveToFile: "Save terminal to file" backend: + deleteGuard: + noPath: "Nothing was deleted. A path is missing." + root: "Nothing was deleted. '{}' is the root directory." + containsHome: "Nothing was deleted. '{}' is or contains the home directory '{}'." + windowsName: "Nothing was deleted. The server uses Windows paths, where '{}' could name a different item." + notVerified: "Nothing was deleted, because the server did not answer in time whether '{}' contains the home directory." rename: targetExists: "An item with this name already exists." invalidName: "The new name must be a plain name in the same directory." From 62f8949ca71bb940c2442dd309485167688de0c1 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:19:50 +0200 Subject: [PATCH 021/131] A sync delete of a protected directory refuses everything below it A sync delete of a protected directory also refuses every planned entry below it, remotely and locally, instead of deleting its content, and the sync dialog paints only the entries that failed; the local delete operation leaves the guard check to the queue --- .../backend/sftp/local_delete_operation.cpp | 8 ---- .../source/backend/sftp/operation_queue.cpp | 38 ++++++++++++++++++- .../frontend/sync_dialog/sync_dialog.cpp | 8 +++- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/backend/source/backend/sftp/local_delete_operation.cpp b/backend/source/backend/sftp/local_delete_operation.cpp index 7362f48a..bbf52ed4 100644 --- a/backend/source/backend/sftp/local_delete_operation.cpp +++ b/backend/source/backend/sftp/local_delete_operation.cpp @@ -19,14 +19,6 @@ std::expected Loc case Preparing: case Prepared: { - for (auto const& target : options_.targets) - { - if (target.empty() || target.isRoot()) - { - Log::error("LocalDeleteOperation: Refusing to delete '{}'.", target.toUtf8()); - return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = target.toUtf8()}); - } - } state_ = Running; [[fallthrough]]; } diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 64515623..3beaadf8 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1568,6 +1568,33 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( auto remoteGuard = remoteDeleteGuard(sftp); remoteGuard.trustScannedBelow(session.remoteRoot()); + // The plan lists a directory before its content, so a refused directory refuses every entry below it. + std::vector refusedRemoteDirectories; + std::vector refusedLocalDirectories; + const auto isBelowAny = [](std::vector const& directories, std::string const& relKey) + { + return std::ranges::any_of( + directories, + [&relKey](std::string const& directory) + { + return relKey.size() > directory.size() && relKey.starts_with(directory) && + relKey[directory.size()] == '/'; + } + ); + }; + const auto refuse = [&result]( + SharedData::Sync::EnqueuePlanEntry& entry, + std::vector& refusedDirectories, + std::string_view side, + std::string const& reason + ) + { + Log::error("Sync: refusing to delete {} '{}' and what is below it: {}", side, entry.relKey, reason); + if (entry.isDirectory) + refusedDirectories.push_back(entry.relKey); + result.failed.push_back(std::move(entry.relKey)); + }; + for (auto& [entry, paths] : plan.entries) { switch (entry.action) @@ -1618,6 +1645,11 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( result.failed.push_back(std::move(entry.relKey)); break; } + if (auto allowed = remoteGuard.check(paths.remote); !allowed) + { + refuse(entry, refusedRemoteDirectories, "remotely", allowed.error().toString()); + break; + } result.remoteDeleteSources.push_back(paths.remote.value()); remoteDeletes.entries.push_back({.target = std::move(paths.remote), .isDirectory = entry.isDirectory}); result.remoteDeletes.push_back(std::move(entry.relKey)); @@ -1627,10 +1659,14 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( { if (isBelowAny(refusedLocalDirectories, entry.relKey)) { - Log::error("Sync: refusing to delete locally: {}", allowed.error()); result.failed.push_back(std::move(entry.relKey)); break; } + if (auto allowed = refuseProtectedLocalDelete(paths.local, localHomes); !allowed) + { + refuse(entry, refusedLocalDirectories, "locally", allowed.error()); + break; + } result.localDeleteSources.push_back(paths.local.toUtf8()); localDeletes.push_back(std::move(paths.local)); result.localDeletes.push_back(std::move(entry.relKey)); diff --git a/frontend/source/frontend/sync_dialog/sync_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_dialog.cpp index 5ea8c275..3237ac1c 100644 --- a/frontend/source/frontend/sync_dialog/sync_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_dialog.cpp @@ -1145,6 +1145,12 @@ struct SyncDialog::Implementation *observer = value; } }; + // The backend lists every failed entry, the rows below a failed directory are not all failed. + const auto setRows = [this, section](std::vector const& rows, double value) + { + for (auto const& row : rows) + *progressFor(section, row) = value; + }; mapBulk(*uploads, result.uploads, result.uploadSources); mapBulk(*downloads, result.downloads, result.downloadSources); @@ -1166,7 +1172,7 @@ struct SyncDialog::Implementation part->map(); } setCovering(result.createdDirectories, 1.1); - setCovering(result.failed, -1.0); + setRows(result.failed, -1.0); Nui::globalEventContext.executeActiveEventsImmediately(); }, [this, request](std::string const& message) From 14269145a90c1503e1f812b79df1a91333ea7a0b Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:21:56 +0200 Subject: [PATCH 022/131] SftpSession checks strand and paths with one requireSetInStrand SftpSession checks the strand and unset paths of an in-strand call with one requireSetInStrand for all its paths, and a listing takes symlink names from its entries instead of a second list --- ssh/include/ssh/sftp_session.hpp | 15 +++++- ssh/source/ssh/sftp_session.cpp | 79 +++++++++++--------------------- 2 files changed, 40 insertions(+), 54 deletions(-) diff --git a/ssh/include/ssh/sftp_session.hpp b/ssh/include/ssh/sftp_session.hpp index 3f4e846e..7f02fd26 100644 --- a/ssh/include/ssh/sftp_session.hpp +++ b/ssh/include/ssh/sftp_session.hpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -442,9 +443,19 @@ namespace SecureShell private: /** - * @brief Refuses an unset path before it reaches the server. + * @brief Asserts the call runs in the processing thread and refuses an unset path before it reaches the + * server, where an empty path would mean the start directory. */ - static std::expected requireSet(Paths::RemotePath const& path); + template + std::expected requireSetInStrand(PathTs const&... paths) const + { + assert(strand_->withinProcessingThread()); + if ((paths.empty() || ...)) + return std::unexpected(unsetPathError()); + return {}; + } + + static SftpError unsetPathError(); /** * @brief The absolute path the server canonicalizes @p path to. diff --git a/ssh/source/ssh/sftp_session.cpp b/ssh/source/ssh/sftp_session.cpp index cbe5fd76..f6d4505c 100644 --- a/ssh/source/ssh/sftp_session.cpp +++ b/ssh/source/ssh/sftp_session.cpp @@ -72,13 +72,11 @@ namespace SecureShell std::expected, SftpSession::Error> SftpSession::listDirectoryInStrand(Paths::RemotePath const& path) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); int closeResult = 0; std::vector entries{}; - std::vector names{}; { std::unique_ptr> dir{ @@ -104,7 +102,6 @@ namespace SecureShell for (; entry != nullptr; entry.reset(sftp_readdir(session_, dir.get()))) { entries.push_back(fromSftpAttributes(entry.get())); - names.emplace_back(entry->name != nullptr ? entry->name : ""); } } @@ -118,13 +115,12 @@ namespace SecureShell return std::unexpected(lastError()); } - for (std::size_t index = 0; index < entries.size(); ++index) + for (auto& entry : entries) { - auto& entry = entries[index]; if (entry.type == SharedData::FileType::Symlink) { // A name that is no valid segment, like "x/..", must not become a path to look up. - const auto name = Paths::PathSegment::make(names[index]); + const auto name = entry.nameSegment(); if (!name) continue; const auto fullPath = path / *name; @@ -164,8 +160,7 @@ namespace SecureShell std::expected SftpSession::createDirectoryInStrand(Paths::RemotePath const& path, std::filesystem::perms permissions) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_mkdir( session_, path.c_str(), static_cast(permissions & std::filesystem::perms::mask) @@ -191,8 +186,7 @@ namespace SecureShell std::filesystem::perms permissions ) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); std::unique_ptr attributes{ sftp_lstat(session_, path.c_str()), sftp_attributes_free @@ -235,8 +229,7 @@ namespace SecureShell std::expected SftpSession::createFileInStrand(Paths::RemotePath const& path, std::filesystem::perms permissions) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); std::unique_ptr> file{ // O_EXCL: an existing file is an error, not reported as created. @@ -274,8 +267,7 @@ namespace SecureShell std::expected SftpSession::removeFileInStrand(Paths::RemotePath const& path) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_unlink(session_, path.c_str()); if (result != SSH_OK) @@ -295,8 +287,7 @@ namespace SecureShell std::expected SftpSession::removeDirectoryInStrand(Paths::RemotePath const& path) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_rmdir(session_, path.c_str()); if (result != SSH_OK) @@ -324,7 +315,7 @@ namespace SecureShell std::vector nonEmpties; for (const auto& path : directories) { - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); std::unique_ptr attributes{ sftp_lstat(session_, path.c_str()), sftp_attributes_free @@ -395,7 +386,7 @@ namespace SecureShell { for (const auto& path : paths) { - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); std::unique_ptr attributes{ sftp_lstat(session_, path.c_str()), sftp_attributes_free @@ -424,8 +415,7 @@ namespace SecureShell std::expected SftpSession::statInStrand(Paths::RemotePath const& path) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); const auto callStart = std::chrono::steady_clock::now(); std::unique_ptr attributes{ @@ -450,8 +440,7 @@ namespace SecureShell std::expected SftpSession::lstatInStrand(Paths::RemotePath const& path) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); const auto callStart = std::chrono::steady_clock::now(); std::unique_ptr attributes{ @@ -478,8 +467,7 @@ namespace SecureShell std::expected SftpSession::statInStrand(Paths::RemotePath const& path, sftp_attributes attributes) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_setstat(session_, path.c_str(), attributes); if (result != SSH_OK) @@ -501,10 +489,7 @@ namespace SecureShell std::expected SftpSession::renameInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(source); !valid) - return std::unexpected(valid.error()); - if (auto valid = requireSet(destination); !valid) + if (auto valid = requireSetInStrand(source, destination); !valid) return std::unexpected(valid.error()); auto result = sftp_rename(session_, source.c_str(), destination.c_str()); if (result != SSH_OK) @@ -703,8 +688,7 @@ namespace SecureShell std::expected SftpSession::chownInStrand(Paths::RemotePath const& path, uid_t owner, gid_t group) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_chown(session_, path.c_str(), owner, group); if (result != SSH_OK) @@ -726,8 +710,7 @@ namespace SecureShell std::expected SftpSession::chmodInStrand(Paths::RemotePath const& path, std::filesystem::perms perms) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_chmod(session_, path.c_str(), static_cast(perms)); if (result != SSH_OK) @@ -814,8 +797,7 @@ namespace SecureShell std::expected SftpSession::readSmallFileInStrand(Paths::RemotePath const& path, std::size_t maximumSize) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); const std::unique_ptr file{ sftp_open(session_, path.c_str(), O_RDONLY, 0), sftp_close @@ -842,7 +824,7 @@ namespace SecureShell SftpSession::userDirectoriesInStrand(Paths::RemotePath const& home) { constexpr std::size_t maximumFileSize = 64u * 1024u; - if (auto valid = requireSet(home); !valid) + if (auto valid = requireSetInStrand(home); !valid) return std::unexpected(valid.error()); return readSmallFileInStrand(home / *Paths::RelativePath::make(".config/user-dirs.dirs"), maximumFileSize) .transform( @@ -872,17 +854,13 @@ namespace SecureShell ); } - std::expected SftpSession::requireSet(Paths::RemotePath const& path) + SftpError SftpSession::unsetPathError() { - if (!path.empty()) - return {}; - return std::unexpected( - Error{ - .message = "The remote path is not set", - .sftpError = SSH_FX_FAILURE, - .wrapperError = WrapperErrors::InvalidPath, - } - ); + return Error{ + .message = "The remote path is not set", + .sftpError = SSH_FX_FAILURE, + .wrapperError = WrapperErrors::InvalidPath, + }; } SftpError SftpSession::lastError() const @@ -920,8 +898,7 @@ namespace SecureShell std::filesystem::perms permissions ) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); const auto callStart = std::chrono::steady_clock::now(); std::unique_ptr> file{ @@ -968,8 +945,7 @@ namespace SecureShell std::expected SftpSession::readLinkDeepInStrand(Paths::RemotePath const& path, int maxDepth) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto currentPath = path; std::optional linkContent{}; @@ -1049,8 +1025,7 @@ namespace SecureShell std::expected SftpSession::createSymLinkInStrand(std::string const& target, Paths::RemotePath const& linkPath) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(linkPath); !valid) + if (auto valid = requireSetInStrand(linkPath); !valid) return std::unexpected(valid.error()); const auto result = sftp_symlink(session_, target.c_str(), linkPath.c_str()); if (result != SSH_OK) From a15329f08e359c696766be048df22df1ce93d968 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:24:17 +0200 Subject: [PATCH 023/131] The frontend asks for the local path platform before loading the state The frontend asks the backend for its local path platform before it loads the state, so local paths read at startup follow the backend's rules instead of POSIX ones --- frontend/source/frontend/main.cpp | 149 +++++++++++++++---------- frontend/source/frontend/main_page.cpp | 8 -- 2 files changed, 88 insertions(+), 69 deletions(-) diff --git a/frontend/source/frontend/main.cpp b/frontend/source/frontend/main.cpp index 375cda59..993a3c45 100644 --- a/frontend/source/frontend/main.cpp +++ b/frontend/source/frontend/main.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -38,6 +39,92 @@ namespace } } +void loadState() +{ + persistence->load( + [](std::optional const&, Persistence::StateHolder&, Persistence::LoadWarnings const&) + { + Log::info("State loaded, setting up frontend."); + frontendEvents = std::make_unique(); + frontendEvents->onLanguageChanged.value() = persistence->stateCache().localizationOptions.languageCode; + frontendEvents->selectedTheme.value() = persistence->stateCache().uiOptions.theme; + frontendEvents->darkLightMode.value() = persistence->stateCache().uiOptions.darkLightMode; + + themeController = std::make_unique(*frontendEvents); + + persistence->loadLanguageFiles( + [](std::optional lang) + { + Log::info("Language file loaded, continuing setup."); + try + { + language = std::make_unique( + frontendEvents.get(), + lang.value_or( + []() + { + auto res = nlohmann::json::object(); + res["en_US"] = nlohmann::json::object(); + return res; + }() + ) + ); + + Log::info("Language file loaded."); + Log::info("Creating main page."); + mainPage = + std::make_unique(persistence.get(), frontendEvents.get(), *themeController); + + Log::info("Creating DOM."); + dom = std::make_unique(); + + Log::info("Rendering main page into DOM."); + dom->setBody(Nui::Elements::body{}(mainPage->render())); + + Log::info("Calling setup completion function"); + mainPage->onSetupComplete(); + + Log::debug("Dumping nui_rpc.backend and nui_rpc.frontend for debugging:"); + auto nuiRpc = Nui::val::global("nui_rpc"); + if (!nuiRpc.isUndefined() && nuiRpc.hasOwnProperty("backend")) + { + Log::debug("nui_rpc.backend:"); + printKeys(nuiRpc["backend"]); + Log::debug("nui_rpc.frontend:"); + printKeys(nuiRpc["frontend"]); + } + } + catch (std::exception const& exc) + { + Log::error("Failed to continue setup after loading language file: {}", exc.what()); + } + } + ); + } + ); +} + +/** + * @brief Takes the platform of the backend for local paths, then runs @p then. Local paths follow the rules of the + * machine the backend runs on, not WebAssembly's, so nothing may read one before. + */ +void withLocalPlatform(void (*then)()) +{ + Nui::RpcClient::callWithBackChannel( + "Main::getVersion", + [then](Nui::val response) + { + if (response.hasOwnProperty("localPlatform")) + { + Paths::defaultLocalPlatform() = response["localPlatform"].as() == "windows" + ? Paths::LocalPlatform::Windows + : Paths::LocalPlatform::Posix; + } + then(); + } + ); +} + bool tryLoad(std::shared_ptr const& setupWait) { static int counter = 0; @@ -58,67 +145,7 @@ bool tryLoad(std::shared_ptr const& setupWait) Log::info("terminalUtility available."); persistence = std::make_unique(); - persistence->load( - [](std::optional const&, Persistence::StateHolder&, Persistence::LoadWarnings const&) - { - Log::info("State loaded, setting up frontend."); - frontendEvents = std::make_unique(); - frontendEvents->onLanguageChanged.value() = persistence->stateCache().localizationOptions.languageCode; - frontendEvents->selectedTheme.value() = persistence->stateCache().uiOptions.theme; - frontendEvents->darkLightMode.value() = persistence->stateCache().uiOptions.darkLightMode; - - themeController = std::make_unique(*frontendEvents); - - persistence->loadLanguageFiles( - [](std::optional lang) - { - Log::info("Language file loaded, continuing setup."); - try - { - language = std::make_unique( - frontendEvents.get(), - lang.value_or( - []() - { - auto res = nlohmann::json::object(); - res["en_US"] = nlohmann::json::object(); - return res; - }() - ) - ); - - Log::info("Language file loaded."); - Log::info("Creating main page."); - mainPage = - std::make_unique(persistence.get(), frontendEvents.get(), *themeController); - - Log::info("Creating DOM."); - dom = std::make_unique(); - - Log::info("Rendering main page into DOM."); - dom->setBody(Nui::Elements::body{}(mainPage->render())); - - Log::info("Calling setup completion function"); - mainPage->onSetupComplete(); - - Log::debug("Dumping nui_rpc.backend and nui_rpc.frontend for debugging:"); - auto nuiRpc = Nui::val::global("nui_rpc"); - if (!nuiRpc.isUndefined() && nuiRpc.hasOwnProperty("backend")) - { - Log::debug("nui_rpc.backend:"); - printKeys(nuiRpc["backend"]); - Log::debug("nui_rpc.frontend:"); - printKeys(nuiRpc["frontend"]); - } - } - catch (std::exception const& exc) - { - Log::error("Failed to continue setup after loading language file: {}", exc.what()); - } - } - ); - } - ); + withLocalPlatform(&loadState); } else Log::info("Waiting for terminalUtility to be available."); diff --git a/frontend/source/frontend/main_page.cpp b/frontend/source/frontend/main_page.cpp index 59c05814..6e607615 100644 --- a/frontend/source/frontend/main_page.cpp +++ b/frontend/source/frontend/main_page.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -133,13 +132,6 @@ void MainPage::onSetupComplete() "Main::getVersion", [this](Nui::val response) { - // Local paths are parsed by the rules of the machine the backend runs on, not WebAssembly's. - if (response.hasOwnProperty("localPlatform")) - { - Paths::defaultLocalPlatform() = response["localPlatform"].as() == "windows" - ? Paths::LocalPlatform::Windows - : Paths::LocalPlatform::Posix; - } impl_->events->appVersion = AppVersion{ .version = response["version"].as(), .gitTag = response["gitTag"].as(), From c60369a9c9121093d1476cf86a9a9be221cddba7 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:25:05 +0200 Subject: [PATCH 024/131] Document one delete operation per bulk delete request The docs of BulkDeleteRequest and DeleteOperation::appendScanResult describe one delete operation per request and how a scan result is checked --- backend/include/backend/sftp/delete_operation.hpp | 6 ++++-- .../shared_data/file_operations/bulk_delete_request.hpp | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/include/backend/sftp/delete_operation.hpp b/backend/include/backend/sftp/delete_operation.hpp index 8c907408..74d10cf3 100644 --- a/backend/include/backend/sftp/delete_operation.hpp +++ b/backend/include/backend/sftp/delete_operation.hpp @@ -48,7 +48,9 @@ class DeleteOperation : public Operation void setScanResult(std::vector&& entries, std::uint64_t /* totalBytes */); /** - * @brief Adds the scanned content of @p root, which has to lie below it, otherwise nothing is deleted. + * @brief Adds the entries of a scan of @p root as targets. The first entry is @p root itself, every other one + * lies below it through its parents. A second root, an invalid name or the root directory refuses the + * scan, then nothing is deleted. */ void appendScanResult(Paths::RemotePath const& root, std::vector const& entries); @@ -120,7 +122,7 @@ class DeleteOperation : public Operation std::size_t totalToDelete_{0}; /** @brief Deleted from the back, so a scanned directory goes after its content. */ std::vector targets_{}; - /** @brief A scanned entry was not a valid path within remotePath_, nothing may be deleted. */ + /** @brief Why a scan result was refused, nothing may be deleted then. */ std::optional invalidScanEntry_{}; std::vector> failedEntries_{}; std::unique_ptr> scanPhase_{}; diff --git a/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp b/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp index f5ec99ca..afdb367f 100644 --- a/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp +++ b/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp @@ -25,8 +25,8 @@ namespace SharedData BOOST_DESCRIBE_STRUCT(BulkDeleteEntry, (), (target, isDirectory, sizeBytes)) /** - * @brief Single-RPC payload for deleting many remote items. Files become one aggregate delete, every directory - * becomes a delete of its own that scans first. + * @brief Single-RPC payload for deleting many remote items as one delete operation: files are deleted as listed, + * every directory is scanned first and deleted with its content. */ struct BulkDeleteRequest { From c1c0d5bf7480a8b974fc520a24d14f0018bea02d Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:34:46 +0200 Subject: [PATCH 025/131] paths/utf8.hpp is the one UTF-8 module paths/utf8.hpp is the one UTF-8 module: besides validation it sanitizes (U+FFFD for every byte outside a valid sequence), truncates at a code point boundary and converts to and from UTF-16 (wide text on Windows) without throwing; Utility::Utf8 decodes and encodes through it, and the snippet error excerpt and the typed line backspace cut with truncatedUtf8 --- .../command_snippets_panel.cpp | 11 +- utility/include/utility/utf8.hpp | 4 +- utility/source/utility/typed_line_buffer.cpp | 7 +- utility/source/utility/utf8.cpp | 107 ++---------------- 4 files changed, 18 insertions(+), 111 deletions(-) diff --git a/frontend/source/frontend/session_components/command_snippets_panel.cpp b/frontend/source/frontend/session_components/command_snippets_panel.cpp index 3ffa6737..43a7be5f 100644 --- a/frontend/source/frontend/session_components/command_snippets_panel.cpp +++ b/frontend/source/frontend/session_components/command_snippets_panel.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -45,6 +46,7 @@ #include #include #include +#include #include #include @@ -474,14 +476,11 @@ struct CommandSnippetsPanel::Implementation if (!unmatched) return {}; constexpr std::size_t excerptLength = 24u; - auto excerptEnd = std::min(command.size(), *unmatched + excerptLength); - // Never cut a UTF-8 sequence in half. - while (excerptEnd < command.size() && (static_cast(command[excerptEnd]) & 0xC0u) == 0x80u) - --excerptEnd; - const auto excerpt = command.substr(*unmatched, excerptEnd - *unmatched); + const auto rest = std::string_view{command}.substr(*unmatched); + const auto excerpt = Paths::truncatedUtf8(rest, excerptLength); return fmt::format( fmt::runtime(language->get("commandSnippetsPanel", "unmatchedBraces")), - excerptEnd < command.size() ? fmt::format("{}...", excerpt) : excerpt + excerpt.size() < rest.size() ? fmt::format("{}...", excerpt) : std::string{excerpt} ); } diff --git a/utility/include/utility/utf8.hpp b/utility/include/utility/utf8.hpp index 698c35dd..43279ddc 100644 --- a/utility/include/utility/utf8.hpp +++ b/utility/include/utility/utf8.hpp @@ -6,8 +6,8 @@ namespace Utility::Utf8 { /** - * @brief Decodes UTF-8 into code points. Malformed or truncated sequences, overlong forms and surrogates each - * become one U+FFFD, so arbitrary bytes never fail to decode. + * @brief Decodes UTF-8 into code points. Every byte that is not part of a valid sequence (malformed, truncated, + * overlong, surrogate) becomes one U+FFFD, so arbitrary bytes never fail to decode. * * @param utf8 The text to decode. */ diff --git a/utility/source/utility/typed_line_buffer.cpp b/utility/source/utility/typed_line_buffer.cpp index c9426f75..165c0ada 100644 --- a/utility/source/utility/typed_line_buffer.cpp +++ b/utility/source/utility/typed_line_buffer.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include @@ -56,11 +58,8 @@ namespace Utility case '\x7f': case '\b': { - // A multibyte character goes as a whole: its continuation bytes, then its lead. - while (!line_.empty() && (static_cast(line_.back()) & 0xC0) == 0x80) - line_.pop_back(); if (!line_.empty()) - line_.pop_back(); + line_.resize(Paths::truncatedUtf8(line_, line_.size() - 1u).size()); continue; } case '\x03': // Ctrl-C diff --git a/utility/source/utility/utf8.cpp b/utility/source/utility/utf8.cpp index 099bc9d0..70be7489 100644 --- a/utility/source/utility/utf8.cpp +++ b/utility/source/utility/utf8.cpp @@ -1,84 +1,20 @@ #include -#include +#include namespace Utility::Utf8 { - namespace - { - constexpr char32_t replacementCharacter = 0xFFFDu; - - bool isContinuationByte(unsigned char byte) - { - return (byte & 0xC0u) == 0x80u; - } - - bool isScalarValue(char32_t codePoint) - { - return codePoint <= 0x10FFFFu && (codePoint < 0xD800u || codePoint > 0xDFFFu); - } - } - std::u32string decode(std::string_view utf8) { std::u32string result; result.reserve(utf8.size()); - - std::size_t position = 0; - while (position < utf8.size()) - { - const auto leadByte = static_cast(utf8[position]); - if (leadByte < 0x80u) - { - result.push_back(leadByte); - ++position; - continue; - } - - std::size_t length = 0; - char32_t codePoint = 0; - char32_t minimum = 0; - if ((leadByte & 0xE0u) == 0xC0u) - { - length = 2; - codePoint = leadByte & 0x1Fu; - minimum = 0x80u; - } - else if ((leadByte & 0xF0u) == 0xE0u) - { - length = 3; - codePoint = leadByte & 0x0Fu; - minimum = 0x800u; - } - else if ((leadByte & 0xF8u) == 0xF0u) - { - length = 4; - codePoint = leadByte & 0x07u; - minimum = 0x10000u; - } - else + Paths::Detail::forEachUtf8CodePoint( + utf8, + [&result](char32_t codePoint) { - result.push_back(replacementCharacter); - ++position; - continue; - } - - std::size_t consumed = 1; - while ( - consumed < length && position + consumed < utf8.size() && - isContinuationByte(static_cast(utf8[position + consumed])) - ) - { - codePoint = (codePoint << 6u) | (static_cast(utf8[position + consumed]) & 0x3Fu); - ++consumed; - } - - if (consumed != length || codePoint < minimum || !isScalarValue(codePoint)) - result.push_back(replacementCharacter); - else result.push_back(codePoint); - position += consumed; - } + } + ); return result; } @@ -86,35 +22,8 @@ namespace Utility::Utf8 { std::string result; result.reserve(codePoints.size()); - - for (auto codePoint : codePoints) - { - if (!isScalarValue(codePoint)) - codePoint = replacementCharacter; - - if (codePoint < 0x80u) - { - result.push_back(static_cast(codePoint)); - } - else if (codePoint < 0x800u) - { - result.push_back(static_cast(0xC0u | (codePoint >> 6u))); - result.push_back(static_cast(0x80u | (codePoint & 0x3Fu))); - } - else if (codePoint < 0x10000u) - { - result.push_back(static_cast(0xE0u | (codePoint >> 12u))); - result.push_back(static_cast(0x80u | ((codePoint >> 6u) & 0x3Fu))); - result.push_back(static_cast(0x80u | (codePoint & 0x3Fu))); - } - else - { - result.push_back(static_cast(0xF0u | (codePoint >> 18u))); - result.push_back(static_cast(0x80u | ((codePoint >> 12u) & 0x3Fu))); - result.push_back(static_cast(0x80u | ((codePoint >> 6u) & 0x3Fu))); - result.push_back(static_cast(0x80u | (codePoint & 0x3Fu))); - } - } + for (const auto codePoint : codePoints) + Paths::Detail::appendUtf8(result, codePoint); return result; } } From c7abeff86c49d48c8e0359741f11a8d76868a576 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:43:01 +0200 Subject: [PATCH 026/131] Native paths convert to UTF-8 without throwing Native paths convert to UTF-8 without throwing: nativeToUtf8 replaces bytes that are not UTF-8 (POSIX) and unpaired surrogates (Windows) with U+FFFD, convertNativeToUtf8 also says whether that happened, checkedNativeToUtf8 and LocalPath::make refuse such text with InvalidUtf8, and nativeFromUtf8 converts through the wide text on Windows; Utility::errorMessage gives an error code or filesystem error as UTF-8, read with FormatMessageW for Windows system errors --- paths/include/paths/local_path.hpp | 16 +++-- paths/include/paths/native.hpp | 75 ++++++++++++++++++---- utility/include/utility/error_message.hpp | 27 ++++++++ utility/source/utility/CMakeLists.txt | 1 + utility/source/utility/error_message.cpp | 78 +++++++++++++++++++++++ 5 files changed, 180 insertions(+), 17 deletions(-) create mode 100644 utility/include/utility/error_message.hpp create mode 100644 utility/source/utility/error_message.cpp diff --git a/paths/include/paths/local_path.hpp b/paths/include/paths/local_path.hpp index 49fffda0..29344aa9 100644 --- a/paths/include/paths/local_path.hpp +++ b/paths/include/paths/local_path.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -182,19 +183,22 @@ namespace Paths #ifndef __EMSCRIPTEN__ /** - * @brief Takes a native path, which has to be absolute. + * @brief Takes a native path, which has to be absolute. Refused with InvalidUtf8 when its text is not + * UTF-8 (POSIX) or has an unpaired surrogate (Windows). Never throws. */ static std::expected make(std::filesystem::path const& path) { - const auto utf8 = path.generic_u8string(); - return fromUtf8(std::string_view{reinterpret_cast(utf8.data()), utf8.size()}); + return checkedNativeToUtf8(path).and_then( + [](std::string const& utf8) + { + return fromUtf8(utf8); + } + ); } std::filesystem::path native() const { - return std::filesystem::path{ - std::u8string{reinterpret_cast(value_.data()), value_.size()} - }.make_preferred(); + return nativeFromUtf8(value_).make_preferred(); } #endif diff --git a/paths/include/paths/native.hpp b/paths/include/paths/native.hpp index 7440378c..f7cd31d7 100644 --- a/paths/include/paths/native.hpp +++ b/paths/include/paths/native.hpp @@ -1,29 +1,82 @@ #pragma once -#ifndef __EMSCRIPTEN__ -# include -# include -# include +#include +#include + +#include +#include +#include +#include +#include namespace Paths { /** - * @brief The text of a native path as generic UTF-8. For names and link literals, which are no LocalPath; an - * absolute path goes through LocalPath::make. + * @brief Native text converted to UTF-8, and whether that was lossless. + */ + struct NativeText + { + /** + * @brief The text as generic ('/' separated) UTF-8, always valid: U+FFFD stands for every byte that is not + * UTF-8 (POSIX) and every unpaired surrogate (Windows). + */ + std::string utf8{}; + /** + * @brief False when the native text had to be replaced in places, then @ref utf8 only shows it. + */ + bool lossless{true}; + }; + + /** + * @brief The text of a native path as generic UTF-8 and whether it converted losslessly. Never throws. + */ + inline NativeText convertNativeToUtf8(std::filesystem::path const& path) + { +#ifdef _WIN32 + const auto wide = path.generic_wstring(); + return NativeText{ + .utf8 = utf16ToUtf8(std::wstring_view{wide}), + .lossless = isValidUtf16(std::wstring_view{wide}), + }; +#else + auto const& bytes = path.native(); + return NativeText{.utf8 = sanitizedUtf8(bytes), .lossless = isValidUtf8(bytes)}; +#endif + } + + /** + * @brief The text of a native path as generic UTF-8, for display and logs: what is not UTF-8 becomes U+FFFD. + * A name or path that is acted on goes through @ref checkedNativeToUtf8 or LocalPath::make. */ inline std::string nativeToUtf8(std::filesystem::path const& path) { - const auto text = path.generic_u8string(); - return std::string{reinterpret_cast(text.data()), text.size()}; + return convertNativeToUtf8(path).utf8; + } + + /** + * @brief The text of a native path as generic UTF-8, refused with InvalidUtf8 when it does not convert + * losslessly (the error input shows it with U+FFFD). + */ + inline std::expected checkedNativeToUtf8(std::filesystem::path const& path) + { + auto converted = convertNativeToUtf8(path); + if (!converted.lossless) + return std::unexpected(PathError{.reason = PathRejection::InvalidUtf8, .input = std::move(converted.utf8)}); + return std::move(converted.utf8); } /** - * @brief A native path from UTF-8 text, the reverse of @ref nativeToUtf8. An absolute path goes through - * LocalPath::fromUtf8, which validates it. + * @brief A native path from UTF-8 text, the reverse of @ref nativeToUtf8. Never throws: on Windows every byte + * that is not UTF-8 becomes U+FFFD, so text from a server is checked with isValidUtf8 first. An absolute + * path goes through LocalPath::fromUtf8, which validates it. */ inline std::filesystem::path nativeFromUtf8(std::string_view utf8) { - return std::filesystem::path{std::u8string_view{reinterpret_cast(utf8.data()), utf8.size()}}; +#ifdef _WIN32 + return std::filesystem::path{utf8ToWide(utf8)}; +#else + return std::filesystem::path{std::string{utf8}}; +#endif } } #endif diff --git a/utility/include/utility/error_message.hpp b/utility/include/utility/error_message.hpp new file mode 100644 index 00000000..34bf11c6 --- /dev/null +++ b/utility/include/utility/error_message.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include + +namespace Utility +{ + /** + * @brief The text of @p error as valid UTF-8. On Windows a system error is read with FormatMessageW, which the + * narrow message() would give in the ANSI code page; any other text has every byte that is not UTF-8 + * replaced by U+FFFD. + */ + std::string errorMessage(std::error_code const& error); + + /** + * @brief The text of a filesystem error as valid UTF-8: the message of its code and the paths it names. + */ + std::string errorMessage(std::filesystem::filesystem_error const& error); + + /** + * @brief The text of any exception as valid UTF-8: a filesystem error as above, any other what() with every byte + * that is not UTF-8 replaced by U+FFFD. + */ + std::string errorMessage(std::exception const& error); +} diff --git a/utility/source/utility/CMakeLists.txt b/utility/source/utility/CMakeLists.txt index a669f51a..0c048ba8 100644 --- a/utility/source/utility/CMakeLists.txt +++ b/utility/source/utility/CMakeLists.txt @@ -21,6 +21,7 @@ else() utility PRIVATE node/node.cpp + error_message.cpp permissions.cpp rename.cpp link_safe_remove.cpp diff --git a/utility/source/utility/error_message.cpp b/utility/source/utility/error_message.cpp new file mode 100644 index 00000000..0627765e --- /dev/null +++ b/utility/source/utility/error_message.cpp @@ -0,0 +1,78 @@ +#include + +#include +#include + +#include + +#include +#include + +#ifdef _WIN32 +# include +#endif + +namespace Utility +{ + namespace + { +#ifdef _WIN32 + struct LocalFreeDeleter + { + void operator()(wchar_t* buffer) const noexcept + { + LocalFree(buffer); + } + }; + + std::string systemMessage(int code) + { + wchar_t* buffer = nullptr; + const auto length = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + nullptr, + static_cast(code), + 0, + reinterpret_cast(&buffer), + 0, + nullptr + ); + const std::unique_ptr owned{buffer}; + if (length == 0 || !owned) + return fmt::format("System error {}", code); + std::wstring_view text{owned.get(), length}; + while (!text.empty() && (text.back() == L'\r' || text.back() == L'\n' || text.back() == L' ')) + text.remove_suffix(1); + return Paths::wideToUtf8(text); + } +#endif + } + + std::string errorMessage(std::error_code const& error) + { +#ifdef _WIN32 + if (error.category() == std::system_category()) + return systemMessage(error.value()); +#endif + return Paths::sanitizedUtf8(error.message()); + } + + std::string errorMessage(std::filesystem::filesystem_error const& error) + { + const auto message = errorMessage(error.code()); + if (error.path1().empty()) + return message; + if (error.path2().empty()) + return fmt::format("{}: '{}'", message, Paths::nativeToUtf8(error.path1())); + return fmt::format( + "{}: '{}', '{}'", message, Paths::nativeToUtf8(error.path1()), Paths::nativeToUtf8(error.path2()) + ); + } + + std::string errorMessage(std::exception const& error) + { + if (const auto* filesystemError = dynamic_cast(&error)) + return errorMessage(*filesystemError); + return Paths::sanitizedUtf8(error.what()); + } +} From e7e66837e20e0cd87756ce9e57897e9e86ca5888 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:46:12 +0200 Subject: [PATCH 027/131] A listed name that is not UTF-8 is shown sanitized and never acted on A listed name that is not UTF-8 is kept for display with U+FFFD and nameIsValidUtf8 false, and DirectoryEntry::nameSegment refuses it, so walkers, sync scans and explorer actions reject it as InvalidName and never act on a different item called like its display text; server owner, group, ACL and link literals and local owner and group names are sanitized the same way, a downloaded link whose literal is not UTF-8 and an uploaded link whose target is not are refused, and the mock server can list raw name bytes --- backend/source/backend/rpc_filesystem.cpp | 16 ++++-- .../backend/sftp/bulk_transfer_operation.cpp | 52 ++++++++++++----- .../backend/sftp/download_operation.cpp | 2 +- .../backend/sftp/local_scan_operation.cpp | 50 +++++------------ .../source/backend/sftp/scan_operation.cpp | 4 +- .../source/backend/sftp/upload_operation.cpp | 21 ++++++- .../file_explorer/local_side_model.cpp | 12 +++- .../file_explorer/remote_side_model.cpp | 2 +- .../include/shared_data/directory_entry.hpp | 56 +++++++++++-------- .../sync/tree_directory_walker.hpp | 10 ++-- .../source/shared_data/directory_entry.cpp | 3 + ssh/source/ssh/file_information.cpp | 18 ++++-- ssh/source/ssh/sftp_session.cpp | 2 +- .../include/utility/directory_traversal.hpp | 15 +++-- 14 files changed, 163 insertions(+), 100 deletions(-) diff --git a/backend/source/backend/rpc_filesystem.cpp b/backend/source/backend/rpc_filesystem.cpp index f164bfc7..93b9ecb1 100644 --- a/backend/source/backend/rpc_filesystem.cpp +++ b/backend/source/backend/rpc_filesystem.cpp @@ -68,7 +68,7 @@ namespace struct passwd* result = nullptr; std::string name; if (::getpwuid_r(uid, &pwd, buffer, sizeof(buffer), &result) == 0 && result) - name = result->pw_name; + name = Paths::sanitizedUtf8(result->pw_name); return users.emplace(uid, std::move(name)).first->second; } @@ -82,7 +82,7 @@ namespace struct group* result = nullptr; std::string name; if (::getgrgid_r(gid, &grp, buffer, sizeof(buffer), &result) == 0 && result) - name = result->gr_name; + name = Paths::sanitizedUtf8(result->gr_name); return groups.emplace(gid, std::move(name)).first->second; } }; @@ -144,13 +144,13 @@ namespace struct passwd pwd{}; struct passwd* pwdResult = nullptr; if (::getpwuid_r(st.st_uid, &pwd, pwdBuf, sizeof(pwdBuf), &pwdResult) == 0 && pwdResult) - entry.owner = pwdResult->pw_name; + entry.owner = Paths::sanitizedUtf8(pwdResult->pw_name); char grpBuf[1024]; struct group grp{}; struct group* grpResult = nullptr; if (::getgrgid_r(st.st_gid, &grp, grpBuf, sizeof(grpBuf), &grpResult) == 0 && grpResult) - entry.group = grpResult->gr_name; + entry.group = Paths::sanitizedUtf8(grpResult->gr_name); } #endif @@ -414,7 +414,9 @@ void RpcFilesystem::registerListFiles() const auto entryPath = entry.path(); SharedData::DirectoryEntry dirEntry{}; - dirEntry.name = Paths::nativeToUtf8(entryPath.filename()); + auto name = Paths::convertNativeToUtf8(entryPath.filename()); + dirEntry.name = std::move(name.utf8); + dirEntry.nameIsValidUtf8 = name.lossless; dirEntry.type = SharedData::fileTypeFromStdFilesystemType(entry.symlink_status().type()); dirEntry.permissions = entry.symlink_status().permissions(); @@ -559,7 +561,9 @@ void RpcFilesystem::registerProperties() auto const& path = *pathParameter; SharedData::DirectoryEntry entry{}; - entry.name = Paths::nativeToUtf8(path.filename()); + auto name = Paths::convertNativeToUtf8(path.filename()); + entry.name = std::move(name.utf8); + entry.nameIsValidUtf8 = name.lossless; std::string resolvedPath{}; std::error_code symEc; diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 82989cac..08d66cef 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -520,22 +520,46 @@ BulkTransferOperation::entryPaths(std::size_t entryIndex) const return Error{.type = ErrorType::InvalidName, .extraInfo = error.toString()}; }; - // The names from the entry up to, but not including, the root. - std::vector names; - for (auto const* current = &entries_[entryIndex]; current->parent; current = &entries_[*current->parent]) +namespace Detail +{ + std::expected treeEntryPaths( + std::vector const& entries, + std::size_t entryIndex, + Paths::RemotePath const& remoteRoot, + Paths::LocalPath const& localRoot + ) { - if (*current->parent >= entries_.size()) - return std::unexpected(Error{.type = ErrorType::ImplementationError, .extraInfo = "Parent out of range"}); - names.push_back(¤t->name); - } + using ErrorType = Operation::ErrorType; + if (remoteRoot.empty() || localRoot.empty()) + return std::unexpected( + Operation::Error{.type = ErrorType::ImplementationError, .extraInfo = "A tree needs both roots"} + ); + if (entryIndex >= entries.size()) + return std::unexpected( + Operation::Error{.type = ErrorType::ImplementationError, .extraInfo = "Entry out of range"} + ); - Paths::RelativePath relative{}; - for (auto name = names.rbegin(); name != names.rend(); ++name) - { - auto segment = Paths::PathSegment::make(**name); - if (!segment) - return std::unexpected(invalidName(segment.error())); - relative = relative / *segment; + const auto refused = [](Paths::PathError const& error) + { + if (error.reason == Paths::PathRejection::UnknownParent) + return Operation::Error{.type = ErrorType::ImplementationError, .extraInfo = "Parent out of range"}; + return Operation::Error{.type = ErrorType::InvalidName, .extraInfo = error.toString()}; + }; + return Utility::walkedRelativePath(entries, entries[entryIndex]) + .and_then( + [&](Paths::RelativePath const& relative) + { + return localRoot.join(relative).transform( + [&](Paths::LocalPath local) + { + return BulkTransferOperation::EntryPaths{ + .remote = remoteRoot / relative, .local = std::move(local) + }; + } + ); + } + ) + .transform_error(refused); } return common_.localPath.join(relative) diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index d7b996e8..e34f6832 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -288,7 +288,7 @@ std::expected DownloadOperation::handleSymlinkIn } std::filesystem::remove(tempNativePath_, ec); ec.clear(); - const auto linkContent = Paths::nativeFromUtf8(readResult.value().linkContent); + const auto linkContent = Paths::nativeFromUtf8(linkText); if (linkEntry && linkEntry->isDirectory()) std::filesystem::create_directory_symlink(linkContent, tempNativePath_, ec); else diff --git a/backend/source/backend/sftp/local_scan_operation.cpp b/backend/source/backend/sftp/local_scan_operation.cpp index 8d74bdf8..c8fc3809 100644 --- a/backend/source/backend/sftp/local_scan_operation.cpp +++ b/backend/source/backend/sftp/local_scan_operation.cpp @@ -134,33 +134,12 @@ LocalScanOperation::scanner(Paths::RelativePath const& relative) if (::lstat(entry.path().c_str(), &st) == 0) mtimeSecs = static_cast(st.st_mtime); #endif - } - else - { - std::error_code errc{}; - const auto ftime = entry.last_write_time(errc); - if (!errc) - { - const auto sctime = std::chrono::file_clock::to_sys(ftime); - mtimeSecs = static_cast( - std::chrono::duration_cast(sctime.time_since_epoch()).count() - ); - } - } - - std::optional linkTarget{}; - if (isSymlink) - { - std::error_code errc{}; - const auto target = std::filesystem::read_symlink(entry.path(), errc); - if (!errc) - linkTarget = Paths::nativeToUtf8(target); - } - - const auto name = Paths::nativeToUtf8(entry.path().filename()); - if (ignoreHidden_ && !name.empty() && name.front() == '.') - continue; - if (respectIgnoreFiles_ && !ignoreMatcher_.empty()) + } + else + { + std::error_code errc{}; + const auto ftime = entry.last_write_time(errc); + if (!errc) { // A name that is no segment cannot match a rule, the walker rejects it. const auto segment = Paths::PathSegment::make(name); @@ -169,16 +148,13 @@ LocalScanOperation::scanner(Paths::RelativePath const& relative) continue; } - entries.push_back( - SharedData::DirectoryEntry{ - .name = name, - .type = type, - .size = entry.is_regular_file() ? entry.file_size() : 0, - .permissions = Utility::localPermissions(status), - .mtime = mtimeSecs, - .linkTarget = std::move(linkTarget), - } - ); + std::optional linkTarget{}; + if (isSymlink) + { + std::error_code errc{}; + const auto target = std::filesystem::read_symlink(entry.path(), errc); + if (!errc) + linkTarget = Paths::nativeToUtf8(target); } catch (const std::filesystem::filesystem_error& e) { diff --git a/backend/source/backend/sftp/scan_operation.cpp b/backend/source/backend/sftp/scan_operation.cpp index 7dc3c004..615cab27 100644 --- a/backend/source/backend/sftp/scan_operation.cpp +++ b/backend/source/backend/sftp/scan_operation.cpp @@ -109,7 +109,7 @@ ScanOperation::scanner(Paths::RelativePath const& relative) continue; if (entry.type != SharedData::DirectoryEntry::FileType::Regular) continue; - if (auto content = readRemoteFile(*sftp_, path / *Paths::PathSegment::make(name), futureTimeout_)) + if (auto content = readRemoteFile(*sftp, path / *entry.nameSegment(), futureTimeout_)) ignoreMatcher_.addFile(relative, *content); } @@ -121,7 +121,7 @@ ScanOperation::scanner(Paths::RelativePath const& relative) for (auto& entry : entries) { // A name that is no segment cannot match a rule, the walker rejects it. - const auto segment = Paths::PathSegment::make(entry.name); + const auto segment = entry.nameSegment(); const bool isDir = entry.type == SharedData::DirectoryEntry::FileType::Directory; if (segment && ignoreMatcher_.isIgnored(relative / *segment, isDir)) continue; diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index e9173522..37ca9889 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -306,7 +307,23 @@ std::expected UploadOperation::handleSymlinkInStra return {}; } - const auto target = std::filesystem::read_symlink(localNativePath_); + std::error_code readError{}; + const auto target = std::filesystem::read_symlink(localNativePath_, readError); + if (readError) + { + Log::error( + "UploadOperation: Cannot read the symlink '{}': {}.", + options_.localPath.toUtf8(), + Utility::errorMessage(readError) + ); + return std::unexpected(Error{.type = ErrorType::FileStatFailed, .extraInfo = Utility::errorMessage(readError)}); + } + const auto targetText = Paths::checkedNativeToUtf8(target); + if (!targetText) + { + Log::error("UploadOperation: The symlink '{}' points to a name that is not UTF-8.", options_.localPath.toUtf8()); + return std::unexpected(Error{.type = ErrorType::InvalidName, .extraInfo = targetText.error().toString()}); + } if (options_.createMissingDirectories) { @@ -317,7 +334,7 @@ std::expected UploadOperation::handleSymlinkInStra // The link is created under the temp name and renamed into place, so an existing destination is // never removed before its replacement exists. std::ignore = sftp_->removeFileInStrand(tempPath_); - const auto result = sftp_->createSymLinkInStrand(Paths::nativeToUtf8(target), tempPath_); + const auto result = sftp_->createSymLinkInStrand(*targetText, tempPath_); if (!result.has_value()) { Log::error( diff --git a/frontend/source/frontend/file_explorer/local_side_model.cpp b/frontend/source/frontend/file_explorer/local_side_model.cpp index f8189763..675bf69c 100644 --- a/frontend/source/frontend/file_explorer/local_side_model.cpp +++ b/frontend/source/frontend/file_explorer/local_side_model.cpp @@ -885,6 +885,8 @@ void LocalSideModel::onProperties(NuiFileExplorer::Item const& item) .mtimeNsec = pickU32("mtimeNsec"), .acl = pickString("acl"), }; + if (source.hasOwnProperty("nameIsValidUtf8")) + out.nameIsValidUtf8 = source["nameIsValidUtf8"].template as(); if (source.hasOwnProperty("linkTarget")) { const auto link = source["linkTarget"]; @@ -1046,6 +1048,8 @@ void LocalSideModel::navigateTo(std::filesystem::path const& path) .mtime = pickU64("mtime"), .mtimeNsec = pickU32("mtimeNsec"), }; + if (source.hasOwnProperty("nameIsValidUtf8")) + entry.nameIsValidUtf8 = source["nameIsValidUtf8"].template as(); if (source.hasOwnProperty("linkTarget")) { const auto link = source["linkTarget"]; @@ -1492,7 +1496,13 @@ std::expected LocalSideModel::localPathOf(Nu { if (item.isParentLink) return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::DotOrDotDot, .input = ".."}); - return Paths::LocalPathSegment::make(item.name) + return item.nameSegment() + .and_then( + [](Paths::PathSegment const& segment) + { + return Paths::LocalPathSegment::make(segment); + } + ) .and_then( [&item](Paths::LocalPathSegment const&) { diff --git a/frontend/source/frontend/file_explorer/remote_side_model.cpp b/frontend/source/frontend/file_explorer/remote_side_model.cpp index d512bbaf..df7da412 100644 --- a/frontend/source/frontend/file_explorer/remote_side_model.cpp +++ b/frontend/source/frontend/file_explorer/remote_side_model.cpp @@ -286,7 +286,7 @@ std::expected RemoteSideModel::remotePathOf { if (item.isParentLink) return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::DotOrDotDot, .input = ".."}); - return Paths::PathSegment::make(item.name) + return item.nameSegment() .and_then( [&item](Paths::PathSegment const&) { diff --git a/shared_data/include/shared_data/directory_entry.hpp b/shared_data/include/shared_data/directory_entry.hpp index 8ecdb0f8..7a97fd64 100644 --- a/shared_data/include/shared_data/directory_entry.hpp +++ b/shared_data/include/shared_data/directory_entry.hpp @@ -88,17 +88,30 @@ namespace SharedData using FileType = SharedData::FileType; /** - * @brief The name as listed, UTF-8. A server or a hostile local name can be anything, like "..", "x/.." or - * empty, so it only becomes part of a path through Paths::PathSegment::make. Empty for the root of a - * walk and for stat results. + * @brief The name as listed, always valid UTF-8. A server or a hostile local name can be anything, like "..", + * "x/.." or empty, so it only becomes part of a path through @ref nameSegment. A listed name that was + * not UTF-8 is kept for display with U+FFFD for every invalid byte and @ref nameIsValidUtf8 false. + * Empty for the root of a walk and for stat results. */ std::string name{}; + /** + * @brief False when the listed name was not UTF-8 and @ref name only shows it: such an entry can be listed + * but never named in a path, so no action can hit a different item that is called like the display + * text. + */ + bool nameIsValidUtf8{true}; std::uint32_t flags{0}; FileType type{FileType::Unknown}; std::uint64_t size{0}; std::uint32_t uid{0}; std::uint32_t gid{0}; + /** + * @brief The owner name, valid UTF-8 (U+FFFD for every invalid byte the server sent). + */ std::string owner{}; + /** + * @brief The group name, valid UTF-8 (U+FFFD for every invalid byte the server sent). + */ std::string group{}; std::filesystem::perms permissions{std::filesystem::perms::unknown}; std::uint64_t atime{0}; @@ -108,12 +121,26 @@ namespace SharedData std::uint64_t mtime{0}; std::uint32_t mtimeNsec{0}; std::string acl{}; - // For symlinks: raw value returned by readlink (i.e. the link literal, not the resolved target) as UTF-8. - // nullopt when not a symlink or when unavailable. This is what the sync diff compares by. + /** + * @brief For symlinks: the link literal readlink returned (not the resolved target), valid UTF-8 with U+FFFD + * for every invalid byte. nullopt when not a symlink or when unavailable. The sync diff compares by it, + * a transfer of the link reads the literal itself. + */ std::optional linkTarget{}; // For symlinks: the full entry of the symlink target (if known). nullptr means unknown. std::shared_ptr resolvedTarget{nullptr}; + /** + * @brief The name as a path segment. Refused like Paths::PathSegment::make refuses it, and with InvalidUtf8 + * when the listed name was not UTF-8. + */ + std::expected nameSegment() const + { + if (!nameIsValidUtf8) + return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::InvalidUtf8, .input = name}); + return Paths::PathSegment::make(name); + } + bool isDirectory() const { return type == FileType::Directory; @@ -229,24 +256,7 @@ namespace SharedData inline std::expected relativePath(std::vector const& entries, DirectoryEntry const& entry) { - if (!entry.parent) - return Paths::RelativePath{}; - const auto parentIndex = entry.parent.value(); - if (parentIndex >= entries.size()) - throw std::out_of_range("Parent index is out of range"); - return Paths::PathSegment::make(entry.name) - .and_then( - [&entries, parentIndex](Paths::PathSegment const& segment) - { - return relativePath(entries, entries[parentIndex]) - .transform( - [&segment](Paths::RelativePath const& parent) - { - return parent / segment; - } - ); - } - ); + return Utility::walkedRelativePath(entries, entry); } void to_json(nlohmann::json& j, DirectoryEntry const& entry); diff --git a/shared_data/include/shared_data/sync/tree_directory_walker.hpp b/shared_data/include/shared_data/sync/tree_directory_walker.hpp index 5537bf68..16d26f1d 100644 --- a/shared_data/include/shared_data/sync/tree_directory_walker.hpp +++ b/shared_data/include/shared_data/sync/tree_directory_walker.hpp @@ -145,9 +145,9 @@ namespace SharedData::Sync if (raw.type == SharedData::FileType::Regular) totalBytes_ += raw.size; - // The listed name as it is, "x/.." stays an invalid name. + // The listed name as it is, "x/.." or a name that was not UTF-8 stays an invalid name. + const auto availability = nameAvailability(raw); std::string nameUtf8 = std::move(raw.name); - const auto availability = nameAvailability(nameUtf8, raw.type); children.push_back( ScanNode{ .name = std::move(nameUtf8), @@ -244,9 +244,11 @@ namespace SharedData::Sync type == SharedData::FileType::Symlink; } - Availability nameAvailability(std::string const& name, SharedData::FileType type) const + Availability nameAvailability(SharedData::DirectoryEntry const& entry) const { - const auto segment = Paths::PathSegment::make(name); + auto const& name = entry.name; + const auto type = entry.type; + const auto segment = entry.nameSegment(); if (!segment) return Availability::InvalidName; if (requireLocalNames_ && !Paths::LocalPathSegment::make(*segment).has_value()) diff --git a/shared_data/source/shared_data/directory_entry.cpp b/shared_data/source/shared_data/directory_entry.cpp index 61265b8b..81172095 100644 --- a/shared_data/source/shared_data/directory_entry.cpp +++ b/shared_data/source/shared_data/directory_entry.cpp @@ -12,6 +12,8 @@ namespace SharedData {"mtime", entry.mtime}, {"mtimeNsec", entry.mtimeNsec}, }; + if (!entry.nameIsValidUtf8) + j["nameIsValidUtf8"] = false; if (entry.flags != 0) j["flags"] = entry.flags; if (entry.uid != 0) @@ -46,6 +48,7 @@ namespace SharedData j.at("name").get_to(entry.name); j.at("type").get_to(entry.type); + getOptional("nameIsValidUtf8", entry.nameIsValidUtf8); getOptional("flags", entry.flags); getOptional("size", entry.size); getOptional("uid", entry.uid); diff --git a/ssh/source/ssh/file_information.cpp b/ssh/source/ssh/file_information.cpp index ea1016d8..3ebcc35f 100644 --- a/ssh/source/ssh/file_information.cpp +++ b/ssh/source/ssh/file_information.cpp @@ -1,19 +1,29 @@ #include +#include + +#include +#include + namespace SecureShell { FileInformation fromSftpAttributes(sftp_attributes attributes) { + const auto serverText = [](char const* text) + { + return text ? Paths::sanitizedUtf8(text) : std::string{}; + }; return FileInformation{ SharedData::DirectoryEntry{ - .name = attributes->name ? std::string{attributes->name} : std::string{}, + .name = serverText(attributes->name), + .nameIsValidUtf8 = attributes->name == nullptr || Paths::isValidUtf8(attributes->name), .flags = attributes->flags, .type = static_cast(attributes->type), .size = attributes->size, .uid = attributes->uid, .gid = attributes->gid, - .owner = attributes->owner ? std::string{attributes->owner} : std::string{}, - .group = attributes->group ? std::string{attributes->group} : std::string{}, + .owner = serverText(attributes->owner), + .group = serverText(attributes->group), .permissions = static_cast(attributes->permissions), .atime = attributes->atime, .atimeNsec = attributes->atime_nseconds, @@ -22,7 +32,7 @@ namespace SecureShell .mtime = attributes->mtime, .mtimeNsec = attributes->mtime_nseconds, .acl = attributes->acl - ? std::string{ssh_string_get_char(attributes->acl), ssh_string_len(attributes->acl)} + ? Paths::sanitizedUtf8({ssh_string_get_char(attributes->acl), ssh_string_len(attributes->acl)}) : std::string{}, }, }; diff --git a/ssh/source/ssh/sftp_session.cpp b/ssh/source/ssh/sftp_session.cpp index f6d4505c..533e758b 100644 --- a/ssh/source/ssh/sftp_session.cpp +++ b/ssh/source/ssh/sftp_session.cpp @@ -131,7 +131,7 @@ namespace SecureShell if (linkLiteral) { const auto* raw = static_cast(linkLiteral.get()); - entry.linkTarget = std::string{raw ? raw : ""}; + entry.linkTarget = Paths::sanitizedUtf8(raw ? raw : ""); } std::unique_ptr targetAttrs{ diff --git a/utility/include/utility/directory_traversal.hpp b/utility/include/utility/directory_traversal.hpp index c71faabb..ffd2386f 100644 --- a/utility/include/utility/directory_traversal.hpp +++ b/utility/include/utility/directory_traversal.hpp @@ -133,7 +133,12 @@ namespace Utility if (parentIndex >= entries_.size()) throw std::out_of_range("Parent index is out of range"); // Only valid segments are adopted, see moveEntries. - return relativePath(entries_[parentIndex]) / *Paths::PathSegment::make(nameOf(entry)); + return entry.nameSegment().transform( + [this, parentIndex](Paths::PathSegment const& segment) + { + return directoryPaths_[parentIndex] / segment; + } + ); } /** @@ -165,8 +170,9 @@ namespace Utility } /** - * @brief Entries whose name is not a valid path segment, like ".." or "a/b" from a server. They are never - * walked or returned by @ref entries, their parent index is set so their full path can be shown. + * @brief Entries whose name is not a valid path segment, like ".." or "a/b" from a server or a name that was + * not UTF-8. They are never walked or returned by @ref entries, their parent index is set so their + * full path can be shown. */ std::vector const& rejectedEntries() const { @@ -186,7 +192,8 @@ namespace Utility const auto name = nameOf(entry); if (name == "." || name == "..") continue; - if (!Paths::PathSegment::make(name).has_value()) + auto segment = entry.nameSegment(); + if (!segment.has_value()) { rejectedEntries_.push_back(std::move(entry)); continue; From e693c59e11a2358134237bb369508a8c29bb9137 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:51:11 +0200 Subject: [PATCH 028/131] Calls into the frontend never throw on text that is not UTF-8 Calls into the frontend never throw on text that is not UTF-8: RpcHelper::callRemoteSafely sends such a call again with U+FFFD for every invalid byte and logs any other failure, RPC replies, operation queue and process events and the session, main and file tracking calls go through it, and the logger forwards sanitized messages; the log file is opened by its native name, wide on Windows --- CMakeLists.txt | 3 + backend/include/backend/rpc_helper.hpp | 105 +++++++++++++----- .../file_tracking/temp_dir_instancing.cpp | 3 +- backend/source/backend/main.cpp | 7 +- .../source/backend/process/process_store.cpp | 99 ++++++++++------- backend/source/backend/session.cpp | 4 +- backend/source/backend/session_manager.cpp | 5 +- .../source/backend/sftp/operation_queue.cpp | 88 +++++++++------ .../backend/windows/windows_drag_drop.cpp | 3 +- log/source/log/logger_backend.cpp | 17 +-- utility/include/utility/json_utf8.hpp | 39 +++++++ 11 files changed, 261 insertions(+), 112 deletions(-) create mode 100644 utility/include/utility/json_utf8.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cfa040cf..9f5f74f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,9 @@ project(nui-sftp VERSION "${PROJECT_VERSION}" LANGUAGES CXX) set(SPDLOG_FMT_EXTERNAL ON CACHE BOOL "Use an external fmt library (provide it manually)" FORCE) set(SPDLOG_BUILD_SHARED OFF CACHE BOOL "Dont build shared spdlog" FORCE) +if (WIN32) + set(SPDLOG_WCHAR_FILENAMES ON CACHE BOOL "Open log files by their wide name" FORCE) +endif() set(NUI_FILE_EXPLORER_MEMORY64 ON CACHE BOOL "Enable MEMORY64 for nui-file-explorer" FORCE) set(JSON_Diagnostics ON CACHE BOOL "Enable json diagnostics" FORCE) set(NUI_JSON_DIAGNOSTICS ON CACHE BOOL "Enable json diagnostics for nlohmann_json through nui" FORCE) diff --git a/backend/include/backend/rpc_helper.hpp b/backend/include/backend/rpc_helper.hpp index fd57b0bc..0c9ba1cd 100644 --- a/backend/include/backend/rpc_helper.hpp +++ b/backend/include/backend/rpc_helper.hpp @@ -1,9 +1,12 @@ #pragma once +#include +#include +#include + #include #include #include -#include #include #include #include @@ -14,7 +17,9 @@ #include #include #include +#include #include +#include // TODO: split and document this file! namespace RpcHelper @@ -56,6 +61,71 @@ namespace RpcHelper template using pop_front_t = typename pop_front::type; + + /** + * @brief @p argument for a second try of a call: numbers as they are, everything else as JSON with U+FFFD + * for every byte that is not UTF-8. + */ + template + auto withValidUtf8Argument(T const& argument) + { + if constexpr (std::is_arithmetic_v) + return argument; + else + return Utility::withValidUtf8(nlohmann::json(argument)); + } + } + + /** + * @brief Calls @p name in the frontend like Nui::RpcHub::callRemote, but never throws. Text that is not UTF-8, + * which the JSON dump refuses, is sent again with U+FFFD for every invalid byte; any other failure is + * logged. + */ + template + void callRemoteSafely(Nui::RpcHub const& hub, std::string const& name, Args const&... arguments) noexcept + { + const auto logFailure = [&name](std::exception const& error) noexcept + { + try + { + Log::error("Failed to call '{}' in the frontend: {}", name, Utility::errorMessage(error)); + } + catch (...) + {} + }; + constexpr int invalidUtf8 = 316; + try + { + hub.callRemote(name, arguments...); + return; + } + catch (nlohmann::json::type_error const& error) + { + if (error.id != invalidUtf8) + return logFailure(error); + } + catch (std::exception const& error) + { + return logFailure(error); + } + + try + { + Log::warn("The call of '{}' in the frontend carries text that is not UTF-8, it is sent replaced.", name); + hub.callRemote(name, Detail::withValidUtf8Argument(arguments)...); + } + catch (std::exception const& error) + { + logFailure(error); + } + } + + /** + * @brief @ref callRemoteSafely for a JSON value, which may be written as a braced list. + */ + inline void callRemoteSafely(Nui::RpcHub const& hub, std::string const& name, nlohmann::json const& json) noexcept + { + callRemoteSafely(hub, name, json); } class RpcInCorrectThread @@ -77,14 +147,7 @@ namespace RpcHelper wnd_->runInJavascriptThread( [this, json = std::move(json)]() { - try - { - hub_->callRemote(responseId_, json); - } - catch (const std::exception& e) - { - Log::error("Failed to call rpc respond '{}': {}", responseId_, e.what()); - } + callRemoteSafely(*hub_, responseId_, json); } ); } @@ -113,7 +176,11 @@ namespace RpcHelper { if (called_) { - Log::warn("RPC response with id '{}' already sent. Not sending: {}", responseId_, json.dump(4)); + Log::warn( + "RPC response with id '{}' already sent. Not sending: {}", + responseId_, + json.dump(4, ' ', false, nlohmann::json::error_handler_t::replace) + ); return; } called_ = true; @@ -121,14 +188,7 @@ namespace RpcHelper wnd_->runInJavascriptThread( [hub = hub_, responseId = std::move(responseId_), json = std::move(json)]() { - try - { - hub->callRemote(responseId, json); - } - catch (const std::exception& e) - { - Log::error("Failed to call rpc respond '{}': {}", responseId, e.what()); - } + callRemoteSafely(*hub, responseId, json); } ); } @@ -145,14 +205,7 @@ namespace RpcHelper wnd_->runInJavascriptThread( [hub = hub_, responseId = std::move(responseId_), message]() { - try - { - hub->callRemote(responseId, {{"success", false}, {"error", message}}); - } - catch (const std::exception& e) - { - Log::error("Failed to call rpc respond '{}': {}", responseId, e.what()); - } + callRemoteSafely(*hub, responseId, {{"success", false}, {"error", message}}); } ); } diff --git a/backend/source/backend/file_tracking/temp_dir_instancing.cpp b/backend/source/backend/file_tracking/temp_dir_instancing.cpp index 4133f8c9..54a2b949 100644 --- a/backend/source/backend/file_tracking/temp_dir_instancing.cpp +++ b/backend/source/backend/file_tracking/temp_dir_instancing.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -111,7 +112,7 @@ namespace FileTracking strand_, [hub = hub_, name = eventName_, pld = std::move(payload)]() mutable { - hub->callRemote(name, pld); + RpcHelper::callRemoteSafely(*hub, name, pld); } ); } diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index 1a1977ac..b9e69923 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -1,4 +1,5 @@ #include +#include #ifdef _WIN32 # include @@ -378,7 +379,8 @@ void Main::registerInitialWarningGetter() false; #endif - hub_.callRemote( + RpcHelper::callRemoteSafely( + hub_, responseId, nlohmann::json{ {"warnings", initialPersistenceLoadWarnings_}, @@ -395,7 +397,8 @@ void Main::registerVersionGetter() "Main::getVersion", [this](std::string responseId) { - hub_.callRemote( + RpcHelper::callRemoteSafely( + hub_, responseId, nlohmann::json{ {"version", VERSION}, diff --git a/backend/source/backend/process/process_store.cpp b/backend/source/backend/process/process_store.cpp index 888ec0b5..78aa3c16 100644 --- a/backend/source/backend/process/process_store.cpp +++ b/backend/source/backend/process/process_store.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -171,7 +172,9 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) wnd_->runInJavascriptThread( [this, procId, msg]() { - hub_->callRemote(msg["responseId"].get(), nlohmann::json{{"id", procId}}); + RpcHelper::callRemoteSafely( + *hub_, msg["responseId"].get(), nlohmann::json{{"id", procId}} + ); } ); } @@ -185,7 +188,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) wnd_->runInJavascriptThread( [this, receptacle, procId, data]() { - hub_->callRemote(receptacle, nlohmann::json{{"id", procId}, {"data", data}}); + RpcHelper::callRemoteSafely(*hub_, receptacle, nlohmann::json{{"id", procId}, {"data", data}}); } ); } @@ -198,7 +201,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) wnd_->runInJavascriptThread( [this, procId]() { - hub_->callRemote("execTerminalExit_" + procId, nlohmann::json{{"id", procId}}); + RpcHelper::callRemoteSafely(*hub_, "execTerminalExit_" + procId, nlohmann::json{{"id", procId}}); } ); } @@ -211,7 +214,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) wnd_->runInJavascriptThread( [this, responseId, errMsg]() { - hub_->callRemote(responseId, nlohmann::json{{"error", errMsg}}); + RpcHelper::callRemoteSafely(*hub_, responseId, nlohmann::json{{"error", errMsg}}); } ); return; @@ -222,7 +225,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) { if (procsList.empty()) { - hub_->callRemote(responseId, nlohmann::json{{"error", "No processes found"}}); + RpcHelper::callRemoteSafely(*hub_, responseId, nlohmann::json{{"error", "No processes found"}}); return; } nlohmann::json j = nlohmann::json::object(); @@ -231,7 +234,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) {"cmdline", procsList.front()["cmdline"]}, }; j["all"] = procsList; - hub_->callRemote(responseId, j); + RpcHelper::callRemoteSafely(*hub_, responseId, j); } ); } @@ -268,7 +271,7 @@ void ProcessStore::notifyChildExit(Nui::RpcHub& hub, std::string const& id) process->second->exit(); processes_.erase(process); - hub.callRemote("execTerminalExit_" + id, nlohmann::json{{"id", id}}); + RpcHelper::callRemoteSafely(hub, "execTerminalExit_" + id, nlohmann::json{{"id", id}}); } void ProcessStore::notifyChildExit(Nui::RpcHub& hub, long long pid) @@ -280,7 +283,9 @@ void ProcessStore::notifyChildExit(Nui::RpcHub& hub, long long pid) process->exit(); const auto idCopy = id; processes_.erase(id); - hub.callRemote("execTerminalExit_" + idCopy, nlohmann::json{{"id", idCopy}, {"pid", pid}}); + RpcHelper::callRemoteSafely( + hub, "execTerminalExit_" + idCopy, nlohmann::json{{"id", idCopy}, {"pid", pid}} + ); return; } } @@ -368,7 +373,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) using PtyType = PTY::PseudoTerminal; #endif - hub->callRemote(responseId, nlohmann::json{{"id", processId}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"id", processId}}); if (processes_[processId]->getState(ProcessAttachedState::ProcessInfo).isPty) { @@ -380,8 +385,10 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) wnd->runInJavascriptThread( [hub, stdoutReceptacle, id, msg = std::string{message}]() { - hub->callRemote( - stdoutReceptacle, nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} + RpcHelper::callRemoteSafely( + *hub, + stdoutReceptacle, + nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} ); } ); @@ -391,8 +398,10 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) wnd->runInJavascriptThread( [hub, stderrReceptacle, id, msg = std::string{message}]() { - hub->callRemote( - stderrReceptacle, nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} + RpcHelper::callRemoteSafely( + *hub, + stderrReceptacle, + nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} ); } ); @@ -408,8 +417,10 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) wnd->runInJavascriptThread( [hub, stdoutReceptacle, id, msg = std::string{message}]() { - hub->callRemote( - stdoutReceptacle, nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} + RpcHelper::callRemoteSafely( + *hub, + stdoutReceptacle, + nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} ); } ); @@ -420,8 +431,10 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) wnd->runInJavascriptThread( [hub, stderrReceptacle, id, msg = std::string{message}]() { - hub->callRemote( - stderrReceptacle, nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} + RpcHelper::callRemoteSafely( + *hub, + stderrReceptacle, + nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} ); } ); @@ -432,8 +445,8 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) } catch (std::exception const& e) { - Log::error("Failed to spawn process: {}", e.what()); - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + Log::error("Failed to spawn process: {}", Utility::errorMessage(e)); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -456,7 +469,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) if (forkPool_) forkPool_->send(nlohmann::json{{"id", id}, {"command", "kill"}}); forkPoolProcesses_.erase(iter); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); return; } } @@ -464,7 +477,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } auto& proc = process->second; @@ -475,12 +488,12 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) process->second->exit(0s); processes_.erase(process); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); #endif } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -502,7 +515,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) if (forkPool_) forkPool_->send(nlohmann::json{{"id", id}, {"command", "kill"}}); forkPoolProcesses_.erase(iter); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); return; } } @@ -510,7 +523,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } auto& proc = process->second; @@ -521,12 +534,12 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) process->second->exit(); processes_.erase(process); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); #endif } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -551,7 +564,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) {"payload", {{"data", data}}}, } ); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); return; } } @@ -560,7 +573,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } @@ -578,11 +591,11 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) else process->second->write(Roar::base64Decode(data) + "\r"); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -608,7 +621,9 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) } ); else - hub->callRemote(responseId, nlohmann::json{{"error", "No fork pool available"}}); + RpcHelper::callRemoteSafely( + *hub, responseId, nlohmann::json{{"error", "No fork pool available"}} + ); return; } } @@ -616,7 +631,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } @@ -629,13 +644,13 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) if (process->second->getState(ProcessAttachedState::ProcessInfo).isPty) { #ifdef _WIN32 - hub->callRemote(responseId, nlohmann::json{{"error", "Not implemented"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Not implemented"}}); #else auto& pty = process->second->getState(ProcessAttachedState::PseudoConsole); const auto procs = pty.listProcessesUnderPty(); if (procs.empty()) { - hub->callRemote(responseId, nlohmann::json{{"error", "No processes found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "No processes found"}}); return; } nlohmann::json j = nlohmann::json::object(); @@ -651,17 +666,17 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) {"cmdline", p.cmdline}, }); } - hub->callRemote(responseId, j); + RpcHelper::callRemoteSafely(*hub, responseId, j); #endif } else { - hub->callRemote(responseId, nlohmann::json{{"error", "Process is not a PTY"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process is not a PTY"}}); } } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -688,7 +703,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) {"payload", {{"cols", cols}, {"rows", rows}}}, } ); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); return; } } @@ -697,7 +712,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } @@ -718,11 +733,11 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) process->second->signal(SIGWINCH); #endif - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index aec08ab2..ca005d94 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -1548,7 +1549,8 @@ void Session::registerRpcSftpRecomputeSyncDiff() { // Matches the OperationQueue::rpcName scheme // the frontend OperationQueue listens on. - parent->hub_->callRemote( + RpcHelper::callRemoteSafely( + *parent->hub_, fmt::format("OperationQueue::{}::onSyncDiffProgress", parent->id_.value()), syncSessionId, std::to_string(compared) diff --git a/backend/source/backend/session_manager.cpp b/backend/source/backend/session_manager.cpp index e8005fcb..2cf70bfd 100644 --- a/backend/source/backend/session_manager.cpp +++ b/backend/source/backend/session_manager.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -193,7 +194,9 @@ void SessionManager::addSession( { try { - hub_->callRemote(fmt::format("Session::{}::onDisconnect", sessionId.value())); + RpcHelper::callRemoteSafely( + *hub_, fmt::format("Session::{}::onDisconnect", sessionId.value()) + ); } catch (const std::exception& e) { diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 3beaadf8..f3aa065d 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -249,7 +250,8 @@ auto OperationQueue::makeScanProgressCallback(std::string_view eventName, Ids::O auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::ScanProgress{ .operationId = operationId, @@ -280,7 +282,7 @@ auto OperationQueue::makeBulkProgressCallback(std::string_view eventName, Ids::O auto self = weak.lock(); if (!self) return; - self->hub_->callRemote(name, progress); + RpcHelper::callRemoteSafely(*self->hub_, name, progress); } ); }; @@ -345,7 +347,8 @@ void OperationQueue::moveOperation(Ids::OperationId operationId, std::size_t new // on this to apply or discard the user's intended reorder. auto emitResolution = [&self, &operationId](std::size_t resolvedIndex, bool applied) { - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, self->rpcName("onOperationsReordered"), SharedData::OperationsReordered{ .operationId = operationId, @@ -437,7 +440,7 @@ void OperationQueue::completeOperation(SharedData::OperationCompleted&& operatio if (self->completionObserver_) self->completionObserver_(operationCompleted); - self->hub_->callRemote(self->rpcName("onOperationCompleted"), std::move(operationCompleted)); + RpcHelper::callRemoteSafely(*self->hub_, self->rpcName("onOperationCompleted"), operationCompleted); } ); } @@ -737,7 +740,8 @@ std::expected OperationQueue::addDownloadOperation( auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::TransferProgress{ .operationId = operationId, @@ -755,7 +759,8 @@ std::expected OperationQueue::addDownloadOperation( targetQueue.emplace_back(operationId, std::make_unique(sftp, std::move(opts))); Log::info("Calling OperationQueue::{}::onOperationAdded", sessionId_.value()); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -800,7 +805,8 @@ std::expected OperationQueue::addDownloadOperation( auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; targetQueue.emplace_back(operationId, std::move(bulk)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -948,7 +954,8 @@ std::expected OperationQueue::addArchiveDownloadOperatio auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::TransferProgress{ .operationId = operationId, @@ -989,7 +996,8 @@ std::expected OperationQueue::addArchiveDownloadOperatio totalPayloadBytes, directoryRootPaths.size() ); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1075,7 +1083,8 @@ std::expected OperationQueue::addArchiveUploadOperation( auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::TransferProgress{ .operationId = operationId, @@ -1110,7 +1119,8 @@ std::expected OperationQueue::addArchiveUploadOperation( estimatedTotalBytes, directoryRootPaths.size() ); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1228,7 +1238,8 @@ std::size_t OperationQueue::addBulkDownloadOperation( bulk->setPrescannedFileList(std::move(files)); targetQueue.emplace_back(bulkOpId, std::move(bulk)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = bulkOpId, @@ -1317,7 +1328,8 @@ std::expected OperationQueue::addUploadOperation( auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::TransferProgress{ .operationId = operationId, @@ -1334,7 +1346,8 @@ std::expected OperationQueue::addUploadOperation( auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; targetQueue.emplace_back(operationId, std::make_unique(sftp, std::move(opts))); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1373,7 +1386,8 @@ std::expected OperationQueue::addUploadOperation( auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; targetQueue.emplace_back(operationId, std::move(bulk)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1498,7 +1512,8 @@ std::size_t OperationQueue::addBulkUploadOperation( bulk->setPrescannedFileList(std::move(files)); targetQueue.emplace_back(bulkOpId, std::move(bulk)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = bulkOpId, @@ -1757,7 +1772,8 @@ std::expected OperationQueue::addLocalDeleteOperation( auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::BulkDeleteProgress{ .operationId = operationId, @@ -1771,7 +1787,8 @@ std::expected OperationQueue::addLocalDeleteOperation( }) ); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1844,7 +1861,8 @@ std::expected OperationQueue::addDeleteOperation( auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::BulkDeleteProgress{ .operationId = operationId, @@ -1876,7 +1894,8 @@ std::expected OperationQueue::addDeleteOperation( targetQueue.emplace_back(operationId, std::move(operation)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1924,7 +1943,8 @@ std::size_t OperationQueue::addBulkDeleteOperation( auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::BulkDeleteProgress{ .operationId = bulkOperationId, @@ -1987,7 +2007,8 @@ std::size_t OperationQueue::addBulkDeleteOperation( operation->setTargets(std::move(fileTargets)); targetQueue.emplace_back(bulkOperationId, std::move(operation)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = bulkOperationId, @@ -2031,7 +2052,8 @@ std::expected OperationQueue::addRenameOperation( auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; targetQueue.emplace_back(operationId, std::move(operation)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -2082,8 +2104,8 @@ void OperationQueue::addSyncScanOperation( return; if (!tree) { - self->hub_->callRemote( - self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/false, tree.error() + RpcHelper::callRemoteSafely( + *self->hub_, self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/false, tree.error() ); return; } @@ -2094,7 +2116,9 @@ void OperationQueue::addSyncScanOperation( sess->setRemoteTreeInStrand(std::move(tree)); } ); - self->hub_->callRemote(self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/false); + RpcHelper::callRemoteSafely( + *self->hub_, self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/false + ); }; syncScanCallbacks_[localScanId.value()] = @@ -2108,8 +2132,8 @@ void OperationQueue::addSyncScanOperation( return; if (!tree) { - self->hub_->callRemote( - self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/true, tree.error() + RpcHelper::callRemoteSafely( + *self->hub_, self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/true, tree.error() ); return; } @@ -2120,7 +2144,7 @@ void OperationQueue::addSyncScanOperation( sess->setLocalTreeInStrand(std::move(tree)); } ); - self->hub_->callRemote(self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/true); + RpcHelper::callRemoteSafely(*self->hub_, self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/true); }; // Queue remote scan — build-tree mode. @@ -2149,7 +2173,8 @@ void OperationQueue::addSyncScanOperation( ); priorityOperations_.emplace_back(remoteScanId, std::move(remoteScan)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = remoteScanId, @@ -2171,7 +2196,8 @@ void OperationQueue::addSyncScanOperation( }); priorityOperations_.emplace_back(localScanId, std::move(localScan)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = localScanId, diff --git a/backend/source/backend/windows/windows_drag_drop.cpp b/backend/source/backend/windows/windows_drag_drop.cpp index 2cbe6297..e71dbdfa 100644 --- a/backend/source/backend/windows/windows_drag_drop.cpp +++ b/backend/source/backend/windows/windows_drag_drop.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -141,7 +142,7 @@ EnableWindowsDragDrop::EnableWindowsDragDrop(Nui::Window& wnd, Nui::RpcHub& hub) if (messageJson.contains("subdir")) reply["subdir"] = messageJson["subdir"]; - hub.callRemote("SessionArea::onFilesDropped", reply); + RpcHelper::callRemoteSafely(hub, "SessionArea::onFilesDropped", reply); } catch (std::exception const& exc) { diff --git a/log/source/log/logger_backend.cpp b/log/source/log/logger_backend.cpp index 556d67e7..9849cdd0 100644 --- a/log/source/log/logger_backend.cpp +++ b/log/source/log/logger_backend.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include @@ -29,16 +31,17 @@ namespace Log // ---- File sink (optional) ---- if (!disableFileLog) { - fs::path log_dir = - fmt::format(fmt::runtime(sinkDir.generic_string()), fmt::arg("appName", Constants::appName)); + const auto logDirectory = Paths::nativeFromUtf8( + fmt::format(fmt::runtime(Paths::nativeToUtf8(sinkDir)), fmt::arg("appName", Constants::appName)) + ); std::error_code ec; - fs::create_directories(log_dir, ec); // best-effort + fs::create_directories(logDirectory, ec); // best-effort - fs::path log_file = log_dir / (std::string{Constants::appName} + ".log"); + const auto logFile = logDirectory / Paths::nativeFromUtf8(fmt::format("{}.log", Constants::appName)); - // 5 MB per file, keep 3 rotated files + // 5 MB per file, keep 3 rotated files. The native name is wide on Windows (SPDLOG_WCHAR_FILENAMES). auto file_sink = - std::make_shared(log_file.string(), 5 * 1024 * 1024, 3, true); + std::make_shared(logFile.native(), 5 * 1024 * 1024, 3, true); file_sink->set_level(spdlogLevel); dist_sink->add_sink(file_sink); @@ -135,7 +138,7 @@ namespace Log { try { - hub->callRemote("log", static_cast(level), msg); + hub->callRemote("log", static_cast(level), Paths::sanitizedUtf8(msg)); } catch (const std::exception& e) { diff --git a/utility/include/utility/json_utf8.hpp b/utility/include/utility/json_utf8.hpp new file mode 100644 index 00000000..a5c6698b --- /dev/null +++ b/utility/include/utility/json_utf8.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include + +#include + +#include + +namespace Utility +{ + /** + * @brief @p json with every string and object key that is not valid UTF-8 written with U+FFFD for each invalid + * byte, so dump() never refuses it. Valid text stays as it is. + */ + inline nlohmann::json withValidUtf8(nlohmann::json json) + { + if (json.is_string()) + { + auto const& text = json.get_ref(); + if (!Paths::isValidUtf8(text)) + json = Paths::sanitizedUtf8(text); + return json; + } + if (json.is_array()) + { + for (auto& element : json) + element = withValidUtf8(std::move(element)); + return json; + } + if (json.is_object()) + { + nlohmann::json result = nlohmann::json::object(); + for (auto& [key, value] : json.items()) + result[Paths::sanitizedUtf8(key)] = withValidUtf8(std::move(value)); + return result; + } + return json; + } +} From 15ad7072d0cd26933e6ba3912d7e7312d66bc255 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:00:21 +0200 Subject: [PATCH 029/131] Error texts and local paths reach logs and the UI as UTF-8 Error texts and local paths reach logs, replies, operation errors and the UI as UTF-8: error codes and exceptions go through Utility::errorMessage, paths through nativeToUtf8 and nativeFromUtf8 instead of the narrow string(), which uses the ANSI code page on Windows and throws outside it; language files are read from a stream opened by the native path, environment variables, the user name, known folders and drive labels convert from their wide text on Windows, and the test temporary directory is handed over with the wide environment API --- .../backend/process/json_process_io.hpp | 9 +- backend/include/backend/rpc_helper.hpp | 4 +- .../backend/windows/cotask_mem_string.hpp | 4 +- backend/source/backend/existence_check.cpp | 3 +- .../temp_dir_instance_manager.cpp | 30 +++-- .../file_tracking/temp_dir_instancing.cpp | 15 ++- backend/source/backend/linux/opener_linux.cpp | 23 ++-- backend/source/backend/local_delete_guard.cpp | 1 + backend/source/backend/mac/opener_mac.mm | 11 +- backend/source/backend/main.cpp | 9 +- .../source/backend/process/environment.cpp | 28 +++-- backend/source/backend/process/fork_pool.cpp | 11 +- .../source/backend/process/process_store.cpp | 1 + backend/source/backend/program_options.cpp | 3 +- .../source/backend/pty/posix/process_list.cpp | 4 +- backend/source/backend/pty/posix/pty.cpp | 3 +- backend/source/backend/rpc_filesystem.cpp | 109 ++++-------------- backend/source/backend/rpc_system.cpp | 7 +- backend/source/backend/session.cpp | 61 +++------- backend/source/backend/session_manager.cpp | 17 +-- .../sftp/archive_download_operation.cpp | 11 +- .../backend/sftp/archive_upload_operation.cpp | 5 +- .../backend/sftp/bulk_download_operation.cpp | 23 +++- .../backend/sftp/download_operation.cpp | 15 ++- .../backend/sftp/local_delete_operation.cpp | 28 +---- .../backend/sftp/local_scan_operation.cpp | 22 ++-- .../source/backend/sftp/operation_queue.cpp | 5 +- backend/source/backend/theme_finder.cpp | 12 +- .../backend/windows/windows_drag_drop.cpp | 5 +- .../support/default_places_posix.cpp | 5 +- .../support/default_places_windows.cpp | 14 +-- .../support/windows_drives.cpp | 11 +- paths/include/paths/native.hpp | 1 - .../persistence/state_holder_backend.cpp | 37 +++--- tar-archive/source/tar_archive/archive.cpp | 5 +- .../tar_archive/compression/factory.cpp | 3 +- .../tar_archive/compression/raw_stream.cpp | 13 +-- .../utility/process_temporary_directory.hpp | 13 ++- .../include/utility/temporary_directory.hpp | 15 ++- utility/source/utility/node/node.cpp | 24 ++-- utility/source/utility/resources.cpp | 26 +++-- utility/source/utility/shell_integration.cpp | 3 +- 42 files changed, 328 insertions(+), 321 deletions(-) diff --git a/backend/include/backend/process/json_process_io.hpp b/backend/include/backend/process/json_process_io.hpp index f67ecafd..8d6ba5ee 100644 --- a/backend/include/backend/process/json_process_io.hpp +++ b/backend/include/backend/process/json_process_io.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -90,7 +91,7 @@ class JsonFraming } catch (std::exception const& exc) { - onResult(std::unexpected(std::string{"JSON parse error: "} + exc.what())); + onResult(std::unexpected(fmt::format("JSON parse error: {}", Utility::errorMessage(exc)))); return false; } } @@ -168,7 +169,7 @@ class JsonProcessIo : public JsonFraming boost::asio::buffer(*frame), [this, frame](boost::system::error_code err, std::size_t bytes) { - spdlog::debug("Sent {} byte(s), error: {}", bytes, err ? err.message() : "none"); + spdlog::debug("Sent {} byte(s), error: {}", bytes, err ? Utility::errorMessage(err) : "none"); if (err) { // The callbacks may own this object, so nothing here is touched after they ran. @@ -194,8 +195,8 @@ class JsonProcessIo : public JsonFraming { if (ec) { - spdlog::error("Error reading from input: {}", ec.message()); - onResult_(std::unexpected(ec.message())); + spdlog::error("Error reading from input: {}", Utility::errorMessage(ec)); + onResult_(std::unexpected(Utility::errorMessage(ec))); return; } readBufferAccum_.append(readBufferCurrent_, 0, bytes_transferred); diff --git a/backend/include/backend/rpc_helper.hpp b/backend/include/backend/rpc_helper.hpp index 0c9ba1cd..08575bbc 100644 --- a/backend/include/backend/rpc_helper.hpp +++ b/backend/include/backend/rpc_helper.hpp @@ -300,7 +300,7 @@ namespace RpcHelper } catch (const std::exception& e) { - (*rpcOnce)({{"error", e.what()}}); + (*rpcOnce)({{"error", Utility::errorMessage(e)}}); } }; } @@ -418,7 +418,7 @@ namespace RpcHelper [func = std::forward(func)](auto const& ec) { if (ec) - Log::info("Timer canceled in within_strand_do_delayed: {}", ec.message()); + Log::info("Timer canceled in within_strand_do_delayed: {}", Utility::errorMessage(ec)); // Run func anyway, if its because of a shutdown, it will stop early. func(); diff --git a/backend/include/backend/windows/cotask_mem_string.hpp b/backend/include/backend/windows/cotask_mem_string.hpp index 68db1c9a..aa6bc28c 100644 --- a/backend/include/backend/windows/cotask_mem_string.hpp +++ b/backend/include/backend/windows/cotask_mem_string.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -55,7 +55,7 @@ class CoTaskMemString std::string toUtf8String() const { - return Nui::utf16ToUtf8(toStandardString()); + return Paths::wideToUtf8(toStandardString()); } private: diff --git a/backend/source/backend/existence_check.cpp b/backend/source/backend/existence_check.cpp index 85b293ea..403e588e 100644 --- a/backend/source/backend/existence_check.cpp +++ b/backend/source/backend/existence_check.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -26,7 +27,7 @@ std::expected, std::string> localPathsExist(std::vector +#include #include #include #include @@ -63,7 +64,7 @@ namespace FileTracking { namespace fs = std::filesystem; - auto const dirName = dirPath.filename().string(); + auto const dirName = Paths::nativeToUtf8(dirPath.filename()); for (auto const& sid : skipIds) if (sid == dirName) return false; @@ -151,11 +152,18 @@ namespace FileTracking if (!isOrphanedDir(dirPath, skipIds)) continue; - Log::info("forceCleanOrphanedInstanceDirs: removing orphaned instance '{}'", dirPath.filename().string()); + Log::info( + "forceCleanOrphanedInstanceDirs: removing orphaned instance '{}'", + Paths::nativeToUtf8(dirPath.filename()) + ); std::error_code rmErr; Utility::removeAllNoFollow(dirPath, rmErr); if (rmErr) - Log::warn("forceCleanOrphanedInstanceDirs: failed to remove '{}': {}", dirPath.filename().string(), rmErr.message()); + Log::warn( + "forceCleanOrphanedInstanceDirs: failed to remove '{}': {}", + Paths::nativeToUtf8(dirPath.filename()), + Utility::errorMessage(rmErr) + ); else ++removed; } @@ -182,7 +190,7 @@ namespace FileTracking continue; auto const& dirPath = entry.path(); - auto const dirName = dirPath.filename().string(); + auto const dirName = Paths::nativeToUtf8(dirPath.filename()); bool skip = false; for (auto const& sid : skipIds) @@ -228,7 +236,7 @@ namespace FileTracking Log::warn( "scanAndCleanDeadInstanceDirs: could not write deadAt for '{}': {}", dirName, - exc.what() + Utility::errorMessage(exc) ); } }; @@ -254,7 +262,7 @@ namespace FileTracking Log::warn( "scanAndCleanDeadInstanceDirs: error reading metadata for '{}': {}", dirName, - exc.what() + Utility::errorMessage(exc) ); } @@ -272,7 +280,7 @@ namespace FileTracking std::error_code rmErr; Utility::removeAllNoFollow(dirPath, rmErr); if (rmErr) - Log::warn("Failed to remove dead instance '{}': {}", dirName, rmErr.message()); + Log::warn("Failed to remove dead instance '{}': {}", dirName, Utility::errorMessage(rmErr)); } } } // namespace @@ -434,7 +442,11 @@ namespace FileTracking }); } if (walkError) - Log::warn("FileTracking::fileStates: walking '{}' failed: {}", instanceId, walkError.message()); + Log::warn( + "FileTracking::fileStates: walking '{}' failed: {}", + instanceId, + Utility::errorMessage(walkError) + ); reply({{"success", true}, {"files", std::move(files)}}); } ); @@ -483,7 +495,7 @@ namespace FileTracking } catch (std::exception const& exc) { - Log::error("TempDirInstanceManager::createInstance failed: {}", exc.what()); + Log::error("TempDirInstanceManager::createInstance failed: {}", Utility::errorMessage(exc)); return nullptr; } } diff --git a/backend/source/backend/file_tracking/temp_dir_instancing.cpp b/backend/source/backend/file_tracking/temp_dir_instancing.cpp index 54a2b949..a29c0088 100644 --- a/backend/source/backend/file_tracking/temp_dir_instancing.cpp +++ b/backend/source/backend/file_tracking/temp_dir_instancing.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -37,7 +38,9 @@ namespace FileTracking std::error_code err; std::filesystem::create_directories(dir, err); if (err) - throw std::runtime_error{"Failed to create instance directory: " + err.message()}; + throw std::runtime_error{ + fmt::format("Failed to create instance directory: {}", Utility::errorMessage(err)) + }; return dir; } @@ -265,7 +268,11 @@ namespace FileTracking std::error_code err; Utility::removeAllNoFollow(impl_->instanceDir, err); if (err) - Log::warn("cleanupNow: failed to remove '{}': {}", impl_->instanceDir.string(), err.message()); + Log::warn( + "cleanupNow: failed to remove '{}': {}", + Paths::nativeToUtf8(impl_->instanceDir), + Utility::errorMessage(err) + ); impl_->valid = false; } @@ -306,7 +313,9 @@ namespace FileTracking } catch (std::exception const& exc) { - Log::error("writeDeadTimestamp: failed for instance '{}': {}", impl_->instanceId, exc.what()); + Log::error( + "writeDeadTimestamp: failed for instance '{}': {}", impl_->instanceId, Utility::errorMessage(exc) + ); } } } diff --git a/backend/source/backend/linux/opener_linux.cpp b/backend/source/backend/linux/opener_linux.cpp index 19e87821..f33be552 100644 --- a/backend/source/backend/linux/opener_linux.cpp +++ b/backend/source/backend/linux/opener_linux.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -419,7 +420,7 @@ std::optional Opener::riskOf(std::filesystem::path const& path) con return std::string{"backend.opener.risk.program"}; // Types a desktop starts as programs instead of showing them. static constexpr std::string_view runnable[] = {".desktop", ".appimage", ".run", ".jar"}; - auto extension = path.extension().string(); + auto extension = Paths::nativeToUtf8(path.extension()); std::ranges::transform(extension, extension.begin(), [](unsigned char character) { return std::tolower(character); }); if (std::ranges::find(runnable, std::string_view{extension}) != std::end(runnable)) return std::string{"backend.opener.risk.dangerousType"}; @@ -428,7 +429,12 @@ std::optional Opener::riskOf(std::filesystem::path const& path) con std::expected Opener::openFile(std::filesystem::path const& path, bool openWith) { - Log::info("Opener: openFile path='{}' openWith={} parentWindow='{}'", path.string(), openWith, impl_->parentWindow); + Log::info( + "Opener: openFile path='{}' openWith={} parentWindow='{}'", + Paths::nativeToUtf8(path), + openWith, + impl_->parentWindow + ); if (!impl_->connection) { @@ -439,7 +445,7 @@ std::expected Opener::openFile(std::filesystem::path const& p Utility::FdGuard fd{::open(path.c_str(), O_RDONLY)}; if (!fd.valid()) { - Log::error("Opener: failed to open fd for '{}': {}", path.string(), std::strerror(errno)); + Log::error("Opener: failed to open fd for '{}': {}", Paths::nativeToUtf8(path), std::strerror(errno)); return std::unexpected{Utility::localizedMessage("backend.opener.openFileFailed", std::strerror(errno))}; } @@ -496,8 +502,7 @@ SharedData::OpenerCapabilities Opener::capabilities() const std::expected Opener::openInFileManager(std::filesystem::path const& path) { - Log::info( - "Opener: openInFileManager path='{}' parentWindow='{}'", path.string(), impl_->parentWindow); + Log::info("Opener: openInFileManager path='{}' parentWindow='{}'", Paths::nativeToUtf8(path), impl_->parentWindow); if (!impl_->connection) { @@ -512,7 +517,11 @@ std::expected Opener::openInFileManager(std::filesystem::path { auto result = launchDefaultForPath(path); if (!result) - Log::error("Opener: g_app_info_launch_default_for_uri failed for '{}': {}", path.string(), result.error()); + Log::error( + "Opener: g_app_info_launch_default_for_uri failed for '{}': {}", + Paths::nativeToUtf8(path), + result.error() + ); return result; } @@ -522,7 +531,7 @@ std::expected Opener::openInFileManager(std::filesystem::path Utility::FdGuard fd{::open(path.c_str(), O_RDONLY)}; if (!fd.valid()) { - Log::error("Opener: failed to open fd for '{}': {}", path.string(), std::strerror(errno)); + Log::error("Opener: failed to open fd for '{}': {}", Paths::nativeToUtf8(path), std::strerror(errno)); return std::unexpected{Utility::localizedMessage("backend.opener.openFileFailed", std::strerror(errno))}; } diff --git a/backend/source/backend/local_delete_guard.cpp b/backend/source/backend/local_delete_guard.cpp index 3643013e..51212eab 100644 --- a/backend/source/backend/local_delete_guard.cpp +++ b/backend/source/backend/local_delete_guard.cpp @@ -1,4 +1,5 @@ #include +#include #include #include diff --git a/backend/source/backend/mac/opener_mac.mm b/backend/source/backend/mac/opener_mac.mm index 215d7e96..b6d44451 100644 --- a/backend/source/backend/mac/opener_mac.mm +++ b/backend/source/backend/mac/opener_mac.mm @@ -1,4 +1,5 @@ #include +#include #include #include @@ -82,7 +83,7 @@ void chooseApplicationAndOpen(NSURL* url) static constexpr std::string_view runnable[] = { ".command", ".tool", ".terminal", ".pkg", ".mpkg", ".workflow", ".scpt", ".applescript", ".jar", ".webloc", }; - auto extension = path.extension().string(); + auto extension = Paths::nativeToUtf8(path.extension()); std::ranges::transform(extension, extension.begin(), [](unsigned char character) { return std::tolower(character); }); if (std::ranges::find(runnable, std::string_view{extension}) != std::end(runnable)) return std::string{"backend.opener.risk.dangerousType"}; @@ -104,7 +105,7 @@ void chooseApplicationAndOpen(NSURL* url) std::expected Opener::openFile(std::filesystem::path const& path, bool openWith) { - Log::info("Opener: openFile path='{}' openWith={}", path.string(), openWith); + Log::info("Opener: openFile path='{}' openWith={}", Paths::nativeToUtf8(path), openWith); std::error_code existsError; if (!path.is_absolute() || !std::filesystem::exists(path, existsError)) @@ -123,7 +124,7 @@ void chooseApplicationAndOpen(NSURL* url) } if (![[NSWorkspace sharedWorkspace] openURL:url]) - return std::unexpected{Utility::localizedMessage("backend.opener.mac.openFailed", path.string())}; + return std::unexpected{Utility::localizedMessage("backend.opener.mac.openFailed", Paths::nativeToUtf8(path))}; } return {}; } @@ -136,7 +137,7 @@ void chooseApplicationAndOpen(NSURL* url) std::expected Opener::openInFileManager(std::filesystem::path const& path) { - Log::info("Opener: openInFileManager path='{}'", path.string()); + Log::info("Opener: openInFileManager path='{}'", Paths::nativeToUtf8(path)); std::error_code directoryCheckError; const bool isDirectory = std::filesystem::is_directory(path, directoryCheckError); @@ -148,7 +149,7 @@ void chooseApplicationAndOpen(NSURL* url) { NSString* const pathString = [NSString stringWithUTF8String:path.c_str()]; if (![workspace selectFile:nil inFileViewerRootedAtPath:pathString]) - return std::unexpected{Utility::localizedMessage("backend.opener.mac.openFailed", path.string())}; + return std::unexpected{Utility::localizedMessage("backend.opener.mac.openFailed", Paths::nativeToUtf8(path))}; return {}; } [workspace activateFileViewerSelectingURLs:@[fileUrl(path, false)]]; diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index b9e69923..6d47f6db 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -140,7 +141,7 @@ namespace // Check if file exists and return 404 if not if (!std::filesystem::exists(file)) { - Log::error("File not found: '{}'", file.string()); + Log::error("File not found: '{}'", Paths::nativeToUtf8(file)); return CustomSchemeResponse{ .statusCode = 404, .reasonPhrase = "Not Found", @@ -150,11 +151,11 @@ namespace // Do not forget to allow CORS {"Access-Control-Allow-Origin"s, "*"s}, }, - .body = "Not Found: "s + file.string(), + .body = fmt::format("Not Found: {}", Paths::nativeToUtf8(file)), }; } - Log::debug("Serving file: '{}'", file.string()); + Log::debug("Serving file: '{}'", Paths::nativeToUtf8(file)); // Stream the file body via bodyFile — avoids loading the whole file into memory // (matters for large assets like DWARF blobs). Empty files yield a 204. @@ -165,7 +166,7 @@ namespace // Roar's MIME table has no entry for .wasm; emit application/wasm explicitly so that // WebAssembly.instantiateStreaming() accepts the response and compiles the module as // bytes arrive (otherwise the browser falls back to buffering the full binary first). - const auto ext = file.extension().string(); + const auto ext = Paths::nativeToUtf8(file.extension()); const auto mime = (ext == ".wasm") ? std::string{"application/wasm"} : Roar::extensionToMime(ext).value_or("application/octet-stream"); diff --git a/backend/source/backend/process/environment.cpp b/backend/source/backend/process/environment.cpp index 3d4b4286..feb94a32 100644 --- a/backend/source/backend/process/environment.cpp +++ b/backend/source/backend/process/environment.cpp @@ -1,8 +1,10 @@ #include #include +#include #include +#include #include #include #include @@ -13,6 +15,20 @@ namespace { constexpr std::string_view originalValuePrefix = "NUI_SFTP_APPIMAGE_ORIGINAL_"; constexpr std::string_view unsetMarkerPrefix = "NUI_SFTP_APPIMAGE_UNSET_"; + + /** + * @brief The text of an environment key or value as UTF-8, converted from the wide text on Windows. + */ + template + std::string utf8Of(ViewT const& view) + { + const auto native = view.native(); +#ifdef _WIN32 + return Paths::wideToUtf8(std::wstring_view{native.data(), native.size()}); +#else + return std::string{native.data(), native.size()}; +#endif + } } Environment::Environment( @@ -38,17 +54,7 @@ void Environment::loadFromCurrent() for (auto iter = currentEnv.begin(); iter != currentEnv.end(); ++iter) { auto deref = *iter; - environment_.emplace( - [&]() { - if (deref.key().size() > 0) - return deref.key().string(); - return std::string{}; - }(), - [&]() { - if (deref.value().size() > 0) - return deref.value().string(); - return std::string{}; - }()); + environment_.emplace(utf8Of(deref.key()), utf8Of(deref.value())); } restoreHostEnvironment(); } diff --git a/backend/source/backend/process/fork_pool.cpp b/backend/source/backend/process/fork_pool.cpp index 4d9336f9..4f410472 100644 --- a/backend/source/backend/process/fork_pool.cpp +++ b/backend/source/backend/process/fork_pool.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -249,8 +250,10 @@ namespace } catch (std::exception const& exc) { - spdlog::error("[worker] command exception: cmd='{}' id='{}' what='{}'", cmd, procId, exc.what()); - sendJson({{"id", procId}, {"type", "error"}, {"message", exc.what()}}); + spdlog::error( + "[worker] command exception: cmd='{}' id='{}' what='{}'", cmd, procId, Utility::errorMessage(exc) + ); + sendJson({{"id", procId}, {"type", "error"}, {"message", Utility::errorMessage(exc)}}); } } @@ -584,9 +587,7 @@ namespace auto logPath = Nui::resolvePath("%state_home2%/nui-sftp/logs/worker.log"); std::error_code mkdirEc; std::filesystem::create_directories(logPath.parent_path(), mkdirEc); - return std::make_shared( - logPath.string(), 2 * 1024 * 1024, 3, true - ); + return std::make_shared(logPath.native(), 2 * 1024 * 1024, 3, true); } catch (std::exception const&) { diff --git a/backend/source/backend/process/process_store.cpp b/backend/source/backend/process/process_store.cpp index 78aa3c16..34b89523 100644 --- a/backend/source/backend/process/process_store.cpp +++ b/backend/source/backend/process/process_store.cpp @@ -1,4 +1,5 @@ #include +#include #include #include diff --git a/backend/source/backend/program_options.cpp b/backend/source/backend/program_options.cpp index 77b858eb..3e1cc18f 100644 --- a/backend/source/backend/program_options.cpp +++ b/backend/source/backend/program_options.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -34,7 +35,7 @@ std::optional parseProgramOptions(int argc, char const* const* a } catch (std::exception& e) { - std::cerr << "Error parsing program options: " << e.what() << "\n"; + std::cerr << "Error parsing program options: " << Utility::errorMessage(e) << "\n"; return std::nullopt; } } \ No newline at end of file diff --git a/backend/source/backend/pty/posix/process_list.cpp b/backend/source/backend/pty/posix/process_list.cpp index f3701cf9..7ebbdeb2 100644 --- a/backend/source/backend/pty/posix/process_list.cpp +++ b/backend/source/backend/pty/posix/process_list.cpp @@ -93,7 +93,7 @@ namespace PTY iterator.increment(iterationError)) { auto const& entry = *iterator; - const auto id = entry.path().filename().string(); + const auto id = entry.path().filename().native(); if (id.empty() || !std::all_of( id.begin(), @@ -110,7 +110,7 @@ namespace PTY // Unreadable entries (other users' processes) fail here and are skipped. std::error_code linkError; const auto target = std::filesystem::read_symlink(entry.path() / "fd" / "0", linkError); - if (linkError || target.string() != terminalName) + if (linkError || target.native() != terminalName) continue; std::ifstream cmdlineFile{entry.path() / "cmdline"}; diff --git a/backend/source/backend/pty/posix/pty.cpp b/backend/source/backend/pty/posix/pty.cpp index e385e74a..df49b8bf 100644 --- a/backend/source/backend/pty/posix/pty.cpp +++ b/backend/source/backend/pty/posix/pty.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -347,7 +348,7 @@ namespace PTY const boost::system::error_code& error ) { - Log::error("Failed to launch the process on the pty: {}", error.message()); + Log::error("Failed to launch the process on the pty: {}", Utility::errorMessage(error)); terminal_->impl_->closeInput(); close(terminal_->impl_->master); diff --git a/backend/source/backend/rpc_filesystem.cpp b/backend/source/backend/rpc_filesystem.cpp index 93b9ecb1..615375b3 100644 --- a/backend/source/backend/rpc_filesystem.cpp +++ b/backend/source/backend/rpc_filesystem.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -196,66 +197,6 @@ RpcFilesystem::RpcFilesystem( registerOpenerCapabilities(); } -void RpcFilesystem::registerRemove() -{ - on("RpcFilesystem::remove") - .perform( - [this](RpcHelper::RpcOnce&& reply, nlohmann::json const& parameters) - { - if (options_.preventDeletion) - { - Log::warn("RpcFilesystem::remove called but deletion is prevented by configuration."); - return reply.error("backend.rpcFilesystem.deletionPrevented"); - } - - Log::info("RpcFilesystem::remove called with parameters: {}", parameters.dump(4)); - - if (!RpcHelper::ParameterVerifyView{reply, "RpcFilesystem::remove", parameters}.hasValueDeep("path")) - { - Log::error("RpcFilesystem::remove missing required parameter 'path'."); - return reply.error("Missing required parameter 'path'."); - } - - const bool recursive = parameters.value("recursive", false); - const auto pathUtf8 = parameters["path"].get(); - const auto target = localDeleteTarget(pathUtf8, localHomeDirectories()); - if (!target) - { - Log::error("RpcFilesystem::remove refused: {}", target.error()); - return reply.error(Utility::localizedMessage("backend.rpcFilesystem.removeRefused", pathUtf8)); - } - const auto path = target->native(); - - if (!recursive) - { - std::error_code ec; - if (std::filesystem::remove(path, ec)) - { - Log::info("Successfully removed file: {}", pathUtf8); - return reply({{"success", true}}); - } - - Log::error("Failed to remove file: {}: {}", pathUtf8, ec.message()); - return reply.error(ec.message()); - } - - std::error_code ec; - const auto filesRemoved = Utility::removeAllNoFollow(path, ec); - if (ec) - { - Log::error("Failed to recursively remove path: {}: {}", pathUtf8, ec.message()); - return reply.error(ec.message()); - } - Log::info( - "Successfully recursively removed path: {} ({} files/directories removed)", - pathUtf8, - filesRemoved - ); - return reply({{"success", true}, {"removedCount", filesRemoved}}); - } - ); -} - void RpcFilesystem::registerRemoveMultiple() { on("RpcFilesystem::removeSome") @@ -281,24 +222,10 @@ void RpcFilesystem::registerRemoveMultiple() const auto removal = removeLocalItems(pathsUtf8, localHomeDirectories()); if (!removal) { - const auto path = target.native(); - const auto& pathUtf8 = target.toUtf8(); - std::error_code ec; - if (recursive) - { - removedCount += Utility::removeAllNoFollow(path, ec); - } - else - { - if (std::filesystem::remove(path, ec)) - ++removedCount; - } - - if (ec) - { - Log::error("Failed to remove path '{}': {}", pathUtf8, ec.message()); - failures.push_back(fmt::format("{}: {}", pathUtf8, ec.message())); - } + Log::error("RpcFilesystem::removeSome refused: {}", removal.error().reason); + return reply.error( + Utility::localizedMessage("backend.rpcFilesystem.removeRefused", removal.error().path) + ); } if (!failures.empty()) { @@ -401,9 +328,11 @@ void RpcFilesystem::registerListFiles() if (ec) { Log::error( - "Failed to list files in directory '{}': {}", Paths::nativeToUtf8(directoryPath), ec.message() + "Failed to list files in directory '{}': {}", + Paths::nativeToUtf8(directoryPath), + Utility::errorMessage(ec) ); - return reply.error(ec.message()); + return reply.error(Utility::errorMessage(ec)); } #ifndef _WIN32 @@ -484,8 +413,8 @@ void RpcFilesystem::registerCreateFile() std::ofstream fileStream(filePath, std::ios_base::noreplace | std::ios_base::binary); if (!fileStream) { - Log::error("Failed to create file '{}': {}", filePathUtf8, ec.message()); - return reply.error(ec.message()); + Log::error("Failed to create file '{}': {}", filePathUtf8, Utility::errorMessage(ec)); + return reply.error(Utility::errorMessage(ec)); } Log::info("Successfully created file '{}'", filePathUtf8); @@ -527,8 +456,8 @@ void RpcFilesystem::registerCreateDirectory() std::filesystem::create_directory(directoryPath, ec); if (ec) { - Log::error("Failed to create directory '{}': {}", directoryPathUtf8, ec.message()); - return reply.error(ec.message()); + Log::error("Failed to create directory '{}': {}", directoryPathUtf8, Utility::errorMessage(ec)); + return reply.error(Utility::errorMessage(ec)); } Log::info("Successfully created directory '{}'", directoryPathUtf8); @@ -570,8 +499,12 @@ void RpcFilesystem::registerProperties() const auto symStatus = std::filesystem::symlink_status(path, symEc); if (symEc) { - Log::error("Failed to get properties for path '{}': {}", Paths::nativeToUtf8(path), symEc.message()); - return reply.error(symEc.message()); + Log::error( + "Failed to get properties for path '{}': {}", + Paths::nativeToUtf8(path), + Utility::errorMessage(symEc) + ); + return reply.error(Utility::errorMessage(symEc)); } entry.type = SharedData::fileTypeFromStdFilesystemType(symStatus.type()); entry.permissions = symStatus.permissions(); @@ -679,8 +612,8 @@ void RpcFilesystem::registerDoesExist() const auto status = std::filesystem::exists(*path, ec); if (ec) { - Log::error("Failed to get properties for path '{}': {}", pathUtf8, ec.message()); - return reply.error(ec.message()); + Log::error("Failed to get properties for path '{}': {}", pathUtf8, Utility::errorMessage(ec)); + return reply.error(Utility::errorMessage(ec)); } Log::info("Successfully retrieved existence for path '{}'", pathUtf8); diff --git a/backend/source/backend/rpc_system.cpp b/backend/source/backend/rpc_system.cpp index aba3024c..ecc983c5 100644 --- a/backend/source/backend/rpc_system.cpp +++ b/backend/source/backend/rpc_system.cpp @@ -1,5 +1,6 @@ #include #include +#include #ifdef _WIN32 # include @@ -48,7 +49,7 @@ std::string RpcSystem::username() const DWORD username_len = UNLEN + 1; if (GetUserNameW(username, &username_len)) { - usernameMemo_ = Nui::utf16ToUtf8(std::wstring(username)); + usernameMemo_ = Paths::wideToUtf8(std::wstring_view{username}); return usernameMemo_; } else @@ -60,14 +61,14 @@ std::string RpcSystem::username() const char username[loginNameMaximum]; if (getlogin_r(username, sizeof(username)) == 0) { - usernameMemo_ = std::string(username); + usernameMemo_ = Paths::sanitizedUtf8(username); return usernameMemo_; } else { struct passwd* pwd = getpwuid(getuid()); if (pwd) - return pwd->pw_name; + return Paths::sanitizedUtf8(pwd->pw_name); Log::error("RpcSystem::username: getlogin_r failed"); return "unknown"; } diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index ca005d94..d2fc30e9 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -52,20 +53,7 @@ namespace * @brief A JSON request from the frontend, or the message to reply with when it does not parse. Paths are * validated while parsing. */ - template - std::expected parseRequest(nlohmann::json const& json, std::string_view what) - { - try - { - return json.get(); - } - catch (std::exception const& exception) - { - return std::unexpected(fmt::format("Invalid {}: {}", what, exception.what())); - } - } - - std::expected, std::string> parseRemotePaths(std::vector const& paths) + nlohmann::json enqueuedReply(std::size_t enqueued) { std::vector result; result.reserve(paths.size()); @@ -996,18 +984,12 @@ void Session::registerRpcSftpAddArchiveDownloadOperation() if (!self) return reply({{"error", "Session no longer exists"}}); - // An invalid remote path fails the parse. - std::vector roots; - try - { - roots = nlohmann::json::parse(rootsJson).get>(); - } - catch (std::exception const& exc) - { - Log::error("addArchiveDownload: failed to parse roots JSON: {}", exc.what()); - return reply({{"error", std::string{"Invalid roots payload: "} + exc.what()}}); - } - auto archivePath = parseLocalPath(localArchivePath); + auto roots = RpcHelper::parseRequest>( + nlohmann::json::parse(rootsJson, nullptr, false), "archive roots" + ); + if (!roots) + return reply.error(roots.error()); + auto archivePath = RpcHelper::parseLocalPath(localArchivePath); if (!archivePath) return reply.error(archivePath.error()); @@ -1631,15 +1613,11 @@ void Session::registerRpcSftpExecuteSyncSelection() if (!self || !self->operationQueue_) return reply.error("Session no longer exists"); - SharedData::Sync::SyncExecutionRequest request{}; - try - { - request = requestJson.get(); - } - catch (std::exception const& exception) - { - return reply.error(fmt::format("Invalid sync request: {}", exception.what())); - } + auto parsed = + RpcHelper::parseRequest(requestJson, "sync request"); + if (!parsed) + return reply.error(parsed.error()); + auto request = *std::move(parsed); auto session = self->operationQueue_->syncSession(request.syncSessionId); if (!session) @@ -1875,16 +1853,9 @@ void Session::registerRpcSftpAddBulkDeleteOperation() if (!self) return reply({{"error", "Session no longer exists"}}); - // Parsed here, so an invalid target is answered with an error instead of no reply at all. - SharedData::BulkDeleteRequest request{}; - try - { - request = requestJson.get(); - } - catch (std::exception const& exception) - { - return reply.error(fmt::format("Invalid bulk delete: {}", exception.what())); - } + auto request = RpcHelper::parseRequest(requestJson, "bulk delete"); + if (!request) + return reply.error(request.error()); self->withSftpChannelDo( Ids::makeChannelId(channelIdString), diff --git a/backend/source/backend/session_manager.cpp b/backend/source/backend/session_manager.cpp index 2cf70bfd..4c010b13 100644 --- a/backend/source/backend/session_manager.cpp +++ b/backend/source/backend/session_manager.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -203,7 +204,9 @@ void SessionManager::addSession( // There should never anything throw here, but lets log just in case and dont let an // exception escape out of this lambda. Log::error( - "Failed to call Session::{}::onDisconnect: {}.", sessionId.value(), e.what() + "Failed to call Session::{}::onDisconnect: {}.", + sessionId.value(), + Utility::errorMessage(e) ); } } @@ -328,8 +331,8 @@ void SessionManager::registerRpcSessionDisconnect() } catch (std::exception const& e) { - Log::error("Error disconnecting to ssh server: {}", e.what()); - return reply({{"error", e.what()}}); + Log::error("Error disconnecting to ssh server: {}", Utility::errorMessage(e)); + return reply({{"error", Utility::errorMessage(e)}}); } } ); @@ -379,8 +382,8 @@ void SessionManager::registerRpcAdoptBulkResume() } catch (std::exception const& e) { - Log::error("Error in adoptBulkResume: {}", e.what()); - return reply({{"error", e.what()}}); + Log::error("Error in adoptBulkResume: {}", Utility::errorMessage(e)); + return reply({{"error", Utility::errorMessage(e)}}); } } ); @@ -404,8 +407,8 @@ void SessionManager::registerRpcDiscardBulkResume() } catch (std::exception const& e) { - Log::error("Error in discardBulkResume: {}", e.what()); - return reply({{"error", e.what()}}); + Log::error("Error in discardBulkResume: {}", Utility::errorMessage(e)); + return reply({{"error", Utility::errorMessage(e)}}); } } ); diff --git a/backend/source/backend/sftp/archive_download_operation.cpp b/backend/source/backend/sftp/archive_download_operation.cpp index 248becdb..43a8f33e 100644 --- a/backend/source/backend/sftp/archive_download_operation.cpp +++ b/backend/source/backend/sftp/archive_download_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -271,8 +272,8 @@ std::expected ArchiveDownloadOperation::prepareInStrand( { Log::error( "ArchiveDownloadOperation: cannot create parents of '{}': {}", - tempPath_.generic_string(), - mkErr.message() + Paths::nativeToUtf8(tempPath_), + Utility::errorMessage(mkErr) ); return std::unexpected(Error{.type = ErrorType::CannotCreateDirectory}); } @@ -291,7 +292,7 @@ std::expected ArchiveDownloadOperation::prepareInStrand( // Opening for writing follows a symlink planted at the temp path, out of the target directory. if (std::error_code linkError{}; std::filesystem::is_symlink(std::filesystem::symlink_status(tempPath_, linkError))) { - Log::error("ArchiveDownloadOperation: temp path '{}' is a symlink.", tempPath_.generic_string()); + Log::error("ArchiveDownloadOperation: temp path '{}' is a symlink.", Paths::nativeToUtf8(tempPath_)); return std::unexpected(Error{.type = ErrorType::OpenFailure, .extraInfo = "The temporary file path is a symlink"}); } @@ -658,7 +659,9 @@ std::expected ArchiveDownloadOperation::finaliseArchiveI if (options_.markAsDownloaded) { if (const auto marked = Utility::markAsDownloaded(tempPath_); marked) - Log::warn("ArchiveDownloadOperation: Cannot mark the archive as downloaded: {}", marked.message()); + Log::warn( + "ArchiveDownloadOperation: Cannot mark the archive as downloaded: {}", Utility::errorMessage(marked) + ); } if (const auto moved = Utility::moveIntoPlace(tempPath_, localArchiveNativePath_, options_.mayOverwrite); !moved) diff --git a/backend/source/backend/sftp/archive_upload_operation.cpp b/backend/source/backend/sftp/archive_upload_operation.cpp index 8ff65196..285585b1 100644 --- a/backend/source/backend/sftp/archive_upload_operation.cpp +++ b/backend/source/backend/sftp/archive_upload_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -92,8 +93,8 @@ namespace { Log::warn( "ArchiveUploadOperation: cannot stat '{}': {}", - localPath.generic_string(), - statErr.message() + Paths::nativeToUtf8(localPath), + Utility::errorMessage(statErr) ); return std::nullopt; } diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index 92839af2..ee190c22 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -50,10 +51,14 @@ BulkDownloadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPath const auto link = Utility::firstLinkBelow(common().localPath.native(), paths.local.native(), error); if (error) { - return std::unexpected(Error{ - .type = ErrorType::FileStatFailed, - .extraInfo = fmt::format("Inspecting local directory: {}: {}", paths.local.toUtf8(), error.message()) - }); + return std::unexpected( + Error{ + .type = ErrorType::FileStatFailed, + .extraInfo = fmt::format( + "Inspecting local directory: {}: {}", paths.local.toUtf8(), Utility::errorMessage(error) + ) + } + ); } if (link) { @@ -76,11 +81,17 @@ BulkDownloadOperation::createLocalDirectory(Paths::LocalPath const& localPath, S std::filesystem::create_directories(path, ec); if (ec) { - Log::error("BulkDownloadOperation: Failed to create local directory: {}: {}", path.string(), ec.message()); + Log::error( + "BulkDownloadOperation: Failed to create local directory: {}: {}", + Paths::nativeToUtf8(path), + Utility::errorMessage(ec) + ); return std::unexpected( Error{ .type = ErrorType::CannotCreateDirectory, - .extraInfo = fmt::format("Creating local directory: {}: {}", path.string(), ec.message()) + .extraInfo = fmt::format( + "Creating local directory: {}: {}", Paths::nativeToUtf8(path), Utility::errorMessage(ec) + ) } ); } diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index e34f6832..3af0751b 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -271,7 +272,7 @@ std::expected DownloadOperation::handleSymlinkIn Log::error( "DownloadOperation: Failed to create parent dirs for symlink '{}': {}.", options_.localPath.toUtf8(), - ec.message() + Utility::errorMessage(ec) ); return std::unexpected(Error{.type = ErrorType::CannotCreateDirectory}); } @@ -427,7 +428,9 @@ DownloadOperation::refuseLinkedPath(std::filesystem::path const& path) const link = path; if (error) { - Log::error("DownloadOperation: Cannot inspect '{}': {}.", Paths::nativeToUtf8(path), error.message()); + Log::error( + "DownloadOperation: Cannot inspect '{}': {}.", Paths::nativeToUtf8(path), Utility::errorMessage(error) + ); return std::unexpected(Error{.type = ErrorType::FileStatFailed, .extraInfo = Paths::nativeToUtf8(path)}); } if (link) @@ -461,7 +464,7 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) Log::error( "DownloadOperation: Failed to create parent directories for '{}': {}.", Paths::nativeToUtf8(tempPath), - mkErr.message() + Utility::errorMessage(mkErr) ); return std::unexpected(Error{.type = ErrorType::CannotCreateDirectory}); } @@ -825,7 +828,11 @@ std::expected DownloadOperation::finalizeInStran if (options_.markAsDownloaded) { if (const auto marked = Utility::markAsDownloaded(tempPath); marked) - Log::warn("DownloadOperation: Cannot mark '{}' as downloaded: {}", Paths::nativeToUtf8(tempPath), marked.message()); + Log::warn( + "DownloadOperation: Cannot mark '{}' as downloaded: {}", + Paths::nativeToUtf8(tempPath), + Utility::errorMessage(marked) + ); } // Times and permissions go on the temp file, so nothing can fail once the destination is replaced. diff --git a/backend/source/backend/sftp/local_delete_operation.cpp b/backend/source/backend/sftp/local_delete_operation.cpp index bbf52ed4..8f522aa3 100644 --- a/backend/source/backend/sftp/local_delete_operation.cpp +++ b/backend/source/backend/sftp/local_delete_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -27,28 +28,11 @@ std::expected Loc const auto total = options_.targets.size(); if (deletedThusFar_ < total) { - auto const& target = options_.targets[total - deletedThusFar_ - 1]; - std::error_code error{}; - if (Utility::isLinkOrJunction(target.native(), error) && !error) - Utility::removeAllNoFollow(target.native(), error); - else if (!error) - std::filesystem::remove(target.native(), error); - // One item that cannot be deleted does not keep the others, its directories then stay as well. - if (error) - { - Log::error("LocalDeleteOperation: Failed to delete '{}': {}", target.toUtf8(), error.message()); - failedEntries_.emplace_back( - target.toUtf8(), - Error{ - .type = ErrorType::DeleteFailed, - .extraInfo = fmt::format("{}: {}", target.toUtf8(), error.message()) - } - ); - } - ++deletedThusFar_; - options_.itemsRemovedProgress(target.toUtf8(), deletedThusFar_, total); - if (deletedThusFar_ < total) - return WorkStatus::MoreWork; + auto const& [path, error] = failedEntries_.front(); + return enterErrorState(Error{ + .type = ErrorType::DeleteFailed, + .extraInfo = fmt::format("{}: {}", path, error.extraInfo.value_or(std::string{})), + }); } // Nothing deleted at all is a failure, some items left behind a partial success. if (total > 0 && failedEntries_.size() == total) diff --git a/backend/source/backend/sftp/local_scan_operation.cpp b/backend/source/backend/sftp/local_scan_operation.cpp index c8fc3809..40c41f43 100644 --- a/backend/source/backend/sftp/local_scan_operation.cpp +++ b/backend/source/backend/sftp/local_scan_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -158,10 +159,11 @@ LocalScanOperation::scanner(Paths::RelativePath const& relative) } catch (const std::filesystem::filesystem_error& e) { - Log::error("LocalScanOperation: Filesystem error while scanning '{}': {}", path.generic_string(), e.what()); - return enterErrorState>( - {.type = ErrorType::FilesystemError, .extraInfo = e.what()} - ); + // A name that is no segment cannot match a rule, the walker rejects it. + const auto segment = Paths::PathSegment::make(name); + if (segment && + ignoreMatcher_.isIgnored(relative / *segment, type == SharedData::DirectoryEntry::FileType::Directory)) + continue; } } return {std::move(entries)}; @@ -179,7 +181,9 @@ LocalScanOperation::listingFailed( errorCode == std::errc::permission_denied || errorCode == std::errc::no_such_file_or_directory; if (belowRoot && deniedOrVanished) { - Log::warn("LocalScanOperation: Cannot list '{}': {}", path.generic_string(), errorCode.message()); + Log::warn( + "LocalScanOperation: Cannot list '{}': {}", Paths::nativeToUtf8(path), Utility::errorMessage(errorCode) + ); if (!buildTree_) { // Left out of the entries, the consumer reports it instead of transferring an empty directory. @@ -189,13 +193,15 @@ LocalScanOperation::listingFailed( return std::unexpected( Error{ .type = ErrorType::DirectoryUnreadable, - .extraInfo = fmt::format("{}: {}", path.generic_string(), errorCode.message()), + .extraInfo = fmt::format("{}: {}", Paths::nativeToUtf8(path), Utility::errorMessage(errorCode)), } ); } - Log::error("LocalScanOperation: Failed to list '{}': {}", path.generic_string(), errorCode.message()); + Log::error( + "LocalScanOperation: Failed to list '{}': {}", Paths::nativeToUtf8(path), Utility::errorMessage(errorCode) + ); return enterErrorState>( - {.type = ErrorType::FilesystemError, .extraInfo = errorCode.message()} + {.type = ErrorType::FilesystemError, .extraInfo = Utility::errorMessage(errorCode)} ); } diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index f3aa065d..07578bc1 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -1284,7 +1285,7 @@ std::expected OperationQueue::addUploadOperation( if (ec) { - Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), ec.message()); + Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), Utility::errorMessage(ec)); return std::unexpected(Operation::Error{.type = Operation::ErrorType::FileStatFailed}); } @@ -1295,7 +1296,7 @@ std::expected OperationQueue::addUploadOperation( stat = std::filesystem::status(localNativePath, ec); if (ec) { - Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), ec.message()); + Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), Utility::errorMessage(ec)); return std::unexpected(Operation::Error{.type = Operation::ErrorType::FileStatFailed}); } } diff --git a/backend/source/backend/theme_finder.cpp b/backend/source/backend/theme_finder.cpp index 8ed35c55..11f596e9 100644 --- a/backend/source/backend/theme_finder.cpp +++ b/backend/source/backend/theme_finder.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -9,7 +11,7 @@ ThemeFinder::ThemeFinder(std::filesystem::path const& relativeRoot, AppWideEvent events.onReloadThemes, [&events, relativeRoot](bool) { - Log::info("Reloading themes... relativeRoot='{}'", relativeRoot.generic_string()); + Log::info("Reloading themes... relativeRoot='{}'", Paths::nativeToUtf8(relativeRoot)); events.availableThemes = findAvailableThemes(relativeRoot); Log::info("availableThemes updated, count={}", events.availableThemes.value().size()); events.availableThemes.eventContext().sync(); @@ -17,15 +19,15 @@ ThemeFinder::ThemeFinder(std::filesystem::path const& relativeRoot, AppWideEvent } )} { - Log::info("ThemeFinder constructed with relativeRoot='{}'", relativeRoot.generic_string()); + Log::info("ThemeFinder constructed with relativeRoot='{}'", Paths::nativeToUtf8(relativeRoot)); } std::vector ThemeFinder::findAvailableThemes(std::filesystem::path const& relativeRoot) { - Log::info("findAvailableThemes called with relativeRoot='{}'", relativeRoot.generic_string()); + Log::info("findAvailableThemes called with relativeRoot='{}'", Paths::nativeToUtf8(relativeRoot)); std::error_code ec; const bool rootExists = std::filesystem::exists(relativeRoot, ec); - Log::info("relativeRoot exists={}, ec='{}'", rootExists, ec.message()); + Log::info("relativeRoot exists={}, ec='{}'", rootExists, Utility::errorMessage(ec)); const auto files = findFilesInSearchPaths(relativeRoot, "themes/*.css"); Log::info("findFilesInSearchPaths returned {} file(s).", files.size()); @@ -33,7 +35,7 @@ std::vector ThemeFinder::findAvailableThemes(std::filesys themes.reserve(files.size()); for (const auto& file : files) { - Log::info("Found theme file: '{}', stem='{}'", file.generic_string(), file.stem().generic_string()); + Log::info("Found theme file: '{}', stem='{}'", Paths::nativeToUtf8(file), Paths::nativeToUtf8(file.stem())); themes.push_back(file.stem()); } Log::info("Theme discovery complete, found {} themes.", themes.size()); diff --git a/backend/source/backend/windows/windows_drag_drop.cpp b/backend/source/backend/windows/windows_drag_drop.cpp index e71dbdfa..57b4a129 100644 --- a/backend/source/backend/windows/windows_drag_drop.cpp +++ b/backend/source/backend/windows/windows_drag_drop.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -74,7 +75,7 @@ EnableWindowsDragDrop::EnableWindowsDragDrop(Nui::Window& wnd, Nui::RpcHub& hub) catch (std::exception const& exc) { std::wcout << L"Failed to parse message as JSON." << std::endl; - std::cout << exc.what(); + std::cout << Utility::errorMessage(exc); return S_OK; } @@ -147,7 +148,7 @@ EnableWindowsDragDrop::EnableWindowsDragDrop(Nui::Window& wnd, Nui::RpcHub& hub) catch (std::exception const& exc) { std::wcout << L"Failed to send drop message to RPC hub." << std::endl; - std::cout << exc.what(); + std::cout << Utility::errorMessage(exc); } return S_OK; diff --git a/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp b/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp index 887e2439..1fc63722 100644 --- a/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp +++ b/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -71,7 +72,7 @@ namespace NuiFileExplorer if (home.empty()) return result; - result.push_back({{"kind", "home"}, {"name", "Home"}, {"path", home.generic_string()}}); + result.push_back({{"kind", "home"}, {"name", "Home"}, {"path", Paths::nativeToUtf8(home)}}); const auto userDirectories = readUserDirectories(home); for (auto const& definition : Utility::defaultPlaces) @@ -83,7 +84,7 @@ namespace NuiFileExplorer result.push_back( {{"kind", std::string{definition.kind}}, {"name", std::string{definition.name}}, - {"path", path.generic_string()}} + {"path", Paths::nativeToUtf8(path)}} ); } return result; diff --git a/nui-file-explorer/source/nui-file-explorer/support/default_places_windows.cpp b/nui-file-explorer/source/nui-file-explorer/support/default_places_windows.cpp index 70fca4f1..e3bc2e99 100644 --- a/nui-file-explorer/source/nui-file-explorer/support/default_places_windows.cpp +++ b/nui-file-explorer/source/nui-file-explorer/support/default_places_windows.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -9,6 +10,7 @@ #include #include +#include #include namespace NuiFileExplorer @@ -18,13 +20,11 @@ namespace NuiFileExplorer std::string resolveKnownFolder(KNOWNFOLDERID const& folderId) { PWSTR path = nullptr; - if (SUCCEEDED(SHGetKnownFolderPath(folderId, KF_FLAG_DEFAULT, nullptr, &path))) - { - std::wstring wide{path}; - CoTaskMemFree(path); - return std::filesystem::path{wide}.generic_string(); - } - return {}; + const auto result = SHGetKnownFolderPath(folderId, KF_FLAG_DEFAULT, nullptr, &path); + const std::unique_ptr owned{path, &CoTaskMemFree}; + if (FAILED(result) || !owned) + return {}; + return Paths::nativeToUtf8(std::filesystem::path{owned.get()}); } struct FolderDef diff --git a/nui-file-explorer/source/nui-file-explorer/support/windows_drives.cpp b/nui-file-explorer/source/nui-file-explorer/support/windows_drives.cpp index 59b8f169..7c26bf48 100644 --- a/nui-file-explorer/source/nui-file-explorer/support/windows_drives.cpp +++ b/nui-file-explorer/source/nui-file-explorer/support/windows_drives.cpp @@ -1,5 +1,8 @@ #include +#include +#include +#include #include #ifndef WIN32_LEAN_AND_MEAN @@ -23,12 +26,12 @@ namespace NuiFileExplorer if (!wide.empty()) { // "My Label (C:)" - auto letter = std::filesystem::path{rootPath}.root_name().string(); - return std::filesystem::path{wide}.string() + " (" + letter + ")"; + const auto letter = Paths::nativeToUtf8(std::filesystem::path{rootPath}.root_name()); + return fmt::format("{} ({})", Paths::wideToUtf8(wide), letter); } } // Fallback: just the drive letter - return std::filesystem::path{rootPath}.root_name().string(); + return Paths::nativeToUtf8(std::filesystem::path{rootPath}.root_name()); } } @@ -61,7 +64,7 @@ namespace NuiFileExplorer continue; auto displayName = driveDisplayName(root); - auto genericPath = std::filesystem::path{root}.generic_string(); + auto genericPath = Paths::nativeToUtf8(std::filesystem::path{root}); result.push_back({ {"name", displayName}, diff --git a/paths/include/paths/native.hpp b/paths/include/paths/native.hpp index f7cd31d7..04105db6 100644 --- a/paths/include/paths/native.hpp +++ b/paths/include/paths/native.hpp @@ -79,4 +79,3 @@ namespace Paths #endif } } -#endif diff --git a/persistence/source/persistence/state_holder_backend.cpp b/persistence/source/persistence/state_holder_backend.cpp index 5ad4f03e..d8acd670 100644 --- a/persistence/source/persistence/state_holder_backend.cpp +++ b/persistence/source/persistence/state_holder_backend.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -26,7 +28,7 @@ namespace Persistence void setupPersistence() { const auto path = Nui::resolvePath(Constants::persistencePath); - const auto parentPath = std::filesystem::path{path}.parent_path().string(); + const auto parentPath = std::filesystem::path{path}.parent_path(); if (!std::filesystem::exists(parentPath)) std::filesystem::create_directories(parentPath); @@ -39,7 +41,8 @@ namespace Persistence const auto now = std::chrono::system_clock::now(); const auto time = fmt::format("{:%Y-%m-%d_%H-%M-%S}", now); - return path.parent_path() / (path.filename().string() + ".backup_" + time); + return path.parent_path() / + Paths::nativeFromUtf8(fmt::format("{}.backup_{}", Paths::nativeToUtf8(path.filename()), time)); }(); { @@ -48,16 +51,16 @@ namespace Persistence writer << reader.rdbuf(); } - Log::info("Copied config file to backup: {}", backupFileName.string()); + Log::info("Copied config file to backup: {}", Paths::nativeToUtf8(backupFileName)); return backupFileName; } LoadWarning unreadableConfig(std::filesystem::path const& path, std::exception const& exception) { - Log::error("Failed to parse config file: {}", exception.what()); + Log::error("Failed to parse config file: {}", Utility::errorMessage(exception)); return LoadWarning{ .kind = LoadWarningKind::ConfigUnreadable, - .arguments = {makeBackup(path).string(), exception.what()}, + .arguments = {Paths::nativeToUtf8(makeBackup(path)), Utility::errorMessage(exception)}, }; } } @@ -134,8 +137,8 @@ namespace Persistence } catch (std::exception const& e) { - Log::error("Failed to load config file: {}", e.what()); - error = e.what(); + Log::error("Failed to load config file: {}", Utility::errorMessage(e)); + error = Utility::errorMessage(e); } onLoad(error, *this, cachedWarnings_); @@ -461,7 +464,7 @@ namespace Persistence } catch (std::exception const& e) { - Log::error("Failed to save config file: {}", e.what()); + Log::error("Failed to save config file: {}", Utility::errorMessage(e)); throw; } } @@ -527,7 +530,8 @@ namespace Persistence hub.callRemote( responseId, nlohmann::json{ - {"error", Utility::localizedMessage("backend.persistence.saveFailed", e.what())}, + {"error", + Utility::localizedMessage("backend.persistence.saveFailed", Utility::errorMessage(e))}, } ); } @@ -583,7 +587,14 @@ namespace Persistence try { - YAML::Node config = YAML::LoadFile(filePath.generic_string()); + std::ifstream file{filePath, std::ios::binary}; + if (!file) + { + Log::error("Cannot open the language file '{}'.", Paths::nativeToUtf8(filePath)); + onLoadComplete(std::nullopt); + return; + } + YAML::Node config = YAML::Load(file); auto translateNode = [&](this const auto& translateNode, YAML::Node const& node) -> nlohmann::json { if (node.IsScalar()) @@ -605,7 +616,7 @@ namespace Persistence } catch (std::exception const& e) { - Log::error("Failed to parse language file: {}", e.what()); + Log::error("Failed to parse language file: {}", Utility::errorMessage(e)); onLoadComplete(std::nullopt); return; } @@ -635,11 +646,11 @@ namespace Persistence ); if (!content) { - Log::error("Failed to load language file: {}", file.string()); + Log::error("Failed to load language file: {}", Paths::nativeToUtf8(file)); onLoadComplete(std::nullopt); return; } - assembled[file.stem().string()] = *content; + assembled[Paths::nativeToUtf8(file.stem())] = *content; } onLoadComplete(std::move(assembled)); } diff --git a/tar-archive/source/tar_archive/archive.cpp b/tar-archive/source/tar_archive/archive.cpp index b39bb431..4104d0cb 100644 --- a/tar-archive/source/tar_archive/archive.cpp +++ b/tar-archive/source/tar_archive/archive.cpp @@ -2,6 +2,9 @@ #include #include "compression_internal.hpp" +#include + +#include #include @@ -18,7 +21,7 @@ namespace TarArchive { if (!std::filesystem::exists(path_)) return std::unexpected(makeError( - TarErrorCode::FileNotFound, "archive does not exist: " + path_.string() + TarErrorCode::FileNotFound, fmt::format("archive does not exist: {}", Paths::nativeToUtf8(path_)) )); const Compression byExtension = compressionFromExtension(path_); auto byMagic = detectCompressionByMagic(path_); diff --git a/tar-archive/source/tar_archive/compression/factory.cpp b/tar-archive/source/tar_archive/compression/factory.cpp index b70dbc5d..9c9884e2 100644 --- a/tar-archive/source/tar_archive/compression/factory.cpp +++ b/tar-archive/source/tar_archive/compression/factory.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -27,7 +28,7 @@ namespace TarArchive std::string lowercaseExtension(std::filesystem::path const& path) { - return toLower(path.extension().string()); + return toLower(Paths::nativeToUtf8(path.extension())); } struct MagicSignature diff --git a/tar-archive/source/tar_archive/compression/raw_stream.cpp b/tar-archive/source/tar_archive/compression/raw_stream.cpp index a89f8d9e..f4625245 100644 --- a/tar-archive/source/tar_archive/compression/raw_stream.cpp +++ b/tar-archive/source/tar_archive/compression/raw_stream.cpp @@ -2,6 +2,9 @@ #include #include +#include + +#include #include #include @@ -15,9 +18,7 @@ namespace TarArchive::Detail std::ofstream output{path, std::ios::binary | std::ios::trunc}; if (!output.is_open()) return std::unexpected(makeError( - TarErrorCode::IoError, - "could not open '" + path.string() + "' for writing", - errno + TarErrorCode::IoError, fmt::format("could not open '{}' for writing", Paths::nativeToUtf8(path)), errno )); return output; } @@ -26,14 +27,12 @@ namespace TarArchive::Detail { if (!std::filesystem::exists(path)) return std::unexpected(makeError( - TarErrorCode::FileNotFound, "archive does not exist: " + path.string() + TarErrorCode::FileNotFound, fmt::format("archive does not exist: {}", Paths::nativeToUtf8(path)) )); std::ifstream input{path, std::ios::binary}; if (!input.is_open()) return std::unexpected(makeError( - TarErrorCode::IoError, - "could not open '" + path.string() + "' for reading", - errno + TarErrorCode::IoError, fmt::format("could not open '{}' for reading", Paths::nativeToUtf8(path)), errno )); return input; } diff --git a/utility/include/utility/process_temporary_directory.hpp b/utility/include/utility/process_temporary_directory.hpp index 2b8622bf..130195e0 100644 --- a/utility/include/utility/process_temporary_directory.hpp +++ b/utility/include/utility/process_temporary_directory.hpp @@ -19,7 +19,12 @@ namespace Utility public: explicit ProcessTemporaryDirectory(std::filesystem::path const& base) { - if (char const* inherited = std::getenv(environmentName); inherited != nullptr && *inherited != '\0') +#ifdef _WIN32 + auto const* inherited = ::_wgetenv(environmentName); +#else + auto const* inherited = std::getenv(environmentName); +#endif + if (inherited != nullptr && *inherited != 0) { path_ = std::filesystem::path{inherited}; std::filesystem::create_directories(path_); @@ -28,7 +33,7 @@ namespace Utility owned_.emplace(base, true); path_ = owned_->path(); #ifdef _WIN32 - ::_putenv_s(environmentName, path_.string().c_str()); + ::_wputenv_s(environmentName, path_.c_str()); #else ::setenv(environmentName, path_.c_str(), 1); #endif @@ -40,7 +45,11 @@ namespace Utility } private: +#ifdef _WIN32 + static constexpr wchar_t const* environmentName = L"NUI_SFTP_TEST_PROCESS_TEMPORARY"; +#else static constexpr char const* environmentName = "NUI_SFTP_TEST_PROCESS_TEMPORARY"; +#endif std::optional owned_{}; std::filesystem::path path_{}; diff --git a/utility/include/utility/temporary_directory.hpp b/utility/include/utility/temporary_directory.hpp index 7bde3acc..cdd74c0d 100644 --- a/utility/include/utility/temporary_directory.hpp +++ b/utility/include/utility/temporary_directory.hpp @@ -1,14 +1,15 @@ #pragma once #include - -#include +#include #include #include -#include #include +#include +#include +#include #include namespace Utility @@ -29,7 +30,9 @@ namespace Utility } else if (!std::filesystem::is_directory(path_)) { - throw std::runtime_error(std::string{"Given path exists and is not a directory: "} + path_.string()); + throw std::runtime_error( + fmt::format("Given path exists and is not a directory: {}", Paths::nativeToUtf8(path_)) + ); } if (valid && randomSubFolder) { @@ -40,7 +43,9 @@ namespace Utility } if (!valid) { - throw std::runtime_error(std::string{"Could not setup temporary directory: "} + path_.string()); + throw std::runtime_error( + fmt::format("Could not setup temporary directory: {}", Paths::nativeToUtf8(path_)) + ); } } ~TemporaryDirectory() diff --git a/utility/source/utility/node/node.cpp b/utility/source/utility/node/node.cpp index 56cd6560..0aa69c04 100644 --- a/utility/source/utility/node/node.cpp +++ b/utility/source/utility/node/node.cpp @@ -1,4 +1,6 @@ #include +#include +#include #pragma clang diagnostic push #pragma clang diagnostic ignored "-Weverything" @@ -45,7 +47,7 @@ namespace SecureShell::Test boost::asio::write(stdinPipe, boost::asio::buffer(commandWithNewline), ec); if (ec) { - std::cerr << "Failed to write command: " << ec.message() << std::endl; + std::cerr << "Failed to write command: " << Utility::errorMessage(ec) << std::endl; return; } // Node on Windows cannot finish exiting while a read on its stdin pipe is pending. @@ -71,7 +73,7 @@ namespace SecureShell::Test boost::asio::write(stdinPipe, boost::asio::buffer(commandWithNewline), ec); if (ec) { - std::cerr << "Failed to write command: " << ec.message() << std::endl; + std::cerr << "Failed to write command: " << Utility::errorMessage(ec) << std::endl; return false; } @@ -82,7 +84,7 @@ namespace SecureShell::Test const auto readAmount = stdoutPipe.read_some(boost::asio::buffer(buffer), ec); if (ec || readAmount == 0) { - std::cerr << "Failed to read command acknowledgement: " << ec.message() << std::endl; + std::cerr << "Failed to read command acknowledgement: " << Utility::errorMessage(ec) << std::endl; return false; } received.append(buffer.data(), readAmount); @@ -103,7 +105,7 @@ namespace SecureShell::Test const int exitCode = process(*this)->wait(errc); if (errc && errc != boost::system::errc::no_child_process) { - std::cerr << "node process wait failed: " << errc.message() << std::endl; + std::cerr << "node process wait failed: " << Utility::errorMessage(errc) << std::endl; } return exitCode; } @@ -119,7 +121,7 @@ namespace SecureShell::Test } catch (std::exception const& e) { - std::cerr << "Failed to terminate process: " << e.what() << std::endl; + std::cerr << "Failed to terminate process: " << Utility::errorMessage(e) << std::endl; } } @@ -134,7 +136,7 @@ namespace SecureShell::Test packageJsonFile << packageJson.dump(4); } - const auto command = "\"cd "s + directory.generic_string() + " && npm install\""; + const auto command = fmt::format("\"cd {} && npm install\"", Paths::nativeToUtf8(directory)); #ifdef _WIN32 const auto shell = MSYS2_BASH; const auto npmShellArgs = std::vector{ @@ -156,7 +158,7 @@ namespace SecureShell::Test shell, npmShellArgs, bp2::process_environment{bp2::environment::current()}, - bp2::process_start_dir{directory.generic_string()}}; + bp2::process_start_dir{directory.native()}}; child.wait(); } @@ -201,7 +203,7 @@ namespace SecureShell::Test node, std::vector{"main.mjs", "--log-file=./log.txt"}, bp2::process_environment{bp2::environment::current()}, - bp2::process_start_dir{isolateDirectory.path().generic_string()}, + bp2::process_start_dir{isolateDirectory.path().native()}, bp2::process_stdio{ .in = result->stdinPipe, .out = result->stdoutPipe, @@ -223,7 +225,7 @@ namespace SecureShell::Test } catch (std::exception const& e) { - std::cerr << "Failed to terminate process: " << e.what() << std::endl; + std::cerr << "Failed to terminate process: " << Utility::errorMessage(e) << std::endl; } } }); @@ -236,7 +238,7 @@ namespace SecureShell::Test int readAmount = result->stdoutPipe.read_some(boost::asio::buffer(buffer), ec); if (ec) { - std::cerr << "Failed to read port: " << ec.message() << std::endl; + std::cerr << "Failed to read port: " << Utility::errorMessage(ec) << std::endl; return result; } if (readAmount == 0) @@ -249,7 +251,7 @@ namespace SecureShell::Test } catch (std::exception const& e) { - std::cerr << "Failed to parse port object: " << e.what() << std::endl; + std::cerr << "Failed to parse port object: " << Utility::errorMessage(e) << std::endl; return result; } diff --git a/utility/source/utility/resources.cpp b/utility/source/utility/resources.cpp index f50960ae..40733422 100644 --- a/utility/source/utility/resources.cpp +++ b/utility/source/utility/resources.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #ifndef NDEBUG # include #endif @@ -59,10 +60,11 @@ namespace const auto& seg = segments[index]; if (!hasGlobChars(seg)) { - const auto next = currentDir / seg; + const auto nativeSegment = Paths::nativeFromUtf8(seg); + const auto next = currentDir / nativeSegment; std::error_code ec; if (std::filesystem::exists(next, ec)) - matchPatternInDirImpl(next, segments, index + 1, relative / seg, results); + matchPatternInDirImpl(next, segments, index + 1, relative / nativeSegment, results); } else { @@ -71,9 +73,11 @@ namespace return; for (const auto& entry : std::filesystem::directory_iterator(currentDir, ec)) { - const auto name = entry.path().filename().string(); + const auto name = Paths::nativeToUtf8(entry.path().filename()); if (segmentMatchesPattern(name, seg)) - matchPatternInDirImpl(entry.path(), segments, index + 1, relative / name, results); + matchPatternInDirImpl( + entry.path(), segments, index + 1, relative / entry.path().filename(), results + ); } } } @@ -83,8 +87,8 @@ namespace std::vector searchPaths; #ifndef NDEBUG - searchPaths.push_back(std::filesystem::path{SOURCE_DIR}); - searchPaths.push_back(std::filesystem::path{SOURCE_DIR} / "static"); + searchPaths.push_back(Paths::nativeFromUtf8(SOURCE_DIR)); + searchPaths.push_back(Paths::nativeFromUtf8(SOURCE_DIR) / "static"); #endif searchPaths.push_back(relativeRoot); searchPaths.push_back( @@ -161,7 +165,7 @@ bool pointsToWithinDir(std::filesystem::path const& relativeRoot, std::filesyste { // make absolute and check if it starts with relativeRoot const auto absPath = std::filesystem::absolute(path); - return absPath.string().starts_with(relativeRoot.string()); + return Paths::nativeToUtf8(absPath).starts_with(Paths::nativeToUtf8(relativeRoot)); } std::vector getThemeDirs(std::filesystem::path const& relativeRoot) @@ -175,12 +179,12 @@ matchPatternInDir(std::filesystem::path const& baseDir, std::filesystem::path co std::vector segments; for (auto const& part : relativePattern) { - const auto s = part.string(); + const auto s = Paths::nativeToUtf8(part); if (!s.empty() && s != "." && s != "/" && s != "\\") segments.push_back(s); } std::vector results; - matchPatternInDirImpl(baseDir.generic_string(), segments, 0, {}, results); + matchPatternInDirImpl(baseDir, segments, 0, {}, results); return results; } @@ -195,7 +199,7 @@ std::vector findFilesInSearchPaths( { for (auto const& relPath : matchPatternInDir(searchPath, relativePattern)) { - if (seen.insert(relPath.generic_string()).second) + if (seen.insert(Paths::nativeToUtf8(relPath)).second) result.push_back(searchPath / relPath); } } @@ -231,7 +235,7 @@ mapUrlToFile(std::filesystem::path const& resourceDir, std::string const& urlPat const auto path = [&]() -> std::optional { // make path relative to / to avoid directory traversal - const auto relative = std::filesystem::relative(urlPathString, "/"); + const auto relative = std::filesystem::relative(Paths::nativeFromUtf8(urlPathString), "/"); if (endsWith(".css") && relative.parent_path().filename() == "themes") return firstOf(findFilesInSearchPaths(getThemeDirs(resourceDir), relative.filename())); diff --git a/utility/source/utility/shell_integration.cpp b/utility/source/utility/shell_integration.cpp index 3e6907c0..09561ae0 100644 --- a/utility/source/utility/shell_integration.cpp +++ b/utility/source/utility/shell_integration.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -99,7 +100,7 @@ namespace ShellIntegration ShellKind detectShellKind(std::filesystem::path const& command) { - auto name = command.filename().string(); + auto name = Paths::nativeToUtf8(command.filename()); std::ranges::transform(name, name.begin(), [](char character) { return static_cast(std::tolower(static_cast(character))); }); From 04634358d31bba76d6b639d7e8e1a438d8e43f9e Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:05:25 +0200 Subject: [PATCH 030/131] Starting a process uses native and wide text, not the ANSI code page Starting a process looks up and runs the executable by its native name, and on Windows passes the arguments and the environment as wide text converted from UTF-8, so no part of the command goes through the ANSI code page --- backend/source/backend/process/process.cpp | 25 +++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/backend/source/backend/process/process.cpp b/backend/source/backend/process/process.cpp index 041612d8..1dd5439b 100644 --- a/backend/source/backend/process/process.cpp +++ b/backend/source/backend/process/process.cpp @@ -1,6 +1,9 @@ #include #include +#include +#include + #include #include @@ -8,6 +11,9 @@ #include #include #include +#include +#include +#include namespace bp2 = boost::process::v2; @@ -303,12 +309,17 @@ void Process::spawn( { if (key.empty() || value.empty()) continue; +#ifdef _WIN32 + env.emplace(Paths::utf8ToWide(key), Paths::utf8ToWide(value)); +#else env.emplace(key, value); +#endif } - auto executable = bp2::environment::find_executable(processName, env); + const boost::filesystem::path name{Paths::nativeFromUtf8(processName).native()}; + auto executable = bp2::environment::find_executable(name, env); if (executable.empty()) - executable = processName; + executable = name; #ifdef _WIN32 // cmd.exe reads its own path as given: in "C:/Windows/System32/cmd.exe" it takes "/c" as a switch and runs // "md.exe" instead of starting a shell. @@ -326,10 +337,18 @@ void Process::spawn( // The app is a GUI program without a console, a console child would open a window of its own. bp2::windows::default_launcher windowlessLauncher{}; windowlessLauncher.creation_flags |= CREATE_NO_WINDOW; + const auto wideArguments = arguments | + std::views::transform( + [](std::string const& argument) + { + return Paths::utf8ToWide(argument); + } + ) | + std::ranges::to(); impl_->child = std::make_unique(windowlessLauncher( impl_->executor, executable, - arguments, + wideArguments, bp2::process_environment{env}, bp2::process_stdio(impl_->stdinPipe, impl_->stdoutPipe, impl_->stderrPipe) )); From d9822a878111b9a6b5edba5ed2883a6dd793e8e9 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:05:25 +0200 Subject: [PATCH 031/131] Key files are read by their native path and imported from memory Key files are read by their native path and imported from memory (ssh_pki_import_privkey_base64, an OpenSSH public key line through ssh_pki_import_pubkey_base64), so a key under any name loads on Windows; libssh is given the ssh directory and known hosts file as UTF-8 --- ssh/include/ssh/key_file.hpp | 47 ++++++++++++++++++ ssh/source/ssh/CMakeLists.txt | 1 + ssh/source/ssh/key_file.cpp | 93 +++++++++++++++++++++++++++++++++++ ssh/source/ssh/session.cpp | 37 ++++---------- 4 files changed, 150 insertions(+), 28 deletions(-) create mode 100644 ssh/include/ssh/key_file.hpp create mode 100644 ssh/source/ssh/key_file.cpp diff --git a/ssh/include/ssh/key_file.hpp b/ssh/include/ssh/key_file.hpp new file mode 100644 index 00000000..c968441d --- /dev/null +++ b/ssh/include/ssh/key_file.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace SecureShell +{ + namespace Detail + { + /** + * @brief Overwrites @p target with zero bytes through a volatile pointer and clears it, so sensitive data + * does not linger in process memory. + */ + void secureWipe(std::string& target) noexcept; + + /** + * @brief The key type name and base64 blob of an OpenSSH public key line "type base64 [comment]", nothing + * for any other text. + */ + std::optional> splitPublicKeyLine(std::string_view text); + } + + /** + * @brief Imports the private key file @p keyFile, read by its native path so any name works on Windows. Returns + * what ssh_pki_import_privkey_file would: SSH_EOF when the file cannot be read, SSH_ERROR when it holds no + * valid key or the passphrase is wrong. + */ + int importPrivateKeyFile( + std::filesystem::path const& keyFile, + char const* passphrase, + ssh_auth_callback askPass, + void* userData, + ssh_key* key + ); + + /** + * @brief Imports the public key file @p keyFile, read by its native path. An OpenSSH public key line is imported + * from memory, any other format (like PEM) through libssh by the UTF-8 path. SSH_EOF when the file cannot + * be read. + */ + int importPublicKeyFile(std::filesystem::path const& keyFile, ssh_key* key); +} diff --git a/ssh/source/ssh/CMakeLists.txt b/ssh/source/ssh/CMakeLists.txt index 77bd4247..e0333a24 100644 --- a/ssh/source/ssh/CMakeLists.txt +++ b/ssh/source/ssh/CMakeLists.txt @@ -9,6 +9,7 @@ add_library( sftp_session.cpp file_stream.cpp file_information.cpp + key_file.cpp ) target_include_directories( diff --git a/ssh/source/ssh/key_file.cpp b/ssh/source/ssh/key_file.cpp new file mode 100644 index 00000000..582cdb51 --- /dev/null +++ b/ssh/source/ssh/key_file.cpp @@ -0,0 +1,93 @@ +#include +#include + +#include + +#include +#include +#include + +namespace SecureShell +{ + namespace + { + std::optional readKeyFile(std::filesystem::path const& keyFile) + { + std::ifstream stream{keyFile, std::ios::binary}; + if (!stream) + return std::nullopt; + std::string content{std::istreambuf_iterator{stream}, std::istreambuf_iterator{}}; + if (stream.bad()) + { + Detail::secureWipe(content); + return std::nullopt; + } + return content; + } + + bool isSpace(char character) + { + return std::isspace(static_cast(character)) != 0; + } + + std::string_view nextToken(std::string_view& text) + { + std::size_t start = 0; + while (start < text.size() && isSpace(text[start])) + ++start; + std::size_t end = start; + while (end < text.size() && !isSpace(text[end])) + ++end; + const auto token = text.substr(start, end - start); + text.remove_prefix(end); + return token; + } + } + + namespace Detail + { + void secureWipe(std::string& target) noexcept + { + volatile char* volatileData = target.data(); + for (std::size_t index = 0; index < target.size(); ++index) + volatileData[index] = '\0'; + target.clear(); + } + + std::optional> splitPublicKeyLine(std::string_view text) + { + const auto type = nextToken(text); + const auto blob = nextToken(text); + if (type.empty() || blob.empty() || type.starts_with("-----") || + ssh_key_type_from_name(std::string{type}.c_str()) == SSH_KEYTYPE_UNKNOWN) + return std::nullopt; + return std::pair{std::string{type}, std::string{blob}}; + } + } + + int importPrivateKeyFile( + std::filesystem::path const& keyFile, + char const* passphrase, + ssh_auth_callback askPass, + void* userData, + ssh_key* key + ) + { + auto content = readKeyFile(keyFile); + if (!content) + return SSH_EOF; + const auto result = ssh_pki_import_privkey_base64(content->c_str(), passphrase, askPass, userData, key); + Detail::secureWipe(*content); + return result; + } + + int importPublicKeyFile(std::filesystem::path const& keyFile, ssh_key* key) + { + const auto content = readKeyFile(keyFile); + if (!content) + return SSH_EOF; + if (const auto line = Detail::splitPublicKeyLine(*content); line) + return ssh_pki_import_pubkey_base64(line->second.c_str(), ssh_key_type_from_name(line->first.c_str()), key); + return ssh_pki_import_pubkey_file(Paths::nativeToUtf8(keyFile).c_str(), key); + } +} diff --git a/ssh/source/ssh/session.cpp b/ssh/source/ssh/session.cpp index 779f0af8..152c7183 100644 --- a/ssh/source/ssh/session.cpp +++ b/ssh/source/ssh/session.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -11,23 +12,9 @@ namespace SecureShell { namespace { - /** - * @brief Overwrite the contents of a string buffer in place so sensitive - * data does not linger in process memory after use. - * @param target String whose storage will be overwritten with zero bytes - * through a volatile pointer to prevent optimizer elision. - */ - void secureWipe(std::string& target) - { - volatile char* volatilePtr = target.data(); - for (std::size_t index = 0; index < target.size(); ++index) - volatilePtr[index] = '\0'; - target.clear(); - } - /** * @brief Describes why reading a key file failed, libssh only returns SSH_EOF or SSH_ERROR for it. - * @param importResult Return value of ssh_pki_import_pubkey_file or ssh_pki_import_privkey_file. + * @param importResult Return value of importPublicKeyFile or importPrivateKeyFile. * @param keyKind "public" or "private". * @param keyFile The key file that was read. */ @@ -352,7 +339,7 @@ namespace SecureShell { lastAttemptedAppliedOption = "SSH Directory"; return static_cast(*session).setOption( - SSH_OPTIONS_SSH_DIR, sshOptions.sshDirectory.value().generic_string().c_str() + SSH_OPTIONS_SSH_DIR, Paths::nativeToUtf8(sshOptions.sshDirectory.value()).c_str() ); } return 0; @@ -378,7 +365,7 @@ namespace SecureShell { lastAttemptedAppliedOption = "Known Hosts File"; return static_cast(*session).setOption( - SSH_OPTIONS_KNOWNHOSTS, sshOptions.knownHostsFile.value().generic_string().c_str() + SSH_OPTIONS_KNOWNHOSTS, Paths::nativeToUtf8(sshOptions.knownHostsFile.value()).c_str() ); } return 0; @@ -634,9 +621,7 @@ namespace SecureShell { if (maybeSshKeyPublic) { - return ssh_pki_import_pubkey_file( - Paths::nativeToUtf8(*maybeSshKeyPublic).c_str(), &publicKey.underlyingReferenced() - ); + return importPublicKeyFile(*maybeSshKeyPublic, &publicKey.underlyingReferenced()); } return SSH_OK; }, @@ -654,12 +639,8 @@ namespace SecureShell }, [&sshKeyPrivate, &privateKey, askPass, askPassUserDataKeyPhrase]() { - return ssh_pki_import_privkey_file( - Paths::nativeToUtf8(sshKeyPrivate).c_str(), - nullptr, - askPass, - askPassUserDataKeyPhrase, - &privateKey.underlyingReferenced() + return importPrivateKeyFile( + sshKeyPrivate, nullptr, askPass, askPassUserDataKeyPhrase, &privateKey.underlyingReferenced() ); }, [&privateKey, &session]() @@ -699,7 +680,7 @@ namespace SecureShell if (pwFromCache.has_value()) { buf = pwFromCache.value(); - secureWipe(*pwFromCache); + Detail::secureWipe(*pwFromCache); pwFromCache.reset(); const auto result = static_cast(*session).userauthPassword(buf.data()); if (result == SSH_AUTH_SUCCESS) @@ -721,7 +702,7 @@ namespace SecureShell return (int)SSH_AUTH_DENIED; } ); - secureWipe(buf); + Detail::secureWipe(buf); } if (keyFailure && result.result == SSH_EOF) From a4cff3d50e5e949e4153d50d5b1025f940c0f8c0 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:06:55 +0200 Subject: [PATCH 032/131] The Windows executable runs with the UTF-8 active code page The Windows executable embeds a manifest whose activeCodePage is UTF-8, so narrow Windows APIs called by third-party code (libssh, yaml-cpp, spdlog) read and return UTF-8 instead of the ANSI code page --- CMakeLists.txt | 5 ++++- _cmake/windows/nui-sftp.manifest | 8 ++++++++ _cmake/windows/nui-sftp.rc.in | 3 +++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 _cmake/windows/nui-sftp.manifest diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f5f74f6..177f81af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,6 +91,9 @@ if (WIN32) set_target_properties(${PROJECT_NAME} PROPERTIES WIN32_EXECUTABLE TRUE) include("${CMAKE_CURRENT_LIST_DIR}/_cmake/windows/icon.cmake") file(TO_CMAKE_PATH "${NUI_SFTP_ICO}" NUI_SFTP_ICO_FORWARD) + # activeCodePage UTF-8: narrow Windows APIs in third-party code (libssh, yaml-cpp, spdlog) read UTF-8. + set(NUI_SFTP_MANIFEST "${CMAKE_CURRENT_LIST_DIR}/_cmake/windows/nui-sftp.manifest") + file(TO_CMAKE_PATH "${NUI_SFTP_MANIFEST}" NUI_SFTP_MANIFEST_FORWARD) configure_file( "${CMAKE_CURRENT_LIST_DIR}/_cmake/windows/nui-sftp.rc.in" "${CMAKE_BINARY_DIR}/generated/nui-sftp.rc" @@ -99,7 +102,7 @@ if (WIN32) target_sources(${PROJECT_NAME} PRIVATE "${CMAKE_BINARY_DIR}/generated/nui-sftp.rc") set_source_files_properties( "${CMAKE_BINARY_DIR}/generated/nui-sftp.rc" - PROPERTIES OBJECT_DEPENDS "${NUI_SFTP_ICO}" + PROPERTIES OBJECT_DEPENDS "${NUI_SFTP_ICO};${NUI_SFTP_MANIFEST}" ) add_dependencies(${PROJECT_NAME} nui-sftp-icon) elseif (APPLE AND NOT EMSCRIPTEN) diff --git a/_cmake/windows/nui-sftp.manifest b/_cmake/windows/nui-sftp.manifest new file mode 100644 index 00000000..f2fff698 --- /dev/null +++ b/_cmake/windows/nui-sftp.manifest @@ -0,0 +1,8 @@ + + + + + UTF-8 + + + diff --git a/_cmake/windows/nui-sftp.rc.in b/_cmake/windows/nui-sftp.rc.in index d975a860..715a13a9 100644 --- a/_cmake/windows/nui-sftp.rc.in +++ b/_cmake/windows/nui-sftp.rc.in @@ -4,6 +4,9 @@ // (Windows picks the lowest-numbered ICON resource). 1 ICON "@NUI_SFTP_ICO_FORWARD@" +// The application manifest: CREATEPROCESS_MANIFEST_RESOURCE_ID (1) of type RT_MANIFEST (24). +1 24 "@NUI_SFTP_MANIFEST_FORWARD@" + VS_VERSION_INFO VERSIONINFO FILEVERSION @SEMVER_MAJOR@,@SEMVER_MINOR@,@SEMVER_PATCH@,0 PRODUCTVERSION @SEMVER_MAJOR@,@SEMVER_MINOR@,@SEMVER_PATCH@,0 From 717f8b53db19248ac0a86c51ebe3179bf2d813b2 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:08:56 +0200 Subject: [PATCH 033/131] Help and --console output on Windows writes UTF-8 Help and --console output on Windows writes UTF-8: the attached console switches its output (and, for a console stdin, input) code page to UTF-8 for as long as the program runs and gets its previous code pages back at exit, and a stream that cannot be reopened on the console is left alone --- .../include/backend/windows/main_windows.hpp | 31 +++++++- backend/source/backend/main.cpp | 23 +++--- .../source/backend/windows/main_windows.cpp | 73 +++++++++++++++---- 3 files changed, 101 insertions(+), 26 deletions(-) diff --git a/backend/include/backend/windows/main_windows.hpp b/backend/include/backend/windows/main_windows.hpp index fca772ad..3638c871 100644 --- a/backend/include/backend/windows/main_windows.hpp +++ b/backend/include/backend/windows/main_windows.hpp @@ -10,8 +10,35 @@ struct Main::PlatformSpecifics EnableWindowsDragDrop enableDragDrop; }; +/** + * @brief The console the standard streams were pointed at. It writes and reads UTF-8 while this lives and gets its + * previous code pages back when this is destroyed, since the console belongs to the parent as well. + */ +class ConsoleAttachment +{ + public: + ConsoleAttachment() = default; + ~ConsoleAttachment(); + ConsoleAttachment(ConsoleAttachment&& other) noexcept; + ConsoleAttachment& operator=(ConsoleAttachment&& other) noexcept; + ConsoleAttachment(ConsoleAttachment const&) = delete; + ConsoleAttachment& operator=(ConsoleAttachment const&) = delete; + + private: + friend ConsoleAttachment attachParentConsole(); + + /** + * @brief The output code page to restore, 0 when it was not changed. + */ + unsigned int previousOutputCodePage_{0}; + /** + * @brief The input code page to restore, 0 when it was not changed. + */ + unsigned int previousInputCodePage_{0}; +}; + /** * @brief Attaches to the console of the process that started this one, if there is one, and points the standard - * streams at it. The program is a GUI program, without this its output goes nowhere. + * streams without a file or pipe at it. The program is a GUI program, without this its output goes nowhere. */ -void attachParentConsole(); \ No newline at end of file +ConsoleAttachment attachParentConsole(); \ No newline at end of file diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index 6d47f6db..4766419b 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -38,6 +38,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -452,15 +456,16 @@ int main(int const argc, char const* const* argv) static_cast(Utility::processUmask()); #ifdef _WIN32 // A GUI program has no console; help and --console write to the one it was started from. - for (int index = 1; index < argc; ++index) - { - const std::string_view argument{argv[index]}; - if (argument == "--help" || argument == "--console") - { - attachParentConsole(); - break; - } - } + const auto arguments = std::span{argv, static_cast(argc)} | std::views::drop(1); + const auto console = std::ranges::any_of( + arguments, + [](std::string_view argument) + { + return argument == "--help" || argument == "--console"; + } + ) + ? attachParentConsole() + : ConsoleAttachment{}; #endif auto options = parseProgramOptions(argc, argv); if (!options) diff --git a/backend/source/backend/windows/main_windows.cpp b/backend/source/backend/windows/main_windows.cpp index 8ce15627..8c49aad5 100644 --- a/backend/source/backend/windows/main_windows.cpp +++ b/backend/source/backend/windows/main_windows.cpp @@ -1,9 +1,10 @@ #include -#include - #include #include +#include + +#include Main::PlatformSpecifics::PlatformSpecifics(Nui::Window& window, Nui::RpcHub& hub) : enableDragDrop{window, hub} @@ -19,26 +20,68 @@ namespace const auto handle = GetStdHandle(standardHandle); return handle != nullptr && handle != INVALID_HANDLE_VALUE && GetFileType(handle) != FILE_TYPE_UNKNOWN; } + + bool reopen(char const* device, char const* mode, FILE* target) + { + FILE* stream = nullptr; + return freopen_s(&stream, device, mode, target) == 0; + } +} + +ConsoleAttachment::~ConsoleAttachment() +{ + std::cout.flush(); + std::cerr.flush(); + std::fflush(stdout); + std::fflush(stderr); + if (previousOutputCodePage_ != 0) + SetConsoleOutputCP(previousOutputCodePage_); + if (previousInputCodePage_ != 0) + SetConsoleCP(previousInputCodePage_); +} + +ConsoleAttachment::ConsoleAttachment(ConsoleAttachment&& other) noexcept + : previousOutputCodePage_{std::exchange(other.previousOutputCodePage_, 0u)} + , previousInputCodePage_{std::exchange(other.previousInputCodePage_, 0u)} +{} + +ConsoleAttachment& ConsoleAttachment::operator=(ConsoleAttachment&& other) noexcept +{ + if (this != &other) + { + previousOutputCodePage_ = std::exchange(other.previousOutputCodePage_, 0u); + previousInputCodePage_ = std::exchange(other.previousInputCodePage_, 0u); + } + return *this; } -void attachParentConsole() +ConsoleAttachment attachParentConsole() { // Redirected streams keep their file or pipe, only the ones without a target go to the console. const bool outputUsable = isUsable(STD_OUTPUT_HANDLE); const bool errorUsable = isUsable(STD_ERROR_HANDLE); const bool inputUsable = isUsable(STD_INPUT_HANDLE); if (outputUsable && errorUsable && inputUsable) - return; + return {}; if (!AttachConsole(ATTACH_PARENT_PROCESS)) - return; - FILE* stream = nullptr; - if (!outputUsable) - freopen_s(&stream, "CONOUT$", "w", stdout); - if (!errorUsable) - freopen_s(&stream, "CONOUT$", "w", stderr); - if (!inputUsable) - freopen_s(&stream, "CONIN$", "r", stdin); - std::cout.clear(); - std::cerr.clear(); - std::cin.clear(); + return {}; + + ConsoleAttachment attachment{}; + const bool outputReopened = !outputUsable && reopen("CONOUT$", "w", stdout); + const bool errorReopened = !errorUsable && reopen("CONOUT$", "w", stderr); + const bool inputReopened = !inputUsable && reopen("CONIN$", "r", stdin); + if (outputReopened || errorReopened) + { + attachment.previousOutputCodePage_ = GetConsoleOutputCP(); + SetConsoleOutputCP(CP_UTF8); + std::cout.clear(); + std::cerr.clear(); + } + if (inputReopened) + { + attachment.previousInputCodePage_ = GetConsoleCP(); + SetConsoleCP(CP_UTF8); + std::cin.clear(); + } + return attachment; } From 831913d806904ddd39321b2dd0d5e3b821ed1de4 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:11:13 +0200 Subject: [PATCH 034/131] Case conversion changes ASCII letters only Case conversion changes ASCII letters only and never passes a byte of a multibyte UTF-8 sequence to the C locale functions (also in the naming convention converter and the terminal debug print); the settings search compares the caret with the input's length in UTF-16 units like selectionStart, and a long value is cut at a code point boundary --- .../settings/search/settings_search.cpp | 7 ++- .../frontend/terminal/terminal_channel.cpp | 2 +- .../utility/algorithm/case_convert.hpp | 55 ++++++++++++------- .../utility/convert_naming_convention.hpp | 22 ++++---- 4 files changed, 53 insertions(+), 33 deletions(-) diff --git a/frontend/source/frontend/settings/search/settings_search.cpp b/frontend/source/frontend/settings/search/settings_search.cpp index 34ba356d..53aad7aa 100644 --- a/frontend/source/frontend/settings/search/settings_search.cpp +++ b/frontend/source/frontend/settings/search/settings_search.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -137,7 +138,7 @@ namespace { auto text = value.is_string() ? value.get() : value.dump(); if (text.size() > maximumValueLength) - text = text.substr(0, maximumValueLength) + "..."; + return fmt::format("{}...", Paths::truncatedUtf8(text, maximumValueLength)); return text; } @@ -506,8 +507,8 @@ struct SettingsSearch::Implementation const auto key = event["key"].as(); const auto count = selectableCount(); const auto caret = inputElement.isUndefined() ? 0 : inputElement["selectionStart"].as(); - const auto valueLength = - inputElement.isUndefined() ? 0 : static_cast(inputElement["value"].as().size()); + // selectionStart counts UTF-16 code units, like the length of the JavaScript string. + const auto valueLength = inputElement.isUndefined() ? 0 : inputElement["value"]["length"].as(); const auto consume = [&event]() { diff --git a/frontend/source/frontend/terminal/terminal_channel.cpp b/frontend/source/frontend/terminal/terminal_channel.cpp index 9732d67e..4ae59924 100644 --- a/frontend/source/frontend/terminal/terminal_channel.cpp +++ b/frontend/source/frontend/terminal/terminal_channel.cpp @@ -235,7 +235,7 @@ namespace { debugPrint += "\\n"; } - else if (std::isprint(c)) + else if (std::isprint(static_cast(c))) { debugPrint += c; } diff --git a/utility/include/utility/algorithm/case_convert.hpp b/utility/include/utility/algorithm/case_convert.hpp index 72373fb1..ca77bce5 100644 --- a/utility/include/utility/algorithm/case_convert.hpp +++ b/utility/include/utility/algorithm/case_convert.hpp @@ -9,19 +9,42 @@ namespace Utility::Algorithm { /** - * @brief Converts the passed string to upper case by out paramter. + * @brief @p character in lower case when it is an ASCII letter, unchanged otherwise. Bytes of a multibyte UTF-8 + * sequence are never touched and the locale plays no part. + */ + constexpr char asciiToLower(char character) noexcept + { + return character >= 'A' && character <= 'Z' ? static_cast(character - 'A' + 'a') : character; + } + + /** + * @brief @p character in upper case when it is an ASCII letter, unchanged otherwise, see @ref asciiToLower. + */ + constexpr char asciiToUpper(char character) noexcept + { + return character >= 'a' && character <= 'z' ? static_cast(character - 'a' + 'A') : character; + } + + /** + * @brief True for the ASCII upper case letters. + */ + constexpr bool isAsciiUpper(char character) noexcept + { + return character >= 'A' && character <= 'Z'; + } + + /** + * @brief Converts the ASCII letters of the passed string to upper case by out paramter. * * @param input The string to convert. */ inline void toUpperCaseInplace(std::string& input) { - std::transform(input.begin(), input.end(), input.begin(), [](char c) { - return std::toupper(c); - }); + std::ranges::transform(input, input.begin(), asciiToUpper); } /** - * @brief Converts the passed string to upper case and returns it. + * @brief Converts the ASCII letters of the passed string to upper case and returns it. * * @param input The string to convert. * @return std::string The string in upper case. @@ -29,26 +52,22 @@ namespace Utility::Algorithm inline std::string toUpperCase(std::string const& input) { std::string result(input.size(), '\0'); - std::transform(input.begin(), input.end(), result.begin(), [](char c) { - return std::toupper(c); - }); + std::ranges::transform(input, result.begin(), asciiToUpper); return result; } /** - * @brief Converts the passed string to lower case by out paramter. + * @brief Converts the ASCII letters of the passed string to lower case by out paramter. * * @param input The string to convert. */ inline void toLowerCaseInplace(std::string& input) { - std::transform(input.begin(), input.end(), input.begin(), [](char c) { - return std::tolower(c); - }); + std::ranges::transform(input, input.begin(), asciiToLower); } /** - * @brief Converts the passed string to lower case and returns it. + * @brief Converts the ASCII letters of the passed string to lower case and returns it. * * @param input The string to convert. * @return std::string The string in lower case. @@ -56,14 +75,12 @@ namespace Utility::Algorithm inline std::string toLowerCase(std::string const& input) { std::string result(input.size(), '\0'); - std::transform(input.begin(), input.end(), result.begin(), [](char c) { - return std::tolower(c); - }); + std::ranges::transform(input, result.begin(), asciiToLower); return result; } /** - * @brief Case-insensitive lexicographic comparison of two strings. + * @brief Lexicographic comparison of two strings, ASCII letters compared without case. * * @param lhs The left-hand string. * @param rhs The right-hand string. @@ -75,8 +92,8 @@ namespace Utility::Algorithm const auto commonLength = std::min(lhs.size(), rhs.size()); for (std::size_t index = 0; index < commonLength; ++index) { - const auto leftChar = std::tolower(static_cast(lhs[index])); - const auto rightChar = std::tolower(static_cast(rhs[index])); + const auto leftChar = static_cast(asciiToLower(lhs[index])); + const auto rightChar = static_cast(asciiToLower(rhs[index])); if (leftChar != rightChar) return leftChar < rightChar ? -1 : 1; } diff --git a/utility/include/utility/convert_naming_convention.hpp b/utility/include/utility/convert_naming_convention.hpp index 63f6d17a..d8db38ec 100644 --- a/utility/include/utility/convert_naming_convention.hpp +++ b/utility/include/utility/convert_naming_convention.hpp @@ -1,7 +1,9 @@ #pragma once -#include +#include + #include +#include namespace Utility { @@ -19,7 +21,7 @@ namespace Utility { if (!normalizedSegments[i].empty()) { - result += std::toupper(normalizedSegments[i][0]); + result += Algorithm::asciiToUpper(normalizedSegments[i][0]); result += normalizedSegments[i].substr(1); } } @@ -36,7 +38,7 @@ namespace Utility { if (!segment.empty()) { - result += std::toupper(segment[0]); + result += Algorithm::asciiToUpper(segment[0]); result += segment.substr(1); } } @@ -67,7 +69,7 @@ namespace Utility { result += '_'; for (auto const& ch : normalizedSegments[i]) - result += std::toupper(ch); + result += Algorithm::asciiToUpper(ch); } return result; } @@ -110,12 +112,12 @@ namespace Utility std::string currentSegment{}; for (auto const& ch : input) { - if (std::isupper(ch) && !currentSegment.empty()) + if (Algorithm::isAsciiUpper(ch) && !currentSegment.empty()) { result.normalizedSegments.push_back(currentSegment); currentSegment.clear(); } - currentSegment += std::tolower(ch); + currentSegment += Algorithm::asciiToLower(ch); } if (!currentSegment.empty()) result.normalizedSegments.push_back(currentSegment); @@ -131,12 +133,12 @@ namespace Utility std::string currentSegment{}; for (auto const& ch : input) { - if (std::isupper(ch) && !currentSegment.empty()) + if (Algorithm::isAsciiUpper(ch) && !currentSegment.empty()) { result.normalizedSegments.push_back(currentSegment); currentSegment.clear(); } - currentSegment += std::tolower(ch); + currentSegment += Algorithm::asciiToLower(ch); } if (!currentSegment.empty()) result.normalizedSegments.push_back(currentSegment); @@ -159,7 +161,7 @@ namespace Utility } else { - currentSegment += std::tolower(ch); + currentSegment += Algorithm::asciiToLower(ch); } } if (!currentSegment.empty()) @@ -183,7 +185,7 @@ namespace Utility } else { - currentSegment += std::tolower(ch); + currentSegment += Algorithm::asciiToLower(ch); } } if (!currentSegment.empty()) From 9c31593eedbfd77ccd4d7a18ccfa75d4ac5ddc45 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:13:10 +0200 Subject: [PATCH 035/131] The tar writer stores non-ASCII names in PAX records The tar writer gives every member whose name or link target has a non-ASCII byte a PAX path or linkpath record, read as UTF-8, and a name too long for the ustar fields is cut at a code point boundary there; archive uploads and downloads of non-ASCII names are tested --- tar-archive/source/tar_archive/header.cpp | 32 ++++++++++++++++------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tar-archive/source/tar_archive/header.cpp b/tar-archive/source/tar_archive/header.cpp index 49e9ce20..c6ed9212 100644 --- a/tar-archive/source/tar_archive/header.cpp +++ b/tar-archive/source/tar_archive/header.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -227,6 +228,17 @@ namespace TarArchive return std::nullopt; } + bool hasNonAsciiByte(std::string_view text) + { + return std::ranges::any_of( + text, + [](char character) + { + return static_cast(character) >= 0x80u; + } + ); + } + std::string normaliseArchivePath(std::string value) { while (!value.empty() && value.front() == '/') @@ -647,20 +659,21 @@ namespace TarArchive std::string truncatedPath = normalisedPath; std::string truncatedLink = linkTargetText; + // A name with non-ASCII bytes also gets a PAX record, which readers take as UTF-8, the ustar fields have no + // encoding of their own. auto split = trySplitUstarPath(normalisedPath, isDirectory); - const bool pathNeedsPax = !split.has_value(); - if (pathNeedsPax) - { + if (!split.has_value() || hasNonAsciiByte(normalisedPath)) paxEntries.emplace_back("path", normalisedPath); - truncatedPath = normalisedPath.substr(0u, fieldName.length); + if (!split.has_value()) + { + truncatedPath = std::string{Paths::truncatedUtf8(normalisedPath, fieldName.length)}; split = std::pair{std::string{}, truncatedPath}; } - const bool linkNeedsPax = linkTargetText.size() > fieldLinkName.length; - if (linkNeedsPax) + if (linkTargetText.size() > fieldLinkName.length || hasNonAsciiByte(linkTargetText)) { paxEntries.emplace_back("linkpath", linkTargetText); - truncatedLink = linkTargetText.substr(0u, fieldLinkName.length); + truncatedLink = std::string{Paths::truncatedUtf8(linkTargetText, fieldLinkName.length)}; } if (sizeExceedsOctal) @@ -683,9 +696,8 @@ namespace TarArchive const std::string paxName = "PaxHeaders/" + (truncatedPath.empty() ? std::string{"entry"} : truncatedPath); const auto paxSplit = trySplitUstarPath(paxName, false); - const std::string paxNameField = paxSplit - ? paxSplit->second - : paxName.substr(0u, fieldName.length); + const std::string paxNameField = + paxSplit ? paxSplit->second : std::string{Paths::truncatedUtf8(paxName, fieldName.length)}; const std::string paxPrefixField = paxSplit ? paxSplit->first : std::string{}; RawRecord paxRecord; From 298cfe0f53109bbf61ea1adcdddca696a104522d Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:42:59 +0200 Subject: [PATCH 036/131] Link-safe removal goes on past failures and names the failed item Link-safe removal goes on past an item that cannot be removed and names that item: removeAllNoFollow returns every failure by its own path and keeps only the directories containing one, inspects each entry once (the directory iterator's type on POSIX), and on Windows clears a read-only attribute and retries; NoFollowTreeWalk counts or removes a tree in bounded steps, removeNoFollow removes a single item without following it, and on Windows only reparse points tagged as name surrogates (symlinks, junctions, WSL symlinks) count as links, so cloud placeholders, deduplicated files and AF_UNIX sockets are ordinary items; firstLinkBelow stops at the first missing component and can skip directories it verified before --- utility/include/utility/link_safe_remove.hpp | 99 ++++++- .../include/utility/no_follow_tree_walk.hpp | 105 ++++++++ utility/source/utility/CMakeLists.txt | 1 + utility/source/utility/link_safe_remove.cpp | 254 +++++++++++++----- .../source/utility/no_follow_inspection.hpp | 37 +++ .../source/utility/no_follow_tree_walk.cpp | 172 ++++++++++++ 6 files changed, 595 insertions(+), 73 deletions(-) create mode 100644 utility/include/utility/no_follow_tree_walk.hpp create mode 100644 utility/source/utility/no_follow_inspection.hpp create mode 100644 utility/source/utility/no_follow_tree_walk.cpp diff --git a/utility/include/utility/link_safe_remove.hpp b/utility/include/utility/link_safe_remove.hpp index 711d770f..0e398e05 100644 --- a/utility/include/utility/link_safe_remove.hpp +++ b/utility/include/utility/link_safe_remove.hpp @@ -2,29 +2,116 @@ #include #include +#include #include +#include #include +#include namespace Utility { + namespace Detail + { + /** + * @brief True for a Windows reparse tag that makes its item a link: a name surrogate such as a symlink + * (IO_REPARSE_TAG_SYMLINK), a junction (IO_REPARSE_TAG_MOUNT_POINT) or a WSL symlink + * (IO_REPARSE_TAG_LX_SYMLINK). Cloud file placeholders, deduplicated files and AF_UNIX sockets are + * reparse points too, but no links. + */ + bool isLinkReparseTag(std::uint32_t tag) noexcept; + } + + /** + * @brief What an item is, inspected without following it. + */ + enum class NoFollowKind + { + Missing, + /** @brief A symlink or, on Windows, a junction or another name surrogate reparse point. */ + Link, + Directory, + /** @brief A regular file or any other item that is no directory and no link. */ + Other + }; + + /** + * @brief Inspects @p path without following it. A missing path is no error. + */ + NoFollowKind noFollowKindOf(std::filesystem::path const& path, std::error_code& error); + /** - * @brief True for a symlink and, on Windows, for every reparse point such as a junction. std::filesystem may - * report a junction as a plain directory, and walking into it leaves the tree it belongs to. + * @brief True for a symlink and, on Windows, for a junction or another reparse point whose tag is a name + * surrogate (see Detail::isLinkReparseTag). std::filesystem may report a junction as a plain directory, + * and walking into it leaves the tree it belongs to. */ bool isLinkOrJunction(std::filesystem::path const& path, std::error_code& error); + /** + * @brief Directories already found to be real directories, so @ref firstLinkBelow inspects each of them once. + * One set belongs to one operation, which writes below a single root. Thread safe. + */ + class VerifiedDirectories + { + public: + bool contains(std::filesystem::path const& directory) const; + void insert(std::filesystem::path const& directory); + + private: + mutable std::mutex mutex_{}; + std::set directories_{}; + }; + /** * @brief Finds the first existing component of @p path below @p root (root excluded, @p path included) that is - * a symlink or junction. Writing through it would leave @p root. + * a symlink or junction. Writing through it would leave @p root. Components after the first missing one + * are not inspected, they cannot exist. + * @param verified Directories inspected before, skipped here; the real directories found are added to it. * @return The offending component, or nothing when there is none. @p error is set when @p path is not lexically * below @p root or a component cannot be inspected. */ - std::optional - firstLinkBelow(std::filesystem::path const& root, std::filesystem::path const& path, std::error_code& error); + std::optional firstLinkBelow( + std::filesystem::path const& root, + std::filesystem::path const& path, + std::error_code& error, + VerifiedDirectories* verified = nullptr + ); + + /** + * @brief Removes @p path itself without following it: a file, a link (never its target) or an empty directory. + * On Windows a read-only item has its read-only attribute cleared and is removed again. + * @return True when @p path was removed, false when it did not exist or @p error is set. + */ + bool removeNoFollow(std::filesystem::path const& path, std::error_code& error); + + /** + * @brief An item that could not be removed, and why. + */ + struct RemoveFailure + { + std::filesystem::path path; + std::error_code error; + }; + + /** + * @brief The outcome of @ref removeAllNoFollow. + */ + struct RemoveAllResult + { + /** @brief Every entry removed, the given path included. */ + std::uintmax_t removed{0}; + /** @brief The items that stopped the removal, by their own path. Their parent directories stay as well. */ + std::vector failures{}; + }; /** * @brief Removes @p path and, for a real directory, everything below it. A symlink or junction is removed - * itself and never followed, so nothing outside @p path is touched. + * itself and never followed, so nothing outside @p path is touched. An item that cannot be removed does + * not stop the others, only the directories containing it stay. + */ + RemoveAllResult removeAllNoFollow(std::filesystem::path const& path); + + /** + * @brief @ref removeAllNoFollow that sets @p error to the error of the first item that could not be removed. * @return The number of removed entries, 0 when @p path does not exist. */ std::uintmax_t removeAllNoFollow(std::filesystem::path const& path, std::error_code& error); diff --git a/utility/include/utility/no_follow_tree_walk.hpp b/utility/include/utility/no_follow_tree_walk.hpp new file mode 100644 index 00000000..156e5608 --- /dev/null +++ b/utility/include/utility/no_follow_tree_walk.hpp @@ -0,0 +1,105 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace Utility +{ + /** + * @brief Walks a path and, for a real directory, the tree below it in bounded steps without following links or + * junctions. It either counts the entries or removes them, the content of a directory before the + * directory itself. A link is counted or removed itself. + * + * An item that cannot be removed does not stop the walk, only the directories containing it stay. + */ + class NoFollowTreeWalk + { + public: + enum class Mode + { + Count, + Remove + }; + + NoFollowTreeWalk(std::filesystem::path root, Mode mode); + ~NoFollowTreeWalk() = default; + NoFollowTreeWalk(NoFollowTreeWalk const&) = delete; + NoFollowTreeWalk& operator=(NoFollowTreeWalk const&) = delete; + NoFollowTreeWalk(NoFollowTreeWalk&&) = default; + NoFollowTreeWalk& operator=(NoFollowTreeWalk&&) = default; + + /** + * @brief Works until the walk is done or @p deadline has passed, handling at least one entry per call. + * @return True when the walk is done. + */ + bool step(std::chrono::steady_clock::time_point deadline); + + bool done() const noexcept + { + return done_; + } + + /** + * @brief The entries counted or removed so far, the root included. + */ + std::uintmax_t entries() const noexcept + { + return entries_; + } + + /** + * @brief Count mode: the size of the regular files counted so far. + */ + std::uintmax_t bytes() const noexcept + { + return bytes_; + } + + /** + * @brief Remove mode: the items that could not be removed or listed, by their own path. + */ + std::vector const& failures() const noexcept + { + return failures_; + } + + /** + * @brief The entry handled last, empty before the first step. + */ + std::filesystem::path const& lastPath() const noexcept + { + return lastPath_; + } + + private: + struct Frame + { + std::filesystem::path directory; + std::filesystem::directory_iterator iterator; + /** @brief Something below could not be removed, so the directory stays. */ + bool keepsContent{false}; + }; + + void start(); + void handleEntry(std::filesystem::directory_entry const& entry); + void enter(std::filesystem::path const& directory); + void finishDirectory(); + void finishItem(std::filesystem::path const& path, NoFollowKind kind, bool directoryEntry, std::uintmax_t size); + void fail(std::filesystem::path const& path, std::error_code const& error); + + private: + std::filesystem::path root_; + Mode mode_; + bool started_{false}; + bool done_{false}; + std::vector stack_{}; + std::uintmax_t entries_{0}; + std::uintmax_t bytes_{0}; + std::vector failures_{}; + std::filesystem::path lastPath_{}; + }; +} diff --git a/utility/source/utility/CMakeLists.txt b/utility/source/utility/CMakeLists.txt index 0c048ba8..8544627e 100644 --- a/utility/source/utility/CMakeLists.txt +++ b/utility/source/utility/CMakeLists.txt @@ -25,6 +25,7 @@ else() permissions.cpp rename.cpp link_safe_remove.cpp + no_follow_tree_walk.cpp download_mark.cpp resources.cpp ) diff --git a/utility/source/utility/link_safe_remove.cpp b/utility/source/utility/link_safe_remove.cpp index b2ed6b2c..a1048b54 100644 --- a/utility/source/utility/link_safe_remove.cpp +++ b/utility/source/utility/link_safe_remove.cpp @@ -1,4 +1,9 @@ #include +#include +#include "no_follow_inspection.hpp" + +#include +#include #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN @@ -7,29 +12,165 @@ namespace Utility { - bool isLinkOrJunction(std::filesystem::path const& path, std::error_code& error) + namespace Detail { - error.clear(); + bool isLinkReparseTag(std::uint32_t tag) noexcept + { + // IsReparseTagNameSurrogate: the item stands for another named item, like a symlink or junction does. + constexpr std::uint32_t nameSurrogateBit = 0x20000000u; + return (tag & nameSurrogateBit) != 0; + } + #ifdef _WIN32 - const auto attributes = ::GetFileAttributesW(path.c_str()); - if (attributes == INVALID_FILE_ATTRIBUTES) + namespace + { + struct FindCloser + { + void operator()(HANDLE handle) const noexcept + { + if (handle != INVALID_HANDLE_VALUE) + ::FindClose(handle); + } + }; + using FindHandle = std::unique_ptr; + + bool isMissingError(DWORD lastError) + { + return lastError == ERROR_FILE_NOT_FOUND || lastError == ERROR_PATH_NOT_FOUND; + } + } + + NoFollowInspection inspectNoFollow(std::filesystem::path const& path, std::error_code& error) + { + error.clear(); + const auto attributes = ::GetFileAttributesW(path.c_str()); + if (attributes == INVALID_FILE_ATTRIBUTES) + { + const auto lastError = ::GetLastError(); + if (!isMissingError(lastError)) + error = std::error_code{static_cast(lastError), std::system_category()}; + return {}; + } + const bool directoryEntry = (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + if ((attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) + { + WIN32_FIND_DATAW data{}; + const FindHandle find{::FindFirstFileW(path.c_str(), &data)}; + if (find.get() == INVALID_HANDLE_VALUE) + { + const auto lastError = ::GetLastError(); + if (!isMissingError(lastError)) + error = std::error_code{static_cast(lastError), std::system_category()}; + return {}; + } + if (isLinkReparseTag(data.dwReserved0)) + return {.kind = NoFollowKind::Link, .directoryEntry = directoryEntry}; + } + return { + .kind = directoryEntry ? NoFollowKind::Directory : NoFollowKind::Other, + .directoryEntry = directoryEntry, + }; + } + + bool removeInspected(std::filesystem::path const& path, NoFollowInspection const& inspection, std::error_code& error) { - const auto lastError = ::GetLastError(); - if (lastError != ERROR_FILE_NOT_FOUND && lastError != ERROR_PATH_NOT_FOUND) - error = std::error_code{static_cast(lastError), std::system_category()}; + error.clear(); + if (inspection.kind == NoFollowKind::Missing) + return false; + // A junction or directory link is removed like an empty directory, its target stays. + const auto removeOnce = [&path, &inspection]() + { + return inspection.directoryEntry ? ::RemoveDirectoryW(path.c_str()) != 0 : ::DeleteFileW(path.c_str()) != 0; + }; + if (removeOnce()) + return true; + auto lastError = ::GetLastError(); + if (isMissingError(lastError)) + return false; + if (lastError == ERROR_ACCESS_DENIED && inspection.kind != NoFollowKind::Link) + { + const auto attributes = ::GetFileAttributesW(path.c_str()); + if (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_READONLY) != 0) + { + const auto writable = attributes & ~static_cast(FILE_ATTRIBUTE_READONLY); + if (::SetFileAttributesW(path.c_str(), writable == 0 ? FILE_ATTRIBUTE_NORMAL : writable) != 0) + { + if (removeOnce()) + return true; + lastError = ::GetLastError(); + ::SetFileAttributesW(path.c_str(), attributes); + } + } + } + error = std::error_code{static_cast(lastError), std::system_category()}; return false; } - return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0; #else - const auto status = std::filesystem::symlink_status(path, error); - if (status.type() == std::filesystem::file_type::not_found) + NoFollowInspection inspectionOf(std::filesystem::file_status const& status) + { + switch (status.type()) + { + case std::filesystem::file_type::none: + case std::filesystem::file_type::not_found: + return {}; + case std::filesystem::file_type::symlink: + return {.kind = NoFollowKind::Link}; + case std::filesystem::file_type::directory: + return {.kind = NoFollowKind::Directory, .directoryEntry = true}; + default: + return {.kind = NoFollowKind::Other}; + } + } + + NoFollowInspection inspectNoFollow(std::filesystem::path const& path, std::error_code& error) + { + const auto status = std::filesystem::symlink_status(path, error); + if (status.type() == std::filesystem::file_type::not_found) + error.clear(); + if (error) + return {}; + return inspectionOf(status); + } + + bool removeInspected(std::filesystem::path const& path, NoFollowInspection const& inspection, std::error_code& error) + { error.clear(); - return !error && std::filesystem::is_symlink(status); + if (inspection.kind == NoFollowKind::Missing) + return false; + // remove(3) unlinks a link itself and removes a directory only when it is empty. + return std::filesystem::remove(path, error); + } #endif } - std::optional - firstLinkBelow(std::filesystem::path const& root, std::filesystem::path const& path, std::error_code& error) + NoFollowKind noFollowKindOf(std::filesystem::path const& path, std::error_code& error) + { + return Detail::inspectNoFollow(path, error).kind; + } + + bool isLinkOrJunction(std::filesystem::path const& path, std::error_code& error) + { + return noFollowKindOf(path, error) == NoFollowKind::Link; + } + + bool VerifiedDirectories::contains(std::filesystem::path const& directory) const + { + const std::scoped_lock lock{mutex_}; + return directories_.contains(directory); + } + + void VerifiedDirectories::insert(std::filesystem::path const& directory) + { + const std::scoped_lock lock{mutex_}; + directories_.insert(directory); + } + + std::optional firstLinkBelow( + std::filesystem::path const& root, + std::filesystem::path const& path, + std::error_code& error, + VerifiedDirectories* verified + ) { error.clear(); const auto relative = path.lexically_relative(root); @@ -44,70 +185,49 @@ namespace Utility if (component == ".") continue; current /= component; - const bool isLink = isLinkOrJunction(current, error); + if (verified && verified->contains(current)) + continue; + const auto kind = noFollowKindOf(current, error); if (error) return std::nullopt; - if (isLink) - return current; + switch (kind) + { + case NoFollowKind::Link: + return current; + case NoFollowKind::Missing: + return std::nullopt; + case NoFollowKind::Directory: + { + if (verified) + verified->insert(current); + break; + } + case NoFollowKind::Other: + break; + } } return std::nullopt; } - namespace + bool removeNoFollow(std::filesystem::path const& path, std::error_code& error) { - bool removeLink(std::filesystem::path const& path, std::error_code& error) - { -#ifdef _WIN32 - // A junction or directory symlink is removed like an empty directory, its target stays. - const auto attributes = ::GetFileAttributesW(path.c_str()); - const bool removed = (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0 ? ::RemoveDirectoryW(path.c_str()) != 0 - : ::DeleteFileW(path.c_str()) != 0; - if (!removed) - error = std::error_code{static_cast(::GetLastError()), std::system_category()}; - return removed; -#else - return std::filesystem::remove(path, error); -#endif - } + const auto inspection = Detail::inspectNoFollow(path, error); + if (error) + return false; + return Detail::removeInspected(path, inspection, error); } - std::uintmax_t removeAllNoFollow(std::filesystem::path const& path, std::error_code& error) + RemoveAllResult removeAllNoFollow(std::filesystem::path const& path) { - error.clear(); - const bool isLink = isLinkOrJunction(path, error); - if (error) - return 0; - if (isLink) - return removeLink(path, error) ? 1u : 0u; - - const auto status = std::filesystem::symlink_status(path, error); - if (status.type() == std::filesystem::file_type::not_found) - { - error.clear(); - return 0; - } - if (error) - return 0; + NoFollowTreeWalk walk{path, NoFollowTreeWalk::Mode::Remove}; + walk.step(std::chrono::steady_clock::time_point::max()); + return {.removed = walk.entries(), .failures = walk.failures()}; + } - std::uintmax_t removed = 0; - if (std::filesystem::is_directory(status)) - { - std::filesystem::directory_iterator iter{path, error}; - if (error) - return removed; - for (; iter != std::filesystem::directory_iterator{}; iter.increment(error)) - { - if (error) - return removed; - removed += removeAllNoFollow(iter->path(), error); - if (error) - return removed; - } - if (error) - return removed; - } - if (std::filesystem::remove(path, error)) - ++removed; - return removed; + std::uintmax_t removeAllNoFollow(std::filesystem::path const& path, std::error_code& error) + { + auto result = removeAllNoFollow(path); + error = result.failures.empty() ? std::error_code{} : result.failures.front().error; + return result.removed; } } diff --git a/utility/source/utility/no_follow_inspection.hpp b/utility/source/utility/no_follow_inspection.hpp new file mode 100644 index 00000000..336c142d --- /dev/null +++ b/utility/source/utility/no_follow_inspection.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include +#include + +namespace Utility::Detail +{ + /** + * @brief What an item is, inspected without following it, with what removing it needs to know. + */ + struct NoFollowInspection + { + NoFollowKind kind{NoFollowKind::Missing}; + /** @brief The item has the directory attribute, which on Windows includes directory links and junctions. */ + bool directoryEntry{false}; + }; + + /** + * @brief Inspects @p path without following it, a missing path is no error. + */ + NoFollowInspection inspectNoFollow(std::filesystem::path const& path, std::error_code& error); + +#ifndef _WIN32 + /** + * @brief The inspection a symlink_status gives, which on POSIX is all there is to know. + */ + NoFollowInspection inspectionOf(std::filesystem::file_status const& status); +#endif + + /** + * @brief Removes the item at @p path that @p inspection describes, never following it. + * @return True when it was removed, false when it was gone or @p error is set. + */ + bool removeInspected(std::filesystem::path const& path, NoFollowInspection const& inspection, std::error_code& error); +} diff --git a/utility/source/utility/no_follow_tree_walk.cpp b/utility/source/utility/no_follow_tree_walk.cpp new file mode 100644 index 00000000..5e807576 --- /dev/null +++ b/utility/source/utility/no_follow_tree_walk.cpp @@ -0,0 +1,172 @@ +#include +#include "no_follow_inspection.hpp" + +#include + +namespace Utility +{ + namespace + { + Detail::NoFollowInspection inspectEntry(std::filesystem::directory_entry const& entry, std::error_code& error) + { +#ifdef _WIN32 + // The cached type does not tell a junction from a directory, the reparse tag does. + return Detail::inspectNoFollow(entry.path(), error); +#else + const auto status = entry.symlink_status(error); + if (status.type() == std::filesystem::file_type::not_found) + error.clear(); + if (error) + return {}; + return Detail::inspectionOf(status); +#endif + } + } + + NoFollowTreeWalk::NoFollowTreeWalk(std::filesystem::path root, Mode mode) + : root_{std::move(root)} + , mode_{mode} + {} + + bool NoFollowTreeWalk::step(std::chrono::steady_clock::time_point deadline) + { + if (done_) + return true; + if (!started_) + { + started_ = true; + start(); + } + do + { + if (stack_.empty()) + { + done_ = true; + return true; + } + auto& frame = stack_.back(); + if (frame.iterator == std::filesystem::directory_iterator{}) + { + finishDirectory(); + continue; + } + const auto entry = *frame.iterator; + std::error_code error{}; + frame.iterator.increment(error); + if (error) + { + fail(frame.directory, error); + frame.iterator = std::filesystem::directory_iterator{}; + } + handleEntry(entry); + } while (std::chrono::steady_clock::now() < deadline); + return false; + } + + void NoFollowTreeWalk::start() + { + std::error_code error{}; + const auto inspection = Detail::inspectNoFollow(root_, error); + if (error) + return fail(root_, error); + switch (inspection.kind) + { + case NoFollowKind::Missing: + return; + case NoFollowKind::Directory: + return enter(root_); + case NoFollowKind::Link: + case NoFollowKind::Other: + { + const auto size = mode_ == Mode::Count && inspection.kind == NoFollowKind::Other + ? std::filesystem::file_size(root_, error) + : std::uintmax_t{0}; + return finishItem(root_, inspection.kind, inspection.directoryEntry, error ? 0 : size); + } + } + } + + void NoFollowTreeWalk::handleEntry(std::filesystem::directory_entry const& entry) + { + std::error_code error{}; + const auto inspection = inspectEntry(entry, error); + if (error) + return fail(entry.path(), error); + switch (inspection.kind) + { + case NoFollowKind::Missing: + return; + case NoFollowKind::Directory: + return enter(entry.path()); + case NoFollowKind::Link: + case NoFollowKind::Other: + { + const bool countsBytes = mode_ == Mode::Count && inspection.kind == NoFollowKind::Other && + entry.is_regular_file(error); + const auto size = countsBytes ? entry.file_size(error) : std::uintmax_t{0}; + return finishItem(entry.path(), inspection.kind, inspection.directoryEntry, error ? 0 : size); + } + } + } + + void NoFollowTreeWalk::enter(std::filesystem::path const& directory) + { + lastPath_ = directory; + std::error_code error{}; + std::filesystem::directory_iterator iterator{directory, error}; + if (error) + { + if (mode_ == Mode::Count) + { + ++entries_; + return; + } + return fail(directory, error); + } + stack_.push_back(Frame{.directory = directory, .iterator = std::move(iterator)}); + } + + void NoFollowTreeWalk::finishDirectory() + { + auto frame = std::move(stack_.back()); + stack_.pop_back(); + if (frame.keepsContent) + { + if (!stack_.empty()) + stack_.back().keepsContent = true; + return; + } + finishItem(frame.directory, NoFollowKind::Directory, true, 0); + } + + void NoFollowTreeWalk::finishItem( + std::filesystem::path const& path, + NoFollowKind kind, + bool directoryEntry, + std::uintmax_t size + ) + { + lastPath_ = path; + if (mode_ == Mode::Count) + { + ++entries_; + bytes_ += size; + return; + } + std::error_code error{}; + if (Detail::removeInspected(path, {.kind = kind, .directoryEntry = directoryEntry}, error)) + ++entries_; + else if (error) + fail(path, error); + } + + void NoFollowTreeWalk::fail(std::filesystem::path const& path, std::error_code const& error) + { + lastPath_ = path; + if (mode_ == Mode::Count) + return; + failures_.push_back({.path = path, .error = error}); + if (!stack_.empty()) + stack_.back().keepsContent = true; + } +} From 50257fe6c4a271be00a45ed05676c069db258aa3 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:44:00 +0200 Subject: [PATCH 037/131] A bulk download inspects each local directory once for links A bulk download inspects each local directory once for links: the directories found to be real are shared by the bulk and its downloads, and only the components not verified yet are looked at --- backend/include/backend/sftp/download_operation.hpp | 4 ++++ backend/source/backend/sftp/bulk_download_operation.cpp | 8 ++++++-- backend/source/backend/sftp/download_operation.cpp | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/backend/include/backend/sftp/download_operation.hpp b/backend/include/backend/sftp/download_operation.hpp index e0ecc8f1..c9da4f95 100644 --- a/backend/include/backend/sftp/download_operation.hpp +++ b/backend/include/backend/sftp/download_operation.hpp @@ -8,9 +8,11 @@ #include #include #include +#include #include #include +#include #include class DownloadOperation : public Operation @@ -51,6 +53,8 @@ class DownloadOperation : public Operation * a link planted inside a downloaded tree cannot redirect the download out of it. */ Paths::LocalPath linkFreeRoot{}; + /** @brief Directories below @ref linkFreeRoot already found to be no links, shared by a bulk download. */ + std::shared_ptr verifiedDirectories{}; }; SecureShell::ProcessingStrand* strand() const override diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index ee190c22..31dc7df0 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -27,7 +27,9 @@ namespace BulkDownloadOperation::BulkDownloadOperation(SecureShell::SftpSession& sftp, BulkDownloadOperationOptions options) : BulkTransferOperation{sftp, toCommon(options)} , individualOptions_{std::move(options.individualOptions)} -{} +{ + individualOptions_.verifiedDirectories = std::make_shared(); +} BulkDownloadOperation::~BulkDownloadOperation() = default; @@ -48,7 +50,9 @@ BulkDownloadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPath { // create_directories follows a link planted inside the tree and would create and fill directories outside it. std::error_code error{}; - const auto link = Utility::firstLinkBelow(common().localPath.native(), paths.local.native(), error); + const auto link = Utility::firstLinkBelow( + common().localPath.native(), paths.local.native(), error, individualOptions_.verifiedDirectories.get() + ); if (error) { return std::unexpected( diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index 3af0751b..e0240b7c 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -423,7 +423,9 @@ DownloadOperation::refuseLinkedPath(std::filesystem::path const& path) const std::error_code error{}; std::optional link{}; if (!options_.linkFreeRoot.empty()) - link = Utility::firstLinkBelow(options_.linkFreeRoot.native(), path, error); + link = Utility::firstLinkBelow( + options_.linkFreeRoot.native(), path, error, options_.verifiedDirectories.get() + ); else if (Utility::isLinkOrJunction(path, error)) link = path; if (error) From e0669b5c84266661862f802205cd6ec3973260e1 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:50:21 +0200 Subject: [PATCH 038/131] The queued local delete can delete whole trees The queued local delete can delete whole trees: in recursive mode it counts every target first and then removes it with its content, links and junctions removed themselves and never followed; every work step deletes until 20 ms are used and progress goes out at most ten times a second plus once at the end; a failure names the item that failed, the card is labelled with the directory the items lie in and can refresh the local listing, and a session that ends cancels its local deletes; Session::addLocalDelete queues such a delete after the local delete guard and the deletion setting allowed it --- backend/include/backend/session.hpp | 12 +- backend/include/backend/session_manager.hpp | 8 + .../backend/sftp/local_delete_operation.hpp | 63 +++++++- .../include/backend/sftp/operation_queue.hpp | 14 +- backend/source/backend/main.cpp | 2 + backend/source/backend/session.cpp | 39 ++++- backend/source/backend/session_manager.cpp | 8 +- .../backend/sftp/local_delete_operation.cpp | 146 +++++++++++++++++- .../source/backend/sftp/operation_queue.cpp | 46 +++++- 9 files changed, 320 insertions(+), 18 deletions(-) diff --git a/backend/include/backend/session.hpp b/backend/include/backend/session.hpp index bc0d81d3..e4e2c07a 100644 --- a/backend/include/backend/session.hpp +++ b/backend/include/backend/session.hpp @@ -36,7 +36,8 @@ class Session Nui::Window& wnd, Nui::RpcHub& hub, Persistence::SftpOptions const& sftpOptions, - BulkResumeRegistry* bulkResumeRegistry + BulkResumeRegistry* bulkResumeRegistry, + std::shared_ptr localOperationQueue ); Session(const Session&) = delete; @@ -205,6 +206,13 @@ class Session void registerRpcSftpAddBulkDownloadOperation(); void registerRpcSftpAddBulkUploadOperation(); void registerRpcSftpAddBulkDeleteOperation(); + + /** + * @brief Handles "Session::{id}::addLocalDelete" (operation id, local paths): queues the recursive delete of the + * local items in the local operation queue, held while the queue of this session is paused, see + * LocalOperationQueue::addExplorerDelete. + */ + void registerRpcAddLocalDeleteOperation(); void registerRpcSftpExistsBatch(); void registerRpcSftpOpenSyncSession(); void registerRpcSftpRecomputeSyncDiff(); @@ -299,4 +307,6 @@ class Session std::unordered_map, Ids::IdHash> sftpChannels_{}; /** @brief The temp file suffix of uploads, which names where a replaced item is kept during a rename. */ std::string remoteTempFileSuffix_; + /** @brief Runs the local deletes of this session, which go on when it ends. */ + std::shared_ptr localOperationQueue_; }; \ No newline at end of file diff --git a/backend/include/backend/session_manager.hpp b/backend/include/backend/session_manager.hpp index 708a313c..e4190ba1 100644 --- a/backend/include/backend/session_manager.hpp +++ b/backend/include/backend/session_manager.hpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -39,6 +40,11 @@ class SessionManager void registerRpc(); + /** + * @brief Whether the sessions refuse local deletes, as the local filesystem options say. Refused until set. + */ + void preventLocalDeletion(bool prevent); + void addPasswordProvider(int priority, PasswordProvider* provider); void joinSessionAdder(); void addSession( @@ -113,6 +119,8 @@ class SessionManager std::atomic_bool updateDispatchRunning_{false}; Persistence::StateHolder* stateHolder_{}; std::unordered_map, Ids::IdHash> sessions_{}; + /** @brief Read by the local operation queue on each local delete the explorer asks for. */ + std::shared_ptr localDeletionPrevented_{std::make_shared(true)}; std::map passwordProviders_{}; std::mutex passwordRequestMutex_{}; diff --git a/backend/include/backend/sftp/local_delete_operation.hpp b/backend/include/backend/sftp/local_delete_operation.hpp index 9f4f10d8..238e8423 100644 --- a/backend/include/backend/sftp/local_delete_operation.hpp +++ b/backend/include/backend/sftp/local_delete_operation.hpp @@ -2,10 +2,13 @@ #include #include +#include #include #include #include +#include +#include #include #include @@ -20,12 +23,43 @@ class LocalDeleteOperation : public Operation public: struct LocalDeleteOperationOptions { + /** + * @brief The item handled last, the entries handled so far and the total. Called at most every + * @ref progressInterval and once more when the deletion ends. + */ std::function itemsRemovedProgress = [](std::string const&, auto, auto) {}; - /** @brief Deleted from the back, so a directory goes before its content in this list. */ + /** + * @brief Recursive only: the bytes and entries counted before the deletion starts, at most every + * @ref progressInterval and once when the count is done. + */ + std::function scanProgress = [](auto, auto) {}; + /** @brief Deleted from the back, so without @ref recursive a directory goes before its content in this list. */ std::vector targets{}; + /** + * @brief Every target goes with everything below it, counted first. Without, exactly the targets are removed, + * a directory only once it is empty. + */ + bool recursive{false}; + /** + * @brief Empty, or what a sync compare saw at each target, index aligned with @ref targets. A target that no + * longer holds it is not deleted and fails with ChangedSinceCompare. + */ + std::vector comparedStates{}; + /** @brief How long one work step deletes before it hands back to the queue. */ + std::chrono::milliseconds stepBudget{20}; + std::chrono::milliseconds progressInterval{100}; }; + /** @brief At most this many failed items are listed, the others are only counted and logged. */ + constexpr static std::size_t maximumListedFailures = 1000; + + /** + * @brief The deepest directory that every one of @p targets lies below, the first target itself when it is a + * root. Unset when there are no targets or they share no directory. + */ + static std::optional commonParentOf(std::span targets); + explicit LocalDeleteOperation(LocalDeleteOperationOptions options); ~LocalDeleteOperation() override = default; LocalDeleteOperation(LocalDeleteOperation const&) = delete; @@ -58,19 +92,42 @@ class LocalDeleteOperation : public Operation return 1; } + /** + * @brief Deletes until the step budget is used. Completes as a partial success when some items could not be + * deleted, and fails only when not a single target could be deleted. An item that is already gone counts + * as deleted. + */ std::expected work() override; std::expected cancel(bool adoptCancelState) override; /** - * @brief Items that could not be deleted, with why. The rest was still deleted. + * @brief Items that could not be deleted, with why, at most @ref maximumListedFailures. The rest was still deleted. */ std::vector> const& getFailed() const noexcept { return failedEntries_; } + private: + bool workFlat(std::chrono::steady_clock::time_point deadline); + bool workRecursive(std::chrono::steady_clock::time_point deadline); + void finishWalk(); + void recordFailure(std::filesystem::path const& path, std::error_code const& error); + void recordFailure(std::string const& path, Error error); + void reportProgress(bool final); + std::uint64_t handledEntries() const; + private: LocalDeleteOperationOptions options_; std::vector> failedEntries_{}; - std::size_t deletedThusFar_{0}; + std::size_t failureCount_{0}; + std::size_t targetIndex_{0}; + std::size_t completedTargets_{0}; + bool counting_{false}; + std::optional walk_{}; + std::uint64_t countedEntries_{0}; + std::uint64_t countedBytes_{0}; + std::uint64_t removedEntries_{0}; + std::string lastItem_{}; + std::chrono::steady_clock::time_point lastProgress_{}; }; diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index 68480e09..d87c32a3 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -256,15 +256,25 @@ class OperationQueue ); /** - * @brief Queues the delete of local items as one operation with one card. Refused as a whole when one of them - * is a root directory or contains the home directory. + * @brief Queues the delete of local items as one operation with one card, labelled with the directory they lie + * in. Refused as a whole when one of them is a root directory or contains the home directory. + * @param recursive Every target goes with its content, see LocalDeleteOperationOptions::recursive. + * @param insertRefresh The card refreshes the local listing when the delete ends. */ std::expected addLocalDeleteOperation( Ids::OperationId operationId, std::vector targets, + bool recursive, + bool insertRefresh, SharedData::OperationMode mode = SharedData::OperationMode::Queued ); + /** + * @brief Ends every queued and running local delete as canceled, telling the frontend. A local delete cannot be + * resumed by another session, so a session ending stops it this way instead of leaving its card behind. + */ + void cancelLocalDeletes(); + /** * @brief A guard for the remote deletes of one request, sharing the home directory this queue looked up. */ diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index 4766419b..d4f37272 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -316,9 +316,11 @@ void Main::registerRpc() }, *opener_ ); + sshSessionManager_->preventLocalDeletion(true); return; } + sshSessionManager_->preventLocalDeletion(holder.stateCache().localFilesystemOptions.preventDeletion); rpcFilesystem_ = std::make_unique( window_.getExecutor(), window_, diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index d2fc30e9..a25532c9 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -85,17 +86,26 @@ Session::Session( Nui::Window& wnd, Nui::RpcHub& hub, Persistence::SftpOptions const& sftpOptions, - BulkResumeRegistry* bulkResumeRegistry + BulkResumeRegistry* bulkResumeRegistry, + std::shared_ptr localOperationQueue ) : RpcHelper::StrandRpc{executor, std::move(strand), wnd, hub} , id_{std::move(id)} , session_{std::move(session)} - , operationQueue_{std::make_shared< - OperationQueue>(executor_, strand_, wnd, hub, sftpOptions, id_, sftpOptions.concurrency.value_or(Persistence::defaultSftpConcurrency))} + , operationQueue_{std::make_shared( + executor_, + strand_, + wnd, + hub, + sftpOptions, + id_, + sftpOptions.concurrency.value_or(Persistence::defaultSftpConcurrency) + )} , bulkResumeRegistry_{bulkResumeRegistry} , remoteTempFileSuffix_{Persistence::effectiveTempFileSuffix( sftpOptions.uploadOptions ? sftpOptions.uploadOptions->tempFileSuffix : std::nullopt )} + , localOperationQueue_{std::move(localOperationQueue)} {} Session::~Session() @@ -137,6 +147,7 @@ void Session::start() self->registerRpcSftpAddBulkDownloadOperation(); self->registerRpcSftpAddBulkUploadOperation(); self->registerRpcSftpAddBulkDeleteOperation(); + self->registerRpcAddLocalDeleteOperation(); self->registerRpcSftpExistsBatch(); self->registerRpcSftpOpenSyncSession(); self->registerRpcSftpRecomputeSyncDiff(); @@ -190,6 +201,7 @@ void Session::stop() // registry TTL) handle cleanup if no resume occurs. if (self->bulkResumeRegistry_) self->operationQueue_->dumpBulkResumes(*self->bulkResumeRegistry_); + self->localOperationQueue_->release(self->id_); self->operationQueue_->cancelAll(); self->operationQueue_.reset(); @@ -1875,6 +1887,27 @@ void Session::registerRpcSftpAddBulkDeleteOperation() ); } +void Session::registerRpcAddLocalDeleteOperation() +{ + on(fmt::format("Session::{}::addLocalDelete", id_.value())) + .perform( + [weak = weak_from_this()]( + RpcHelper::RpcOnce&& reply, std::string const& operationIdString, std::vector const& paths + ) + { + auto self = weak.lock(); + if (!self || !self->operationQueue_) + return reply.error("Session no longer exists"); + const auto added = self->localOperationQueue_->addExplorerDelete( + Ids::makeOperationId(operationIdString), paths, self->id_, self->operationQueue_->paused() + ); + if (!added) + return reply.error(added.error()); + reply(SharedData::success()); + } + ); +} + void Session::registerRpcSftpExistsBatch() { // Batched existence probe over SFTP. Lets the frontend ask whether N diff --git a/backend/source/backend/session_manager.cpp b/backend/source/backend/session_manager.cpp index 4c010b13..ed77e5d3 100644 --- a/backend/source/backend/session_manager.cpp +++ b/backend/source/backend/session_manager.cpp @@ -107,6 +107,11 @@ SessionManager::SessionManager( , stateHolder_{&stateHolder} {} +void SessionManager::preventLocalDeletion(bool prevent) +{ + localDeletionPrevented_->store(prevent); +} + SessionManager::~SessionManager() { for (auto& [id, session] : sessions_) @@ -224,7 +229,8 @@ void SessionManager::addSession( *wnd_, *hub_, sessionOptions.sftpOptions.value(), - &bulkResumeRegistry_ + &bulkResumeRegistry_, + localOperationQueue_ ); const auto emplaced = sessions_.emplace(sessionId, session); if (!emplaced.second) diff --git a/backend/source/backend/sftp/local_delete_operation.cpp b/backend/source/backend/sftp/local_delete_operation.cpp index 8f522aa3..f7a9c082 100644 --- a/backend/source/backend/sftp/local_delete_operation.cpp +++ b/backend/source/backend/sftp/local_delete_operation.cpp @@ -1,13 +1,21 @@ #include +#include #include #include +#include #include #include +std::optional LocalDeleteOperation::commonParentOf(std::span targets) +{ + return Paths::commonParentOf(targets); +} + LocalDeleteOperation::LocalDeleteOperation(LocalDeleteOperationOptions options) : Operation{} , options_{std::move(options)} + , counting_{options_.recursive} {} std::expected LocalDeleteOperation::work() @@ -25,8 +33,13 @@ std::expected Loc } case Running: { - const auto total = options_.targets.size(); - if (deletedThusFar_ < total) + const auto deadline = std::chrono::steady_clock::now() + options_.stepBudget; + const bool finished = options_.recursive ? workRecursive(deadline) : workFlat(deadline); + reportProgress(finished); + if (!finished) + return WorkStatus::MoreWork; + walk_.reset(); + if (failureCount_ > 0 && removedEntries_ == 0 && completedTargets_ == 0) { auto const& [path, error] = failedEntries_.front(); return enterErrorState(Error{ @@ -34,9 +47,6 @@ std::expected Loc .extraInfo = fmt::format("{}: {}", path, error.extraInfo.value_or(std::string{})), }); } - // Nothing deleted at all is a failure, some items left behind a partial success. - if (total > 0 && failedEntries_.size() == total) - return enterErrorState(failedEntries_.back().second); state_ = Completed; return WorkStatus::Complete; } @@ -56,8 +66,134 @@ std::expected Loc return enterErrorState({.type = ErrorType::UnknownWorkState}); } +bool LocalDeleteOperation::workFlat(std::chrono::steady_clock::time_point deadline) +{ + auto const& targets = options_.targets; + do + { + if (targetIndex_ >= targets.size()) + return true; + const auto index = targets.size() - targetIndex_ - 1; + auto const& target = targets[index]; + const auto compared = index < options_.comparedStates.size() + ? std::optional{options_.comparedStates[index]} + : std::nullopt; + if (auto unchanged = ComparedStateCheck::requireUnchangedLocal(target.native(), compared); !unchanged) + { + recordFailure(target.toUtf8(), std::move(unchanged).error()); + } + else + { + std::error_code error{}; + if (Utility::removeNoFollow(target.native(), error)) + ++removedEntries_; + if (error) + recordFailure(target.native(), error); + else + ++completedTargets_; + } + lastItem_ = target.toUtf8(); + ++targetIndex_; + } while (std::chrono::steady_clock::now() < deadline); + return targetIndex_ >= targets.size(); +} + +bool LocalDeleteOperation::workRecursive(std::chrono::steady_clock::time_point deadline) +{ + auto const& targets = options_.targets; + while (true) + { + if (!walk_) + { + if (targetIndex_ >= targets.size()) + { + if (!counting_) + return true; + counting_ = false; + targetIndex_ = 0; + options_.scanProgress(countedBytes_, countedEntries_); + continue; + } + walk_.emplace( + targets[targets.size() - targetIndex_ - 1].native(), + counting_ ? Utility::NoFollowTreeWalk::Mode::Count : Utility::NoFollowTreeWalk::Mode::Remove + ); + } + const bool walked = walk_->step(deadline); + if (!walked) + return false; + finishWalk(); + if (std::chrono::steady_clock::now() >= deadline) + return false; + } +} + +void LocalDeleteOperation::finishWalk() +{ + if (counting_) + { + countedEntries_ += walk_->entries(); + countedBytes_ += walk_->bytes(); + } + else + { + removedEntries_ += walk_->entries(); + lastItem_ = Paths::nativeToUtf8(walk_->lastPath()); + for (auto const& failure : walk_->failures()) + recordFailure(failure.path, failure.error); + if (walk_->failures().empty()) + ++completedTargets_; + } + walk_.reset(); + ++targetIndex_; +} + +void LocalDeleteOperation::recordFailure(std::filesystem::path const& path, std::error_code const& error) +{ + recordFailure(Paths::nativeToUtf8(path), Error{.type = ErrorType::DeleteFailed, .extraInfo = Utility::errorMessage(error)}); +} + +void LocalDeleteOperation::recordFailure(std::string const& path, Error error) +{ + ++failureCount_; + if (failedEntries_.size() >= maximumListedFailures) + return; + Log::error("LocalDeleteOperation: Failed to delete '{}': {}", path, error.toString()); + failedEntries_.emplace_back(path, std::move(error)); + if (failedEntries_.size() == maximumListedFailures) + Log::error("LocalDeleteOperation: Further failures are counted, not listed."); +} + +std::uint64_t LocalDeleteOperation::handledEntries() const +{ + if (!options_.recursive) + return targetIndex_; + const auto running = walk_ && !counting_ ? walk_->entries() + walk_->failures().size() : 0; + return removedEntries_ + failureCount_ + running; +} + +void LocalDeleteOperation::reportProgress(bool final) +{ + const auto now = std::chrono::steady_clock::now(); + if (!final && now - lastProgress_ < options_.progressInterval) + return; + lastProgress_ = now; + if (counting_) + { + const auto entries = countedEntries_ + (walk_ ? walk_->entries() : 0); + const auto bytes = countedBytes_ + (walk_ ? walk_->bytes() : 0); + return options_.scanProgress(bytes, entries); + } + const auto current = handledEntries(); + const auto total = options_.recursive ? std::max(countedEntries_, current) : options_.targets.size(); + if (walk_) + lastItem_ = Paths::nativeToUtf8(walk_->lastPath()); + options_.itemsRemovedProgress(lastItem_, current, total); +} + std::expected LocalDeleteOperation::cancel(bool adoptCancelState) { + walk_.reset(); if (adoptCancelState) state_ = OperationState::Canceled; return {}; diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 07578bc1..1ac10e54 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1722,7 +1722,15 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( result.remoteDeletes.clear(); result.remoteDeleteSources.clear(); } - if (!localDeletes.empty() && !addLocalDeleteOperation(request.localDeleteId, std::move(localDeletes), request.mode)) + if (!localDeletes.empty() && + !localDeleteQueue.addDelete({ + .operationId = request.localDeleteId, + .targets = std::move(localDeletes), + .mode = request.mode, + .owner = sessionId_, + .held = paused_, + .comparedStates = std::move(localDeletesCompared), + })) { std::ranges::move(result.localDeletes, std::back_inserter(result.failed)); result.localDeletes.clear(); @@ -1744,6 +1752,8 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( std::expected OperationQueue::addLocalDeleteOperation( Ids::OperationId operationId, std::vector targets, + bool recursive, + bool insertRefresh, SharedData::OperationMode mode ) { @@ -1760,7 +1770,7 @@ std::expected OperationQueue::addLocalDeleteOperation( } } - const auto commonParent = targets.front().parent().value_or(targets.front()); + auto parent = LocalDeleteOperation::commonParentOf(targets); auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; targetQueue.emplace_back( operationId, @@ -1784,7 +1794,15 @@ std::expected OperationQueue::addLocalDeleteOperation( } ); }, + .scanProgress = + [progress = makeScanProgressCallback("onScanProgress", operationId)]( + std::uint64_t bytes, std::uint64_t entries + ) + { + progress(bytes, entries, entries); + }, .targets = std::move(targets), + .recursive = recursive, }) ); @@ -1795,12 +1813,34 @@ std::expected OperationQueue::addLocalDeleteOperation( .operationId = operationId, .type = SharedData::OperationType::LocalDelete, .mode = mode, - .localPath = commonParent, + .insertRefresh = insertRefresh, + .localPath = std::move(parent), + .scanPhase = recursive, } ); return {}; } +void OperationQueue::cancelLocalDeletes() +{ + // Assumed in strand + for (auto* queue : {&priorityOperations_, &operations_}) + { + std::erase_if( + *queue, + [this](auto& entry) + { + auto& [id, operation] = entry; + if (operation->type() != SharedData::OperationType::LocalDelete) + return false; + std::ignore = operation->cancel(true); + completeOperation(makeCompletedOperation(CompletionReason::Canceled, id, *operation)); + return true; + } + ); + } +} + RemoteDeleteGuard OperationQueue::remoteDeleteGuard(SecureShell::SftpSession& sftp) { return RemoteDeleteGuard{sftp, remoteHome_, sftpOpts_.operationTimeout.value_or(defaultFutureTimeout)}; From dbf38aa16eadcd69c13af157c7cc0ba6fd93bf71 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:52:28 +0200 Subject: [PATCH 039/131] RpcFilesystem::removeSome removes items without following links RpcFilesystem::removeSome removes items without following links: without recursion each item by itself (a directory only when empty), with recursion through the link-safe tree removal, and a failure lists the item that failed once by its own path; the unused RpcFilesystem::remove handler is gone --- backend/include/backend/local_removal.hpp | 37 +++++++++++++++++++ backend/include/backend/rpc_filesystem.hpp | 1 - backend/source/backend/local_removal.cpp | 43 ++++++++++++++++++++++ backend/source/backend/rpc_filesystem.cpp | 15 ++++---- 4 files changed, 87 insertions(+), 9 deletions(-) create mode 100644 backend/include/backend/local_removal.hpp create mode 100644 backend/source/backend/local_removal.cpp diff --git a/backend/include/backend/local_removal.hpp b/backend/include/backend/local_removal.hpp new file mode 100644 index 00000000..62cc55e7 --- /dev/null +++ b/backend/include/backend/local_removal.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +/** + * @brief What @ref removeLocalItems removed. + */ +struct LocalRemoval +{ + std::uintmax_t removed{0}; + /** @brief "path: reason" for every item that could not be removed, by the item's own path. */ + std::vector failures{}; +}; + +/** + * @brief A path that @ref removeLocalItems refused, so it removed nothing. + */ +struct LocalRemovalRefusal +{ + std::string path; + std::string reason; +}; + +/** + * @brief Removes each of the local items @p pathsUtf8 by itself, so a directory only when it is empty, without + * following links or junctions. An item that cannot be removed does not stop the others. Nothing is removed + * when one of the paths is refused by localDeleteTarget. A directory with its content goes through the + * LocalOperationQueue instead. + */ +std::expected +removeLocalItems(std::span pathsUtf8, std::vector const& homes); diff --git a/backend/include/backend/rpc_filesystem.hpp b/backend/include/backend/rpc_filesystem.hpp index 2fed420a..c50a2dc9 100644 --- a/backend/include/backend/rpc_filesystem.hpp +++ b/backend/include/backend/rpc_filesystem.hpp @@ -19,7 +19,6 @@ class RpcFilesystem : public RpcHelper::StrandRpc ); private: - void registerRemove(); void registerRemoveMultiple(); void registerRename(); void registerListFiles(); diff --git a/backend/source/backend/local_removal.cpp b/backend/source/backend/local_removal.cpp new file mode 100644 index 00000000..98827881 --- /dev/null +++ b/backend/source/backend/local_removal.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +std::expected +removeLocalItems(std::span pathsUtf8, std::vector const& homes) +{ + std::vector targets; + targets.reserve(pathsUtf8.size()); + for (auto const& pathUtf8 : pathsUtf8) + { + auto target = localDeleteTarget(pathUtf8, homes); + if (!target) + return std::unexpected(LocalRemovalRefusal{.path = pathUtf8, .reason = std::move(target).error()}); + targets.push_back(*std::move(target)); + } + + LocalRemoval removal{}; + const auto addFailure = [&removal](std::filesystem::path const& path, std::error_code const& error) + { + const auto pathUtf8 = Paths::nativeToUtf8(path); + const auto message = Utility::errorMessage(error); + Log::error("Failed to remove local item '{}': {}", pathUtf8, message); + removal.failures.push_back(fmt::format("{}: {}", pathUtf8, message)); + }; + for (auto const& target : targets) + { + std::error_code error{}; + if (Utility::removeNoFollow(target.native(), error)) + ++removal.removed; + if (error) + addFailure(target.native(), error); + } + return removal; +} diff --git a/backend/source/backend/rpc_filesystem.cpp b/backend/source/backend/rpc_filesystem.cpp index 615375b3..68b11151 100644 --- a/backend/source/backend/rpc_filesystem.cpp +++ b/backend/source/backend/rpc_filesystem.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -8,6 +7,7 @@ #include #include #include +#include #include #include @@ -181,7 +181,6 @@ RpcFilesystem::RpcFilesystem( , opener_{&opener} , riskyFilePolicy_{std::move(riskyFilePolicy)} { - registerRemove(); registerRemoveMultiple(); registerRename(); registerListFiles(); @@ -205,7 +204,7 @@ void RpcFilesystem::registerRemoveMultiple() { if (options_.preventDeletion) { - Log::warn("RpcFilesystem::remove called but deletion is prevented by configuration."); + Log::warn("RpcFilesystem::removeSome called but deletion is prevented by configuration."); return reply.error("backend.rpcFilesystem.deletionPrevented"); } @@ -227,18 +226,18 @@ void RpcFilesystem::registerRemoveMultiple() Utility::localizedMessage("backend.rpcFilesystem.removeRefused", removal.error().path) ); } - if (!failures.empty()) + if (!removal->failures.empty()) { return reply.error( Utility::localizedMessage( "backend.rpcFilesystem.removeSomeFailed", - failures.size(), - fmt::format("{}", fmt::join(failures, "\n")) + removal->failures.size(), + fmt::format("{}", fmt::join(removal->failures, "\n")) ) ); } - Log::info("Successfully removed all specified paths."); - return reply({{"success", true}, {"removedCount", removedCount}}); + Log::info("RpcFilesystem::removeSome removed {} entries.", removal->removed); + return reply({{"success", true}, {"removedCount", removal->removed}}); } ); } From 157bace2cb3fa8890c13cff330c4599e012bdec3 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:53:10 +0200 Subject: [PATCH 040/131] Add Utility::LifetimeToken for callbacks that outlive their owner Utility::LifetimeToken lets a callback check that the object that made it still exists: guarded wraps a callable so it does nothing once its owner is gone, and a copied or moved owner gets a token of its own --- utility/include/utility/lifetime_token.hpp | 83 ++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 utility/include/utility/lifetime_token.hpp diff --git a/utility/include/utility/lifetime_token.hpp b/utility/include/utility/lifetime_token.hpp new file mode 100644 index 00000000..0bdd64b4 --- /dev/null +++ b/utility/include/utility/lifetime_token.hpp @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include + +namespace Utility +{ + /** + * @brief Tells a callback whether the object that made it still exists. The object holds the token as a member + * and wraps callbacks that capture it with @ref guarded, which then do nothing once it is gone. + * + * A token belongs to the address of its owner: a copied or moved owner gets a new token, and assigning one owner + * to another keeps each one's own token, so callbacks of the old address never run on the new one. + */ + class LifetimeToken + { + public: + LifetimeToken() = default; + ~LifetimeToken() = default; + LifetimeToken(LifetimeToken const&) + : LifetimeToken{} + {} + LifetimeToken(LifetimeToken&&) noexcept + : LifetimeToken{} + {} + LifetimeToken& operator=(LifetimeToken const&) noexcept + { + return *this; + } + LifetimeToken& operator=(LifetimeToken&&) noexcept + { + return *this; + } + + /** + * @brief @p callable wrapped so it runs only while this token exists. The wrapper takes the same parameters, + * so code that inspects them, like an RPC back channel, sees the original signature. @p callable has + * one call operator, which returns nothing: a call after the owner is gone has nothing to return. + */ + template + auto guarded(CallableT&& callable) const + { + return guardedFor(std::forward(callable), &std::remove_cvref_t::operator()); + } + + /** + * @brief Checks whether the owner still exists, for callbacks that need the check without @ref guarded. + */ + std::weak_ptr watch() const + { + return alive_; + } + + private: + template + auto guardedFor(CallableT&& callable, void (ClassT::*)(ArgumentTs...) const) const + { + return guardedWith(std::forward(callable)); + } + + template + auto guardedFor(CallableT&& callable, void (ClassT::*)(ArgumentTs...)) const + { + return guardedWith(std::forward(callable)); + } + + template + auto guardedWith(CallableT&& callable) const + { + return [alive = std::weak_ptr{alive_}, + callable = std::forward(callable)](ArgumentTs... arguments) mutable + { + if (const auto locked = alive.lock(); !locked) + return; + callable(std::forward(arguments)...); + }; + } + + private: + std::shared_ptr alive_{std::make_shared(true)}; + }; +} From 837fd67e94943c5e48ae627042785bc8f431fbb8 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:01:23 +0200 Subject: [PATCH 041/131] Local deletes of a selection with a directory are queued with a card Deleting a local selection that contains a directory queues it as a local delete with a card (scan, progress, pause, cancel) that refreshes the local listing when it ends, so other local actions no longer wait for it; a selection of files only is still deleted at once, without recursion; the reply and dialog callbacks of the local delete do nothing once the side model is gone --- .../file_explorer/local_side_model.hpp | 10 ++ .../frontend/file_explorer/side_model.hpp | 3 + .../session_components/operation_queue.hpp | 16 ++ .../displayed_delete_operation.hpp | 2 +- .../file_explorer/local_side_model.cpp | 161 +++++++++++------- .../session_components/operation_queue.cpp | 41 ++++- 6 files changed, 174 insertions(+), 59 deletions(-) diff --git a/frontend/include/frontend/file_explorer/local_side_model.hpp b/frontend/include/frontend/file_explorer/local_side_model.hpp index b3abac5f..0381147c 100644 --- a/frontend/include/frontend/file_explorer/local_side_model.hpp +++ b/frontend/include/frontend/file_explorer/local_side_model.hpp @@ -153,6 +153,16 @@ class LocalSideModel void enqueueSingleUpload(SharedData::BulkUploadEntry const& entry, bool allowOverwrite, bool insertRefresh); + /** + * @brief Deletes @p targets through the backend at once, each item by itself, and refreshes the listing. + */ + void deleteFilesDirectly(std::vector const& targets); + + /** + * @brief Shows why deleting local items failed. + */ + void showDeleteFailed(std::string const& reason); + private: /** * @brief Ask the backend once which external-open operations the platform can perform diff --git a/frontend/include/frontend/file_explorer/side_model.hpp b/frontend/include/frontend/file_explorer/side_model.hpp index 1801f57e..11be4529 100644 --- a/frontend/include/frontend/file_explorer/side_model.hpp +++ b/frontend/include/frontend/file_explorer/side_model.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -171,4 +172,6 @@ class SideModel : public NuiFileExplorer::ISideModel std::function refreshCallback_{nullptr}; bool reapplySelectionOnce_{false}; std::string dropMetadata_{}; + /** @brief Guards callbacks that can arrive after the side model is gone, like RPC replies and dialog closes. */ + Utility::LifetimeToken lifetime_{}; }; \ No newline at end of file diff --git a/frontend/include/frontend/session_components/operation_queue.hpp b/frontend/include/frontend/session_components/operation_queue.hpp index 9b4d3d4c..c5700920 100644 --- a/frontend/include/frontend/session_components/operation_queue.hpp +++ b/frontend/include/frontend/session_components/operation_queue.hpp @@ -187,6 +187,22 @@ class OperationQueue std::function onEnqueued = {} ); + /** + * @brief Queues the delete of local items with their content in the backend's local operation queue, as one + * operation whose card shows here and refreshes the local listing when it ends. Needs no connection; with + * one the delete waits while this queue is paused. @p onAck gets the error when the backend refused it, + * nothing when it was queued. + */ + void enqueueLocalDelete( + std::vector const& targets, + std::function const& error)> onAck + ); + + /** + * @brief The queue belongs to a connected session and can take operations. + */ + bool isActive() const; + void addCompletionCallback(Ids::OperationId const& opId, std::function callback); /** @brief Register a callback that receives the source paths of the entries a bulk operation * could not transfer. Runs on completion, before the completion callback, because the diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp index 1f4e5f70..d5610706 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp @@ -78,7 +78,7 @@ struct DisplayedDeleteOperation : public OperationCard std::optional resumableDescriptor() const override { - // A local delete runs in this process and needs no connection to be redone. + // A local delete runs in the local operation queue, which a reconnect does not touch. if (isCompletedState() || type_ == SharedData::OperationType::LocalDelete) return std::nullopt; // A bulk delete is re-issued by the backend from its own backup. diff --git a/frontend/source/frontend/file_explorer/local_side_model.cpp b/frontend/source/frontend/file_explorer/local_side_model.cpp index 675bf69c..0e885107 100644 --- a/frontend/source/frontend/file_explorer/local_side_model.cpp +++ b/frontend/source/frontend/file_explorer/local_side_model.cpp @@ -23,8 +23,11 @@ #include #include +#include + #include #include +#include #include #include #include @@ -490,25 +493,41 @@ void LocalSideModel::onDelete(std::vector const& items) return; std::vector listItems; - bool anyDirectory = false; - for (const auto& item : items) - { - listItems.push_back({item.name, item.fullPath.generic_string()}); - anyDirectory = anyDirectory || item.type == NuiFileExplorer::Item::Type::Directory; - } + listItems.reserve(items.size()); + std::ranges::transform( + items, + std::back_inserter(listItems), + [](NuiFileExplorer::Item const& item) + { + return ConfirmDialog::OpenOptions::ListElement{ + .text = item.name, .description = item.fullPath.generic_string() + }; + } + ); + const bool anyDirectory = std::ranges::any_of( + items, + [](NuiFileExplorer::Item const& item) + { + return item.type == NuiFileExplorer::Item::Type::Directory; + } + ); // Local directories go with their content in one step, unlike remote ones there is no second question. - auto text = language->get("localSideModel", "deleteSelectedConfirm"); - if (anyDirectory) - text += "\n" + language->get("localSideModel", "deleteDirectoriesWithContent"); + auto text = anyDirectory ? fmt::format( + "{}\n{}", + language->get("localSideModel", "deleteSelectedConfirm"), + language->get("localSideModel", "deleteDirectoriesWithContent") + ) + : language->get("localSideModel", "deleteSelectedConfirm"); - confirmDialog_->open( - {.styleVariant = ScriptNuiComponents::StyleVariant::Warning, - .headerText = language->get("localSideModel", "deleteLocalItemsQuestion"), - .text = std::move(text), - .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, - .focusButton = ConfirmDialog::Button::No, - .listItems = listItems, - .onClose = [this, items](std::optional button) + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Warning, + .headerText = language->get("localSideModel", "deleteLocalItemsQuestion"), + .text = std::move(text), + .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, + .focusButton = ConfirmDialog::Button::No, + .listItems = std::move(listItems), + .onClose = lifetime_.guarded( + [this, items, anyDirectory](std::optional button) { if (!button || *button != ConfirmDialog::Button::Yes) { @@ -516,59 +535,87 @@ void LocalSideModel::onDelete(std::vector const& items) return; } - Nui::val args = Nui::val::object(); - args.set("paths", Nui::val::array()); - args.set("recursive", true); - + std::vector targets; + targets.reserve(items.size()); for (const auto& item : items) { auto path = localPathOf(item); if (!path) return showPathRefused(language->get("localSideModel", "deleteFilesFailedTitle"), path.error()); - args["paths"].call("push", path->toUtf8()); + targets.push_back(*std::move(path)); } - Nui::RpcClient::callWithBackChannel( - "RpcFilesystem::removeSome", - [this](Nui::val val) - { - if (!val.hasOwnProperty("success")) + // Deleting a directory can take long, so it runs in the local operation queue with a card of its own. + if (!anyDirectory) + return deleteFilesDirectly(targets); + if (!operationQueue_) + { + Log::warn("Cannot queue the delete of local items: no operation queue is set"); + return showDeleteFailed(language->get("operationQueue", "unknownError")); + } + operationQueue_->enqueueLocalDelete( + targets, + lifetime_.guarded( + [this](std::optional const& error) { - Log::error("Invalid response from RpcFilesystem::remove"); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = language->get("localSideModel", "deleteFilesFailedTitle"), - .text = language->get("localSideModel", "invalidResponseFromBackend"), - .buttons = ConfirmDialog::Button::Ok, - }); - return; + if (!error) + return; + const auto reason = language->translate(*error); + Log::error("Failed to queue the delete of local items: {}", reason); + showDeleteFailed(reason); } + ) + ); + } + ), + }); +} - const auto success = val["success"].as(); - if (!success) - { - const auto error = language->translate(val["error"].as()); - Log::error("Failed to delete files: {}", error); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = language->get("localSideModel", "deleteFilesFailedTitle"), - .text = error, - .buttons = ConfirmDialog::Button::Ok, - }); - // The other items may be gone, the listing has to show it. - navigateTo(currentPath_.value()); - return; - } +void LocalSideModel::deleteFilesDirectly(std::vector const& targets) +{ + Nui::val args = Nui::val::object(); + args.set("paths", Nui::val::array()); + for (auto const& target : targets) + args["paths"].call("push", target.toUtf8()); - Log::info("Files deleted successfully"); - // Refresh list: - navigateTo(currentPath_.value()); - }, - args - ); - }} + Nui::RpcClient::callWithBackChannel( + "RpcFilesystem::removeSome", + lifetime_.guarded( + [this](Nui::val val) + { + const auto error = [&val]() -> std::optional + { + if (!val.hasOwnProperty("success")) + return language->get("localSideModel", "invalidResponseFromBackend"); + if (val["success"].as()) + return std::nullopt; + return language->translate(val["error"].as()); + }(); + if (error) + { + Log::error("Failed to delete local items: {}", *error); + showDeleteFailed(*error); + } + else + Log::info("Local items deleted"); + // The listing shows what is gone, also after a failure that kept some items. + navigateTo(currentPath_.value()); + } + ), + args ); } + +void LocalSideModel::showDeleteFailed(std::string const& reason) +{ + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = language->get("localSideModel", "deleteFilesFailedTitle"), + .text = reason, + .buttons = ConfirmDialog::Button::Ok, + }); +} + void LocalSideModel::onTransfer( std::vector const& items, std::optional const& subDir diff --git a/frontend/source/frontend/session_components/operation_queue.cpp b/frontend/source/frontend/session_components/operation_queue.cpp index 8ac8f71f..35fd5477 100644 --- a/frontend/source/frontend/session_components/operation_queue.cpp +++ b/frontend/source/frontend/session_components/operation_queue.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -802,7 +803,8 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) cancelOperation(operation); }, impl_->autoClean, - std::function{} + localRefresh, + added.scanPhase.value_or(false) ); } else if (added.type == SharedData::OperationType::BulkDownload) @@ -1082,6 +1084,7 @@ void OperationQueue::onScanProgress(SharedData::ScanProgress const& progress) } case SharedData::OperationType::Delete: case SharedData::OperationType::BulkDelete: + case SharedData::OperationType::LocalDelete: { if (auto* card = operation->getCardSpecifically(); card) card->setScanProgress(progress.totalBytes, progress.totalScanned); @@ -1363,6 +1366,42 @@ void OperationQueue::onOperationCompleted(Nui::val val) Nui::globalEventContext.executeActiveEventsImmediately(); } +void OperationQueue::enqueueLocalDelete( + std::vector const& targets, + std::function const& error)> onAck +) +{ + std::vector paths; + paths.reserve(targets.size()); + std::ranges::transform(targets, std::back_inserter(paths), &Paths::LocalPath::toUtf8); + Log::info("Frontend Operation Queue local delete: {} items", paths.size()); + + const auto operationId = Ids::generateOperationId(); + LocalOperationRouter::instance().host(operationId, *this); + // With a connection the delete is held while the queue of the connection is paused. + const auto rpcName = isActive() ? fmt::format("Session::{}::addLocalDelete", impl_->sessionId.value()) + : std::string{"LocalOperationQueue::addDelete"}; + Nui::RpcClient::callWithBackChannel( + rpcName, + [onAck = std::move(onAck), operationId](SharedData::ErrorOrSuccess<> const& result) + { + if (!result) + { + LocalOperationRouter::instance().forget(operationId); + return onAck(result.error); + } + onAck(std::nullopt); + }, + operationId.value(), + paths + ); +} + +bool OperationQueue::isActive() const +{ + return impl_->sessionId.isValid(); +} + void OperationQueue::addCompletionCallback(Ids::OperationId const& opId, std::function callback) { impl_->completionCallbacks[opId.value()] = std::move(callback); From 811b75dbb618aa86c82e899fc2e4febdc9431c33 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:15:15 +0200 Subject: [PATCH 042/131] The sync diff compares each side with a per-pair baseline The sync diff can compare each entry with a per-pair baseline: every side of an entry is checked against its own record (type, size and time, a symlink by its target), so the clocks of the two sides never meet; an entry changed on both sides stays blocked for as long as it keeps its record, only a changed side is transferred, and a delete needs the entry and everything below it recorded and unchanged; without a baseline the legacy last sync time still applies, and directory modification times count as changes in neither --- shared_data/include/shared_data/sync/diff.hpp | 15 +- .../shared_data/sync/sync_baseline.hpp | 76 +++++++++ shared_data/source/shared_data/sync/diff.cpp | 97 ++++-------- .../source/shared_data/sync/sync_baseline.cpp | 147 ++++++++++++++++++ 4 files changed, 256 insertions(+), 79 deletions(-) create mode 100644 shared_data/include/shared_data/sync/sync_baseline.hpp create mode 100644 shared_data/source/shared_data/sync/sync_baseline.cpp diff --git a/shared_data/include/shared_data/sync/diff.hpp b/shared_data/include/shared_data/sync/diff.hpp index 1d9add0d..605fab16 100644 --- a/shared_data/include/shared_data/sync/diff.hpp +++ b/shared_data/include/shared_data/sync/diff.hpp @@ -36,8 +36,10 @@ namespace SharedData::Sync /// The local filesystem folds case (Windows, macOS), set by the backend. Remote names that differ only in /// case would land on one local file then. bool localCaseInsensitive{false}; - /// When this pair was last synchronized (seconds since the epoch); a side whose modification time is later - /// changed since. Unset for the first sync, newest wins then. + /** + * @brief The server looks like it folds case (Windows or macOS paths), set by the backend. Names that differ + * only in case or normalization would land on one remote file then. + */ std::optional lastSyncTime{}; }; BOOST_DESCRIBE_STRUCT( @@ -114,14 +116,11 @@ namespace SharedData::Sync * whose @ref ScanNode::childrenKnown is false, * - `options.ignoreHidden` filters per-name at every level. * + * A file that differs on both sides goes from the newer side to the other one in Both mode. + * * @see DiffSink::onProgress heartbeats fire every 512 compares. */ - void diffScanTrees( - ScanNode const& local, - ScanNode const& remote, - DiffOptions const& options, - DiffSink& sink - ); + void diffScanTrees(ScanNode const& local, ScanNode const& remote, DiffOptions const& options, DiffSink& sink); inline void to_json(nlohmann::json& j, DiffOptions const& s) { diff --git a/shared_data/include/shared_data/sync/sync_baseline.hpp b/shared_data/include/shared_data/sync/sync_baseline.hpp new file mode 100644 index 00000000..ae4bcedc --- /dev/null +++ b/shared_data/include/shared_data/sync/sync_baseline.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include + +namespace SharedData::Sync +{ + /** + * @brief One side of an entry as sync left it, compared with the next scan of the same side only, so the + * clocks of the two sides never meet. + */ + struct RecordedSide + { + FileType type{FileType::Unknown}; + std::uint64_t size{0}; + std::uint64_t mtime{0}; + /** @brief The link literal of a symlink, which compares by it instead of size and time. */ + std::optional linkTarget{}; + + bool operator==(RecordedSide const&) const = default; + }; + + /** + * @brief Both sides of an entry after it was last synchronized or found in agreement. + */ + struct SyncRecord + { + RecordedSide local{}; + RecordedSide remote{}; + + bool operator==(SyncRecord const&) const = default; + }; + + /** + * @brief What sync knows about one pair (saved session, local root, remote root) from earlier runs: a record per + * relKey for every entry that was synchronized successfully or found equal on both sides. An entry without + * a record was never in agreement. + */ + struct SyncBaseline + { + std::unordered_map records{}; + + bool operator==(SyncBaseline const&) const = default; + }; + + /** + * @brief How @p node is recorded: type, size and modification time, the link literal for a symlink. + */ + RecordedSide recordedSideOf(ScanNode const& node); + + /** + * @brief True when @p current is not what @p recorded holds. A directory changes only by becoming something + * else, its modification time and size do not count. A symlink compares by its literal. + */ + bool sideChanged(RecordedSide const& recorded, ScanNode const& current); + + /** + * @brief The record of an entry transferred from @p source: both sides hold what the source held. + */ + SyncRecord transferredRecordOf(ScanNode const& source); + + /** + * @brief Records every entry listed on both sides that the diff sees as equal, directories included, walking + * into directories present on both sides. + */ + SyncBaseline agreementOf(ScanNode const& local, ScanNode const& remote); + + void to_json(nlohmann::json& json, SyncBaseline const& baseline); + void from_json(nlohmann::json const& json, SyncBaseline& baseline); +} diff --git a/shared_data/source/shared_data/sync/diff.cpp b/shared_data/source/shared_data/sync/diff.cpp index fe5a7907..a2898758 100644 --- a/shared_data/source/shared_data/sync/diff.cpp +++ b/shared_data/source/shared_data/sync/diff.cpp @@ -17,6 +17,16 @@ namespace SharedData::Sync { constexpr std::uint64_t progressCheckpointInterval = 512; + /** + * @brief State of one merge walk. + */ + struct Walk + { + DiffOptions const& options; + DiffSink& sink; + std::uint64_t compared{0}; + }; + std::string joinRel(std::string const& parentRelKey, std::string const& name) { if (parentRelKey.empty()) @@ -138,16 +148,11 @@ namespace SharedData::Sync } } - /** - * @brief Forward declaration — recursion point for directory pairs. - */ void mergeChildren( std::vector const& localChildren, std::vector const& remoteChildren, std::string const& parentRelKey, - DiffOptions const& options, - DiffSink& sink, - std::uint64_t& comparedCounter + Walk& walk ); std::optional blockReasonOf(ScanNode const& node) @@ -166,36 +171,16 @@ namespace SharedData::Sync return BlockReason::Excluded; } - /** - * @brief The latest modification time in @p node and everything listed below it. - */ - std::uint64_t newestModification(ScanNode const& node) - { - auto newest = node.mtime; - for (auto const& child : node.children) - newest = std::max(newest, newestModification(child)); - return newest; - } - - /** - * @brief A delete must not remove what appeared or changed on that side since the last sync, nobody has - * seen it on the other side. - */ - bool changedSinceLastSync(ScanNode const& node, DiffOptions const& options) - { - return options.lastSyncTime && newestModification(node) > *options.lastSyncTime; - } - void handleBothPresent( ScanNode const& localChild, ScanNode const& remoteChild, std::string const& relKey, std::string const& parentRelKey, - DiffOptions const& options, - DiffSink& sink, - std::uint64_t& comparedCounter + Walk& walk ) { + auto const& options = walk.options; + auto& sink = walk.sink; // Unknown or untransferable on one side: neither side may be changed for this name. if (auto reason = blockReasonOf(localChild).or_else([&]() { return blockReasonOf(remoteChild); }); reason) { @@ -220,32 +205,13 @@ namespace SharedData::Sync if (sameDir) { if (options.recursive) - mergeChildren(localChild.children, remoteChild.children, relKey, options, sink, comparedCounter); + mergeChildren(localChild.children, remoteChild.children, relKey, walk); return; } if (!entriesDiffer(localChild, remoteChild)) return; - // With a known last sync, the side changed since then is the one that holds new data. - if (options.lastSyncTime) - { - const bool localChanged = localChild.mtime > *options.lastSyncTime; - const bool remoteChanged = remoteChild.mtime > *options.lastSyncTime; - const bool destinationChanged = (options.direction == Direction::Upload && remoteChanged) || - (options.direction == Direction::Download && localChanged); - if (options.direction == Direction::Both && localChanged && remoteChanged) - { - sink.onBlocked(relKey, BlockReason::ChangedOnBothSides); - return; - } - if (destinationChanged) - { - sink.onBlocked(relKey, BlockReason::ChangedOnDestination); - return; - } - } - const auto action = chooseTwoSidedAction(localChild, remoteChild, options); if (!action) return; @@ -331,11 +297,11 @@ namespace SharedData::Sync std::vector const& localChildren, std::vector const& remoteChildren, std::string const& parentRelKey, - DiffOptions const& options, - DiffSink& sink, - std::uint64_t& comparedCounter + Walk& walk ) { + auto const& options = walk.options; + auto& sink = walk.sink; std::size_t li = 0; std::size_t ri = 0; const auto caseConflicts = caseConflictsOf(localChildren, remoteChildren, options); @@ -348,9 +314,9 @@ namespace SharedData::Sync }; const auto bumpAndCheckpoint = [&]() { - ++comparedCounter; - if (comparedCounter % progressCheckpointInterval == 0) - sink.onProgress(comparedCounter); + ++walk.compared; + if (walk.compared % progressCheckpointInterval == 0) + sink.onProgress(walk.compared); }; while (li < localChildren.size() || ri < remoteChildren.size()) @@ -373,7 +339,7 @@ namespace SharedData::Sync if ((!options.ignoreHidden || !isHiddenName(lc.name)) && !blockedForCase(lc.name)) { const auto relKey = joinRel(parentRelKey, lc.name); - handleBothPresent(lc, rc, relKey, parentRelKey, options, sink, comparedCounter); + handleBothPresent(lc, rc, relKey, parentRelKey, walk); } ++li; ++ri; @@ -389,9 +355,6 @@ namespace SharedData::Sync else if (auto action = chooseLocalOnlyAction(lc, options); action) { const auto relKey = joinRel(parentRelKey, lc.name); - if (*action == Action::DeleteLocal && changedSinceLastSync(lc, options)) - sink.onBlocked(relKey, BlockReason::ChangedOnDestination); - else emitOneSided( makeOneSidedRow(lc, relKey, *action, true), *action, parentRelKey, options, sink ); @@ -410,9 +373,6 @@ namespace SharedData::Sync else if (auto action = chooseRemoteOnlyAction(rc, options); action) { const auto relKey = joinRel(parentRelKey, rc.name); - if (*action == Action::DeleteRemote && changedSinceLastSync(rc, options)) - sink.onBlocked(relKey, BlockReason::ChangedOnDestination); - else emitOneSided( makeOneSidedRow(rc, relKey, *action, false), *action, parentRelKey, options, sink ); @@ -492,17 +452,12 @@ namespace SharedData::Sync return false; } - void diffScanTrees( - ScanNode const& local, - ScanNode const& remote, - DiffOptions const& options, - DiffSink& sink - ) + void diffScanTrees(ScanNode const& local, ScanNode const& remote, DiffOptions const& options, DiffSink& sink) { - std::uint64_t compared = 0; - mergeChildren(local.children, remote.children, std::string{}, options, sink, compared); + Walk walk{.options = options, .sink = sink}; + mergeChildren(local.children, remote.children, std::string{}, walk); // Final heartbeat so observers see the completed count even if it didn't // land on a checkpoint boundary. - sink.onProgress(compared); + sink.onProgress(walk.compared); } } diff --git a/shared_data/source/shared_data/sync/sync_baseline.cpp b/shared_data/source/shared_data/sync/sync_baseline.cpp new file mode 100644 index 00000000..bd78ef5b --- /dev/null +++ b/shared_data/source/shared_data/sync/sync_baseline.cpp @@ -0,0 +1,147 @@ +#include +#include + +#include +#include + +namespace SharedData::Sync +{ + namespace + { + std::string joinRelKey(std::string const& parent, std::string const& name) + { + return parent.empty() ? name : fmt::format("{}/{}", parent, name); + } + + void collectAgreement( + std::vector const& localChildren, + std::vector const& remoteChildren, + std::string const& parentRelKey, + SyncBaseline& baseline + ) + { + auto localIter = localChildren.begin(); + auto remoteIter = remoteChildren.begin(); + while (localIter != localChildren.end() && remoteIter != remoteChildren.end()) + { + if (localIter->name < remoteIter->name) + { + ++localIter; + continue; + } + if (remoteIter->name < localIter->name) + { + ++remoteIter; + continue; + } + auto const& local = *localIter++; + auto const& remote = *remoteIter++; + const bool listed = + local.availability == Availability::Listed && remote.availability == Availability::Listed; + const bool directories = local.type == FileType::Directory && remote.type == FileType::Directory; + if (!listed || local.type != remote.type || (!directories && entriesDiffer(local, remote))) + continue; + auto relKey = joinRelKey(parentRelKey, local.name); + if (directories) + collectAgreement(local.children, remote.children, relKey, baseline); + baseline.records.insert_or_assign( + std::move(relKey), SyncRecord{.local = recordedSideOf(local), .remote = recordedSideOf(remote)} + ); + } + } + + nlohmann::json sideToJson(RecordedSide const& side) + { + return nlohmann::json::array({static_cast(side.type), side.size, side.mtime}); + } + + RecordedSide sideFromJson(nlohmann::json const& json, std::size_t offset) + { + return RecordedSide{ + .type = static_cast(json.at(offset).get()), + .size = json.at(offset + 1).get(), + .mtime = json.at(offset + 2).get(), + }; + } + + std::optional linkTargetFromJson(nlohmann::json const& json, std::size_t index) + { + if (json.size() <= index || json.at(index).is_null()) + return std::nullopt; + return json.at(index).get(); + } + } + + RecordedSide recordedSideOf(ScanNode const& node) + { + return RecordedSide{ + .type = node.type, + .size = node.type == FileType::Directory ? 0u : node.size, + .mtime = node.type == FileType::Directory ? 0u : node.mtime, + .linkTarget = node.type == FileType::Symlink ? node.linkTarget : std::nullopt, + }; + } + + bool sideChanged(RecordedSide const& recorded, ScanNode const& current) + { + if (recorded.type != current.type) + return true; + switch (current.type) + { + case FileType::Directory: + return false; + case FileType::Symlink: + return recorded.linkTarget != current.linkTarget; + default: + return recorded.size != current.size || recorded.mtime != current.mtime; + } + } + + SyncRecord transferredRecordOf(ScanNode const& source) + { + const auto side = recordedSideOf(source); + return SyncRecord{.local = side, .remote = side}; + } + + SyncBaseline agreementOf(ScanNode const& local, ScanNode const& remote) + { + SyncBaseline baseline; + collectAgreement(local.children, remote.children, std::string{}, baseline); + return baseline; + } + + void to_json(nlohmann::json& json, SyncBaseline const& baseline) + { + auto records = nlohmann::json::object(); + for (auto const& [relKey, record] : baseline.records) + { + auto entry = sideToJson(record.local); + const auto remote = sideToJson(record.remote); + entry.insert(entry.end(), remote.begin(), remote.end()); + if (record.local.linkTarget || record.remote.linkTarget) + { + entry.push_back(record.local.linkTarget ? nlohmann::json(*record.local.linkTarget) : nlohmann::json()); + entry.push_back( + record.remote.linkTarget ? nlohmann::json(*record.remote.linkTarget) : nlohmann::json() + ); + } + records[relKey] = std::move(entry); + } + json = nlohmann::json{{"records", std::move(records)}}; + } + + void from_json(nlohmann::json const& json, SyncBaseline& baseline) + { + baseline.records.clear(); + for (auto const& [relKey, entry] : json.at("records").items()) + { + auto local = sideFromJson(entry, 0); + auto remote = sideFromJson(entry, 3); + local.linkTarget = linkTargetFromJson(entry, 6); + remote.linkTarget = linkTargetFromJson(entry, 7); + baseline.records.insert_or_assign( + relKey, SyncRecord{.local = std::move(local), .remote = std::move(remote)} + ); + } + } +} From 6355991a7598dc89ba1c10b29577626354d8591b Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:23:38 +0200 Subject: [PATCH 043/131] A prescanned bulk works below its request's roots A prescanned bulk upload or download works below the roots its request names, the sync roots for a sync, and otherwise below the closest directories holding all its entries, so files of sibling directories no longer fail the link check against the first file's parent; empty directories of a sync are entries of its bulks (link check, permissions, pause, cancel, a failure of their own) instead of being created at once --- .../backend/sftp/bulk_download_operation.hpp | 2 + .../backend/sftp/bulk_transfer_operation.hpp | 3 +- .../backend/sftp/bulk_upload_operation.hpp | 5 + .../backend/sftp/bulk_download_operation.cpp | 6 +- .../backend/sftp/bulk_transfer_operation.cpp | 3 +- .../backend/sftp/bulk_upload_operation.cpp | 46 +++++- .../backend/sftp/local_delete_operation.cpp | 1 + .../source/backend/sftp/operation_queue.cpp | 143 ++++++++---------- .../frontend/sync_dialog/sync_dialog.cpp | 24 --- paths/include/paths/common_parent.hpp | 35 +++++ .../file_operations/bulk_transfer_request.hpp | 37 ++++- .../shared_data/sync/sync_execution.hpp | 3 - 12 files changed, 186 insertions(+), 122 deletions(-) create mode 100644 paths/include/paths/common_parent.hpp diff --git a/backend/include/backend/sftp/bulk_download_operation.hpp b/backend/include/backend/sftp/bulk_download_operation.hpp index c9b76075..b8875d0f 100644 --- a/backend/include/backend/sftp/bulk_download_operation.hpp +++ b/backend/include/backend/sftp/bulk_download_operation.hpp @@ -51,6 +51,8 @@ class BulkDownloadOperation : public BulkTransferOperation std::uint64_t mtime{0}; std::uint32_t mtimeNsec{0}; bool mayOverwrite{false}; + /** @brief A directory created without its content, with the link check of every local directory. */ + bool isDirectory{false}; }; void setPrescannedFileList(std::vector files); diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index f40265f0..c39b1a63 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -160,7 +160,8 @@ class BulkTransferOperation : public Operation BulkTransferOperation(SecureShell::SftpSession& sftp, CommonOptions common); /** - * @brief Flat-list mode: every entry is a file, @p paths holds both of its ends (index aligned). + * @brief Flat-list mode: every entry is a file or a directory created without content, @p paths holds both of + * its ends (index aligned). A directory that cannot be created fails only its own entry. */ void setPrescannedEntries( std::vector entries, diff --git a/backend/include/backend/sftp/bulk_upload_operation.hpp b/backend/include/backend/sftp/bulk_upload_operation.hpp index 4eb22e1e..fbbe41dc 100644 --- a/backend/include/backend/sftp/bulk_upload_operation.hpp +++ b/backend/include/backend/sftp/bulk_upload_operation.hpp @@ -40,6 +40,8 @@ class BulkUploadOperation : public BulkTransferOperation Paths::RemotePath destination; std::uint64_t sizeBytes; bool mayOverwrite{false}; + /** @brief A directory created without its content, its missing parents below the bulk's root with it. */ + bool isDirectory{false}; }; void setPrescannedFileList(std::vector files); @@ -65,6 +67,9 @@ class BulkUploadOperation : public BulkTransferOperation private: std::filesystem::perms determinePerms(SharedData::DirectoryEntry const& entry) const; + /** + * @brief Creates the missing directories between the bulk's remote root and @p path, not @p path itself. + */ std::expected createRemoteDirectoryInStrand(Paths::RemotePath const& path, SharedData::DirectoryEntry const& entry); diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index 31dc7df0..13034949 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -200,11 +200,11 @@ void BulkDownloadOperation::setPrescannedFileList(std::vector fi // Flat entries carry their name only, the paths come from the prescanned paths. if (const auto name = file.source.filename(); name) entry.name = name->value(); - entry.type = SharedData::FileType::Regular; - entry.size = file.sizeBytes; + entry.type = file.isDirectory ? SharedData::FileType::Directory : SharedData::FileType::Regular; + entry.size = file.isDirectory ? 0u : file.sizeBytes; entry.mtime = file.mtime; entry.mtimeNsec = file.mtimeNsec; - totalBytes += file.sizeBytes; + totalBytes += entry.size; entries.push_back(std::move(entry)); paths.push_back( EntryPaths{ diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 08d66cef..47315a80 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -241,7 +241,8 @@ std::expected BulkTransferOperation::fillSlo const auto result = createDirectoryInStrand(index, *paths); if (!result.has_value()) { - if (!isSkippableDirectoryError(result.error())) + // Without a tree, nothing else depends on a directory entry. + if (!prescanned() && !isSkippableDirectoryError(result.error())) return std::unexpected(result.error()); Log::warn("{}: Skipping directory '{}': {}", logName(), displayPath(index), result.error().toString()); failedEntries_.emplace_back(failureSource(index), result.error()); diff --git a/backend/source/backend/sftp/bulk_upload_operation.cpp b/backend/source/backend/sftp/bulk_upload_operation.cpp index c09678ae..bbf6eab9 100644 --- a/backend/source/backend/sftp/bulk_upload_operation.cpp +++ b/backend/source/backend/sftp/bulk_upload_operation.cpp @@ -4,6 +4,9 @@ #include #include +#include +#include + namespace { BulkTransferOperation::CommonOptions toCommon(BulkUploadOperation::BulkUploadOperationOptions const& options) @@ -42,9 +45,46 @@ BulkUploadOperation::prepareRootInStrand(SharedData::DirectoryEntry const& root) std::expected BulkUploadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) { + // A scanned directory comes after its parent, a prescanned one may be the first entry below it. + if (prescanned() && individualOptions_.createMissingDirectories) + { + if (auto parents = createMissingParentsInStrand(sftp, paths.remote); !parents) + return parents; + } return createRemoteDirectoryInStrand(paths.remote, entries()[entryIndex]); } +std::expected +BulkUploadOperation::createMissingParentsInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& path) +{ + auto const& root = common().remotePath; + std::vector missing; + for (auto parent = path.parent(); parent && *parent != root && parent->isWithin(root); parent = parent->parent()) + { + const auto stat = sftp.lstatInStrand(*parent); + if (stat) + break; + if (stat.error().sftpError != SSH_FX_NO_SUCH_FILE) + { + return std::unexpected( + Error{ + .type = ErrorType::SftpError, + .sftpError = stat.error(), + .extraInfo = fmt::format("Inspecting remote directory: {}", parent->value()) + } + ); + } + missing.push_back(*parent); + } + const SharedData::DirectoryEntry directory{.type = SharedData::FileType::Directory}; + for (auto const& parent : missing | std::views::reverse) + { + if (auto created = createRemoteDirectoryInStrand(sftp, parent, directory); !created) + return created; + } + return {}; +} + std::expected BulkUploadOperation::createRemoteDirectoryInStrand( Paths::RemotePath const& path, SharedData::DirectoryEntry const& entry @@ -139,9 +179,9 @@ void BulkUploadOperation::setPrescannedFileList(std::vector file // Flat entries carry their name only, the paths come from the prescanned paths. if (const auto name = file.source.filename(); name) entry.name = name->value(); - entry.type = SharedData::FileType::Regular; - entry.size = file.sizeBytes; - totalBytes += file.sizeBytes; + entry.type = file.isDirectory ? SharedData::FileType::Directory : SharedData::FileType::Regular; + entry.size = file.isDirectory ? 0u : file.sizeBytes; + totalBytes += entry.size; entries.push_back(std::move(entry)); paths.push_back( EntryPaths{ diff --git a/backend/source/backend/sftp/local_delete_operation.cpp b/backend/source/backend/sftp/local_delete_operation.cpp index f7a9c082..e60663f7 100644 --- a/backend/source/backend/sftp/local_delete_operation.cpp +++ b/backend/source/backend/sftp/local_delete_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 1ac10e54..f015bf28 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,9 @@ #include #include #include +#include +#include +#include using namespace std::chrono_literals; @@ -38,6 +42,19 @@ namespace return path; } + /** + * @brief The closest directory holding the path @p member of every file of a prescanned bulk. + */ + template + PathT commonParentOf(std::vector const& files, PathT FileT::* member) + { + std::vector paths; + paths.reserve(files.size()); + std::ranges::transform(files, std::back_inserter(paths), member); + auto parent = Paths::commonParentOf(std::span{paths}); + return parent ? *std::move(parent) : PathT{}; + } + OperationQueue::OperationCompleted makeCompletedOperation( OperationQueue::CompletionReason reason, Ids::OperationId operationId, @@ -1155,19 +1172,10 @@ std::size_t OperationQueue::addBulkDownloadOperation( const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); auto& targetQueue = (request.mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - std::vector files; - files.reserve(request.entries.size()); - std::size_t directoryCount = 0; - // Each directory resumes as its own unit under its own id, so the card's backup holds only the files. - SharedData::BulkDownloadRequest fileUnit{ - .allowOverwrite = request.allowOverwrite, - .insertRefresh = request.insertRefresh, - .mode = request.mode, - }; - for (std::size_t idx = 0; idx < request.entries.size(); ++idx) - { - auto const& entry = request.entries[idx]; - if (entry.isDirectory) + return splitBulkRequest( + std::move(request), + bulkCardId, + [&](std::size_t index, SharedData::BulkDownloadEntry const& entry, SharedData::BulkDownloadRequest const& unit) { const auto result = addDownloadOperation( sftp, @@ -1205,6 +1213,7 @@ std::size_t OperationQueue::addBulkDownloadOperation( .mtime = entry.mtime, .mtimeNsec = entry.mtimeNsec, .mayOverwrite = entry.mayOverwrite, + .isDirectory = entry.isDirectory, } ); } @@ -1213,13 +1222,16 @@ std::size_t OperationQueue::addBulkDownloadOperation( { // Dedicated operationId for the aggregate bulk-download card, reserved // by the frontend separately from the per-entry ids to avoid collisions - // (entry-0's id would clash when entry 0 is a directory). The first - // file entry drives the UI's "localPath"/"remotePath" labels for the - // card — arbitrary pick; per-file paths show up in progress events. + // (entry-0's id would clash when entry 0 is a directory). const auto bulkOpId = bulkCardId; - // The card's labels, a file directly below the root labels with the root itself. - const auto remoteLabel = files.front().source.parent().value_or(Paths::RemotePath::root()); - const auto localLabel = files.front().destination.parent().value_or(files.front().destination); + const auto remoteLabel = request.remoteRoot + ? *request.remoteRoot + : commonParentOf(files, &BulkDownloadOperation::PrescannedFile::source); + const auto localLabel = request.localRoot + ? *request.localRoot + : commonParentOf(files, &BulkDownloadOperation::PrescannedFile::destination); + fileUnit.remoteRoot = remoteLabel; + fileUnit.localRoot = localLabel; auto downloadOpts = resolveDownloadOptions(transferOptions, resolvedTimeout); // Sync plans list files of directories that do not exist locally yet. @@ -1437,19 +1449,10 @@ std::size_t OperationQueue::addBulkUploadOperation( const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); auto& targetQueue = (request.mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - std::vector files; - files.reserve(request.entries.size()); - std::size_t directoryCount = 0; - // Each directory resumes as its own unit under its own id, so the card's backup holds only the files. - SharedData::BulkUploadRequest fileUnit{ - .allowOverwrite = request.allowOverwrite, - .insertRefresh = request.insertRefresh, - .mode = request.mode, - }; - for (std::size_t idx = 0; idx < request.entries.size(); ++idx) - { - auto const& entry = request.entries[idx]; - if (entry.isDirectory) + return splitBulkRequest( + std::move(request), + bulkCardId, + [&](std::size_t index, SharedData::BulkUploadEntry const& entry, SharedData::BulkUploadRequest const& unit) { const auto result = addUploadOperation( sftp, @@ -1485,6 +1488,7 @@ std::size_t OperationQueue::addBulkUploadOperation( .destination = entry.destination, .sizeBytes = entry.sizeBytes, .mayOverwrite = entry.mayOverwrite, + .isDirectory = entry.isDirectory, } ); } @@ -1493,8 +1497,13 @@ std::size_t OperationQueue::addBulkUploadOperation( { // Separate reserved id — see addBulkDownloadOperation for rationale. const auto bulkOpId = bulkCardId; - const auto localLabel = files.front().source.parent().value_or(files.front().source); - const auto remoteLabel = files.front().destination.parent().value_or(Paths::RemotePath::root()); + const auto localLabel = request.localRoot ? *request.localRoot + : commonParentOf(files, &BulkUploadOperation::PrescannedFile::source); + const auto remoteLabel = request.remoteRoot + ? *request.remoteRoot + : commonParentOf(files, &BulkUploadOperation::PrescannedFile::destination); + fileUnit.remoteRoot = remoteLabel; + fileUnit.localRoot = localLabel; auto uploadOpts = resolveUploadOptions(transferOptions, resolvedTimeout); uploadOpts.createMissingDirectories = true; @@ -1535,35 +1544,6 @@ std::size_t OperationQueue::addBulkUploadOperation( return request.entries.size(); } -namespace -{ - /** - * @brief Creates @p directory and its missing parents below @p root, which has to exist. - */ - bool createRemoteDirectoryBelow( - SecureShell::SftpSession& sftp, - Paths::RemotePath const& root, - Paths::RemotePath const& directory, - std::chrono::seconds timeout - ) - { - std::vector chain; - for (auto current = std::optional{directory}; current && *current != root && current->isWithin(root); - current = current->parent()) - chain.push_back(*current); - for (auto iter = chain.rbegin(); iter != chain.rend(); ++iter) - { - auto created = sftp.createDirectoryIfItDoesntExist(*iter); - if (created.wait_for(timeout) != std::future_status::ready || !created.get().has_value()) - { - Log::error("Sync: could not create remote directory '{}'.", iter->value()); - return false; - } - } - return true; - } -} - SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( SecureShell::SftpSession& sftp, SyncSession const& session, @@ -1573,11 +1553,18 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( { // Assumed in strand using SharedData::Sync::Action; - const auto timeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); SharedData::Sync::SyncExecutionResult result{.failed = std::move(plan.refused)}; - SharedData::BulkUploadRequest uploads{.allowOverwrite = true, .mode = request.mode}; - SharedData::BulkDownloadRequest downloads{.allowOverwrite = true, .mode = request.mode}; + SharedData::BulkUploadRequest uploads{ + .mode = request.mode, + .localRoot = session.localRoot(), + .remoteRoot = session.remoteRoot(), + }; + SharedData::BulkDownloadRequest downloads{ + .mode = request.mode, + .localRoot = session.localRoot(), + .remoteRoot = session.remoteRoot(), + }; SharedData::BulkDeleteRequest remoteDeletes{.mode = request.mode, .listedContentOnly = true}; std::vector localDeletes; const auto localHomes = localHomeDirectories(); @@ -1617,37 +1604,28 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( { case Action::Upload: { - if (entry.isDirectory) - { - auto& target = createRemoteDirectoryBelow(sftp, session.remoteRoot(), paths.remote, timeout) - ? result.createdDirectories - : result.failed; - target.push_back(std::move(entry.relKey)); - break; - } result.uploadSources.push_back(paths.local.toUtf8()); uploads.entries.push_back({ .source = std::move(paths.local), .destination = std::move(paths.remote), .sizeBytes = entry.sizeBytes, + .isDirectory = entry.isDirectory, + .withoutContent = entry.isDirectory, + .mayOverwrite = planned.compared.exists, + .comparedDestination = comparedDestination, }); result.uploads.push_back(std::move(entry.relKey)); break; } case Action::Download: { - if (entry.isDirectory) - { - std::error_code error{}; - std::filesystem::create_directories(paths.local.native(), error); - (error ? result.failed : result.createdDirectories).push_back(std::move(entry.relKey)); - break; - } result.downloadSources.push_back(paths.remote.value()); downloads.entries.push_back({ .source = std::move(paths.remote), .destination = std::move(paths.local), .sizeBytes = entry.sizeBytes, + .isDirectory = entry.isDirectory, + .withoutContent = entry.isDirectory, .mtime = entry.mtime, .mtimeNsec = entry.mtimeNsec, }); @@ -1738,12 +1716,11 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( } Log::info( - "Sync: queued {} uploads, {} downloads, {} remote and {} local deletes, created {} directories, {} failed", + "Sync: queued {} uploads, {} downloads, {} remote and {} local deletes, {} failed", result.uploads.size(), result.downloads.size(), result.remoteDeletes.size(), result.localDeletes.size(), - result.createdDirectories.size(), result.failed.size() ); return result; diff --git a/frontend/source/frontend/sync_dialog/sync_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_dialog.cpp index 3237ac1c..f1951d94 100644 --- a/frontend/source/frontend/sync_dialog/sync_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_dialog.cpp @@ -583,21 +583,6 @@ struct SyncDialog::Implementation return entry; } - /** @brief Observer of @p relKey plus those of the loaded file rows below it, so a directory - * entry paints the rows it covers. - */ - std::vector progressCovering(DiffSection section, std::string const& relKey) - { - std::vector observers{progressFor(section, relKey)}; - const auto prefix = relKey + "/"; - for (auto const& [key, observer] : progressMapFor(section)) - { - if (key.starts_with(prefix)) - observers.push_back(observer); - } - return observers; - } - ScriptNuiComponents::Tree::ChildrenLoader makeChildrenLoader(DiffSection section) { return [this, section]( @@ -1137,14 +1122,6 @@ struct SyncDialog::Implementation part.indexBySource.emplace(sources[index], index); } }; - const auto setCovering = [this, section](std::vector const& rows, double value) - { - for (auto const& row : rows) - { - for (auto const& observer : progressCovering(section, row)) - *observer = value; - } - }; // The backend lists every failed entry, the rows below a failed directory are not all failed. const auto setRows = [this, section](std::vector const& rows, double value) { @@ -1171,7 +1148,6 @@ struct SyncDialog::Implementation else part->map(); } - setCovering(result.createdDirectories, 1.1); setRows(result.failed, -1.0); Nui::globalEventContext.executeActiveEventsImmediately(); }, diff --git a/paths/include/paths/common_parent.hpp b/paths/include/paths/common_parent.hpp new file mode 100644 index 00000000..9ed07cc9 --- /dev/null +++ b/paths/include/paths/common_parent.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +namespace Paths +{ + /** + * @brief The closest directory that holds every one of @p paths below it. A path without a parent (a root) + * stands for itself, nothing for no paths. Works for LocalPath and RemotePath. + */ + template + std::optional commonParentOf(std::span paths) + { + if (paths.empty()) + return std::nullopt; + auto candidate = paths.front().parent(); + if (!candidate) + return paths.front(); + const auto containsAll = [&paths](PathT const& directory) + { + return std::ranges::all_of( + paths, + [&directory](PathT const& path) + { + return path != directory && path.isWithin(directory); + } + ); + }; + while (candidate && !containsAll(*candidate)) + candidate = candidate->parent(); + return candidate; + } +} diff --git a/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp b/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp index aee44469..b7d439b8 100644 --- a/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp +++ b/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp @@ -8,6 +8,7 @@ #include #include +#include #include namespace SharedData @@ -22,13 +23,22 @@ namespace SharedData /** @brief Trusted for files, the backend does not stat them. */ std::uint64_t sizeBytes{0}; bool isDirectory{false}; + /** + * @brief A directory that is created without its content, as an entry of the files' bulk instead of a bulk + * of its own that scans it. + */ + bool withoutContent{false}; /** * @brief The user agreed to replace this existing destination. Without it, or the request's allowOverwrite, * a destination that appeared after the existence check is not replaced. */ bool mayOverwrite{false}; }; - BOOST_DESCRIBE_STRUCT(BulkUploadEntry, (), (source, destination, sizeBytes, isDirectory, mayOverwrite)) + BOOST_DESCRIBE_STRUCT( + BulkUploadEntry, + (), + (source, destination, sizeBytes, isDirectory, withoutContent, mayOverwrite) + ) /** * @brief One remote item of a bulk download. A directory is scanned and downloaded with its content. @@ -40,6 +50,8 @@ namespace SharedData /** @brief Trusted for files, the backend does not stat them. */ std::uint64_t sizeBytes{0}; bool isDirectory{false}; + /** @brief See @ref BulkUploadEntry::withoutContent. */ + bool withoutContent{false}; /** @brief Modification time of the remote file, restored on the local copy so the next sync sees no change. */ std::uint64_t mtime{0}; std::uint32_t mtimeNsec{0}; @@ -49,7 +61,7 @@ namespace SharedData BOOST_DESCRIBE_STRUCT( BulkDownloadEntry, (), - (source, destination, sizeBytes, isDirectory, mtime, mtimeNsec, mayOverwrite) + (source, destination, sizeBytes, isDirectory, withoutContent, mtime, mtimeNsec, mayOverwrite) ) /** @@ -62,8 +74,14 @@ namespace SharedData bool allowOverwrite{false}; bool insertRefresh{false}; OperationMode mode{OperationMode::Queued}; + /** + * @brief The directories the files' bulk works below, the labels of its card. Unset, the closest directories + * holding every source and every destination. + */ + std::optional localRoot{}; + std::optional remoteRoot{}; }; - BOOST_DESCRIBE_STRUCT(BulkUploadRequest, (), (entries, allowOverwrite, insertRefresh, mode)) + BOOST_DESCRIBE_STRUCT(BulkUploadRequest, (), (entries, allowOverwrite, insertRefresh, mode, localRoot, remoteRoot)) /** * @brief Download analogue of @ref BulkUploadRequest. @@ -74,6 +92,17 @@ namespace SharedData bool allowOverwrite{false}; bool insertRefresh{false}; OperationMode mode{OperationMode::Queued}; + /** + * @brief The directories the files' bulk works below, the labels of its card. No link may lie between the + * local one and a destination. Unset, the closest directories holding every source and every + * destination. + */ + std::optional localRoot{}; + std::optional remoteRoot{}; }; - BOOST_DESCRIBE_STRUCT(BulkDownloadRequest, (), (entries, allowOverwrite, insertRefresh, mode)) + BOOST_DESCRIBE_STRUCT( + BulkDownloadRequest, + (), + (entries, allowOverwrite, insertRefresh, mode, localRoot, remoteRoot) + ) } diff --git a/shared_data/include/shared_data/sync/sync_execution.hpp b/shared_data/include/shared_data/sync/sync_execution.hpp index fe000bf2..c71b3303 100644 --- a/shared_data/include/shared_data/sync/sync_execution.hpp +++ b/shared_data/include/shared_data/sync/sync_execution.hpp @@ -54,8 +54,6 @@ namespace SharedData::Sync std::vector remoteDeleteSources{}; std::vector localDeletes{}; std::vector localDeleteSources{}; - /** @brief Directories without content, created right away. */ - std::vector createdDirectories{}; /** @brief Rows that were refused or failed right away, nothing was queued for them. */ std::vector failed{}; }; @@ -70,7 +68,6 @@ namespace SharedData::Sync remoteDeleteSources, localDeletes, localDeleteSources, - createdDirectories, failed) ) From d61bd3a2a674ee10876c7740dc03f7eca202cab3 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:31:05 +0200 Subject: [PATCH 044/131] Sync keeps a baseline per pair of a saved session Sync keeps a baseline per pair of a saved session in the state directory, one file per pair named by a hash of its key (session, local root, remote root): queueing a plan records every entry found equal on both sides, and each operation of the plan records the entries it finished once it completes, a transfer both sides of its source and a delete by removing the record, so blocked, deselected, failed and cancelled entries keep what they had; every compare reads it, the legacy last sync time only counts for pairs without one and the frontend still writes it for older builds; completions of bulk transfers name their failed entries by index too --- .../backend/sftp/bulk_transfer_operation.hpp | 8 + .../include/backend/sftp/operation_queue.hpp | 31 ++- .../backend/sync/sync_baseline_store.hpp | 67 +++++ backend/include/backend/sync/sync_session.hpp | 3 +- backend/source/backend/session.cpp | 6 +- .../source/backend/sftp/operation_queue.cpp | 155 +++++++++--- .../backend/sync/sync_baseline_store.cpp | 228 ++++++++++++++++++ frontend/source/frontend/session.cpp | 51 +--- .../include/persistence/state/state.hpp | 5 - .../file_operations/operation_completed.hpp | 8 +- 10 files changed, 455 insertions(+), 107 deletions(-) create mode 100644 backend/include/backend/sync/sync_baseline_store.hpp create mode 100644 backend/source/backend/sync/sync_baseline_store.cpp diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index c39b1a63..d3c49ea4 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -141,6 +141,14 @@ class BulkTransferOperation : public Operation */ std::vector> getFailed() const; + /** + * @brief The entries that were skipped by their index, and why. + */ + std::vector const& indexedFailures() const noexcept + { + return indexedFailures_; + } + std::size_t inFlightCount() const noexcept { return slots_.size(); diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index d87c32a3..db9897f7 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -244,9 +244,10 @@ class OperationQueue ); /** - * @brief Queues a resolved sync plan under the operation ids of @p request: files of each direction as one - * bulk transfer, remote and local deletes as one operation each. Directories without content are created - * right away. Every path comes from @p plan, which resolved them below the sync roots. + * @brief Queues a resolved sync plan under the operation ids of @p request: files and directories without + * content of each direction as one bulk transfer below the sync roots, remote deletes as one operation + * and local deletes as one operation of @p localDeleteQueue, held while this queue is paused. Every path + * comes from @p plan, which resolved them below the sync roots. */ SharedData::Sync::SyncExecutionResult executeSyncPlan( SecureShell::SftpSession& sftp, @@ -288,8 +289,10 @@ class OperationQueue SharedData::OperationMode mode = SharedData::OperationMode::Queued ); - /** @brief Queues a remote scan and a local scan as priority operations for sync comparison. - * When each scan completes the results are sent to the frontend via onSyncScanResult. + /** + * @brief Opens a @ref SyncSession and kicks off both scans as priority operations. The scans build + * ScanNode trees directly and feed them into the session on completion. + * Emits @c onSyncScanPhaseDone(sessionId, isLocal) for each side. * * @param sftp SFTP session to use for the remote scan. * @param remoteScanId Pre-assigned operation ID for the remote scan. @@ -297,11 +300,6 @@ class OperationQueue * @param remotePath Remote directory root to scan. * @param localPath Local directory root to scan. */ - /** - * @brief Opens a @ref SyncSession and kicks off both scans. The scans build - * ScanNode trees directly and feed them into the session on completion. - * Emits @c onSyncScanPhaseDone(sessionId, isLocal) for each side. - */ void addSyncScanOperation( SecureShell::SftpSession& sftp, Ids::SyncSessionId syncSessionId, @@ -386,6 +384,19 @@ class OperationQueue */ void deliverSyncScan(Ids::OperationId const& id, Operation& operation); + /** + * @brief The callback of the scan of one side of a sync: hands the tree to @p session through @p setTree on the + * session's strand and tells the frontend the side is done, or cancels the other scan @p siblingScanId and + * reports the failure. + */ + auto makeSyncScanCallback( + Ids::SyncSessionId syncSessionId, + std::weak_ptr session, + Ids::OperationId siblingScanId, + bool isLocal, + void (SyncSession::*setTree)(SharedData::Sync::ScanNode) + ) -> std::function)>; + private: Persistence::SftpOptions sftpOpts_{}; Ids::SessionId sessionId_{}; diff --git a/backend/include/backend/sync/sync_baseline_store.hpp b/backend/include/backend/sync/sync_baseline_store.hpp new file mode 100644 index 00000000..e7a847b5 --- /dev/null +++ b/backend/include/backend/sync/sync_baseline_store.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/** + * @brief Keeps the @ref SharedData::Sync::SyncBaseline of every sync pair, one JSON file per pair in a directory of + * its own, named by a hash of the pair's session and one of the pair key. The file also holds the key, so a + * hash collision reads as no baseline instead of another pair's records. + * + * Every access goes to the file under one mutex shared by all stores, so sync sessions and finished operations of + * the same pair can update it from any thread without losing each other's changes. + */ +class SyncBaselineStore +{ + public: + explicit SyncBaselineStore(std::filesystem::path directory); + + /** + * @brief The store below the state directory of the program. + */ + static std::filesystem::path defaultDirectory(); + + /** + * @brief Names a pair: the saved session, the local root and the remote root. + */ + static std::string + pairKey(std::string_view sessionName, Paths::LocalPath const& localRoot, Paths::RemotePath const& remoteRoot); + + /** + * @brief The file of @p pairKey below @p directory. + */ + static std::filesystem::path fileOf(std::filesystem::path const& directory, std::string_view pairKey); + + /** + * @brief Removes the baselines of every pair whose session is not one of @p sessionNames, like a deleted or + * renamed session. + * @return How many were removed. + */ + std::size_t pruneAllBut(std::vector const& sessionNames) const; + + /** + * @brief The baseline of @p pairKey, nothing when the pair was never synchronized. A file that cannot be read + * counts as a pair with history but no records, which protects every delete. + */ + std::optional load(std::string const& pairKey) const; + + /** + * @brief Reads the baseline of @p pairKey (empty when there is none), lets @p change modify it and writes it + * back, which also creates the file of a pair synchronized for the first time. + * @return false when it could not be written. + */ + bool update(std::string const& pairKey, std::function const& change) const; + + private: + std::optional loadLocked(std::string const& pairKey) const; + + std::filesystem::path directory_; +}; diff --git a/backend/include/backend/sync/sync_session.hpp b/backend/include/backend/sync/sync_session.hpp index 8f4a4f51..f93b039c 100644 --- a/backend/include/backend/sync/sync_session.hpp +++ b/backend/include/backend/sync/sync_session.hpp @@ -184,7 +184,8 @@ class SyncSession : public std::enable_shared_from_this }; /** - * @brief @ref buildEnqueuePlanInStrand with every entry resolved below the session roots. + * @brief The plan of a sparse selection (an entry in @p selectedRelKeys selects it and everything below it, see + * @ref SharedData::Sync::DiffTreeStore::buildPlan) with every entry resolved below the session roots. * Precondition: running on @ref strand(). */ ResolvedPlan resolvePlanInStrand( diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index a25532c9..fb1fd5ad 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -1477,7 +1477,11 @@ void Session::registerRpcSftpOpenSyncSession() localPath = *std::move(localRoot), respectIgnoreFiles, recursive, - ignoreHidden](RpcHelper::RpcOnce&& reply, auto&& channel) + ignoreHidden]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel + ) { auto self = weak.lock(); if (!self) diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index f015bf28..4688910a 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -87,44 +87,11 @@ namespace }, [reason, operationId, error](LocalScanOperation const& op) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .localPath = unlessEmpty(op.localPath()), - .error = error, - }; - }, - [reason, operationId, error](BulkDownloadOperation const& op) - { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - .failedEntries = op.getFailed(), - }; + completed.remotePath = unlessEmpty(scan.remotePath()); }, [reason, operationId, error](UploadOperation const& op) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .localPath = unlessEmpty(op.localPath()), - .remotePath = unlessEmpty(op.remotePath()), - .error = error, - }; - }, - [reason, operationId, error](BulkUploadOperation const& op) - { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - .failedEntries = op.getFailed(), - }; + completed.localPath = unlessEmpty(scan.localPath()); }, [reason, operationId, error](DeleteOperation const& op) { @@ -317,6 +284,7 @@ void OperationQueue::cancelAll() self->operations_.clear(); self->priorityOperations_.clear(); + self->syncRecordings_.clear(); auto waiting = std::exchange(self->syncScanCallbacks_, {}); for (auto& [id, callback] : waiting) callback(std::unexpected(std::string{"The scan was cancelled."})); @@ -456,6 +424,7 @@ void OperationQueue::completeOperation(SharedData::OperationCompleted&& operatio // a backup for reconnect adoption. self->bulkResumeStash_.erase(operationCompleted.operationId); + self->recordSyncCompletion(operationCompleted); if (self->completionObserver_) self->completionObserver_(operationCompleted); RpcHelper::callRemoteSafely(*self->hub_, self->rpcName("onOperationCompleted"), operationCompleted); @@ -1598,12 +1567,26 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( result.failed.push_back(std::move(entry.relKey)); }; - for (auto& [entry, paths] : plan.entries) + auto const& pairKey = session.baselineKey(); + auto const& baselines = session.baselines(); + const auto makeRecording = [&]() { + return SyncRecording{.pairKey = pairKey.value_or(std::string{}), .store = baselines}; + }; + auto uploadRecording = makeRecording(); + auto downloadRecording = makeRecording(); + auto remoteDeleteRecording = makeRecording(); + auto localDeleteRecording = makeRecording(); + + for (auto& planned : plan.entries) + { + auto& entry = planned.entry; + auto& paths = planned.paths; switch (entry.action) { case Action::Upload: { + uploadRecording.entries.emplace_back(entry.relKey, std::move(planned.record)); result.uploadSources.push_back(paths.local.toUtf8()); uploads.entries.push_back({ .source = std::move(paths.local), @@ -1619,6 +1602,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( } case Action::Download: { + downloadRecording.entries.emplace_back(entry.relKey, std::move(planned.record)); result.downloadSources.push_back(paths.remote.value()); downloads.entries.push_back({ .source = std::move(paths.remote), @@ -1644,8 +1628,14 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( refuse(entry, refusedRemoteDirectories, "remotely", allowed.error().toString()); break; } + remoteDeleteRecording.entries.emplace_back(entry.relKey, std::nullopt); + remoteDeleteRecording.sources.push_back(paths.remote.value()); result.remoteDeleteSources.push_back(paths.remote.value()); - remoteDeletes.entries.push_back({.target = std::move(paths.remote), .isDirectory = entry.isDirectory}); + remoteDeletes.entries.push_back({ + .target = std::move(paths.remote), + .isDirectory = entry.isDirectory, + .compared = std::move(planned.compared), + }); result.remoteDeletes.push_back(std::move(entry.relKey)); break; } @@ -1661,6 +1651,8 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( refuse(entry, refusedLocalDirectories, "locally", allowed.error()); break; } + localDeleteRecording.entries.emplace_back(entry.relKey, std::nullopt); + localDeleteRecording.sources.push_back(paths.local.toUtf8()); result.localDeleteSources.push_back(paths.local.toUtf8()); localDeletes.push_back(std::move(paths.local)); result.localDeletes.push_back(std::move(entry.relKey)); @@ -1715,6 +1707,27 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( result.localDeleteSources.clear(); } + if (pairKey && baselines) + { + baselines->update( + *pairKey, + [&plan](SharedData::Sync::SyncBaseline& baseline) + { + for (auto& [relKey, record] : plan.agreement.records) + baseline.records.insert_or_assign(relKey, std::move(record)); + } + ); + const auto keep = [this](Ids::OperationId const& operationId, bool queued, SyncRecording recording) + { + if (queued && !recording.entries.empty()) + syncRecordings_.insert_or_assign(operationId.value(), std::move(recording)); + }; + keep(request.uploadId, !result.uploads.empty(), std::move(uploadRecording)); + keep(request.downloadId, !result.downloads.empty(), std::move(downloadRecording)); + keep(request.remoteDeleteId, !result.remoteDeletes.empty(), std::move(remoteDeleteRecording)); + keep(request.localDeleteId, !result.localDeletes.empty(), std::move(localDeleteRecording)); + } + Log::info( "Sync: queued {} uploads, {} downloads, {} remote and {} local deletes, {} failed", result.uploads.size(), @@ -1731,7 +1744,8 @@ std::expected OperationQueue::addLocalDeleteOperation( std::vector targets, bool recursive, bool insertRefresh, - SharedData::OperationMode mode + SharedData::OperationMode mode, + std::vector comparedStates ) { // Assumed in strand @@ -2094,16 +2108,22 @@ void OperationQueue::addSyncScanOperation( Paths::LocalPath const& localPath, bool respectIgnoreFiles, bool recursive, - bool ignoreHidden + bool ignoreHidden, + std::optional const& sessionName ) { // Create (or replace) the SyncSession. A fresh id from the frontend is the // norm; re-opening over an existing id is handled as "discard previous state". + const bool keepsBaseline = sessionName && !sessionName->empty() && syncBaselines_; auto session = std::make_shared( SyncSession::Options{ .sessionId = syncSessionId, .localRoot = localPath, .remoteRoot = remotePath, + .baselineKey = keepsBaseline + ? std::optional{SyncBaselineStore::pairKey(*sessionName, localPath, remotePath)} + : std::nullopt, + .baselines = keepsBaseline ? syncBaselines_ : nullptr, }, executor_ ); @@ -2226,6 +2246,63 @@ void OperationQueue::addSyncScanOperation( ); } +void OperationQueue::setSyncBaselineStore(std::shared_ptr store) +{ + syncBaselines_ = std::move(store); +} + +void OperationQueue::recordSyncCompletion(OperationCompleted const& completed) +{ + auto found = syncRecordings_.extract(completed.operationId.value()); + if (found.empty() || completed.reason != SharedData::OperationCompletionReason::Completed) + return; + auto& recording = found.mapped(); + + std::vector failed(recording.entries.size(), false); + for (const auto index : completed.failedEntryIndices) + { + if (index < failed.size()) + failed[index] = true; + } + if (!recording.sources.empty() && !completed.failedEntries.empty()) + { + std::unordered_set failedSources; + for (auto const& [source, error] : completed.failedEntries) + failedSources.insert(source); + for (std::size_t index = 0; index < recording.sources.size() && index < failed.size(); ++index) + failed[index] = failed[index] || failedSources.contains(recording.sources[index]); + } + + recording.store->update( + recording.pairKey, + [&recording, &failed](SharedData::Sync::SyncBaseline& baseline) + { + const SharedData::Sync::SyncRecord directory{ + .local = {.type = SharedData::FileType::Directory}, + .remote = {.type = SharedData::FileType::Directory}, + }; + for (std::size_t index = 0; index < recording.entries.size(); ++index) + { + if (failed[index]) + continue; + auto const& [relKey, record] = recording.entries[index]; + if (!record) + { + baseline.records.erase(relKey); + continue; + } + baseline.records.insert_or_assign(relKey, *record); + // The transfer created or found every directory above it on both sides. + for ( + auto slash = relKey.rfind('/'); slash != std::string::npos && slash > 0; + slash = relKey.rfind('/', slash - 1) + ) + baseline.records.insert_or_assign(relKey.substr(0, slash), directory); + } + } + ); +} + std::shared_ptr OperationQueue::syncSession(Ids::SyncSessionId const& sessionId) const { const auto iter = syncSessions_.find(sessionId.value()); diff --git a/backend/source/backend/sync/sync_baseline_store.cpp b/backend/source/backend/sync/sync_baseline_store.cpp new file mode 100644 index 00000000..9ec730aa --- /dev/null +++ b/backend/source/backend/sync/sync_baseline_store.cpp @@ -0,0 +1,228 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + constexpr int formatVersion = 1; + + std::mutex& storeMutex() + { + static std::mutex mutex; + return mutex; + } + + /** + * @brief The session part of a pair key, see SyncBaselineStore::pairKey. + */ + std::string_view sessionOf(std::string_view pairKey) + { + return pairKey.substr(0, pairKey.find('\n')); + } + + std::uint64_t fnv1a(std::string_view text) + { + std::uint64_t hash = 14695981039346656037ull; + for (const auto character : text) + { + hash ^= static_cast(character); + hash *= 1099511628211ull; + } + return hash; + } + + /** + * @brief The file's JSON, nothing when it does not exist, an error text when it cannot be read. + */ + std::expected, std::string> readJson(std::filesystem::path const& file) + { + std::error_code error{}; + if (!std::filesystem::exists(file, error)) + { + if (error) + return std::unexpected(Utility::errorMessage(error)); + return std::nullopt; + } + std::ifstream stream{file, std::ios::binary}; + if (!stream) + return std::unexpected(std::string{"cannot open"}); + auto json = nlohmann::json::parse(stream, nullptr, false); + if (json.is_discarded()) + return std::unexpected(std::string{"not JSON"}); + return json; + } +} + +SyncBaselineStore::SyncBaselineStore(std::filesystem::path directory) + : directory_{std::move(directory)} +{} + +std::filesystem::path SyncBaselineStore::defaultDirectory() +{ + return Nui::resolvePath(std::filesystem::path{std::string{Constants::syncBaselinePath}}); +} + +std::string SyncBaselineStore::pairKey( + std::string_view sessionName, + Paths::LocalPath const& localRoot, + Paths::RemotePath const& remoteRoot +) +{ + return fmt::format("{}\n{}\n{}", sessionName, localRoot.toUtf8(), remoteRoot.value()); +} + +std::filesystem::path SyncBaselineStore::fileOf(std::filesystem::path const& directory, std::string_view pairKey) +{ + return directory / fmt::format("{:016x}-{:016x}.json", fnv1a(sessionOf(pairKey)), fnv1a(pairKey)); +} + +std::size_t SyncBaselineStore::pruneAllBut(std::vector const& sessionNames) const +{ + std::scoped_lock lock{storeMutex()}; + std::unordered_set kept; + for (auto const& name : sessionNames) + kept.insert(fmt::format("{:016x}-", fnv1a(name))); + + std::vector orphans; + std::error_code error{}; + for ( + auto iter = std::filesystem::directory_iterator{directory_, error}; + !error && iter != std::filesystem::directory_iterator{}; + iter.increment(error) + ) + { + const auto name = Paths::nativeToUtf8(iter->path().filename()); + // Session hash, a dash, pair hash and ".json". + if (name.size() == 16 + 1 + 16 + 5 && name.ends_with(".json") && !kept.contains(name.substr(0, 17))) + orphans.push_back(iter->path()); + } + + std::size_t removed = 0; + for (auto const& orphan : orphans) + { + std::error_code removeError{}; + if (std::filesystem::remove(orphan, removeError)) + ++removed; + else if (removeError) + Log::warn( + "Cannot remove the sync baseline '{}': {}", + Paths::nativeToUtf8(orphan), + Utility::errorMessage(removeError) + ); + } + if (removed > 0) + Log::info("Removed {} sync baselines of sessions that no longer exist.", removed); + return removed; +} + +std::optional SyncBaselineStore::loadLocked(std::string const& pairKey) const +{ + const auto file = fileOf(directory_, pairKey); + const auto json = readJson(file); + if (!json) + { + Log::error( + "Sync baseline '{}' cannot be read ({}), every delete of its pair is refused.", + Paths::nativeToUtf8(file), + json.error() + ); + return SharedData::Sync::SyncBaseline{}; + } + if (!*json) + return std::nullopt; + auto const& content = **json; + if (!content.is_object() || content.value("pair", std::string{}) != pairKey) + { + Log::warn("Sync baseline '{}' belongs to another pair, ignored.", Paths::nativeToUtf8(file)); + return std::nullopt; + } + try + { + return content.get(); + } + catch (std::exception const& exception) + { + Log::error( + "Sync baseline '{}' is damaged ({}), every delete of its pair is refused.", + Paths::nativeToUtf8(file), + Utility::errorMessage(exception) + ); + return SharedData::Sync::SyncBaseline{}; + } +} + +std::optional SyncBaselineStore::load(std::string const& pairKey) const +{ + std::scoped_lock lock{storeMutex()}; + return loadLocked(pairKey); +} + +bool SyncBaselineStore::update( + std::string const& pairKey, + std::function const& change +) const +{ + std::scoped_lock lock{storeMutex()}; + const auto file = fileOf(directory_, pairKey); + if (const auto existing = readJson(file); + existing && *existing && (*existing)->value("pair", std::string{}) != pairKey) + { + Log::error("Sync baseline '{}' belongs to another pair, not replaced.", Paths::nativeToUtf8(file)); + return false; + } + + auto baseline = loadLocked(pairKey).value_or(SharedData::Sync::SyncBaseline{}); + change(baseline); + + nlohmann::json json = baseline; + json["pair"] = pairKey; + json["version"] = formatVersion; + + std::error_code error{}; + std::filesystem::create_directories(directory_, error); + if (error) + { + Log::error( + "Cannot create the sync baseline directory '{}': {}", + Paths::nativeToUtf8(directory_), + Utility::errorMessage(error) + ); + return false; + } + auto temporary = file; + temporary += ".part"; + { + std::ofstream stream{temporary, std::ios::binary | std::ios::trunc}; + stream << json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); + if (!stream.flush()) + { + Log::error("Cannot write the sync baseline '{}'.", Paths::nativeToUtf8(temporary)); + return false; + } + } + if (auto moved = Utility::moveIntoPlace(temporary, file, /*replace=*/true); !moved) + { + Log::error( + "Cannot replace the sync baseline '{}': {}", Paths::nativeToUtf8(file), Utility::errorMessage(moved.error()) + ); + std::filesystem::remove(temporary, error); + return false; + } + return true; +} diff --git a/frontend/source/frontend/session.cpp b/frontend/source/frontend/session.cpp index 4a3ea8fc..668ce104 100644 --- a/frontend/source/frontend/session.cpp +++ b/frontend/source/frontend/session.cpp @@ -83,17 +83,9 @@ struct Session::Implementation std::unique_ptr layoutInitializer; SyncDialog syncDialog; SyncProgressDialog syncProgressDialog; - // Holds the active BackendSyncProvider between progress-dialog open and - // sync-dialog close. Destroyed (→ provider.close() → backend session - // released) when replaced or explicitly reset. - std::shared_ptr syncProvider_; // Last local+remote paths given to startSyncFlow; used by the dialog's // Recompare path to re-issue the flow without asking the user again. std::filesystem::path lastSyncLocalPath_{}; - /** @brief Names the sync pair of the running flow in State::syncBaselines, unset without a saved session. */ - std::optional syncBaselineKey_{}; - /** @brief When the running flow started its compare, recorded as the baseline once it is synchronized. */ - std::uint64_t syncFlowStart_{0}; std::filesystem::path lastSyncRemotePath_{}; std::unique_ptr connectionLossOverlay; std::unique_ptr snapshotManager; @@ -190,25 +182,7 @@ struct Session::Implementation { // Recompare: rerun the whole scan-and-diff flow with the dialog's current // settings, reusing the cached local+remote roots. The old provider is - // replaced by a fresh one; the backend session it held is closed on - // destruction. - // A synchronized selection makes the time its compare started the baseline of the next sync. - syncDialog.setOnSynchronized( - [this]() - { - if (!syncBaselineKey_ || !this->stateHolder) - return; - this->stateHolder->loadModifySave( - [key = *syncBaselineKey_, time = syncFlowStart_](Persistence::State& state) - { - if (!state.syncBaselines) - state.syncBaselines.emplace(); - (*state.syncBaselines)[key] = time; - } - ); - } - ); - + // replaced by a fresh one. syncDialog.setOnRecompareRequested( [this](SyncDialog::RecompareRequest req) { @@ -240,28 +214,7 @@ struct Session::Implementation lastSyncLocalPath_ = localPath; lastSyncRemotePath_ = remotePath; - // Only a saved session has a stable identity to remember the last sync of. - syncBaselineKey_.reset(); - std::optional lastSyncTime{}; - if (persistenceName && !persistenceName->empty()) - { - syncBaselineKey_ = - fmt::format("{}\n{}\n{}", *persistenceName, localPath.generic_string(), remotePath.generic_string()); - if (auto const& baselines = stateHolder->stateCache().syncBaselines; baselines) - { - if (const auto found = baselines->find(*syncBaselineKey_); found != baselines->end()) - lastSyncTime = found->second; - } - } - syncFlowStart_ = static_cast( - std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() - ); - initialDiffOptions.lastSyncTime = lastSyncTime; - - // Replace the old provider before opening the progress dialog so its - // backend session (if any) is closed before the new one opens. - syncProvider_ = std::make_shared(&operationQueue); - + // The progress dialog holds the provider while it scans and compares, then hands it to the sync dialog. syncProgressDialog.open( syncProvider_, localPath, diff --git a/persistence/include/persistence/state/state.hpp b/persistence/include/persistence/state/state.hpp index 98db154e..098b2c91 100644 --- a/persistence/include/persistence/state/state.hpp +++ b/persistence/include/persistence/state/state.hpp @@ -59,11 +59,6 @@ namespace Persistence LocalizationOptions localizationOptions{}; std::optional lastDirectConnect{}; std::optional lastDirectConnectInheritables{}; - /** - * @brief When each sync pair (session, local root and remote root) was last synchronized, in seconds since - * the epoch. Sync compares modification times against it to see which side changed since. - */ - std::optional> syncBaselines{}; State fullyResolve() const; diff --git a/shared_data/include/shared_data/file_operations/operation_completed.hpp b/shared_data/include/shared_data/file_operations/operation_completed.hpp index 5b27fdc3..3ad6af00 100644 --- a/shared_data/include/shared_data/file_operations/operation_completed.hpp +++ b/shared_data/include/shared_data/file_operations/operation_completed.hpp @@ -10,8 +10,10 @@ #include -#include #include +#include +#include +#include namespace SharedData { @@ -27,10 +29,12 @@ namespace SharedData std::optional error{std::nullopt}; /** @brief The skipped entries of a bulk by their source path, a remote path for downloads. */ std::vector> failedEntries{}; + /** @brief The skipped entries of a prescanned bulk up- or download by their index. */ + std::vector indexedFailures{}; }; BOOST_DESCRIBE_STRUCT( OperationCompleted, (), - (reason, operationId, completionTime, localPath, remotePath, error, failedEntries) + (reason, operationId, completionTime, localPath, remotePath, error, failedEntries, indexedFailures) ) } \ No newline at end of file From 65801f9fc660e694a502c48a82f14ffbe2788bc0 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:37:24 +0200 Subject: [PATCH 045/131] Synchronize is off while a compare runs; refusals show their reason Synchronize is off while a compare runs or after one was cancelled, and a refused selection shows its reason, marks its rows failed and records nothing; the legacy last sync time is written once the backend accepted a synchronize; the progress dialog holds the provider until the first diff, a failure or a cancel and then hands it to the sync dialog, which lets go of it on close, and the dialog removes the callbacks of its executions when it closes or opens another provider, ignores replies once it is gone, and keeps progress observers for loaded rows only; transfer failures map to rows by entry index, so a sync result lists paths for deletes only, and a closed provider rejects instead of never answering --- backend/source/backend/session.cpp | 16 +- .../source/backend/sftp/operation_queue.cpp | 2 - .../session_components/operation_queue.hpp | 11 +- .../sync_dialog/backend_sync_provider.hpp | 7 +- .../frontend/sync_dialog/sync_dialog.hpp | 11 +- .../sync_dialog/sync_progress_dialog.hpp | 9 +- frontend/source/frontend/session.cpp | 8 +- .../session_components/operation_queue.cpp | 18 +- .../sync_dialog/backend_sync_provider.cpp | 21 +- .../frontend/sync_dialog/sync_dialog.cpp | 462 ++++++++++++------ .../sync_dialog/sync_progress_dialog.cpp | 16 +- .../shared_data/sync/sync_execution.hpp | 12 +- static/assets/languages/de_DE.yaml | 4 + static/assets/languages/en_US.yaml | 4 + 14 files changed, 383 insertions(+), 218 deletions(-) diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index fb1fd5ad..861d2446 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -1642,20 +1642,28 @@ void Session::registerRpcSftpExecuteSyncSelection() auto replyShared = std::make_shared(std::move(reply)); boost::asio::post( session->strand(), - [self, session, request = std::move(request), channelIdString, replyShared]() mutable + [weak, session, request = std::move(request), channelIdString, replyShared]() mutable { if (session->generation() != request.generation) - return replyShared->error("stale generation"); + return replyShared->error(Utility::localizedMessage("backend.sync.staleComparison")); + auto self = weak.lock(); + if (!self) + return replyShared->error("Session no longer exists"); auto plan = session->resolvePlanInStrand( request.section, std::unordered_set{request.relKeys.begin(), request.relKeys.end()} ); self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [self, session, plan = std::move(plan), request = std::move(request)]( - RpcHelper::RpcOnce&& reply, auto&& channel + [session, plan = std::move(plan), request = std::move(request)]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel ) mutable { + // A stopped session has no queue any more. + if (!self.operationQueue_) + return reply.error("Session no longer exists"); auto result = self->operationQueue_->executeSyncPlan( *channel, *session, std::move(plan), request ); diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 4688910a..039a397d 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1587,7 +1587,6 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( case Action::Upload: { uploadRecording.entries.emplace_back(entry.relKey, std::move(planned.record)); - result.uploadSources.push_back(paths.local.toUtf8()); uploads.entries.push_back({ .source = std::move(paths.local), .destination = std::move(paths.remote), @@ -1603,7 +1602,6 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( case Action::Download: { downloadRecording.entries.emplace_back(entry.relKey, std::move(planned.record)); - result.downloadSources.push_back(paths.remote.value()); downloads.entries.push_back({ .source = std::move(paths.remote), .destination = std::move(paths.local), diff --git a/frontend/include/frontend/session_components/operation_queue.hpp b/frontend/include/frontend/session_components/operation_queue.hpp index c5700920..40671e19 100644 --- a/frontend/include/frontend/session_components/operation_queue.hpp +++ b/frontend/include/frontend/session_components/operation_queue.hpp @@ -204,14 +204,11 @@ class OperationQueue bool isActive() const; void addCompletionCallback(Ids::OperationId const& opId, std::function callback); - /** @brief Register a callback that receives the source paths of the entries a bulk operation - * could not transfer. Runs on completion, before the completion callback, because the - * final BulkProgress can arrive after the completion and is dropped then. + /** @brief Register a callback that receives the entries a bulk operation could not transfer or + * delete. Runs on completion, before the completion callback, because the final BulkProgress + * can arrive after the completion and is dropped then. */ - void addFailedEntriesCallback( - Ids::OperationId const& opId, - std::function const& failedPaths)> callback - ); + void addFailedEntriesCallback(Ids::OperationId const& opId, FailedEntriesCallback callback); /** @brief Register a callback that receives progress as a 0.0–1.0 fraction for an upload/download. * Automatically removed when the operation completes. */ diff --git a/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp b/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp index fdc62019..2b56b097 100644 --- a/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp +++ b/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp @@ -30,9 +30,10 @@ class OperationQueue; * - @ref cancelDiff flips the backend cancel flag. * - @ref close releases backend memory. Idempotent. * - * The provider owns the lifecycle — callers should construct it once per sync flow - * and destroy it when the dialog closes. The destructor calls @ref close, replies that - * arrive afterwards are dropped. + * The provider owns the lifecycle: one per sync flow, held by the progress dialog until the + * first diff landed and then by the sync dialog until it closes or opens with another one. + * The destructor calls @ref close, replies that arrive afterwards are dropped. A call on a + * closed provider rejects (or reports a cancelled diff) right away. */ class BackendSyncProvider { diff --git a/frontend/include/frontend/sync_dialog/sync_dialog.hpp b/frontend/include/frontend/sync_dialog/sync_dialog.hpp index 61109863..e9a2570a 100644 --- a/frontend/include/frontend/sync_dialog/sync_dialog.hpp +++ b/frontend/include/frontend/sync_dialog/sync_dialog.hpp @@ -41,13 +41,13 @@ class SyncDialog * @brief Opens the dialog against a backend-resident sync session whose first * diff has already completed. * - * @param provider Non-owning handle to the session; must outlive the dialog. + * @param provider The sync session, kept until the dialog closes or opens with another one. Callbacks of + * executions under a previous provider are dropped then. * @param snapshot The result of the initial recompute: summary and section roots. - * @param localPath Local scan root (for path column formatting only — data is + * @param localPath Local scan root (for path column formatting only, data is * served lazily from @p provider). * @param remotePath Remote scan root. * @param scanned Settings the scans of @p provider ran with. - * @param lastSyncTime When this pair was last synchronized, every recompute compares against it. */ void open( std::shared_ptr provider, @@ -58,11 +58,6 @@ class SyncDialog std::optional lastSyncTime ); - /** - * @brief Called when the user synchronized the whole selection, so the session can record the sync. - */ - void setOnSynchronized(std::function callback); - /** * @brief Scan + diff settings snapshot handed to the Recompare callback. */ diff --git a/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp b/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp index 7e9aeebb..8ec3cdf7 100644 --- a/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp +++ b/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp @@ -32,10 +32,11 @@ class SyncProgressDialog * The dialog keeps showing the Comparing spinner during the diff walk; * light diffs finish within a frame so the phase effectively flashes. * - * @param provider Non-owning handle to the sync session. Lifetime must - * outlive this call (typically owned by @c Session). + * @param provider The sync session, held until the first diff landed, a scan failed or the + * dialog was cancelled. * @param initialOptions The diff options to apply for the first recompute. - * @param onDone Called exactly once on success. Not invoked on cancel. + * @param onDone Called exactly once on success with the provider, which this dialog lets go + * of then. Not invoked on cancel or failure. */ void open( std::shared_ptr provider, @@ -45,7 +46,7 @@ class SyncProgressDialog bool recursive, bool ignoreHidden, SharedData::Sync::DiffOptions initialOptions, - std::function onDone + std::function, SharedData::Sync::DiffSnapshot)> onDone ); /** diff --git a/frontend/source/frontend/session.cpp b/frontend/source/frontend/session.cpp index 668ce104..79c6283d 100644 --- a/frontend/source/frontend/session.cpp +++ b/frontend/source/frontend/session.cpp @@ -216,18 +216,18 @@ struct Session::Implementation // The progress dialog holds the provider while it scans and compares, then hands it to the sync dialog. syncProgressDialog.open( - syncProvider_, + std::make_shared(&operationQueue), localPath, remotePath, scan.respectIgnoreFiles, scan.recursive, scan.ignoreHidden, initialDiffOptions, - [this, provider = syncProvider_, localPath, remotePath, scan, lastSyncTime]( - SharedData::Sync::DiffSnapshot snapshot + [this, localPath, remotePath, scan]( + std::shared_ptr provider, SharedData::Sync::DiffSnapshot snapshot ) { - syncDialog.open(provider, std::move(snapshot), localPath, remotePath, scan, lastSyncTime); + syncDialog.open(std::move(provider), std::move(snapshot), localPath, remotePath, scan); Nui::globalEventContext.executeActiveEventsImmediately(); } ); diff --git a/frontend/source/frontend/session_components/operation_queue.cpp b/frontend/source/frontend/session_components/operation_queue.cpp index 35fd5477..23b1ca72 100644 --- a/frontend/source/frontend/session_components/operation_queue.cpp +++ b/frontend/source/frontend/session_components/operation_queue.cpp @@ -87,8 +87,7 @@ struct OperationQueue::Implementation Nui::TimerHandle autoCleanTimer; std::unordered_map> completionCallbacks; - std::unordered_map const&)>> - failedEntriesCallbacks; + std::unordered_map failedEntriesCallbacks; std::unordered_map> transferProgressCallbacks; // Keyed by the aggregate bulk operation id (first pre-generated opId for // bulk up/download, or the aggregate id for bulk delete). Auto-erased on @@ -1296,14 +1295,8 @@ void OperationQueue::onOperationCompleted(Nui::val val) { auto callback = std::move(failedIt->second); impl_->failedEntriesCallbacks.erase(failedIt); - if (!completed.failedEntries.empty()) - { - std::vector failedPaths; - failedPaths.reserve(completed.failedEntries.size()); - for (auto const& entry : completed.failedEntries) - failedPaths.push_back(entry.first); - callback(failedPaths); - } + if (!completed.failedEntries.empty() || !completed.indexedFailures.empty()) + callback(completed.failedEntries, completed.indexedFailures); } bool partial = false; @@ -1407,10 +1400,7 @@ void OperationQueue::addCompletionCallback(Ids::OperationId const& opId, std::fu impl_->completionCallbacks[opId.value()] = std::move(callback); } -void OperationQueue::addFailedEntriesCallback( - Ids::OperationId const& opId, - std::function const&)> callback -) +void OperationQueue::addFailedEntriesCallback(Ids::OperationId const& opId, FailedEntriesCallback callback) { impl_->failedEntriesCallbacks[opId.value()] = std::move(callback); } diff --git a/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp index e8048136..7c40cf02 100644 --- a/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp +++ b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp @@ -88,7 +88,12 @@ void BackendSyncProvider::recompute( ) { if (!queue_ || closed_) + { + SharedData::Sync::DiffSnapshot nothing{}; + nothing.summary.cancelled = true; + onSnapshot(std::move(nothing)); return; + } queue_->recomputeSyncDiff( sessionId_, @@ -115,7 +120,7 @@ void BackendSyncProvider::loadChildren( ) { if (!queue_ || closed_) - return; + return onRejected("closed"); const auto expectedGeneration = generation_; queue_->loadSyncDiffChildren( @@ -135,7 +140,11 @@ void BackendSyncProvider::loadChildren( return; onResolved(std::move(nodes)); }, - std::move(onRejected) + [alive = std::weak_ptr{alive_}, onRejected = std::move(onRejected)](std::string const& message) + { + if (alive.lock()) + onRejected(message); + } ); } @@ -146,7 +155,7 @@ void BackendSyncProvider::executeSelection( ) { if (!queue_ || closed_) - return; + return onRejected("closed"); request.syncSessionId = sessionId_; request.generation = generation_; @@ -158,7 +167,11 @@ void BackendSyncProvider::executeSelection( if (alive.lock()) onResolved(std::move(result)); }, - std::move(onRejected) + [alive = std::weak_ptr{alive_}, onRejected = std::move(onRejected)](std::string const& message) + { + if (alive.lock()) + onRejected(message); + } ); } diff --git a/frontend/source/frontend/sync_dialog/sync_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_dialog.cpp index f1951d94..6e5a0e85 100644 --- a/frontend/source/frontend/sync_dialog/sync_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_dialog.cpp @@ -3,8 +3,10 @@ #include #include #include +#include #include +#include #include #include @@ -41,7 +43,6 @@ #include #include #include -#include #include #include @@ -265,6 +266,8 @@ struct SyncDialog::Implementation bool selectionTouched{false}; /// What the tree shows, to skip re-rendering a section a recompute left unchanged. std::vector applied{}; + /** @brief What executions reported per row, see setRowState. */ + std::unordered_map executionStates{}; }; std::array sections_{}; @@ -279,9 +282,14 @@ struct SyncDialog::Implementation ConfirmDialog* confirmDialog_; OperationQueue* operationQueue_; std::function onRecompareRequested_{}; - std::function onSynchronized_{}; - /** @brief When the shown pair was last synchronized, part of every diff. */ - std::optional lastSyncTime_{}; + /** + * @brief False while a recompute runs or after one was cancelled: the backend's diff is not the one the rows + * show, so Synchronize is off. + */ + Nui::Observed diffCurrent_{true}; + /** @brief Operations of executions whose callbacks this dialog registered with the queue. */ + std::vector registeredOperations_{}; + Utility::LifetimeToken lifetime_{}; explicit Implementation(ConfirmDialog* confirmDialog, OperationQueue* operationQueue) : confirmDialog_{confirmDialog} @@ -291,6 +299,15 @@ struct SyncDialog::Implementation initTrees(); } + ~Implementation() + { + dropRegistrations(); + } + Implementation(Implementation const&) = delete; + Implementation& operator=(Implementation const&) = delete; + Implementation(Implementation&&) = delete; + Implementation& operator=(Implementation&&) = delete; + SectionState& stateFor(DiffSection section) { return sections_[static_cast(section)]; @@ -562,7 +579,10 @@ struct SyncDialog::Implementation } } - /** @brief Remembers rows the tree is about to show and gives file rows their progress observer. */ + /** + * @brief Remembers rows the tree is about to show and gives file rows their progress observer, which starts + * with what an execution already reported for the row. + */ void recordRows(DiffSection section, std::vector const& rows) { auto& state = stateFor(section); @@ -570,17 +590,64 @@ struct SyncDialog::Implementation for (auto const& row : rows) { state.rows.insert_or_assign(row.relKey, row); - if (!row.isDirectory) - progress.try_emplace(row.relKey, std::make_shared>(0.0)); + if (row.isDirectory) + continue; + const auto reported = state.executionStates.find(row.relKey); + progress.try_emplace( + row.relKey, + std::make_shared>( + reported == state.executionStates.end() ? 0.0 : reported->second + ) + ); } } - ProgressObserver progressFor(DiffSection section, std::string const& relKey) + /** + * @brief Shows the progress of a row: a fraction while it runs, above 1 when done, below 0 when it failed. Only + * loaded rows have an observer, the others pick the state up when they load. + */ + void setRowState(DiffSection section, std::string const& relKey, double value) + { + auto& state = stateFor(section); + state.executionStates.insert_or_assign(relKey, value); + if (value >= 0.0) + state.failureReasons.erase(relKey); + auto const& progress = progressMapFor(section); + if (const auto observer = progress.find(relKey); observer != progress.end()) + *observer->second = value; + } + + /** + * @brief Shows a row as failed with the reason of @p type, for the failures a row explains: a destination or + * delete target that changed after the compare. A sync entry may only replace what the compare saw, so a + * destination that appeared since fails as existing. + */ + void setRowFailed(DiffSection section, std::string const& relKey, SharedData::OperationErrorType type) + { + using enum SharedData::OperationErrorType; + if (type == ChangedSinceCompare || type == FileExists) + stateFor(section).failureReasons.insert_or_assign( + relKey, std::string{language->get("syncDialog", "changedSinceCompare")} + ); + setRowState(section, relKey, -1.0); + // A row shown as failed already only gets its reason. + auto const& progress = progressMapFor(section); + if (const auto observer = progress.find(relKey); observer != progress.end()) + observer->second->modify(); + } + + std::string failureReasonOf(DiffSection section, std::string const& relKey) const { - auto& entry = progressMapFor(section)[relKey]; - if (!entry) - entry = std::make_shared>(0.0); - return entry; + auto const& reasons = stateFor(section).failureReasons; + const auto found = reasons.find(relKey); + return found == reasons.end() ? std::string{} : found->second; + } + + double rowState(DiffSection section, std::string const& relKey) const + { + auto const& states = stateFor(section).executionStates; + const auto found = states.find(relKey); + return found == states.end() ? 0.0 : found->second; } ScriptNuiComponents::Tree::ChildrenLoader makeChildrenLoader(DiffSection section) @@ -701,6 +768,13 @@ struct SyncDialog::Implementation return !state.rows.contains(entry.first); } ); + std::erase_if( + state.executionStates, + [&state](auto const& entry) + { + return !state.rows.contains(entry.first); + } + ); auto const& selection = selectionFor(section); auto& set = selection->value(); @@ -799,6 +873,7 @@ struct SyncDialog::Implementation return; } recomputeInFlight_ = true; + diffCurrent_ = false; provider_->recompute( buildDiffOptions(), SharedData::Sync::LoadedParents{ @@ -806,19 +881,25 @@ struct SyncDialog::Implementation .downloads = loadedParentsOf(DiffSection::Download), .deletes = loadedParentsOf(DiffSection::Delete), }, - [this](SharedData::Sync::DiffSnapshot snapshot) - { - recomputeInFlight_ = false; - if (recomputeQueued_) + lifetime_.guarded( + [this](SharedData::Sync::DiffSnapshot snapshot) { - recomputeQueued_ = false; - recomputeDiff(); - return; + recomputeInFlight_ = false; + if (recomputeQueued_) + { + recomputeQueued_ = false; + recomputeDiff(); + return; + } + if (snapshot.summary.cancelled) + { + Nui::globalEventContext.executeActiveEventsImmediately(); + return; + } + diffCurrent_ = true; + applySnapshot(std::move(snapshot), /*collapseZero=*/true); } - if (snapshot.summary.cancelled) - return; - applySnapshot(std::move(snapshot), /*collapseZero=*/true); - } + ) ); } @@ -946,9 +1027,12 @@ struct SyncDialog::Implementation enqueueSelection(section, {relKey}, SharedData::OperationMode::PriorityQueued); } + /** + * @brief Queues the selection of every section. + */ void enqueueOperations() { - if (!provider_) + if (!provider_ || !diffCurrent_.value()) return; for (const auto section : {DiffSection::Upload, DiffSection::Download, DiffSection::Delete}) { @@ -964,55 +1048,86 @@ struct SyncDialog::Implementation */ struct SyncPart { - /** @brief One observer per entry index; a directory delete lists its content as entries of their own. */ - std::vector observers{}; + DiffSection section{DiffSection::Upload}; + /** @brief The row of every entry index; a directory delete lists its content as entries of their own. */ + std::vector rows{}; + /** @brief Deletes report failures by path, transfers by entry index. */ std::unordered_map indexBySource{}; + std::vector failed{}; std::set seenInFlight{}; bool mapped{false}; std::optional completion{}; - std::vector failedSources{}; + std::vector> failedSources{}; + std::vector failedIndices{}; + }; - void markFailedSources() - { - for (auto const& source : failedSources) - { - if (const auto iter = indexBySource.find(source); iter != indexBySource.end()) - *observers[iter->second] = -1.0; - } - failedSources.clear(); - } + /** + * @brief Shows the entry @p index as failed. Without @p type the reason is not known yet, a later report may + * add it. + */ + void markFailedEntry(SyncPart& part, std::size_t index, std::optional type = {}) + { + if (index >= part.rows.size() || (part.failed[index] && !type)) + return; + part.failed[index] = true; + if (type) + setRowFailed(part.section, part.rows[index], *type); + else + setRowState(part.section, part.rows[index], -1.0); + } - void complete() + void markReportedFailures(SyncPart& part) + { + for (auto const& [source, type] : std::exchange(part.failedSources, {})) { - for (auto const& observer : observers) - { - // Rows already marked failed stay failed on an otherwise successful operation. - if (!(*completion && observer->value() < 0.0)) - *observer = *completion ? 1.1 : -1.0; - } + if (const auto iter = part.indexBySource.find(source); iter != part.indexBySource.end()) + markFailedEntry(part, iter->second, type); } + for (auto const& failure : std::exchange(part.failedIndices, {})) + markFailedEntry(part, failure.index, failure.type); + } - /** @brief The backend answered with the rows of this part. */ - void map() + void completePart(SyncPart& part) + { + // Rows already marked failed stay failed on an otherwise successful operation. + for (std::size_t index = 0; index < part.rows.size(); ++index) { - mapped = true; - markFailedSources(); - if (completion) - complete(); + if (!part.failed[index]) + setRowState(part.section, part.rows[index], *part.completion ? 1.1 : -1.0); } - }; + } + /** @brief The backend answered with the rows of @p part. */ + void mapPart(SyncPart& part, std::vector rows, std::vector const& sources) + { + part.rows = std::move(rows); + part.failed.assign(part.rows.size(), false); + for (std::size_t index = 0; index < sources.size() && index < part.rows.size(); ++index) + part.indexBySource.emplace(sources[index], index); + part.mapped = true; + markReportedFailures(part); + if (part.completion) + completePart(part); + } + + /** + * @brief Registers the completion of @p operationId with the queue, kept for as long as this dialog shows the + * execution, see @ref dropRegistrations. + */ void watchCompletion(Ids::OperationId const& operationId, std::shared_ptr part) { + registeredOperations_.push_back(operationId); operationQueue_->addCompletionCallback( operationId, - [part](bool success) - { - part->completion = success; - if (part->mapped) - part->complete(); - Nui::globalEventContext.executeActiveEventsImmediately(); - } + lifetime_.guarded( + [this, part](bool success) + { + part->completion = success; + if (part->mapped) + completePart(*part); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ) ); } @@ -1021,64 +1136,92 @@ struct SyncDialog::Implementation { operationQueue_->addBulkProgressCallback( operationId, - [part](SharedData::BulkProgress const& progress) - { - if (!part->mapped) - return; - auto& observers = part->observers; - for (const auto entryIndex : progress.failedEntryIndices) + lifetime_.guarded( + [this, part](SharedData::BulkProgress const& progress) { - if (entryIndex < observers.size() && observers[entryIndex]->value() >= 0.0) - *observers[entryIndex] = -1.0; - } - std::set stillInFlight; - for (auto const& file : progress.inFlight) - { - if (file.entryIndex >= observers.size()) - continue; - stillInFlight.insert(file.entryIndex); - part->seenInFlight.insert(file.entryIndex); - if (file.totalBytes > 0) - *observers[file.entryIndex] = - static_cast(file.bytes) / static_cast(file.totalBytes); - } - std::erase_if( - part->seenInFlight, - [&](std::uint64_t entryIndex) + if (!part->mapped) + return; + for (const auto entryIndex : progress.failedEntryIndices) + markFailedEntry(*part, entryIndex); + std::set stillInFlight; + for (auto const& file : progress.inFlight) { - if (stillInFlight.contains(entryIndex)) - return false; - auto& observer = observers[entryIndex]; - if (observer->value() >= 0.0 && observer->value() < 1.0) - *observer = 1.1; - return true; + if (file.entryIndex >= part->rows.size() || part->failed[file.entryIndex]) + continue; + stillInFlight.insert(file.entryIndex); + part->seenInFlight.insert(file.entryIndex); + if (file.totalBytes > 0) + setRowState( + part->section, + part->rows[file.entryIndex], + static_cast(file.bytes) / static_cast(file.totalBytes) + ); } - ); - Nui::globalEventContext.executeActiveEventsImmediately(); - } + std::erase_if( + part->seenInFlight, + [&](std::uint64_t entryIndex) + { + if (stillInFlight.contains(entryIndex)) + return false; + auto const& row = part->rows[entryIndex]; + if (!part->failed[entryIndex] && rowState(part->section, row) < 1.0) + setRowState(part->section, row, 1.1); + return true; + } + ); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ) ); watchFailedEntries(operationId, std::move(part)); } - /** @brief Failed entries arrive by their source path and mark the rows they belong to; then completion. */ + /** @brief Failed entries arrive on completion, by source path and for bulk transfers by index; then completion. */ void watchFailedEntries(Ids::OperationId const& operationId, std::shared_ptr part) { operationQueue_->addFailedEntriesCallback( operationId, - [part](std::vector const& failedPaths) - { - for (auto const& path : failedPaths) - part->failedSources.push_back(path); - if (part->mapped) - part->markFailedSources(); - } + lifetime_.guarded( + [this, part]( + std::vector> const& failedEntries, + std::vector const& indexedFailures + ) + { + std::ranges::transform( + failedEntries, + std::back_inserter(part->failedSources), + [](auto const& failure) + { + return std::pair{failure.first, failure.second.type}; + } + ); + part->failedIndices.insert( + part->failedIndices.end(), indexedFailures.begin(), indexedFailures.end() + ); + if (part->mapped) + markReportedFailures(*part); + } + ) ); watchCompletion(operationId, std::move(part)); } + /** + * @brief Removes the callbacks of every execution this dialog registered with the queue. Their operations keep + * running, only the rows they would paint are gone. + */ + void dropRegistrations() + { + if (!operationQueue_) + return; + for (auto const& operationId : std::exchange(registeredOperations_, {})) + operationQueue_->removeCallbacks(operationId); + } + /** * @brief Has the backend plan and queue the sparse selection @p relKeys. The operation ids are chosen here and - * listened to before the request goes out, the answer maps them to rows. + * listened to before the request goes out, the answer maps them to rows. A refused request queues + * nothing, its rows show as failed and the reason is shown. */ void enqueueSelection(DiffSection section, std::vector relKeys, SharedData::OperationMode mode) { @@ -1094,10 +1237,14 @@ struct SyncDialog::Implementation .remoteDeleteId = Ids::generateOperationId(), .localDeleteId = Ids::generateOperationId(), }; - auto uploads = std::make_shared(); - auto downloads = std::make_shared(); - auto remoteDeletes = std::make_shared(); - auto localDeletes = std::make_shared(); + const auto makePart = [section]() + { + return std::make_shared(SyncPart{.section = section}); + }; + auto uploads = makePart(); + auto downloads = makePart(); + auto remoteDeletes = makePart(); + auto localDeletes = makePart(); watchBulkPart(request.uploadId, uploads); watchBulkPart(request.downloadId, downloads); watchFailedEntries(request.remoteDeleteId, remoteDeletes); @@ -1105,59 +1252,53 @@ struct SyncDialog::Implementation provider_->executeSelection( request, - [this, section, request, uploads, downloads, remoteDeletes, localDeletes]( - SharedData::Sync::SyncExecutionResult result - ) - { - const auto mapBulk = [this, section]( - SyncPart& part, - std::vector const& rows, - std::vector const& sources - ) + lifetime_.guarded( + [this, section, request, uploads, downloads, remoteDeletes, localDeletes]( + SharedData::Sync::SyncExecutionResult result + ) { - for (std::size_t index = 0; index < rows.size(); ++index) + mapPart(*uploads, std::move(result.uploads), {}); + mapPart(*downloads, std::move(result.downloads), {}); + mapPart(*remoteDeletes, std::move(result.remoteDeletes), result.remoteDeleteSources); + mapPart(*localDeletes, std::move(result.localDeletes), result.localDeleteSources); + + // A part without rows was never queued and will not complete. + for ( + auto const& [operationId, part] : std::array{ + std::pair{request.uploadId, uploads}, + std::pair{request.downloadId, downloads}, + std::pair{request.remoteDeleteId, remoteDeletes}, + std::pair{request.localDeleteId, localDeletes}, + } + ) { - part.observers.push_back(progressFor(section, rows[index])); - if (index < sources.size()) - part.indexBySource.emplace(sources[index], index); + if (part->rows.empty()) + operationQueue_->removeCallbacks(operationId); } - }; - // The backend lists every failed entry, the rows below a failed directory are not all failed. - const auto setRows = [this, section](std::vector const& rows, double value) - { - for (auto const& row : rows) - *progressFor(section, row) = value; - }; - - mapBulk(*uploads, result.uploads, result.uploadSources); - mapBulk(*downloads, result.downloads, result.downloadSources); - mapBulk(*remoteDeletes, result.remoteDeletes, result.remoteDeleteSources); - mapBulk(*localDeletes, result.localDeletes, result.localDeleteSources); - - // A part without rows was never queued and will not complete. - const std::array parts{ - std::pair{request.uploadId, uploads}, - std::pair{request.downloadId, downloads}, - std::pair{request.remoteDeleteId, remoteDeletes}, - std::pair{request.localDeleteId, localDeletes}, - }; - for (auto const& [operationId, part] : parts) + for (auto const& row : result.failed) + setRowState(section, row, -1.0); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ), + lifetime_.guarded( + [this, section, request](std::string const& message) { - if (part->observers.empty()) + Log::warn("Sync: the backend did not run the selection: {}", message); + for ( + auto const& operationId : + {request.uploadId, request.downloadId, request.remoteDeleteId, request.localDeleteId} + ) operationQueue_->removeCallbacks(operationId); - else - part->map(); + for (auto const& row : request.relKeys) + setRowState(section, row, -1.0); + Notifications::error( + fmt::format( + fmt::runtime(language->get("syncDialog", "executionRejected")), language->translate(message) + ) + ); + Nui::globalEventContext.executeActiveEventsImmediately(); } - setRows(result.failed, -1.0); - Nui::globalEventContext.executeActiveEventsImmediately(); - }, - [this, request](std::string const& message) - { - Log::error("Sync: the backend did not run the selection: {}", message); - for (auto const& operationId : - {request.uploadId, request.downloadId, request.remoteDeleteId, request.localDeleteId}) - operationQueue_->removeCallbacks(operationId); - } + ) ); } }; @@ -1324,7 +1465,10 @@ void SyncDialog::open( std::optional lastSyncTime ) { + if (impl_->provider_ != provider) + impl_->dropRegistrations(); impl_->provider_ = std::move(provider); + impl_->diffCurrent_ = true; impl_->lastSyncTime_ = lastSyncTime; impl_->localPath_ = std::move(localPath); impl_->remotePath_ = std::move(remotePath); @@ -1451,6 +1595,9 @@ Nui::ElementRenderer SyncDialog::operator()() onClick = [this]() { impl_->open_ = false; impl_->minimized_ = false; + // Closing ends the flow: the backend session and the rows' callbacks go. + impl_->dropRegistrations(); + impl_->provider_.reset(); if (impl_->operationQueue_) impl_->operationQueue_->hideMinimizedSync(); Nui::globalEventContext.executeActiveEventsImmediately(); @@ -1773,7 +1920,17 @@ Nui::ElementRenderer SyncDialog::operator()() .text = language->getObserved("syncDialog", "synchronize"), .icon = Ui5Icons::synchronize(), .attributes = { + disabled = observe(impl_->diffCurrent_).generate([this]() -> std::optional { + return impl_->diffCurrent_.value() ? std::nullopt : std::optional{true}; + }), + Nui::Attributes::title = observe(impl_->diffCurrent_).generate([this]() { + return impl_->diffCurrent_.value() + ? std::string{} + : language->get("syncDialog", "synchronizeUnavailable"); + }), onClick = [this](Nui::val) { + if (!impl_->diffCurrent_.value()) + return; const bool deletes = impl_->selectedTotalsOf(DiffSection::Delete).first > 0; impl_->confirmDialog_->open({ .styleVariant = deletes ? Snc::StyleVariant::Danger : Snc::StyleVariant::Warning, @@ -1782,11 +1939,8 @@ Nui::ElementRenderer SyncDialog::operator()() .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, .focusButton = deletes ? std::optional{ConfirmDialog::Button::No} : std::nullopt, .onClose = [this](std::optional btn) { - if (btn != ConfirmDialog::Button::Yes) - return; - impl_->enqueueOperations(); - if (impl_->onSynchronized_) - impl_->onSynchronized_(); + if (btn == ConfirmDialog::Button::Yes) + impl_->enqueueOperations(); } }); } diff --git a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp index 1a729a61..6f359963 100644 --- a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp @@ -61,7 +61,7 @@ struct SyncProgressDialog::Implementation std::shared_ptr provider_{}; SharedData::Sync::DiffOptions initialOptions_{}; - std::function onDone_{}; + std::function, SharedData::Sync::DiffSnapshot)> onDone_{}; // Cancel token; replaced on each open() to invalidate stale callback captures. // The provider also has its own cancel flag that kills the backend walk — this @@ -86,7 +86,7 @@ void SyncProgressDialog::open( bool recursive, bool ignoreHidden, SharedData::Sync::DiffOptions initialOptions, - std::function onDone + std::function, SharedData::Sync::DiffSnapshot)> onDone ) { // Invalidate any stale callbacks from a previous open() by flipping the old @@ -150,8 +150,9 @@ void SyncProgressDialog::open( impl_->phase_ = SharedData::SyncPhase::Done; impl_->open_ = false; Nui::globalEventContext.executeActiveEventsImmediately(); - if (impl_->onDone_) - impl_->onDone_(std::move(snapshot)); + auto provider = std::exchange(impl_->provider_, nullptr); + if (auto onDone = std::exchange(impl_->onDone_, nullptr); onDone) + onDone(std::move(provider), std::move(snapshot)); } ); }, @@ -162,6 +163,8 @@ void SyncProgressDialog::open( return; impl_->failureReason_ = reason; impl_->phase_ = SharedData::SyncPhase::Failed; + impl_->onDone_ = nullptr; + impl_->provider_.reset(); Nui::globalEventContext.executeActiveEventsImmediately(); }, // onDiffProgress @@ -181,8 +184,9 @@ void SyncProgressDialog::cancel() impl_->cancelToken_ = std::make_shared(false); impl_->open_ = false; impl_->phase_ = SharedData::SyncPhase::Idle; - if (impl_->provider_) - impl_->provider_->cancelDiff(); + impl_->onDone_ = nullptr; + if (auto provider = std::exchange(impl_->provider_, nullptr); provider) + provider->cancelDiff(); Nui::globalEventContext.executeActiveEventsImmediately(); } diff --git a/shared_data/include/shared_data/sync/sync_execution.hpp b/shared_data/include/shared_data/sync/sync_execution.hpp index c71b3303..f2978d45 100644 --- a/shared_data/include/shared_data/sync/sync_execution.hpp +++ b/shared_data/include/shared_data/sync/sync_execution.hpp @@ -13,8 +13,8 @@ namespace SharedData::Sync { /** - * @brief Asks the backend to plan and queue the selected rows of one section. Paths never travel: the backend - * resolves every row below its own roots. + * @brief Asks the backend to plan and queue the selected rows of one section. The request names rows only, the + * backend resolves every row below its own roots. * * The operation ids are chosen by the frontend, so it can listen to them before the reply arrives. An id is * unused when its part of the plan is empty. @@ -43,14 +43,12 @@ namespace SharedData::Sync */ struct SyncExecutionResult { - /** @brief Rows of the bulk upload, in the order of its entry indices. */ + /** @brief Rows of the bulk upload, in the order of its entry indices, by which it reports failures. */ std::vector uploads{}; - /** @brief The source each upload row reports when it fails, in the same order. */ - std::vector uploadSources{}; std::vector downloads{}; - std::vector downloadSources{}; /** @brief Every listed item of a directory delete is a row of its own, its content after it. */ std::vector remoteDeletes{}; + /** @brief The path each delete row reports when it fails, in the same order. */ std::vector remoteDeleteSources{}; std::vector localDeletes{}; std::vector localDeleteSources{}; @@ -61,9 +59,7 @@ namespace SharedData::Sync SyncExecutionResult, (), (uploads, - uploadSources, downloads, - downloadSources, remoteDeletes, remoteDeleteSources, localDeletes, diff --git a/static/assets/languages/de_DE.yaml b/static/assets/languages/de_DE.yaml index fa6e84cc..a9b334d3 100644 --- a/static/assets/languages/de_DE.yaml +++ b/static/assets/languages/de_DE.yaml @@ -199,6 +199,8 @@ syncDialog: confirmDeleteRow: "'{}' {} jetzt löschen?" blockedNotice: "{} Einträge bleiben unverändert, weil sie nicht lesbar oder nicht übertragbar sind, ein Verzeichnis auf einen symbolischen Link trifft sich ihre Namen nur in der Groß- und Kleinschreibung unterscheiden oder sie sich seit der letzten Synchronisation so geändert haben, dass Daten verloren gingen. Zum Anzeigen klicken." blockedTitle: "Unveränderte Einträge" + synchronizeUnavailable: "Der Vergleich läuft noch oder wurde abgebrochen. Vergleichen Sie erneut, um zu synchronisieren." + executionRejected: "Diese Einträge wurden nicht synchronisiert: {}" blockReason: Unreadable: "nicht lesbar" Excluded: "nicht übertragbar" @@ -1202,3 +1204,5 @@ backend: connectFailedUnspecified: "Verbindung zum SSH-Server aus unbekanntem Grund fehlgeschlagen." persistence: saveFailed: "Der Zustand konnte nicht gespeichert werden: {}" + sync: + staleComparison: "der Vergleich hat sich seit der Anzeige geändert, bitte erneut vergleichen" diff --git a/static/assets/languages/en_US.yaml b/static/assets/languages/en_US.yaml index 7929001d..a0850209 100644 --- a/static/assets/languages/en_US.yaml +++ b/static/assets/languages/en_US.yaml @@ -139,6 +139,8 @@ syncDialog: confirmDeleteRow: "Delete '{}' {} now?" blockedNotice: "{} entries are left alone because they cannot be read, cannot be transferred, a directory faces a symlink, their names differ only in case, or they changed since the last sync where that would lose data. Click to see them." blockedTitle: "Entries left alone" + synchronizeUnavailable: "The comparison is running or was cancelled. Compare again to synchronize." + executionRejected: "These entries were not synchronized: {}" blockReason: Unreadable: "unreadable" Excluded: "not transferable" @@ -1204,3 +1206,5 @@ backend: connectFailedUnspecified: "Failed to connect to ssh server for unspecified reason." persistence: saveFailed: "Failed to save state to disk: {}" + sync: + staleComparison: "the comparison changed since they were shown, compare again" From f90d136921878e8a9b0350c14e30d037237efa1c Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:40:42 +0200 Subject: [PATCH 046/131] A bulk delete goes on past directories it cannot scan A bulk delete goes on past a directory it cannot scan and past items refused or not looked up in time, each of them a failed entry of its card, so the rest is still deleted; the scan counts of its directories add up instead of starting over for each one; delete progress of remote, bulk and local deletes goes out through one makeDeleteProgressCallback, and DeleteOperation takes its scans through addScanRoot only --- .../include/backend/sftp/delete_operation.hpp | 11 +- .../include/backend/sftp/operation_queue.hpp | 6 + backend/include/backend/sftp/scan_phase.hpp | 24 +++- .../backend/sftp/sequential_scan_progress.hpp | 42 +++++++ .../source/backend/sftp/delete_operation.cpp | 26 ++++- .../source/backend/sftp/operation_queue.cpp | 106 ++++++++---------- 6 files changed, 141 insertions(+), 74 deletions(-) create mode 100644 backend/include/backend/sftp/sequential_scan_progress.hpp diff --git a/backend/include/backend/sftp/delete_operation.hpp b/backend/include/backend/sftp/delete_operation.hpp index 74d10cf3..9770c7ec 100644 --- a/backend/include/backend/sftp/delete_operation.hpp +++ b/backend/include/backend/sftp/delete_operation.hpp @@ -60,14 +60,15 @@ class DeleteOperation : public Operation void setTargets(std::vector targets); /** - * @brief Scans with @p scan before deleting; every scanned entry is deleted, deepest first. + * @brief Scans with @p scan before deleting; every scanned entry is deleted, deepest first. One operation can + * take several roots. A root whose scan fails is a failed entry and the other roots are still deleted. */ - void scanFirst(std::unique_ptr scan); + void addScanRoot(std::unique_ptr scan); /** - * @brief Like @ref scanFirst for one more directory, so one operation deletes several directories. + * @brief Counts @p target as an item that could not be deleted, like one refused before it was queued. */ - void addScanRoot(std::unique_ptr scan); + void addFailedTarget(Paths::RemotePath const& target, Error error); bool scanning() const noexcept { @@ -120,6 +121,8 @@ class DeleteOperation : public Operation std::chrono::seconds futureTimeout_; std::size_t deletedThusFar_{0}; std::size_t totalToDelete_{0}; + /** @brief Targets whose removal failed, the operation fails when that is all of them and nothing else. */ + std::size_t removalFailures_{0}; /** @brief Deleted from the back, so a scanned directory goes after its content. */ std::vector targets_{}; /** @brief Why a scan result was refused, nothing may be deleted then. */ diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index db9897f7..d1614027 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -371,6 +371,12 @@ class OperationQueue auto makeBulkProgressCallback(std::string_view eventName, Ids::OperationId operationId) -> BulkProgressCallback; + /** + * @brief Sends the delete progress of @p operationId (current item, items handled, items in all) to the frontend. + */ + auto makeDeleteProgressCallback(Ids::OperationId operationId) + -> std::function; + private: bool workQueue(std::deque>>& queue); diff --git a/backend/include/backend/sftp/scan_phase.hpp b/backend/include/backend/sftp/scan_phase.hpp index ad5a3a82..92e9d39d 100644 --- a/backend/include/backend/sftp/scan_phase.hpp +++ b/backend/include/backend/sftp/scan_phase.hpp @@ -39,8 +39,17 @@ class ScanPhase : public ScanPhaseDriver */ using DeliverFunction = std::function; - explicit ScanPhase(DeliverFunction deliver) + /** + * @brief Receives a scan that failed, the phase then goes on with the next root. + */ + using FailFunction = std::function; + + /** + * @param onFailed Without it, a failed scan ends the phase with its error. + */ + explicit ScanPhase(DeliverFunction deliver, FailFunction onFailed = {}) : deliver_{std::move(deliver)} + , onFailed_{std::move(onFailed)} {} void addRoot(std::unique_ptr scan) @@ -56,11 +65,15 @@ class ScanPhase : public ScanPhaseDriver auto& current = *scans_.front(); auto result = current.work(); if (!result.has_value()) - return std::unexpected(std::move(result).error()); - if (result.value() != Operation::WorkStatus::Complete) + { + if (!onFailed_) + return std::unexpected(std::move(result).error()); + onFailed_(current, result.error()); + } + else if (result.value() != Operation::WorkStatus::Complete) return Operation::WorkStatus::MoreWork; - - deliver_(current); + else + deliver_(current); scans_.pop_front(); return scans_.empty() ? Operation::WorkStatus::Complete : Operation::WorkStatus::MoreWork; } @@ -73,6 +86,7 @@ class ScanPhase : public ScanPhaseDriver private: DeliverFunction deliver_; + FailFunction onFailed_; std::deque> scans_{}; }; diff --git a/backend/include/backend/sftp/sequential_scan_progress.hpp b/backend/include/backend/sftp/sequential_scan_progress.hpp new file mode 100644 index 00000000..d0298310 --- /dev/null +++ b/backend/include/backend/sftp/sequential_scan_progress.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace Detail +{ + using ScanProgressFunction = std::function; + + /** + * @brief Hands out a progress callback per scan of scans that run one after another, each reporting the running + * total of all of them (bytes, index, entries scanned) to @p report. + */ + inline std::function sequentialScanProgress(ScanProgressFunction report) + { + struct Totals + { + std::size_t roots{0}; + std::size_t current{0}; + std::array finished{}; + std::array running{}; + }; + auto totals = std::make_shared(); + return [totals, report = std::move(report)]() -> ScanProgressFunction + { + const auto root = totals->roots++; + return [totals, report, root](std::uint64_t bytes, std::uint64_t index, std::uint64_t scanned) + { + if (root != totals->current) + { + std::ranges::transform(totals->finished, totals->running, totals->finished.begin(), std::plus{}); + totals->current = root; + } + totals->running = {bytes, index, scanned}; + report(totals->finished[0] + bytes, totals->finished[1] + index, totals->finished[2] + scanned); + }; + }; + } +} diff --git a/backend/source/backend/sftp/delete_operation.cpp b/backend/source/backend/sftp/delete_operation.cpp index 55af74a3..5472ab58 100644 --- a/backend/source/backend/sftp/delete_operation.cpp +++ b/backend/source/backend/sftp/delete_operation.cpp @@ -86,7 +86,7 @@ std::expected DeleteOperati return WorkStatus::MoreWork; } // Nothing deleted at all is a failure, some items left behind a partial success. - if (totalToDelete_ > 0 && failedEntries_.size() == totalToDelete_) + if (!failedEntries_.empty() && removalFailures_ == totalToDelete_) return enterErrorState(failedEntries_.back().second); state_ = Completed; return WorkStatus::Complete; @@ -136,6 +136,7 @@ std::expected DeleteOperation::removeOnce(Paths::R if (const auto result = fut.get(); !result.has_value() && result.error().sftpError != SSH_FX_NO_SUCH_FILE) { Log::error("DeleteOperation: Failed to delete '{}': {}", path.value(), result.error().toString()); + ++removalFailures_; failedEntries_.emplace_back( path.value(), Error{.type = ErrorType::DeleteFailed, .sftpError = result.error(), .extraInfo = path.value()} ); @@ -200,11 +201,6 @@ void DeleteOperation::setTargets(std::vector targets) totalToDelete_ = targets_.size(); } -void DeleteOperation::scanFirst(std::unique_ptr scan) -{ - addScanRoot(std::move(scan)); -} - void DeleteOperation::addScanRoot(std::unique_ptr scan) { if (!scanPhase_) @@ -213,12 +209,30 @@ void DeleteOperation::addScanRoot(std::unique_ptr scan) [this](ScanOperation& finished) { appendScanResult(finished.remotePath(), finished.ejectEntries()); + }, + [this](ScanOperation& failed, Error const& error) + { + Log::error( + "DeleteOperation: Scan of '{}' failed, nothing of it is deleted: {}", + failed.remotePath().value(), + error.toString() + ); + // The other roots are still deleted, the failure alone does not mean the scans are incomplete. + scanDelivered_ = true; + addFailedTarget(failed.remotePath(), error); } ); } scanPhase_->addRoot(std::move(scan)); } +void DeleteOperation::addFailedTarget(Paths::RemotePath const& target, Error error) +{ + if (!error.extraInfo) + error.extraInfo = target.value(); + failedEntries_.emplace_back(target.value(), std::move(error)); +} + std::expected DeleteOperation::cancel(bool adoptCancelState) { if (scanPhase_) diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 039a397d..3d66bf1a 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,8 @@ #include #include +#include +#include #include #include #include @@ -250,6 +253,29 @@ auto OperationQueue::makeScanProgressCallback(std::string_view eventName, Ids::O }; } +auto OperationQueue::makeDeleteProgressCallback(Ids::OperationId operationId) + -> std::function +{ + return [weak = weak_from_this(), operationId, name = rpcName("onDeleteProgress")]( + std::string const& path, std::uint64_t itemsDeleted, std::uint64_t totalItems + ) + { + auto self = weak.lock(); + if (!self) + return; + RpcHelper::callRemoteSafely( + *self->hub_, + name, + SharedData::BulkDeleteProgress{ + .operationId = operationId, + .currentFile = path, + .filesDeleted = itemsDeleted, + .totalFiles = totalItems, + } + ); + }; +} + auto OperationQueue::makeBulkProgressCallback(std::string_view eventName, Ids::OperationId operationId) -> BulkProgressCallback { @@ -1764,25 +1790,7 @@ std::expected OperationQueue::addLocalDeleteOperation( targetQueue.emplace_back( operationId, std::make_unique(LocalDeleteOperation::LocalDeleteOperationOptions{ - .itemsRemovedProgress = - [weak = weak_from_this(), operationId, name = rpcName("onDeleteProgress")]( - std::string const& path, std::uint64_t itemsDeleted, std::uint64_t totalItems - ) - { - auto self = weak.lock(); - if (!self) - return; - RpcHelper::callRemoteSafely( - *self->hub_, - name, - SharedData::BulkDeleteProgress{ - .operationId = operationId, - .currentFile = path, - .filesDeleted = itemsDeleted, - .totalFiles = totalItems, - } - ); - }, + .itemsRemovedProgress = makeDeleteProgressCallback(operationId), .scanProgress = [progress = makeScanProgressCallback("onScanProgress", operationId)]( std::uint64_t bytes, std::uint64_t entries @@ -1883,25 +1891,7 @@ std::expected OperationQueue::addDeleteOperation( auto operation = std::make_unique( sftp, DeleteOperation::DeleteOperationOptions{ - .filesRemovedProgress = - [weak = weak_from_this(), operationId, name = rpcName("onDeleteProgress")]( - auto const& path, std::uint64_t filesDeleted, std::uint64_t totalFiles - ) - { - auto self = weak.lock(); - if (!self) - return; - RpcHelper::callRemoteSafely( - *self->hub_, - name, - SharedData::BulkDeleteProgress{ - .operationId = operationId, - .currentFile = path, - .filesDeleted = filesDeleted, - .totalFiles = totalFiles, - } - ); - }, + .filesRemovedProgress = makeDeleteProgressCallback(operationId), .remotePath = remotePath, .futureTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout), .recursive = isBulk, @@ -1910,7 +1900,7 @@ std::expected OperationQueue::addDeleteOperation( if (isBulk) { // The scan is part of the delete, so cancelling or moving the card acts on both. - operation->scanFirst( + operation->addScanRoot( std::make_unique( sftp, ScanOperation::ScanOperationOptions{ @@ -1965,25 +1955,7 @@ std::size_t OperationQueue::addBulkDeleteOperation( auto operation = std::make_unique( sftp, DeleteOperation::DeleteOperationOptions{ - .filesRemovedProgress = - [weak = weak_from_this(), bulkOperationId, name = rpcName("onDeleteProgress")]( - auto const& path, std::uint64_t filesDeleted, std::uint64_t totalFiles - ) - { - auto self = weak.lock(); - if (!self) - return; - RpcHelper::callRemoteSafely( - *self->hub_, - name, - SharedData::BulkDeleteProgress{ - .operationId = bulkOperationId, - .currentFile = path, - .filesDeleted = filesDeleted, - .totalFiles = totalFiles, - } - ); - }, + .filesRemovedProgress = makeDeleteProgressCallback(bulkOperationId), .futureTimeout = resolvedTimeout, .recursive = true, } @@ -1991,11 +1963,24 @@ std::size_t OperationQueue::addBulkDeleteOperation( std::vector fileTargets; std::size_t scannedDirectories = 0; + std::size_t failedTargets = 0; + const auto failTarget = [&](Paths::RemotePath const& target, Operation::Error error) + { + operation->addFailedTarget(target, std::move(error)); + ++failedTargets; + }; + // The roots are scanned one after another, their counts add up on the card. + const auto scanProgressOfNextRoot = + Detail::sequentialScanProgress(makeScanProgressCallback("onScanProgress", bulkOperationId)); for (auto const& entry : request.entries) { if (auto allowed = guard.check(entry.target); !allowed) { Log::error("Bulk delete: refusing to delete: {}", allowed.error().toString()); + failTarget( + entry.target, + Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = allowed.error().toString()} + ); continue; } if (entry.isDirectory && !request.listedContentOnly) @@ -2005,6 +1990,7 @@ std::size_t OperationQueue::addBulkDeleteOperation( if (statFuture.wait_for(resolvedTimeout) != std::future_status::ready) { Log::error("Bulk delete: lstat timed out for '{}'", entry.target.value()); + failTarget(entry.target, Operation::Error{.type = Operation::ErrorType::FutureTimeout}); continue; } const auto information = statFuture.get(); @@ -2014,7 +2000,7 @@ std::size_t OperationQueue::addBulkDeleteOperation( std::make_unique( sftp, ScanOperation::ScanOperationOptions{ - .progressCallback = makeScanProgressCallback("onScanProgress", bulkOperationId), + .progressCallback = scanProgressOfNextRoot(), .remotePath = entry.target, .futureTimeout = resolvedTimeout, } @@ -2030,6 +2016,8 @@ std::size_t OperationQueue::addBulkDeleteOperation( const auto queued = fileTargets.size() + scannedDirectories; if (queued == 0) return 0; + if (failedTargets > 0) + Log::warn("Bulk delete: {} items were refused, they are reported as failed", failedTargets); // Parent of the first item, so the card can read "Deleting items in " before any progress arrives. const auto commonParent = request.entries.front().target.parent().value_or(Paths::RemotePath::root()); From 883d38012a301535c012e29d1fb26bc6d5decba2 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:43:44 +0200 Subject: [PATCH 047/131] When one scan of a sync fails, the other one is cancelled When one scan of a sync fails, the other one is cancelled and its card ends as cancelled; the reason reaches the progress dialog as a translatable message naming the side that could not be listed --- .../include/backend/sftp/operation_queue.hpp | 6 + .../source/backend/sftp/operation_queue.cpp | 108 ++++++++---------- .../sync_dialog/sync_progress_dialog.cpp | 2 +- static/assets/languages/de_DE.yaml | 3 + static/assets/languages/en_US.yaml | 3 + 5 files changed, 63 insertions(+), 59 deletions(-) diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index d1614027..411aa68d 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -382,9 +382,15 @@ class OperationQueue /** * @brief Tells the sync session waiting for the scan @p id, if any, that it failed or was cancelled. + * @param reason A localized message, see Utility::localizedMessage. */ void failSyncScan(Ids::OperationId const& id, std::string const& reason); + /** + * @brief Drops the scan @p id of a sync whose other scan failed, without telling its session again. + */ + void cancelSiblingScan(Ids::OperationId const& id); + /** * @brief Hands the tree of a completed standalone scan to the sync session registered for its id. */ diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 3d66bf1a..6ba53e87 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -313,7 +314,7 @@ void OperationQueue::cancelAll() self->syncRecordings_.clear(); auto waiting = std::exchange(self->syncScanCallbacks_, {}); for (auto& [id, callback] : waiting) - callback(std::unexpected(std::string{"The scan was cancelled."})); + callback(std::unexpected(Utility::localizedMessage("backend.sync.scanCancelled"))); Log::info("All operations in the queue have been canceled."); } ); @@ -339,7 +340,7 @@ void OperationQueue::cancel(Ids::OperationId id) const auto erasedFromRegular = std::erase_if(self->operations_, cancelPredicate); if (erasedFromRegular == 0) std::erase_if(self->priorityOperations_, cancelPredicate); - self->failSyncScan(id, "The scan was cancelled."); + self->failSyncScan(id, Utility::localizedMessage("backend.sync.scanCancelled")); } ); } @@ -606,7 +607,14 @@ bool OperationQueue::workQueue(std::dequetype() == SharedData::OperationType::LocalScan ? "backend.sync.localScanFailed" + : "backend.sync.remoteScanFailed", + item.result.error().toString() + ) + ); completeOperation( makeCompletedOperation(OperationQueue::CompletionReason::Failed, id, *operation, item.result.error()) ); @@ -631,6 +639,39 @@ bool OperationQueue::workQueue(std::dequecancel(true); + return true; + }; + if (std::erase_if(self->priorityOperations_, matches) + std::erase_if(self->operations_, matches) > 0) + { + self->completeOperation( + SharedData::OperationCompleted{ + .reason = SharedData::OperationCompletionReason::Canceled, + .operationId = id, + .completionTime = std::chrono::system_clock::now(), + } + ); + } + } + ); +} + void OperationQueue::failSyncScan(Ids::OperationId const& id, std::string const& reason) { auto callback = syncScanCallbacks_.extract(id.value()); @@ -2115,61 +2156,12 @@ void OperationQueue::addSyncScanOperation( ); syncSessions_[syncSessionId.value()] = session; - // Scan completion callbacks hand the ScanNode trees into the session on its - // own strand, then emit onSyncScanPhaseDone to the frontend. - syncScanCallbacks_[remoteScanId.value()] = - [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}]( - std::expected tree - ) - { - auto self = weak.lock(); - auto sess = weakSession.lock(); - if (!self || !sess) - return; - if (!tree) - { - RpcHelper::callRemoteSafely( - *self->hub_, self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/false, tree.error() - ); - return; - } - boost::asio::post( - sess->strand(), - [sess, tree = std::move(*tree)]() mutable - { - sess->setRemoteTreeInStrand(std::move(tree)); - } - ); - RpcHelper::callRemoteSafely( - *self->hub_, self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/false - ); - }; - - syncScanCallbacks_[localScanId.value()] = - [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}]( - std::expected tree - ) - { - auto self = weak.lock(); - auto sess = weakSession.lock(); - if (!self || !sess) - return; - if (!tree) - { - RpcHelper::callRemoteSafely( - *self->hub_, self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/true, tree.error() - ); - return; - } - boost::asio::post( - sess->strand(), - [sess, tree = std::move(*tree)]() mutable - { - sess->setLocalTreeInStrand(std::move(tree)); - } - ); - RpcHelper::callRemoteSafely(*self->hub_, self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/true); - }; + syncScanCallbacks_[remoteScanId.value()] = makeSyncScanCallback( + syncSessionId, session, localScanId, /*isLocal=*/false, &SyncSession::setRemoteTreeInStrand + ); + syncScanCallbacks_[localScanId.value()] = makeSyncScanCallback( + syncSessionId, session, remoteScanId, /*isLocal=*/true, &SyncSession::setLocalTreeInStrand + ); // Queue remote scan — build-tree mode. // Temp files of transfers on either side, a sync neither copies nor deletes them. diff --git a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp index 6f359963..e1d4c08a 100644 --- a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp @@ -161,7 +161,7 @@ void SyncProgressDialog::open( { if (*token) return; - impl_->failureReason_ = reason; + impl_->failureReason_ = language->translate(reason); impl_->phase_ = SharedData::SyncPhase::Failed; impl_->onDone_ = nullptr; impl_->provider_.reset(); diff --git a/static/assets/languages/de_DE.yaml b/static/assets/languages/de_DE.yaml index a9b334d3..ecf096b3 100644 --- a/static/assets/languages/de_DE.yaml +++ b/static/assets/languages/de_DE.yaml @@ -1206,3 +1206,6 @@ backend: saveFailed: "Der Zustand konnte nicht gespeichert werden: {}" sync: staleComparison: "der Vergleich hat sich seit der Anzeige geändert, bitte erneut vergleichen" + scanCancelled: "Das Auflisten wurde abgebrochen." + localScanFailed: "Das lokale Verzeichnis konnte nicht aufgelistet werden: {}" + remoteScanFailed: "Das Verzeichnis auf dem Server konnte nicht aufgelistet werden: {}" diff --git a/static/assets/languages/en_US.yaml b/static/assets/languages/en_US.yaml index a0850209..0721ee53 100644 --- a/static/assets/languages/en_US.yaml +++ b/static/assets/languages/en_US.yaml @@ -1208,3 +1208,6 @@ backend: saveFailed: "Failed to save state to disk: {}" sync: staleComparison: "the comparison changed since they were shown, compare again" + scanCancelled: "The listing was cancelled." + localScanFailed: "The local directory could not be listed: {}" + remoteScanFailed: "The directory on the server could not be listed: {}" From 849f304ca55c1d602a59ff9bd73f861e5ce3696f Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:48:28 +0200 Subject: [PATCH 048/131] Sync matches names through one case and normalization fold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync finds names that meet as one file through one fold, caseFoldedName, which lowercases Latin-1, Latin Extended-A, Greek, Cyrillic and fullwidth letters and decomposes their precomposed forms, so case variants like Ä and ä and NFC against NFD spellings are blocked as case conflicts; whether the local root folds case is looked up on the root itself (an entry under another case, else a probe file) instead of assumed per platform; the diff tree store and the delete protection share one isPlannable filter --- backend/source/backend/sync/sync_session.cpp | 8 +- .../include/shared_data/sync/case_fold.hpp | 20 ++ shared_data/include/shared_data/sync/diff.hpp | 6 + shared_data/source/shared_data/CMakeLists.txt | 1 + .../source/shared_data/sync/case_fold.cpp | 319 ++++++++++++++++++ shared_data/source/shared_data/sync/diff.cpp | 37 +- .../shared_data/sync/diff_tree_store.cpp | 26 +- 7 files changed, 370 insertions(+), 47 deletions(-) create mode 100644 shared_data/include/shared_data/sync/case_fold.hpp create mode 100644 shared_data/source/shared_data/sync/case_fold.cpp diff --git a/backend/source/backend/sync/sync_session.cpp b/backend/source/backend/sync/sync_session.cpp index 083de7b9..01e2709f 100644 --- a/backend/source/backend/sync/sync_session.cpp +++ b/backend/source/backend/sync/sync_session.cpp @@ -85,12 +85,8 @@ DiffSummary SyncSession::recomputeDiffInStrand( ) { auto options = requestedOptions; -#if defined(_WIN32) || defined(__APPLE__) - // NTFS and APFS fold case by default. - options.localCaseInsensitive = true; -#else - options.localCaseInsensitive = false; -#endif + options.localCaseInsensitive = options_.localFoldsCase; + options.remoteCaseInsensitive = options_.remoteFoldsCase; uploads_.clear(options); downloads_.clear(options); deletes_.clear(options); diff --git a/shared_data/include/shared_data/sync/case_fold.hpp b/shared_data/include/shared_data/sync/case_fold.hpp new file mode 100644 index 00000000..a1cd54a2 --- /dev/null +++ b/shared_data/include/shared_data/sync/case_fold.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace SharedData::Sync +{ + /** + * @brief The key under which names meet on a filesystem that folds case or normalization, like NTFS, APFS and + * HFS+: every letter is lowercased (ASCII, Latin-1, Latin Extended-A, Greek, Cyrillic, fullwidth Latin) + * and every precomposed letter of those scripts is decomposed into its base and combining marks, so an + * NFC and an NFD spelling of a name get the same key. Text that is not UTF-8 is its own key. + */ + std::string caseFoldedName(std::string_view name); + + /** + * @brief False when @p name is its own case folded key, which holds for most names and costs no allocation. + */ + bool foldingChanges(std::string_view name); +} diff --git a/shared_data/include/shared_data/sync/diff.hpp b/shared_data/include/shared_data/sync/diff.hpp index 605fab16..9aa7e596 100644 --- a/shared_data/include/shared_data/sync/diff.hpp +++ b/shared_data/include/shared_data/sync/diff.hpp @@ -104,6 +104,12 @@ namespace SharedData::Sync /** @brief True for names the "ignore hidden" option drops. */ bool isHiddenName(std::string_view name); + /** + * @brief Whether a child inside a directory that is transferred or deleted as a whole is part of it: listed and + * not dropped as hidden. + */ + bool isPlannable(ScanNode const& child, DiffOptions const& options); + /** * @brief Parallel merge walk of two sorted scan trees. * diff --git a/shared_data/source/shared_data/CMakeLists.txt b/shared_data/source/shared_data/CMakeLists.txt index 5511a387..01e22cfa 100644 --- a/shared_data/source/shared_data/CMakeLists.txt +++ b/shared_data/source/shared_data/CMakeLists.txt @@ -3,6 +3,7 @@ add_library( STATIC directory_entry.cpp ignore_rules.cpp + sync/case_fold.cpp sync/diff.cpp sync/diff_tree_store.cpp sync/enqueue_minimizer.cpp diff --git a/shared_data/source/shared_data/sync/case_fold.cpp b/shared_data/source/shared_data/sync/case_fold.cpp new file mode 100644 index 00000000..3919d9e5 --- /dev/null +++ b/shared_data/source/shared_data/sync/case_fold.cpp @@ -0,0 +1,319 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace SharedData::Sync +{ + namespace + { + struct Decomposition + { + char32_t composed; + char32_t base; + char32_t mark; + }; + + /** + * @brief Canonical decompositions into two code points of Latin-1, Latin Extended-A, Greek and Cyrillic, + * sorted by the composed code point. A base can decompose again. + */ + constexpr std::array decompositions = std::to_array( + {{0xC0u, 0x41u, 0x300u}, + {0xC1u, 0x41u, 0x301u}, + {0xC2u, 0x41u, 0x302u}, + {0xC3u, 0x41u, 0x303u}, + {0xC4u, 0x41u, 0x308u}, + {0xC5u, 0x41u, 0x30Au}, + {0xC7u, 0x43u, 0x327u}, + {0xC8u, 0x45u, 0x300u}, + {0xC9u, 0x45u, 0x301u}, + {0xCAu, 0x45u, 0x302u}, + {0xCBu, 0x45u, 0x308u}, + {0xCCu, 0x49u, 0x300u}, + {0xCDu, 0x49u, 0x301u}, + {0xCEu, 0x49u, 0x302u}, + {0xCFu, 0x49u, 0x308u}, + {0xD1u, 0x4Eu, 0x303u}, + {0xD2u, 0x4Fu, 0x300u}, + {0xD3u, 0x4Fu, 0x301u}, + {0xD4u, 0x4Fu, 0x302u}, + {0xD5u, 0x4Fu, 0x303u}, + {0xD6u, 0x4Fu, 0x308u}, + {0xD9u, 0x55u, 0x300u}, + {0xDAu, 0x55u, 0x301u}, + {0xDBu, 0x55u, 0x302u}, + {0xDCu, 0x55u, 0x308u}, + {0xDDu, 0x59u, 0x301u}, + {0xE0u, 0x61u, 0x300u}, + {0xE1u, 0x61u, 0x301u}, + {0xE2u, 0x61u, 0x302u}, + {0xE3u, 0x61u, 0x303u}, + {0xE4u, 0x61u, 0x308u}, + {0xE5u, 0x61u, 0x30Au}, + {0xE7u, 0x63u, 0x327u}, + {0xE8u, 0x65u, 0x300u}, + {0xE9u, 0x65u, 0x301u}, + {0xEAu, 0x65u, 0x302u}, + {0xEBu, 0x65u, 0x308u}, + {0xECu, 0x69u, 0x300u}, + {0xEDu, 0x69u, 0x301u}, + {0xEEu, 0x69u, 0x302u}, + {0xEFu, 0x69u, 0x308u}, + {0xF1u, 0x6Eu, 0x303u}, + {0xF2u, 0x6Fu, 0x300u}, + {0xF3u, 0x6Fu, 0x301u}, + {0xF4u, 0x6Fu, 0x302u}, + {0xF5u, 0x6Fu, 0x303u}, + {0xF6u, 0x6Fu, 0x308u}, + {0xF9u, 0x75u, 0x300u}, + {0xFAu, 0x75u, 0x301u}, + {0xFBu, 0x75u, 0x302u}, + {0xFCu, 0x75u, 0x308u}, + {0xFDu, 0x79u, 0x301u}, + {0xFFu, 0x79u, 0x308u}, + {0x100u, 0x41u, 0x304u}, + {0x101u, 0x61u, 0x304u}, + {0x102u, 0x41u, 0x306u}, + {0x103u, 0x61u, 0x306u}, + {0x104u, 0x41u, 0x328u}, + {0x105u, 0x61u, 0x328u}, + {0x106u, 0x43u, 0x301u}, + {0x107u, 0x63u, 0x301u}, + {0x108u, 0x43u, 0x302u}, + {0x109u, 0x63u, 0x302u}, + {0x10Au, 0x43u, 0x307u}, + {0x10Bu, 0x63u, 0x307u}, + {0x10Cu, 0x43u, 0x30Cu}, + {0x10Du, 0x63u, 0x30Cu}, + {0x10Eu, 0x44u, 0x30Cu}, + {0x10Fu, 0x64u, 0x30Cu}, + {0x112u, 0x45u, 0x304u}, + {0x113u, 0x65u, 0x304u}, + {0x114u, 0x45u, 0x306u}, + {0x115u, 0x65u, 0x306u}, + {0x116u, 0x45u, 0x307u}, + {0x117u, 0x65u, 0x307u}, + {0x118u, 0x45u, 0x328u}, + {0x119u, 0x65u, 0x328u}, + {0x11Au, 0x45u, 0x30Cu}, + {0x11Bu, 0x65u, 0x30Cu}, + {0x11Cu, 0x47u, 0x302u}, + {0x11Du, 0x67u, 0x302u}, + {0x11Eu, 0x47u, 0x306u}, + {0x11Fu, 0x67u, 0x306u}, + {0x120u, 0x47u, 0x307u}, + {0x121u, 0x67u, 0x307u}, + {0x122u, 0x47u, 0x327u}, + {0x123u, 0x67u, 0x327u}, + {0x124u, 0x48u, 0x302u}, + {0x125u, 0x68u, 0x302u}, + {0x128u, 0x49u, 0x303u}, + {0x129u, 0x69u, 0x303u}, + {0x12Au, 0x49u, 0x304u}, + {0x12Bu, 0x69u, 0x304u}, + {0x12Cu, 0x49u, 0x306u}, + {0x12Du, 0x69u, 0x306u}, + {0x12Eu, 0x49u, 0x328u}, + {0x12Fu, 0x69u, 0x328u}, + {0x130u, 0x49u, 0x307u}, + {0x134u, 0x4Au, 0x302u}, + {0x135u, 0x6Au, 0x302u}, + {0x136u, 0x4Bu, 0x327u}, + {0x137u, 0x6Bu, 0x327u}, + {0x139u, 0x4Cu, 0x301u}, + {0x13Au, 0x6Cu, 0x301u}, + {0x13Bu, 0x4Cu, 0x327u}, + {0x13Cu, 0x6Cu, 0x327u}, + {0x13Du, 0x4Cu, 0x30Cu}, + {0x13Eu, 0x6Cu, 0x30Cu}, + {0x143u, 0x4Eu, 0x301u}, + {0x144u, 0x6Eu, 0x301u}, + {0x145u, 0x4Eu, 0x327u}, + {0x146u, 0x6Eu, 0x327u}, + {0x147u, 0x4Eu, 0x30Cu}, + {0x148u, 0x6Eu, 0x30Cu}, + {0x14Cu, 0x4Fu, 0x304u}, + {0x14Du, 0x6Fu, 0x304u}, + {0x14Eu, 0x4Fu, 0x306u}, + {0x14Fu, 0x6Fu, 0x306u}, + {0x150u, 0x4Fu, 0x30Bu}, + {0x151u, 0x6Fu, 0x30Bu}, + {0x154u, 0x52u, 0x301u}, + {0x155u, 0x72u, 0x301u}, + {0x156u, 0x52u, 0x327u}, + {0x157u, 0x72u, 0x327u}, + {0x158u, 0x52u, 0x30Cu}, + {0x159u, 0x72u, 0x30Cu}, + {0x15Au, 0x53u, 0x301u}, + {0x15Bu, 0x73u, 0x301u}, + {0x15Cu, 0x53u, 0x302u}, + {0x15Du, 0x73u, 0x302u}, + {0x15Eu, 0x53u, 0x327u}, + {0x15Fu, 0x73u, 0x327u}, + {0x160u, 0x53u, 0x30Cu}, + {0x161u, 0x73u, 0x30Cu}, + {0x162u, 0x54u, 0x327u}, + {0x163u, 0x74u, 0x327u}, + {0x164u, 0x54u, 0x30Cu}, + {0x165u, 0x74u, 0x30Cu}, + {0x168u, 0x55u, 0x303u}, + {0x169u, 0x75u, 0x303u}, + {0x16Au, 0x55u, 0x304u}, + {0x16Bu, 0x75u, 0x304u}, + {0x16Cu, 0x55u, 0x306u}, + {0x16Du, 0x75u, 0x306u}, + {0x16Eu, 0x55u, 0x30Au}, + {0x16Fu, 0x75u, 0x30Au}, + {0x170u, 0x55u, 0x30Bu}, + {0x171u, 0x75u, 0x30Bu}, + {0x172u, 0x55u, 0x328u}, + {0x173u, 0x75u, 0x328u}, + {0x174u, 0x57u, 0x302u}, + {0x175u, 0x77u, 0x302u}, + {0x176u, 0x59u, 0x302u}, + {0x177u, 0x79u, 0x302u}, + {0x178u, 0x59u, 0x308u}, + {0x179u, 0x5Au, 0x301u}, + {0x17Au, 0x7Au, 0x301u}, + {0x17Bu, 0x5Au, 0x307u}, + {0x17Cu, 0x7Au, 0x307u}, + {0x17Du, 0x5Au, 0x30Cu}, + {0x17Eu, 0x7Au, 0x30Cu}, + {0x385u, 0xA8u, 0x301u}, + {0x386u, 0x391u, 0x301u}, + {0x388u, 0x395u, 0x301u}, + {0x389u, 0x397u, 0x301u}, + {0x38Au, 0x399u, 0x301u}, + {0x38Cu, 0x39Fu, 0x301u}, + {0x38Eu, 0x3A5u, 0x301u}, + {0x38Fu, 0x3A9u, 0x301u}, + {0x390u, 0x3CAu, 0x301u}, + {0x3AAu, 0x399u, 0x308u}, + {0x3ABu, 0x3A5u, 0x308u}, + {0x3ACu, 0x3B1u, 0x301u}, + {0x3ADu, 0x3B5u, 0x301u}, + {0x3AEu, 0x3B7u, 0x301u}, + {0x3AFu, 0x3B9u, 0x301u}, + {0x3B0u, 0x3CBu, 0x301u}, + {0x3CAu, 0x3B9u, 0x308u}, + {0x3CBu, 0x3C5u, 0x308u}, + {0x3CCu, 0x3BFu, 0x301u}, + {0x3CDu, 0x3C5u, 0x301u}, + {0x3CEu, 0x3C9u, 0x301u}, + {0x3D3u, 0x3D2u, 0x301u}, + {0x3D4u, 0x3D2u, 0x308u}, + {0x400u, 0x415u, 0x300u}, + {0x401u, 0x415u, 0x308u}, + {0x403u, 0x413u, 0x301u}, + {0x407u, 0x406u, 0x308u}, + {0x40Cu, 0x41Au, 0x301u}, + {0x40Du, 0x418u, 0x300u}, + {0x40Eu, 0x423u, 0x306u}, + {0x419u, 0x418u, 0x306u}, + {0x439u, 0x438u, 0x306u}, + {0x450u, 0x435u, 0x300u}, + {0x451u, 0x435u, 0x308u}, + {0x453u, 0x433u, 0x301u}, + {0x457u, 0x456u, 0x308u}, + {0x45Cu, 0x43Au, 0x301u}, + {0x45Du, 0x438u, 0x300u}, + {0x45Eu, 0x443u, 0x306u}, + {0x476u, 0x474u, 0x30Fu}, + {0x477u, 0x475u, 0x30Fu}, + {0x4C1u, 0x416u, 0x306u}, + {0x4C2u, 0x436u, 0x306u}, + {0x4D0u, 0x410u, 0x306u}, + {0x4D1u, 0x430u, 0x306u}, + {0x4D2u, 0x410u, 0x308u}, + {0x4D3u, 0x430u, 0x308u}, + {0x4D6u, 0x415u, 0x306u}, + {0x4D7u, 0x435u, 0x306u}, + {0x4DAu, 0x4D8u, 0x308u}, + {0x4DBu, 0x4D9u, 0x308u}, + {0x4DCu, 0x416u, 0x308u}, + {0x4DDu, 0x436u, 0x308u}, + {0x4DEu, 0x417u, 0x308u}, + {0x4DFu, 0x437u, 0x308u}, + {0x4E2u, 0x418u, 0x304u}, + {0x4E3u, 0x438u, 0x304u}, + {0x4E4u, 0x418u, 0x308u}, + {0x4E5u, 0x438u, 0x308u}, + {0x4E6u, 0x41Eu, 0x308u}, + {0x4E7u, 0x43Eu, 0x308u}, + {0x4EAu, 0x4E8u, 0x308u}, + {0x4EBu, 0x4E9u, 0x308u}, + {0x4ECu, 0x42Du, 0x308u}, + {0x4EDu, 0x44Du, 0x308u}, + {0x4EEu, 0x423u, 0x304u}, + {0x4EFu, 0x443u, 0x304u}, + {0x4F0u, 0x423u, 0x308u}, + {0x4F1u, 0x443u, 0x308u}, + {0x4F2u, 0x423u, 0x30Bu}, + {0x4F3u, 0x443u, 0x30Bu}, + {0x4F4u, 0x427u, 0x308u}, + {0x4F5u, 0x447u, 0x308u}, + {0x4F8u, 0x42Bu, 0x308u}, + {0x4F9u, 0x44Bu, 0x308u}} + ); + + char32_t foldedLetter(char32_t codePoint) + { + // Simple case folding differs from the lowercase mapping in these. + switch (codePoint) + { + case 0xB5u: + return 0x3BCu; + case 0x17Fu: + return U's'; + case 0x3C2u: + return 0x3C3u; + default: + return Utility::FuzzySearch::toLowerCase(codePoint); + } + } + + void appendDecomposed(char32_t codePoint, std::u32string& out) + { + const auto found = std::ranges::lower_bound(decompositions, codePoint, {}, &Decomposition::composed); + if (found == decompositions.end() || found->composed != codePoint) + { + out.push_back(foldedLetter(codePoint)); + return; + } + appendDecomposed(found->base, out); + out.push_back(found->mark); + } + } + + bool foldingChanges(std::string_view name) + { + return std::ranges::any_of( + name, + [](char character) + { + const auto byte = static_cast(character); + return byte >= 0x80u || (byte >= 'A' && byte <= 'Z'); + } + ); + } + + std::string caseFoldedName(std::string_view name) + { + if (!foldingChanges(name)) + return std::string{name}; + if (!Paths::isValidUtf8(name)) + return std::string{name}; + std::u32string folded; + folded.reserve(name.size()); + for (const auto codePoint : Utility::Utf8::decode(name)) + appendDecomposed(codePoint, folded); + return Utility::Utf8::encode(folded); + } +} diff --git a/shared_data/source/shared_data/sync/diff.cpp b/shared_data/source/shared_data/sync/diff.cpp index a2898758..a17c0f9f 100644 --- a/shared_data/source/shared_data/sync/diff.cpp +++ b/shared_data/source/shared_data/sync/diff.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -222,22 +223,11 @@ namespace SharedData::Sync sink.emitDownload(std::move(row), parentRelKey); } - std::string foldedCase(std::string_view name) - { - std::string folded{name}; - for (auto& character : folded) - { - if (character >= 'A' && character <= 'Z') - character = static_cast(character - 'A' + 'a'); - } - return folded; - } - /** - * @brief Names of one directory that would meet as one file on a case folding filesystem: a local-only and a - * remote-only name that differ only in case, and remote names that differ only in case when the local - * filesystem folds case. Transferring either could overwrite the other and a delete could hit both. - * ASCII letters only. + * @brief Names of one directory that would meet as one file on a destination that folds case: variants that + * differ only in case or normalization across the sides, and variants on the side transferred from. + * Transferring one could overwrite the other and a delete could hit both. Only the directions whose + * destination folds count, a case sensitive destination stores every variant. See @ref caseFoldedName. */ std::unordered_set caseConflictsOf( std::vector const& localChildren, @@ -245,18 +235,18 @@ namespace SharedData::Sync DiffOptions const& options ) { - const auto hasUpper = [](std::vector const& children) + const auto anyFolds = [](std::vector const& children) { return std::ranges::any_of( children, [](ScanNode const& node) { - return std::ranges::any_of(node.name, [](char character) { return character >= 'A' && character <= 'Z'; }); + return foldingChanges(node.name); } ); }; - // Without an upper case letter folding changes nothing, which is the common case. - if (!hasUpper(localChildren) && !hasUpper(remoteChildren)) + // Lower case ASCII names fold to themselves, which is the common case. + if (!anyFolds(localChildren) && !anyFolds(remoteChildren)) return {}; struct Variants @@ -266,9 +256,9 @@ namespace SharedData::Sync }; std::unordered_map byFolded; for (auto const& node : localChildren) - byFolded[foldedCase(node.name)].local.push_back(node.name); + byFolded[caseFoldedName(node.name)].local.push_back(node.name); for (auto const& node : remoteChildren) - byFolded[foldedCase(node.name)].remote.push_back(node.name); + byFolded[caseFoldedName(node.name)].remote.push_back(node.name); std::unordered_set conflicts; for (auto const& [folded, variants] : byFolded) @@ -415,6 +405,11 @@ namespace SharedData::Sync return !name.empty() && name.front() == '.'; } + bool isPlannable(ScanNode const& child, DiffOptions const& options) + { + return child.availability == Availability::Listed && (!options.ignoreHidden || !isHiddenName(child.name)); + } + /** * @brief Link literals compare with repeated separators collapsed, "a//b" points where "a/b" does. */ diff --git a/shared_data/source/shared_data/sync/diff_tree_store.cpp b/shared_data/source/shared_data/sync/diff_tree_store.cpp index 1962649d..f277da24 100644 --- a/shared_data/source/shared_data/sync/diff_tree_store.cpp +++ b/shared_data/source/shared_data/sync/diff_tree_store.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -192,9 +193,7 @@ namespace SharedData::Sync keys.reserve(directory->children.size()); for (auto const& child : directory->children) { - if (options_.ignoreHidden && isHiddenName(child.name)) - continue; - if (child.availability != Availability::Listed) + if (!isPlannable(child, options_)) continue; auto childKey = joinRelKey(parentRelKey, child.name); nodes_.insert_or_assign(childKey, makeOneSidedRow(child, childKey, parent.action, sourceIsLocal)); @@ -227,30 +226,19 @@ namespace SharedData::Sync // Downloaded onto a case folding filesystem, names that differ only in case would meet as one file. std::unordered_map foldedCounts; const bool foldsCase = action == Action::Download && options_.localCaseInsensitive; - const auto folded = [](std::string name) - { - for (auto& character : name) - { - if (character >= 'A' && character <= 'Z') - character = static_cast(character - 'A' + 'a'); - } - return name; - }; if (foldsCase) { for (auto const& child : directory.children) - ++foldedCounts[folded(child.name)]; + ++foldedCounts[caseFoldedName(child.name)]; } bool anyChild = false; for (auto const& child : directory.children) { - if (options_.ignoreHidden && isHiddenName(child.name)) - continue; // Unknown or untransferable entries are never planned, not even inside a transferred directory. - if (child.availability != Availability::Listed) + if (!isPlannable(child, options_)) continue; - if (foldsCase && foldedCounts[folded(child.name)] > 1) + if (foldsCase && foldedCounts[caseFoldedName(child.name)] > 1) continue; anyChild = true; auto childKey = joinRelKey(relKey, child.name); @@ -282,9 +270,7 @@ namespace SharedData::Sync ); for (auto const& child : directory.children) { - if (options_.ignoreHidden && isHiddenName(child.name)) - continue; - if (child.availability != Availability::Listed) + if (!isPlannable(child, options_)) continue; auto childKey = joinRelKey(relKey, child.name); if (child.type == FileType::Directory) From 63e0e72f420d4c60e51ace13bf2767a58ec0affd Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:50:30 +0200 Subject: [PATCH 049/131] The remote existence check lists a directory only for ten children The remote existence check lists a directory only when at least ten of its children are asked about, and then without resolving symlinks; the others get an lstat each, as do the children of a directory that may not be listed (a drop box) and a name the listing holds only in another case or normalization, which a server folding names would find --- backend/include/backend/existence_check.hpp | 12 +- backend/source/backend/existence_check.cpp | 165 +++++++++++++------- ssh/include/ssh/sftp_session.hpp | 8 +- ssh/source/ssh/sftp_session.cpp | 10 +- 4 files changed, 128 insertions(+), 67 deletions(-) diff --git a/backend/include/backend/existence_check.hpp b/backend/include/backend/existence_check.hpp index d985d39a..3c906c36 100644 --- a/backend/include/backend/existence_check.hpp +++ b/backend/include/backend/existence_check.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -14,8 +15,15 @@ std::expected, std::string> localPathsExist(std::vector const& paths); /** - * @brief Whether each of @p paths exists on the server, with the same rules as localPathsExist. Few paths are - * checked with one lstat each, more with one listing per parent directory. + * @brief How many queried children of one remote directory are checked by listing it instead of an lstat each. + */ +constexpr std::size_t remoteListingThreshold = 10; + +/** + * @brief Whether each of @p paths exists on the server, with the same rules as localPathsExist. The children of a + * directory with at least @ref remoteListingThreshold of them queried are checked with one listing of it + * (names only), the others with one lstat each. A directory that cannot be listed for its permissions has its + * children looked up one by one, and so has a name only a case or normalization variant of is listed. */ std::expected, std::string> remotePathsExist( SecureShell::SftpSession& session, diff --git a/backend/source/backend/existence_check.cpp b/backend/source/backend/existence_check.cpp index 403e588e..cac367fa 100644 --- a/backend/source/backend/existence_check.cpp +++ b/backend/source/backend/existence_check.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -6,6 +7,7 @@ #include #include +#include #include #include #include @@ -35,79 +37,104 @@ std::expected, std::string> localPathsExist(std::vector, std::string> remotePathsExistByStat( + /** + * @brief Whether @p path exists, nothing but "no such file" counts as absent. + */ + std::expected remotePathExists( SecureShell::SftpSession& session, - std::vector const& paths, + Paths::RemotePath const& path, std::chrono::milliseconds timeout ) { - std::vector results; - results.reserve(paths.size()); - for (auto const& path : paths) + auto future = session.lstat(path); + if (future.wait_for(timeout) != std::future_status::ready) + return std::unexpected(fmt::format("'{}': stat timed out", path.value())); + const auto result = future.get(); + if (!result && result.error().sftpError != SSH_FX_NO_SUCH_FILE) + return std::unexpected(fmt::format("'{}': {}", path.value(), result.error().toString())); + return result.has_value(); + } + + /** + * @brief The names in @p parent, without resolving symlinks. Nothing for a directory that cannot be listed + * because of its permissions, like a drop box; an empty set for one that does not exist yet. + */ + std::expected>, std::string> + listNames(SecureShell::SftpSession& session, Paths::RemotePath const& parent, std::chrono::milliseconds timeout) + { + auto future = session.listDirectory(parent, /*resolveLinks=*/false); + if (future.wait_for(timeout) != std::future_status::ready) + return std::unexpected(fmt::format("'{}': listing timed out", parent.value())); + const auto result = future.get(); + if (!result) { - auto future = session.lstat(path); - if (future.wait_for(timeout) != std::future_status::ready) - return std::unexpected(fmt::format("'{}': stat timed out", path.value())); - const auto result = future.get(); - if (!result && result.error().sftpError != SSH_FX_NO_SUCH_FILE) - return std::unexpected(fmt::format("'{}': {}", path.value(), result.error().toString())); - results.push_back(result.has_value()); + if (result.error().sftpError == SSH_FX_NO_SUCH_FILE) + return std::unordered_set{}; + if (result.error().sftpError == SSH_FX_PERMISSION_DENIED) + return std::nullopt; + return std::unexpected(fmt::format("'{}': {}", parent.value(), result.error().toString())); } - return results; + std::unordered_set names; + names.reserve(result->size()); + for (auto const& entry : *result) + names.insert(entry.name); + return names; } /** - * @brief One listing per parent directory instead of a stat per path. A parent that does not exist yet holds - * nothing. + * @brief Whether the queried children of one parent exist, from one listing of it. A name the listing lacks + * while it holds a case or normalization variant is looked up, since the server may fold names. */ - std::expected, std::string> remotePathsExistByListing( + std::expected checkByListing( SecureShell::SftpSession& session, + Paths::RemotePath const& parent, std::vector const& paths, - std::chrono::milliseconds timeout + std::vector const& indices, + std::chrono::milliseconds timeout, + std::vector& results ) { - const auto listParent = - [&session, timeout](Paths::RemotePath const& parent) -> std::expected, std::string> + auto names = listNames(session, parent, timeout); + if (!names) + return std::unexpected(names.error()); + if (!*names) { - auto future = session.listDirectory(parent); - if (future.wait_for(timeout) != std::future_status::ready) - return std::unexpected(fmt::format("'{}': listing timed out", parent.value())); - const auto result = future.get(); - if (!result) + for (const auto index : indices) { - if (result.error().sftpError == SSH_FX_NO_SUCH_FILE) - return std::unordered_set{}; - return std::unexpected(fmt::format("'{}': {}", parent.value(), result.error().toString())); + auto exists = remotePathExists(session, paths[index], timeout); + if (!exists) + return std::unexpected(exists.error()); + results[index] = *exists; } - std::unordered_set names; - names.reserve(result->size()); - for (auto const& entry : *result) - names.insert(entry.name); - return names; - }; + return {}; + } - std::unordered_map> listingByParent; - std::vector results; - results.reserve(paths.size()); - for (auto const& path : paths) + std::optional> foldedNames{}; + for (const auto index : indices) { - const auto parent = path.parent(); - if (!parent) - return std::unexpected(fmt::format("'{}' has no parent directory", path.value())); - auto iter = listingByParent.find(parent->value()); - if (iter == listingByParent.end()) + const auto name = std::string{paths[index].filename()->view()}; + if ((*names)->contains(name)) + { + results[index] = true; + continue; + } + if (!foldedNames) + { + foldedNames.emplace(); + for (auto const& listed : **names) + foldedNames->insert(SharedData::Sync::caseFoldedName(listed)); + } + if (!foldedNames->contains(SharedData::Sync::caseFoldedName(name))) { - auto listing = listParent(*parent); - if (!listing) - return std::unexpected(listing.error()); - iter = listingByParent.emplace(parent->value(), *std::move(listing)).first; + results[index] = false; + continue; } - results.push_back(iter->second.contains(std::string{path.filename()->view()})); + auto exists = remotePathExists(session, paths[index], timeout); + if (!exists) + return std::unexpected(exists.error()); + results[index] = *exists; } - return results; + return {}; } } @@ -119,14 +146,36 @@ std::expected, std::string> remotePathsExist( { std::vector remotePaths; remotePaths.reserve(paths.size()); + // Parents in the order they first appear, with the indices of their queried children. + std::vector>> byParent; + std::unordered_map parentIndex; for (auto const& path : paths) { - auto remotePath = Paths::RemotePath::make(path); - if (!remotePath) - return std::unexpected(remotePath.error().toString()); - remotePaths.push_back(*std::move(remotePath)); + const auto parent = remotePaths[index].parent(); + if (!parent) + return std::unexpected(fmt::format("'{}' has no parent directory", remotePaths[index].value())); + const auto [found, inserted] = parentIndex.try_emplace(parent->value(), byParent.size()); + if (inserted) + byParent.emplace_back(*parent, std::vector{}); + byParent[found->second].second.push_back(index); + } + + std::vector results(remotePaths.size(), false); + for (auto const& [parent, indices] : byParent) + { + if (indices.size() >= remoteListingThreshold) + { + if (auto checked = checkByListing(session, parent, remotePaths, indices, timeout, results); !checked) + return std::unexpected(checked.error()); + continue; + } + for (const auto index : indices) + { + auto exists = remotePathExists(session, remotePaths[index], timeout); + if (!exists) + return std::unexpected(exists.error()); + results[index] = *exists; + } } - if (remotePaths.size() < listingThreshold) - return remotePathsExistByStat(session, remotePaths, timeout); - return remotePathsExistByListing(session, remotePaths, timeout); + return results; } diff --git a/ssh/include/ssh/sftp_session.hpp b/ssh/include/ssh/sftp_session.hpp index 7f02fd26..0b6d9e9b 100644 --- a/ssh/include/ssh/sftp_session.hpp +++ b/ssh/include/ssh/sftp_session.hpp @@ -72,18 +72,22 @@ namespace SecureShell * @brief Lists the contents of a directory. * * @param path + * @param resolveLinks Reads the literal and the target of every symlink, two more round trips each. Without + * it a symlink is listed as the link itself only. * @return std::future, Error>> */ std::future, Error>> - listDirectory(Paths::RemotePath const& path); + listDirectory(Paths::RemotePath const& path, bool resolveLinks = true); /** * @brief In-strand variant of listDirectory. Must be called from within the processing thread. * * @param path + * @param resolveLinks See @ref listDirectory. * @return std::expected, Error> */ - std::expected, Error> listDirectoryInStrand(Paths::RemotePath const& path); + std::expected, Error> + listDirectoryInStrand(Paths::RemotePath const& path, bool resolveLinks = true); /** * @brief Create a directory. diff --git a/ssh/source/ssh/sftp_session.cpp b/ssh/source/ssh/sftp_session.cpp index 533e758b..46e7277e 100644 --- a/ssh/source/ssh/sftp_session.cpp +++ b/ssh/source/ssh/sftp_session.cpp @@ -70,7 +70,7 @@ namespace SecureShell } std::expected, SftpSession::Error> - SftpSession::listDirectoryInStrand(Paths::RemotePath const& path) + SftpSession::listDirectoryInStrand(Paths::RemotePath const& path, bool resolveLinks) { if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); @@ -117,7 +117,7 @@ namespace SecureShell for (auto& entry : entries) { - if (entry.type == SharedData::FileType::Symlink) + if (resolveLinks && entry.type == SharedData::FileType::Symlink) { // A name that is no valid segment, like "x/..", must not become a path to look up. const auto name = entry.nameSegment(); @@ -147,12 +147,12 @@ namespace SecureShell } std::future, SftpSession::Error>> - SftpSession::listDirectory(Paths::RemotePath const& path) + SftpSession::listDirectory(Paths::RemotePath const& path, bool resolveLinks) { return performPromise( - [this, path]() + [this, path, resolveLinks]() { - return listDirectoryInStrand(path); + return listDirectoryInStrand(path, resolveLinks); } ); } From 28c70d6bab71298afe8b66ddb9ea507ec2fdd849 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:51:24 +0200 Subject: [PATCH 050/131] Sync scans exclude the asides of replacing moves Sync scans also exclude the asides a replacing move leaves under the temp suffix plus .old, like the temp files themselves --- .../source/backend/sftp/operation_queue.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 6ba53e87..2606b899 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -2164,14 +2164,19 @@ void OperationQueue::addSyncScanOperation( ); // Queue remote scan — build-tree mode. - // Temp files of transfers on either side, a sync neither copies nor deletes them. + // Temp files of transfers on either side and the asides of replaced destinations, a sync neither copies nor + // deletes them. + const auto uploadSuffix = Persistence::effectiveTempFileSuffix( + sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}).tempFileSuffix + ); + const auto downloadSuffix = Persistence::effectiveTempFileSuffix( + sftpOpts_.downloadOptions.value_or(Persistence::DownloadOptions{}).tempFileSuffix + ); std::vector temporarySuffixes{ - Persistence::effectiveTempFileSuffix( - sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}).tempFileSuffix - ), - Persistence::effectiveTempFileSuffix( - sftpOpts_.downloadOptions.value_or(Persistence::DownloadOptions{}).tempFileSuffix - ), + uploadSuffix, + downloadSuffix, + Persistence::asideFileSuffix(uploadSuffix), + Persistence::asideFileSuffix(downloadSuffix), }; auto remoteScan = std::make_unique( From 1a2bec1bca88db7093aa1d291526e4fcc9cf0322 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:53:20 +0200 Subject: [PATCH 051/131] A directory upload never creates an unreadable subdirectory empty A directory upload no longer creates an unreadable subdirectory empty on the server: its entry keeps an unknown type that no consumer creates, next to the failed entry that reports it; on Windows the links and junctions a directory upload leaves out are failed entries of its card instead of a silent skip --- .../backend/sftp/bulk_transfer_operation.hpp | 7 ++++ .../backend/sftp/local_scan_operation.hpp | 28 ++++++++++++--- .../backend/sftp/bulk_transfer_operation.cpp | 15 ++++++++ .../backend/sftp/local_scan_operation.cpp | 34 +++++++++++-------- 4 files changed, 65 insertions(+), 19 deletions(-) diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index d3c49ea4..1d978199 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -114,6 +114,8 @@ class BulkTransferOperation : public Operation addRejectedEntries(finished.rejectedPaths()); if constexpr (requires { finished.unreadableDirectories(); }) addUnreadableDirectories(finished.unreadableDirectories()); + if constexpr (requires { finished.skippedLinks(); }) + addSkippedLinks(finished.skippedLinks()); setScanResult(finished.ejectEntries(), scannedBytes); } ); @@ -131,6 +133,11 @@ class BulkTransferOperation : public Operation */ void addRejectedEntries(std::vector const& paths); + /** + * @brief Records links the scan left out as failed, they are not transferred. + */ + void addSkippedLinks(std::vector const& paths); + /** * @brief Records directories the scan could not list as failed, their content is not transferred. */ diff --git a/backend/include/backend/sftp/local_scan_operation.hpp b/backend/include/backend/sftp/local_scan_operation.hpp index cf4d8da8..f96f1432 100644 --- a/backend/include/backend/sftp/local_scan_operation.hpp +++ b/backend/include/backend/sftp/local_scan_operation.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -116,16 +117,19 @@ class LocalScanOperation : public Operation /** * @brief Eject the scanned directory entries. Careful!: The internal list is moved out. - * Only valid when @ref buildsTree() is false. + * Only valid when @ref buildsTree() is false. A directory that could not be listed is kept as an entry + * of unknown type, which no consumer creates, see @ref unreadableDirectories. */ std::vector ejectEntries() { - return withWalkerDo( + auto entries = withWalkerDo( [](auto& walker) { return std::move(walker).ejectEntries(); } ); + markUnreadable(entries); + return entries; } /** @@ -149,14 +153,22 @@ class LocalScanOperation : public Operation } /** - * @brief Full paths of directories below the root that could not be listed. Flat scans only, they are left out - * of the entries; the tree of a sync scan keeps them as unreadable nodes. + * @brief Full paths of directories below the root that could not be listed. Flat scans only, their entries are + * never created; the tree of a sync scan keeps them as unreadable nodes. */ std::vector const& unreadableDirectories() const noexcept { return unreadableDirectories_; } + /** + * @brief Full paths of links and junctions a flat scan on Windows leaves out, they are not uploaded from there. + */ + std::vector const& skippedLinks() const noexcept + { + return skippedLinks_; + } + /** * @brief Eject the scanned node tree. Only valid when @ref buildsTree() is true. */ @@ -193,8 +205,16 @@ class LocalScanOperation : public Operation std::error_code const& errorCode ); + /** + * @brief Gives the entries of @ref unreadableDirectories an unknown type. + */ + void markUnreadable(std::vector& entries) const; + private: std::vector unreadableDirectories_{}; + /** @brief The unreadable directories below the root, as relative paths. */ + std::unordered_set unreadableRelatives_{}; + std::vector skippedLinks_{}; Paths::LocalPath localPath_; std::function progressCallback_; diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 47315a80..542de75a 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -492,6 +492,21 @@ void BulkTransferOperation::addRejectedEntries(std::vector const& p } } +void BulkTransferOperation::addSkippedLinks(std::vector const& paths) +{ + for (auto const& path : paths) + { + Log::warn("{}: Skipping the link '{}', links are not transferred from here.", logName(), path); + failedEntries_.emplace_back( + path, + Error{ + .type = ErrorType::OperationNotPossibleOnFileType, + .extraInfo = "Links and junctions are not transferred" + } + ); + } +} + void BulkTransferOperation::addUnreadableDirectories(std::vector const& paths) { for (auto const& path : paths) diff --git a/backend/source/backend/sftp/local_scan_operation.cpp b/backend/source/backend/sftp/local_scan_operation.cpp index 40c41f43..94e4d031 100644 --- a/backend/source/backend/sftp/local_scan_operation.cpp +++ b/backend/source/backend/sftp/local_scan_operation.cpp @@ -102,20 +102,9 @@ LocalScanOperation::scanner(Paths::RelativePath const& relative) if (isSymlink) { #ifdef _WIN32 - // Windows symlinks and junctions are not transferred. A sync tree keeps them as excluded - // placeholders, so the name still counts as present and the remote side is left alone. - if (!buildTree_) - continue; - type = SharedData::DirectoryEntry::FileType::Unknown; -#else - type = SharedData::DirectoryEntry::FileType::Symlink; -#endif - } - else if (std::filesystem::is_directory(status)) - type = SharedData::DirectoryEntry::FileType::Directory; - else if (std::filesystem::is_regular_file(status)) - type = SharedData::DirectoryEntry::FileType::Regular; - else + // Windows symlinks and junctions are not transferred. A sync tree keeps them as excluded + // placeholders, so the name still counts as present and the remote side is left alone. + if (!buildTree_) { // Sockets, devices and FIFOs are not uploadable, a sync tree keeps them as excluded placeholders. if (!buildTree_) @@ -186,8 +175,9 @@ LocalScanOperation::listingFailed( ); if (!buildTree_) { - // Left out of the entries, the consumer reports it instead of transferring an empty directory. + // The consumer reports it instead of transferring an empty directory. unreadableDirectories_.push_back(Paths::nativeToUtf8(path)); + unreadableRelatives_.insert(relative.value()); return std::vector{}; } return std::unexpected( @@ -205,6 +195,20 @@ LocalScanOperation::listingFailed( ); } +void LocalScanOperation::markUnreadable(std::vector& entries) const +{ + if (unreadableRelatives_.empty()) + return; + for (auto& entry : entries) + { + if (!entry.isDirectory()) + continue; + if (const auto relative = SharedData::relativePath(entries, entry); + relative && unreadableRelatives_.contains(relative->value())) + entry.type = SharedData::FileType::Unknown; + } +} + std::uint64_t LocalScanOperation::totalBytes() const { return walker_->totalBytes(); From 406bc0922164208773920f301c7fcdd81cad413b Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:54:42 +0200 Subject: [PATCH 052/131] A dropped link or junction is described as the link itself A dropped link or junction is described as the link itself instead of what it points to, like explorer uploads see it, and a drop that is not an absolute path or cannot be inspected is left out with a log line --- backend/include/backend/dropped_entry.hpp | 6 +- backend/source/backend/dropped_entry.cpp | 72 +++++++++++++++-------- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/backend/include/backend/dropped_entry.hpp b/backend/include/backend/dropped_entry.hpp index 20dc1653..cc60630b 100644 --- a/backend/include/backend/dropped_entry.hpp +++ b/backend/include/backend/dropped_entry.hpp @@ -6,7 +6,9 @@ #include /** - * @brief Describes a file dropped onto the window from the desktop, with just enough detail for the transfer. - * @param path Local path of the dropped file or directory, nothing when it is not an absolute path. + * @brief Describes a file dropped onto the window from the desktop, with just enough detail for the transfer. A + * dropped link or junction is a symlink, never what it points to. + * @param path Local path of the dropped item. Nothing, and a log line, when it is not an absolute path or cannot be + * inspected. */ std::optional droppedItemFromPath(std::filesystem::path const& path); diff --git a/backend/source/backend/dropped_entry.cpp b/backend/source/backend/dropped_entry.cpp index 1e76b3e7..f15b97c1 100644 --- a/backend/source/backend/dropped_entry.cpp +++ b/backend/source/backend/dropped_entry.cpp @@ -1,36 +1,62 @@ #include +#include +#include +#include +#include -std::optional droppedItemFromPath(std::filesystem::path const& path) +#include + +namespace { - using FileType = SharedData::DirectoryEntry::FileType; + /** + * @brief The type of what lies at @p path itself: a link or junction is a symlink, never what it points to, + * like every upload from the explorer sees it. + */ + SharedData::FileType droppedTypeOf(std::filesystem::path const& path, std::filesystem::file_status const& status) + { + using FileType = SharedData::FileType; + std::error_code linkError{}; + if (std::filesystem::is_symlink(status) || Utility::isLinkOrJunction(path, linkError)) + return FileType::Symlink; + if (std::filesystem::is_directory(status)) + return FileType::Directory; + if (std::filesystem::is_regular_file(status)) + return FileType::Regular; + if (std::filesystem::is_socket(status)) + return FileType::Socket; + if (std::filesystem::is_character_file(status)) + return FileType::CharDevice; + if (std::filesystem::is_block_file(status)) + return FileType::BlockDevice; + if (std::filesystem::is_other(status)) + return FileType::Special; + return FileType::Unknown; + } +} +std::optional droppedItemFromPath(std::filesystem::path const& path) +{ auto localPath = Paths::LocalPath::make(path); if (!localPath) + { + Log::warn("Drop: '{}' is ignored: {}", Paths::nativeToUtf8(path), localPath.error().toString()); + return std::nullopt; + } + std::error_code statusError{}; + const auto status = std::filesystem::symlink_status(path, statusError); + if (status.type() == std::filesystem::file_type::not_found || statusError) + { + Log::warn( + "Drop: '{}' is ignored, it cannot be inspected: {}", + localPath->toUtf8(), + statusError ? Utility::errorMessage(statusError) : std::string{"not found"} + ); return std::nullopt; + } const auto name = localPath->filename(); auto entry = SharedData::DirectoryEntry{ .name = name ? name->value() : std::string{}, - .type = [&path]() - { - std::error_code statusError; - const auto status = std::filesystem::status(path, statusError); - if (std::filesystem::is_directory(status)) - return FileType::Directory; - else if (std::filesystem::is_regular_file(status)) - return FileType::Regular; - else if (std::filesystem::is_symlink(status)) - return FileType::Symlink; - else if (std::filesystem::is_socket(status)) - return FileType::Socket; - else if (std::filesystem::is_character_file(status)) - return FileType::CharDevice; - else if (std::filesystem::is_block_file(status)) - return FileType::BlockDevice; - else if (std::filesystem::is_other(status)) - return FileType::Special; - else - return FileType::Unknown; - }() + .type = droppedTypeOf(path, status), }; return SharedData::DroppedItem{.path = *std::move(localPath), .entry = std::move(entry)}; } From 0186411268741998f66553835c06ed80ac86eeb0 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:56:24 +0200 Subject: [PATCH 053/131] EnqueuePlanEntry is a plain struct; stale comments are corrected EnqueuePlanEntry is a plain struct with a doc that says what a plan lists, without the unused JSON and frontend conversions; SyncSession plans only through resolvePlanInStrand; the bulk delete card is labelled with the directory all its items lie in; stale comments of the bulk delete and the sync selection RPC describe what they do --- backend/include/backend/sync/sync_session.hpp | 14 ++----- backend/source/backend/session.cpp | 3 +- .../source/backend/sftp/operation_queue.cpp | 4 +- .../file_explorer/remote_side_model.cpp | 6 +-- .../shared_data/sync/enqueue_plan_entry.hpp | 39 +++---------------- 5 files changed, 14 insertions(+), 52 deletions(-) diff --git a/backend/include/backend/sync/sync_session.hpp b/backend/include/backend/sync/sync_session.hpp index f93b039c..23305270 100644 --- a/backend/include/backend/sync/sync_session.hpp +++ b/backend/include/backend/sync/sync_session.hpp @@ -160,16 +160,6 @@ class SyncSession : public std::enable_shared_from_this std::string const& parentRelKey ); - /** - * @brief Computes the enqueue plan from a SPARSE selection set: an entry X in - * @p selectedRelKeys means "X and every descendant is selected". See - * @ref SharedData::Sync::DiffTreeStore::buildPlan. Precondition: running on @ref strand(). - */ - std::vector buildEnqueuePlanInStrand( - SharedData::Sync::DiffSection section, - std::unordered_set const& selectedRelKeys - ) const; - struct ResolvedPlanEntry { SharedData::Sync::EnqueuePlanEntry entry; @@ -194,6 +184,10 @@ class SyncSession : public std::enable_shared_from_this ) const; private: + std::vector buildEnqueuePlanInStrand( + SharedData::Sync::DiffSection section, + std::unordered_set const& selectedRelKeys + ) const; SharedData::Sync::DiffTreeStore& storeFor(SharedData::Sync::DiffSection section); SharedData::Sync::DiffTreeStore const& storeFor(SharedData::Sync::DiffSection section) const; SharedData::Sync::ScanTrees scanTrees() const; diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index 861d2446..48a956a0 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -1617,8 +1617,7 @@ void Session::registerRpcSftpLoadSyncDiffChildren() void Session::registerRpcSftpExecuteSyncSelection() { - // The backend plans the selection and resolves every row below its own sync roots, so no absolute path - // round-trips through the frontend. + // Plans the selection and resolves every row below the sync roots of the session. on(fmt::format("Session::{}::sftp::executeSyncSelection", id_.value())) .perform( [weak = weak_from_this()]( diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 2606b899..7f6bfac4 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -2060,8 +2060,8 @@ std::size_t OperationQueue::addBulkDeleteOperation( if (failedTargets > 0) Log::warn("Bulk delete: {} items were refused, they are reported as failed", failedTargets); - // Parent of the first item, so the card can read "Deleting items in " before any progress arrives. - const auto commonParent = request.entries.front().target.parent().value_or(Paths::RemotePath::root()); + // The card reads "Deleting items in " before any progress arrives. + const auto commonParent = commonParentOf(request.entries, &SharedData::BulkDeleteEntry::target); const auto fileCount = fileTargets.size(); operation->setTargets(std::move(fileTargets)); targetQueue.emplace_back(bulkOperationId, std::move(operation)); diff --git a/frontend/source/frontend/file_explorer/remote_side_model.cpp b/frontend/source/frontend/file_explorer/remote_side_model.cpp index df7da412..0cbaa2cc 100644 --- a/frontend/source/frontend/file_explorer/remote_side_model.cpp +++ b/frontend/source/frontend/file_explorer/remote_side_model.cpp @@ -514,10 +514,8 @@ void RemoteSideModel::enqueueDeletes( std::vector filesAndEmptyDirs ) { - // Combine into a single bulk-delete request. File / empty-dir entries - // become one bulk-delete card; non-empty directories become individual - // Scan+Delete pairs (their isDirectory=true triggers the recursive flow - // backend-side). One RPC for the whole user action. + // One bulk-delete request and card for the whole user action. Non-empty directories are scanned and deleted + // with their content by the backend (isDirectory), files and empty directories are deleted as they are. std::vector entries; entries.reserve(filesAndEmptyDirs.size() + nonEmpties.size()); for (auto& path : filesAndEmptyDirs) diff --git a/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp b/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp index 8c14244c..ec94584c 100644 --- a/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp +++ b/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp @@ -1,29 +1,19 @@ #pragma once -#include #include -#include - #include #include namespace SharedData::Sync { /** - * @brief One row of the backend-computed enqueue plan. - * - * The backend collapses the frontend's selected relKey set into a minimal list - * that: + * @brief One entry of the plan @ref DiffTreeStore::buildPlan makes from a sparse selection. * - * - emits a one-sided directory as a single bulk-dir entry when every descendant - * is selected (saves N transfers for N descendants), - * - emits an individual leaf otherwise, - * - skips intermediate directory rows with partial selection — the enqueue - * side creates missing directories as needed via the bulk upload/download - * flow. - * - * Entries carry no paths: the backend resolves @ref relKey below its own sync roots. + * A transfer lists the files and symlinks of a selected directory, plus every directory that has no + * transferred content and so has to be created by itself; the files create their parents. A delete lists a + * directory before its listed content. Entries name rows by relKey, the backend resolves them below its own + * sync roots. */ struct EnqueuePlanEntry { @@ -34,23 +24,4 @@ namespace SharedData::Sync std::uint32_t mtimeNsec{0}; bool isDirectory{false}; }; - BOOST_DESCRIBE_STRUCT( - EnqueuePlanEntry, - (), - (relKey, action, sizeBytes, mtime, mtimeNsec, isDirectory) - ) - - inline void to_json(nlohmann::json& j, EnqueuePlanEntry const& e) - { - SharedData::to_json(j, e); - } - inline void from_json(nlohmann::json const& j, EnqueuePlanEntry& e) - { - SharedData::from_json(j, e); - } - -#ifdef NUI_FRONTEND - // No to_val/from_val forwarder for EnqueuePlanEntry — described-struct path - // in nui's convertToVal handles it natively. -#endif } From 3463ba910c8997b039a055bd13ba2f36c6d7235f Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:59:06 +0200 Subject: [PATCH 054/131] Sync baseline files carry a hash of their session in the name Sync baseline files carry a hash of their session in the name, and once the state loaded the baselines of sessions that no longer exist (deleted or renamed) are removed --- backend/source/backend/main.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index d4f37272..d5bbb384 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,7 @@ #include #include #include +#include #include #ifndef _WIN32 @@ -321,6 +323,13 @@ void Main::registerRpc() } sshSessionManager_->preventLocalDeletion(holder.stateCache().localFilesystemOptions.preventDeletion); + { + auto const& sessions = holder.stateCache().sessions; + std::vector sessionNames; + sessionNames.reserve(sessions.size()); + std::ranges::copy(std::views::keys(sessions), std::back_inserter(sessionNames)); + SyncBaselineStore{SyncBaselineStore::defaultDirectory()}.pruneAllBut(sessionNames); + } rpcFilesystem_ = std::make_unique( window_.getExecutor(), window_, From b315b911baa7371276c4fdcdcc9aa136f93bc2d9 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:04:30 +0200 Subject: [PATCH 055/131] Format the sync, existence check and bulk changes with clang-format The code of the sync baseline, existence check, bulk and sync dialog changes is formatted with the project's clang-format, and a doc comment of the sync dialog names what it registers --- frontend/source/frontend/sync_dialog/backend_sync_provider.cpp | 3 ++- shared_data/source/shared_data/sync/diff.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp index 7c40cf02..c19e75cc 100644 --- a/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp +++ b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp @@ -161,7 +161,8 @@ void BackendSyncProvider::executeSelection( request.generation = generation_; queue_->executeSyncSelection( std::move(request), - [alive = std::weak_ptr{alive_}, onResolved = std::move(onResolved)](SharedData::Sync::SyncExecutionResult result) + [alive = std::weak_ptr{alive_}, + onResolved = std::move(onResolved)](SharedData::Sync::SyncExecutionResult result) { // The operations are queued either way, the reply only maps them to rows of a dialog that may be gone. if (alive.lock()) diff --git a/shared_data/source/shared_data/sync/diff.cpp b/shared_data/source/shared_data/sync/diff.cpp index a17c0f9f..eb5f26b2 100644 --- a/shared_data/source/shared_data/sync/diff.cpp +++ b/shared_data/source/shared_data/sync/diff.cpp @@ -303,7 +303,8 @@ namespace SharedData::Sync return true; }; - const auto bumpAndCheckpoint = [&]() { + const auto bumpAndCheckpoint = [&]() + { ++walk.compared; if (walk.compared % progressCheckpointInterval == 0) sink.onProgress(walk.compared); From bb6bf2674c9c605af67b657b14e9904b565a44e0 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:16:52 +0200 Subject: [PATCH 056/131] Operations hold their SFTP session weakly and lock it per work step Operations hold their SFTP session weakly and lock it for each work step, so a channel closed while operations are queued or running ends them with the new ChannelClosed error instead of leaving them a dangling session; the in-strand steps get the locked session passed in, posted tasks lock it themselves, and a remote scan waits for a listing once instead of twice --- .../sftp/archive_download_operation.hpp | 10 +- .../backend/sftp/archive_upload_operation.hpp | 10 +- .../backend/sftp/bulk_download_operation.hpp | 8 +- .../backend/sftp/bulk_transfer_operation.hpp | 23 ++- .../backend/sftp/bulk_upload_operation.hpp | 15 +- .../include/backend/sftp/delete_operation.hpp | 5 +- .../backend/sftp/download_operation.hpp | 12 +- backend/include/backend/sftp/operation.hpp | 43 ++++- .../include/backend/sftp/rename_operation.hpp | 5 +- .../include/backend/sftp/scan_operation.hpp | 5 +- .../include/backend/sftp/upload_operation.hpp | 17 +- .../sftp/archive_download_operation.cpp | 34 ++-- .../backend/sftp/archive_upload_operation.cpp | 41 +++-- .../backend/sftp/bulk_download_operation.cpp | 16 +- .../backend/sftp/bulk_transfer_operation.cpp | 41 +++-- .../backend/sftp/bulk_upload_operation.cpp | 23 ++- .../source/backend/sftp/delete_operation.cpp | 39 ++++- .../backend/sftp/download_operation.cpp | 70 +++++--- .../source/backend/sftp/operation_queue.cpp | 163 +++--------------- .../source/backend/sftp/rename_operation.cpp | 12 +- .../source/backend/sftp/scan_operation.cpp | 12 +- .../source/backend/sftp/upload_operation.cpp | 120 ++++++++----- .../file_operations/operation_error_type.hpp | 4 +- 23 files changed, 400 insertions(+), 328 deletions(-) diff --git a/backend/include/backend/sftp/archive_download_operation.hpp b/backend/include/backend/sftp/archive_download_operation.hpp index a7305ed4..6330b666 100644 --- a/backend/include/backend/sftp/archive_download_operation.hpp +++ b/backend/include/backend/sftp/archive_download_operation.hpp @@ -94,7 +94,7 @@ class ArchiveDownloadOperation : public Operation std::chrono::seconds futureTimeout{5}; }; - ArchiveDownloadOperation(SecureShell::SftpSession& sftp, Options options); + ArchiveDownloadOperation(std::weak_ptr sftp, Options options); ~ArchiveDownloadOperation() override; ArchiveDownloadOperation(ArchiveDownloadOperation const&) = delete; ArchiveDownloadOperation& operator=(ArchiveDownloadOperation const&) = delete; @@ -137,7 +137,7 @@ class ArchiveDownloadOperation : public Operation SecureShell::ProcessingStrand* strand() const override { - return sftp_ ? sftp_->strand() : nullptr; + return strandOf(sftp_); } std::expected cancel(bool adoptCancelState) override; @@ -177,7 +177,7 @@ class ArchiveDownloadOperation : public Operation Paths::RemotePath source{}; }; - std::expected prepareInStrand(); + std::expected prepareInStrand(SecureShell::SftpSession& sftp); std::expected flattenRootsInStrand(); /** * @brief Expand a preceding ScanOperation's flat entry vector into the @@ -192,7 +192,7 @@ class ArchiveDownloadOperation : public Operation Paths::RemotePath const& remoteRoot, Paths::RelativePath const& archivePrefix ); - std::expected openNextEntryInStrand(); + std::expected openNextEntryInStrand(SecureShell::SftpSession& sftp); std::expected readChunkInStrand(); std::expected finaliseEntryInStrand(); std::expected finaliseArchiveInStrand(); @@ -201,7 +201,7 @@ class ArchiveDownloadOperation : public Operation void cleanup(); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Options options_; std::vector resolvedEntries_{}; std::optional writer_{std::nullopt}; diff --git a/backend/include/backend/sftp/archive_upload_operation.hpp b/backend/include/backend/sftp/archive_upload_operation.hpp index c0eddf13..6e13ab0d 100644 --- a/backend/include/backend/sftp/archive_upload_operation.hpp +++ b/backend/include/backend/sftp/archive_upload_operation.hpp @@ -75,7 +75,7 @@ class ArchiveUploadOperation : public Operation std::chrono::seconds futureTimeout{5}; }; - ArchiveUploadOperation(SecureShell::SftpSession& sftp, Options options); + ArchiveUploadOperation(std::weak_ptr sftp, Options options); ~ArchiveUploadOperation() override; ArchiveUploadOperation(ArchiveUploadOperation const&) = delete; ArchiveUploadOperation& operator=(ArchiveUploadOperation const&) = delete; @@ -118,7 +118,7 @@ class ArchiveUploadOperation : public Operation SecureShell::ProcessingStrand* strand() const override { - return sftp_ ? sftp_->strand() : nullptr; + return strandOf(sftp_); } std::expected cancel(bool adoptCancelState) override; @@ -153,7 +153,7 @@ class ArchiveUploadOperation : public Operation Paths::LocalPath source{}; }; - std::expected prepareInStrand(); + std::expected prepareInStrand(SecureShell::SftpSession& sftp); std::expected flattenRoots(); /** * @brief Expand a preceding LocalScanOperation's flat entry vector into @@ -170,13 +170,13 @@ class ArchiveUploadOperation : public Operation std::expected openNextEntryInStrand(); std::expected readChunkInStrand(); std::expected finaliseEntryInStrand(); - std::expected finaliseArchiveInStrand(); + std::expected finaliseArchiveInStrand(SecureShell::SftpSession& sftp); bool isPayloadEntry(ResolvedEntry const& entry) const noexcept; void cleanup(); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Options options_; std::vector resolvedEntries_{}; std::optional writer_{std::nullopt}; diff --git a/backend/include/backend/sftp/bulk_download_operation.hpp b/backend/include/backend/sftp/bulk_download_operation.hpp index b8875d0f..580f47ca 100644 --- a/backend/include/backend/sftp/bulk_download_operation.hpp +++ b/backend/include/backend/sftp/bulk_download_operation.hpp @@ -28,7 +28,7 @@ class BulkDownloadOperation : public BulkTransferOperation std::chrono::milliseconds progressEmitInterval{100}; }; - BulkDownloadOperation(SecureShell::SftpSession& sftp, BulkDownloadOperationOptions options); + BulkDownloadOperation(std::weak_ptr sftp, BulkDownloadOperationOptions options); ~BulkDownloadOperation() override; BulkDownloadOperation(BulkDownloadOperation const&) = delete; BulkDownloadOperation(BulkDownloadOperation&&) = delete; @@ -57,8 +57,10 @@ class BulkDownloadOperation : public BulkTransferOperation void setPrescannedFileList(std::vector files); protected: - std::expected prepareRootInStrand(SharedData::DirectoryEntry const& root) override; - std::expected createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) override; + std::expected + prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) override; + std::expected + createDirectoryInStrand(SecureShell::SftpSession& sftp, std::size_t entryIndex, EntryPaths const& paths) override; std::unique_ptr makeChild( std::size_t entryIndex, EntryPaths const& paths, diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index 1d978199..f13d8bab 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -172,7 +172,7 @@ class BulkTransferOperation : public Operation } protected: - BulkTransferOperation(SecureShell::SftpSession& sftp, CommonOptions common); + BulkTransferOperation(std::weak_ptr sftp, CommonOptions common); /** * @brief Flat-list mode: every entry is a file or a directory created without content, @p paths holds both of @@ -215,9 +215,12 @@ class BulkTransferOperation : public Operation return entries_; } - SecureShell::SftpSession& sftp() const noexcept + /** + * @brief The session the children are given, held weakly like the bulk's own. + */ + std::weak_ptr const& sftp() const noexcept { - return *sftp_; + return sftp_; } CommonOptions const& common() const noexcept @@ -228,12 +231,14 @@ class BulkTransferOperation : public Operation /** * @brief Tree mode only: creates the bulk's root directory from entries()[0]. */ - virtual std::expected prepareRootInStrand(SharedData::DirectoryEntry const& root) = 0; + virtual std::expected + prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) = 0; /** * @brief Creates the directory described by the entry at @p entryIndex. */ - virtual std::expected createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) = 0; + virtual std::expected + createDirectoryInStrand(SecureShell::SftpSession& sftp, std::size_t entryIndex, EntryPaths const& paths) = 0; /** * @brief Builds the child transfer for the file at @p entryIndex. @@ -288,9 +293,9 @@ class BulkTransferOperation : public Operation bool finished{false}; }; - std::expected start(); - std::expected step(); - std::expected fillSlots(); + std::expected start(SecureShell::SftpSession& sftp); + std::expected step(SecureShell::SftpSession& sftp); + std::expected fillSlots(SecureShell::SftpSession& sftp); std::expected stepSlots(); void finishSlot(Slot& slot); void cancelChildrenInStrand(bool adoptCancelState); @@ -320,7 +325,7 @@ class BulkTransferOperation : public Operation Flat, }; - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; CommonOptions common_; std::unique_ptr scanPhase_{}; EntrySource entrySource_{EntrySource::None}; diff --git a/backend/include/backend/sftp/bulk_upload_operation.hpp b/backend/include/backend/sftp/bulk_upload_operation.hpp index fbbe41dc..848d0921 100644 --- a/backend/include/backend/sftp/bulk_upload_operation.hpp +++ b/backend/include/backend/sftp/bulk_upload_operation.hpp @@ -24,7 +24,7 @@ class BulkUploadOperation : public BulkTransferOperation std::chrono::milliseconds progressEmitInterval{100}; }; - BulkUploadOperation(SecureShell::SftpSession& sftp, BulkUploadOperationOptions options); + BulkUploadOperation(std::weak_ptr sftp, BulkUploadOperationOptions options); ~BulkUploadOperation() override; BulkUploadOperation(BulkUploadOperation const&) = delete; BulkUploadOperation(BulkUploadOperation&&) = delete; @@ -46,8 +46,10 @@ class BulkUploadOperation : public BulkTransferOperation void setPrescannedFileList(std::vector files); protected: - std::expected prepareRootInStrand(SharedData::DirectoryEntry const& root) override; - std::expected createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) override; + std::expected + prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) override; + std::expected + createDirectoryInStrand(SecureShell::SftpSession& sftp, std::size_t entryIndex, EntryPaths const& paths) override; std::unique_ptr makeChild( std::size_t entryIndex, EntryPaths const& paths, @@ -71,7 +73,12 @@ class BulkUploadOperation : public BulkTransferOperation * @brief Creates the missing directories between the bulk's remote root and @p path, not @p path itself. */ std::expected - createRemoteDirectoryInStrand(Paths::RemotePath const& path, SharedData::DirectoryEntry const& entry); + createMissingParentsInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& path); + std::expected createRemoteDirectoryInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& path, + SharedData::DirectoryEntry const& entry + ); private: UploadOperation::UploadOperationOptions individualOptions_; diff --git a/backend/include/backend/sftp/delete_operation.hpp b/backend/include/backend/sftp/delete_operation.hpp index 9770c7ec..bc835020 100644 --- a/backend/include/backend/sftp/delete_operation.hpp +++ b/backend/include/backend/sftp/delete_operation.hpp @@ -9,6 +9,7 @@ #include #include +#include #include class DeleteOperation : public Operation @@ -33,7 +34,7 @@ class DeleteOperation : public Operation return false; } - DeleteOperation(SecureShell::SftpSession& sftp, DeleteOperationOptions options); + DeleteOperation(std::weak_ptr sftp, DeleteOperationOptions options); ~DeleteOperation() override; DeleteOperation(DeleteOperation const&) = delete; DeleteOperation(DeleteOperation&&) = delete; @@ -115,7 +116,7 @@ class DeleteOperation : public Operation std::expected removeOnce(Paths::RemotePath const& path); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Paths::RemotePath remotePath_; std::function filesRemovedProgress_; std::chrono::seconds futureTimeout_; diff --git a/backend/include/backend/sftp/download_operation.hpp b/backend/include/backend/sftp/download_operation.hpp index c9da4f95..5c4fe2d6 100644 --- a/backend/include/backend/sftp/download_operation.hpp +++ b/backend/include/backend/sftp/download_operation.hpp @@ -61,10 +61,10 @@ class DownloadOperation : public Operation { if (auto stream = fileStream_.lock(); stream) return stream->strand(); - return sftp_ ? sftp_->strand() : nullptr; + return strandOf(sftp_); } - DownloadOperation(SecureShell::SftpSession& sftp, DownloadOperationOptions options); + DownloadOperation(std::weak_ptr sftp, DownloadOperationOptions options); ~DownloadOperation() override; DownloadOperation(DownloadOperation const&) = delete; DownloadOperation(DownloadOperation&&) = delete; @@ -126,12 +126,12 @@ class DownloadOperation : public Operation std::expected readOnceInStrand(); std::expected handleSymlink(); - std::expected handleSymlinkInStrand(); + std::expected handleSymlinkInStrand(SecureShell::SftpSession& sftp); bool commitBufferToFile(SecureShell::IFileStream::SignedSizeType bytesRead); /// Runs the remote-heavy portion of prepare() under a single strand umbrella. - std::expected prepareInStrand(); + std::expected prepareInStrand(SecureShell::SftpSession& sftp); std::expected openOrAdoptFileInStrand(SecureShell::IFileStream& stream); @@ -142,7 +142,7 @@ class DownloadOperation : public Operation std::expected refuseLinkedPath(std::filesystem::path const& path) const; std::expected - readSymlinkInStrand(Paths::RemotePath const& remoteFullPath); + readSymlinkInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& remoteFullPath); /** * @brief The error of a file that ended before its listed size arrived. @@ -167,7 +167,7 @@ class DownloadOperation : public Operation void cleanup(); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; std::weak_ptr fileStream_; DownloadOperationOptions options_; /** @brief options_.remotePath, unset when none was given. */ diff --git a/backend/include/backend/sftp/operation.hpp b/backend/include/backend/sftp/operation.hpp index 373aedb3..73f2c1a9 100644 --- a/backend/include/backend/sftp/operation.hpp +++ b/backend/include/backend/sftp/operation.hpp @@ -8,14 +8,16 @@ #include #include #include +#include #include #include #include -#include #include +#include +#include #include class Operation @@ -168,6 +170,45 @@ class Operation state_ = newState; } + /** + * @brief The operation completed, failed or was canceled. + */ + bool hasEnded() const noexcept + { + return state_ >= OperationState::Completed; + } + + /** + * @brief The session behind @p session for one work step, none once its channel was closed or is closing. + * Operations hold their session weakly and lock it per step, so a closed channel ends them instead of + * leaving them a dangling session. + */ + static std::shared_ptr openSession(std::weak_ptr const& session) + { + auto locked = session.lock(); + if (!locked || !locked->strand() || locked->strand()->isFinalized()) + return nullptr; + return locked; + } + + /** + * @brief The processing strand of @p session, none once its channel was closed or is closing. It stays valid + * until the channel is closed, which happens on the queue's strand. + */ + static SecureShell::ProcessingStrand* strandOf(std::weak_ptr const& session) + { + const auto locked = openSession(session); + return locked ? locked->strand() : nullptr; + } + + /** + * @brief The error of an operation whose SFTP channel was closed before it ended. + */ + static Error channelClosedError() + { + return Error{.type = ErrorType::ChannelClosed, .extraInfo = "The SFTP channel was closed"}; + } + /** * @brief Waits until every task queued on @p strand so far has run, so none of them outlives the operation that * queued it. Returns at once on the processing thread itself and on a finalized strand. diff --git a/backend/include/backend/sftp/rename_operation.hpp b/backend/include/backend/sftp/rename_operation.hpp index d573c0fe..91c4a3dd 100644 --- a/backend/include/backend/sftp/rename_operation.hpp +++ b/backend/include/backend/sftp/rename_operation.hpp @@ -6,6 +6,7 @@ #include #include +#include class RenameOperation : public Operation { @@ -17,7 +18,7 @@ class RenameOperation : public Operation std::chrono::seconds futureTimeout{5}; }; - RenameOperation(SecureShell::SftpSession& sftp, RenameOperationOptions options); + RenameOperation(std::weak_ptr sftp, RenameOperationOptions options); ~RenameOperation() override = default; RenameOperation(RenameOperation const&) = delete; @@ -63,7 +64,7 @@ class RenameOperation : public Operation } private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Paths::RemotePath sourcePath_; Paths::RemotePath destinationPath_; std::chrono::seconds futureTimeout_; diff --git a/backend/include/backend/sftp/scan_operation.hpp b/backend/include/backend/sftp/scan_operation.hpp index 2d85228a..721a0ab3 100644 --- a/backend/include/backend/sftp/scan_operation.hpp +++ b/backend/include/backend/sftp/scan_operation.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -46,7 +47,7 @@ class ScanOperation : public Operation return false; } - ScanOperation(SecureShell::SftpSession& sftp, ScanOperationOptions options); + ScanOperation(std::weak_ptr sftp, ScanOperationOptions options); ~ScanOperation() override; ScanOperation(ScanOperation const&) = delete; ScanOperation(ScanOperation&&) = delete; @@ -172,7 +173,7 @@ class ScanOperation : public Operation std::expected, Error> scanner(Paths::RelativePath const& relative); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Paths::RemotePath remotePath_; std::function progressCallback_; diff --git a/backend/include/backend/sftp/upload_operation.hpp b/backend/include/backend/sftp/upload_operation.hpp index 87ccee97..ceebb9e1 100644 --- a/backend/include/backend/sftp/upload_operation.hpp +++ b/backend/include/backend/sftp/upload_operation.hpp @@ -46,7 +46,7 @@ class UploadOperation : public Operation SecureShell::ProcessingStrand* strand() const override; - UploadOperation(SecureShell::SftpSession& sftp, UploadOperationOptions options); + UploadOperation(std::weak_ptr sftp, UploadOperationOptions options); ~UploadOperation() override; UploadOperation(UploadOperation const&) = delete; UploadOperation(UploadOperation&&) = delete; @@ -107,20 +107,20 @@ class UploadOperation : public Operation SecureShell::IFileStream::SignedSizeType commitFileToBuffer(SecureShell::IFileStream::SignedSizeType bytes); std::expected handleSymlink(); - std::expected handleSymlinkInStrand(); + std::expected handleSymlinkInStrand(SecureShell::SftpSession& sftp); - std::expected openOrAdoptFileInStrand(); + std::expected openOrAdoptFileInStrand(SecureShell::SftpSession& sftp); /// Remote-heavy portion of prepare() — runs under a single strand umbrella. - std::expected prepareInStrand(); + std::expected prepareInStrand(SecureShell::SftpSession& sftp); /// Remote-heavy portion of finalize() — runs under a single strand umbrella. - std::expected finalizeInStrand(); + std::expected finalizeInStrand(SecureShell::SftpSession& sftp); /** * @brief Whether the existing remote temp file of @p tempSize bytes ends like the same range of the source. */ - bool tempBelongsToSourceInStrand(std::uint64_t tempSize); + bool tempBelongsToSourceInStrand(SecureShell::SftpSession& sftp, std::uint64_t tempSize); /** @brief Ensures every ancestor directory of @p dir exists on the remote, creating * any missing ones. Idempotent: existing directories are left alone. @@ -129,7 +129,8 @@ class UploadOperation : public Operation * the parent, so deeply-nested missing paths are created in one pass. * Must be called from within the processing thread. */ - std::expected ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& dir); + std::expected + ensureRemoteDirectoryExistsInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& dir); std::filesystem::perms determinePerms(std::filesystem::perms localPerms) const; @@ -141,7 +142,7 @@ class UploadOperation : public Operation void cleanup(); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; std::weak_ptr fileStream_; UploadOperationOptions options_; /** @brief options_.remotePath, unset when none was given. */ diff --git a/backend/source/backend/sftp/archive_download_operation.cpp b/backend/source/backend/sftp/archive_download_operation.cpp index 43a8f33e..4c1cde76 100644 --- a/backend/source/backend/sftp/archive_download_operation.cpp +++ b/backend/source/backend/sftp/archive_download_operation.cpp @@ -119,9 +119,9 @@ bool ArchiveDownloadOperation::isPayloadEntry(ResolvedEntry const& entry) const return isRegularFile(entry.header); } -ArchiveDownloadOperation::ArchiveDownloadOperation(SecureShell::SftpSession& sftp, Options options) +ArchiveDownloadOperation::ArchiveDownloadOperation(std::weak_ptr sftp, Options options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , options_{std::move(options)} { options_.tempFileSuffix = Persistence::effectiveTempFileSuffix(options_.tempFileSuffix); @@ -150,7 +150,15 @@ std::expected ArchiveDownloadOperation: return result; } - auto future = sftp_->performPromise([this]() { return workInStrand(); }); + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto future = sftp->performPromise( + [this]() + { + return workInStrand(); + } + ); if (future.wait_for(options_.futureTimeout) != std::future_status::ready) { Log::error("ArchiveDownloadOperation: work umbrella timed out."); @@ -172,6 +180,10 @@ ArchiveDownloadOperation::workInStrand() ); } + const auto sftp = openSession(sftp_); + if (!sftp && !hasEnded()) + return enterErrorState(channelClosedError()); + switch (state_) { case NotStarted: @@ -190,7 +202,7 @@ ArchiveDownloadOperation::workInStrand() if (std::error_code existsError{}; std::filesystem::exists(localArchiveNativePath_, existsError) && !options_.mayOverwrite) return enterErrorState({.type = ErrorType::FileExists}); - const auto prepareResult = prepareInStrand(); + const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) return enterErrorState(prepareResult.error()); state_ = Prepared; @@ -219,7 +231,7 @@ ArchiveDownloadOperation::workInStrand() } else { - const auto openResult = openNextEntryInStrand(); + const auto openResult = openNextEntryInStrand(*sftp); if (!openResult.has_value()) return enterErrorState(openResult.error()); return WorkStatus::MoreWork; @@ -262,7 +274,7 @@ ArchiveDownloadOperation::workInStrand() return enterErrorState({.type = ErrorType::UnknownWorkState}); } -std::expected ArchiveDownloadOperation::prepareInStrand() +std::expected ArchiveDownloadOperation::prepareInStrand(SecureShell::SftpSession& sftp) { if (options_.createMissingDirectories) { @@ -308,7 +320,7 @@ std::expected ArchiveDownloadOperation::prepareInStrand( } writer_ = TarArchive::Writer::makeFromSink(std::move(*sinkOrError)); - buffer_ = sftp_->bufferProvider().lease(totalPayloadBytes_); + buffer_ = sftp.bufferProvider().lease(totalPayloadBytes_); if (buffer_.empty()) { Log::error("ArchiveDownloadOperation: No transfer buffer available from pool."); @@ -429,7 +441,7 @@ void ArchiveDownloadOperation::appendPrescannedInStrand( } } -std::expected ArchiveDownloadOperation::openNextEntryInStrand() +std::expected ArchiveDownloadOperation::openNextEntryInStrand(SecureShell::SftpSession& sftp) { if (!writer_) return std::unexpected(Error{ @@ -460,10 +472,8 @@ std::expected ArchiveDownloadOperation::openNextEntryInS return {}; } - const auto streamOpenResult = sftp_->openFileInStrand( - resolved.source, - SecureShell::SftpSession::OpenType::Read, - std::filesystem::perms::unknown + const auto streamOpenResult = sftp.openFileInStrand( + resolved.source, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown ); if (!streamOpenResult.has_value()) { diff --git a/backend/source/backend/sftp/archive_upload_operation.cpp b/backend/source/backend/sftp/archive_upload_operation.cpp index 285585b1..e2a44437 100644 --- a/backend/source/backend/sftp/archive_upload_operation.cpp +++ b/backend/source/backend/sftp/archive_upload_operation.cpp @@ -131,9 +131,9 @@ bool ArchiveUploadOperation::isPayloadEntry(ResolvedEntry const& entry) const no return entry.header.type == SharedData::FileType::Regular; } -ArchiveUploadOperation::ArchiveUploadOperation(SecureShell::SftpSession& sftp, Options options) +ArchiveUploadOperation::ArchiveUploadOperation(std::weak_ptr sftp, Options options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , options_{std::move(options)} { remoteArchivePath_ = options_.remoteArchivePath; @@ -156,7 +156,15 @@ std::expected ArchiveUploadOperation::w return result; } - auto future = sftp_->performPromise([this]() { return workInStrand(); }); + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto future = sftp->performPromise( + [this]() + { + return workInStrand(); + } + ); if (future.wait_for(options_.futureTimeout) != std::future_status::ready) { Log::error("ArchiveUploadOperation: work umbrella timed out."); @@ -182,6 +190,10 @@ std::expected ArchiveUploadOperation::w return enterErrorState(*incompleteScan_); } + const auto sftp = openSession(sftp_); + if (!sftp && !hasEnded()) + return enterErrorState(channelClosedError()); + switch (state_) { case NotStarted: @@ -198,7 +210,7 @@ std::expected ArchiveUploadOperation::w {.type = ErrorType::InvalidPath, .extraInfo = "archive has no source paths"} ); - const auto prepareResult = prepareInStrand(); + const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) return enterErrorState(prepareResult.error()); state_ = Prepared; @@ -243,7 +255,7 @@ std::expected ArchiveUploadOperation::w } case Finalizing: { - const auto finaliseResult = finaliseArchiveInStrand(); + const auto finaliseResult = finaliseArchiveInStrand(*sftp); if (!finaliseResult.has_value()) return enterErrorState(finaliseResult.error()); state_ = Completed; @@ -389,7 +401,7 @@ void ArchiveUploadOperation::appendPrescannedFromScan( } } -std::expected ArchiveUploadOperation::prepareInStrand() +std::expected ArchiveUploadOperation::prepareInStrand(SecureShell::SftpSession& sftp) { if (remoteArchivePath_.empty() || remoteArchivePath_.isRoot() || tempPath_.empty()) { @@ -412,12 +424,12 @@ std::expected ArchiveUploadOperation::prepareInStrand() }); // Fail early instead of packing everything first, the final rename refuses to replace it anyway. - if (!options_.mayOverwrite && sftp_->lstatInStrand(remoteArchivePath_).has_value()) + if (!options_.mayOverwrite && sftp.lstatInStrand(remoteArchivePath_).has_value()) return std::unexpected(Error{.type = ErrorType::FileExists}); // Written under the temp name, so a failed or cancelled upload never leaves a truncated destination. using OpenType = SecureShell::SftpSession::OpenType; - const auto openResult = sftp_->openFileInStrand( + const auto openResult = sftp.openFileInStrand( tempPath_, OpenType::Write | OpenType::Create | OpenType::Truncate, std::filesystem::perms{0644u} ); if (!openResult.has_value()) @@ -461,7 +473,7 @@ std::expected ArchiveUploadOperation::prepareInStrand() writer_ = TarArchive::Writer::makeFromSink(std::move(*wrappedSink)); - buffer_ = sftp_->bufferProvider().lease(totalPayloadBytes_); + buffer_ = sftp.bufferProvider().lease(totalPayloadBytes_); if (buffer_.empty()) { Log::error("ArchiveUploadOperation: No transfer buffer available from pool."); @@ -635,7 +647,7 @@ std::expected ArchiveUploadOperation::finaliseEntryInStr return {}; } -std::expected ArchiveUploadOperation::finaliseArchiveInStrand() +std::expected ArchiveUploadOperation::finaliseArchiveInStrand(SecureShell::SftpSession& sftp) { if (!writer_) return std::unexpected(Error{ @@ -718,10 +730,13 @@ void ArchiveUploadOperation::cleanup() if (tempCreated_) { tempCreated_ = false; - if (auto* processingStrand = sftp_->strand(); processingStrand && processingStrand->withinProcessingThread()) - std::ignore = sftp_->removeFileInStrand(tempPath_); + const auto sftp = openSession(sftp_); + if (!sftp) + Log::warn("ArchiveUploadOperation: The channel is closed, '{}' stays on the server.", tempPath_.value()); + else if (sftp->strand()->withinProcessingThread()) + std::ignore = sftp->removeFileInStrand(tempPath_); else - std::ignore = sftp_->removeFile(tempPath_).wait_for(options_.futureTimeout); + std::ignore = sftp->removeFile(tempPath_).wait_for(options_.futureTimeout); } } diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index 13034949..ada3e50b 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -24,8 +24,11 @@ namespace } } -BulkDownloadOperation::BulkDownloadOperation(SecureShell::SftpSession& sftp, BulkDownloadOperationOptions options) - : BulkTransferOperation{sftp, toCommon(options)} +BulkDownloadOperation::BulkDownloadOperation( + std::weak_ptr sftp, + BulkDownloadOperationOptions options +) + : BulkTransferOperation{std::move(sftp), toCommon(options)} , individualOptions_{std::move(options.individualOptions)} { individualOptions_.verifiedDirectories = std::make_shared(); @@ -39,14 +42,17 @@ SharedData::OperationType BulkDownloadOperation::type() const } std::expected -BulkDownloadOperation::prepareRootInStrand(SharedData::DirectoryEntry const& root) +BulkDownloadOperation::prepareRootInStrand(SecureShell::SftpSession&, SharedData::DirectoryEntry const& root) { // Base directory of the download, everything after this will be relative to this: return createLocalDirectory(common().localPath, root); } -std::expected -BulkDownloadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) +std::expected BulkDownloadOperation::createDirectoryInStrand( + SecureShell::SftpSession&, + std::size_t entryIndex, + EntryPaths const& paths +) { // create_directories follows a link planted inside the tree and would create and fill directories outside it. std::error_code error{}; diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 542de75a..2b9a4508 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -6,9 +6,9 @@ using namespace std::chrono_literals; -BulkTransferOperation::BulkTransferOperation(SecureShell::SftpSession& sftp, CommonOptions common) +BulkTransferOperation::BulkTransferOperation(std::weak_ptr sftp, CommonOptions common) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , common_{std::move(common)} , budget_{std::max(1, common_.concurrency)} {} @@ -29,7 +29,10 @@ std::expected B return result; } - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return workInStrand(); @@ -58,13 +61,15 @@ std::expected B switch (state()) { case (NotStarted): - return start(); case (Preparing): - [[fallthrough]]; case (Prepared): - [[fallthrough]]; case (Running): - return step(); + { + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + return state() == NotStarted ? start(*sftp) : step(*sftp); + } case (Finalizing): { enterState(Completed); @@ -97,7 +102,8 @@ std::expected B return enterErrorState({.type = ErrorType::UnknownWorkState}); } -std::expected BulkTransferOperation::start() +std::expected +BulkTransferOperation::start(SecureShell::SftpSession& sftp) { if (entrySource_ == EntrySource::None) { @@ -141,7 +147,7 @@ std::expected B Error{.type = ErrorType::ImplementationError, .extraInfo = "A tree needs both roots."} ); } - const auto result = prepareRootInStrand(root); + const auto result = prepareRootInStrand(sftp, root); if (!result.has_value()) return enterErrorState(result.error()); cursor_ = 1; @@ -152,7 +158,8 @@ std::expected B return WorkStatus::MoreWork; } -std::expected BulkTransferOperation::step() +std::expected +BulkTransferOperation::step(SecureShell::SftpSession& sftp) { if (cursor_ > entries_.size()) { @@ -164,7 +171,7 @@ std::expected B } const auto slotsBefore = slots_.size(); - if (const auto filled = fillSlots(); !filled.has_value()) + if (const auto filled = fillSlots(sftp); !filled.has_value()) return enterErrorState(filled.error()); // Report newly started files before stepping them: small files finish within this very // step and would otherwise never be seen in flight. @@ -196,7 +203,7 @@ std::expected B return WorkStatus::MoreWork; } -std::expected BulkTransferOperation::fillSlots() +std::expected BulkTransferOperation::fillSlots(SecureShell::SftpSession& sftp) { // Directories are created inline and each blocks the strand (a round trip or two on // upload), so a step creates only a few before yielding to the queue. @@ -238,7 +245,7 @@ std::expected BulkTransferOperation::fillSlo { // Created inline while the cursor passes it, so files listed after it only start once // it exists. Scan order lists parents before children. - const auto result = createDirectoryInStrand(index, *paths); + const auto result = createDirectoryInStrand(sftp, index, *paths); if (!result.has_value()) { // Without a tree, nothing else depends on a directory entry. @@ -639,12 +646,12 @@ std::expected BulkTransferOperation::cancel( // Each child's cancel and destructor would block on the processing thread separately; // hop once and do all of them there. - auto* processingStrand = sftp_->strand(); - if (!slots_.empty() && processingStrand && !processingStrand->withinProcessingThread()) + const auto sftp = openSession(sftp_); + if (!slots_.empty() && sftp && !sftp->strand()->withinProcessingThread()) { // Only wait, never get(): a finalized strand hands back an exceptional future and this // runs from the destructor too. - auto fut = sftp_->performPromise( + auto fut = sftp->performPromise( [this, adoptCancelState]() -> bool { cancelChildrenInStrand(adoptCancelState); @@ -674,5 +681,5 @@ void BulkTransferOperation::pause(bool doPause) SecureShell::ProcessingStrand* BulkTransferOperation::strand() const { - return sftp_->strand(); + return strandOf(sftp_); } diff --git a/backend/source/backend/sftp/bulk_upload_operation.cpp b/backend/source/backend/sftp/bulk_upload_operation.cpp index bbf6eab9..1018adc9 100644 --- a/backend/source/backend/sftp/bulk_upload_operation.cpp +++ b/backend/source/backend/sftp/bulk_upload_operation.cpp @@ -23,8 +23,11 @@ namespace } } -BulkUploadOperation::BulkUploadOperation(SecureShell::SftpSession& sftp, BulkUploadOperationOptions options) - : BulkTransferOperation{sftp, toCommon(options)} +BulkUploadOperation::BulkUploadOperation( + std::weak_ptr sftp, + BulkUploadOperationOptions options +) + : BulkTransferOperation{std::move(sftp), toCommon(options)} , individualOptions_{std::move(options.individualOptions)} {} @@ -36,14 +39,17 @@ SharedData::OperationType BulkUploadOperation::type() const } std::expected -BulkUploadOperation::prepareRootInStrand(SharedData::DirectoryEntry const& root) +BulkUploadOperation::prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) { // Base directory of the upload, everything after this will be relative to this: - return createRemoteDirectoryInStrand(common().remotePath, root); + return createRemoteDirectoryInStrand(sftp, common().remotePath, root); } -std::expected -BulkUploadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) +std::expected BulkUploadOperation::createDirectoryInStrand( + SecureShell::SftpSession& sftp, + std::size_t entryIndex, + EntryPaths const& paths +) { // A scanned directory comes after its parent, a prescanned one may be the first entry below it. if (prescanned() && individualOptions_.createMissingDirectories) @@ -51,7 +57,7 @@ BulkUploadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPaths if (auto parents = createMissingParentsInStrand(sftp, paths.remote); !parents) return parents; } - return createRemoteDirectoryInStrand(paths.remote, entries()[entryIndex]); + return createRemoteDirectoryInStrand(sftp, paths.remote, entries()[entryIndex]); } std::expected @@ -86,11 +92,12 @@ BulkUploadOperation::createMissingParentsInStrand(SecureShell::SftpSession& sftp } std::expected BulkUploadOperation::createRemoteDirectoryInStrand( + SecureShell::SftpSession& sftp, Paths::RemotePath const& path, SharedData::DirectoryEntry const& entry ) { - const auto result = sftp().createDirectoryIfItDoesntExistInStrand(path, determinePerms(entry)); + const auto result = sftp.createDirectoryIfItDoesntExistInStrand(path, determinePerms(entry)); if (!result.has_value()) { Log::error("BulkUploadOperation: Failed to create remote sftp directory: {}.", result.error().message); diff --git a/backend/source/backend/sftp/delete_operation.cpp b/backend/source/backend/sftp/delete_operation.cpp index 5472ab58..d742ec3d 100644 --- a/backend/source/backend/sftp/delete_operation.cpp +++ b/backend/source/backend/sftp/delete_operation.cpp @@ -3,9 +3,9 @@ #include -DeleteOperation::DeleteOperation(SecureShell::SftpSession& sftp, DeleteOperationOptions options) +DeleteOperation::DeleteOperation(std::weak_ptr sftp, DeleteOperationOptions options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , remotePath_{std::move(options.remotePath)} , filesRemovedProgress_{std::move(options.filesRemovedProgress)} , futureTimeout_{options.futureTimeout} @@ -16,7 +16,7 @@ DeleteOperation::~DeleteOperation() SecureShell::ProcessingStrand* DeleteOperation::strand() const { - return sftp_->strand(); + return strandOf(sftp_); } std::expected DeleteOperation::work() @@ -127,7 +127,38 @@ std::expected DeleteOperati std::expected DeleteOperation::removeOnce(Paths::RemotePath const& path) { - auto fut = sftp_->removeAll({path}); + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto const& path = target.path; + auto fut = sftp->performPromise( + [weak = std::weak_ptr{sftp}, path, compared = target.compared]() -> std::expected + { + const auto session = weak.lock(); + if (!session) + return std::unexpected(channelClosedError()); + return ComparedStateCheck::requireUnchangedRemoteInStrand(*session, path, compared) + .and_then( + [&]() -> std::expected + { + const auto knownType = compared ? std::optional{compared->type} : std::nullopt; + // An item that is already gone, after a resume for one, counts as deleted. + if (auto removed = removeOneInStrand(*session, path, knownType); + !removed && removed.error().sftpError != SSH_FX_NO_SUCH_FILE) + { + return std::unexpected( + Error{ + .type = ErrorType::DeleteFailed, + .sftpError = std::move(removed).error(), + .extraInfo = path.value(), + } + ); + } + return {}; + } + ); + } + ); if (fut.wait_for(futureTimeout_) != std::future_status::ready) return enterErrorState({.type = ErrorType::FutureTimeout}); diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index e0240b7c..5b3cb1ab 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -14,9 +14,9 @@ using namespace std::chrono_literals; -DownloadOperation::DownloadOperation(SecureShell::SftpSession& sftp, DownloadOperationOptions options) +DownloadOperation::DownloadOperation(std::weak_ptr sftp, DownloadOperationOptions options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , options_{std::move(options)} , localFile_{} { @@ -41,7 +41,10 @@ std::expected DownloadO // Drive one work() step from outside the strand. OperationQueue normally calls // workInStrand directly inside its batched umbrella, so this wrapper is only used by // callers that aren't already on the strand (standalone ops, tests). - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return workInStrand(); @@ -59,6 +62,10 @@ std::expected DownloadO { using enum OperationState; + const auto sftp = openSession(sftp_); + if (!sftp && !hasEnded()) + return enterErrorState(channelClosedError()); + switch (state_) { case (NotStarted): @@ -83,7 +90,7 @@ std::expected DownloadO ); return enterErrorState({.type = ErrorType::FileExists}); } - const auto prepareResult = prepareInStrand(); + const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) { Log::error("DownloadOperation: Failed to prepare operation: {}", prepareResult.error().toString()); @@ -97,7 +104,7 @@ std::expected DownloadO state_ = Running; if (options_.entry->isSymlink() && options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) { - const auto result = handleSymlinkInStrand(); + const auto result = handleSymlinkInStrand(*sftp); if (!result.has_value()) { Log::error("DownloadOperation: Failed to handle symlink: {}", result.error().toString()); @@ -229,10 +236,16 @@ std::expected DownloadO std::expected DownloadOperation::handleSymlink() { - auto fut = sftp_->performPromise( - [this]() + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + auto fut = sftp->performPromise( + [this]() -> std::expected { - return handleSymlinkInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return handleSymlinkInStrand(*sftp); } ); if (fut.wait_for(options_.futureTimeout) != std::future_status::ready) @@ -243,7 +256,7 @@ std::expected DownloadOperation::handleSymlink() return fut.get(); } -std::expected DownloadOperation::handleSymlinkInStrand() +std::expected DownloadOperation::handleSymlinkInStrand(SecureShell::SftpSession& sftp) { if (options_.symlinkHandling == Persistence::SymlinkHandling::SkipSymlink) { @@ -251,7 +264,7 @@ std::expected DownloadOperation::handleSymlinkIn return {}; } - const auto readResult = readSymlinkInStrand(remotePath_); + const auto readResult = readSymlinkInStrand(sftp, remotePath_); if (!readResult.has_value()) return std::unexpected(readResult.error()); auto const& linkEntry = readResult.value().targetInfo; @@ -317,7 +330,10 @@ std::expected DownloadOperation::handleSymlinkIn std::expected DownloadOperation::readOnce() { - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return readOnceInStrand(); @@ -574,9 +590,9 @@ std::expected DownloadOperation::restrictTempFil } std::expected -DownloadOperation::readSymlinkInStrand(Paths::RemotePath const& remoteFullPath) +DownloadOperation::readSymlinkInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& remoteFullPath) { - auto deepLinkResult = sftp_->readLinkDeepInStrand(remoteFullPath); + auto deepLinkResult = sftp.readLinkDeepInStrand(remoteFullPath); if (!deepLinkResult.has_value()) { Log::error( @@ -614,10 +630,16 @@ std::expected DownloadOperation::prepare() } } - auto fut = sftp_->performPromise( - [this]() + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( + [this]() -> std::expected { - return prepareInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return prepareInStrand(*sftp); } ); if (fut.wait_for(options_.futureTimeout) != std::future_status::ready) @@ -631,7 +653,7 @@ std::expected DownloadOperation::prepare() return {}; } -std::expected DownloadOperation::prepareInStrand() +std::expected DownloadOperation::prepareInStrand(SecureShell::SftpSession& sftp) { if (remotePath_.empty() || remotePath_.isRoot()) { @@ -641,7 +663,7 @@ std::expected DownloadOperation::prepareInStrand if (!options_.entry) { - const auto fileInfo = sftp_->lstatInStrand(remotePath_); + const auto fileInfo = sftp.lstatInStrand(remotePath_); if (!fileInfo.has_value()) { Log::error("DownloadOperation: Failed to stat file."); @@ -654,9 +676,8 @@ std::expected DownloadOperation::prepareInStrand if (options_.entry->isSymlink() && options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) return {}; - const auto streamOpenResult = sftp_->openFileInStrand( - remotePath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown - ); + const auto streamOpenResult = + sftp.openFileInStrand(remotePath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown); if (!streamOpenResult.has_value()) { Log::error("DownloadOperation: Failed to open remote sftp file: {}.", streamOpenResult.error().message); @@ -702,7 +723,7 @@ std::expected DownloadOperation::prepareInStrand return std::unexpected(openResult.error()); } - buffer_ = sftp_->bufferProvider().leaseForTransfer(options_.entry->size, stream->readLengthLimit()); + buffer_ = sftp.bufferProvider().leaseForTransfer(options_.entry->size, stream->readLengthLimit()); if (buffer_.empty()) { Log::error("DownloadOperation: No transfer buffer available from pool."); @@ -783,7 +804,10 @@ void DownloadOperation::pause(bool doPause) std::expected DownloadOperation::finalize() { - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return finalizeInStrand(); diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 7f6bfac4..3761f081 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -810,7 +810,9 @@ std::expected OperationQueue::addDownloadOperation( }; auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - targetQueue.emplace_back(operationId, std::make_unique(sftp, std::move(opts))); + targetQueue.emplace_back( + operationId, std::make_unique(sftp.weak_from_this(), std::move(opts)) + ); Log::info("Calling OperationQueue::{}::onOperationAdded", sessionId_.value()); RpcHelper::callRemoteSafely( @@ -833,7 +835,7 @@ std::expected OperationQueue::addDownloadOperation( else if (result->isDirectory()) { auto bulk = std::make_unique( - sftp, + sftp.weak_from_this(), BulkDownloadOperation::BulkDownloadOperationOptions{ .overallProgressCallback = makeBulkProgressCallback("onBulkDownloadProgress", operationId), .remotePath = remotePath, @@ -847,7 +849,7 @@ std::expected OperationQueue::addDownloadOperation( // The scan is part of the bulk, so cancelling or moving the card acts on both. bulk->scanFirst( std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ .progressCallback = makeScanProgressCallback("onScanProgress", operationId), .remotePath = remotePath, @@ -1026,12 +1028,12 @@ std::expected OperationQueue::addArchiveDownloadOperatio auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; // Directory roots are scanned by the archive operation itself before it packs. - auto archive = std::make_unique(sftp, std::move(opts)); + auto archive = std::make_unique(sftp.weak_from_this(), std::move(opts)); for (auto const& rootPath : directoryRootPaths) { archive->addScanRoot( std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ .progressCallback = makeScanProgressCallback("onScanProgress", operationId), .remotePath = rootPath, @@ -1155,7 +1157,7 @@ std::expected OperationQueue::addArchiveUploadOperation( auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; // Directory roots are scanned by the archive operation itself before it packs. - auto archive = std::make_unique(sftp, std::move(opts)); + auto archive = std::make_unique(sftp.weak_from_this(), std::move(opts)); for (auto const& rootPath : directoryRootPaths) { archive->addScanRoot(std::make_unique(LocalScanOperation::ScanOperationOptions{ @@ -1240,74 +1242,7 @@ std::size_t OperationQueue::addBulkDownloadOperation( ++directoryCount; continue; } - fileUnit.entries.push_back(entry); - files.push_back( - BulkDownloadOperation::PrescannedFile{ - .source = entry.source, - .destination = entry.destination, - .sizeBytes = entry.sizeBytes, - .mtime = entry.mtime, - .mtimeNsec = entry.mtimeNsec, - .mayOverwrite = entry.mayOverwrite, - .isDirectory = entry.isDirectory, - } - ); - } - - if (!files.empty()) - { - // Dedicated operationId for the aggregate bulk-download card, reserved - // by the frontend separately from the per-entry ids to avoid collisions - // (entry-0's id would clash when entry 0 is a directory). - const auto bulkOpId = bulkCardId; - const auto remoteLabel = request.remoteRoot - ? *request.remoteRoot - : commonParentOf(files, &BulkDownloadOperation::PrescannedFile::source); - const auto localLabel = request.localRoot - ? *request.localRoot - : commonParentOf(files, &BulkDownloadOperation::PrescannedFile::destination); - fileUnit.remoteRoot = remoteLabel; - fileUnit.localRoot = localLabel; - - auto downloadOpts = resolveDownloadOptions(transferOptions, resolvedTimeout); - // Sync plans list files of directories that do not exist locally yet. - downloadOpts.createMissingDirectories = true; - - auto bulk = std::make_unique( - sftp, - BulkDownloadOperation::BulkDownloadOperationOptions{ - .overallProgressCallback = makeBulkProgressCallback("onBulkDownloadProgress", bulkOpId), - .remotePath = remoteLabel, - .localPath = localLabel, - .individualOptions = std::move(downloadOpts), - .failFast = transferOptions.failFast.value_or(false), - .concurrency = bulkConcurrency_, - } - ); - bulk->setPrescannedFileList(std::move(files)); - - targetQueue.emplace_back(bulkOpId, std::move(bulk)); - RpcHelper::callRemoteSafely( - *hub_, - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = bulkOpId, - .type = SharedData::OperationType::BulkDownload, - .mode = request.mode, - .insertRefresh = request.insertRefresh, - .localPath = localLabel, - .remotePath = remoteLabel, - } - ); - } - - Log::info("Bulk download: queued one bulk card (for files) + {} directory entries", directoryCount); - - // Stashed under the card id so a teardown can hand it to the resume registry. - if (!fileUnit.entries.empty()) - bulkResumeStash_.insert_or_assign(bulkCardId, BulkResumeEntry{.request = std::move(fileUnit)}); - - return request.entries.size(); + ); } std::expected OperationQueue::addUploadOperation( @@ -1393,7 +1328,9 @@ std::expected OperationQueue::addUploadOperation( }; auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - targetQueue.emplace_back(operationId, std::make_unique(sftp, std::move(opts))); + targetQueue.emplace_back( + operationId, std::make_unique(sftp.weak_from_this(), std::move(opts)) + ); RpcHelper::callRemoteSafely( *hub_, @@ -1414,7 +1351,7 @@ std::expected OperationQueue::addUploadOperation( else if (stat.type() == std::filesystem::file_type::directory) { auto bulk = std::make_unique( - sftp, + sftp.weak_from_this(), BulkUploadOperation::BulkUploadOperationOptions{ .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", operationId), .remotePath = remotePath, @@ -1517,67 +1454,7 @@ std::size_t OperationQueue::addBulkUploadOperation( ++directoryCount; continue; } - fileUnit.entries.push_back(entry); - files.push_back( - BulkUploadOperation::PrescannedFile{ - .source = entry.source, - .destination = entry.destination, - .sizeBytes = entry.sizeBytes, - .mayOverwrite = entry.mayOverwrite, - .isDirectory = entry.isDirectory, - } - ); - } - - if (!files.empty()) - { - // Separate reserved id — see addBulkDownloadOperation for rationale. - const auto bulkOpId = bulkCardId; - const auto localLabel = request.localRoot ? *request.localRoot - : commonParentOf(files, &BulkUploadOperation::PrescannedFile::source); - const auto remoteLabel = request.remoteRoot - ? *request.remoteRoot - : commonParentOf(files, &BulkUploadOperation::PrescannedFile::destination); - fileUnit.remoteRoot = remoteLabel; - fileUnit.localRoot = localLabel; - - auto uploadOpts = resolveUploadOptions(transferOptions, resolvedTimeout); - uploadOpts.createMissingDirectories = true; - - auto bulk = std::make_unique( - sftp, - BulkUploadOperation::BulkUploadOperationOptions{ - .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", bulkOpId), - .remotePath = remoteLabel, - .localPath = localLabel, - .individualOptions = std::move(uploadOpts), - .failFast = transferOptions.failFast.value_or(false), - .concurrency = bulkConcurrency_, - } - ); - bulk->setPrescannedFileList(std::move(files)); - - targetQueue.emplace_back(bulkOpId, std::move(bulk)); - RpcHelper::callRemoteSafely( - *hub_, - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = bulkOpId, - .type = SharedData::OperationType::BulkUpload, - .mode = request.mode, - .insertRefresh = request.insertRefresh, - .localPath = localLabel, - .remotePath = remoteLabel, - } - ); - } - - Log::info("Bulk upload: queued one bulk card (for files) + {} directory entries", directoryCount); - - if (!fileUnit.entries.empty()) - bulkResumeStash_.insert_or_assign(bulkCardId, BulkResumeEntry{.request = std::move(fileUnit)}); - - return request.entries.size(); + ); } SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( @@ -1930,7 +1807,7 @@ std::expected OperationQueue::addDeleteOperation( const auto isBulk = recursive && isDirectory; auto operation = std::make_unique( - sftp, + sftp.weak_from_this(), DeleteOperation::DeleteOperationOptions{ .filesRemovedProgress = makeDeleteProgressCallback(operationId), .remotePath = remotePath, @@ -1943,7 +1820,7 @@ std::expected OperationQueue::addDeleteOperation( // The scan is part of the delete, so cancelling or moving the card acts on both. operation->addScanRoot( std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ .progressCallback = makeScanProgressCallback("onScanProgress", operationId), .remotePath = remotePath, @@ -1994,7 +1871,7 @@ std::size_t OperationQueue::addBulkDeleteOperation( const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); auto operation = std::make_unique( - sftp, + sftp.weak_from_this(), DeleteOperation::DeleteOperationOptions{ .filesRemovedProgress = makeDeleteProgressCallback(bulkOperationId), .futureTimeout = resolvedTimeout, @@ -2039,7 +1916,7 @@ std::size_t OperationQueue::addBulkDeleteOperation( { operation->addScanRoot( std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ .progressCallback = scanProgressOfNextRoot(), .remotePath = entry.target, @@ -2100,7 +1977,7 @@ std::expected OperationQueue::addRenameOperation( // Assumed in strand auto operation = std::make_unique( - sftp, + sftp.weak_from_this(), RenameOperation::RenameOperationOptions{ .sourcePath = sourcePath, .destinationPath = destinationPath, @@ -2180,7 +2057,7 @@ void OperationQueue::addSyncScanOperation( }; auto remoteScan = std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ .progressCallback = makeScanProgressCallback("onScanProgress", remoteScanId), .remotePath = remotePath, diff --git a/backend/source/backend/sftp/rename_operation.cpp b/backend/source/backend/sftp/rename_operation.cpp index ac2e294e..35e2473c 100644 --- a/backend/source/backend/sftp/rename_operation.cpp +++ b/backend/source/backend/sftp/rename_operation.cpp @@ -2,9 +2,9 @@ #include -RenameOperation::RenameOperation(SecureShell::SftpSession& sftp, RenameOperationOptions options) +RenameOperation::RenameOperation(std::weak_ptr sftp, RenameOperationOptions options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , sourcePath_{std::move(options.sourcePath)} , destinationPath_{std::move(options.destinationPath)} , futureTimeout_{options.futureTimeout} @@ -12,7 +12,7 @@ RenameOperation::RenameOperation(SecureShell::SftpSession& sftp, RenameOperation SecureShell::ProcessingStrand* RenameOperation::strand() const { - return sftp_->strand(); + return strandOf(sftp_); } std::expected RenameOperation::work() @@ -30,6 +30,9 @@ std::expected RenameOperati } case (Running): { + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); // A queued rename mirrors a local move and must not replace what is at the destination. // The task outlives this operation when the wait times out. auto fut = sftp->performPromise( @@ -37,6 +40,9 @@ std::expected RenameOperati source = sourcePath_, destination = destinationPath_]() -> std::expected { + const auto sftp = session.lock(); + if (!sftp) + return std::unexpected(SecureShell::SftpError{.message = "The SFTP channel was closed"}); return sftp->renameNoReplaceInStrand(source, destination); } ); diff --git a/backend/source/backend/sftp/scan_operation.cpp b/backend/source/backend/sftp/scan_operation.cpp index 615cab27..4ae1f105 100644 --- a/backend/source/backend/sftp/scan_operation.cpp +++ b/backend/source/backend/sftp/scan_operation.cpp @@ -7,8 +7,8 @@ #include #include -ScanOperation::ScanOperation(SecureShell::SftpSession& sftp, ScanOperationOptions options) - : sftp_(&sftp) +ScanOperation::ScanOperation(std::weak_ptr sftp, ScanOperationOptions options) + : sftp_{std::move(sftp)} , remotePath_{std::move(options.remotePath)} , progressCallback_{std::move(options.progressCallback)} , futureTimeout_{options.futureTimeout} @@ -62,9 +62,11 @@ ScanOperation::scanner(Paths::RelativePath const& relative) return std::vector{}; rootScanned_ = true; + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState>(channelClosedError()); const auto path = remotePath_ / relative; - auto fut = sftp_->listDirectory(path); - fut.wait_for(futureTimeout_); + auto fut = sftp->listDirectory(path); if (fut.wait_for(futureTimeout_) != std::future_status::ready) return enterErrorState>({.type = ErrorType::FutureTimeout}); @@ -226,5 +228,5 @@ std::expected ScanOperation::cancel(bool adoptCancel SecureShell::ProcessingStrand* ScanOperation::strand() const { - return sftp_->strand(); + return strandOf(sftp_); } \ No newline at end of file diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index 37ca9889..dd8778ac 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -12,9 +12,9 @@ #include #include -UploadOperation::UploadOperation(SecureShell::SftpSession& sftp, UploadOperationOptions options) +UploadOperation::UploadOperation(std::weak_ptr sftp, UploadOperationOptions options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , fileStream_{} , options_{std::move(options)} , localFile_{} @@ -34,9 +34,7 @@ UploadOperation::~UploadOperation() SecureShell::ProcessingStrand* UploadOperation::strand() const { - if (!sftp_) - return nullptr; - return sftp_->strand(); + return strandOf(sftp_); } void UploadOperation::pause(bool doPause) @@ -47,7 +45,10 @@ void UploadOperation::pause(bool doPause) std::expected UploadOperation::work() { - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return workInStrand(); @@ -65,6 +66,10 @@ std::expected UploadOperati { using enum OperationState; + const auto sftp = openSession(sftp_); + if (!sftp && !hasEnded()) + return enterErrorState(channelClosedError()); + switch (state_) { case (NotStarted): @@ -124,7 +129,7 @@ std::expected UploadOperati Log::error("UploadOperation: Failed to seek in local file: {}", options_.localPath.toUtf8()); return enterErrorState({.type = ErrorType::FileSeekFailure}); } - const auto prepareResult = prepareInStrand(); + const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) { Log::error("UploadOperation: Failed to prepare operation: {}", prepareResult.error().toString()); @@ -139,7 +144,7 @@ std::expected UploadOperati state_ = Running; if (isSymlink_ && options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) { - const auto result = handleSymlinkInStrand(); + const auto result = handleSymlinkInStrand(*sftp); if (!result.has_value()) { Log::error("UploadOperation: Failed to handle symlink: {}", result.error().toString()); @@ -227,7 +232,7 @@ std::expected UploadOperati case (Finalizing): { localFile_.close(); - const auto finalizeResult = finalizeInStrand(); + const auto finalizeResult = finalizeInStrand(*sftp); if (!finalizeResult.has_value()) { Log::error("UploadOperation: Failed to finalize operation: {}", finalizeResult.error().toString()); @@ -285,10 +290,16 @@ UploadOperation::commitFileToBuffer(SecureShell::IFileStream::SignedSizeType byt std::expected UploadOperation::handleSymlink() { - auto fut = sftp_->performPromise( - [this]() + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + auto fut = sftp->performPromise( + [this]() -> std::expected { - return handleSymlinkInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return handleSymlinkInStrand(*sftp); } ); if (fut.wait_for(options_.futureTimeout) != std::future_status::ready) @@ -299,7 +310,7 @@ std::expected UploadOperation::handleSymlink() return fut.get(); } -std::expected UploadOperation::handleSymlinkInStrand() +std::expected UploadOperation::handleSymlinkInStrand(SecureShell::SftpSession& sftp) { if (options_.symlinkHandling == Persistence::SymlinkHandling::SkipSymlink) { @@ -327,14 +338,14 @@ std::expected UploadOperation::handleSymlinkInStra if (options_.createMissingDirectories) { - if (auto res = ensureRemoteDirectoryExistsInStrand(*remotePath_.parent()); !res.has_value()) + if (auto res = ensureRemoteDirectoryExistsInStrand(sftp, *remotePath_.parent()); !res.has_value()) return std::unexpected(res.error()); } // The link is created under the temp name and renamed into place, so an existing destination is // never removed before its replacement exists. - std::ignore = sftp_->removeFileInStrand(tempPath_); - const auto result = sftp_->createSymLinkInStrand(*targetText, tempPath_); + std::ignore = sftp.removeFileInStrand(tempPath_); + const auto result = sftp.createSymLinkInStrand(*targetText, tempPath_); if (!result.has_value()) { Log::error( @@ -343,7 +354,7 @@ std::expected UploadOperation::handleSymlinkInStra return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = result.error()}); } - const auto moved = sftp_->moveIntoPlaceInStrand( + const auto moved = sftp.moveIntoPlaceInStrand( tempPath_, remotePath_, options_.mayOverwrite, @@ -352,7 +363,7 @@ std::expected UploadOperation::handleSymlinkInStra ); if (!moved.has_value()) { - std::ignore = sftp_->removeFileInStrand(tempPath_); + std::ignore = sftp.removeFileInStrand(tempPath_); Log::error( "UploadOperation: Failed to move symlink to '{}': {}.", options_.remotePath.value(), moved.error().message ); @@ -363,7 +374,10 @@ std::expected UploadOperation::handleSymlinkInStra std::expected UploadOperation::writeOnce() { - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return writeOnceInStrand(); @@ -424,9 +438,9 @@ std::expected UploadOperation::writeOnceInStrand() return leftToUpload_ > 0; } -std::expected UploadOperation::openOrAdoptFileInStrand() +std::expected UploadOperation::openOrAdoptFileInStrand(SecureShell::SftpSession& sftp) { - const auto result = sftp_->lstatInStrand(remotePath_); + const auto result = sftp.lstatInStrand(remotePath_); if (!result.has_value()) { const auto err = result.error(); @@ -449,7 +463,7 @@ std::expected UploadOperation::openOrAdoptFileInSt } } - const auto tempResult = sftp_->lstatInStrand(tempPath_); + const auto tempResult = sftp.lstatInStrand(tempPath_); // Permissions of what is uploaded: the target of a followed symlink, not the link. std::error_code statusError{}; @@ -464,7 +478,7 @@ std::expected UploadOperation::openOrAdoptFileInSt Log::debug("UploadOperation: Remote temp file is larger than local file, do not adopt file."); // Do not adopt file } - else if (options_.tryContinue && !tempBelongsToSourceInStrand(tempResult->size)) + else if (options_.tryContinue && !tempBelongsToSourceInStrand(sftp, tempResult->size)) { Log::info( "UploadOperation: Remote temp file '{}' is not a part of this source, starting over.", tempPath_.value() @@ -474,7 +488,7 @@ std::expected UploadOperation::openOrAdoptFileInSt { Log::debug("UploadOperation: Continuing upload to existing temp file."); - const auto openResult = sftp_->openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Write, perms); + const auto openResult = sftp.openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Write, perms); if (!openResult.has_value()) { Log::error("Failed to open remote sftp file for continue: {}", openResult.error().message); @@ -505,13 +519,13 @@ std::expected UploadOperation::openOrAdoptFileInSt if (options_.createMissingDirectories) { - if (auto res = ensureRemoteDirectoryExistsInStrand(*tempPath_.parent()); !res.has_value()) + if (auto res = ensureRemoteDirectoryExistsInStrand(sftp, *tempPath_.parent()); !res.has_value()) return std::unexpected(res.error()); } // Open temp file part regularly, freshly: Log::debug("UploadOperation: Starting new upload to '{}'.", tempPath_.value()); - const auto openResult = sftp_->openFileInStrand( + const auto openResult = sftp.openFileInStrand( tempPath_, SecureShell::SftpSession::OpenType::Write | SecureShell::SftpSession::OpenType::Create | SecureShell::SftpSession::OpenType::Truncate, @@ -528,13 +542,13 @@ std::expected UploadOperation::openOrAdoptFileInSt return {}; } -bool UploadOperation::tempBelongsToSourceInStrand(std::uint64_t tempSize) +bool UploadOperation::tempBelongsToSourceInStrand(SecureShell::SftpSession& sftp, std::uint64_t tempSize) { // Another program's partial file or a part of an older version of the source must not be continued. if (tempSize == 0) return false; const auto opened = - sftp_->openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::none); + sftp.openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::none); if (!opened) return false; auto stream = opened->lock(); @@ -546,12 +560,12 @@ bool UploadOperation::tempBelongsToSourceInStrand(std::uint64_t tempSize) } std::expected -UploadOperation::ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& dir) +UploadOperation::ensureRemoteDirectoryExistsInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& dir) { if (dir.isRoot()) return {}; - const auto statResult = sftp_->lstatInStrand(dir); + const auto statResult = sftp.lstatInStrand(dir); if (statResult.has_value()) { if (statResult->type == SharedData::FileType::Directory) @@ -572,7 +586,7 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& di if (statResult.error().sftpError != SSH_FX_NO_SUCH_FILE) return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = statResult.error()}); - if (auto parent = ensureRemoteDirectoryExistsInStrand(dir.parent().value_or(Paths::RemotePath::root())); + if (auto parent = ensureRemoteDirectoryExistsInStrand(sftp, dir.parent().value_or(Paths::RemotePath::root())); !parent.has_value()) return parent; @@ -580,7 +594,7 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& di std::filesystem::perms::owner_all | std::filesystem::perms::group_read | std::filesystem::perms::group_exec | std::filesystem::perms::others_read | std::filesystem::perms::others_exec ); - const auto mkdirResult = sftp_->createDirectoryIfItDoesntExistInStrand(dir, dirPerms); + const auto mkdirResult = sftp.createDirectoryIfItDoesntExistInStrand(dir, dirPerms); if (!mkdirResult.has_value()) { Log::error( @@ -650,10 +664,16 @@ std::expected UploadOperation::prepare() return enterErrorState({.type = ErrorType::FileSeekFailure}); } - auto umbrellaFut = sftp_->performPromise( - [this]() + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto umbrellaFut = sftp->performPromise( + [this]() -> std::expected { - return prepareInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return prepareInStrand(*sftp); } ); if (umbrellaFut.wait_for(options_.futureTimeout) != std::future_status::ready) @@ -675,9 +695,9 @@ std::expected UploadOperation::prepare() return {}; } -std::expected UploadOperation::prepareInStrand() +std::expected UploadOperation::prepareInStrand(SecureShell::SftpSession& sftp) { - if (auto result = openOrAdoptFileInStrand(); !result.has_value()) + if (auto result = openOrAdoptFileInStrand(sftp); !result.has_value()) return std::unexpected(result.error()); auto stream = fileStream_.lock(); @@ -687,7 +707,7 @@ std::expected UploadOperation::prepareInStrand() return std::unexpected(Error{.type = ErrorType::FileStreamExpired}); } - buffer_ = sftp_->bufferProvider().leaseForTransfer(totalSize_, stream->writeLengthLimit()); + buffer_ = sftp.bufferProvider().leaseForTransfer(totalSize_, stream->writeLengthLimit()); if (buffer_.empty()) { Log::error("UploadOperation: No transfer buffer available from pool."); @@ -741,10 +761,16 @@ std::expected UploadOperation::finalize() localFile_.close(); - auto fut = sftp_->performPromise( - [this]() + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + auto fut = sftp->performPromise( + [this]() -> std::expected { - return finalizeInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return finalizeInStrand(*sftp); } ); // finalize has more work than a single call (stat + maybe removeFile + rename + setstat), @@ -764,7 +790,7 @@ std::expected UploadOperation::finalize() return {}; } -std::expected UploadOperation::finalizeInStrand() +std::expected UploadOperation::finalizeInStrand(SecureShell::SftpSession& sftp) { auto stream = fileStream_.lock(); if (!stream) @@ -790,9 +816,9 @@ std::expected UploadOperation::finalizeInStrand() stream->closeInStrand(); // The modification time goes on the temp file, so nothing can fail once the destination is replaced. - applyModificationTimeInStrand(); + applyModificationTimeInStrand(sftp); - const auto moved = sftp_->moveIntoPlaceInStrand( + const auto moved = sftp.moveIntoPlaceInStrand( tempPath_, remotePath_, options_.mayOverwrite, @@ -802,14 +828,14 @@ std::expected UploadOperation::finalizeInStrand() if (!moved.has_value()) { Log::error("UploadOperation: Failed to move '{}' into place: {}", options_.remotePath.value(), moved.error().message); - if (auto removed = sftp_->removeFileInStrand(tempPath_); !removed.has_value()) + if (auto removed = sftp.removeFileInStrand(tempPath_); !removed.has_value()) Log::warn("UploadOperation: Cannot remove '{}': {}", tempPath_.value(), removed.error().message); return std::unexpected(remoteMoveError(moved.error())); } return {}; } -void UploadOperation::applyModificationTimeInStrand() +void UploadOperation::applyModificationTimeInStrand(SecureShell::SftpSession& sftp) { std::error_code timeError{}; const auto localTime = std::filesystem::last_write_time(localNativePath_, timeError); @@ -830,7 +856,7 @@ void UploadOperation::applyModificationTimeInStrand() attributes.flags = SSH_FILEXFER_ATTR_ACMODTIME; attributes.atime = seconds; attributes.mtime = seconds; - if (const auto applied = sftp_->statInStrand(tempPath_, &attributes); !applied.has_value()) + if (const auto applied = sftp.statInStrand(tempPath_, &attributes); !applied.has_value()) Log::warn("UploadOperation: Failed to set mtime on '{}': {}.", tempPath_.value(), applied.error().message); } diff --git a/shared_data/include/shared_data/file_operations/operation_error_type.hpp b/shared_data/include/shared_data/file_operations/operation_error_type.hpp index 20ce72dd..8349d97c 100644 --- a/shared_data/include/shared_data/file_operations/operation_error_type.hpp +++ b/shared_data/include/shared_data/file_operations/operation_error_type.hpp @@ -47,6 +47,8 @@ namespace SharedData ScanIncomplete, DirectoryUnreadable, InvalidName, - LinkInLocalPath + LinkInLocalPath, + ChannelClosed, + ChangedSinceCompare ); } \ No newline at end of file From bde04587e013ada6cfc1641e7190a82419b44335 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:18:38 +0200 Subject: [PATCH 057/131] A connection's operation queue starts unless its profile pauses it A connection's operation queue starts running when its profile does not start it paused: the frontend unpauses the backend queue, which is always created paused, once it reported its initial state --- .../source/frontend/session_components/operation_queue.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/source/frontend/session_components/operation_queue.cpp b/frontend/source/frontend/session_components/operation_queue.cpp index 23b1ca72..e6a45964 100644 --- a/frontend/source/frontend/session_components/operation_queue.cpp +++ b/frontend/source/frontend/session_components/operation_queue.cpp @@ -59,6 +59,8 @@ struct OperationQueue::Implementation std::vector onUpdate; Nui::Observed paused{true}; + /** @brief The profile starts the queue of every connection paused, the backend creates it paused. */ + bool startsPaused{true}; std::shared_ptr> autoClean{std::make_shared>(false)}; ObservedRandomAccessMap priorityOperations; ObservedRandomAccessMap operations; @@ -311,7 +313,8 @@ OperationQueue::OperationQueue( ), } { - impl_->paused = queueOptions.startInPausedState.value_or(true); + impl_->startsPaused = queueOptions.startInPausedState.value_or(true); + impl_->paused = impl_->startsPaused; *impl_->autoClean = queueOptions.autoRemoveCompletedOperations.value_or(false); impl_->liveQueuePageSize = std::clamp(queueOptions.liveQueuePageSize.value_or(200), 1, 1000); @@ -1670,6 +1673,8 @@ void OperationQueue::onIsPaused(SharedData::ErrorOrSuccess Log::info("Initial pause state of operation queue is: {}", result.paused ? "paused" : "running"); impl_->paused = result.paused; Nui::globalEventContext.executeActiveEventsImmediately(); + if (result.paused && !impl_->startsPaused) + unpause(); } else { From 94021ca4a8908d27076b206cebfdf0975cd7e5e4 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:25:50 +0200 Subject: [PATCH 058/131] Every SFTP request on a channel is answered Every SFTP request on a channel is answered: the channel helpers look the session up weakly themselves and reply with the message of an exception thrown on the strand, a moved RpcOnce counts as answered so a late error never goes out under an empty id, and the queueing requests share queueOnChannel; remote and local request paths and JSON requests parse through one set of RpcHelper functions in rpc_path_parameters, also used by the local filesystem RPCs and the existence checks --- backend/include/backend/rpc_helper.hpp | 32 +- .../include/backend/rpc_path_parameters.hpp | 56 ++ backend/include/backend/session.hpp | 146 +++-- backend/source/backend/existence_check.cpp | 21 +- backend/source/backend/rpc_filesystem.cpp | 25 +- .../source/backend/rpc_path_parameters.cpp | 56 ++ backend/source/backend/session.cpp | 570 +++++++----------- 7 files changed, 469 insertions(+), 437 deletions(-) create mode 100644 backend/include/backend/rpc_path_parameters.hpp create mode 100644 backend/source/backend/rpc_path_parameters.cpp diff --git a/backend/include/backend/rpc_helper.hpp b/backend/include/backend/rpc_helper.hpp index 08575bbc..a70de7d6 100644 --- a/backend/include/backend/rpc_helper.hpp +++ b/backend/include/backend/rpc_helper.hpp @@ -20,6 +20,7 @@ #include #include #include +#include // TODO: split and document this file! namespace RpcHelper @@ -166,12 +167,39 @@ namespace RpcHelper , hub_{&hub} , responseId_{responseId} {} - RpcOnce(RpcOnce&&) = default; - RpcOnce& operator=(RpcOnce&&) = default; + /** + * @brief Takes over the reply of @p other, which counts as answered from then on, so a reply through it is + * dropped with a warning instead of going out under an empty id. + */ + RpcOnce(RpcOnce&& other) noexcept + : wnd_{other.wnd_} + , hub_{other.hub_} + , responseId_{std::move(other.responseId_)} + , called_{std::exchange(other.called_, true)} + {} + RpcOnce& operator=(RpcOnce&& other) noexcept + { + if (this != &other) + { + wnd_ = other.wnd_; + hub_ = other.hub_; + responseId_ = std::move(other.responseId_); + called_ = std::exchange(other.called_, true); + } + return *this; + } RpcOnce(RpcOnce const&) = delete; RpcOnce& operator=(RpcOnce const&) = delete; ~RpcOnce() = default; + /** + * @brief The reply was sent, or handed to another RpcOnce. + */ + bool answered() const noexcept + { + return called_; + } + void operator()(nlohmann::json&& json) const { if (called_) diff --git a/backend/include/backend/rpc_path_parameters.hpp b/backend/include/backend/rpc_path_parameters.hpp new file mode 100644 index 00000000..d0804189 --- /dev/null +++ b/backend/include/backend/rpc_path_parameters.hpp @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace RpcHelper +{ + /** + * @brief A remote path from the frontend, or the message to reply with when it is not a valid absolute path. + */ + std::expected parseRemotePath(std::string const& path); + + /** + * @brief The remote paths from the frontend, or the message about the first one that is not a valid absolute path. + */ + std::expected, std::string> parseRemotePaths(std::vector const& paths); + + /** + * @brief A local path from the frontend, or the message to reply with when it is not a valid absolute path. + */ + std::expected parseLocalPath(std::string const& path); + + /** + * @brief The local paths from the frontend, or the message about the first one that is not a valid absolute path. + */ + std::expected, std::string> parseLocalPaths(std::vector const& paths); + + /** + * @brief A JSON request from the frontend, or the message to reply with when it does not parse. Paths are + * validated while parsing. + * + * @param what Names the request in the message, e.g. "bulk download". + */ + template + std::expected parseRequest(nlohmann::json const& json, std::string_view what) + { + try + { + return json.get(); + } + catch (std::exception const& exception) + { + return std::unexpected(fmt::format("Invalid {}: {}", what, Utility::errorMessage(exception))); + } + } +} diff --git a/backend/include/backend/session.hpp b/backend/include/backend/session.hpp index e4e2c07a..928ca067 100644 --- a/backend/include/backend/session.hpp +++ b/backend/include/backend/session.hpp @@ -8,10 +8,14 @@ #include #include -#include -#include #include +#include +#include +#include #include +#include +#include +#include /** * @brief This session is the implementation equivalent of one tab in the UI. @@ -226,65 +230,121 @@ class Session void removeSftpChannel(Ids::ChannelId channelId); - template - void withChannelDo(Ids::ChannelId channelId, FunctionT&& func, ReplyCallable&& reply) + /** + * @brief Runs @p func on the strand with the channel @p channelId of @p channels, as (reply, channel) or + * (reply, session, channel). @p reply is always answered: when the session is gone, when there is no such + * channel, and with the message of an exception @p func throws. + */ + template + void withLockedChannelDo( + std::unordered_map, Ids::IdHash> Session::* channels, + void (Session::*remove)(Ids::ChannelId), + Ids::ChannelId channelId, + FunctionT&& func, + RpcHelper::RpcOnce&& reply + ) { within_strand_do( - [this, + [weak = weak_from_this(), + channels, + remove, channelId = std::move(channelId), func = std::forward(func), - reply = std::forward(reply)]() mutable + reply = std::move(reply)]() mutable { - if (auto iter = channels_.find(channelId); iter != channels_.end()) + auto self = weak.lock(); + if (!self) + return reply.error("Session no longer exists"); + auto& map = (*self).*channels; + const auto iter = map.find(channelId); + if (iter == map.end()) { - if (auto channel = iter->second.lock(); channel) - { - func(std::move(reply), std::move(channel)); - } + Log::error("No channel found with id: {}", channelId.value()); + return reply.error("No channel found with id"); + } + auto channel = iter->second.lock(); + if (!channel) + { + Log::error("Failed to lock channel with id: {}", channelId.value()); + ((*self).*remove)(channelId); + return reply.error("Failed to lock channel"); + } + try + { + if constexpr ( + std::is_invocable_v> + ) + func(std::move(reply), *self, std::move(channel)); else - { - Log::error("Failed to lock channel with id: {}", channelId.value()); - removeChannel(channelId); - return reply({{"error", "Failed to lock channel"}}); - } + func(std::move(reply), std::move(channel)); } - else + catch (std::exception const& exception) { - Log::error("No channel found with id: {}", channelId.value()); - return reply({{"error", "No channel found with id"}}); + Log::error( + "A call on channel '{}' failed: {}", channelId.value(), Utility::errorMessage(exception) + ); + reply.error(Utility::errorMessage(exception)); + } + catch (...) + { + Log::error("A call on channel '{}' failed with an unknown exception", channelId.value()); + reply.error("Unknown error"); } } ); } - template - void withSftpChannelDo(Ids::ChannelId channelId, FunctionT&& func, ReplyCallable&& reply) + template + void withChannelDo(Ids::ChannelId channelId, FunctionT&& func, RpcHelper::RpcOnce&& reply) { - within_strand_do( - [this, - channelId = std::move(channelId), - func = std::forward(func), - reply = std::forward(reply)]() mutable + withLockedChannelDo( + &Session::channels_, + &Session::removeChannel, + std::move(channelId), + std::forward(func), + std::move(reply) + ); + } + + template + void withSftpChannelDo(Ids::ChannelId channelId, FunctionT&& func, RpcHelper::RpcOnce&& reply) + { + withLockedChannelDo( + &Session::sftpChannels_, + &Session::removeSftpChannel, + std::move(channelId), + std::forward(func), + std::move(reply) + ); + } + + /** + * @brief Queues operations on the SFTP channel @p channelId with @p queue, called on the strand with the operation + * queue and the channel, and answers @p reply with the JSON it returns or with the message it fails with, + * which the log names as the failure to queue @p description. A queued request resets the queue throttle. + */ + template + void queueOnChannel(RpcHelper::RpcOnce&& reply, Ids::ChannelId channelId, std::string description, QueueT&& queue) + { + withSftpChannelDo( + std::move(channelId), + [description = std::move(description), queue = std::forward(queue)]( + RpcHelper::RpcOnce&& reply, Session& self, std::shared_ptr const& channel + ) mutable { - if (auto iter = sftpChannels_.find(channelId); iter != sftpChannels_.end()) - { - if (auto channel = iter->second.lock(); channel) - { - func(std::move(reply), std::move(channel)); - } - else - { - Log::error("Failed to lock channel with id: {}", channelId.value()); - removeSftpChannel(channelId); - return reply({{"error", "Failed to lock channel"}}); - } - } - else + if (!self.operationQueue_) + return reply.error("Session no longer exists"); + std::expected queued = queue(*self.operationQueue_, *channel); + if (!queued) { - Log::error("No channel found with id: {}", channelId.value()); - return reply({{"error", "No channel found with id"}}); + Log::error("Failed to queue {}: {}", description, queued.error()); + return reply.error(queued.error()); } - } + Log::info("Queued {}", description); + self.resetQueueThrottle(); + reply(*std::move(queued)); + }, + std::move(reply) ); } diff --git a/backend/source/backend/existence_check.cpp b/backend/source/backend/existence_check.cpp index cac367fa..2585d2a5 100644 --- a/backend/source/backend/existence_check.cpp +++ b/backend/source/backend/existence_check.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -14,15 +15,15 @@ std::expected, std::string> localPathsExist(std::vector const& paths) { + const auto localPaths = RpcHelper::parseLocalPaths(paths); + if (!localPaths) + return std::unexpected(localPaths.error()); std::vector results; - results.reserve(paths.size()); - for (auto const& pathUtf8 : paths) + results.reserve(localPaths->size()); + for (auto const& path : *localPaths) { - const auto path = Paths::LocalPath::fromUtf8(pathUtf8); - if (!path) - return std::unexpected(path.error().toString()); std::error_code ec; - const auto status = std::filesystem::symlink_status(path->native(), ec); + const auto status = std::filesystem::symlink_status(path.native(), ec); if (status.type() == std::filesystem::file_type::not_found) { results.push_back(false); @@ -144,12 +145,14 @@ std::expected, std::string> remotePathsExist( std::chrono::milliseconds timeout ) { - std::vector remotePaths; - remotePaths.reserve(paths.size()); + const auto parsed = RpcHelper::parseRemotePaths(paths); + if (!parsed) + return std::unexpected(parsed.error()); + auto const& remotePaths = *parsed; // Parents in the order they first appear, with the indices of their queried children. std::vector>> byParent; std::unordered_map parentIndex; - for (auto const& path : paths) + for (std::size_t index = 0; index < remotePaths.size(); ++index) { const auto parent = remotePaths[index].parent(); if (!parent) diff --git a/backend/source/backend/rpc_filesystem.cpp b/backend/source/backend/rpc_filesystem.cpp index 68b11151..41ed9bf3 100644 --- a/backend/source/backend/rpc_filesystem.cpp +++ b/backend/source/backend/rpc_filesystem.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -34,19 +35,12 @@ namespace */ std::expected localPathParameter(std::string const& utf8) { - return Paths::LocalPath::fromUtf8(utf8) - .transform( - [](Paths::LocalPath const& path) - { - return path.native(); - } - ) - .transform_error( - [](Paths::PathError const& error) - { - return error.toString(); - } - ); + return RpcHelper::parseLocalPath(utf8).transform( + [](Paths::LocalPath const& path) + { + return path.native(); + } + ); } #ifndef _WIN32 @@ -623,10 +617,7 @@ void RpcFilesystem::registerDoesExist() void RpcFilesystem::registerDoesExistBatch() { - // Batched form of RpcFilesystem::exists. Lets the frontend ask about a - // whole transfer's worth of destination paths in a single round-trip - // instead of N — eliminating the per-file network latency that made - // multi-file drag/drop downloads feel slow. + // Whether each of N local paths exists, in one RPC. on("RpcFilesystem::existsBatch") .perform( [](RpcHelper::RpcOnce&& reply, std::vector const& paths) diff --git a/backend/source/backend/rpc_path_parameters.cpp b/backend/source/backend/rpc_path_parameters.cpp new file mode 100644 index 00000000..a10044b8 --- /dev/null +++ b/backend/source/backend/rpc_path_parameters.cpp @@ -0,0 +1,56 @@ +#include + +#include + +#include + +namespace RpcHelper +{ + namespace + { + template + std::expected, std::string> parseAll(std::vector const& paths, ParseT parse) + { + std::vector parsed; + parsed.reserve(paths.size()); + for (auto const& path : paths) + { + auto one = parse(path); + if (!one) + return std::unexpected(std::move(one).error()); + parsed.push_back(*std::move(one)); + } + return parsed; + } + } + + std::expected parseRemotePath(std::string const& path) + { + return Paths::RemotePath::make(path).transform_error( + [](Paths::PathError const& error) + { + return fmt::format("Invalid remote path: {}", error.toString()); + } + ); + } + + std::expected, std::string> parseRemotePaths(std::vector const& paths) + { + return parseAll(paths, parseRemotePath); + } + + std::expected parseLocalPath(std::string const& path) + { + return Paths::LocalPath::fromUtf8(path).transform_error( + [](Paths::PathError const& error) + { + return fmt::format("Invalid local path: {}", error.toString()); + } + ); + } + + std::expected, std::string> parseLocalPaths(std::vector const& paths) + { + return parseAll(paths, parseLocalPath); + } +} diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index 48a956a0..f564ad4a 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -25,47 +26,31 @@ using namespace std::chrono_literals; namespace { /** - * @brief A remote path from the frontend, or the message to reply with when it is not a valid absolute path. + * @brief The reply to a request that queued one operation, or why it could not be queued. */ - std::expected parseRemotePath(std::string const& path) + std::expected queuedReply(std::expected const& queued) { - return Paths::RemotePath::make(path).transform_error( - [](Paths::PathError const& error) - { - return fmt::format("Invalid remote path: {}", error.toString()); - } - ); - } - - /** - * @brief A local path from the frontend, or the message to reply with when it is not a valid absolute path. - */ - std::expected parseLocalPath(std::string const& path) - { - return Paths::LocalPath::fromUtf8(path).transform_error( - [](Paths::PathError const& error) - { - return fmt::format("Invalid local path: {}", error.toString()); - } - ); + return queued + .transform( + []() + { + return nlohmann::json{{"success", true}}; + } + ) + .transform_error( + [](Operation::Error const& error) + { + return error.toString(); + } + ); } /** - * @brief A JSON request from the frontend, or the message to reply with when it does not parse. Paths are - * validated while parsing. + * @brief The reply to a request that queued @p enqueued entries. */ nlohmann::json enqueuedReply(std::size_t enqueued) { - std::vector result; - result.reserve(paths.size()); - for (auto const& path : paths) - { - auto parsed = parseRemotePath(path); - if (!parsed) - return std::unexpected(std::move(parsed).error()); - result.push_back(*std::move(parsed)); - } - return result; + return {{"success", true}, {"enqueued", enqueued}}; } /** @@ -650,7 +635,7 @@ void Session::registerRpcSftpListDirectory() Ids::makeChannelId(channelIdString), [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - const auto remotePath = parseRemotePath(path); + const auto remotePath = RpcHelper::parseRemotePath(path); if (!remotePath) return reply.error(remotePath.error()); auto fut = channel->listDirectory(*remotePath); @@ -725,7 +710,7 @@ void Session::registerRpcSftpUserDirectories() Ids::makeChannelId(channelIdString), [home](RpcHelper::RpcOnce&& reply, auto&& channel) { - const auto homePath = parseRemotePath(home); + const auto homePath = RpcHelper::parseRemotePath(home); if (!homePath) return reply.error(homePath.error()); auto fut = channel->userDirectories(*homePath); @@ -769,18 +754,19 @@ void Session::registerRpcQueuedRemoves() if (!self) return reply.error("Session no longer exists"); - auto parsed = parseRemotePaths(pathStrings); + auto parsed = RpcHelper::parseRemotePaths(pathStrings); if (!parsed) return reply.error(parsed.error()); self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), paths = *std::move(parsed), recursive, insertRefresh, mode]( - RpcHelper::RpcOnce&& reply, auto&& channel + [paths = *std::move(parsed), recursive, insertRefresh, mode]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel ) mutable { - auto self = weak.lock(); - if (!self) + if (!self.operationQueue_) return reply.error("Session no longer exists"); auto guard = self.operationQueue_->remoteDeleteGuard(*channel); @@ -791,7 +777,7 @@ void Session::registerRpcQueuedRemoves() for (auto const& path : paths) { const auto newOperationId = Ids::generateOperationId(); - const auto result = self->operationQueue_->addDeleteOperation( + const auto result = self.operationQueue_->addDeleteOperation( guard, *channel, newOperationId, @@ -832,7 +818,7 @@ void Session::registerRpcSftpCreateDirectory() Ids::makeChannelId(channelIdString), [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - const auto remotePath = parseRemotePath(path); + const auto remotePath = RpcHelper::parseRemotePath(path); if (!remotePath) return reply.error(remotePath.error()); auto fut = channel->createDirectory(*remotePath); @@ -870,7 +856,7 @@ void Session::registerRpcSftpCreateFile() Ids::makeChannelId(channelIdString), [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - const auto remotePath = parseRemotePath(path); + const auto remotePath = RpcHelper::parseRemotePath(path); if (!remotePath) return reply.error(remotePath.error()); auto fut = channel->createFile(*remotePath); @@ -909,34 +895,34 @@ void Session::registerRpcSftpAddDownloadOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - auto remote = parseRemotePath(remotePath); + auto remote = RpcHelper::parseRemotePath(remotePath); if (!remote) return reply.error(remote.error()); - auto local = parseLocalPath(localPath); + auto local = RpcHelper::parseLocalPath(localPath); if (!local) return reply.error(local.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the download '{}' of '{}' to '{}'", newOperationIdString, remote->value(), local->toUtf8() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), local = *std::move(local), remote = *std::move(remote), allowOverwrite, bigFile, insertRefresh, createMissingDirectories, - mode](RpcHelper::RpcOnce&& reply, auto&& channel) + mode](OperationQueue& queue, SecureShell::SftpSession& channel) { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto result = self->operationQueue_->addDownloadOperation( - *channel, - Ids::makeOperationId(newOperationIdString), + return queuedReply(queue.addDownloadOperation( + channel, + operationId, local, remote, allowOverwrite, @@ -944,30 +930,8 @@ void Session::registerRpcSftpAddDownloadOperation() insertRefresh, createMissingDirectories, static_cast(mode) - ); - - if (!result.has_value()) - { - Log::error( - "Failed to add download operation for file '{}' to '{}': {}", - remote.value(), - local.toUtf8(), - result.error().toString() - ); - return reply.error("Failed to add download operation: " + result.error().toString()); - } - - Log::info( - "Added download operation with id '{}' for file '{}' to '{}'", - newOperationIdString, - remote.value(), - local.toUtf8() - ); - - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + )); + } ); } ); @@ -976,8 +940,6 @@ void Session::registerRpcSftpAddDownloadOperation() void Session::registerRpcSftpAddArchiveDownloadOperation() { // One RPC to pack N remote files into a single local tar[.ext] archive. - // The entries list is trusted (no per-entry lstat) and the operation runs - // sequentially on the SFTP strand like a regular DownloadOperation. on(fmt::format("Session::{}::sftp::addArchiveDownload", id_.value())) .perform( [weak = weak_from_this()]( @@ -994,7 +956,7 @@ void Session::registerRpcSftpAddArchiveDownloadOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); auto roots = RpcHelper::parseRequest>( nlohmann::json::parse(rootsJson, nullptr, false), "archive roots" @@ -1005,57 +967,35 @@ void Session::registerRpcSftpAddArchiveDownloadOperation() if (!archivePath) return reply.error(archivePath.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the archive download '{}' of {} roots to '{}'", + newOperationIdString, + roots->size(), + archivePath->toUtf8() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, - roots = std::move(roots), - localArchivePath = *std::move(archivePath), + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), + roots = *std::move(roots), + archivePath = *std::move(archivePath), compressionCodec, compressionLevel, mayOverwrite, - mode](RpcHelper::RpcOnce&& reply, auto&& channel) mutable + mode](OperationQueue& queue, SecureShell::SftpSession& channel) mutable { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto codec = static_cast(compressionCodec); - - const auto result = self->operationQueue_->addArchiveDownloadOperation( - *channel, - Ids::makeOperationId(newOperationIdString), + return queuedReply(queue.addArchiveDownloadOperation( + channel, + operationId, std::move(roots), - localArchivePath, - codec, + archivePath, + static_cast(compressionCodec), compressionLevel, mayOverwrite, static_cast(mode) - ); - - if (!result.has_value()) - { - Log::error( - "Failed to add archive-download operation to '{}': {}", - localArchivePath.toUtf8(), - result.error().toString() - ); - return reply.error( - "Failed to add archive-download operation: " + result.error().toString() - ); - } - - Log::info( - "Added archive-download operation '{}' → '{}' ({} roots)", - newOperationIdString, - localArchivePath.toUtf8(), - roots.size() - ); - - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + )); + } ); } ); @@ -1079,65 +1019,44 @@ void Session::registerRpcSftpAddArchiveUploadOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - std::vector localPaths; - localPaths.reserve(localPathStrings.size()); - for (auto const& pathString : localPathStrings) - { - auto localPath = parseLocalPath(pathString); - if (!localPath) - return reply.error(localPath.error()); - localPaths.push_back(*std::move(localPath)); - } - auto archivePath = parseRemotePath(remoteArchivePath); + auto localPaths = RpcHelper::parseLocalPaths(localPathStrings); + if (!localPaths) + return reply.error(localPaths.error()); + auto archivePath = RpcHelper::parseRemotePath(remoteArchivePath); if (!archivePath) return reply.error(archivePath.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the archive upload '{}' of {} paths to '{}'", + newOperationIdString, + localPaths->size(), + archivePath->value() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, - localPaths = std::move(localPaths), - remoteArchivePath = *std::move(archivePath), + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), + localPaths = *std::move(localPaths), + archivePath = *std::move(archivePath), compressionCodec, compressionLevel, mayOverwrite, - mode](RpcHelper::RpcOnce&& reply, auto&& channel) mutable + mode](OperationQueue& queue, SecureShell::SftpSession& channel) mutable { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto codec = static_cast(compressionCodec); - const auto result = self->operationQueue_->addArchiveUploadOperation( - *channel, - Ids::makeOperationId(newOperationIdString), + return queuedReply(queue.addArchiveUploadOperation( + channel, + operationId, std::move(localPaths), - remoteArchivePath, - codec, + archivePath, + static_cast(compressionCodec), compressionLevel, mayOverwrite, static_cast(mode) - ); - if (!result.has_value()) - { - Log::error( - "Failed to add archive-upload operation to '{}': {}", - remoteArchivePath.value(), - result.error().toString() - ); - return reply.error("Failed to add archive-upload operation: " + result.error().toString()); - } - Log::info( - "Added archive-upload operation '{}' → '{}'", - newOperationIdString, - remoteArchivePath.value() - ); - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + )); + } ); } ); @@ -1145,59 +1064,47 @@ void Session::registerRpcSftpAddArchiveUploadOperation() void Session::registerRpcSftpAddBulkDownloadOperation() { - // One RPC for a whole batch. The handler performs all queueing inside a - // single strand dispatch so the SSH thread-hop cost is paid once per - // batch instead of once per file. Files are trusted (no lstat per entry). + // One RPC and one strand dispatch for a whole batch. Files are trusted, there is no lstat per entry. on(fmt::format("Session::{}::sftp::addBulkDownload", id_.value())) .perform( [weak = weak_from_this()]( RpcHelper::RpcOnce&& reply, std::string const& channelIdString, - std::vector const& operationIdStrings, + std::vector&& operationIdStrings, nlohmann::json const& requestJson ) { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); - - // Parsed here, so an invalid path is answered with an error instead of no reply at all. - auto parsed = parseRequest(requestJson, "bulk download"); - if (!parsed) - return reply.error(parsed.error()); - auto request = *std::move(parsed); + return reply.error("Session no longer exists"); - // Expect N per-entry ids + 1 dedicated aggregate-bulk-card id at the end. - if (operationIdStrings.size() != request.entries.size() + 1) + auto request = RpcHelper::parseRequest(requestJson, "bulk download"); + if (!request) + return reply.error(request.error()); + // An id per entry and the id of the card of the files at the end. + if (operationIdStrings.size() != request->entries.size() + 1) return reply.error("addBulkDownload: operationIds and entries length mismatch"); - self->withSftpChannelDo( + auto description = fmt::format("the bulk download of {} entries", request->entries.size()); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - operationIdStrings, - request](RpcHelper::RpcOnce&& reply, auto&& channel) + std::move(description), + [operationIds = std::move(operationIdStrings), request = *std::move(request)]( + OperationQueue& queue, SecureShell::SftpSession& channel + ) mutable -> std::expected { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto bulkCardId = Ids::makeOperationId(operationIdStrings.back()); - const auto enqueued = self->operationQueue_->addBulkDownloadOperation( - *channel, - request, - [&operationIdStrings](std::size_t idx) + const auto bulkCardId = Ids::makeOperationId(operationIds.back()); + return enqueuedReply(queue.addBulkDownloadOperation( + channel, + std::move(request), + [&operationIds](std::size_t index) { - return Ids::makeOperationId(operationIdStrings[idx]); + return Ids::makeOperationId(operationIds[index]); }, bulkCardId - ); - - Log::info("addBulkDownload: queued {}/{} entries", enqueued, request.entries.size()); - - self->resetQueueThrottle(); - reply({{"success", true}, {"enqueued", enqueued}}); - }, - std::move(reply) + )); + } ); } ); @@ -1205,55 +1112,46 @@ void Session::registerRpcSftpAddBulkDownloadOperation() void Session::registerRpcSftpAddBulkUploadOperation() { - // See registerRpcSftpAddBulkDownloadOperation — same contract for upload. + // Same contract as addBulkDownload. on(fmt::format("Session::{}::sftp::addBulkUpload", id_.value())) .perform( [weak = weak_from_this()]( RpcHelper::RpcOnce&& reply, std::string const& channelIdString, - std::vector const& operationIdStrings, + std::vector&& operationIdStrings, nlohmann::json const& requestJson ) { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); - - // Parsed here, so an invalid path is answered with an error instead of no reply at all. - auto parsed = parseRequest(requestJson, "bulk upload"); - if (!parsed) - return reply.error(parsed.error()); - auto request = *std::move(parsed); + return reply.error("Session no longer exists"); - if (operationIdStrings.size() != request.entries.size() + 1) + auto request = RpcHelper::parseRequest(requestJson, "bulk upload"); + if (!request) + return reply.error(request.error()); + if (operationIdStrings.size() != request->entries.size() + 1) return reply.error("addBulkUpload: operationIds and entries length mismatch"); - self->withSftpChannelDo( + auto description = fmt::format("the bulk upload of {} entries", request->entries.size()); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - operationIdStrings, - request](RpcHelper::RpcOnce&& reply, auto&& channel) + std::move(description), + [operationIds = std::move(operationIdStrings), request = *std::move(request)]( + OperationQueue& queue, SecureShell::SftpSession& channel + ) mutable -> std::expected { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto bulkCardId = Ids::makeOperationId(operationIdStrings.back()); - const auto enqueued = self->operationQueue_->addBulkUploadOperation( - *channel, - request, - [&operationIdStrings](std::size_t idx) + const auto bulkCardId = Ids::makeOperationId(operationIds.back()); + return enqueuedReply(queue.addBulkUploadOperation( + channel, + std::move(request), + [&operationIds](std::size_t index) { - return Ids::makeOperationId(operationIdStrings[idx]); + return Ids::makeOperationId(operationIds[index]); }, bulkCardId - ); - - Log::info("addBulkUpload: queued {}/{} entries", enqueued, request.entries.size()); - self->resetQueueThrottle(); - reply({{"success", true}, {"enqueued", enqueued}}); - }, - std::move(reply) + )); + } ); } ); @@ -1278,34 +1176,34 @@ void Session::registerRpcSftpAddUploadOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - auto local = parseLocalPath(localPath); + auto local = RpcHelper::parseLocalPath(localPath); if (!local) return reply.error(local.error()); - auto remote = parseRemotePath(remotePath); + auto remote = RpcHelper::parseRemotePath(remotePath); if (!remote) return reply.error(remote.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the upload '{}' of '{}' to '{}'", newOperationIdString, local->toUtf8(), remote->value() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), local = *std::move(local), remote = *std::move(remote), allowOverwrite, bigFile, insertRefresh, createMissingDirectories, - mode](RpcHelper::RpcOnce&& reply, auto&& channel) + mode](OperationQueue& queue, SecureShell::SftpSession& channel) { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto result = self->operationQueue_->addUploadOperation( - *channel, - Ids::makeOperationId(newOperationIdString), + return queuedReply(queue.addUploadOperation( + channel, + operationId, local, remote, allowOverwrite, @@ -1313,30 +1211,8 @@ void Session::registerRpcSftpAddUploadOperation() insertRefresh, createMissingDirectories, static_cast(mode) - ); - - if (!result.has_value()) - { - Log::error( - "Failed to add upload operation for file '{}' to '{}': {}", - local.toUtf8(), - remote.value(), - result.error().toString() - ); - return reply.error("Failed to add upload operation: " + result.error().toString()); - } - - Log::info( - "Added upload operation with id '{}' for file '{}' to '{}'", - newOperationIdString, - local.toUtf8(), - remote.value() - ); - - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + )); + } ); } ); @@ -1357,57 +1233,31 @@ void Session::registerRpcSftpAddRenameOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - auto source = parseRemotePath(sourcePath); + auto source = RpcHelper::parseRemotePath(sourcePath); if (!source) return reply.error(source.error()); - auto destination = parseRemotePath(destinationPath); + auto destination = RpcHelper::parseRemotePath(destinationPath); if (!destination) return reply.error(destination.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the rename '{}' of '{}' to '{}'", newOperationIdString, source->value(), destination->value() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), source = *std::move(source), destination = *std::move(destination), - mode](RpcHelper::RpcOnce&& reply, auto&& channel) + mode](OperationQueue& queue, SecureShell::SftpSession& channel) { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto result = self->operationQueue_->addRenameOperation( - *channel, - Ids::makeOperationId(newOperationIdString), - source, - destination, - static_cast(mode) - ); - - if (!result.has_value()) - { - Log::error( - "Failed to add rename operation for '{}' to '{}': {}", - source.value(), - destination.value(), - result.error().toString() - ); - return reply.error("Failed to add rename operation: " + result.error().toString()); - } - - Log::info( - "Added rename operation with id '{}' for '{}' to '{}'", - newOperationIdString, - source.value(), - destination.value() - ); - - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + return queuedReply(queue.addRenameOperation( + channel, operationId, source, destination, static_cast(mode) + )); + } ); } ); @@ -1460,17 +1310,16 @@ void Session::registerRpcSftpOpenSyncSession() return reply({{"error", "Session no longer exists"}}); // The roots every synchronized path is resolved against, refused here when they are not absolute. - auto remoteRoot = parseRemotePath(remotePath); + auto remoteRoot = RpcHelper::parseRemotePath(remotePath); if (!remoteRoot) return reply.error(remoteRoot.error()); - auto localRoot = Paths::LocalPath::fromUtf8(localPath); + auto localRoot = RpcHelper::parseLocalPath(localPath); if (!localRoot) - return reply.error(fmt::format("Invalid local path: {}", localRoot.error().toString())); + return reply.error(localRoot.error()); self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - syncSessionIdString, + [syncSessionIdString, remoteScanIdString, localScanIdString, remotePath = *std::move(remoteRoot), @@ -1483,11 +1332,10 @@ void Session::registerRpcSftpOpenSyncSession() std::shared_ptr const& channel ) { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); + if (!self.operationQueue_) + return reply.error("Session no longer exists"); - self->operationQueue_->addSyncScanOperation( + self.operationQueue_->addSyncScanOperation( *channel, Ids::makeSyncSessionId(syncSessionIdString), Ids::makeOperationId(remoteScanIdString), @@ -1499,7 +1347,7 @@ void Session::registerRpcSftpOpenSyncSession() ignoreHidden ); - self->resetQueueThrottle(); + self.resetQueueThrottle(); reply({{"success", true}}); }, std::move(reply) @@ -1663,10 +1511,10 @@ void Session::registerRpcSftpExecuteSyncSelection() // A stopped session has no queue any more. if (!self.operationQueue_) return reply.error("Session no longer exists"); - auto result = self->operationQueue_->executeSyncPlan( - *channel, *session, std::move(plan), request + auto result = self.operationQueue_->executeSyncPlan( + *channel, *session, std::move(plan), request, *self.localOperationQueue_ ); - self->resetQueueThrottle(); + self.resetQueueThrottle(); nlohmann::json json; SharedData::to_json(json, result); reply(std::move(json)); @@ -1729,13 +1577,9 @@ void Session::registerRpcSftpPreDeleteChecks() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), directories](RpcHelper::RpcOnce&& reply, auto&& channel) + [directories](RpcHelper::RpcOnce&& reply, auto&& channel) { - auto self = weak.lock(); - if (!self) - return reply.error("Session no longer exists"); - - auto remotePaths = parseRemotePaths(directories); + auto remotePaths = RpcHelper::parseRemotePaths(directories); if (!remotePaths) return reply.error(remotePaths.error()); auto fut = channel->filterOutEmptyDirectories(*std::move(remotePaths)); @@ -1766,13 +1610,15 @@ void Session::registerRpcSftpDeleteFiles() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), paths](RpcHelper::RpcOnce&& reply, auto&& channel) + [paths]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel + ) { - auto self = weak.lock(); - if (!self) + if (!self.operationQueue_) return reply.error("Session no longer exists"); - - auto remotePaths = parseRemotePaths(paths); + auto remotePaths = RpcHelper::parseRemotePaths(paths); if (!remotePaths) return reply.error(remotePaths.error()); auto guard = self.operationQueue_->remoteDeleteGuard(*channel); @@ -1811,14 +1657,12 @@ void Session::registerRpcSftpRename() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), oldName, newName, overwrite]( - RpcHelper::RpcOnce&& reply, auto&& channel + [oldName, newName, overwrite]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel ) { - auto self = weak.lock(); - if (!self) - return reply.error("Session no longer exists"); - const auto oldPath = Paths::RemotePath::make(oldName); const auto newPath = Paths::RemotePath::make(newName); if (!oldPath || !newPath || !oldPath->parent() || oldPath->parent() != newPath->parent()) @@ -1860,9 +1704,7 @@ void Session::registerRpcSftpRename() void Session::registerRpcSftpAddBulkDeleteOperation() { - // One RPC for the whole bulk delete. Frontend pre-allocates a single - // OperationId for the aggregate file-bulk card; the backend may also - // create extra single-Delete cards for any directory entries. + // One RPC and one card for the whole bulk delete, under the operation id the frontend reserved. on(fmt::format("Session::{}::sftp::addBulkDelete", id_.value())) .perform( [weak = weak_from_this()]( @@ -1874,17 +1716,20 @@ void Session::registerRpcSftpAddBulkDeleteOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); auto request = RpcHelper::parseRequest(requestJson, "bulk delete"); if (!request) return reply.error(request.error()); - self->withSftpChannelDo( + auto description = fmt::format("the bulk delete of {} entries", request->entries.size()); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - bulkOperationIdString, - request](RpcHelper::RpcOnce&& reply, auto&& channel) + std::move(description), + [operationId = Ids::makeOperationId(bulkOperationIdString), request = *std::move(request)]( + OperationQueue& queue, SecureShell::SftpSession& channel + ) -> std::expected { auto guard = queue.remoteDeleteGuard(channel); if (const auto refusal = @@ -1921,11 +1766,8 @@ void Session::registerRpcAddLocalDeleteOperation() void Session::registerRpcSftpExistsBatch() { - // Batched existence probe over SFTP. Lets the frontend ask whether N - // remote paths exist in a single round-trip — eliminates the per-file - // SFTP stat latency that made multi-file uploads feel slow. Only "no such - // file" is absent, any other failure fails the whole batch: the frontend - // overwrites what it takes for absent. + // Whether each of N remote paths exists, in one RPC. Only "no such file" is absent, any other failure fails the + // whole batch: the frontend overwrites what it takes for absent. on(fmt::format("Session::{}::sftp::existsBatch", id_.value())) .perform( [weak = weak_from_this()]( @@ -1969,13 +1811,9 @@ void Session::registerRpcSftpStat() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), path](RpcHelper::RpcOnce&& reply, auto&& channel) + [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - auto self = weak.lock(); - if (!self) - return reply.error("Session no longer exists"); - - const auto remotePath = parseRemotePath(path); + const auto remotePath = RpcHelper::parseRemotePath(path); if (!remotePath) return reply.error(remotePath.error()); auto fut = channel->stat(*remotePath); From 836149cdc9e26f212f3a8831fbdfb8e14fa4ca9f Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:33:20 +0200 Subject: [PATCH 059/131] A bulk resumed after a reconnect leaves out what it finished A bulk transfer resumed after a reconnect leaves out what it finished: the backup of a bulk that has its entries becomes a flat list of the entries it has not transferred or created yet, below its roots, so finished files are neither sent again nor reported as existing; a backup of more than one directory is refused instead of completing a card twice. Bulk requests are moved instead of copied and split through one shared helper, which returns how many entries were really queued and still queues the card of a request without entries, so a resumed card with nothing left completes --- .../backend/sftp/bulk_download_operation.hpp | 9 + .../backend/sftp/bulk_resume_registry.hpp | 15 +- .../backend/sftp/bulk_transfer_operation.hpp | 21 +- .../backend/sftp/bulk_upload_operation.hpp | 8 + .../include/backend/sftp/operation_queue.hpp | 106 +++-- .../backend/sftp/bulk_download_operation.cpp | 38 ++ .../backend/sftp/bulk_transfer_operation.cpp | 27 +- .../backend/sftp/bulk_upload_operation.cpp | 28 ++ .../source/backend/sftp/operation_queue.cpp | 404 ++++++++++++++---- 9 files changed, 512 insertions(+), 144 deletions(-) diff --git a/backend/include/backend/sftp/bulk_download_operation.hpp b/backend/include/backend/sftp/bulk_download_operation.hpp index 580f47ca..02c5ba44 100644 --- a/backend/include/backend/sftp/bulk_download_operation.hpp +++ b/backend/include/backend/sftp/bulk_download_operation.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -56,6 +57,14 @@ class BulkDownloadOperation : public BulkTransferOperation }; void setPrescannedFileList(std::vector files); + /** + * @brief The request that resumes this bulk on another connection: the entries it has not finished, as a flat + * list below its roots, with the options of @p request. None while the entries are not known yet, the + * backup then stays as it is. Also none when a remote symlink is among them, which a flat list could only + * follow. + */ + std::optional resumeRequest(SharedData::BulkDownloadRequest request) const; + protected: std::expected prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) override; diff --git a/backend/include/backend/sftp/bulk_resume_registry.hpp b/backend/include/backend/sftp/bulk_resume_registry.hpp index 25a5b3af..62d3ee68 100644 --- a/backend/include/backend/sftp/bulk_resume_registry.hpp +++ b/backend/include/backend/sftp/bulk_resume_registry.hpp @@ -11,18 +11,23 @@ #include #include +/** + * @brief The request a backup re-issues, its type picks the addBulk* path. + */ +using BulkResumeRequest = + std::variant; + /** * @brief Carries the state needed to re-enqueue a bulk operation against a * replacement Session after a seamless reconnect. * - * One entry per frontend card: the request's type picks the addBulk* - * path. A transfer entry holds either the files of one bulk card or a - * single directory, so adopting it re-issues exactly that card under - * its own id. + * One entry per frontend card. A transfer entry holds either the files + * of one bulk card or a single directory, so adopting it re-issues + * exactly that card under its own id. */ struct BulkResumeEntry { - std::variant request; + BulkResumeRequest request; std::chrono::steady_clock::time_point storedAt{std::chrono::steady_clock::now()}; }; diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index f13d8bab..08c2afc8 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -161,6 +161,20 @@ class BulkTransferOperation : public Operation return slots_.size(); } + /** + * @brief The entries are known: the scan delivered them or a flat list was set. + */ + bool hasEntries() const noexcept + { + return entrySource_ != EntrySource::None; + } + + /** + * @brief The indices of the entries a resume has to do again: every entry that was neither transferred, nor + * created as a directory, nor skipped for its type. Failed and in flight entries are among them. + */ + std::vector unfinishedEntryIndices() const; + std::uint64_t completedCount() const noexcept { return completedEntries_; @@ -297,7 +311,10 @@ class BulkTransferOperation : public Operation std::expected step(SecureShell::SftpSession& sftp); std::expected fillSlots(SecureShell::SftpSession& sftp); std::expected stepSlots(); - void finishSlot(Slot& slot); + /** + * @brief Counts the entry of @p slot as done, as @p transferred or as a failure recorded for the user. + */ + void finishSlot(Slot& slot, bool transferred); void cancelChildrenInStrand(bool adoptCancelState); Slot* slotFor(std::size_t entryIndex) noexcept; std::uint64_t fileCount() const noexcept; @@ -342,6 +359,8 @@ class BulkTransferOperation : public Operation std::uint64_t completedBytes_{0}; std::uint64_t totalBytes_{0}; int budget_{1}; + /** @brief Per entry: transferred, created or skipped for its type, so a resume leaves it out. */ + std::vector finishedEntries_{}; std::vector> failedEntries_{}; std::vector failedEntryIndices_{}; // Rolling bulk-level throughput. Per-file rates reset between files, which made the number diff --git a/backend/include/backend/sftp/bulk_upload_operation.hpp b/backend/include/backend/sftp/bulk_upload_operation.hpp index 848d0921..0de6c5da 100644 --- a/backend/include/backend/sftp/bulk_upload_operation.hpp +++ b/backend/include/backend/sftp/bulk_upload_operation.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -45,6 +46,13 @@ class BulkUploadOperation : public BulkTransferOperation }; void setPrescannedFileList(std::vector files); + /** + * @brief The request that resumes this bulk on another connection: the entries it has not finished, as a flat + * list below its roots, with the options of @p request. None while the entries are not known yet, the + * backup then stays as it is. + */ + std::optional resumeRequest(SharedData::BulkUploadRequest request) const; + protected: std::expected prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) override; diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index 411aa68d..f14694b1 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -179,40 +179,35 @@ class OperationQueue ); /** - * @brief Bulk-add download operations from a pre-known entry list. - * Files are taken at face value (no lstat — sizes come from the - * caller) and turned into individual DownloadOperation cards. - * Directory entries fall back to the existing Scan+BulkDownload - * flow. Everything happens inside a single strand dispatch so - * only one SSH-thread context switch is paid for the batch. + * @brief Queues the entries of @p request: each directory with its content as a bulk of its own that scans it + * first, under the id @p operationIdFor gives its index; all files, and directories created without their + * content, as one bulk under @p bulkCardId. Sizes and times of files are taken as given, without an lstat. + * Each queued part keeps a resume backup under its id. * - * @param sftp SFTP session to use. - * @param request Full bulk request (file + directory entries). - * @param operationIdFor Callback producing a fresh OperationId per entry - * index (frontend reserves them so it can register - * completion callbacks before the RPC returns). - * @return Number of entries successfully enqueued. + * @param sftp SFTP session to use. + * @param request The entries; the card of the files works below its roots, unset the closest directories + * holding every source and every destination. + * @param operationIdFor The id of the entry at an index, reserved by the frontend so it can register callbacks + * before the reply. + * @param bulkCardId The id of the card of the files, reserved by the frontend apart from the entry ids. + * @return Number of entries queued: the files and directories of the card and the directories queued on their + * own. A request without any entry still queues the card, which then completes at once. */ std::size_t addBulkDownloadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkDownloadRequest const& request, - std::function operationIdFor, - Ids::OperationId bulkCardId + SharedData::BulkDownloadRequest request, + std::function const& operationIdFor, + Ids::OperationId const& bulkCardId ); - /** @brief Upload analogue of addBulkDownloadOperation. File entries skip - * the per-file local lstat + RPC round-trip; directories fall - * back to the existing LocalScan+BulkUpload flow. - * - * @param bulkCardId Separate id reserved by the frontend for the aggregate - * BulkUpload card that covers all file entries. Must not - * collide with any of the per-entry ids produced by - * @p operationIdFor. */ + /** + * @brief Upload analogue of @ref addBulkDownloadOperation. + */ std::size_t addBulkUploadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkUploadRequest const& request, - std::function operationIdFor, - Ids::OperationId bulkCardId + SharedData::BulkUploadRequest request, + std::function const& operationIdFor, + Ids::OperationId const& bulkCardId ); /** @@ -339,24 +334,21 @@ class OperationQueue bool hasPriorityWork() const; /** - * @brief Moves every still-tracked bulk-operation backup into @p - * registry, keyed by the bulk's OperationId. Called by Session - * during teardown so a seamless reconnect can adopt them. + * @brief Moves every resume backup into @p registry, keyed by the id of its card. Called by Session during + * teardown so a seamless reconnect can adopt them. * - * Covers both queued-not-started bulks and bulks already running - * — for the latter we re-issue the full original entry list and - * rely on the per-file tryContinue path to skip whole files - * already present (the in-progress file's bytes are not - * byte-checkpointed here). + * A bulk transfer that has its entries already leaves only the entries it has not finished, as a flat + * list below its roots, and records how many entries and bytes it finished, which the resumed card counts + * as done; a running file in it starts over unless its partial file can be continued. A bulk that did not + * start yet or is still scanning keeps its request. */ void dumpBulkResumes(BulkResumeRegistry& registry); /** - * @brief Re-enqueues the bulk operation described by @p entry against - * this queue, reusing @p operationId as the bulk-card id so the - * frontend's existing card surface stays attached. Called from - * Session::adoptBulkResumes after pulling @p entry out of the - * registry. + * @brief Re-enqueues the backup @p entry under @p operationId as the id of every part, so the frontend's card + * stays attached. Refuses a backup of more than one directory, or of a directory and other entries, + * which would complete the card more than once. Called from Session::adoptBulkResumes after pulling + * @p entry out of the registry. */ void adoptBulkResume(SecureShell::SftpSession& sftp, Ids::OperationId const& operationId, BulkResumeEntry entry); @@ -380,6 +372,38 @@ class OperationQueue private: bool workQueue(std::deque>>& queue); + /** + * @brief The queue an operation of @p mode goes to. + */ + auto queueFor(SharedData::OperationMode mode) + -> std::deque>>&; + + /** + * @brief Shared part of the bulk transfer requests: each directory with content goes to @p queueDirectory as + * (index, entry, request options) and counts when it returns true, every other entry into one request that + * @p queueFiles queues as the card @p bulkCardId and sets the roots of. That request is the card's backup. + * + * @return Number of entries queued. + */ + template + std::size_t splitBulkRequest( + RequestT request, + Ids::OperationId const& bulkCardId, + QueueDirectoryT&& queueDirectory, + QueueFilesT&& queueFiles + ); + + /** + * @brief The queued or running operation @p operationId, none when there is none. + */ + Operation* findOperation(Ids::OperationId const& operationId); + + /** + * @brief Reduces the backup @p entry of @p operation to what the operation has not finished yet and records + * what it left out in @p entry, which the resumed card counts as done. + */ + static void leaveOutFinished(Operation const& operation, BulkResumeEntry& entry); + /** * @brief Tells the sync session waiting for the scan @p id, if any, that it failed or was cancelled. * @param reason A localized message, see Utility::localizedMessage. @@ -426,9 +450,7 @@ class OperationQueue std::map)>> syncScanCallbacks_{}; - // SyncSessions active on this channel. Keyed by SyncSessionId.value() because - // Ids::IdHash isn't specialized on the strongly-typed id; the existing - // bulkResumeStash_ does the specialization — we stay string-keyed for simplicity. + // SyncSessions active on this channel, keyed by SyncSessionId.value(). std::map> syncSessions_{}; // Per-bulk backup snapshot, populated when a bulk-add path enqueues an diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index ada3e50b..8c1587ca 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -6,6 +6,7 @@ #include #include +#include #include namespace @@ -220,3 +221,40 @@ void BulkDownloadOperation::setPrescannedFileList(std::vector fi } setPrescannedEntries(std::move(entries), std::move(paths), totalBytes); } + +std::optional +BulkDownloadOperation::resumeRequest(SharedData::BulkDownloadRequest request) const +{ + if (!hasEntries()) + return std::nullopt; + const auto unfinished = unfinishedEntryIndices(); + const auto isSymlink = [this](std::size_t index) + { + return entries()[index].isSymlink(); + }; + if (std::ranges::any_of(unfinished, isSymlink)) + return std::nullopt; + + request.entries.clear(); + for (const auto index : unfinished) + { + auto const& entry = entries()[index]; + const auto paths = entryPaths(index); + if (!paths || (!entry.isRegularFile() && !entry.isDirectory())) + continue; + request.entries.push_back({ + .source = paths->remote, + .destination = paths->local, + .sizeBytes = entry.isDirectory() ? 0u : entry.size, + .isDirectory = entry.isDirectory(), + .withoutContent = entry.isDirectory(), + .mtime = entry.mtime, + .mtimeNsec = entry.mtimeNsec, + .mayOverwrite = paths->mayOverwrite, + .comparedDestination = paths->comparedDestination, + }); + } + request.remoteRoot = common().remotePath; + request.localRoot = common().localPath; + return request; +} diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 2b9a4508..5c6d078e 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -150,6 +150,7 @@ BulkTransferOperation::start(SecureShell::SftpSession& sftp) const auto result = prepareRootInStrand(sftp, root); if (!result.has_value()) return enterErrorState(result.error()); + finishedEntries_[0] = true; cursor_ = 1; } @@ -255,6 +256,10 @@ std::expected BulkTransferOperation::fillSlo failedEntries_.emplace_back(failureSource(index), result.error()); failedEntryIndices_.push_back(static_cast(index)); } + else + { + finishedEntries_[index] = true; + } ++completedEntries_; ++directoriesCreated; continue; @@ -277,6 +282,7 @@ std::expected BulkTransferOperation::fillSlo continue; } Log::warn("{}: Skipping unsupported file type for entry: {}.", logName(), displayPath(index)); + finishedEntries_[index] = true; ++completedEntries_; } return {}; @@ -290,7 +296,7 @@ std::expected BulkTransferOperation::stepSlo if (result.has_value()) { if (result.value() == WorkStatus::Complete) - finishSlot(slot); + finishSlot(slot, true); continue; } @@ -317,7 +323,7 @@ std::expected BulkTransferOperation::stepSlo // Not critical for the bulk as a whole: record the entry for the user and move on. failedEntries_.emplace_back(failureSource(slot.entryIndex), error); failedEntryIndices_.push_back(static_cast(slot.entryIndex)); - finishSlot(slot); + finishSlot(slot, false); if (common_.failFast) { Log::error("{}: failFast is enabled, failing the whole operation due to the error.", logName()); @@ -330,8 +336,10 @@ std::expected BulkTransferOperation::stepSlo return {}; } -void BulkTransferOperation::finishSlot(Slot& slot) +void BulkTransferOperation::finishSlot(Slot& slot, bool transferred) { + if (transferred) + finishedEntries_[slot.entryIndex] = true; // Advance by the entry's reported size, not the child's measured size: totalBytes_ is // summed from the same reported sizes, so the two converge exactly. completedBytes_ += entries_[slot.entryIndex].size; @@ -471,6 +479,7 @@ void BulkTransferOperation::setParallelBudget(int slots) void BulkTransferOperation::setScanResult(std::vector&& entries, std::uint64_t totalBytes) { entries_ = std::move(entries); + finishedEntries_.assign(entries_.size(), false); totalBytes_ = totalBytes; prescannedPaths_.clear(); entrySource_ = EntrySource::Tree; @@ -483,6 +492,7 @@ void BulkTransferOperation::setPrescannedEntries( ) { entries_ = std::move(entries); + finishedEntries_.assign(entries_.size(), false); prescannedPaths_ = std::move(paths); totalBytes_ = totalBytes; entrySource_ = EntrySource::Flat; @@ -530,6 +540,17 @@ std::vector> BulkTransferOp return failedEntries_; } +std::vector BulkTransferOperation::unfinishedEntryIndices() const +{ + std::vector unfinished; + for (std::size_t index = 0; index < finishedEntries_.size(); ++index) + { + if (!finishedEntries_[index]) + unfinished.push_back(index); + } + return unfinished; +} + std::expected BulkTransferOperation::entryPaths(std::size_t entryIndex) const { diff --git a/backend/source/backend/sftp/bulk_upload_operation.cpp b/backend/source/backend/sftp/bulk_upload_operation.cpp index 1018adc9..8380ff49 100644 --- a/backend/source/backend/sftp/bulk_upload_operation.cpp +++ b/backend/source/backend/sftp/bulk_upload_operation.cpp @@ -198,3 +198,31 @@ void BulkUploadOperation::setPrescannedFileList(std::vector file } setPrescannedEntries(std::move(entries), std::move(paths), totalBytes); } + +std::optional +BulkUploadOperation::resumeRequest(SharedData::BulkUploadRequest request) const +{ + if (!hasEntries()) + return std::nullopt; + + request.entries.clear(); + for (const auto index : unfinishedEntryIndices()) + { + auto const& entry = entries()[index]; + const auto paths = entryPaths(index); + if (!paths || (!entry.isRegularFile() && !entry.isSymlink() && !entry.isDirectory())) + continue; + request.entries.push_back({ + .source = paths->local, + .destination = paths->remote, + .sizeBytes = entry.isDirectory() ? 0u : entry.size, + .isDirectory = entry.isDirectory(), + .withoutContent = entry.isDirectory(), + .mayOverwrite = paths->mayOverwrite, + .comparedDestination = paths->comparedDestination, + }); + } + request.remoteRoot = common().remotePath; + request.localRoot = common().localPath; + return request; +} diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 3761f081..f33fda74 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -158,6 +158,24 @@ namespace ); } + /** + * @brief The resume backup of the directory transfer @p source to @p destination: the directory alone, a resume + * scans it again. + */ + template + RequestT directoryResumeRequest( + SourceT const& source, + DestinationT const& destination, + bool allowOverwrite, + bool insertRefresh, + SharedData::OperationMode mode + ) + { + RequestT request{.allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, .mode = mode}; + request.entries.push_back({.source = source, .destination = destination, .isDirectory = true}); + return request; + } + DownloadOperation::DownloadOperationOptions resolveDownloadOptions(Persistence::DownloadOptions const& opts, std::chrono::seconds futureTimeout) { @@ -469,45 +487,117 @@ void OperationQueue::dumpBulkResumes(BulkResumeRegistry& registry) if (bulkResumeStash_.empty()) return; Log::info("OperationQueue::dumpBulkResumes: persisting {} bulk-operation backup(s)", bulkResumeStash_.size()); - for (auto& [opId, entry] : bulkResumeStash_) - registry.store(opId, std::move(entry)); + for (auto& [operationId, entry] : bulkResumeStash_) + { + if (auto* operation = findOperation(operationId); operation) + leaveOutFinished(*operation, entry); + registry.store(operationId, std::move(entry)); + } bulkResumeStash_.clear(); } +Operation* OperationQueue::findOperation(Ids::OperationId const& operationId) +{ + for (auto* queue : {&priorityOperations_, &operations_}) + { + const auto found = + std::ranges::find(*queue, operationId, &std::pair>::first); + if (found != queue->end()) + return found->second.get(); + } + return nullptr; +} + +void OperationQueue::leaveOutFinished(Operation const& operation, BulkResumeEntry& entry) +{ + const auto trim = [&entry](auto const& bulk, auto& request) + { + auto remaining = bulk.resumeRequest(request); + if (remaining) + { + request = *std::move(remaining); + entry.finished = bulk.finishedProgress(); + } + else + { + entry.finished = bulk.earlierProgress(); + } + }; + std::visit( + Utility::overloaded{ + [&](SharedData::BulkDownloadRequest& download) + { + if (operation.type() == SharedData::OperationType::BulkDownload) + trim(static_cast(operation), download); + }, + [&](SharedData::BulkUploadRequest& upload) + { + if (operation.type() == SharedData::OperationType::BulkUpload) + trim(static_cast(operation), upload); + }, + [](SharedData::BulkDeleteRequest&) + { + // A delete of an item that is gone already counts as done, the whole request can run again. + }, + }, + entry.request + ); +} + void OperationQueue::adoptBulkResume( SecureShell::SftpSession& sftp, Ids::OperationId const& operationId, BulkResumeEntry entry ) { - Log::info( - "OperationQueue::adoptBulkResume: re-issuing bulk operation '{}' ({} entries)", - operationId.value(), - std::visit( + const auto [entryCount, directoriesWithContent] = std::visit( + Utility::overloaded{ + [](SharedData::BulkDeleteRequest const& request) + { + return std::pair{request.entries.size(), std::size_t{0}}; + }, [](auto const& request) { - return request.entries.size(); + const auto directories = std::ranges::count_if( + request.entries, + [](auto const& transferEntry) + { + return transferEntry.isDirectory && !transferEntry.withoutContent; + } + ); + return std::pair{request.entries.size(), static_cast(directories)}; }, - entry.request - ) + }, + entry.request + ); + // A backup holds the files of one card or a single directory, more would complete the card more than once. + if (directoriesWithContent > 1 || (directoriesWithContent == 1 && entryCount > 1)) + { + Log::error( + "OperationQueue::adoptBulkResume: refusing the backup of '{}', it holds {} directories among {} entries", + operationId.value(), + directoriesWithContent, + entryCount + ); + return; + } + Log::info( + "OperationQueue::adoptBulkResume: re-issuing bulk operation '{}' ({} entries)", operationId.value(), entryCount ); - // A transfer backup holds either the files of one bulk card or a single directory, both resume under the - // stashed id so the surviving card re-attaches. - auto operationIdFor = [&operationId](std::size_t) + const auto operationIdFor = [&operationId](std::size_t) { return operationId; }; - std::visit( Utility::overloaded{ - [&](SharedData::BulkDownloadRequest const& request) + [&](SharedData::BulkDownloadRequest& request) { - addBulkDownloadOperation(sftp, request, operationIdFor, operationId); + addBulkDownloadOperation(sftp, std::move(request), operationIdFor, operationId); }, - [&](SharedData::BulkUploadRequest const& request) + [&](SharedData::BulkUploadRequest& request) { - addBulkUploadOperation(sftp, request, operationIdFor, operationId); + addBulkUploadOperation(sftp, std::move(request), operationIdFor, operationId); }, [&](SharedData::BulkDeleteRequest const& request) { @@ -738,6 +828,50 @@ void OperationQueue::paused(bool pause) ); } +auto OperationQueue::queueFor(SharedData::OperationMode mode) + -> std::deque>>& +{ + return mode == SharedData::OperationMode::PriorityQueued ? priorityOperations_ : operations_; +} + +template +std::size_t OperationQueue::splitBulkRequest( + RequestT request, + Ids::OperationId const& bulkCardId, + QueueDirectoryT&& queueDirectory, + QueueFilesT&& queueFiles +) +{ + // A directory resumes as a unit of its own under its own id, so the card's backup holds the rest. + RequestT unit{ + .allowOverwrite = request.allowOverwrite, + .insertRefresh = request.insertRefresh, + .mode = request.mode, + .localRoot = std::move(request.localRoot), + .remoteRoot = std::move(request.remoteRoot), + }; + std::size_t queued = 0; + for (std::size_t index = 0; index < request.entries.size(); ++index) + { + auto& entry = request.entries[index]; + if (entry.isDirectory && !entry.withoutContent) + { + if (queueDirectory(index, entry, unit)) + ++queued; + continue; + } + unit.entries.push_back(std::move(entry)); + } + // Without any entry the card is still queued, so a resumed card that had nothing left completes. + if (unit.entries.empty() && !request.entries.empty()) + return queued; + + queueFiles(unit); + queued += unit.entries.size(); + bulkResumeStash_.insert_or_assign(bulkCardId, BulkResumeEntry{.request = std::move(unit)}); + return queued; +} + std::expected OperationQueue::addDownloadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, @@ -787,7 +921,7 @@ std::expected OperationQueue::addDownloadOperation( auto self = weak.lock(); if (!self) return; - // Invoked from the SFTP processing strand via workInStrand — marshal back. + // Invoked from the SFTP processing strand via workInStrand, marshal back. self->within_strand_do( [weak, name, operationId, min, max, current, bytesPerSecond]() { @@ -809,7 +943,7 @@ std::expected OperationQueue::addDownloadOperation( ); }; - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); targetQueue.emplace_back( operationId, std::make_unique(sftp.weak_from_this(), std::move(opts)) ); @@ -858,7 +992,7 @@ std::expected OperationQueue::addDownloadOperation( ) ); - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); targetQueue.emplace_back(operationId, std::move(bulk)); RpcHelper::callRemoteSafely( @@ -875,18 +1009,14 @@ std::expected OperationQueue::addDownloadOperation( } ); - // Synthetic single-entry resume backup: re-issuing this through - // adopt walks the same scan+bulk path, so the entry list itself - // doesn't matter — we just need the directory paths preserved. - SharedData::BulkDownloadRequest synthetic{ - .entries = {SharedData::BulkDownloadEntry{ - .source = remotePath, .destination = localPath, .isDirectory = true - }}, - .allowOverwrite = allowOverwrite, - .insertRefresh = insertRefresh, - .mode = mode, - }; - bulkResumeStash_.insert_or_assign(operationId, BulkResumeEntry{.request = std::move(synthetic)}); + bulkResumeStash_.insert_or_assign( + operationId, + BulkResumeEntry{ + .request = directoryResumeRequest( + remotePath, localPath, allowOverwrite, insertRefresh, mode + ) + } + ); return {}; } @@ -1025,7 +1155,7 @@ std::expected OperationQueue::addArchiveDownloadOperatio ); }; - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); // Directory roots are scanned by the archive operation itself before it packs. auto archive = std::make_unique(sftp.weak_from_this(), std::move(opts)); @@ -1154,7 +1284,7 @@ std::expected OperationQueue::addArchiveUploadOperation( ); }; - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); // Directory roots are scanned by the archive operation itself before it packs. auto archive = std::make_unique(sftp.weak_from_this(), std::move(opts)); @@ -1193,22 +1323,17 @@ std::expected OperationQueue::addArchiveUploadOperation( std::size_t OperationQueue::addBulkDownloadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkDownloadRequest const& request, - std::function operationIdFor, - Ids::OperationId bulkCardId + SharedData::BulkDownloadRequest request, + std::function const& operationIdFor, + Ids::OperationId const& bulkCardId ) { - // Assumed in strand. The file portion is collapsed into a SINGLE - // BulkDownloadOperation with prescanned entries — one card on the - // frontend, one SSH-thread worker for the whole batch. Directory - // entries still use the existing Scan+BulkDownload flow (one card - // per directory root). - + // Assumed in strand. auto transferOptions = sftpOpts_.downloadOptions.value_or(Persistence::DownloadOptions{}); if (request.allowOverwrite) transferOptions.mayOverwrite = true; const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); - auto& targetQueue = (request.mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + const auto allowOverwrite = request.allowOverwrite; return splitBulkRequest( std::move(request), @@ -1217,30 +1342,80 @@ std::size_t OperationQueue::addBulkDownloadOperation( { const auto result = addDownloadOperation( sftp, - operationIdFor(idx), + operationIdFor(index), entry.destination, entry.source, - request.allowOverwrite || entry.mayOverwrite, + allowOverwrite || entry.mayOverwrite, /*isBigFile*/ false, - // Propagate the request's flag so manual folder downloads refresh - // the local panel. Sync passes false to avoid refresh spam; a - // manual bulk download passes true and expects one refresh per - // top-level directory on completion. - /*insertRefresh*/ request.insertRefresh, + unit.insertRefresh, /*createMissingDirectories*/ true, - request.mode + unit.mode ); - if (!result.has_value()) + if (!result) { Log::error( "Bulk download: directory entry '{}' failed to queue: {}", entry.source.value(), result.error().toString() ); - continue; } - ++directoryCount; - continue; + return result.has_value(); + }, + [&](SharedData::BulkDownloadRequest& unit) + { + std::vector files; + files.reserve(unit.entries.size()); + std::ranges::transform( + unit.entries, + std::back_inserter(files), + [](SharedData::BulkDownloadEntry const& entry) + { + return BulkDownloadOperation::PrescannedFile{ + .source = entry.source, + .destination = entry.destination, + .sizeBytes = entry.sizeBytes, + .mtime = entry.mtime, + .mtimeNsec = entry.mtimeNsec, + .mayOverwrite = entry.mayOverwrite, + .isDirectory = entry.isDirectory, + .comparedDestination = entry.comparedDestination, + }; + } + ); + unit.remoteRoot = unit.remoteRoot ? *unit.remoteRoot + : commonParentOf(files, &BulkDownloadOperation::PrescannedFile::source); + unit.localRoot = unit.localRoot + ? *unit.localRoot + : commonParentOf(files, &BulkDownloadOperation::PrescannedFile::destination); + + auto downloadOptions = resolveDownloadOptions(transferOptions, resolvedTimeout); + // Sync plans list files of directories that do not exist locally yet. + downloadOptions.createMissingDirectories = true; + auto bulk = std::make_unique( + sftp.weak_from_this(), + BulkDownloadOperation::BulkDownloadOperationOptions{ + .overallProgressCallback = makeBulkProgressCallback("onBulkDownloadProgress", bulkCardId), + .remotePath = *unit.remoteRoot, + .localPath = *unit.localRoot, + .individualOptions = std::move(downloadOptions), + .failFast = transferOptions.failFast.value_or(false), + .concurrency = bulkConcurrency_, + } + ); + bulk->setPrescannedFileList(std::move(files)); + queueFor(unit.mode).emplace_back(bulkCardId, std::move(bulk)); + RpcHelper::callRemoteSafely( + *hub_, + rpcName("onOperationAdded"), + SharedData::OperationAdded{ + .operationId = bulkCardId, + .type = SharedData::OperationType::BulkDownload, + .mode = unit.mode, + .insertRefresh = unit.insertRefresh, + .localPath = *unit.localRoot, + .remotePath = *unit.remoteRoot, + } + ); } ); } @@ -1327,7 +1502,7 @@ std::expected OperationQueue::addUploadOperation( ); }; - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); targetQueue.emplace_back( operationId, std::make_unique(sftp.weak_from_this(), std::move(opts)) ); @@ -1369,7 +1544,7 @@ std::expected OperationQueue::addUploadOperation( }) ); - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); targetQueue.emplace_back(operationId, std::move(bulk)); RpcHelper::callRemoteSafely( @@ -1386,16 +1561,14 @@ std::expected OperationQueue::addUploadOperation( } ); - // Synthetic single-entry resume backup: adopting it walks the same scan and bulk path. - SharedData::BulkUploadRequest synthetic{ - .entries = {SharedData::BulkUploadEntry{ - .source = localPath, .destination = remotePath, .isDirectory = true - }}, - .allowOverwrite = allowOverwrite, - .insertRefresh = insertRefresh, - .mode = mode, - }; - bulkResumeStash_.insert_or_assign(operationId, BulkResumeEntry{.request = std::move(synthetic)}); + bulkResumeStash_.insert_or_assign( + operationId, + BulkResumeEntry{ + .request = directoryResumeRequest( + localPath, remotePath, allowOverwrite, insertRefresh, mode + ) + } + ); return {}; } @@ -1408,19 +1581,17 @@ std::expected OperationQueue::addUploadOperation( std::size_t OperationQueue::addBulkUploadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkUploadRequest const& request, - std::function operationIdFor, - Ids::OperationId bulkCardId + SharedData::BulkUploadRequest request, + std::function const& operationIdFor, + Ids::OperationId const& bulkCardId ) { - // Mirrors addBulkDownloadOperation — file portion collapses into one - // BulkUploadOperation card; directory entries reuse the existing - // LocalScan+BulkUpload flow. + // Assumed in strand. auto transferOptions = sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}); if (request.allowOverwrite) transferOptions.mayOverwrite = true; const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); - auto& targetQueue = (request.mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + const auto allowOverwrite = request.allowOverwrite; return splitBulkRequest( std::move(request), @@ -1429,30 +1600,77 @@ std::size_t OperationQueue::addBulkUploadOperation( { const auto result = addUploadOperation( sftp, - operationIdFor(idx), + operationIdFor(index), entry.source, entry.destination, - request.allowOverwrite || entry.mayOverwrite, + allowOverwrite || entry.mayOverwrite, /*isBigFile*/ false, - // Propagate the request's flag so manual folder uploads refresh - // the remote panel. Sync passes false to avoid refresh spam; a - // manual bulk upload passes true and expects one refresh per - // top-level directory on completion. - /*insertRefresh*/ request.insertRefresh, + unit.insertRefresh, /*createMissingDirectories*/ true, - request.mode + unit.mode ); - if (!result.has_value()) + if (!result) { Log::error( "Bulk upload: directory entry '{}' failed to queue: {}", entry.source.toUtf8(), result.error().toString() ); - continue; } - ++directoryCount; - continue; + return result.has_value(); + }, + [&](SharedData::BulkUploadRequest& unit) + { + std::vector files; + files.reserve(unit.entries.size()); + std::ranges::transform( + unit.entries, + std::back_inserter(files), + [](SharedData::BulkUploadEntry const& entry) + { + return BulkUploadOperation::PrescannedFile{ + .source = entry.source, + .destination = entry.destination, + .sizeBytes = entry.sizeBytes, + .mayOverwrite = entry.mayOverwrite, + .isDirectory = entry.isDirectory, + .comparedDestination = entry.comparedDestination, + }; + } + ); + unit.localRoot = + unit.localRoot ? *unit.localRoot : commonParentOf(files, &BulkUploadOperation::PrescannedFile::source); + unit.remoteRoot = unit.remoteRoot + ? *unit.remoteRoot + : commonParentOf(files, &BulkUploadOperation::PrescannedFile::destination); + + auto uploadOptions = resolveUploadOptions(transferOptions, resolvedTimeout); + uploadOptions.createMissingDirectories = true; + auto bulk = std::make_unique( + sftp.weak_from_this(), + BulkUploadOperation::BulkUploadOperationOptions{ + .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", bulkCardId), + .remotePath = *unit.remoteRoot, + .localPath = *unit.localRoot, + .individualOptions = std::move(uploadOptions), + .failFast = transferOptions.failFast.value_or(false), + .concurrency = bulkConcurrency_, + } + ); + bulk->setPrescannedFileList(std::move(files)); + queueFor(unit.mode).emplace_back(bulkCardId, std::move(bulk)); + RpcHelper::callRemoteSafely( + *hub_, + rpcName("onOperationAdded"), + SharedData::OperationAdded{ + .operationId = bulkCardId, + .type = SharedData::OperationType::BulkUpload, + .mode = unit.mode, + .insertRefresh = unit.insertRefresh, + .localPath = *unit.localRoot, + .remotePath = *unit.remoteRoot, + } + ); } ); } @@ -1607,7 +1825,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( { addBulkUploadOperation( sftp, - uploads, + std::move(uploads), [](std::size_t) { return Ids::generateOperationId(); @@ -1619,7 +1837,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( { addBulkDownloadOperation( sftp, - downloads, + std::move(downloads), [](std::size_t) { return Ids::generateOperationId(); @@ -1704,7 +1922,7 @@ std::expected OperationQueue::addLocalDeleteOperation( } auto parent = LocalDeleteOperation::commonParentOf(targets); - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); targetQueue.emplace_back( operationId, std::make_unique(LocalDeleteOperation::LocalDeleteOperationOptions{ @@ -1785,7 +2003,7 @@ std::expected OperationQueue::addDeleteOperation( ) { // Assumed in strand - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); if (auto allowed = guard.check(remotePath); !allowed) { @@ -1867,7 +2085,7 @@ std::size_t OperationQueue::addBulkDeleteOperation( { // Assumed in strand. One DeleteOperation, so one card and one completion for the whole request: files are // deleted as given, every real directory is scanned first and deleted with its content. - auto& targetQueue = (request.mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(request.mode); const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); auto operation = std::make_unique( @@ -1985,7 +2203,7 @@ std::expected OperationQueue::addRenameOperation( } ); - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); targetQueue.emplace_back(operationId, std::move(operation)); RpcHelper::callRemoteSafely( From 59eac889497c25a2849a5142573def1c6face7a3 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:34:05 +0200 Subject: [PATCH 060/131] The frontend moves bulk entries into requests instead of copying The frontend moves the entries of a bulk download, upload or delete into its request instead of copying them --- .../source/frontend/terminal/file_engine.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/frontend/source/frontend/terminal/file_engine.cpp b/frontend/source/frontend/terminal/file_engine.cpp index 85c59623..a4776d30 100644 --- a/frontend/source/frontend/terminal/file_engine.cpp +++ b/frontend/source/frontend/terminal/file_engine.cpp @@ -688,8 +688,7 @@ void FileEngine::addBulkDownload( { Log::info("Requesting bulk download for {} entries", entries.size()); - // operationIds carries one id per entry plus one dedicated aggregate-bulk-card id - // at the back — see OperationQueue::addBulkDownloadOperation for rationale. + // One operation id per entry and, at the back, the id of the card of the files. if (entries.size() + 1 != operationIds.size()) { Log::error("addBulkDownload: entries and operationIds size mismatch"); @@ -704,7 +703,7 @@ void FileEngine::addBulkDownload( allowOverwrite, insertRefresh, mode, - onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) + onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) mutable { if (!channelId) { @@ -719,7 +718,7 @@ void FileEngine::addBulkDownload( operationIdStrings.push_back(id.value()); SharedData::BulkDownloadRequest request{ - .entries = entries, + .entries = std::move(entries), .allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, .mode = mode, @@ -757,8 +756,7 @@ void FileEngine::addBulkUpload( { Log::info("Requesting bulk upload for {} entries", entries.size()); - // operationIds carries one id per entry plus one dedicated aggregate-bulk-card id - // at the back — see OperationQueue::addBulkUploadOperation for rationale. + // One operation id per entry and, at the back, the id of the card of the files. if (entries.size() + 1 != operationIds.size()) { Log::error("addBulkUpload: entries and operationIds size mismatch"); @@ -773,7 +771,7 @@ void FileEngine::addBulkUpload( allowOverwrite, insertRefresh, mode, - onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) + onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) mutable { if (!channelId) { @@ -788,7 +786,7 @@ void FileEngine::addBulkUpload( operationIdStrings.push_back(id.value()); SharedData::BulkUploadRequest request{ - .entries = entries, + .entries = std::move(entries), .allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, .mode = mode, @@ -831,7 +829,7 @@ void FileEngine::addBulkDelete( bulkOperationId, insertRefresh, mode, - onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) + onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) mutable { if (!channelId) { @@ -841,7 +839,7 @@ void FileEngine::addBulkDelete( } SharedData::BulkDeleteRequest request{ - .entries = entries, + .entries = std::move(entries), .insertRefresh = insertRefresh, .mode = mode, }; From 66598c749e60d25bc6a28f6460d3975998b10ef0 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:35:22 +0200 Subject: [PATCH 061/131] Completions are built from their common fields once Completions are built from their common fields once and only the paths and failures of each operation type are added; both scans of a sync report through one makeSyncScanCallback; setCompletionObserver is documented as the test hook it is --- .../include/backend/sftp/operation_queue.hpp | 3 +- .../source/backend/sftp/operation_queue.cpp | 144 +++++++++--------- 2 files changed, 77 insertions(+), 70 deletions(-) diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index f14694b1..d93b1882 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -327,7 +327,8 @@ class OperationQueue void paused(bool pause); /** - * @brief Observes every completion right before it is sent to the frontend. Used by tests. + * @brief Test hook: @p observer sees every completion right before it is sent to the frontend. The application + * never sets one. */ void setCompletionObserver(std::function observer); diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index f33fda74..02966a21 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -66,96 +66,64 @@ namespace std::optional error = std::nullopt ) { - return operation.visit( + OperationQueue::OperationCompleted completed{ + .reason = reason, + .operationId = std::move(operationId), + .completionTime = std::chrono::system_clock::now(), + .error = std::move(error), + }; + const auto transferPaths = [&completed](auto const& transfer) + { + completed.localPath = unlessEmpty(transfer.localPath()); + completed.remotePath = unlessEmpty(transfer.remotePath()); + }; + const auto bulkFailures = [&completed](BulkTransferOperation const& bulk) + { + completed.failedEntries = bulk.getFailed(); + completed.indexedFailures = bulk.indexedFailures(); + }; + operation.visit( Utility::overloaded( - [reason, operationId, error](DownloadOperation const& op) + [&](DownloadOperation const& download) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .localPath = unlessEmpty(op.localPath()), - .remotePath = unlessEmpty(op.remotePath()), - .error = error, - }; + transferPaths(download); }, - [reason, operationId, error](ScanOperation const& op) + [&](UploadOperation const& upload) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .remotePath = unlessEmpty(op.remotePath()), - .error = error, - }; + transferPaths(upload); }, - [reason, operationId, error](LocalScanOperation const& op) + [&](ScanOperation const& scan) { completed.remotePath = unlessEmpty(scan.remotePath()); }, - [reason, operationId, error](UploadOperation const& op) + [&](LocalScanOperation const& scan) { completed.localPath = unlessEmpty(scan.localPath()); }, - [reason, operationId, error](DeleteOperation const& op) + [&](BulkDownloadOperation const& bulk) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .remotePath = unlessEmpty(op.remotePath()), - .error = error, - .failedEntries = op.getFailed(), - }; - }, - [reason, operationId, error](LocalDeleteOperation const& op) - { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - .failedEntries = op.getFailed(), - }; + bulkFailures(bulk); }, - [reason, operationId, error](RenameOperation const& op) + [&](BulkUploadOperation const& bulk) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .remotePath = unlessEmpty(op.destinationPath()), - .error = error, - }; + bulkFailures(bulk); }, - [reason, operationId, error](ArchiveDownloadOperation const&) + [&](DeleteOperation const& deletion) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - }; + completed.remotePath = unlessEmpty(deletion.remotePath()); + completed.failedEntries = deletion.getFailed(); }, - [reason, operationId, error](ArchiveUploadOperation const&) + [](LocalDeleteOperation const&) {}, + [&](RenameOperation const& rename) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - }; + completed.remotePath = unlessEmpty(rename.destinationPath()); }, - [reason, operationId](std::nullopt_t) - { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - }; - } + [](ArchiveDownloadOperation const&) {}, + [](ArchiveUploadOperation const&) {}, + [](std::nullopt_t) {} ) ); + return completed; } /** @@ -2324,6 +2292,44 @@ void OperationQueue::addSyncScanOperation( ); } +auto OperationQueue::makeSyncScanCallback( + Ids::SyncSessionId syncSessionId, + std::weak_ptr session, + Ids::OperationId siblingScanId, + bool isLocal, + void (SyncSession::*setTree)(SharedData::Sync::ScanNode) +) -> std::function)> +{ + return [weak = weak_from_this(), + syncSessionId = std::move(syncSessionId), + session = std::move(session), + siblingScanId = std::move(siblingScanId), + isLocal, + setTree](std::expected tree) + { + auto self = weak.lock(); + auto locked = session.lock(); + if (!self || !locked) + return; + if (!tree) + { + self->cancelSiblingScan(siblingScanId); + RpcHelper::callRemoteSafely( + *self->hub_, self->rpcName("onSyncScanFailed"), syncSessionId, isLocal, tree.error() + ); + return; + } + boost::asio::post( + locked->strand(), + [locked, setTree, tree = std::move(*tree)]() mutable + { + ((*locked).*setTree)(std::move(tree)); + } + ); + RpcHelper::callRemoteSafely(*self->hub_, self->rpcName("onSyncScanPhaseDone"), syncSessionId, isLocal); + }; +} + void OperationQueue::setSyncBaselineStore(std::shared_ptr store) { syncBaselines_ = std::move(store); From cb77b31b9463f95ecaedc430dcbb43ad1f1d3306 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:38:04 +0200 Subject: [PATCH 062/131] A resumed bulk card shows the whole bulk A resumed bulk card shows the whole bulk: the backup records how many entries and bytes the bulk finished before the reconnect, and the adopted bulk counts them as done in its totals and progress from the start --- .../backend/sftp/bulk_download_operation.hpp | 1 + .../backend/sftp/bulk_resume_registry.hpp | 13 +++++++++ .../backend/sftp/bulk_transfer_operation.hpp | 22 ++++++++++++++ .../backend/sftp/bulk_upload_operation.hpp | 1 + .../backend/sftp/bulk_transfer_operation.cpp | 29 ++++++++++++++++--- .../source/backend/sftp/operation_queue.cpp | 6 ++++ 6 files changed, 68 insertions(+), 4 deletions(-) diff --git a/backend/include/backend/sftp/bulk_download_operation.hpp b/backend/include/backend/sftp/bulk_download_operation.hpp index 02c5ba44..47ae6344 100644 --- a/backend/include/backend/sftp/bulk_download_operation.hpp +++ b/backend/include/backend/sftp/bulk_download_operation.hpp @@ -6,6 +6,7 @@ #include #include +#include #include class BulkDownloadOperation : public BulkTransferOperation diff --git a/backend/include/backend/sftp/bulk_resume_registry.hpp b/backend/include/backend/sftp/bulk_resume_registry.hpp index 62d3ee68..83e49e73 100644 --- a/backend/include/backend/sftp/bulk_resume_registry.hpp +++ b/backend/include/backend/sftp/bulk_resume_registry.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -17,6 +18,16 @@ using BulkResumeRequest = std::variant; +/** + * @brief Entries and bytes of a bulk finished on an earlier connection. Its resumed card counts them as done, so it + * shows the whole bulk instead of a smaller one. + */ +struct BulkResumeProgress +{ + std::uint64_t entries{0}; + std::uint64_t bytes{0}; +}; + /** * @brief Carries the state needed to re-enqueue a bulk operation against a * replacement Session after a seamless reconnect. @@ -28,6 +39,8 @@ using BulkResumeRequest = struct BulkResumeEntry { BulkResumeRequest request; + /** @brief What the bulk finished before the backup, left out of @ref request. */ + BulkResumeProgress finished{}; std::chrono::steady_clock::time_point storedAt{std::chrono::steady_clock::now()}; }; diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index 08c2afc8..36cfabfe 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -175,6 +176,26 @@ class BulkTransferOperation : public Operation */ std::vector unfinishedEntryIndices() const; + /** + * @brief Counts @p progress, what an earlier connection finished of this bulk, as done: its entries and bytes + * are part of the totals and of the progress from the start. Call it after the entries are set. + */ + void countEarlierProgress(BulkResumeProgress progress); + + /** + * @brief What a resume of this bulk leaves out: the entries it finished, without the root of a tree, and the + * bytes of their files, with what an earlier connection finished. + */ + BulkResumeProgress finishedProgress() const; + + /** + * @brief What an earlier connection finished of this bulk, see @ref countEarlierProgress. + */ + BulkResumeProgress earlierProgress() const noexcept + { + return earlierProgress_; + } + std::uint64_t completedCount() const noexcept { return completedEntries_; @@ -359,6 +380,7 @@ class BulkTransferOperation : public Operation std::uint64_t completedBytes_{0}; std::uint64_t totalBytes_{0}; int budget_{1}; + BulkResumeProgress earlierProgress_{}; /** @brief Per entry: transferred, created or skipped for its type, so a resume leaves it out. */ std::vector finishedEntries_{}; std::vector> failedEntries_{}; diff --git a/backend/include/backend/sftp/bulk_upload_operation.hpp b/backend/include/backend/sftp/bulk_upload_operation.hpp index 0de6c5da..4fcd435a 100644 --- a/backend/include/backend/sftp/bulk_upload_operation.hpp +++ b/backend/include/backend/sftp/bulk_upload_operation.hpp @@ -6,6 +6,7 @@ #include #include +#include #include class BulkUploadOperation : public BulkTransferOperation diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 5c6d078e..5c4661ef 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -367,8 +367,8 @@ std::uint64_t BulkTransferOperation::fileCount() const noexcept // Tree mode treats entries_[0] as the synthetic root (not counted toward the user-visible // total); prescanned mode has no root, so every entry counts. if (entries_.empty()) - return 0; - return entries_.size() - (prescanned() ? 0 : 1); + return earlierProgress_.entries; + return earlierProgress_.entries + entries_.size() - (prescanned() ? 0 : 1); } std::uint64_t BulkTransferOperation::bytesNow() const noexcept @@ -480,7 +480,7 @@ void BulkTransferOperation::setScanResult(std::vector> BulkTransferOp return failedEntries_; } +void BulkTransferOperation::countEarlierProgress(BulkResumeProgress progress) +{ + earlierProgress_ = progress; + completedEntries_ += progress.entries; + completedBytes_ += progress.bytes; + totalBytes_ += progress.bytes; +} + +BulkResumeProgress BulkTransferOperation::finishedProgress() const +{ + auto progress = earlierProgress_; + for (std::size_t index = prescanned() ? 0 : 1; index < finishedEntries_.size(); ++index) + { + if (!finishedEntries_[index]) + continue; + ++progress.entries; + progress.bytes += entries_[index].isRegularFile() ? entries_[index].size : 0u; + } + return progress; +} + std::vector BulkTransferOperation::unfinishedEntryIndices() const { std::vector unfinished; diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 02966a21..cba85af3 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -574,6 +574,12 @@ void OperationQueue::adoptBulkResume( }, entry.request ); + + auto* adopted = findOperation(operationId); + if (adopted && + (adopted->type() == SharedData::OperationType::BulkDownload || + adopted->type() == SharedData::OperationType::BulkUpload)) + static_cast(*adopted).countEarlierProgress(entry.finished); } void OperationQueue::deepPause(bool pause) From 67c9d8258628b306aaddd7c992ac8db6b9da6bb0 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:45:37 +0200 Subject: [PATCH 063/131] Walked trees build each directory's path once Walked trees build each directory's path once: DeepDirectoryWalker keeps the path of every directory it adopted, reports a finished walk as done, and returns relative paths as expected values like SharedData::relativePath, which no longer throws for a parent outside the entries but refuses it as UnknownParent; the unused dot-entry template parameter of both walkers is gone, rejected entries and bulk entries without paths are described by one describeWalkedEntry, a bulk transfer builds the paths of an entry once when it starts (Detail::treeEntryPaths for trees), an archive upload computes each member's relative path once and the ignore matcher matches segments without parsing them into a path again --- .../backend/sftp/bulk_transfer_operation.hpp | 26 +++- .../backend/sftp/local_scan_operation.hpp | 37 +----- .../include/backend/sftp/scan_operation.hpp | 35 +----- .../backend/sftp/archive_upload_operation.cpp | 18 +-- .../backend/sftp/bulk_transfer_operation.cpp | 53 +++----- paths/include/paths/path_error.hpp | 6 + .../include/shared_data/directory_entry.hpp | 5 +- .../sync/tree_directory_walker.hpp | 2 +- .../source/shared_data/ignore_rules.cpp | 17 ++- .../include/utility/directory_traversal.hpp | 117 +++++++++++++----- 10 files changed, 163 insertions(+), 153 deletions(-) diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index 36cfabfe..7b5ba3ac 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -225,8 +225,8 @@ class BulkTransferOperation : public Operation } /** - * @brief Both ends of the entry at @p entryIndex. In tree mode the entry's names are joined below the roots, - * so an entry whose name the local side cannot hold fails with InvalidName. + * @brief Both ends of the entry at @p entryIndex. In tree mode the entry's names are joined below the roots + * with @ref Detail::treeEntryPaths. */ std::expected entryPaths(std::size_t entryIndex) const; @@ -240,6 +240,11 @@ class BulkTransferOperation : public Operation */ std::string failureSource(std::size_t entryIndex) const; + /** + * @brief The source of @p paths, the key a failure is reported under. + */ + std::string failureSource(EntryPaths const& paths) const; + /** * @brief The entry below the source root by its names as listed, for entries without valid paths. */ @@ -322,6 +327,8 @@ class BulkTransferOperation : public Operation struct Slot { std::size_t entryIndex; + /** @brief Both ends of the entry, built once when it starts. */ + EntryPaths paths; std::unique_ptr child; std::uint64_t bytes{0}; std::uint64_t totalBytes{0}; @@ -392,3 +399,18 @@ class BulkTransferOperation : public Operation std::int64_t bulkBytesPerSecond_{0}; std::chrono::steady_clock::time_point lastEmit_{}; }; + +namespace Detail +{ + /** + * @brief Both ends of the entry @p entryIndex of a scanned tree: its names on the way from the root joined below + * @p remoteRoot and @p localRoot. InvalidName when a name is no path segment or the local side cannot hold + * it, ImplementationError for a missing root or an index out of range. + */ + std::expected treeEntryPaths( + std::vector const& entries, + std::size_t entryIndex, + Paths::RemotePath const& remoteRoot, + Paths::LocalPath const& localRoot + ); +} diff --git a/backend/include/backend/sftp/local_scan_operation.hpp b/backend/include/backend/sftp/local_scan_operation.hpp index f96f1432..a9b2db96 100644 --- a/backend/include/backend/sftp/local_scan_operation.hpp +++ b/backend/include/backend/sftp/local_scan_operation.hpp @@ -80,14 +80,7 @@ class LocalScanOperation : public Operation auto withWalkerDo(auto&& fn) { - auto scan = [this](Paths::RelativePath const& relative) - { - return scanner(relative); - }; - using WalkerType = Utility::DeepDirectoryWalker; - if (!walker_) - walker_ = std::make_unique(std::move(scan)); - return fn(static_cast(*walker_)); + return ScanWalkers::withFlatWalkerDo(*this, walker_, std::forward(fn)); } /** @@ -96,23 +89,9 @@ class LocalScanOperation : public Operation */ auto withTreeWalkerDo(auto&& fn) { - auto scan = [this](Paths::RelativePath const& relative) - { - return scanner(relative); - }; - using WalkerType = SharedData::Sync::TreeDirectoryWalker; - if (!walker_) - walker_ = std::make_unique( - std::move(scan), - recursive_, - [](Error const& error) - { - return error.type == ErrorType::DirectoryUnreadable; - }, - /*requireLocalNames=*/false, - excludedSuffixes_ - ); - return fn(static_cast(*walker_)); + return ScanWalkers::withTreeWalkerDo( + *this, walker_, recursive_, /*requireLocalNames=*/false, excludedSuffixes_, std::forward(fn) + ); } /** @@ -141,13 +120,7 @@ class LocalScanOperation : public Operation return withWalkerDo( [this](auto& walker) { - std::vector paths; - auto root = localPath_.toUtf8(); - if (!root.ends_with('/')) - root.push_back('/'); - for (auto const& entry : walker.rejectedEntries()) - paths.push_back(root + walker.describeRejected(entry)); - return paths; + return ScanWalkers::rejectedPaths(walker, localPath_.toUtf8()); } ); } diff --git a/backend/include/backend/sftp/scan_operation.hpp b/backend/include/backend/sftp/scan_operation.hpp index 721a0ab3..090a50a5 100644 --- a/backend/include/backend/sftp/scan_operation.hpp +++ b/backend/include/backend/sftp/scan_operation.hpp @@ -80,14 +80,7 @@ class ScanOperation : public Operation auto withWalkerDo(auto&& fn) { - auto scan = [this](Paths::RelativePath const& relative) - { - return scanner(relative); - }; - using WalkerType = Utility::DeepDirectoryWalker; - if (!walker_) - walker_ = std::make_unique(std::move(scan)); - return fn(static_cast(*walker_)); + return ScanWalkers::withFlatWalkerDo(*this, walker_, std::forward(fn)); } /** @@ -96,23 +89,9 @@ class ScanOperation : public Operation */ auto withTreeWalkerDo(auto&& fn) { - auto scan = [this](Paths::RelativePath const& relative) - { - return scanner(relative); - }; - using WalkerType = SharedData::Sync::TreeDirectoryWalker; - if (!walker_) - walker_ = std::make_unique( - std::move(scan), - recursive_, - [](Error const& error) - { - return error.type == ErrorType::DirectoryUnreadable; - }, - /*requireLocalNames=*/true, - excludedSuffixes_ - ); - return fn(static_cast(*walker_)); + return ScanWalkers::withTreeWalkerDo( + *this, walker_, recursive_, /*requireLocalNames=*/true, excludedSuffixes_, std::forward(fn) + ); } /** @@ -138,11 +117,7 @@ class ScanOperation : public Operation return withWalkerDo( [this](auto& walker) { - std::vector paths; - const auto root = remotePath_.isRoot() ? std::string{} : remotePath_.value(); - for (auto const& entry : walker.rejectedEntries()) - paths.push_back(root + "/" + walker.describeRejected(entry)); - return paths; + return ScanWalkers::rejectedPaths(walker, remotePath_.value()); } ); } diff --git a/backend/source/backend/sftp/archive_upload_operation.cpp b/backend/source/backend/sftp/archive_upload_operation.cpp index e2a44437..ce25476d 100644 --- a/backend/source/backend/sftp/archive_upload_operation.cpp +++ b/backend/source/backend/sftp/archive_upload_operation.cpp @@ -355,24 +355,24 @@ void ArchiveUploadOperation::appendPrescannedFromScan( { // Index 0 is the synthetic scan root (the directory itself); the caller // emitted its tar header already, so we start at 1 and consume the - // walker's flat depth-first entry list — mirrors the download op's + // walker's flat depth-first entry list, like the download op's // appendPrescannedInStrand. for (std::size_t idx = 1u; idx < entries.size(); ++idx) { auto const& child = entries[idx]; - auto source = SharedData::relativePath(entries, child) - .and_then( - [&localRoot](Paths::RelativePath const& relative) - { - return localRoot.join(relative); - } - ); + const auto relative = SharedData::relativePath(entries, child); + auto source = relative.and_then( + [&localRoot](Paths::RelativePath const& path) + { + return localRoot.join(path); + } + ); if (!source) { Log::warn("ArchiveUploadOperation: skipping pre-scanned child: {}", source.error().toString()); continue; } - const auto memberPath = archivePrefix / *SharedData::relativePath(entries, child); + const auto memberPath = archivePrefix / *relative; if (child.type == SharedData::FileType::Regular) { diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 5c4661ef..876c8452 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include @@ -278,7 +279,9 @@ std::expected BulkTransferOperation::fillSlo } } ); - slots_.push_back(Slot{.entryIndex = index, .child = std::move(child), .totalBytes = entry.size}); + slots_.push_back( + Slot{.entryIndex = index, .paths = *paths, .child = std::move(child), .totalBytes = entry.size} + ); continue; } Log::warn("{}: Skipping unsupported file type for entry: {}.", logName(), displayPath(index)); @@ -317,11 +320,11 @@ std::expected BulkTransferOperation::stepSlo continue; } - Log::error("{}: Transfer failed for: {}: {}", logName(), displayPath(slot.entryIndex), error.toString()); + Log::error("{}: Transfer failed for: {}: {}", logName(), slot.paths.remote.value(), error.toString()); if (isSkippableError(error)) { // Not critical for the bulk as a whole: record the entry for the user and move on. - failedEntries_.emplace_back(failureSource(slot.entryIndex), error); + failedEntries_.emplace_back(failureSource(slot.paths), error); failedEntryIndices_.push_back(static_cast(slot.entryIndex)); finishSlot(slot, false); if (common_.failFast) @@ -406,7 +409,7 @@ void BulkTransferOperation::emitProgress(bool force) continue; update.inFlight.push_back( SharedData::BulkFileProgress{ - .file = displayPath(slot.entryIndex), + .file = slot.paths.remote.value(), .entryIndex = slot.entryIndex, .bytes = slot.bytes, .totalBytes = slot.totalBytes, @@ -577,13 +580,8 @@ BulkTransferOperation::entryPaths(std::size_t entryIndex) const { if (prescanned()) return prescannedPaths_[entryIndex]; - if (common_.remotePath.empty() || common_.localPath.empty()) - return std::unexpected(Error{.type = ErrorType::ImplementationError, .extraInfo = "A tree needs both roots"}); - - const auto invalidName = [](Paths::PathError const& error) - { - return Error{.type = ErrorType::InvalidName, .extraInfo = error.toString()}; - }; + return Detail::treeEntryPaths(entries_, entryIndex, common_.remotePath, common_.localPath); +} namespace Detail { @@ -626,15 +624,6 @@ namespace Detail ) .transform_error(refused); } - - return common_.localPath.join(relative) - .transform( - [this, &relative](Paths::LocalPath local) - { - return EntryPaths{.remote = common_.remotePath / relative, .local = std::move(local)}; - } - ) - .transform_error(invalidName); } std::string BulkTransferOperation::displayPath(std::size_t entryIndex) const @@ -647,27 +636,21 @@ std::string BulkTransferOperation::displayPath(std::size_t entryIndex) const std::string BulkTransferOperation::failureSource(std::size_t entryIndex) const { if (auto paths = entryPaths(entryIndex); paths) - return sourceIsRemote() ? paths->remote.value() : paths->local.toUtf8(); + return failureSource(*paths); return describeEntry(entryIndex); } +std::string BulkTransferOperation::failureSource(EntryPaths const& paths) const +{ + return sourceIsRemote() ? paths.remote.value() : paths.local.toUtf8(); +} + std::string BulkTransferOperation::describeEntry(std::size_t entryIndex) const { // The names as listed, one of them is no path segment or the entry would have paths. - std::vector names; - for (auto const* current = &entries_[entryIndex]; current->parent && *current->parent < entries_.size(); - current = &entries_[*current->parent]) - { - names.push_back(current->name); - } - auto root = sourceIsRemote() ? common_.remotePath.value() : common_.localPath.toUtf8(); - for (auto name = names.rbegin(); name != names.rend(); ++name) - { - if (!root.ends_with('/')) - root.push_back('/'); - root += *name; - } - return root; + return Utility::describeWalkedEntry( + sourceIsRemote() ? common_.remotePath.value() : common_.localPath.toUtf8(), entries_, entries_[entryIndex] + ); } void BulkTransferOperation::cancelChildrenInStrand(bool adoptCancelState) diff --git a/paths/include/paths/path_error.hpp b/paths/include/paths/path_error.hpp index 2077c366..5c0b7b76 100644 --- a/paths/include/paths/path_error.hpp +++ b/paths/include/paths/path_error.hpp @@ -46,6 +46,10 @@ namespace Paths * @brief A name longer than the 255 UTF-16 code units Windows allows. */ NameTooLong, + /** + * @brief An entry of a walked tree whose parent is not one of its entries. + */ + UnknownParent, }; inline std::string_view toString(PathRejection rejection) @@ -76,6 +80,8 @@ namespace Paths return "is a device path"; case PathRejection::NameTooLong: return "has a name that is too long"; + case PathRejection::UnknownParent: + return "lies below an entry that was not listed"; } return "is invalid"; } diff --git a/shared_data/include/shared_data/directory_entry.hpp b/shared_data/include/shared_data/directory_entry.hpp index 7a97fd64..3da03398 100644 --- a/shared_data/include/shared_data/directory_entry.hpp +++ b/shared_data/include/shared_data/directory_entry.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -251,7 +253,8 @@ namespace SharedData /** * @brief The path of @p entry below the root of its walk, following the parent indices. Empty for the root, - * which is the entry without a parent. Refused when a name on the way is no path segment. + * which is the entry without a parent. Refused when a name on the way is no path segment or a parent index + * lies outside @p entries. */ inline std::expected relativePath(std::vector const& entries, DirectoryEntry const& entry) diff --git a/shared_data/include/shared_data/sync/tree_directory_walker.hpp b/shared_data/include/shared_data/sync/tree_directory_walker.hpp index 16d26f1d..74a749c9 100644 --- a/shared_data/include/shared_data/sync/tree_directory_walker.hpp +++ b/shared_data/include/shared_data/sync/tree_directory_walker.hpp @@ -34,7 +34,7 @@ namespace SharedData::Sync * No second pass is needed. Each directory's children are assigned once, sorted, before any of * them is queued, so the queued pointers stay valid. */ - template + template requires std::is_invocable_r_v< std::expected, WalkErrorType>, ScannerT, diff --git a/shared_data/source/shared_data/ignore_rules.cpp b/shared_data/source/shared_data/ignore_rules.cpp index 7dcffaba..e072466e 100644 --- a/shared_data/source/shared_data/ignore_rules.cpp +++ b/shared_data/source/shared_data/ignore_rules.cpp @@ -1,5 +1,6 @@ #include +#include #include namespace SharedData @@ -135,9 +136,8 @@ namespace SharedData continue; const auto relToOriginStr = ruleSet.originDir.empty() - ? relPath.value() - : relPath.value().substr(ruleSet.originDir.value().size() + 1u); - const auto relToOrigin = *Paths::RelativePath::make(relToOriginStr); + ? std::string_view{relPath.value()} + : std::string_view{relPath.value()}.substr(ruleSet.originDir.value().size() + 1u); for (auto const& pattern : ruleSet.patterns) { @@ -151,14 +151,13 @@ namespace SharedData } else { - for (auto const& component : relToOrigin.segments()) - { - if (globMatch(pattern.pattern, component.view())) + matched = std::ranges::any_of( + Paths::Detail::splitAtSlashes(relToOriginStr), + [&pattern](std::string_view component) { - matched = true; - break; + return globMatch(pattern.pattern, component); } - } + ); if (!matched) matched = globMatch(pattern.pattern, relToOriginStr); } diff --git a/utility/include/utility/directory_traversal.hpp b/utility/include/utility/directory_traversal.hpp index ffd2386f..171dcdd3 100644 --- a/utility/include/utility/directory_traversal.hpp +++ b/utility/include/utility/directory_traversal.hpp @@ -1,8 +1,12 @@ #pragma once +#include #include #include +#include +#include + #include #include #include @@ -10,6 +14,8 @@ #include #include #include +#include +#include namespace Utility { @@ -35,12 +41,61 @@ namespace Utility std::uint64_t totalBytes_{0}; }; + /** + * @brief The path of @p entry below the root of its walk, following the parent indices through @p entries. + * Empty for the root, which is the entry without a parent. Refused when a name on the way is no path + * segment or a parent index lies outside @p entries. + */ + template + std::expected + walkedRelativePath(std::vector const& entries, EntryT const& entry) + { + std::vector ancestry; + for (auto const* current = &entry; current->parent; current = &entries[*current->parent]) + { + if (*current->parent >= entries.size()) + return std::unexpected( + Paths::PathError{.reason = Paths::PathRejection::UnknownParent, .input = current->name} + ); + ancestry.push_back(current); + } + Paths::RelativePath relative{}; + for (auto const* ancestor : ancestry | std::views::reverse) + { + auto segment = ancestor->nameSegment(); + if (!segment) + return std::unexpected(std::move(segment).error()); + relative = relative / *segment; + } + return relative; + } + + /** + * @brief @p entry of a walk for display: @p root followed by the names as listed on the way down to it, so it can + * name an entry whose name is no path segment. Starts below a parent index outside @p entries. + */ + template + std::string describeWalkedEntry(std::string_view root, std::vector const& entries, EntryT const& entry) + { + std::vector names; + for (auto const* current = &entry; current->parent; current = &entries[*current->parent]) + { + names.push_back(current->name); + if (*current->parent >= entries.size()) + break; + } + const auto below = fmt::format("{}", fmt::join(names | std::views::reverse, "/")); + if (root.empty() || below.empty()) + return root.empty() ? below : std::string{root}; + return root.ends_with('/') ? fmt::format("{}{}", root, below) : fmt::format("{}/{}", root, below); + } + /** * @brief Walks a directory tree breadth first. The walker never sees the root path: the scanner lists the * directory at a path relative to the root, and entries are named relative to their parent. The root - * is entry 0 and has an empty name. + * is entry 0 and has an empty name. Every "." and ".." a scanner lists is dropped. */ - template + template requires std:: is_invocable_r_v, WalkErrorType>, ScannerT, Paths::RelativePath const&> class DeepDirectoryWalker : public BaseDirectoryWalker @@ -53,6 +108,7 @@ namespace Utility , entries_{EntryT{ .type = EntryT::FileType::Directory, }} + , directoryPaths_(1) {} std::vector ejectEntries() && @@ -75,6 +131,7 @@ namespace Utility entries_ = {EntryT{ .type = EntryT::FileType::Directory, }}; + directoryPaths_.assign(1, Paths::RelativePath{}); rejectedEntries_.clear(); currentIndex_ = 0; totalBytes_ = 0; @@ -89,25 +146,17 @@ namespace Utility std::expected walk() { if (completed()) - return false; + return true; if (entries_[currentIndex_].isDirectory()) { - auto result = scanner_(relativePath(entries_[currentIndex_])); + auto result = scanner_(directoryPaths_[currentIndex_]); if (!result) return std::unexpected(std::move(result).error()); moveEntries(std::move(result).value(), currentIndex_); ++currentIndex_; } - else - { - // Log::warn( - // "DeepDirectoryWalker: Encountered non-directory entry '{}' during walk at a point where a " - // "directory was expected, skipping.", - // relativePath(entries_[currentIndex_]).value() - // ); - } // Move over all non dirs for (; currentIndex_ < entries_.size(); ++currentIndex_) @@ -123,15 +172,18 @@ namespace Utility } /** - * @brief The path of a walked entry below the root, empty for the root. + * @brief The path of a walked entry below the root, empty for the root. Refused for an entry whose parent is + * not among the walked entries. */ - Paths::RelativePath relativePath(EntryT const& entry) const + std::expected relativePath(EntryT const& entry) const { if (!entry.parent) - return {}; + return Paths::RelativePath{}; const auto parentIndex = entry.parent.value(); if (parentIndex >= entries_.size()) - throw std::out_of_range("Parent index is out of range"); + return std::unexpected( + Paths::PathError{.reason = Paths::PathRejection::UnknownParent, .input = entry.name} + ); // Only valid segments are adopted, see moveEntries. return entry.nameSegment().transform( [this, parentIndex](Paths::PathSegment const& segment) @@ -146,22 +198,19 @@ namespace Utility */ std::string describeRejected(EntryT const& entry) const { - if (!entry.parent || entry.parent.value() >= entries_.size()) - return nameOf(entry); - const auto parent = relativePath(entries_[entry.parent.value()]); - return parent.empty() ? nameOf(entry) : parent.value() + "/" + nameOf(entry); + return describeWalkedEntry({}, entries_, entry); } std::expected walkAll() { - decltype(walk()) res; - do + while (true) { - res = walk(); - if (!res) - return std::unexpected(std::move(res).error()); - } while (!res.value()); - return {}; + auto result = walk(); + if (!result) + return std::unexpected(std::move(result).error()); + if (*result) + return {}; + } } bool completed() const override @@ -189,7 +238,7 @@ namespace Utility for (auto& entry : newEntries) { entry.parent = parent; - const auto name = nameOf(entry); + auto const& name = entry.name; if (name == "." || name == "..") continue; auto segment = entry.nameSegment(); @@ -198,18 +247,18 @@ namespace Utility rejectedEntries_.push_back(std::move(entry)); continue; } + directoryPaths_.push_back( + entry.isDirectory() ? directoryPaths_[parent] / *segment : Paths::RelativePath{} + ); entries_.push_back(std::move(entry)); } } - static std::string const& nameOf(EntryT const& entry) - { - return entry.name; - } - private: ScannerT scanner_; std::vector entries_{}; + /** @brief Index aligned with entries_: the path of each directory below the root, empty for the others. */ + std::vector directoryPaths_{}; std::vector rejectedEntries_{}; }; -} \ No newline at end of file +} From f03ccfb06466cb3711da0db906393bbf2e986d4f Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:47:12 +0200 Subject: [PATCH 064/131] Remote and local scans share ScanWalkers templates The remote and the local scan create and drive their walkers through shared ScanWalkers templates, listing through a named scanner type, so every use of a scan's walker names the same walker type instead of the closure type of whichever helper created it --- .../backend/sftp/local_scan_operation.hpp | 1 + .../include/backend/sftp/scan_operation.hpp | 1 + backend/include/backend/sftp/scan_walkers.hpp | 99 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 backend/include/backend/sftp/scan_walkers.hpp diff --git a/backend/include/backend/sftp/local_scan_operation.hpp b/backend/include/backend/sftp/local_scan_operation.hpp index a9b2db96..8377d7c3 100644 --- a/backend/include/backend/sftp/local_scan_operation.hpp +++ b/backend/include/backend/sftp/local_scan_operation.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/backend/include/backend/sftp/scan_operation.hpp b/backend/include/backend/sftp/scan_operation.hpp index 090a50a5..719e5831 100644 --- a/backend/include/backend/sftp/scan_operation.hpp +++ b/backend/include/backend/sftp/scan_operation.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/backend/include/backend/sftp/scan_walkers.hpp b/backend/include/backend/sftp/scan_walkers.hpp new file mode 100644 index 00000000..f65b1e95 --- /dev/null +++ b/backend/include/backend/sftp/scan_walkers.hpp @@ -0,0 +1,99 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/** + * @brief The walkers the remote and the local scan share: one walker per scan, created on first use as a flat + * DeepDirectoryWalker or a sync TreeDirectoryWalker, both listing through the scan's scanner. + */ +namespace ScanWalkers +{ + /** + * @brief Lists a directory through @p OperationT::scanner. A named type, so every use of a scan's walker names + * the same walker type. + */ + template + struct ScannerOf + { + OperationT* operation; + + auto operator()(Paths::RelativePath const& relative) const + { + return operation->scanner(relative); + } + }; + + template + using FlatWalker = + Utility::DeepDirectoryWalker>; + + template + using TreeWalker = SharedData::Sync::TreeDirectoryWalker>; + + /** + * @brief Calls @p function with the flat walker of @p operation in @p walker, which it creates when there is none. + */ + template + decltype(auto) + withFlatWalkerDo(OperationT& operation, std::unique_ptr& walker, FunctionT&& function) + { + if (!walker) + walker = std::make_unique>(ScannerOf{&operation}); + return std::forward(function)(static_cast&>(*walker)); + } + + /** + * @brief Calls @p function with the tree walker of @p operation in @p walker, which it creates when there is + * none. A directory below the root that cannot be listed stays in the tree as unreadable. + * + * @param requireLocalNames Names must be creatable on this machine, for remote trees. + */ + template + decltype(auto) withTreeWalkerDo( + OperationT& operation, + std::unique_ptr& walker, + bool recursive, + bool requireLocalNames, + std::vector const& excludedSuffixes, + FunctionT&& function + ) + { + if (!walker) + { + walker = std::make_unique>( + ScannerOf{&operation}, + recursive, + [](Operation::Error const& error) + { + return error.type == Operation::ErrorType::DirectoryUnreadable; + }, + requireLocalNames, + excludedSuffixes + ); + } + return std::forward(function)(static_cast&>(*walker)); + } + + /** + * @brief The entries a flat walk refused for their name, below @p root by their names as listed, for display. + */ + template + std::vector rejectedPaths(WalkerT const& walker, std::string_view root) + { + std::vector paths; + paths.reserve(walker.rejectedEntries().size()); + for (auto const& entry : walker.rejectedEntries()) + paths.push_back(Utility::describeWalkedEntry(root, walker.entries(), entry)); + return paths; + } +} From 203f3f7b844e2b9d8c92adc0bc46d272120a2a8e Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:48:43 +0200 Subject: [PATCH 065/131] An archive download looks up each file root again before packing it An archive download looks up each file root again before packing it, so the tar header carries the size that is read instead of the size of an older listing, and a root that is gone or no longer a file fails the archive --- .../sftp/archive_download_operation.hpp | 5 +++- .../sftp/archive_download_operation.cpp | 28 +++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/backend/include/backend/sftp/archive_download_operation.hpp b/backend/include/backend/sftp/archive_download_operation.hpp index 6330b666..9acda6a7 100644 --- a/backend/include/backend/sftp/archive_download_operation.hpp +++ b/backend/include/backend/sftp/archive_download_operation.hpp @@ -178,7 +178,10 @@ class ArchiveDownloadOperation : public Operation }; std::expected prepareInStrand(SecureShell::SftpSession& sftp); - std::expected flattenRootsInStrand(); + /** + * @brief Resolves the roots into archive members: a file root by a fresh lstat, a directory root from its scan. + */ + std::expected flattenRootsInStrand(SecureShell::SftpSession& sftp); /** * @brief Expand a preceding ScanOperation's flat entry vector into the * op's ResolvedEntry list, prefixing each entry's tar path with diff --git a/backend/source/backend/sftp/archive_download_operation.cpp b/backend/source/backend/sftp/archive_download_operation.cpp index 4c1cde76..7e64c669 100644 --- a/backend/source/backend/sftp/archive_download_operation.cpp +++ b/backend/source/backend/sftp/archive_download_operation.cpp @@ -291,7 +291,7 @@ std::expected ArchiveDownloadOperation::prepareInStrand( } } - const auto flattenResult = flattenRootsInStrand(); + const auto flattenResult = flattenRootsInStrand(sftp); if (!flattenResult.has_value()) return std::unexpected(flattenResult.error()); @@ -335,7 +335,7 @@ std::expected ArchiveDownloadOperation::prepareInStrand( return {}; } -std::expected ArchiveDownloadOperation::flattenRootsInStrand() +std::expected ArchiveDownloadOperation::flattenRootsInStrand(SecureShell::SftpSession& sftp) { for (auto const& root : options_.roots) { @@ -352,8 +352,30 @@ std::expected ArchiveDownloadOperation::flattenRootsInSt if (root.entry.type == SharedData::FileType::Regular) { + // The listing the root came from may be old, the header must carry the size that is read. + const auto current = sftp.lstatInStrand(root.path); + if (!current) + { + Log::error( + "ArchiveDownloadOperation: cannot look up root '{}': {}", + root.path.value(), + current.error().toString() + ); + return std::unexpected( + Error{ + .type = ErrorType::FileStatFailed, + .sftpError = current.error(), + .extraInfo = root.path.value(), + } + ); + } + if (!current->isRegularFile()) + { + Log::error("ArchiveDownloadOperation: root '{}' is no longer a regular file", root.path.value()); + return std::unexpected(Error{.type = ErrorType::NotAFile, .extraInfo = root.path.value()}); + } ResolvedEntry resolved{ - .header = TarArchive::memberHeaderOf(archivePrefix.value(), root.entry), + .header = TarArchive::memberHeaderOf(archivePrefix.value(), *current), .source = root.path, }; totalPayloadBytes_ += resolved.header.size; From 422e99785106bed8bf74e581c0478e7f74a0653f Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:51:02 +0200 Subject: [PATCH 066/131] Uploads and local scans use the error code filesystem overloads Uploads and local scans inspect local files with the error code overloads instead of the throwing ones: an upload checks and opens its source in one openLocalSource, reports a missing source as FileNotFound and an unreadable status as FileStatFailed, and a local scan fails with FilesystemError on an entry it cannot inspect without an exception --- .../include/backend/sftp/upload_operation.hpp | 6 + .../backend/sftp/local_scan_operation.cpp | 109 ++++++++++----- .../source/backend/sftp/upload_operation.cpp | 124 +++++++----------- 3 files changed, 134 insertions(+), 105 deletions(-) diff --git a/backend/include/backend/sftp/upload_operation.hpp b/backend/include/backend/sftp/upload_operation.hpp index ceebb9e1..bbc0b19c 100644 --- a/backend/include/backend/sftp/upload_operation.hpp +++ b/backend/include/backend/sftp/upload_operation.hpp @@ -109,6 +109,12 @@ class UploadOperation : public Operation std::expected handleSymlink(); std::expected handleSymlinkInStrand(SecureShell::SftpSession& sftp); + /** + * @brief Checks the local source and opens it, or for a link that is not followed only notes it is one: a + * regular file or a link, no other type, that exists. + */ + std::expected openLocalSource(); + std::expected openOrAdoptFileInStrand(SecureShell::SftpSession& sftp); /// Remote-heavy portion of prepare() — runs under a single strand umbrella. diff --git a/backend/source/backend/sftp/local_scan_operation.cpp b/backend/source/backend/sftp/local_scan_operation.cpp index 94e4d031..96466259 100644 --- a/backend/source/backend/sftp/local_scan_operation.cpp +++ b/backend/source/backend/sftp/local_scan_operation.cpp @@ -87,42 +87,69 @@ LocalScanOperation::scanner(Paths::RelativePath const& relative) { if (iterErrc) return listingFailed(path, relative, iterErrc); - try + auto const& entry = *iter; + const auto failed = [this, &entry](std::error_code const& error) { - auto const& entry = *iter; - const auto status = entry.symlink_status(); - SharedData::DirectoryEntry::FileType type; + Log::error( + "LocalScanOperation: Cannot inspect '{}': {}", + Paths::nativeToUtf8(entry.path()), + Utility::errorMessage(error) + ); + return enterErrorState>( + {.type = ErrorType::FilesystemError, .extraInfo = Utility::errorMessage(error)} + ); + }; + std::error_code statusError{}; + const auto status = entry.symlink_status(statusError); + if (statusError) + return failed(statusError); + SharedData::DirectoryEntry::FileType type; #ifdef _WIN32 - // A junction is a reparse point that std::filesystem reports as a directory. - std::error_code linkError{}; - const bool isSymlink = std::filesystem::is_symlink(status) || Utility::isLinkOrJunction(entry.path(), linkError); + // A junction is a reparse point that std::filesystem reports as a directory. + std::error_code linkError{}; + const bool isSymlink = + std::filesystem::is_symlink(status) || Utility::isLinkOrJunction(entry.path(), linkError); #else - const bool isSymlink = std::filesystem::is_symlink(status); + const bool isSymlink = std::filesystem::is_symlink(status); #endif - if (isSymlink) - { + if (isSymlink) + { #ifdef _WIN32 // Windows symlinks and junctions are not transferred. A sync tree keeps them as excluded // placeholders, so the name still counts as present and the remote side is left alone. if (!buildTree_) { - // Sockets, devices and FIFOs are not uploadable, a sync tree keeps them as excluded placeholders. - if (!buildTree_) - continue; - type = SharedData::fileTypeFromStdFilesystemType(status.type()); - if (type == SharedData::DirectoryEntry::FileType::Regular || - type == SharedData::DirectoryEntry::FileType::Directory) - type = SharedData::DirectoryEntry::FileType::Unknown; + skippedLinks_.push_back(Paths::nativeToUtf8(entry.path())); + continue; } + type = SharedData::DirectoryEntry::FileType::Unknown; +#else + type = SharedData::DirectoryEntry::FileType::Symlink; +#endif + } + else if (std::filesystem::is_directory(status)) + type = SharedData::DirectoryEntry::FileType::Directory; + else if (std::filesystem::is_regular_file(status)) + type = SharedData::DirectoryEntry::FileType::Regular; + else + { + // Sockets, devices and FIFOs are not uploadable, a sync tree keeps them as excluded placeholders. + if (!buildTree_) + continue; + type = SharedData::fileTypeFromStdFilesystemType(status.type()); + if (type == SharedData::DirectoryEntry::FileType::Regular || + type == SharedData::DirectoryEntry::FileType::Directory) + type = SharedData::DirectoryEntry::FileType::Unknown; + } - std::uint64_t mtimeSecs = 0; - if (isSymlink) - { + std::uint64_t mtimeSecs = 0; + if (isSymlink) + { #ifndef _WIN32 - // For symlinks we want the link's own mtime (lstat), not the target's. - struct ::stat st{}; - if (::lstat(entry.path().c_str(), &st) == 0) - mtimeSecs = static_cast(st.st_mtime); + // For symlinks we want the link's own mtime (lstat), not the target's. + struct ::stat st{}; + if (::lstat(entry.path().c_str(), &st) == 0) + mtimeSecs = static_cast(st.st_mtime); #endif } else @@ -130,13 +157,8 @@ LocalScanOperation::scanner(Paths::RelativePath const& relative) std::error_code errc{}; const auto ftime = entry.last_write_time(errc); if (!errc) - { - // A name that is no segment cannot match a rule, the walker rejects it. - const auto segment = Paths::PathSegment::make(name); - if (segment && - ignoreMatcher_.isIgnored(relative / *segment, type == SharedData::DirectoryEntry::FileType::Directory)) - continue; - } + mtimeSecs = ComparedStateCheck::secondsSinceEpoch(ftime); + } std::optional linkTarget{}; if (isSymlink) @@ -146,7 +168,12 @@ LocalScanOperation::scanner(Paths::RelativePath const& relative) if (!errc) linkTarget = Paths::nativeToUtf8(target); } - catch (const std::filesystem::filesystem_error& e) + + auto converted = Paths::convertNativeToUtf8(entry.path().filename()); + auto const& name = converted.utf8; + if (ignoreHidden_ && !name.empty() && name.front() == '.') + continue; + if (respectIgnoreFiles_ && !ignoreMatcher_.empty()) { // A name that is no segment cannot match a rule, the walker rejects it. const auto segment = Paths::PathSegment::make(name); @@ -154,6 +181,24 @@ LocalScanOperation::scanner(Paths::RelativePath const& relative) ignoreMatcher_.isIgnored(relative / *segment, type == SharedData::DirectoryEntry::FileType::Directory)) continue; } + + // The size of what a followed link points to, like a regular file's own. + std::error_code sizeError{}; + const auto size = entry.is_regular_file(sizeError) ? entry.file_size(sizeError) : 0u; + if (sizeError && !isSymlink) + return failed(sizeError); + + entries.push_back( + SharedData::DirectoryEntry{ + .name = std::move(converted.utf8), + .nameIsValidUtf8 = converted.lossless, + .type = type, + .size = sizeError ? 0u : size, + .permissions = Utility::localPermissions(status), + .mtime = mtimeSecs, + .linkTarget = std::move(linkTarget), + } + ); } return {std::move(entries)}; } diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index dd8778ac..23c52c5a 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -79,56 +79,10 @@ std::expected UploadOperati } case (Preparing): { - // Local-only preconditions replicated from prepare(); remote work lives in - // prepareInStrand() and runs without an inner umbrella. - if (options_.localPath.empty()) + if (const auto opened = openLocalSource(); !opened) + return enterErrorState(opened.error()); + if (!isSymlink_) { - Log::error("UploadOperation: Invalid local path."); - return enterErrorState({.type = ErrorType::InvalidPath}); - } - const auto localStatus = std::filesystem::symlink_status(localNativePath_); - if (!std::filesystem::is_regular_file(localStatus) && !std::filesystem::is_symlink(localStatus)) - { - Log::error( - "UploadOperation: Local path '{}' is not a regular file or symlink.", options_.localPath.toUtf8() - ); - return enterErrorState({.type = ErrorType::NotAFile}); - } - if (std::filesystem::is_symlink(localStatus) && - options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) - { - isSymlink_ = true; - } - else - { - if (!std::filesystem::is_regular_file(std::filesystem::status(localNativePath_))) - { - Log::error( - "UploadOperation: Local path '{}' is a directory, but a file was expected.", - options_.localPath.toUtf8() - ); - return enterErrorState({.type = ErrorType::NotAFile}); - } - if (!std::filesystem::exists(localNativePath_)) - { - Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::FileNotFound}); - } - localFile_.open(localNativePath_, std::ios::binary); - if (!localFile_.is_open()) - { - Log::error("UploadOperation: Failed to open local file: {}", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::OpenFailure}); - } - localFile_.seekg(0, std::ios::end); - leftToUpload_ = static_cast(localFile_.tellg()); - totalSize_ = leftToUpload_; - localFile_.seekg(0, std::ios::beg); - if (!localFile_.good()) - { - Log::error("UploadOperation: Failed to seek in local file: {}", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::FileSeekFailure}); - } const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) { @@ -605,64 +559,88 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(SecureShell::SftpSession& s return {}; } -std::expected UploadOperation::prepare() +std::expected UploadOperation::openLocalSource() { if (options_.localPath.empty()) { Log::error("UploadOperation: Invalid local path."); - return enterErrorState({.type = ErrorType::InvalidPath}); + return std::unexpected(Error{.type = ErrorType::InvalidPath}); } - if (remotePath_.empty() || remotePath_.isRoot() || tempPath_.empty()) + std::error_code statusError{}; + const auto linkStatus = std::filesystem::symlink_status(localNativePath_, statusError); + if (linkStatus.type() == std::filesystem::file_type::not_found) { - Log::error("UploadOperation: Invalid remote path '{}'.", options_.remotePath.value()); - return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = options_.remotePath.value()}); + Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); + return std::unexpected(Error{.type = ErrorType::FileNotFound}); } - const auto localStatus = std::filesystem::symlink_status(localNativePath_); - if (!std::filesystem::is_regular_file(localStatus) && !std::filesystem::is_symlink(localStatus)) + if (statusError) + { + Log::error( + "UploadOperation: Cannot inspect '{}': {}", options_.localPath.toUtf8(), Utility::errorMessage(statusError) + ); + return std::unexpected( + Error{.type = ErrorType::FileStatFailed, .extraInfo = Utility::errorMessage(statusError)} + ); + } + if (!std::filesystem::is_regular_file(linkStatus) && !std::filesystem::is_symlink(linkStatus)) { Log::error("UploadOperation: Local path '{}' is not a regular file or symlink.", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::NotAFile}); + return std::unexpected(Error{.type = ErrorType::NotAFile}); } - - if (std::filesystem::is_symlink(localStatus) && + if (std::filesystem::is_symlink(linkStatus) && options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) { isSymlink_ = true; return {}; } - if (!std::filesystem::is_regular_file(std::filesystem::status(localNativePath_))) + // A followed link must point to an existing regular file. + const auto status = std::filesystem::status(localNativePath_, statusError); + if (status.type() == std::filesystem::file_type::not_found) { - Log::error( - "UploadOperation: Local path '{}' is a directory, but a file was expected.", options_.localPath.toUtf8() - ); - return enterErrorState({.type = ErrorType::NotAFile}); + Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); + return std::unexpected(Error{.type = ErrorType::FileNotFound}); } - - // Initial check. Check again later before rename - if (!std::filesystem::exists(localNativePath_)) + if (statusError || !std::filesystem::is_regular_file(status)) { - Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::FileNotFound}); + Log::error("UploadOperation: Local path '{}' is not a regular file.", options_.localPath.toUtf8()); + return std::unexpected(Error{.type = ErrorType::NotAFile}); } localFile_.open(localNativePath_, std::ios::binary); if (!localFile_.is_open()) { Log::error("UploadOperation: Failed to open local file: {}", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::OpenFailure}); + return std::unexpected(Error{.type = ErrorType::OpenFailure}); } - localFile_.seekg(0, std::ios::end); leftToUpload_ = static_cast(localFile_.tellg()); totalSize_ = leftToUpload_; localFile_.seekg(0, std::ios::beg); - if (!localFile_.good()) { Log::error("UploadOperation: Failed to seek in local file: {}", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::FileSeekFailure}); + return std::unexpected(Error{.type = ErrorType::FileSeekFailure}); } + return {}; +} + +std::expected UploadOperation::prepare() +{ + if (options_.localPath.empty()) + { + Log::error("UploadOperation: Invalid local path."); + return enterErrorState({.type = ErrorType::InvalidPath}); + } + if (remotePath_.empty() || remotePath_.isRoot() || tempPath_.empty()) + { + Log::error("UploadOperation: Invalid remote path '{}'.", options_.remotePath.value()); + return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = options_.remotePath.value()}); + } + if (const auto opened = openLocalSource(); !opened) + return enterErrorState(opened.error()); + if (isSymlink_) + return {}; const auto sftp = openSession(sftp_); if (!sftp) From 477f6ea9803c638e547082a6772d6cf74e4f4d34 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:52:47 +0200 Subject: [PATCH 067/131] A finished delete card no longer shows its scan status A finished delete card no longer shows its scan status, like the transfer and bulk cards; the three cards that scan first share their scan status and its progress setter through ScanningCard --- .../displayed_bulk_operation.hpp | 13 ++++-------- .../displayed_delete_operation.hpp | 17 ++++++---------- .../displayed_transfer_operation.hpp | 13 ++++-------- .../operation_queue/scan_phase_status.hpp | 20 +++++++++++++++++++ 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp index 04e12887..16d386a6 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp @@ -13,7 +13,9 @@ #include #include -struct DisplayedBulkOperation : public OperationCard +struct DisplayedBulkOperation + : public OperationCard + , public ScanningCard { public: DisplayedBulkOperation( @@ -35,9 +37,9 @@ struct DisplayedBulkOperation : public OperationCard std::move(doDeletionCountdown), std::move(onCompleteAction) } + , ScanningCard{scanning} , localPath_{std::move(localPath)} , remotePath_{std::move(remotePath)} - , scanPhase_{scanning} , totalProgressBar_({ .height = std::string{progressHeight}, .min = 0, @@ -52,12 +54,6 @@ struct DisplayedBulkOperation : public OperationCard return true; } - void setScanProgress(std::uint64_t scannedBytes, std::uint64_t scannedEntries) - { - scanPhase_.setProgress(scannedBytes, scannedEntries); - Nui::globalEventContext.executeActiveEventsImmediately(); - } - void setProgress(SharedData::BulkProgress const& progress) { scanPhase_.end(); @@ -342,7 +338,6 @@ struct DisplayedBulkOperation : public OperationCard private: Paths::LocalPath localPath_; Paths::RemotePath remotePath_; - ScanPhaseStatus scanPhase_; Nui::Observed fileCurrentIndex{0ull}; Nui::Observed fileCount{0ull}; Nui::Observed> bytesPerSecond{0}; diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp index d5610706..b2d67175 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp @@ -6,7 +6,9 @@ #include #include -struct DisplayedDeleteOperation : public OperationCard +struct DisplayedDeleteOperation + : public OperationCard + , public ScanningCard { public: DisplayedDeleteOperation( @@ -28,10 +30,10 @@ struct DisplayedDeleteOperation : public OperationCard std::move(doDeletionCountdown), std::move(onCompleteAction) } + , ScanningCard{scanning} , displayPath_{std::move(displayPath)} , target_{std::move(target)} , recursive_{scanning} - , scanPhase_{scanning} , progressBar_({ .height = std::string{progressHeight}, .min = 0, @@ -53,12 +55,6 @@ struct DisplayedDeleteOperation : public OperationCard progressBar_.setZeroAsComplete(); } - void setScanProgress(std::uint64_t scannedBytes, std::uint64_t scannedEntries) - { - scanPhase_.setProgress(scannedBytes, scannedEntries); - Nui::globalEventContext.executeActiveEventsImmediately(); - } - void setProgress(SharedData::BulkDeleteProgress const& progress) { scanPhase_.end(); @@ -123,9 +119,9 @@ struct DisplayedDeleteOperation : public OperationCard span{ class_ = "opq-status-text" }( - observe(filesDeleted_, totalFiles_, scanPhase_.scanning, scanPhase_.totalScanned), + observe(filesDeleted_, totalFiles_, state_, scanPhase_.scanning, scanPhase_.totalScanned), [this](){ - if (scanPhase_.scanning.value()) + if (scanPhase_.scanning.value() && !isCompletedState()) return scanPhase_.text(); return fmt::format("{}/{}", filesDeleted_.value(), totalFiles_.value()); } @@ -145,6 +141,5 @@ struct DisplayedDeleteOperation : public OperationCard Paths::RemotePath target_; /** @brief The delete scanned a directory, so it removes content and resumes doing the same. */ bool recursive_{false}; - ScanPhaseStatus scanPhase_; Components::ProgressBar progressBar_; }; \ No newline at end of file diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp index 43ca67b9..4fc8f4c6 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp @@ -7,7 +7,9 @@ #include -class DisplayedTransferOperation : public OperationCard +class DisplayedTransferOperation + : public OperationCard + , public ScanningCard { public: DisplayedTransferOperation( @@ -30,6 +32,7 @@ class DisplayedTransferOperation : public OperationCard totalBytes{0u}; Nui::Observed totalScanned{0u}; }; + +/** + * @brief The part of a card whose operation scans before its own work: the scan status and its progress setter. + */ +class ScanningCard +{ + public: + explicit ScanningCard(bool scanning) + : scanPhase_{scanning} + {} + + void setScanProgress(std::uint64_t scannedBytes, std::uint64_t scannedEntries) + { + scanPhase_.setProgress(scannedBytes, scannedEntries); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + + protected: + ScanPhaseStatus scanPhase_; +}; From 1b64dbd5403faf9d2a91ef42d3bb77bc16b5c0fd Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:07:58 +0200 Subject: [PATCH 068/131] Sync remembers nothing across runs Sync remembers nothing across runs: the per-pair baseline files, their pruning, the legacy last sync time and the blocks for entries changed since the last sync are gone, and an entry that differs on both sides goes from the newer side to the other one again --- .../backend/sync/sync_baseline_store.hpp | 67 ----- backend/source/backend/main.cpp | 10 - .../source/backend/sftp/operation_queue.cpp | 106 +------- .../backend/sync/sync_baseline_store.cpp | 228 ------------------ .../frontend/sync_dialog/sync_dialog.hpp | 5 +- .../frontend/sync_dialog/sync_dialog.cpp | 10 +- .../include/persistence/state/state.hpp | 4 +- shared_data/include/shared_data/sync/diff.hpp | 18 +- .../include/shared_data/sync/diff_summary.hpp | 11 +- .../shared_data/sync/sync_baseline.hpp | 76 ------ .../source/shared_data/sync/sync_baseline.cpp | 147 ----------- static/assets/languages/de_DE.yaml | 4 +- static/assets/languages/en_US.yaml | 4 +- 13 files changed, 20 insertions(+), 670 deletions(-) delete mode 100644 backend/include/backend/sync/sync_baseline_store.hpp delete mode 100644 backend/source/backend/sync/sync_baseline_store.cpp delete mode 100644 shared_data/include/shared_data/sync/sync_baseline.hpp delete mode 100644 shared_data/source/shared_data/sync/sync_baseline.cpp diff --git a/backend/include/backend/sync/sync_baseline_store.hpp b/backend/include/backend/sync/sync_baseline_store.hpp deleted file mode 100644 index e7a847b5..00000000 --- a/backend/include/backend/sync/sync_baseline_store.hpp +++ /dev/null @@ -1,67 +0,0 @@ -#pragma once - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -/** - * @brief Keeps the @ref SharedData::Sync::SyncBaseline of every sync pair, one JSON file per pair in a directory of - * its own, named by a hash of the pair's session and one of the pair key. The file also holds the key, so a - * hash collision reads as no baseline instead of another pair's records. - * - * Every access goes to the file under one mutex shared by all stores, so sync sessions and finished operations of - * the same pair can update it from any thread without losing each other's changes. - */ -class SyncBaselineStore -{ - public: - explicit SyncBaselineStore(std::filesystem::path directory); - - /** - * @brief The store below the state directory of the program. - */ - static std::filesystem::path defaultDirectory(); - - /** - * @brief Names a pair: the saved session, the local root and the remote root. - */ - static std::string - pairKey(std::string_view sessionName, Paths::LocalPath const& localRoot, Paths::RemotePath const& remoteRoot); - - /** - * @brief The file of @p pairKey below @p directory. - */ - static std::filesystem::path fileOf(std::filesystem::path const& directory, std::string_view pairKey); - - /** - * @brief Removes the baselines of every pair whose session is not one of @p sessionNames, like a deleted or - * renamed session. - * @return How many were removed. - */ - std::size_t pruneAllBut(std::vector const& sessionNames) const; - - /** - * @brief The baseline of @p pairKey, nothing when the pair was never synchronized. A file that cannot be read - * counts as a pair with history but no records, which protects every delete. - */ - std::optional load(std::string const& pairKey) const; - - /** - * @brief Reads the baseline of @p pairKey (empty when there is none), lets @p change modify it and writes it - * back, which also creates the file of a pair synchronized for the first time. - * @return false when it could not be written. - */ - bool update(std::string const& pairKey, std::function const& change) const; - - private: - std::optional loadLocked(std::string const& pairKey) const; - - std::filesystem::path directory_; -}; diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index d5bbb384..5b34c19e 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -40,11 +39,9 @@ #include #include #include -#include #include #include #include -#include #include #ifndef _WIN32 @@ -323,13 +320,6 @@ void Main::registerRpc() } sshSessionManager_->preventLocalDeletion(holder.stateCache().localFilesystemOptions.preventDeletion); - { - auto const& sessions = holder.stateCache().sessions; - std::vector sessionNames; - sessionNames.reserve(sessions.size()); - std::ranges::copy(std::views::keys(sessions), std::back_inserter(sessionNames)); - SyncBaselineStore{SyncBaselineStore::defaultDirectory()}.pruneAllBut(sessionNames); - } rpcFilesystem_ = std::make_unique( window_.getExecutor(), window_, diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index cba85af3..0cc697d0 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include using namespace std::chrono_literals; @@ -297,7 +296,6 @@ void OperationQueue::cancelAll() self->operations_.clear(); self->priorityOperations_.clear(); - self->syncRecordings_.clear(); auto waiting = std::exchange(self->syncScanCallbacks_, {}); for (auto& [id, callback] : waiting) callback(std::unexpected(Utility::localizedMessage("backend.sync.scanCancelled"))); @@ -437,7 +435,6 @@ void OperationQueue::completeOperation(SharedData::OperationCompleted&& operatio // a backup for reconnect adoption. self->bulkResumeStash_.erase(operationCompleted.operationId); - self->recordSyncCompletion(operationCompleted); if (self->completionObserver_) self->completionObserver_(operationCompleted); RpcHelper::callRemoteSafely(*self->hub_, self->rpcName("onOperationCompleted"), operationCompleted); @@ -1703,17 +1700,6 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( result.failed.push_back(std::move(entry.relKey)); }; - auto const& pairKey = session.baselineKey(); - auto const& baselines = session.baselines(); - const auto makeRecording = [&]() - { - return SyncRecording{.pairKey = pairKey.value_or(std::string{}), .store = baselines}; - }; - auto uploadRecording = makeRecording(); - auto downloadRecording = makeRecording(); - auto remoteDeleteRecording = makeRecording(); - auto localDeleteRecording = makeRecording(); - for (auto& planned : plan.entries) { auto& entry = planned.entry; @@ -1722,7 +1708,6 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( { case Action::Upload: { - uploadRecording.entries.emplace_back(entry.relKey, std::move(planned.record)); uploads.entries.push_back({ .source = std::move(paths.local), .destination = std::move(paths.remote), @@ -1737,7 +1722,6 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( } case Action::Download: { - downloadRecording.entries.emplace_back(entry.relKey, std::move(planned.record)); downloads.entries.push_back({ .source = std::move(paths.remote), .destination = std::move(paths.local), @@ -1762,8 +1746,6 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( refuse(entry, refusedRemoteDirectories, "remotely", allowed.error().toString()); break; } - remoteDeleteRecording.entries.emplace_back(entry.relKey, std::nullopt); - remoteDeleteRecording.sources.push_back(paths.remote.value()); result.remoteDeleteSources.push_back(paths.remote.value()); remoteDeletes.entries.push_back({ .target = std::move(paths.remote), @@ -1785,8 +1767,6 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( refuse(entry, refusedLocalDirectories, "locally", allowed.error()); break; } - localDeleteRecording.entries.emplace_back(entry.relKey, std::nullopt); - localDeleteRecording.sources.push_back(paths.local.toUtf8()); result.localDeleteSources.push_back(paths.local.toUtf8()); localDeletes.push_back(std::move(paths.local)); result.localDeletes.push_back(std::move(entry.relKey)); @@ -1841,27 +1821,6 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( result.localDeleteSources.clear(); } - if (pairKey && baselines) - { - baselines->update( - *pairKey, - [&plan](SharedData::Sync::SyncBaseline& baseline) - { - for (auto& [relKey, record] : plan.agreement.records) - baseline.records.insert_or_assign(relKey, std::move(record)); - } - ); - const auto keep = [this](Ids::OperationId const& operationId, bool queued, SyncRecording recording) - { - if (queued && !recording.entries.empty()) - syncRecordings_.insert_or_assign(operationId.value(), std::move(recording)); - }; - keep(request.uploadId, !result.uploads.empty(), std::move(uploadRecording)); - keep(request.downloadId, !result.downloads.empty(), std::move(downloadRecording)); - keep(request.remoteDeleteId, !result.remoteDeletes.empty(), std::move(remoteDeleteRecording)); - keep(request.localDeleteId, !result.localDeletes.empty(), std::move(localDeleteRecording)); - } - Log::info( "Sync: queued {} uploads, {} downloads, {} remote and {} local deletes, {} failed", result.uploads.size(), @@ -2204,22 +2163,16 @@ void OperationQueue::addSyncScanOperation( Paths::LocalPath const& localPath, bool respectIgnoreFiles, bool recursive, - bool ignoreHidden, - std::optional const& sessionName + bool ignoreHidden ) { // Create (or replace) the SyncSession. A fresh id from the frontend is the // norm; re-opening over an existing id is handled as "discard previous state". - const bool keepsBaseline = sessionName && !sessionName->empty() && syncBaselines_; auto session = std::make_shared( SyncSession::Options{ .sessionId = syncSessionId, .localRoot = localPath, .remoteRoot = remotePath, - .baselineKey = keepsBaseline - ? std::optional{SyncBaselineStore::pairKey(*sessionName, localPath, remotePath)} - : std::nullopt, - .baselines = keepsBaseline ? syncBaselines_ : nullptr, }, executor_ ); @@ -2336,63 +2289,6 @@ auto OperationQueue::makeSyncScanCallback( }; } -void OperationQueue::setSyncBaselineStore(std::shared_ptr store) -{ - syncBaselines_ = std::move(store); -} - -void OperationQueue::recordSyncCompletion(OperationCompleted const& completed) -{ - auto found = syncRecordings_.extract(completed.operationId.value()); - if (found.empty() || completed.reason != SharedData::OperationCompletionReason::Completed) - return; - auto& recording = found.mapped(); - - std::vector failed(recording.entries.size(), false); - for (const auto index : completed.failedEntryIndices) - { - if (index < failed.size()) - failed[index] = true; - } - if (!recording.sources.empty() && !completed.failedEntries.empty()) - { - std::unordered_set failedSources; - for (auto const& [source, error] : completed.failedEntries) - failedSources.insert(source); - for (std::size_t index = 0; index < recording.sources.size() && index < failed.size(); ++index) - failed[index] = failed[index] || failedSources.contains(recording.sources[index]); - } - - recording.store->update( - recording.pairKey, - [&recording, &failed](SharedData::Sync::SyncBaseline& baseline) - { - const SharedData::Sync::SyncRecord directory{ - .local = {.type = SharedData::FileType::Directory}, - .remote = {.type = SharedData::FileType::Directory}, - }; - for (std::size_t index = 0; index < recording.entries.size(); ++index) - { - if (failed[index]) - continue; - auto const& [relKey, record] = recording.entries[index]; - if (!record) - { - baseline.records.erase(relKey); - continue; - } - baseline.records.insert_or_assign(relKey, *record); - // The transfer created or found every directory above it on both sides. - for ( - auto slash = relKey.rfind('/'); slash != std::string::npos && slash > 0; - slash = relKey.rfind('/', slash - 1) - ) - baseline.records.insert_or_assign(relKey.substr(0, slash), directory); - } - } - ); -} - std::shared_ptr OperationQueue::syncSession(Ids::SyncSessionId const& sessionId) const { const auto iter = syncSessions_.find(sessionId.value()); diff --git a/backend/source/backend/sync/sync_baseline_store.cpp b/backend/source/backend/sync/sync_baseline_store.cpp deleted file mode 100644 index 9ec730aa..00000000 --- a/backend/source/backend/sync/sync_baseline_store.cpp +++ /dev/null @@ -1,228 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace -{ - constexpr int formatVersion = 1; - - std::mutex& storeMutex() - { - static std::mutex mutex; - return mutex; - } - - /** - * @brief The session part of a pair key, see SyncBaselineStore::pairKey. - */ - std::string_view sessionOf(std::string_view pairKey) - { - return pairKey.substr(0, pairKey.find('\n')); - } - - std::uint64_t fnv1a(std::string_view text) - { - std::uint64_t hash = 14695981039346656037ull; - for (const auto character : text) - { - hash ^= static_cast(character); - hash *= 1099511628211ull; - } - return hash; - } - - /** - * @brief The file's JSON, nothing when it does not exist, an error text when it cannot be read. - */ - std::expected, std::string> readJson(std::filesystem::path const& file) - { - std::error_code error{}; - if (!std::filesystem::exists(file, error)) - { - if (error) - return std::unexpected(Utility::errorMessage(error)); - return std::nullopt; - } - std::ifstream stream{file, std::ios::binary}; - if (!stream) - return std::unexpected(std::string{"cannot open"}); - auto json = nlohmann::json::parse(stream, nullptr, false); - if (json.is_discarded()) - return std::unexpected(std::string{"not JSON"}); - return json; - } -} - -SyncBaselineStore::SyncBaselineStore(std::filesystem::path directory) - : directory_{std::move(directory)} -{} - -std::filesystem::path SyncBaselineStore::defaultDirectory() -{ - return Nui::resolvePath(std::filesystem::path{std::string{Constants::syncBaselinePath}}); -} - -std::string SyncBaselineStore::pairKey( - std::string_view sessionName, - Paths::LocalPath const& localRoot, - Paths::RemotePath const& remoteRoot -) -{ - return fmt::format("{}\n{}\n{}", sessionName, localRoot.toUtf8(), remoteRoot.value()); -} - -std::filesystem::path SyncBaselineStore::fileOf(std::filesystem::path const& directory, std::string_view pairKey) -{ - return directory / fmt::format("{:016x}-{:016x}.json", fnv1a(sessionOf(pairKey)), fnv1a(pairKey)); -} - -std::size_t SyncBaselineStore::pruneAllBut(std::vector const& sessionNames) const -{ - std::scoped_lock lock{storeMutex()}; - std::unordered_set kept; - for (auto const& name : sessionNames) - kept.insert(fmt::format("{:016x}-", fnv1a(name))); - - std::vector orphans; - std::error_code error{}; - for ( - auto iter = std::filesystem::directory_iterator{directory_, error}; - !error && iter != std::filesystem::directory_iterator{}; - iter.increment(error) - ) - { - const auto name = Paths::nativeToUtf8(iter->path().filename()); - // Session hash, a dash, pair hash and ".json". - if (name.size() == 16 + 1 + 16 + 5 && name.ends_with(".json") && !kept.contains(name.substr(0, 17))) - orphans.push_back(iter->path()); - } - - std::size_t removed = 0; - for (auto const& orphan : orphans) - { - std::error_code removeError{}; - if (std::filesystem::remove(orphan, removeError)) - ++removed; - else if (removeError) - Log::warn( - "Cannot remove the sync baseline '{}': {}", - Paths::nativeToUtf8(orphan), - Utility::errorMessage(removeError) - ); - } - if (removed > 0) - Log::info("Removed {} sync baselines of sessions that no longer exist.", removed); - return removed; -} - -std::optional SyncBaselineStore::loadLocked(std::string const& pairKey) const -{ - const auto file = fileOf(directory_, pairKey); - const auto json = readJson(file); - if (!json) - { - Log::error( - "Sync baseline '{}' cannot be read ({}), every delete of its pair is refused.", - Paths::nativeToUtf8(file), - json.error() - ); - return SharedData::Sync::SyncBaseline{}; - } - if (!*json) - return std::nullopt; - auto const& content = **json; - if (!content.is_object() || content.value("pair", std::string{}) != pairKey) - { - Log::warn("Sync baseline '{}' belongs to another pair, ignored.", Paths::nativeToUtf8(file)); - return std::nullopt; - } - try - { - return content.get(); - } - catch (std::exception const& exception) - { - Log::error( - "Sync baseline '{}' is damaged ({}), every delete of its pair is refused.", - Paths::nativeToUtf8(file), - Utility::errorMessage(exception) - ); - return SharedData::Sync::SyncBaseline{}; - } -} - -std::optional SyncBaselineStore::load(std::string const& pairKey) const -{ - std::scoped_lock lock{storeMutex()}; - return loadLocked(pairKey); -} - -bool SyncBaselineStore::update( - std::string const& pairKey, - std::function const& change -) const -{ - std::scoped_lock lock{storeMutex()}; - const auto file = fileOf(directory_, pairKey); - if (const auto existing = readJson(file); - existing && *existing && (*existing)->value("pair", std::string{}) != pairKey) - { - Log::error("Sync baseline '{}' belongs to another pair, not replaced.", Paths::nativeToUtf8(file)); - return false; - } - - auto baseline = loadLocked(pairKey).value_or(SharedData::Sync::SyncBaseline{}); - change(baseline); - - nlohmann::json json = baseline; - json["pair"] = pairKey; - json["version"] = formatVersion; - - std::error_code error{}; - std::filesystem::create_directories(directory_, error); - if (error) - { - Log::error( - "Cannot create the sync baseline directory '{}': {}", - Paths::nativeToUtf8(directory_), - Utility::errorMessage(error) - ); - return false; - } - auto temporary = file; - temporary += ".part"; - { - std::ofstream stream{temporary, std::ios::binary | std::ios::trunc}; - stream << json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); - if (!stream.flush()) - { - Log::error("Cannot write the sync baseline '{}'.", Paths::nativeToUtf8(temporary)); - return false; - } - } - if (auto moved = Utility::moveIntoPlace(temporary, file, /*replace=*/true); !moved) - { - Log::error( - "Cannot replace the sync baseline '{}': {}", Paths::nativeToUtf8(file), Utility::errorMessage(moved.error()) - ); - std::filesystem::remove(temporary, error); - return false; - } - return true; -} diff --git a/frontend/include/frontend/sync_dialog/sync_dialog.hpp b/frontend/include/frontend/sync_dialog/sync_dialog.hpp index e9a2570a..21f5788a 100644 --- a/frontend/include/frontend/sync_dialog/sync_dialog.hpp +++ b/frontend/include/frontend/sync_dialog/sync_dialog.hpp @@ -9,8 +9,6 @@ #include #include -#include -#include #include #include @@ -54,8 +52,7 @@ class SyncDialog SharedData::Sync::DiffSnapshot snapshot, std::filesystem::path localPath, std::filesystem::path remotePath, - ScanSettings scanned, - std::optional lastSyncTime + ScanSettings scanned ); /** diff --git a/frontend/source/frontend/sync_dialog/sync_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_dialog.cpp index 6e5a0e85..384c7014 100644 --- a/frontend/source/frontend/sync_dialog/sync_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_dialog.cpp @@ -837,7 +837,6 @@ struct SyncDialog::Implementation .actionDelete = actionDelete_.value(), .recursive = recursive_.value(), .ignoreHidden = ignoreHidden_.value(), - .lastSyncTime = lastSyncTime_, }; } @@ -1441,11 +1440,6 @@ SyncDialog::~SyncDialog() SyncDialog::SyncDialog(SyncDialog&&) = default; SyncDialog& SyncDialog::operator=(SyncDialog&&) = default; -void SyncDialog::setOnSynchronized(std::function callback) -{ - impl_->onSynchronized_ = std::move(callback); -} - void SyncDialog::setOnRecompareRequested(std::function callback) { impl_->onRecompareRequested_ = std::move(callback); @@ -1461,15 +1455,13 @@ void SyncDialog::open( SharedData::Sync::DiffSnapshot snapshot, std::filesystem::path localPath, std::filesystem::path remotePath, - ScanSettings scanned, - std::optional lastSyncTime + ScanSettings scanned ) { if (impl_->provider_ != provider) impl_->dropRegistrations(); impl_->provider_ = std::move(provider); impl_->diffCurrent_ = true; - impl_->lastSyncTime_ = lastSyncTime; impl_->localPath_ = std::move(localPath); impl_->remotePath_ = std::move(remotePath); impl_->scanned_ = scanned; diff --git a/persistence/include/persistence/state/state.hpp b/persistence/include/persistence/state/state.hpp index 098b2c91..364ccd30 100644 --- a/persistence/include/persistence/state/state.hpp +++ b/persistence/include/persistence/state/state.hpp @@ -14,7 +14,6 @@ #include #include -#include #include #include @@ -90,7 +89,6 @@ namespace Persistence fileTrackingOptions, localFilesystemOptions, lastDirectConnect, - lastDirectConnectInheritables, - syncBaselines) + lastDirectConnectInheritables) ) } \ No newline at end of file diff --git a/shared_data/include/shared_data/sync/diff.hpp b/shared_data/include/shared_data/sync/diff.hpp index 9aa7e596..9e50265c 100644 --- a/shared_data/include/shared_data/sync/diff.hpp +++ b/shared_data/include/shared_data/sync/diff.hpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -33,19 +32,28 @@ namespace SharedData::Sync bool recursive{true}; /// When true, entries with any path segment starting with '.' are dropped. bool ignoreHidden{false}; - /// The local filesystem folds case (Windows, macOS), set by the backend. Remote names that differ only in - /// case would land on one local file then. + /** + * @brief The local filesystem folds case (Windows, macOS), set by the backend. Names that differ only in case + * or normalization would land on one local file then. + */ bool localCaseInsensitive{false}; /** * @brief The server looks like it folds case (Windows or macOS paths), set by the backend. Names that differ * only in case or normalization would land on one remote file then. */ - std::optional lastSyncTime{}; + bool remoteCaseInsensitive{false}; }; BOOST_DESCRIBE_STRUCT( DiffOptions, (), - (direction, actionUpload, actionDownload, actionDelete, recursive, ignoreHidden, localCaseInsensitive, lastSyncTime) + (direction, + actionUpload, + actionDownload, + actionDelete, + recursive, + ignoreHidden, + localCaseInsensitive, + remoteCaseInsensitive) ) /** diff --git a/shared_data/include/shared_data/sync/diff_summary.hpp b/shared_data/include/shared_data/sync/diff_summary.hpp index d8d29dac..bee72953 100644 --- a/shared_data/include/shared_data/sync/diff_summary.hpp +++ b/shared_data/include/shared_data/sync/diff_summary.hpp @@ -13,16 +13,7 @@ namespace SharedData::Sync /** * @brief Why the diff left an entry alone even though it exists on one or both sides. */ - BOOST_DEFINE_ENUM_CLASS( - BlockReason, - Unreadable, - Excluded, - InvalidName, - TypeConflict, - CaseConflict, - ChangedOnBothSides, - ChangedOnDestination - ) + BOOST_DEFINE_ENUM_CLASS(BlockReason, Unreadable, Excluded, InvalidName, TypeConflict, CaseConflict) /** * @brief An entry sync does not act on, with the reason. diff --git a/shared_data/include/shared_data/sync/sync_baseline.hpp b/shared_data/include/shared_data/sync/sync_baseline.hpp deleted file mode 100644 index ae4bcedc..00000000 --- a/shared_data/include/shared_data/sync/sync_baseline.hpp +++ /dev/null @@ -1,76 +0,0 @@ -#pragma once - -#include - -#include - -#include -#include -#include -#include - -namespace SharedData::Sync -{ - /** - * @brief One side of an entry as sync left it, compared with the next scan of the same side only, so the - * clocks of the two sides never meet. - */ - struct RecordedSide - { - FileType type{FileType::Unknown}; - std::uint64_t size{0}; - std::uint64_t mtime{0}; - /** @brief The link literal of a symlink, which compares by it instead of size and time. */ - std::optional linkTarget{}; - - bool operator==(RecordedSide const&) const = default; - }; - - /** - * @brief Both sides of an entry after it was last synchronized or found in agreement. - */ - struct SyncRecord - { - RecordedSide local{}; - RecordedSide remote{}; - - bool operator==(SyncRecord const&) const = default; - }; - - /** - * @brief What sync knows about one pair (saved session, local root, remote root) from earlier runs: a record per - * relKey for every entry that was synchronized successfully or found equal on both sides. An entry without - * a record was never in agreement. - */ - struct SyncBaseline - { - std::unordered_map records{}; - - bool operator==(SyncBaseline const&) const = default; - }; - - /** - * @brief How @p node is recorded: type, size and modification time, the link literal for a symlink. - */ - RecordedSide recordedSideOf(ScanNode const& node); - - /** - * @brief True when @p current is not what @p recorded holds. A directory changes only by becoming something - * else, its modification time and size do not count. A symlink compares by its literal. - */ - bool sideChanged(RecordedSide const& recorded, ScanNode const& current); - - /** - * @brief The record of an entry transferred from @p source: both sides hold what the source held. - */ - SyncRecord transferredRecordOf(ScanNode const& source); - - /** - * @brief Records every entry listed on both sides that the diff sees as equal, directories included, walking - * into directories present on both sides. - */ - SyncBaseline agreementOf(ScanNode const& local, ScanNode const& remote); - - void to_json(nlohmann::json& json, SyncBaseline const& baseline); - void from_json(nlohmann::json const& json, SyncBaseline& baseline); -} diff --git a/shared_data/source/shared_data/sync/sync_baseline.cpp b/shared_data/source/shared_data/sync/sync_baseline.cpp deleted file mode 100644 index bd78ef5b..00000000 --- a/shared_data/source/shared_data/sync/sync_baseline.cpp +++ /dev/null @@ -1,147 +0,0 @@ -#include -#include - -#include -#include - -namespace SharedData::Sync -{ - namespace - { - std::string joinRelKey(std::string const& parent, std::string const& name) - { - return parent.empty() ? name : fmt::format("{}/{}", parent, name); - } - - void collectAgreement( - std::vector const& localChildren, - std::vector const& remoteChildren, - std::string const& parentRelKey, - SyncBaseline& baseline - ) - { - auto localIter = localChildren.begin(); - auto remoteIter = remoteChildren.begin(); - while (localIter != localChildren.end() && remoteIter != remoteChildren.end()) - { - if (localIter->name < remoteIter->name) - { - ++localIter; - continue; - } - if (remoteIter->name < localIter->name) - { - ++remoteIter; - continue; - } - auto const& local = *localIter++; - auto const& remote = *remoteIter++; - const bool listed = - local.availability == Availability::Listed && remote.availability == Availability::Listed; - const bool directories = local.type == FileType::Directory && remote.type == FileType::Directory; - if (!listed || local.type != remote.type || (!directories && entriesDiffer(local, remote))) - continue; - auto relKey = joinRelKey(parentRelKey, local.name); - if (directories) - collectAgreement(local.children, remote.children, relKey, baseline); - baseline.records.insert_or_assign( - std::move(relKey), SyncRecord{.local = recordedSideOf(local), .remote = recordedSideOf(remote)} - ); - } - } - - nlohmann::json sideToJson(RecordedSide const& side) - { - return nlohmann::json::array({static_cast(side.type), side.size, side.mtime}); - } - - RecordedSide sideFromJson(nlohmann::json const& json, std::size_t offset) - { - return RecordedSide{ - .type = static_cast(json.at(offset).get()), - .size = json.at(offset + 1).get(), - .mtime = json.at(offset + 2).get(), - }; - } - - std::optional linkTargetFromJson(nlohmann::json const& json, std::size_t index) - { - if (json.size() <= index || json.at(index).is_null()) - return std::nullopt; - return json.at(index).get(); - } - } - - RecordedSide recordedSideOf(ScanNode const& node) - { - return RecordedSide{ - .type = node.type, - .size = node.type == FileType::Directory ? 0u : node.size, - .mtime = node.type == FileType::Directory ? 0u : node.mtime, - .linkTarget = node.type == FileType::Symlink ? node.linkTarget : std::nullopt, - }; - } - - bool sideChanged(RecordedSide const& recorded, ScanNode const& current) - { - if (recorded.type != current.type) - return true; - switch (current.type) - { - case FileType::Directory: - return false; - case FileType::Symlink: - return recorded.linkTarget != current.linkTarget; - default: - return recorded.size != current.size || recorded.mtime != current.mtime; - } - } - - SyncRecord transferredRecordOf(ScanNode const& source) - { - const auto side = recordedSideOf(source); - return SyncRecord{.local = side, .remote = side}; - } - - SyncBaseline agreementOf(ScanNode const& local, ScanNode const& remote) - { - SyncBaseline baseline; - collectAgreement(local.children, remote.children, std::string{}, baseline); - return baseline; - } - - void to_json(nlohmann::json& json, SyncBaseline const& baseline) - { - auto records = nlohmann::json::object(); - for (auto const& [relKey, record] : baseline.records) - { - auto entry = sideToJson(record.local); - const auto remote = sideToJson(record.remote); - entry.insert(entry.end(), remote.begin(), remote.end()); - if (record.local.linkTarget || record.remote.linkTarget) - { - entry.push_back(record.local.linkTarget ? nlohmann::json(*record.local.linkTarget) : nlohmann::json()); - entry.push_back( - record.remote.linkTarget ? nlohmann::json(*record.remote.linkTarget) : nlohmann::json() - ); - } - records[relKey] = std::move(entry); - } - json = nlohmann::json{{"records", std::move(records)}}; - } - - void from_json(nlohmann::json const& json, SyncBaseline& baseline) - { - baseline.records.clear(); - for (auto const& [relKey, entry] : json.at("records").items()) - { - auto local = sideFromJson(entry, 0); - auto remote = sideFromJson(entry, 3); - local.linkTarget = linkTargetFromJson(entry, 6); - remote.linkTarget = linkTargetFromJson(entry, 7); - baseline.records.insert_or_assign( - relKey, SyncRecord{.local = std::move(local), .remote = std::move(remote)} - ); - } - } -} diff --git a/static/assets/languages/de_DE.yaml b/static/assets/languages/de_DE.yaml index ecf096b3..f0d30382 100644 --- a/static/assets/languages/de_DE.yaml +++ b/static/assets/languages/de_DE.yaml @@ -197,7 +197,7 @@ syncDialog: deleteSideLocal: "lokal" confirmDeleteRowHeader: "Jetzt löschen" confirmDeleteRow: "'{}' {} jetzt löschen?" - blockedNotice: "{} Einträge bleiben unverändert, weil sie nicht lesbar oder nicht übertragbar sind, ein Verzeichnis auf einen symbolischen Link trifft sich ihre Namen nur in der Groß- und Kleinschreibung unterscheiden oder sie sich seit der letzten Synchronisation so geändert haben, dass Daten verloren gingen. Zum Anzeigen klicken." + blockedNotice: "{} Einträge bleiben unverändert, weil sie nicht lesbar oder nicht übertragbar sind, ein Verzeichnis auf einen symbolischen Link trifft oder sich ihre Namen nur in der Groß- und Kleinschreibung unterscheiden. Zum Anzeigen klicken." blockedTitle: "Unveränderte Einträge" synchronizeUnavailable: "Der Vergleich läuft noch oder wurde abgebrochen. Vergleichen Sie erneut, um zu synchronisieren." executionRejected: "Diese Einträge wurden nicht synchronisiert: {}" @@ -207,8 +207,6 @@ syncDialog: InvalidName: "ungültiger Name" TypeConflict: "Verzeichnis gegen symbolischen Link" CaseConflict: "Namen unterscheiden sich nur in der Groß- und Kleinschreibung" - ChangedOnBothSides: "seit der letzten Synchronisation auf beiden Seiten geändert" - ChangedOnDestination: "seit der letzten Synchronisation auf der Seite geändert, die ersetzt oder gelöscht würde" tree: loading: "Lädt…" loadFailed: "Fehlgeschlagen: {}" diff --git a/static/assets/languages/en_US.yaml b/static/assets/languages/en_US.yaml index 0721ee53..f1f3714e 100644 --- a/static/assets/languages/en_US.yaml +++ b/static/assets/languages/en_US.yaml @@ -137,7 +137,7 @@ syncDialog: deleteSideLocal: "locally" confirmDeleteRowHeader: "Delete Now" confirmDeleteRow: "Delete '{}' {} now?" - blockedNotice: "{} entries are left alone because they cannot be read, cannot be transferred, a directory faces a symlink, their names differ only in case, or they changed since the last sync where that would lose data. Click to see them." + blockedNotice: "{} entries are left alone because they cannot be read, cannot be transferred, a directory faces a symlink, or their names differ only in case. Click to see them." blockedTitle: "Entries left alone" synchronizeUnavailable: "The comparison is running or was cancelled. Compare again to synchronize." executionRejected: "These entries were not synchronized: {}" @@ -147,8 +147,6 @@ syncDialog: InvalidName: "invalid name" TypeConflict: "directory against symlink" CaseConflict: "names differ only in case" - ChangedOnBothSides: "changed on both sides since the last sync" - ChangedOnDestination: "changed on the side it would replace or delete since the last sync" tree: loading: "Loading…" loadFailed: "Failed: {}" From b196ac9c70635633514f1195fdbd13307d4fb39a Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:11:04 +0200 Subject: [PATCH 069/131] Case variants block a sync only towards a side that folds case Names that differ only in case or normalization block a sync only towards a side that folds case: the local side folds by its platform (Windows, macOS) without a probe file, the server when its sync root or home looks like Windows or macOS (a drive path, /Users), so a Linux to Linux sync transfers every variant --- backend/include/backend/sync/sync_session.hpp | 8 ++++++++ backend/source/backend/sftp/operation_queue.cpp | 8 ++++++++ shared_data/source/shared_data/sync/diff.cpp | 11 +++++++++-- .../source/shared_data/sync/diff_tree_store.cpp | 5 +++-- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/backend/include/backend/sync/sync_session.hpp b/backend/include/backend/sync/sync_session.hpp index 23305270..0ec4dce5 100644 --- a/backend/include/backend/sync/sync_session.hpp +++ b/backend/include/backend/sync/sync_session.hpp @@ -58,6 +58,14 @@ class SyncSession : public std::enable_shared_from_this /// Number of scan entries (sum of both sides) above which /// @ref SharedData::Sync::DiffSummary::heavyCompare is set to true. std::uint64_t heavyCompareThreshold{5000}; + /** @brief The local filesystem folds case, the platform default: Windows and macOS do, others do not. */ +#if defined(_WIN32) || defined(__APPLE__) + bool localFoldsCase{true}; +#else + bool localFoldsCase{false}; +#endif + /** @brief The server looks like it folds case, see RemoteDeleteGuardDetail::looksCaseFolding. */ + bool remoteFoldsCase{false}; }; SyncSession(Options options, boost::asio::any_io_executor executor); diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 0cc697d0..c3af5f97 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -2168,11 +2168,19 @@ void OperationQueue::addSyncScanOperation( { // Create (or replace) the SyncSession. A fresh id from the frontend is the // norm; re-opening over an existing id is handled as "discard previous state". + const auto remoteFoldsCase = [&]() + { + if (RemoteDeleteGuardDetail::looksCaseFolding(remotePath)) + return true; + const auto home = remoteDeleteGuard(sftp).homeDirectory(); + return home && RemoteDeleteGuardDetail::looksCaseFolding(*home); + }(); auto session = std::make_shared( SyncSession::Options{ .sessionId = syncSessionId, .localRoot = localPath, .remoteRoot = remotePath, + .remoteFoldsCase = remoteFoldsCase, }, executor_ ); diff --git a/shared_data/source/shared_data/sync/diff.cpp b/shared_data/source/shared_data/sync/diff.cpp index eb5f26b2..dbad1f31 100644 --- a/shared_data/source/shared_data/sync/diff.cpp +++ b/shared_data/source/shared_data/sync/diff.cpp @@ -235,6 +235,12 @@ namespace SharedData::Sync DiffOptions const& options ) { + const bool localDestinationFolds = options.direction != Direction::Upload && options.localCaseInsensitive; + const bool remoteDestinationFolds = + options.direction != Direction::Download && options.remoteCaseInsensitive; + if (!localDestinationFolds && !remoteDestinationFolds) + return {}; + const auto anyFolds = [](std::vector const& children) { return std::ranges::any_of( @@ -272,8 +278,9 @@ namespace SharedData::Sync ); } ); - const bool remoteVariants = options.localCaseInsensitive && variants.remote.size() > 1; - if (!crossSide && !remoteVariants) + const bool remoteVariants = localDestinationFolds && variants.remote.size() > 1; + const bool localVariants = remoteDestinationFolds && variants.local.size() > 1; + if (!crossSide && !remoteVariants && !localVariants) continue; for (auto const name : variants.local) conflicts.emplace(name); diff --git a/shared_data/source/shared_data/sync/diff_tree_store.cpp b/shared_data/source/shared_data/sync/diff_tree_store.cpp index f277da24..304a326d 100644 --- a/shared_data/source/shared_data/sync/diff_tree_store.cpp +++ b/shared_data/source/shared_data/sync/diff_tree_store.cpp @@ -223,9 +223,10 @@ namespace SharedData::Sync }; }; - // Downloaded onto a case folding filesystem, names that differ only in case would meet as one file. + // Transferred onto a case folding filesystem, names that differ only in case would meet as one file. std::unordered_map foldedCounts; - const bool foldsCase = action == Action::Download && options_.localCaseInsensitive; + const bool foldsCase = (action == Action::Download && options_.localCaseInsensitive) || + (action == Action::Upload && options_.remoteCaseInsensitive); if (foldsCase) { for (auto const& child : directory.children) From d9e71ff60dba1156677586be15b76ec417f00b86 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:12:34 +0200 Subject: [PATCH 070/131] Add ComparedState and the ChangedSinceCompare error ComparedState describes what a sync compare saw at a path an entry replaces or deletes, comparedStateAt reads it from a scan tree, and unchangedSinceCompare tells whether that path still holds it; operations can fail with the new ChangedSinceCompare error --- .../file_operations/compared_state.hpp | 38 +++++++++++++++++++ .../shared_data/sync/diff_tree_store.hpp | 7 ++++ shared_data/source/shared_data/CMakeLists.txt | 1 + .../file_operations/compared_state.cpp | 34 +++++++++++++++++ .../shared_data/sync/diff_tree_store.cpp | 14 +++++++ 5 files changed, 94 insertions(+) create mode 100644 shared_data/include/shared_data/file_operations/compared_state.hpp create mode 100644 shared_data/source/shared_data/file_operations/compared_state.cpp diff --git a/shared_data/include/shared_data/file_operations/compared_state.hpp b/shared_data/include/shared_data/file_operations/compared_state.hpp new file mode 100644 index 00000000..1cc113c4 --- /dev/null +++ b/shared_data/include/shared_data/file_operations/compared_state.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include + +#include +#include +#include + +namespace SharedData +{ + /** + * @brief What a sync compare saw at a path that one of its entries replaces or deletes: nothing, or the type, + * size and modification time of what was there, the link literal for a symlink. + */ + struct ComparedState + { + bool exists{false}; + FileType type{FileType::Unknown}; + std::uint64_t size{0}; + std::uint64_t mtime{0}; + std::optional linkTarget{}; + + bool operator==(ComparedState const&) const = default; + }; + + /** + * @brief @p entry, as an lstat or a listing describes it, in the terms of a compare. + */ + ComparedState comparedStateOf(DirectoryEntry const& entry); + + /** + * @brief True when @p current is still what the compare saw in @p compared. A path seen absent must still be + * absent. One seen present may be gone now, nothing there can be lost. Otherwise the type must be the + * same, a symlink its literal, a regular file or a special one its size and modification time; a + * directory changes by its type only. + */ + bool unchangedSinceCompare(ComparedState const& compared, ComparedState const& current); +} diff --git a/shared_data/include/shared_data/sync/diff_tree_store.hpp b/shared_data/include/shared_data/sync/diff_tree_store.hpp index de398bfb..42872693 100644 --- a/shared_data/include/shared_data/sync/diff_tree_store.hpp +++ b/shared_data/include/shared_data/sync/diff_tree_store.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -33,6 +34,12 @@ namespace SharedData::Sync */ ScanNode const* findScanNode(ScanNode const& root, std::string_view relKey); + /** + * @brief What the scan @p root saw at @p relKey: absent when no node is there, else its type, size and + * modification time and the link literal of a symlink. + */ + ComparedState comparedStateAt(ScanNode const& root, std::string_view relKey); + /** @brief True when @p action reads from the local side (upload, local delete). */ bool actionSourceIsLocal(Action action); diff --git a/shared_data/source/shared_data/CMakeLists.txt b/shared_data/source/shared_data/CMakeLists.txt index 01e22cfa..5fbaf3b3 100644 --- a/shared_data/source/shared_data/CMakeLists.txt +++ b/shared_data/source/shared_data/CMakeLists.txt @@ -2,6 +2,7 @@ add_library( shared-data STATIC directory_entry.cpp + file_operations/compared_state.cpp ignore_rules.cpp sync/case_fold.cpp sync/diff.cpp diff --git a/shared_data/source/shared_data/file_operations/compared_state.cpp b/shared_data/source/shared_data/file_operations/compared_state.cpp new file mode 100644 index 00000000..6ccce2c5 --- /dev/null +++ b/shared_data/source/shared_data/file_operations/compared_state.cpp @@ -0,0 +1,34 @@ +#include + +namespace SharedData +{ + ComparedState comparedStateOf(DirectoryEntry const& entry) + { + return ComparedState{ + .exists = true, + .type = entry.type, + .size = entry.size, + .mtime = entry.mtime, + .linkTarget = entry.type == FileType::Symlink ? entry.linkTarget : std::nullopt, + }; + } + + bool unchangedSinceCompare(ComparedState const& compared, ComparedState const& current) + { + if (!compared.exists) + return !current.exists; + if (!current.exists) + return true; + if (compared.type != current.type) + return false; + switch (compared.type) + { + case FileType::Directory: + return true; + case FileType::Symlink: + return compared.linkTarget == current.linkTarget; + default: + return compared.size == current.size && compared.mtime == current.mtime; + } + } +} diff --git a/shared_data/source/shared_data/sync/diff_tree_store.cpp b/shared_data/source/shared_data/sync/diff_tree_store.cpp index 304a326d..b123bee9 100644 --- a/shared_data/source/shared_data/sync/diff_tree_store.cpp +++ b/shared_data/source/shared_data/sync/diff_tree_store.cpp @@ -76,6 +76,20 @@ namespace SharedData::Sync return current; } + ComparedState comparedStateAt(ScanNode const& root, std::string_view relKey) + { + auto const* node = findScanNode(root, relKey); + if (!node) + return ComparedState{}; + return ComparedState{ + .exists = true, + .type = node->type, + .size = node->size, + .mtime = node->mtime, + .linkTarget = node->type == FileType::Symlink ? node->linkTarget : std::nullopt, + }; + } + bool actionSourceIsLocal(Action action) { return action == Action::Upload || action == Action::DeleteLocal; From 1e0a696ba8d73fa94dc077bbbc23c74ca3cb0f93 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:28:31 +0200 Subject: [PATCH 071/131] Transfers and deletes can be held to what a sync compare saw Transfers and deletes can be held to what a sync compare saw: a download or upload given the compared state of its destination checks it right before moving the temp file into place, a remote or local delete checks its target right before removing it, and one that changed fails with ChangedSinceCompare and is left as it is while the rest of the bulk goes on --- .../backend/sftp/compared_state_check.hpp | 80 +++++++ .../include/backend/sftp/delete_operation.hpp | 20 +- .../backend/sftp/download_operation.hpp | 7 + .../backend/sftp/local_delete_operation.hpp | 1 + backend/include/backend/sftp/operation.hpp | 45 ++-- .../include/backend/sftp/operation_queue.hpp | 20 -- .../include/backend/sftp/upload_operation.hpp | 7 + .../backend/sftp/bulk_download_operation.cpp | 6 +- .../backend/sftp/compared_state_check.cpp | 197 ++++++++++++++++++ .../source/backend/sftp/delete_operation.cpp | 45 ++-- .../backend/sftp/download_operation.cpp | 2 +- .../backend/sftp/local_delete_operation.cpp | 1 + .../backend/sftp/local_scan_operation.cpp | 2 +- .../source/backend/sftp/operation_queue.cpp | 79 +------ .../source/backend/sftp/upload_operation.cpp | 21 +- .../file_operations/bulk_delete_request.hpp | 7 + ssh/include/ssh/sftp_session.hpp | 6 + ssh/source/ssh/sftp_session.cpp | 12 ++ 18 files changed, 410 insertions(+), 148 deletions(-) create mode 100644 backend/include/backend/sftp/compared_state_check.hpp create mode 100644 backend/source/backend/sftp/compared_state_check.cpp diff --git a/backend/include/backend/sftp/compared_state_check.hpp b/backend/include/backend/sftp/compared_state_check.hpp new file mode 100644 index 00000000..a4997815 --- /dev/null +++ b/backend/include/backend/sftp/compared_state_check.hpp @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/** + * @brief The checks that keep a sync entry from replacing or deleting what changed after its compare. The path is + * inspected right before the move or the removal, a change in the moment between the two goes unnoticed. + */ +namespace ComparedStateCheck +{ + /** + * @brief Whole seconds since the epoch of @p time, how a local scan records modification times. + */ + std::uint64_t secondsSinceEpoch(std::filesystem::file_time_type time); + + /** + * @brief What is at the local @p path now, without following a link, as a local scan sees it. A link or junction + * on Windows has an unknown type, the scan excludes it. + */ + std::expected localStateOf(std::filesystem::path const& path); + + /** + * @brief What is at the remote @p path now, without following a link. + */ + std::expected + remoteStateOfInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& path); + + /** + * @brief Fails with ChangedSinceCompare when the local @p path no longer holds @p compared (see + * SharedData::unchangedSinceCompare), and with FileStatFailed when it cannot be inspected. Without + * @p compared there is nothing to check. + */ + std::expected + requireUnchangedLocal(std::filesystem::path const& path, std::optional const& compared); + + /** + * @brief The remote counterpart of @ref requireUnchangedLocal, failing with SftpError when the server cannot + * tell. + */ + std::expected requireUnchangedRemoteInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& path, + std::optional const& compared + ); + + /** + * @brief Utility::moveIntoPlace of a finished local transfer, once @ref requireUnchangedLocal passed for @p to. + */ + std::expected moveLocalIntoPlace( + std::filesystem::path const& from, + std::filesystem::path const& to, + bool replace, + std::optional const& compared + ); + + /** + * @brief SftpSession::moveIntoPlaceInStrand of a finished remote transfer, once + * @ref requireUnchangedRemoteInStrand passed for @p to. + */ + std::expected moveRemoteIntoPlaceInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& from, + Paths::RemotePath const& to, + bool replace, + std::string const& asideSuffix, + SharedData::FileType sourceType, + std::optional const& compared + ); +} diff --git a/backend/include/backend/sftp/delete_operation.hpp b/backend/include/backend/sftp/delete_operation.hpp index bc835020..1875d65c 100644 --- a/backend/include/backend/sftp/delete_operation.hpp +++ b/backend/include/backend/sftp/delete_operation.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,19 @@ class DeleteOperation : public Operation return false; } + /** + * @brief One item to delete. + */ + struct Target + { + Paths::RemotePath path{}; + /** + * @brief What a sync compare saw there. An item that no longer is what it saw is not deleted but fails with + * ChangedSinceCompare. Unset outside of sync. + */ + std::optional compared{}; + }; + DeleteOperation(std::weak_ptr sftp, DeleteOperationOptions options); ~DeleteOperation() override; DeleteOperation(DeleteOperation const&) = delete; @@ -58,7 +72,7 @@ class DeleteOperation : public Operation /** * @brief Deletes exactly @p targets, last first, without a scan. For the files of a bulk delete. */ - void setTargets(std::vector targets); + void setTargets(std::vector targets); /** * @brief Scans with @p scan before deleting; every scanned entry is deleted, deepest first. One operation can @@ -113,7 +127,7 @@ class DeleteOperation : public Operation } private: - std::expected removeOnce(Paths::RemotePath const& path); + std::expected removeOnce(Target const& target); private: std::weak_ptr sftp_; @@ -125,7 +139,7 @@ class DeleteOperation : public Operation /** @brief Targets whose removal failed, the operation fails when that is all of them and nothing else. */ std::size_t removalFailures_{0}; /** @brief Deleted from the back, so a scanned directory goes after its content. */ - std::vector targets_{}; + std::vector targets_{}; /** @brief Why a scan result was refused, nothing may be deleted then. */ std::optional invalidScanEntry_{}; std::vector> failedEntries_{}; diff --git a/backend/include/backend/sftp/download_operation.hpp b/backend/include/backend/sftp/download_operation.hpp index 5c4fe2d6..3ced0e97 100644 --- a/backend/include/backend/sftp/download_operation.hpp +++ b/backend/include/backend/sftp/download_operation.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -55,6 +56,12 @@ class DownloadOperation : public Operation Paths::LocalPath linkFreeRoot{}; /** @brief Directories below @ref linkFreeRoot already found to be no links, shared by a bulk download. */ std::shared_ptr verifiedDirectories{}; + /** + * @brief What a sync compare saw at the destination. The transfer then fails with ChangedSinceCompare + * instead of replacing a destination that no longer holds it. A source that changed since is still + * transferred as it is now. Unset outside of sync. + */ + std::optional comparedDestination{std::nullopt}; }; SecureShell::ProcessingStrand* strand() const override diff --git a/backend/include/backend/sftp/local_delete_operation.hpp b/backend/include/backend/sftp/local_delete_operation.hpp index 238e8423..fd1cb766 100644 --- a/backend/include/backend/sftp/local_delete_operation.hpp +++ b/backend/include/backend/sftp/local_delete_operation.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include diff --git a/backend/include/backend/sftp/operation.hpp b/backend/include/backend/sftp/operation.hpp index 73f2c1a9..55abc65a 100644 --- a/backend/include/backend/sftp/operation.hpp +++ b/backend/include/backend/sftp/operation.hpp @@ -164,6 +164,29 @@ class Operation return std::unexpected(error_.value()); } + public: + /** + * @brief The error of a finished transfer that could not be moved into place on the server: FileExists when the + * destination exists and may not be replaced. + */ + static Error remoteMoveError(SecureShell::SftpError error) + { + if (error.sftpError == SSH_FX_FILE_ALREADY_EXISTS) + return Error{.type = ErrorType::FileExists}; + return Error{.type = ErrorType::SftpError, .sftpError = std::move(error)}; + } + + /** + * @brief The error of a finished transfer that could not be moved into place on this machine: FileExists when the + * destination exists and may not be replaced. + */ + static Error localMoveError(std::error_code error) + { + if (error == std::errc::file_exists) + return Error{.type = ErrorType::FileExists}; + return Error{.type = ErrorType::RenameFailure}; + } + protected: void enterState(OperationState newState) { @@ -219,28 +242,6 @@ class Operation strand->pushPromiseTask([]() {}).wait(); } - /** - * @brief The error of a finished transfer that could not be moved into place on the server: FileExists when the - * destination exists and may not be replaced. - */ - static Error remoteMoveError(SecureShell::SftpError error) - { - if (error.sftpError == SSH_FX_FILE_ALREADY_EXISTS) - return Error{.type = ErrorType::FileExists}; - return Error{.type = ErrorType::SftpError, .sftpError = std::move(error)}; - } - - /** - * @brief The error of a finished transfer that could not be moved into place on this machine: FileExists when the - * destination exists and may not be replaced. - */ - static Error localMoveError(std::error_code error) - { - if (error == std::errc::file_exists) - return Error{.type = ErrorType::FileExists}; - return Error{.type = ErrorType::RenameFailure}; - } - /** * @brief The error of an async transfer that ended without completing. */ diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index d93b1882..312ae8db 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -251,26 +251,6 @@ class OperationQueue SharedData::Sync::SyncExecutionRequest const& request ); - /** - * @brief Queues the delete of local items as one operation with one card, labelled with the directory they lie - * in. Refused as a whole when one of them is a root directory or contains the home directory. - * @param recursive Every target goes with its content, see LocalDeleteOperationOptions::recursive. - * @param insertRefresh The card refreshes the local listing when the delete ends. - */ - std::expected addLocalDeleteOperation( - Ids::OperationId operationId, - std::vector targets, - bool recursive, - bool insertRefresh, - SharedData::OperationMode mode = SharedData::OperationMode::Queued - ); - - /** - * @brief Ends every queued and running local delete as canceled, telling the frontend. A local delete cannot be - * resumed by another session, so a session ending stops it this way instead of leaving its card behind. - */ - void cancelLocalDeletes(); - /** * @brief A guard for the remote deletes of one request, sharing the home directory this queue looked up. */ diff --git a/backend/include/backend/sftp/upload_operation.hpp b/backend/include/backend/sftp/upload_operation.hpp index bbc0b19c..8ffdc1e2 100644 --- a/backend/include/backend/sftp/upload_operation.hpp +++ b/backend/include/backend/sftp/upload_operation.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -42,6 +43,12 @@ class UploadOperation : public Operation * subtrees whose remote structure may not exist yet. */ bool createMissingDirectories{false}; + /** + * @brief What a sync compare saw at the destination. The transfer then fails with ChangedSinceCompare + * instead of replacing a destination that no longer holds it. A source that changed since is still + * transferred as it is now. Unset outside of sync. + */ + std::optional comparedDestination{std::nullopt}; }; SecureShell::ProcessingStrand* strand() const override; diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index 8c1587ca..532c7a4e 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -137,9 +137,11 @@ bool BulkDownloadOperation::isSkippableError(Error const& error) const { // Missing or unreadable single files, and local files that cannot be written, do not compromise // the rest of the bulk. - // A destination that appeared after the existence check and may not be replaced fails only its own entry. + // A destination that appeared after the existence check and may not be replaced, or one that changed after a + // sync compared it, fails only its own entry. if (error.type == ErrorType::OpenFailure || error.type == ErrorType::CannotCreateDirectory || - error.type == ErrorType::LinkInLocalPath || error.type == ErrorType::FileExists) + error.type == ErrorType::LinkInLocalPath || error.type == ErrorType::FileExists || + error.type == ErrorType::ChangedSinceCompare) return true; return error.sftpError && (error.sftpError->sftpError == SSH_FX_NO_SUCH_FILE || error.sftpError->sftpError == SSH_FX_PERMISSION_DENIED || diff --git a/backend/source/backend/sftp/compared_state_check.cpp b/backend/source/backend/sftp/compared_state_check.cpp new file mode 100644 index 00000000..b33e922f --- /dev/null +++ b/backend/source/backend/sftp/compared_state_check.cpp @@ -0,0 +1,197 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace ComparedStateCheck +{ + namespace + { + Operation::Error changedError(std::string const& path) + { + Log::warn("'{}' changed after the sync compared it, it is neither replaced nor deleted.", path); + return Operation::Error{ + .type = Operation::ErrorType::ChangedSinceCompare, + .extraInfo = fmt::format("'{}' changed after the compare", path), + }; + } + + SharedData::FileType localTypeOf( + [[maybe_unused]] std::filesystem::path const& path, + std::filesystem::file_status status, + [[maybe_unused]] std::error_code& error + ) + { +#ifdef _WIN32 + if (std::filesystem::is_symlink(status) || Utility::isLinkOrJunction(path, error)) + return SharedData::FileType::Unknown; +#else + if (std::filesystem::is_symlink(status)) + return SharedData::FileType::Symlink; +#endif + return SharedData::fileTypeFromStdFilesystemType(status.type()); + } + } + + std::uint64_t secondsSinceEpoch(std::filesystem::file_time_type time) + { + return static_cast( + std::chrono::duration_cast(std::chrono::file_clock::to_sys(time).time_since_epoch()) + .count() + ); + } + + std::expected localStateOf(std::filesystem::path const& path) + { + std::error_code error{}; + const auto status = std::filesystem::symlink_status(path, error); + if (error == std::errc::no_such_file_or_directory || status.type() == std::filesystem::file_type::not_found) + return SharedData::ComparedState{}; + if (error) + return std::unexpected(error); + + SharedData::ComparedState state{.exists = true, .type = localTypeOf(path, status, error)}; + if (error) + return std::unexpected(error); + switch (state.type) + { + case SharedData::FileType::Symlink: + { + const auto literal = std::filesystem::read_symlink(path, error); + if (error) + return std::unexpected(error); + state.linkTarget = Paths::nativeToUtf8(literal); + return state; + } + case SharedData::FileType::Regular: + { + state.size = std::filesystem::file_size(path, error); + if (error) + return std::unexpected(error); + const auto modified = std::filesystem::last_write_time(path, error); + if (error) + return std::unexpected(error); + state.mtime = secondsSinceEpoch(modified); + return state; + } + default: + return state; + } + } + + std::expected + remoteStateOfInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& path) + { + const auto information = sftp.lstatInStrand(path); + if (!information) + { + if (information.error().sftpError == SSH_FX_NO_SUCH_FILE) + return SharedData::ComparedState{}; + return std::unexpected(information.error()); + } + auto state = SharedData::comparedStateOf(*information); + if (state.type != SharedData::FileType::Symlink) + return state; + return sftp.readLinkInStrand(path).transform( + [&state](std::string literal) + { + state.linkTarget = std::move(literal); + return state; + } + ); + } + + std::expected + requireUnchangedLocal(std::filesystem::path const& path, std::optional const& compared) + { + if (!compared) + return {}; + const auto current = localStateOf(path); + if (!current) + { + return std::unexpected( + Operation::Error{ + .type = Operation::ErrorType::FileStatFailed, + .extraInfo = fmt::format( + "Inspecting '{}': {}", Paths::nativeToUtf8(path), Utility::errorMessage(current.error()) + ), + } + ); + } + if (!SharedData::unchangedSinceCompare(*compared, *current)) + return std::unexpected(changedError(Paths::nativeToUtf8(path))); + return {}; + } + + std::expected requireUnchangedRemoteInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& path, + std::optional const& compared + ) + { + if (!compared) + return {}; + const auto current = remoteStateOfInStrand(sftp, path); + if (!current) + return std::unexpected(Operation::Error{.type = Operation::ErrorType::SftpError, .sftpError = current.error()}); + if (!SharedData::unchangedSinceCompare(*compared, *current)) + return std::unexpected(changedError(path.value())); + return {}; + } + + std::expected moveLocalIntoPlace( + std::filesystem::path const& from, + std::filesystem::path const& to, + bool replace, + std::optional const& compared + ) + { + return requireUnchangedLocal(to, compared) + .and_then( + [&]() -> std::expected + { + if (const auto moved = Utility::moveIntoPlace(from, to, replace); !moved) + { + Log::error( + "Failed to move '{}' into place: {}", + Paths::nativeToUtf8(to), + Utility::errorMessage(moved.error()) + ); + return std::unexpected(Operation::localMoveError(moved.error())); + } + return {}; + } + ); + } + + std::expected moveRemoteIntoPlaceInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& from, + Paths::RemotePath const& to, + bool replace, + std::string const& asideSuffix, + SharedData::FileType sourceType, + std::optional const& compared + ) + { + return requireUnchangedRemoteInStrand(sftp, to, compared) + .and_then( + [&]() -> std::expected + { + const auto moved = sftp.moveIntoPlaceInStrand(from, to, replace, asideSuffix, sourceType); + if (!moved) + { + Log::error("Failed to move '{}' into place: {}", to.value(), moved.error().message); + return std::unexpected(Operation::remoteMoveError(moved.error())); + } + return {}; + } + ); + } +} diff --git a/backend/source/backend/sftp/delete_operation.cpp b/backend/source/backend/sftp/delete_operation.cpp index d742ec3d..6f049a50 100644 --- a/backend/source/backend/sftp/delete_operation.cpp +++ b/backend/source/backend/sftp/delete_operation.cpp @@ -1,8 +1,32 @@ +#include #include #include #include +namespace +{ + /** + * @brief Removes @p path by itself, a directory only when it is empty. Its type is looked up unless given. + */ + std::expected removeOneInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& path, + std::optional type + ) + { + if (!type) + { + const auto information = sftp.lstatInStrand(path); + if (!information) + return std::unexpected(information.error()); + type = information->type; + } + return *type == SharedData::FileType::Directory ? sftp.removeDirectoryInStrand(path) + : sftp.removeFileInStrand(path); + } +} + DeleteOperation::DeleteOperation(std::weak_ptr sftp, DeleteOperationOptions options) : Operation{} , sftp_{std::move(sftp)} @@ -47,7 +71,7 @@ std::expected DeleteOperati Log::error("DeleteOperation: Refusing to delete '{}'.", remotePath_.value()); return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = remotePath_.value()}); } - targets_ = {remotePath_}; + targets_ = {Target{.path = remotePath_}}; totalToDelete_ = 1; filesRemovedProgress_ = {}; } @@ -125,7 +149,7 @@ std::expected DeleteOperati return enterErrorState({.type = ErrorType::UnknownWorkState}); } -std::expected DeleteOperation::removeOnce(Paths::RemotePath const& path) +std::expected DeleteOperation::removeOnce(Target const& target) { const auto sftp = openSession(sftp_); if (!sftp) @@ -162,15 +186,12 @@ std::expected DeleteOperation::removeOnce(Paths::R if (fut.wait_for(futureTimeout_) != std::future_status::ready) return enterErrorState({.type = ErrorType::FutureTimeout}); - // One item that cannot be deleted does not keep the others, its directories then stay as well. An item that is - // already gone, after a resume for one, counts as deleted. - if (const auto result = fut.get(); !result.has_value() && result.error().sftpError != SSH_FX_NO_SUCH_FILE) + // One item that cannot be deleted does not keep the others, its directories then stay as well. + if (auto result = fut.get(); !result.has_value()) { Log::error("DeleteOperation: Failed to delete '{}': {}", path.value(), result.error().toString()); ++removalFailures_; - failedEntries_.emplace_back( - path.value(), Error{.type = ErrorType::DeleteFailed, .sftpError = result.error(), .extraInfo = path.value()} - ); + failedEntries_.emplace_back(path.value(), std::move(result).error()); } ++deletedThusFar_; @@ -208,21 +229,21 @@ void DeleteOperation::appendScanResult(Paths::RemotePath const& root, std::vecto invalidScanEntry_ = target.value(); break; } - targets_.push_back(std::move(target)); + targets_.push_back(Target{.path = std::move(target)}); } if (invalidScanEntry_) targets_.clear(); totalToDelete_ = targets_.size(); } -void DeleteOperation::setTargets(std::vector targets) +void DeleteOperation::setTargets(std::vector targets) { scanDelivered_ = true; for (auto& target : targets) { - if (target.empty() || target.isRoot()) + if (target.path.empty() || target.path.isRoot()) { - invalidScanEntry_ = target.value(); + invalidScanEntry_ = target.path.value(); break; } targets_.push_back(std::move(target)); diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index 5b3cb1ab..2d38959f 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,7 +9,6 @@ #include #include #include -#include #include diff --git a/backend/source/backend/sftp/local_delete_operation.cpp b/backend/source/backend/sftp/local_delete_operation.cpp index e60663f7..80b098ea 100644 --- a/backend/source/backend/sftp/local_delete_operation.cpp +++ b/backend/source/backend/sftp/local_delete_operation.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/backend/source/backend/sftp/local_scan_operation.cpp b/backend/source/backend/sftp/local_scan_operation.cpp index 96466259..461a0ce2 100644 --- a/backend/source/backend/sftp/local_scan_operation.cpp +++ b/backend/source/backend/sftp/local_scan_operation.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -7,7 +8,6 @@ #include -#include #include #include #include diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index c3af5f97..8fb97e83 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1832,81 +1832,6 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( return result; } -std::expected OperationQueue::addLocalDeleteOperation( - Ids::OperationId operationId, - std::vector targets, - bool recursive, - bool insertRefresh, - SharedData::OperationMode mode, - std::vector comparedStates -) -{ - // Assumed in strand - if (targets.empty()) - return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = "no targets"}); - const auto homes = localHomeDirectories(); - for (auto const& target : targets) - { - if (auto allowed = refuseProtectedLocalDelete(target, homes); !allowed) - { - Log::error("Refusing to delete locally: {}", allowed.error()); - return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = target.toUtf8()}); - } - } - - auto parent = LocalDeleteOperation::commonParentOf(targets); - auto& targetQueue = queueFor(mode); - targetQueue.emplace_back( - operationId, - std::make_unique(LocalDeleteOperation::LocalDeleteOperationOptions{ - .itemsRemovedProgress = makeDeleteProgressCallback(operationId), - .scanProgress = - [progress = makeScanProgressCallback("onScanProgress", operationId)]( - std::uint64_t bytes, std::uint64_t entries - ) - { - progress(bytes, entries, entries); - }, - .targets = std::move(targets), - .recursive = recursive, - }) - ); - - RpcHelper::callRemoteSafely( - *hub_, - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = operationId, - .type = SharedData::OperationType::LocalDelete, - .mode = mode, - .insertRefresh = insertRefresh, - .localPath = std::move(parent), - .scanPhase = recursive, - } - ); - return {}; -} - -void OperationQueue::cancelLocalDeletes() -{ - // Assumed in strand - for (auto* queue : {&priorityOperations_, &operations_}) - { - std::erase_if( - *queue, - [this](auto& entry) - { - auto& [id, operation] = entry; - if (operation->type() != SharedData::OperationType::LocalDelete) - return false; - std::ignore = operation->cancel(true); - completeOperation(makeCompletedOperation(CompletionReason::Canceled, id, *operation)); - return true; - } - ); - } -} - RemoteDeleteGuard OperationQueue::remoteDeleteGuard(SecureShell::SftpSession& sftp) { return RemoteDeleteGuard{sftp, remoteHome_, sftpOpts_.operationTimeout.value_or(defaultFutureTimeout)}; @@ -2030,7 +1955,7 @@ std::size_t OperationQueue::addBulkDeleteOperation( } ); - std::vector fileTargets; + std::vector fileTargets; std::size_t scannedDirectories = 0; std::size_t failedTargets = 0; const auto failTarget = [&](Paths::RemotePath const& target, Operation::Error error) @@ -2079,7 +2004,7 @@ std::size_t OperationQueue::addBulkDeleteOperation( continue; } } - fileTargets.push_back(entry.target); + fileTargets.push_back(DeleteOperation::Target{.path = entry.target, .compared = entry.compared}); } const auto queued = fileTargets.size() + scannedDirectories; diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index 23c52c5a..82348c4c 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -308,20 +309,19 @@ std::expected UploadOperation::handleSymlinkInStra return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = result.error()}); } - const auto moved = sftp.moveIntoPlaceInStrand( + auto moved = ComparedStateCheck::moveRemoteIntoPlaceInStrand( + sftp, tempPath_, remotePath_, options_.mayOverwrite, Persistence::asideFileSuffix(options_.tempFileSuffix), - SharedData::FileType::Symlink + SharedData::FileType::Symlink, + options_.comparedDestination ); if (!moved.has_value()) { std::ignore = sftp.removeFileInStrand(tempPath_); - Log::error( - "UploadOperation: Failed to move symlink to '{}': {}.", options_.remotePath.value(), moved.error().message - ); - return std::unexpected(remoteMoveError(moved.error())); + return std::unexpected(std::move(moved).error()); } return {}; } @@ -796,19 +796,20 @@ std::expected UploadOperation::finalizeInStrand(Se // The modification time goes on the temp file, so nothing can fail once the destination is replaced. applyModificationTimeInStrand(sftp); - const auto moved = sftp.moveIntoPlaceInStrand( + auto moved = ComparedStateCheck::moveRemoteIntoPlaceInStrand( + sftp, tempPath_, remotePath_, options_.mayOverwrite, Persistence::asideFileSuffix(options_.tempFileSuffix), - SharedData::FileType::Regular + SharedData::FileType::Regular, + options_.comparedDestination ); if (!moved.has_value()) { - Log::error("UploadOperation: Failed to move '{}' into place: {}", options_.remotePath.value(), moved.error().message); if (auto removed = sftp.removeFileInStrand(tempPath_); !removed.has_value()) Log::warn("UploadOperation: Cannot remove '{}': {}", tempPath_.value(), removed.error().message); - return std::unexpected(remoteMoveError(moved.error())); + return std::unexpected(std::move(moved).error()); } return {}; } diff --git a/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp b/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp index afdb367f..8e3d0118 100644 --- a/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp +++ b/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -8,6 +9,7 @@ #include #include +#include #include namespace SharedData @@ -21,6 +23,11 @@ namespace SharedData bool isDirectory{false}; /** @brief Only shown on the card for files. */ std::uint64_t sizeBytes{0}; + /** + * @brief What a sync compare saw at the target, which is then only deleted while it still holds that. Set + * by the backend's sync only, it is not part of the request the frontend sends. + */ + std::optional compared{}; }; BOOST_DESCRIBE_STRUCT(BulkDeleteEntry, (), (target, isDirectory, sizeBytes)) diff --git a/ssh/include/ssh/sftp_session.hpp b/ssh/include/ssh/sftp_session.hpp index 0b6d9e9b..54ce88c5 100644 --- a/ssh/include/ssh/sftp_session.hpp +++ b/ssh/include/ssh/sftp_session.hpp @@ -434,6 +434,12 @@ namespace SecureShell */ std::expected readLinkDeepInStrand(Paths::RemotePath const& path, int maxDepth = 10); + /** + * @brief The literal of the symlink @p path, valid UTF-8 like a listed link target (U+FFFD for every invalid + * byte). Must be called from within the processing thread. + */ + std::expected readLinkInStrand(Paths::RemotePath const& path); + /** * @brief Creates a symlink at @p linkPath with the content @p target, which may be relative. */ diff --git a/ssh/source/ssh/sftp_session.cpp b/ssh/source/ssh/sftp_session.cpp index 46e7277e..0ae8d187 100644 --- a/ssh/source/ssh/sftp_session.cpp +++ b/ssh/source/ssh/sftp_session.cpp @@ -1022,6 +1022,18 @@ namespace SecureShell ); } + std::expected SftpSession::readLinkInStrand(Paths::RemotePath const& path) + { + if (auto valid = requireSetInStrand(path); !valid) + return std::unexpected(valid.error()); + const std::unique_ptr literal{ + sftp_readlink(session_, path.c_str()), ssh_string_free_char + }; + if (!literal) + return std::unexpected(lastError()); + return Paths::sanitizedUtf8(literal.get()); + } + std::expected SftpSession::createSymLinkInStrand(std::string const& target, Paths::RemotePath const& linkPath) { From 96b52c6953cbd392e52fad7d21d0b555534718bf Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:29:17 +0200 Subject: [PATCH 072/131] A sync entry replaces or deletes only what its compare saw A sync entry replaces or deletes only what its compare saw: the plan carries the compared state of each destination and delete target, a destination the compare saw absent is never replaced, and an entry whose destination or target changed after the compare fails while the other entries run; a resumed bulk keeps the compared states of its entries --- .../include/backend/sftp/bulk_download_operation.hpp | 2 ++ .../include/backend/sftp/bulk_transfer_operation.hpp | 3 +++ .../include/backend/sftp/bulk_upload_operation.hpp | 1 + backend/include/backend/sync/sync_session.hpp | 6 ++++++ .../source/backend/sftp/bulk_download_operation.cpp | 6 +++++- backend/source/backend/sftp/bulk_upload_operation.cpp | 6 +++++- backend/source/backend/sftp/operation_queue.cpp | 7 +++++++ backend/source/backend/sync/sync_session.cpp | 11 ++++++++++- .../file_operations/bulk_transfer_request.hpp | 9 +++++++++ 9 files changed, 48 insertions(+), 3 deletions(-) diff --git a/backend/include/backend/sftp/bulk_download_operation.hpp b/backend/include/backend/sftp/bulk_download_operation.hpp index 47ae6344..cb66cf42 100644 --- a/backend/include/backend/sftp/bulk_download_operation.hpp +++ b/backend/include/backend/sftp/bulk_download_operation.hpp @@ -55,6 +55,8 @@ class BulkDownloadOperation : public BulkTransferOperation bool mayOverwrite{false}; /** @brief A directory created without its content, with the link check of every local directory. */ bool isDirectory{false}; + /** @brief See SharedData::BulkDownloadEntry::comparedDestination. */ + std::optional comparedDestination{}; }; void setPrescannedFileList(std::vector files); diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index 7b5ba3ac..49bd838b 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -59,6 +60,8 @@ class BulkTransferOperation : public Operation Paths::LocalPath local{}; /** @brief This entry's destination may be replaced even when the bulk's options do not allow it. */ bool mayOverwrite{false}; + /** @brief See SharedData::BulkUploadEntry::comparedDestination, kept by a resume. */ + std::optional comparedDestination{}; }; ~BulkTransferOperation() override; diff --git a/backend/include/backend/sftp/bulk_upload_operation.hpp b/backend/include/backend/sftp/bulk_upload_operation.hpp index 4fcd435a..44151e98 100644 --- a/backend/include/backend/sftp/bulk_upload_operation.hpp +++ b/backend/include/backend/sftp/bulk_upload_operation.hpp @@ -44,6 +44,7 @@ class BulkUploadOperation : public BulkTransferOperation bool mayOverwrite{false}; /** @brief A directory created without its content, its missing parents below the bulk's root with it. */ bool isDirectory{false}; + std::optional comparedDestination{}; }; void setPrescannedFileList(std::vector files); diff --git a/backend/include/backend/sync/sync_session.hpp b/backend/include/backend/sync/sync_session.hpp index 0ec4dce5..f58bd5a9 100644 --- a/backend/include/backend/sync/sync_session.hpp +++ b/backend/include/backend/sync/sync_session.hpp @@ -172,6 +172,12 @@ class SyncSession : public std::enable_shared_from_this { SharedData::Sync::EnqueuePlanEntry entry; ResolvedPaths paths; + /** + * @brief What the compare saw at the path the entry replaces (the destination of a transfer) or deletes, + * which it may then only replace or delete while the path still holds it. A changed source is + * transferred as it is. + */ + SharedData::ComparedState compared{}; }; struct ResolvedPlan diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index 532c7a4e..b5938ee6 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -122,6 +122,7 @@ std::unique_ptr BulkDownloadOperation::makeChild( downloadOptions.remotePath = paths.remote; downloadOptions.localPath = paths.local; downloadOptions.mayOverwrite = downloadOptions.mayOverwrite || paths.mayOverwrite; + downloadOptions.comparedDestination = paths.comparedDestination; downloadOptions.bigFileOptimized = entry.size >= Constants::bigFileCutOff; downloadOptions.progressCallback = [onProgress = std::move(onProgress)](auto, auto max, auto current, auto) { @@ -217,7 +218,10 @@ void BulkDownloadOperation::setPrescannedFileList(std::vector fi entries.push_back(std::move(entry)); paths.push_back( EntryPaths{ - .remote = std::move(file.source), .local = std::move(file.destination), .mayOverwrite = file.mayOverwrite + .remote = std::move(file.source), + .local = std::move(file.destination), + .mayOverwrite = file.mayOverwrite, + .comparedDestination = std::move(file.comparedDestination), } ); } diff --git a/backend/source/backend/sftp/bulk_upload_operation.cpp b/backend/source/backend/sftp/bulk_upload_operation.cpp index 8380ff49..a54eaacf 100644 --- a/backend/source/backend/sftp/bulk_upload_operation.cpp +++ b/backend/source/backend/sftp/bulk_upload_operation.cpp @@ -124,6 +124,7 @@ std::unique_ptr BulkUploadOperation::makeChild( uploadOptions.remotePath = paths.remote; uploadOptions.localPath = paths.local; uploadOptions.mayOverwrite = uploadOptions.mayOverwrite || paths.mayOverwrite; + uploadOptions.comparedDestination = paths.comparedDestination; uploadOptions.bigFileOptimized = entry.size >= Constants::bigFileCutOff; uploadOptions.progressCallback = [onProgress = std::move(onProgress)](auto, auto max, auto current, auto) { @@ -192,7 +193,10 @@ void BulkUploadOperation::setPrescannedFileList(std::vector file entries.push_back(std::move(entry)); paths.push_back( EntryPaths{ - .remote = std::move(file.destination), .local = std::move(file.source), .mayOverwrite = file.mayOverwrite + .remote = std::move(file.destination), + .local = std::move(file.source), + .mayOverwrite = file.mayOverwrite, + .comparedDestination = std::move(file.comparedDestination), } ); } diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 8fb97e83..8805e1fc 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1657,6 +1657,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( using SharedData::Sync::Action; SharedData::Sync::SyncExecutionResult result{.failed = std::move(plan.refused)}; + // An entry replaces only a destination the compare saw, and only while it still holds what the compare saw. SharedData::BulkUploadRequest uploads{ .mode = request.mode, .localRoot = session.localRoot(), @@ -1669,6 +1670,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( }; SharedData::BulkDeleteRequest remoteDeletes{.mode = request.mode, .listedContentOnly = true}; std::vector localDeletes; + std::vector localDeletesCompared; const auto localHomes = localHomeDirectories(); auto remoteGuard = remoteDeleteGuard(sftp); remoteGuard.trustScannedBelow(session.remoteRoot()); @@ -1704,6 +1706,8 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( { auto& entry = planned.entry; auto& paths = planned.paths; + const auto comparedDestination = + entry.isDirectory ? std::nullopt : std::optional{planned.compared}; switch (entry.action) { case Action::Upload: @@ -1730,6 +1734,8 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( .withoutContent = entry.isDirectory, .mtime = entry.mtime, .mtimeNsec = entry.mtimeNsec, + .mayOverwrite = planned.compared.exists, + .comparedDestination = comparedDestination, }); result.downloads.push_back(std::move(entry.relKey)); break; @@ -1769,6 +1775,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( } result.localDeleteSources.push_back(paths.local.toUtf8()); localDeletes.push_back(std::move(paths.local)); + localDeletesCompared.push_back(std::move(planned.compared)); result.localDeletes.push_back(std::move(entry.relKey)); break; } diff --git a/backend/source/backend/sync/sync_session.cpp b/backend/source/backend/sync/sync_session.cpp index 01e2709f..06352b34 100644 --- a/backend/source/backend/sync/sync_session.cpp +++ b/backend/source/backend/sync/sync_session.cpp @@ -177,6 +177,12 @@ SyncSession::ResolvedPlan SyncSession::resolvePlanInStrand(DiffSection section, std::unordered_set const& selectedRelKeys) const { ResolvedPlan result; + const auto comparedAt = [this](EnqueuePlanEntry const& entry) + { + const bool onLocalSide = entry.action == Action::Download || entry.action == Action::DeleteLocal; + auto const& tree = onLocalSide ? localTree_ : remoteTree_; + return tree ? comparedStateAt(*tree, entry.relKey) : SharedData::ComparedState{}; + }; for (auto& entry : buildEnqueuePlanInStrand(section, selectedRelKeys)) { auto paths = Paths::RelativePath::make(entry.relKey).and_then( @@ -191,7 +197,10 @@ SyncSession::resolvePlanInStrand(DiffSection section, std::unordered_set +#include #include #include #include @@ -33,6 +34,12 @@ namespace SharedData * a destination that appeared after the existence check is not replaced. */ bool mayOverwrite{false}; + /** + * @brief What a sync compare saw at the destination: the entry fails with ChangedSinceCompare instead of + * replacing a destination that no longer holds it. Set by the backend's sync only, it is not part of + * the request the frontend sends. + */ + std::optional comparedDestination{}; }; BOOST_DESCRIBE_STRUCT( BulkUploadEntry, @@ -57,6 +64,8 @@ namespace SharedData std::uint32_t mtimeNsec{0}; /** @brief See @ref BulkUploadEntry::mayOverwrite. */ bool mayOverwrite{false}; + /** @brief See @ref BulkUploadEntry::comparedDestination. */ + std::optional comparedDestination{}; }; BOOST_DESCRIBE_STRUCT( BulkDownloadEntry, From 45e9d25f1c5a88b16e84f443a9a2ae4232d20e38 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:30:32 +0200 Subject: [PATCH 073/131] Sync rows name a destination that changed since the compare A sync row whose destination or delete target changed after the compare shows that as its reason; completions of prescanned bulks name the error type of each skipped entry index, and the failed entries callback hands over the errors with the paths --- .../backend/sftp/bulk_transfer_operation.hpp | 3 ++- .../backend/sftp/bulk_transfer_operation.cpp | 13 +++++++++---- .../session_components/operation_queue.hpp | 10 ++++++++++ .../source/frontend/sync_dialog/sync_dialog.cpp | 16 ++++++++++++++++ .../file_operations/operation_completed.hpp | 10 ++++++++++ static/assets/languages/de_DE.yaml | 1 + static/assets/languages/en_US.yaml | 1 + 7 files changed, 49 insertions(+), 5 deletions(-) diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index 49bd838b..a530dbde 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -394,7 +395,7 @@ class BulkTransferOperation : public Operation /** @brief Per entry: transferred, created or skipped for its type, so a resume leaves it out. */ std::vector finishedEntries_{}; std::vector> failedEntries_{}; - std::vector failedEntryIndices_{}; + std::vector indexedFailures_{}; // Rolling bulk-level throughput. Per-file rates reset between files, which made the number // flicker; sampling the cumulative byte count keeps one coherent rate per bulk. std::chrono::steady_clock::time_point lastBpsSampleTime_{}; diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 876c8452..c8b17055 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -4,6 +4,7 @@ #include #include +#include using namespace std::chrono_literals; @@ -238,7 +239,7 @@ std::expected BulkTransferOperation::fillSlo { Log::warn("{}: Skipping '{}': {}", logName(), displayPath(index), paths.error().toString()); failedEntries_.emplace_back(failureSource(index), paths.error()); - failedEntryIndices_.push_back(static_cast(index)); + indexedFailures_.push_back({.index = static_cast(index), .type = paths.error().type}); completedBytes_ += entry.isRegularFile() ? entry.size : 0u; ++completedEntries_; continue; @@ -255,7 +256,7 @@ std::expected BulkTransferOperation::fillSlo return std::unexpected(result.error()); Log::warn("{}: Skipping directory '{}': {}", logName(), displayPath(index), result.error().toString()); failedEntries_.emplace_back(failureSource(index), result.error()); - failedEntryIndices_.push_back(static_cast(index)); + indexedFailures_.push_back({.index = static_cast(index), .type = result.error().type}); } else { @@ -325,7 +326,7 @@ std::expected BulkTransferOperation::stepSlo { // Not critical for the bulk as a whole: record the entry for the user and move on. failedEntries_.emplace_back(failureSource(slot.paths), error); - failedEntryIndices_.push_back(static_cast(slot.entryIndex)); + indexedFailures_.push_back({.index = static_cast(slot.entryIndex), .type = error.type}); finishSlot(slot, false); if (common_.failFast) { @@ -399,8 +400,12 @@ void BulkTransferOperation::emitProgress(bool force) .bytesTotal = totalBytes_, .bytesPerSecond = 0, .inFlight = {}, - .failedEntryIndices = failedEntryIndices_, + .failedEntryIndices = {}, }; + update.failedEntryIndices.reserve(indexedFailures_.size()); + std::ranges::transform( + indexedFailures_, std::back_inserter(update.failedEntryIndices), &SharedData::IndexedFailure::index + ); update.bytesPerSecond = updateBulkBytesPerSecond(update.bytesCurrent); update.inFlight.reserve(slots_.size() + deferred_.size()); for (auto const& slot : slots_) diff --git a/frontend/include/frontend/session_components/operation_queue.hpp b/frontend/include/frontend/session_components/operation_queue.hpp index 40671e19..9fae215d 100644 --- a/frontend/include/frontend/session_components/operation_queue.hpp +++ b/frontend/include/frontend/session_components/operation_queue.hpp @@ -204,6 +204,16 @@ class OperationQueue bool isActive() const; void addCompletionCallback(Ids::OperationId const& opId, std::function callback); + + /** + * @brief Receives the entries a bulk operation could not transfer or delete with why: by source path and, for a + * prescanned bulk transfer, by entry index. + */ + using FailedEntriesCallback = std::function> const& failedEntries, + std::vector const& indexedFailures + )>; + /** @brief Register a callback that receives the entries a bulk operation could not transfer or * delete. Runs on completion, before the completion callback, because the final BulkProgress * can arrive after the completion and is dropped then. diff --git a/frontend/source/frontend/sync_dialog/sync_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_dialog.cpp index 384c7014..ad7e11b8 100644 --- a/frontend/source/frontend/sync_dialog/sync_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_dialog.cpp @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -268,6 +269,8 @@ struct SyncDialog::Implementation std::vector applied{}; /** @brief What executions reported per row, see setRowState. */ std::unordered_map executionStates{}; + /** @brief Why a failed row failed, for the failures a row explains. */ + std::unordered_map failureReasons{}; }; std::array sections_{}; @@ -775,6 +778,13 @@ struct SyncDialog::Implementation return !state.rows.contains(entry.first); } ); + std::erase_if( + state.failureReasons, + [&state](auto const& entry) + { + return !state.rows.contains(entry.first); + } + ); auto const& selection = selectionFor(section); auto& set = selection->value(); @@ -1399,6 +1409,12 @@ ScriptNuiComponents::Tree::RowContentRenderer SyncDialog::Implementation::makeTr auto progress = progressIter->second; return div{ class_ = rowClass, + "title"_attr = Nui::observe(*progress).generate( + [this, section, relKey]() + { + return failureReasonOf(section, relKey); + } + ), style = Nui::observe(*progress).generate( [progress]() -> std::string { diff --git a/shared_data/include/shared_data/file_operations/operation_completed.hpp b/shared_data/include/shared_data/file_operations/operation_completed.hpp index 3ad6af00..0961c9d2 100644 --- a/shared_data/include/shared_data/file_operations/operation_completed.hpp +++ b/shared_data/include/shared_data/file_operations/operation_completed.hpp @@ -19,6 +19,16 @@ namespace SharedData { BOOST_DEFINE_ENUM_CLASS(OperationCompletionReason, Completed, Canceled, Failed) + /** + * @brief A skipped entry of a prescanned bulk up- or download, by its index, and why it was skipped. + */ + struct IndexedFailure + { + std::uint32_t index{0}; + OperationErrorType type{OperationErrorType::UnknownError}; + }; + BOOST_DESCRIBE_STRUCT(IndexedFailure, (), (index, type)) + struct OperationCompleted { OperationCompletionReason reason; diff --git a/static/assets/languages/de_DE.yaml b/static/assets/languages/de_DE.yaml index f0d30382..a7b0164a 100644 --- a/static/assets/languages/de_DE.yaml +++ b/static/assets/languages/de_DE.yaml @@ -201,6 +201,7 @@ syncDialog: blockedTitle: "Unveränderte Einträge" synchronizeUnavailable: "Der Vergleich läuft noch oder wurde abgebrochen. Vergleichen Sie erneut, um zu synchronisieren." executionRejected: "Diese Einträge wurden nicht synchronisiert: {}" + changedSinceCompare: "Nach dem Vergleich geändert und deshalb unverändert gelassen. Erneut vergleichen, um es zu synchronisieren." blockReason: Unreadable: "nicht lesbar" Excluded: "nicht übertragbar" diff --git a/static/assets/languages/en_US.yaml b/static/assets/languages/en_US.yaml index f1f3714e..606a02d9 100644 --- a/static/assets/languages/en_US.yaml +++ b/static/assets/languages/en_US.yaml @@ -141,6 +141,7 @@ syncDialog: blockedTitle: "Entries left alone" synchronizeUnavailable: "The comparison is running or was cancelled. Compare again to synchronize." executionRejected: "These entries were not synchronized: {}" + changedSinceCompare: "Changed after the compare, so it was left as it is. Compare again to synchronize it." blockReason: Unreadable: "unreadable" Excluded: "not transferable" From 17a8af0b51603b5ad42ee720ad80a67009617398 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:49:25 +0200 Subject: [PATCH 074/131] Local deletes run in a LocalOperationQueue of their own Local deletes run in a LocalOperationQueue of their own, which the session manager owns, so a session that ends or loses its connection no longer cancels them: a delete a session started waits while that session's queue is paused and runs on its own once the session ends, a priority delete is never held, and the queue reports to the frontend under LocalOperationQueue::...; explorer deletes without a session queue through LocalOperationQueue::addDelete, and sync plans queue their local deletes there with the compared states --- .../include/backend/local_operation_queue.hpp | 152 +++++++++ backend/include/backend/session.hpp | 1 + backend/include/backend/session_manager.hpp | 3 + .../include/backend/sftp/operation_queue.hpp | 4 +- .../source/backend/local_operation_queue.cpp | 306 ++++++++++++++++++ backend/source/backend/session.cpp | 1 + backend/source/backend/session_manager.cpp | 2 + .../source/backend/sftp/operation_queue.cpp | 3 +- 8 files changed, 470 insertions(+), 2 deletions(-) create mode 100644 backend/include/backend/local_operation_queue.hpp create mode 100644 backend/source/backend/local_operation_queue.cpp diff --git a/backend/include/backend/local_operation_queue.hpp b/backend/include/backend/local_operation_queue.hpp new file mode 100644 index 00000000..d5df04e9 --- /dev/null +++ b/backend/include/backend/local_operation_queue.hpp @@ -0,0 +1,152 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @brief Runs local deletes apart from the SSH sessions, so closing a tab or losing a connection never stops one. + * + * The deletes run one after another on the strand the queue shares with the sessions, a work step at a time, and + * report to the frontend under "LocalOperationQueue::...". A delete a session started waits while the queue of that + * session is paused, for as long as the session exists. + */ +class LocalOperationQueue + : public RpcHelper::StrandRpc + , public std::enable_shared_from_this +{ + public: + using OperationCompleted = SharedData::OperationCompleted; + + struct DeleteRequest + { + Ids::OperationId operationId{}; + std::vector targets{}; + /** @brief See LocalDeleteOperation::LocalDeleteOperationOptions::recursive. */ + bool recursive{false}; + /** @brief The card refreshes the local listing when the delete ends. */ + bool insertRefresh{false}; + /** @brief A priority delete runs before the others and is never held. */ + SharedData::OperationMode mode{SharedData::OperationMode::Queued}; + /** @brief The session whose paused queue holds the delete, none for a delete without a session. */ + std::optional owner{}; + /** @brief The queue of @ref owner is paused, so the delete starts held. */ + bool held{false}; + /** @brief See LocalDeleteOperation::LocalDeleteOperationOptions::comparedStates. */ + std::vector comparedStates{}; + }; + + LocalOperationQueue( + boost::asio::any_io_executor executor, + std::shared_ptr> strand, + Nui::Window& wnd, + Nui::RpcHub& hub, + std::shared_ptr localDeletionPrevented + ); + ~LocalOperationQueue() override = default; + LocalOperationQueue(LocalOperationQueue const&) = delete; + LocalOperationQueue& operator=(LocalOperationQueue const&) = delete; + LocalOperationQueue(LocalOperationQueue&&) = delete; + LocalOperationQueue& operator=(LocalOperationQueue&&) = delete; + + /** + * @brief Handles "LocalOperationQueue::addDelete" (operation id, local paths), the delete of explorer items + * without a session, and "LocalOperationQueue::cancel" (operation id). + */ + void registerRpc(); + + /** + * @brief Queues a delete with one card, labelled with the directory its targets lie in. Refused as a whole when + * one target is a root directory or contains a home directory. Assumed in strand. + */ + std::expected addDelete(DeleteRequest request); + + /** + * @brief Queues the delete of the explorer items @p pathsUtf8 with their content, as @ref addDelete with a card + * that refreshes the local listing. Refused when deletion is prevented in the settings or a path is not + * an item that may be deleted. Assumed in strand. + * + * @param owner The session whose explorer asks, none without one. + * @param held The queue of @p owner is paused. + * @return A message for Utility::localizedMessage or the language file saying why nothing was queued. + */ + std::expected addExplorerDelete( + Ids::OperationId operationId, + std::vector const& pathsUtf8, + std::optional owner, + bool held + ); + + /** + * @brief Holds the deletes of @p owner, or lets them run again. Assumed in strand. + */ + void hold(Ids::SessionId const& owner, bool held); + + /** + * @brief The deletes of @p owner run on their own from now on, never held again. Assumed in strand. + */ + void release(Ids::SessionId const& owner); + + /** + * @brief Ends the delete @p operationId as canceled and tells the frontend. Assumed in strand. + */ + void cancel(Ids::OperationId const& operationId); + + /** + * @brief Works the first delete that is not held for one step and tells the frontend when it ends. Assumed in + * strand; the queue drives itself, this is public for tests. + * + * @return True while a delete that is not held is left. + */ + bool work(); + + /** + * @brief Test hook: @p observer sees every completion right before it is sent to the frontend. + */ + void setCompletionObserver(std::function observer); + + private: + struct QueuedDelete + { + Ids::OperationId operationId; + std::unique_ptr operation; + std::optional owner; + bool held; + bool priority; + }; + + /** + * @brief Posts the next work step unless one is posted already. + */ + void schedule(); + + void complete( + QueuedDelete const& queued, + SharedData::OperationCompletionReason reason, + std::optional error + ); + std::string rpcName(std::string_view event) const; + + std::shared_ptr localDeletionPrevented_; + std::deque deletes_{}; + bool scheduled_{false}; + std::function completionObserver_{}; +}; diff --git a/backend/include/backend/session.hpp b/backend/include/backend/session.hpp index 928ca067..df21957e 100644 --- a/backend/include/backend/session.hpp +++ b/backend/include/backend/session.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/backend/include/backend/session_manager.hpp b/backend/include/backend/session_manager.hpp index e4190ba1..430c7282 100644 --- a/backend/include/backend/session_manager.hpp +++ b/backend/include/backend/session_manager.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -121,6 +122,8 @@ class SessionManager std::unordered_map, Ids::IdHash> sessions_{}; /** @brief Read by the local operation queue on each local delete the explorer asks for. */ std::shared_ptr localDeletionPrevented_{std::make_shared(true)}; + /** @brief Shared with every session, outlives them all, so a local delete never ends with its session. */ + std::shared_ptr localOperationQueue_; std::map passwordProviders_{}; std::mutex passwordRequestMutex_{}; diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index 312ae8db..34760186 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -248,7 +249,8 @@ class OperationQueue SecureShell::SftpSession& sftp, SyncSession const& session, SyncSession::ResolvedPlan plan, - SharedData::Sync::SyncExecutionRequest const& request + SharedData::Sync::SyncExecutionRequest const& request, + LocalOperationQueue& localDeleteQueue ); /** diff --git a/backend/source/backend/local_operation_queue.cpp b/backend/source/backend/local_operation_queue.cpp new file mode 100644 index 00000000..95fd5ee6 --- /dev/null +++ b/backend/source/backend/local_operation_queue.cpp @@ -0,0 +1,306 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +LocalOperationQueue::LocalOperationQueue( + boost::asio::any_io_executor executor, + std::shared_ptr> strand, + Nui::Window& wnd, + Nui::RpcHub& hub, + std::shared_ptr localDeletionPrevented +) + : RpcHelper::StrandRpc{executor, std::move(strand), wnd, hub} + , localDeletionPrevented_{std::move(localDeletionPrevented)} +{} + +std::string LocalOperationQueue::rpcName(std::string_view event) const +{ + return fmt::format("LocalOperationQueue::{}", event); +} + +void LocalOperationQueue::registerRpc() +{ + on(rpcName("addDelete")) + .perform( + [weak = weak_from_this()]( + RpcHelper::RpcOnce&& reply, std::string const& operationIdString, std::vector const& paths + ) + { + auto self = weak.lock(); + if (!self) + return reply.error("The local operation queue no longer exists"); + const auto added = + self->addExplorerDelete(Ids::makeOperationId(operationIdString), paths, std::nullopt, false); + if (!added) + return reply.error(added.error()); + reply(SharedData::success()); + } + ); + + on(rpcName("cancel")) + .perform( + [weak = weak_from_this()](RpcHelper::RpcOnce&& reply, Ids::OperationId operationId) + { + auto self = weak.lock(); + if (!self) + return reply.error("The local operation queue no longer exists"); + self->cancel(operationId); + reply(SharedData::success()); + } + ); +} + +std::expected LocalOperationQueue::addDelete(DeleteRequest request) +{ + if (request.targets.empty()) + return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = "no targets"}); + const auto homes = localHomeDirectories(); + for (auto const& target : request.targets) + { + if (auto allowed = refuseProtectedLocalDelete(target, homes); !allowed) + { + Log::error("Refusing to delete locally: {}", allowed.error()); + return std::unexpected( + Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = target.toUtf8()} + ); + } + } + + auto parent = LocalDeleteOperation::commonParentOf(request.targets); + const auto priority = request.mode == SharedData::OperationMode::PriorityQueued; + auto operation = std::make_unique(LocalDeleteOperation::LocalDeleteOperationOptions{ + .itemsRemovedProgress = + [weak = weak_from_this(), operationId = request.operationId]( + std::string const& path, std::uint64_t itemsDeleted, std::uint64_t totalItems + ) + { + if (auto self = weak.lock(); self) + { + RpcHelper::callRemoteSafely( + *self->hub_, + self->rpcName("onDeleteProgress"), + SharedData::BulkDeleteProgress{ + .operationId = operationId, + .currentFile = path, + .filesDeleted = itemsDeleted, + .totalFiles = totalItems, + } + ); + } + }, + .scanProgress = + [weak = weak_from_this(), operationId = request.operationId](std::uint64_t bytes, std::uint64_t entries) + { + if (auto self = weak.lock(); self) + { + RpcHelper::callRemoteSafely( + *self->hub_, + self->rpcName("onScanProgress"), + SharedData::ScanProgress{ + .operationId = operationId, + .totalBytes = bytes, + .currentIndex = entries, + .totalScanned = entries, + } + ); + } + }, + .targets = std::move(request.targets), + .recursive = request.recursive, + .comparedStates = std::move(request.comparedStates), + }); + + const auto position = priority ? std::ranges::find_if( + deletes_, + [](QueuedDelete const& queued) + { + return !queued.priority; + } + ) + : deletes_.end(); + deletes_.insert( + position, + QueuedDelete{ + .operationId = request.operationId, + .operation = std::move(operation), + .owner = std::move(request.owner), + .held = request.held && !priority, + .priority = priority, + } + ); + + RpcHelper::callRemoteSafely( + *hub_, + rpcName("onOperationAdded"), + SharedData::OperationAdded{ + .operationId = request.operationId, + .type = SharedData::OperationType::LocalDelete, + .mode = request.mode, + .insertRefresh = request.insertRefresh, + .localPath = std::move(parent), + .scanPhase = request.recursive, + } + ); + schedule(); + return {}; +} + +std::expected LocalOperationQueue::addExplorerDelete( + Ids::OperationId operationId, + std::vector const& pathsUtf8, + std::optional owner, + bool held +) +{ + if (!localDeletionPrevented_ || localDeletionPrevented_->load()) + { + Log::warn("A local delete was asked for, but deletion is prevented by configuration."); + return std::unexpected(std::string{"backend.rpcFilesystem.deletionPrevented"}); + } + + std::vector targets; + targets.reserve(pathsUtf8.size()); + const auto homes = localHomeDirectories(); + for (auto const& path : pathsUtf8) + { + auto target = localDeleteTarget(path, homes); + if (!target) + { + Log::error("Refusing to delete locally: {}", target.error()); + return std::unexpected(Utility::localizedMessage("backend.rpcFilesystem.removeRefused", path)); + } + targets.push_back(*std::move(target)); + } + + return addDelete({ + .operationId = std::move(operationId), + .targets = std::move(targets), + .recursive = true, + .insertRefresh = true, + .owner = std::move(owner), + .held = held, + }) + .transform_error( + [](Operation::Error const& error) + { + return error.toString(); + } + ); +} + +void LocalOperationQueue::hold(Ids::SessionId const& owner, bool held) +{ + for (auto& queued : deletes_) + { + if (queued.owner == owner && !queued.priority) + queued.held = held; + } + if (!held) + schedule(); +} + +void LocalOperationQueue::release(Ids::SessionId const& owner) +{ + for (auto& queued : deletes_) + { + if (queued.owner != owner) + continue; + queued.owner.reset(); + queued.held = false; + } + schedule(); +} + +void LocalOperationQueue::cancel(Ids::OperationId const& operationId) +{ + const auto found = std::ranges::find(deletes_, operationId, &QueuedDelete::operationId); + if (found == deletes_.end()) + { + Log::info("LocalOperationQueue: '{}' is not queued, nothing to cancel.", operationId.value()); + return; + } + std::ignore = found->operation->cancel(true); + complete(*found, SharedData::OperationCompletionReason::Canceled, std::nullopt); + deletes_.erase(found); +} + +bool LocalOperationQueue::work() +{ + const auto runnable = [](QueuedDelete const& queued) + { + return !queued.held; + }; + const auto next = std::ranges::find_if(deletes_, runnable); + if (next == deletes_.end()) + return false; + + const auto result = next->operation->work(); + if (!result) + { + complete(*next, SharedData::OperationCompletionReason::Failed, result.error()); + deletes_.erase(next); + } + else if (*result == Operation::WorkStatus::Complete) + { + complete(*next, SharedData::OperationCompletionReason::Completed, std::nullopt); + deletes_.erase(next); + } + return std::ranges::any_of(deletes_, runnable); +} + +void LocalOperationQueue::schedule() +{ + if (scheduled_) + return; + scheduled_ = true; + boost::asio::post( + *strand_, + [weak = weak_from_this()]() + { + auto self = weak.lock(); + if (!self) + return; + self->scheduled_ = false; + if (self->work()) + self->schedule(); + } + ); +} + +void LocalOperationQueue::complete( + QueuedDelete const& queued, + SharedData::OperationCompletionReason reason, + std::optional error +) +{ + if (error) + Log::error("Local delete failed: {}", error->toString()); + OperationCompleted completed{ + .reason = reason, + .operationId = queued.operationId, + .completionTime = std::chrono::system_clock::now(), + .error = std::move(error), + .failedEntries = queued.operation->getFailed(), + }; + if (completionObserver_) + completionObserver_(completed); + RpcHelper::callRemoteSafely(*hub_, rpcName("onOperationCompleted"), completed); +} + +void LocalOperationQueue::setCompletionObserver(std::function observer) +{ + completionObserver_ = std::move(observer); +} diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index f564ad4a..09b21c70 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -1283,6 +1283,7 @@ void Session::registerOperationQueuePauseUnpause() } self->operationQueue_->paused(pause); + self->localOperationQueue_->hold(self->id_, pause); self->resetQueueThrottle(); return reply(SharedData::success()); } diff --git a/backend/source/backend/session_manager.cpp b/backend/source/backend/session_manager.cpp index ed77e5d3..f572a535 100644 --- a/backend/source/backend/session_manager.cpp +++ b/backend/source/backend/session_manager.cpp @@ -105,6 +105,7 @@ SessionManager::SessionManager( ) : RpcHelper::StrandRpc{executor, wnd, hub} , stateHolder_{&stateHolder} + , localOperationQueue_{std::make_shared(executor_, strand_, wnd, hub, localDeletionPrevented_)} {} void SessionManager::preventLocalDeletion(bool prevent) @@ -277,6 +278,7 @@ void SessionManager::registerRpc() registerRpcSessionDisconnect(); registerRpcAdoptBulkResume(); registerRpcDiscardBulkResume(); + localOperationQueue_->registerRpc(); } void SessionManager::registerRpcSessionConnect() diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 8805e1fc..17d4086a 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1650,7 +1650,8 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( SecureShell::SftpSession& sftp, SyncSession const& session, SyncSession::ResolvedPlan plan, - SharedData::Sync::SyncExecutionRequest const& request + SharedData::Sync::SyncExecutionRequest const& request, + LocalOperationQueue& localDeleteQueue ) { // Assumed in strand From b57991bbd2c740187525f5aa9f72e50b706acfbe Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:54:32 +0200 Subject: [PATCH 075/131] RpcFilesystem::removeSome no longer removes whole trees RpcFilesystem::removeSome removes each item by itself only, a directory only when it is empty; deleting a directory with its content goes through the local operation queue, so no RPC removes a whole tree while the local filesystem calls of every tab wait --- backend/source/backend/rpc_filesystem.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/source/backend/rpc_filesystem.cpp b/backend/source/backend/rpc_filesystem.cpp index 41ed9bf3..a050f112 100644 --- a/backend/source/backend/rpc_filesystem.cpp +++ b/backend/source/backend/rpc_filesystem.cpp @@ -210,7 +210,6 @@ void RpcFilesystem::registerRemoveMultiple() return reply.error("Missing required parameter 'paths'."); } - const bool recursive = parameters.value("recursive", false); const auto pathsUtf8 = parameters["paths"].get>(); const auto removal = removeLocalItems(pathsUtf8, localHomeDirectories()); if (!removal) From b0558450d1314ab535f75ea9d72234674d9adac9 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 03:06:58 +0200 Subject: [PATCH 076/131] Closing a session tab ends its session in the backend Closing a session tab ends its session in the backend: the frontend sends the session id of SessionManager::disconnect as the object the handler reads, which it refused before, so the SSH session, its channels and its operations stayed in the backend until the program ended --- frontend/source/frontend/terminal/ssh_engine.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/source/frontend/terminal/ssh_engine.cpp b/frontend/source/frontend/terminal/ssh_engine.cpp index 89ff3b61..ca4e360d 100644 --- a/frontend/source/frontend/terminal/ssh_engine.cpp +++ b/frontend/source/frontend/terminal/ssh_engine.cpp @@ -99,6 +99,8 @@ void SshTerminalEngine::disconnect(std::function onDisconnect, bool byLo Log::info("Disconnecting session: {}", impl_->sshSessionId.value()); if (!byLossOfConnection) { + Nui::val parameters = Nui::val::object(); + parameters.set("sessionId", impl_->sshSessionId.value()); Nui::RpcClient::callWithBackChannel( "SessionManager::disconnect", [onDisconnect = std::move(onDisconnect)](Nui::val) @@ -107,7 +109,7 @@ void SshTerminalEngine::disconnect(std::function onDisconnect, bool byLo Log::info("Frontend SshEngine: Disconnected"); onDisconnect(); }, - impl_->sshSessionId.value() + parameters ); } else From d4f2caec7d16b95246bc683f045eaf203666e61b Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 03:07:02 +0200 Subject: [PATCH 077/131] Local delete cards show in the queue of the tab that started them Local delete cards show in the queue of the tab that started them and move to another tab's queue when that tab closes, or to the next queue that opens; a delete that ends with no queue open is reported as a notification. Explorer deletes of a selection with a directory always go through the local operation queue, also in tabs without a connection, a files-only selection is still deleted at once without recursion; cancel and Cancel All reach local deletes in the local operation queue, which keeps its own order; a queue counts as connected only with a valid session id --- .../local_operation_router.hpp | 90 +++++++ .../session_components/operation_queue.hpp | 6 +- frontend/source/frontend/CMakeLists.txt | 1 + frontend/source/frontend/session.cpp | 2 + .../local_operation_router.cpp | 231 ++++++++++++++++++ .../session_components/operation_queue.cpp | 49 +++- static/assets/languages/de_DE.yaml | 3 + static/assets/languages/en_US.yaml | 3 + 8 files changed, 379 insertions(+), 6 deletions(-) create mode 100644 frontend/include/frontend/session_components/local_operation_router.hpp create mode 100644 frontend/source/frontend/session_components/local_operation_router.cpp diff --git a/frontend/include/frontend/session_components/local_operation_router.hpp b/frontend/include/frontend/session_components/local_operation_router.hpp new file mode 100644 index 00000000..31c8f2de --- /dev/null +++ b/frontend/include/frontend/session_components/local_operation_router.hpp @@ -0,0 +1,90 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +class OperationQueue; + +/** + * @brief Shows the operations of the backend's LocalOperationQueue, which belong to no connection, in the operation + * queues of the session tabs. + * + * An operation shows in the queue that started it. When that queue goes away, with its tab or on a reconnect, the + * card moves to the queue attached last, and without any queue to the next one that attaches. A delete that ends + * while no queue is attached is reported as a notification. + */ +class LocalOperationRouter +{ + public: + static LocalOperationRouter& instance(); + + LocalOperationRouter(LocalOperationRouter const&) = delete; + LocalOperationRouter& operator=(LocalOperationRouter const&) = delete; + LocalOperationRouter(LocalOperationRouter&&) = delete; + LocalOperationRouter& operator=(LocalOperationRouter&&) = delete; + ~LocalOperationRouter() = default; + + /** + * @brief @p queue can show cards from now on and takes the cards that have no queue. + */ + void attach(OperationQueue& queue); + + /** + * @brief @p queue goes away, its cards move to another queue. + */ + void detach(OperationQueue& queue); + + /** + * @brief The events of @p operationId go to @p queue. Call before the operation is queued. + */ + void host(Ids::OperationId const& operationId, OperationQueue& queue); + + /** + * @brief Drops @p operationId, which was not queued or whose card is gone. + */ + void forget(Ids::OperationId const& operationId); + + /** + * @brief Cancels every operation shown in @p queue and drops it, for a queue that clears its cards. + */ + void cancelHostedBy(OperationQueue const& queue); + + private: + LocalOperationRouter(); + + struct Routed + { + OperationQueue* host{nullptr}; + std::optional added{}; + std::optional scanProgress{}; + std::optional deleteProgress{}; + }; + + void onOperationAdded(SharedData::OperationAdded const& added); + void onScanProgress(SharedData::ScanProgress const& progress); + void onDeleteProgress(SharedData::BulkDeleteProgress const& progress); + void onOperationCompleted(Nui::val val); + + /** + * @brief Shows the card of @p routed in @p queue as far as its operation got. + */ + static void show(Routed& routed, OperationQueue& queue); + + /** + * @brief The queue attached last, nullptr without any. + */ + OperationQueue* newestQueue() const; + + std::vector queues_{}; + std::unordered_map routed_{}; + std::vector registrations_{}; +}; diff --git a/frontend/include/frontend/session_components/operation_queue.hpp b/frontend/include/frontend/session_components/operation_queue.hpp index 9fae215d..32e16d56 100644 --- a/frontend/include/frontend/session_components/operation_queue.hpp +++ b/frontend/include/frontend/session_components/operation_queue.hpp @@ -39,6 +39,7 @@ class LocalSideModel; class RemoteSideModel; +class LocalOperationRouter; class OperationQueue { @@ -60,7 +61,7 @@ class OperationQueue RemoteSideModel* remoteModel ); - ROAR_PIMPL_SPECIAL_FUNCTIONS(OperationQueue); + ROAR_PIMPL_SPECIAL_FUNCTIONS_NO_MOVE(OperationQueue); void activate(std::shared_ptr fileEngine, Ids::SessionId sessionId); @@ -336,6 +337,8 @@ class OperationQueue Nui::ElementRenderer operator()(); private: + friend class LocalOperationRouter; + template void cancelOperation(OperationCard const& operation); void cancelAll(); @@ -353,6 +356,7 @@ class OperationQueue void onDeleteProgress(SharedData::BulkDeleteProgress const& progress); void onScanProgress(SharedData::ScanProgress const& progress); void onOperationCompleted(Nui::val val); + void onOperationCompleted(SharedData::OperationCompleted completed); void onIsPaused(SharedData::ErrorOrSuccess const& result); void onOperationsReordered(SharedData::OperationsReordered const& evt); void requestMoveOperation(Ids::OperationId const& opId, std::size_t newIndex); diff --git a/frontend/source/frontend/CMakeLists.txt b/frontend/source/frontend/CMakeLists.txt index 02c61618..fa02c681 100644 --- a/frontend/source/frontend/CMakeLists.txt +++ b/frontend/source/frontend/CMakeLists.txt @@ -141,6 +141,7 @@ target_sources( session_components/file_tracking.cpp session_components/session_options.cpp session_components/operation_queue.cpp + session_components/local_operation_router.cpp session_components/terminal_panel.cpp session_components/file_explorer_panel.cpp session_components/connection_loss_overlay.cpp diff --git a/frontend/source/frontend/session.cpp b/frontend/source/frontend/session.cpp index 79c6283d..6a495b18 100644 --- a/frontend/source/frontend/session.cpp +++ b/frontend/source/frontend/session.cpp @@ -525,6 +525,8 @@ std::optional Session::getLayout() const void Session::setupFileGrid() { impl_->fileExplorerPanel.setup(); + // Local deletes need no connection, their cards show in this queue from the start. + localSideModel().operationQueue(&impl_->operationQueue); // Sync dialog + progress dialog stay on Session; wire them through the // panel's synchronize callback. Every flow scans and diffs with the diff --git a/frontend/source/frontend/session_components/local_operation_router.cpp b/frontend/source/frontend/session_components/local_operation_router.cpp new file mode 100644 index 00000000..c25ed827 --- /dev/null +++ b/frontend/source/frontend/session_components/local_operation_router.cpp @@ -0,0 +1,231 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include + +namespace +{ + /** + * @brief Tells the user how a delete ended that no queue showed. + */ + void notifyUnshownCompletion(SharedData::OperationCompleted const& completed, std::string const& directory) + { + using enum SharedData::OperationCompletionReason; + switch (completed.reason) + { + case Completed: + { + if (completed.failedEntries.empty()) + return Notifications::success( + fmt::format(fmt::runtime(language->get("operationQueue", "localDeleteDone")), directory) + ); + return Notifications::warning( + fmt::format(fmt::runtime(language->get("operationQueue", "localDeletePartlyDone")), directory) + ); + } + case Failed: + { + return Notifications::error( + fmt::format( + fmt::runtime(language->get("operationQueue", "localDeleteFailed")), + directory, + completed.error ? completed.error->toString() : language->get("operationQueue", "unknownError") + ) + ); + } + case Canceled: + return; + } + } +} + +LocalOperationRouter& LocalOperationRouter::instance() +{ + static LocalOperationRouter router; + return router; +} + +LocalOperationRouter::LocalOperationRouter() +{ + registrations_.push_back( + Nui::RpcClient::autoRegisterFunction( + "LocalOperationQueue::onOperationAdded", + [this](SharedData::OperationAdded const& added) + { + onOperationAdded(added); + } + ) + ); + registrations_.push_back( + Nui::RpcClient::autoRegisterFunction( + "LocalOperationQueue::onScanProgress", + [this](SharedData::ScanProgress const& progress) + { + onScanProgress(progress); + } + ) + ); + registrations_.push_back( + Nui::RpcClient::autoRegisterFunction( + "LocalOperationQueue::onDeleteProgress", + [this](SharedData::BulkDeleteProgress const& progress) + { + onDeleteProgress(progress); + } + ) + ); + registrations_.push_back( + Nui::RpcClient::autoRegisterFunction( + "LocalOperationQueue::onOperationCompleted", + [this](Nui::val val) + { + onOperationCompleted(std::move(val)); + } + ) + ); +} + +void LocalOperationRouter::attach(OperationQueue& queue) +{ + queues_.push_back(&queue); + for (auto& [operationId, routed] : routed_) + { + if (routed.host) + continue; + routed.host = &queue; + show(routed, queue); + } +} + +void LocalOperationRouter::detach(OperationQueue& queue) +{ + std::erase(queues_, &queue); + auto* const successor = newestQueue(); + for (auto& [operationId, routed] : routed_) + { + if (routed.host != &queue) + continue; + routed.host = successor; + if (successor) + show(routed, *successor); + } +} + +void LocalOperationRouter::host(Ids::OperationId const& operationId, OperationQueue& queue) +{ + routed_[operationId].host = &queue; +} + +void LocalOperationRouter::forget(Ids::OperationId const& operationId) +{ + routed_.erase(operationId); +} + +void LocalOperationRouter::cancelHostedBy(OperationQueue const& queue) +{ + std::erase_if( + routed_, + [&queue](auto const& entry) + { + auto const& [operationId, routed] = entry; + if (routed.host != &queue) + return false; + Nui::RpcClient::callWithBackChannel( + "LocalOperationQueue::cancel", + [operationId](SharedData::ErrorOrSuccess<> const& result) + { + if (!result) + Log::warn( + "Failed to cancel the local operation '{}': {}", operationId.value(), result.error.value() + ); + }, + operationId + ); + return true; + } + ); +} + +void LocalOperationRouter::onOperationAdded(SharedData::OperationAdded const& added) +{ + auto [found, inserted] = routed_.try_emplace(added.operationId); + auto& routed = found->second; + if (inserted) + routed.host = newestQueue(); + routed.added = added; + if (routed.host) + routed.host->onOperationAdded(added); +} + +void LocalOperationRouter::onScanProgress(SharedData::ScanProgress const& progress) +{ + const auto found = routed_.find(progress.operationId); + if (found == routed_.end()) + return; + found->second.scanProgress = progress; + if (found->second.host) + found->second.host->onScanProgress(progress); +} + +void LocalOperationRouter::onDeleteProgress(SharedData::BulkDeleteProgress const& progress) +{ + const auto found = routed_.find(progress.operationId); + if (found == routed_.end()) + return; + found->second.deleteProgress = progress; + if (found->second.host) + found->second.host->onDeleteProgress(progress); +} + +void LocalOperationRouter::onOperationCompleted(Nui::val val) +{ + SharedData::OperationCompleted completed{}; + try + { + Nui::convertFromVal(val, completed); + } + catch (std::exception const& exception) + { + Log::error("Failed to read the completion of a local operation: {}", exception.what()); + return; + } + + auto routed = routed_.extract(completed.operationId); + if (routed.empty()) + { + Log::info("Local operation '{}' ended without a card.", completed.operationId.value()); + return; + } + if (auto* const host = routed.mapped().host; host) + return host->onOperationCompleted(std::move(completed)); + + auto const& added = routed.mapped().added; + notifyUnshownCompletion(completed, added && added->localPath ? added->localPath->toUtf8() : std::string{}); +} + +void LocalOperationRouter::show(Routed& routed, OperationQueue& queue) +{ + if (!routed.added) + return; + queue.onOperationAdded(*routed.added); + if (routed.scanProgress) + queue.onScanProgress(*routed.scanProgress); + if (routed.deleteProgress) + queue.onDeleteProgress(*routed.deleteProgress); +} + +OperationQueue* LocalOperationRouter::newestQueue() const +{ + return queues_.empty() ? nullptr : queues_.back(); +} diff --git a/frontend/source/frontend/session_components/operation_queue.cpp b/frontend/source/frontend/session_components/operation_queue.cpp index e6a45964..ad77505b 100644 --- a/frontend/source/frontend/session_components/operation_queue.cpp +++ b/frontend/source/frontend/session_components/operation_queue.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -379,11 +380,17 @@ OperationQueue::OperationQueue( impl_->autoCleanTimer = std::move(handle); } ); + LocalOperationRouter::instance().attach(*this); +} + +OperationQueue::~OperationQueue() +{ + LocalOperationRouter::instance().detach(*this); } -ROAR_PIMPL_SPECIAL_FUNCTIONS_IMPL(OperationQueue); void OperationQueue::cancelAll() { + LocalOperationRouter::instance().cancelHostedBy(*this); impl_->operations.clear(); impl_->priorityOperations.clear(); impl_->history.clear(); @@ -405,10 +412,14 @@ void OperationQueue::cancelOperation(OperationCard const& operation) return; } - auto doCancel = [this, operationId = operation.operationId()]() + auto doCancel = [this, + operationId = operation.operationId(), + cancelName = operation.type() == SharedData::OperationType::LocalDelete + ? std::string{"LocalOperationQueue::cancel"} + : fmt::format("OperationQueue::{}::cancel", impl_->sessionId.value())]() { Nui::RpcClient::callWithBackChannel( - fmt::format("OperationQueue::{}::cancel", impl_->sessionId.value()), + cancelName, [this, operationId](SharedData::ErrorOrSuccess<> const& result) { if (!result) @@ -960,7 +971,7 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) // Kick-to-top is meaningful only for the regular queue. Priority cards do // not get a handler — even though the button is rendered, CSS hides it // when the parent is the priority list (see .opq-priority-list rule). - if (added.mode != SharedData::OperationMode::PriorityQueued) + if (added.mode != SharedData::OperationMode::PriorityQueued && added.type != SharedData::OperationType::LocalDelete) { card->setKickToTopHandler( [this, opId = added.operationId]() @@ -1284,7 +1295,11 @@ void OperationQueue::onOperationCompleted(Nui::val val) Log::error("Failed to convert OperationCompleted: {}", e.what()); return; } + onOperationCompleted(std::move(completed)); +} +void OperationQueue::onOperationCompleted(SharedData::OperationCompleted completed) +{ Log::info("Received operation completed for operation id: {}", completed.operationId.value()); auto* operation = impl_->findOperation(completed.operationId); if (!operation) @@ -1593,7 +1608,22 @@ void OperationQueue::executeSyncSelection( { if (!impl_->fileEngine) return onRejected("No file engine set"); - impl_->fileEngine->executeSyncSelection(std::move(request), std::move(onResolved), std::move(onRejected)); + const auto localDeleteId = request.localDeleteId; + LocalOperationRouter::instance().host(localDeleteId, *this); + impl_->fileEngine->executeSyncSelection( + std::move(request), + [onResolved = std::move(onResolved), localDeleteId](SharedData::Sync::SyncExecutionResult result) + { + if (result.localDeletes.empty()) + LocalOperationRouter::instance().forget(localDeleteId); + onResolved(std::move(result)); + }, + [onRejected = std::move(onRejected), localDeleteId](std::string const& reason) + { + LocalOperationRouter::instance().forget(localDeleteId); + onRejected(reason); + } + ); } void OperationQueue::cancelSyncDiff(Ids::SyncSessionId syncSessionId) @@ -1654,6 +1684,10 @@ void OperationQueue::requestMoveOperation(Ids::OperationId const& opId, std::siz // avoids a wasted round trip and a refused log on the backend side. return; } + // Local operations run in the backend's local operation queue, which keeps its own order. + if (const auto* operation = impl_->findOperation(opId); + !isActive() || (operation && operation->type() == SharedData::OperationType::LocalDelete)) + return; Nui::RpcClient::callWithBackChannel( fmt::format("OperationQueue::{}::moveOperation", impl_->sessionId.value()), [opId](SharedData::ErrorOrSuccess<> const& result) @@ -1707,6 +1741,11 @@ void OperationQueue::togglePause() void OperationQueue::askBackendToCancelAll() { + if (!isActive()) + { + cancelAll(); + return; + } Nui::RpcClient::callWithBackChannel( fmt::format("OperationQueue::{}::cancelAll", impl_->sessionId.value()), [this](SharedData::ErrorOrSuccess<> const& result) diff --git a/static/assets/languages/de_DE.yaml b/static/assets/languages/de_DE.yaml index a7b0164a..eaf0e33c 100644 --- a/static/assets/languages/de_DE.yaml +++ b/static/assets/languages/de_DE.yaml @@ -153,6 +153,9 @@ operationQueue: deletingItem: "Lösche: '{}'" deletingSelectedIn: "Lösche ausgewählte Elemente in {}" deletingLocallyIn: "Lösche lokale Elemente in {}" + localDeleteDone: "Die lokalen Elemente in {} wurden gelöscht." + localDeletePartlyDone: "Nicht alle lokalen Elemente in {} konnten gelöscht werden." + localDeleteFailed: "Die lokalen Elemente in {} konnten nicht gelöscht werden: {}" entriesFailed: "{} Einträge fehlgeschlagen:\n" errorButton: "Fehler!" livePage: "Live" diff --git a/static/assets/languages/en_US.yaml b/static/assets/languages/en_US.yaml index 606a02d9..8af1bdb4 100644 --- a/static/assets/languages/en_US.yaml +++ b/static/assets/languages/en_US.yaml @@ -93,6 +93,9 @@ operationQueue: deletingItem: "Deleting: '{}'" deletingSelectedIn: "Deleting selected items in {}" deletingLocallyIn: "Deleting local items in {}" + localDeleteDone: "The local items in {} were deleted." + localDeletePartlyDone: "Not every local item in {} could be deleted." + localDeleteFailed: "The local items in {} could not be deleted: {}" entriesFailed: "{} entries failed:\n" errorButton: "Error!" livePage: "Live" From e4a7542a640bb5c26663c5f18cd16131f2e97a43 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:22:28 +0200 Subject: [PATCH 078/131] Tests: A local no-replace rename falls back where the filesystem refuses it --- utility/test/utility/test_rename.hpp | 78 ++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/utility/test/utility/test_rename.hpp b/utility/test/utility/test_rename.hpp index f96170f8..ebeae1a7 100644 --- a/utility/test/utility/test_rename.hpp +++ b/utility/test/utility/test_rename.hpp @@ -69,6 +69,84 @@ namespace Utility::Test EXPECT_EQ(read(directory_.path() / "target" / "inside"), "x"); } + TEST_F(RenameNoReplaceTests, RenameUnlessExistsRenamesToAFreeName) + { + write(directory_.path() / "a", "a"); + ASSERT_TRUE(Detail::renameUnlessExists(directory_.path() / "a", directory_.path() / "b").has_value()); + EXPECT_FALSE(std::filesystem::exists(directory_.path() / "a")); + EXPECT_EQ(read(directory_.path() / "b"), "a"); + } + + TEST_F(RenameNoReplaceTests, RenameUnlessExistsRefusesAnExistingFile) + { + write(directory_.path() / "a", "a"); + write(directory_.path() / "b", "b"); + const auto result = Detail::renameUnlessExists(directory_.path() / "a", directory_.path() / "b"); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), std::errc::file_exists); + EXPECT_EQ(read(directory_.path() / "a"), "a"); + EXPECT_EQ(read(directory_.path() / "b"), "b"); + } + +#ifndef _WIN32 + TEST_F(RenameNoReplaceTests, RenameUnlessExistsRefusesADanglingLink) + { + write(directory_.path() / "a", "a"); + std::filesystem::create_symlink(directory_.path() / "missing", directory_.path() / "b"); + const auto result = Detail::renameUnlessExists(directory_.path() / "a", directory_.path() / "b"); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), std::errc::file_exists); + EXPECT_TRUE(std::filesystem::is_symlink(std::filesystem::symlink_status(directory_.path() / "b"))); + EXPECT_EQ(read(directory_.path() / "a"), "a"); + } + + TEST_F(RenameNoReplaceTests, RenameByLinkMovesAFile) + { + write(directory_.path() / "a", "a"); + ASSERT_TRUE(Detail::renameByLink(directory_.path() / "a", directory_.path() / "b").has_value()); + EXPECT_FALSE(std::filesystem::exists(directory_.path() / "a")); + EXPECT_EQ(read(directory_.path() / "b"), "a"); + } + + TEST_F(RenameNoReplaceTests, RenameByLinkRefusesAnExistingFile) + { + write(directory_.path() / "a", "a"); + write(directory_.path() / "b", "b"); + const auto result = Detail::renameByLink(directory_.path() / "a", directory_.path() / "b"); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), std::errc::file_exists); + EXPECT_FALSE(Detail::isNoReplaceUnsupported(result.error())); + EXPECT_EQ(read(directory_.path() / "a"), "a"); + EXPECT_EQ(read(directory_.path() / "b"), "b"); + } +#endif + + TEST(RenameNoReplaceFallbackTests, OnlyMissingSupportFallsBack) + { + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::invalid_argument))); + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::function_not_supported))); + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::operation_not_supported))); + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::not_supported))); + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::operation_not_permitted))); + EXPECT_FALSE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::file_exists))); + EXPECT_FALSE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::no_such_file_or_directory))); + EXPECT_FALSE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::permission_denied))); + } + + TEST_F(RenameNoReplaceTests, MoveIntoPlaceReplacesOnlyWhenAllowed) + { + write(directory_.path() / "a", "a"); + write(directory_.path() / "b", "b"); + const auto refused = moveIntoPlace(directory_.path() / "a", directory_.path() / "b", false); + ASSERT_FALSE(refused.has_value()); + EXPECT_EQ(refused.error(), std::errc::file_exists); + EXPECT_EQ(read(directory_.path() / "b"), "b"); + + ASSERT_TRUE(moveIntoPlace(directory_.path() / "a", directory_.path() / "b", true).has_value()); + EXPECT_FALSE(std::filesystem::exists(directory_.path() / "a")); + EXPECT_EQ(read(directory_.path() / "b"), "a"); + } + TEST(RenameTargetTests, OnlyANewNameInTheSameDirectoryIsARenameInPlace) { const std::filesystem::path old{"/data/old.txt"}; From 91b6415411bcd6f56cf07069c59e70eaf887c4ae Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:26:22 +0200 Subject: [PATCH 079/131] Tests: A remote no-replace rename falls back when the server refuses hard links --- ssh/test/ssh/test_sftp.hpp | 162 ++++++++++++++++-- .../ssh/test_ssh2_servers/sftp_server.mjs | 28 ++- 2 files changed, 176 insertions(+), 14 deletions(-) diff --git a/ssh/test/ssh/test_sftp.hpp b/ssh/test/ssh/test_sftp.hpp index 845b1e8f..6a20527d 100644 --- a/ssh/test/ssh/test_sftp.hpp +++ b/ssh/test/ssh/test_sftp.hpp @@ -1042,28 +1042,164 @@ namespace SecureShell::Test } } + namespace + { + std::expected renameReplace(SftpSession& sftp) + { + auto future = sftp.performPromise( + [&sftp]() + { + return sftp.renameReplaceInStrand( + remote("/home/test/source.txt"), remote("/home/test/target.txt"), ".aside" + ); + } + ); + if (future.wait_for(3s) != std::future_status::ready) + return std::unexpected(SftpError{.message = "timeout"}); + return future.get(); + } + + std::expected renameNoReplace( + SftpSession& sftp, + std::string_view destination, + std::optional sourceType = std::nullopt + ) + { + auto future = sftp.performPromise( + [&sftp, destination, sourceType]() + { + return sftp.renameNoReplaceInStrand(remote("/home/test/source.txt"), remote(destination), sourceType); + } + ); + if (future.wait_for(3s) != std::future_status::ready) + return std::unexpected(SftpError{.message = "timeout"}); + return future.get(); + } + } + TEST_F(SftpTests, FailedRenameReplaceRestoresTheDestination) { CREATE_SERVER_AND_JOINER(Sftp); ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); - // Moving the destination aside succeeds, the rename into place fails. + // Moving the destination aside succeeds, the rename into place fails, moving it back succeeds. + ASSERT_TRUE(serverStartResult->command("failRenames", {{"after", 1}, {"count", 1}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + EXPECT_FALSE(renameReplace(*sftp).has_value()); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "target"); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/target.txt.aside")); + EXPECT_EQ(readRemote(*sftp, "/home/test/source.txt"), "source"); + } + + TEST_F(SftpTests, FailedRollbackNamesWhereTheDestinationIsKept) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); ASSERT_TRUE(serverStartResult->command("failRenames", {{"after", 1}})); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto future = sftp->performPromise( - [&sftp]() - { - return sftp->renameReplaceInStrand(remote("/home/test/source.txt"), remote("/home/test/target.txt"), ".aside"); - } - ); - ASSERT_EQ(future.wait_for(3s), std::future_status::ready); - EXPECT_FALSE(future.get().has_value()); + const auto result = renameReplace(*sftp); + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().message.find("/home/test/target.txt.aside"), std::string::npos) << result.error().message; ASSERT_TRUE(serverStartResult->command("failRenames", {{"enabled", false}})); - // The restore is a rename as well and ran while renames failed, so the original sits aside: never lost. - const bool restored = readRemote(*sftp, "/home/test/target.txt") == "target"; - const bool keptAside = readRemote(*sftp, "/home/test/target.txt.aside") == "target"; - EXPECT_TRUE(restored || keptAside); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt.aside"), "target"); EXPECT_EQ(readRemote(*sftp, "/home/test/source.txt"), "source"); } + + TEST_F(SftpTests, RenameReplaceRemovesALeftoverAside) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt.aside"}, {"content", "old"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + ASSERT_TRUE(renameReplace(*sftp).has_value()); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "source"); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/target.txt.aside")); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/source.txt")); + } + + TEST_F(SftpTests, RenameReplacePutsBackADestinationThatOnlyExistsAside) + { + for (const bool failInPlace : {false, true}) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE( + serverStartResult->command("createFile", {{"path", "/home/test/target.txt.aside"}, {"content", "target"}}) + ); + // Putting it back and moving it aside again succeed, the rename into place fails. + if (failInPlace) + ASSERT_TRUE(serverStartResult->command("failRenames", {{"after", 2}, {"count", 1}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto result = renameReplace(*sftp); + EXPECT_EQ(result.has_value(), !failInPlace); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), failInPlace ? "target" : "source"); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/target.txt.aside")) << "failInPlace: " << failInPlace; + } + } + + TEST_F(SftpTests, RenameNoReplaceRenamesWhenTheServerRefusesHardlinks) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "hardlink@openssh.com"}})); + ASSERT_TRUE(serverStartResult->command("failHardlinks", nlohmann::json::object())); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto refused = renameNoReplace(*sftp, "/home/test/target.txt", SharedData::FileType::Regular); + ASSERT_FALSE(refused.has_value()); + EXPECT_EQ(refused.error().sftpError, SSH_FX_FILE_ALREADY_EXISTS); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "target"); + + ASSERT_TRUE(renameNoReplace(*sftp, "/home/test/moved.txt", SharedData::FileType::Regular).has_value()); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/source.txt")); + EXPECT_EQ(readRemote(*sftp, "/home/test/moved.txt"), "source"); + } + + TEST_F(SftpTests, RenameNoReplaceSucceedsWhenTheLinkedSourceCannotBeRemoved) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "hardlink@openssh.com"}})); + ASSERT_TRUE(serverStartResult->command("failRemoves", nlohmann::json::object())); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + ASSERT_TRUE(renameNoReplace(*sftp, "/home/test/moved.txt").has_value()); + EXPECT_EQ(readRemote(*sftp, "/home/test/moved.txt"), "source"); + } + + TEST_F(SftpTests, MoveIntoPlaceReplacesOnlyWhenAllowed) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto move = [&sftp](bool replace) + { + auto future = sftp->performPromise( + [&sftp, replace]() + { + return sftp->moveIntoPlaceInStrand( + remote("/home/test/source.txt"), remote("/home/test/target.txt"), replace, ".aside" + ); + } + ); + return future.get(); + }; + const auto refused = move(false); + ASSERT_FALSE(refused.has_value()); + EXPECT_EQ(refused.error().sftpError, SSH_FX_FILE_ALREADY_EXISTS); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "target"); + + ASSERT_TRUE(move(true).has_value()); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "source"); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/source.txt")); + } } diff --git a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs index c84a80c9..f9e11748 100644 --- a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs +++ b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs @@ -169,6 +169,10 @@ const faults = { failRenames: false, // Renames that still succeed before failRenames applies, null for none. renamesBeforeFailure: null, + // Renames that fail before renames succeed again, null for all. + renamesFailing: null, + failHardlinks: false, + failRemoves: false, deniedDirectories: new Set(), deniedStats: new Set(), bytesRead: 0, @@ -185,6 +189,11 @@ const renameFails = () => { --faults.renamesBeforeFailure; return false; } + if (faults.renamesFailing !== null) { + if (faults.renamesFailing === 0) + return false; + --faults.renamesFailing; + } return true; }; @@ -598,6 +607,10 @@ const server = new Server({ logMessage('Cannot remove directories'); return sftpStream.status(reqid, STATUS_CODE.FAILURE); } + if (faults.failRemoves) { + logMessage('Fault: failing REMOVE'); + return sftpStream.status(reqid, STATUS_CODE.PERMISSION_DENIED); + } result.parent.removeChild(result.name); @@ -663,6 +676,10 @@ const server = new Server({ } if (name === 'hardlink@openssh.com') { + if (faults.failHardlinks) { + logMessage('Fault: failing hardlink'); + return sftpStream.status(reqid, STATUS_CODE.PERMISSION_DENIED); + } if (newResult !== undefined || oldResult.type !== 'file') return sftpStream.status(reqid, STATUS_CODE.FAILURE); const parent = fakeFilesystem.find(pathUtil.dirname(newPath)); @@ -919,6 +936,14 @@ cli.on('dropAfterBytes', (command) => { cli.on('failRenames', (command) => { faults.failRenames = command.enabled !== false; faults.renamesBeforeFailure = command.after === undefined ? null : Number(command.after); + faults.renamesFailing = command.count === undefined ? null : Number(command.count); +}); +// Hard links answer permission denied, like a filesystem without them or Linux protected_hardlinks. +cli.on('failHardlinks', (command) => { + faults.failHardlinks = command.enabled !== false; +}); +cli.on('failRemoves', (command) => { + faults.failRemoves = command.enabled !== false; }); cli.on('denyDirectory', (command) => { faults.deniedDirectories.add(command.path); @@ -955,7 +980,8 @@ cli.on('createDirectory', (command) => { const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); if (parent === undefined || parent.type !== 'directory') return; - parent.insert(directory({ name: pathUtil.posix.basename(command.path) }, [])); + const stat = command.mode === undefined ? undefined : { mode: Number(command.mode) }; + parent.insert(directory({ name: pathUtil.posix.basename(command.path), stat }, [])); }); cli.on('createSymlink', (command) => { const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); From 1807162f94593be931e3eb07f5f467a4ce89a4b5 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:30:35 +0200 Subject: [PATCH 080/131] Tests: A timed-out queued rename keeps its paths; transfers await their tasks --- .../test/backend/test_transfer_failures.hpp | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/backend/test/backend/test_transfer_failures.hpp b/backend/test/backend/test_transfer_failures.hpp index 4be7d0d7..017fa9c8 100644 --- a/backend/test/backend/test_transfer_failures.hpp +++ b/backend/test/backend/test_transfer_failures.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include "real_server_tests.hpp" @@ -82,6 +83,36 @@ namespace Test Utility::TemporaryDirectory localDirectory_{temporaryRoot, true}; }; + TEST_F(TransferFailureTests, RenameThatTimedOutRunsAfterTheOperationIsGone) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + std::promise release{}; + auto blocked = sftp->performPromise( + [released = release.get_future().share()]() + { + released.wait(); + } + ); + { + RenameOperation operation{ + sftp, + { + .sourcePath = remote("/home/test/file1.txt"), + .destinationPath = remote("/home/test/moved.txt"), + .futureTimeout = std::chrono::seconds{1}, + } + }; + const auto result = operation.work(); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::FutureTimeout); + } + release.set_value(); + blocked.wait(); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/moved.txt"), "Fake file content"); + } + TEST_F(TransferFailureTests, AsyncDownloadReadFailureKeepsExistingFile) { CREATE_SERVER_AND_JOINER(sftpServer); From 3827b7e9e8727fe01c386bcf4327084f3b5e66ee Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:33:11 +0200 Subject: [PATCH 081/131] Tests: A download with reserved space replaces only when every byte arrived --- .../test/backend/test_transfer_failures.hpp | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/backend/test/backend/test_transfer_failures.hpp b/backend/test/backend/test_transfer_failures.hpp index 017fa9c8..31c1b895 100644 --- a/backend/test/backend/test_transfer_failures.hpp +++ b/backend/test/backend/test_transfer_failures.hpp @@ -218,6 +218,91 @@ namespace Test } } + TEST_F(TransferFailureTests, ReservedDownloadOfFileShorterThanListedKeepsExistingFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + for (const bool bigFileOptimized : {false, true}) + { + const auto localPath = localDirectory_.path() / fmt::format("reserved_{}.txt", bigFileOptimized); + writeLocalFile(localPath, "original content"); + + SharedData::DirectoryEntry entry{}; + entry.name = "large.txt"; + entry.type = SharedData::FileType::Regular; + entry.size = 70000u; + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .reserveSpace = true, + .bigFileOptimized = bigFileOptimized, + .entry = entry, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()) << "bigFileOptimized: " << bigFileOptimized; + if (!bigFileOptimized) + EXPECT_EQ(result.error().type, Operation::ErrorType::SizeMismatch) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), "original content") << "bigFileOptimized: " << bigFileOptimized; + } + } + + TEST_F(TransferFailureTests, ReservedDownloadResumesFromPartialFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + for (const bool bigFileOptimized : {false, true}) + { + const auto localPath = localDirectory_.path() / fmt::format("resumed_{}.txt", bigFileOptimized); + writeLocalFile(localPath.generic_string() + ".filepart", alphabet(65536).substr(0, 20000)); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + .reserveSpace = true, + .tryContinue = true, + .bigFileOptimized = bigFileOptimized, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), alphabet(65536)) << "bigFileOptimized: " << bigFileOptimized; + } + } + + TEST_F(TransferFailureTests, ReservedCompletePartialFileIsFinalized) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "file1.txt"; + writeLocalFile(localPath.generic_string() + ".filepart", "Fake file content"); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .reserveSpace = true, + .tryContinue = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), "Fake file content"); + } + TEST_F(TransferFailureTests, AsyncDownloadResumesFromPartialFile) { CREATE_SERVER_AND_JOINER(sftpServer); From 45230dd1d3243b983a3de3881035e9c9c97b373a Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:34:04 +0200 Subject: [PATCH 082/131] Tests: Add Utility::restrictToOwner and Utility::newFilePermissions --- utility/test/utility/test_permissions.hpp | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/utility/test/utility/test_permissions.hpp b/utility/test/utility/test_permissions.hpp index d0fb0cb7..b570dc2d 100644 --- a/utility/test/utility/test_permissions.hpp +++ b/utility/test/utility/test_permissions.hpp @@ -1,10 +1,16 @@ #pragma once #include +#include #include #include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; namespace Utility::Test { @@ -30,4 +36,43 @@ namespace Utility::Test EXPECT_EQ(*inherited, perms::all & ~processUmask()); EXPECT_EQ(*inherited & processUmask(), perms::none); } + +#ifndef _WIN32 + TEST(PermissionsTests, NewFilePermissionsAreThoseOfACreatedFile) + { + const TemporaryDirectory directory{temporaryRoot, true}; + std::ofstream{directory.path() / "created"} << "x"; + EXPECT_EQ(std::filesystem::status(directory.path() / "created").permissions(), newFilePermissions()); + } + + TEST(PermissionsTests, RestrictToOwnerCreatesAPrivateFile) + { + const TemporaryDirectory directory{temporaryRoot, true}; + const auto path = directory.path() / "private"; + ASSERT_TRUE(restrictToOwner(path).has_value()); + EXPECT_EQ(std::filesystem::status(path).permissions(), perms::owner_read | perms::owner_write); + } + + TEST(PermissionsTests, RestrictToOwnerKeepsTheContentOfAnExistingFile) + { + const TemporaryDirectory directory{temporaryRoot, true}; + const auto path = directory.path() / "existing"; + std::ofstream{path} << "content"; + std::filesystem::permissions(path, perms::owner_all | perms::group_read | perms::others_read); + ASSERT_TRUE(restrictToOwner(path).has_value()); + EXPECT_EQ(std::filesystem::status(path).permissions(), perms::owner_read | perms::owner_write); + std::ifstream file{path}; + EXPECT_EQ(std::string(std::istreambuf_iterator{file}, std::istreambuf_iterator{}), "content"); + } + + TEST(PermissionsTests, RestrictToOwnerRefusesALink) + { + const TemporaryDirectory directory{temporaryRoot, true}; + std::ofstream{directory.path() / "target"} << "x"; + const auto before = std::filesystem::status(directory.path() / "target").permissions(); + std::filesystem::create_symlink(directory.path() / "target", directory.path() / "link"); + EXPECT_FALSE(restrictToOwner(directory.path() / "link").has_value()); + EXPECT_EQ(std::filesystem::status(directory.path() / "target").permissions(), before); + } +#endif } From ad5dc28e1570cfe11f440922c3d64805a7c670d6 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:36:39 +0200 Subject: [PATCH 083/131] Tests: Transfers set times and permissions on the temp file before the move --- .../test/backend/test_transfer_failures.hpp | 75 +++++++++++++++++++ .../ssh/test_ssh2_servers/sftp_server.mjs | 2 +- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/backend/test/backend/test_transfer_failures.hpp b/backend/test/backend/test_transfer_failures.hpp index 31c1b895..3a1a80ae 100644 --- a/backend/test/backend/test_transfer_failures.hpp +++ b/backend/test/backend/test_transfer_failures.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "real_server_tests.hpp" #include @@ -113,6 +114,79 @@ namespace Test EXPECT_EQ(readRemoteFile(*sftp, "/home/test/moved.txt"), "Fake file content"); } +#ifndef _WIN32 + TEST_F(TransferFailureTests, DownloadGetsItsPermissionsAndTimeBeforeItIsMovedIntoPlace) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/private.txt"}, {"content", "private"}, {"mode", 0640}} + )); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + struct Case + { + bool inheritPermissions; + std::optional filePermissions; + std::filesystem::perms expected; + }; + using std::filesystem::perms; + const std::vector cases{ + {false, std::nullopt, Utility::newFilePermissions()}, + {true, std::nullopt, (perms::owner_read | perms::owner_write | perms::group_read) & ~Utility::processUmask()}, + {false, perms::owner_read | perms::owner_write, perms::owner_read | perms::owner_write}, + }; + for (std::size_t index = 0; index < cases.size(); ++index) + { + auto const& testCase = cases[index]; + const auto localPath = localDirectory_.path() / fmt::format("private_{}.txt", index); + SharedData::DirectoryEntry entry{}; + entry.name = "private.txt"; + entry.type = SharedData::FileType::Regular; + entry.size = 7u; + entry.mtime = 1700000000u; + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/private.txt"), + .localPath = toLocal(localPath), + .inheritPermissions = testCase.inheritPermissions, + .filePermissions = testCase.filePermissions, + .entry = entry, + .markAsDownloaded = false, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), "private"); + EXPECT_EQ(std::filesystem::status(localPath).permissions(), testCase.expected) << "case " << index; + const auto modified = std::chrono::file_clock::to_sys(std::filesystem::last_write_time(localPath)); + EXPECT_EQ(std::chrono::duration_cast(modified.time_since_epoch()).count(), 1700000000); + } + } + + TEST_F(TransferFailureTests, DownloadTempFileIsOnlyReadableByItsOwner) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "large.txt"; + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + } + }; + ASSERT_TRUE(operation.prepare().has_value()); + EXPECT_EQ( + std::filesystem::status(localPath.generic_string() + ".filepart").permissions(), + std::filesystem::perms::owner_read | std::filesystem::perms::owner_write + ); + } +#endif + TEST_F(TransferFailureTests, AsyncDownloadReadFailureKeepsExistingFile) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -521,6 +595,7 @@ namespace Test const auto result = runToEnd(operation); ASSERT_FALSE(result.has_value()); EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "Fake file content"); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt.filepart"), ""); } } diff --git a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs index f9e11748..205c1791 100644 --- a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs +++ b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs @@ -974,7 +974,7 @@ cli.on('createFile', (command) => { const content = command.size !== undefined ? Buffer.alloc(Number(command.size), command.fill ?? 'x') : (command.content ?? ''); - parent.insert(file(name, content)); + parent.insert(file(name, content, command.mode === undefined ? undefined : { mode: Number(command.mode) })); }); cli.on('createDirectory', (command) => { const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); From f5e1b4ba8495d7847e3107e52a9de5d0e85eac5f Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:37:49 +0200 Subject: [PATCH 084/131] Tests: A downloaded symlink is created under the temp name and moved into place --- .../test/backend/test_download_operation.hpp | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/backend/test/backend/test_download_operation.hpp b/backend/test/backend/test_download_operation.hpp index 761f73b6..0fcb286b 100644 --- a/backend/test/backend/test_download_operation.hpp +++ b/backend/test/backend/test_download_operation.hpp @@ -616,6 +616,43 @@ namespace Test EXPECT_FALSE(std::filesystem::is_regular_file(std::filesystem::symlink_status(localPath))); } + TEST_F(DownloadOperationTests, DownloadRemoteSymlinkReplacesOnlyWhenAllowed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + for (const bool mayOverwrite : {false, true}) + { + const auto localPath = downloadDir_.path() / fmt::format("existing_{}", mayOverwrite); + std::ofstream{localPath} << "existing"; + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/link_to_file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = mayOverwrite, + } + }; + const auto result = [&operation]() + { + for (;;) + { + auto step = operation.work(); + if (!step.has_value() || step.value() == DownloadOperation::WorkStatus::Complete) + return step; + } + }(); + ASSERT_EQ(result.has_value(), mayOverwrite); + if (!mayOverwrite) + EXPECT_EQ(result.error().type, Operation::ErrorType::FileExists); + EXPECT_EQ(std::filesystem::is_symlink(std::filesystem::symlink_status(localPath)), mayOverwrite); + if (mayOverwrite) + EXPECT_EQ(std::filesystem::read_symlink(localPath), "/home/test/file1.txt"); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::symlink_status(localPath.string() + ".filepart"))); + } + } + TEST_F(DownloadOperationTests, DownloadRemoteSymlinkToDirectoryCreatesLocalSymlink) { CREATE_SERVER_AND_JOINER(sftpServer); From 302b184920ef723f2806c7df2cf27ccef2c0b3f1 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:39:23 +0200 Subject: [PATCH 085/131] Tests: A bulk download applies directory permissions once every entry is done --- .../backend/test_bulk_download_operation.hpp | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/backend/test/backend/test_bulk_download_operation.hpp b/backend/test/backend/test_bulk_download_operation.hpp index e3871039..2fee4c9e 100644 --- a/backend/test/backend/test_bulk_download_operation.hpp +++ b/backend/test/backend/test_bulk_download_operation.hpp @@ -5,6 +5,7 @@ #include "real_server_tests.hpp" #include +#include #include #include @@ -1127,5 +1128,53 @@ namespace Test EXPECT_EQ(mode & std::filesystem::perms::others_write, std::filesystem::perms::none); EXPECT_TRUE(std::filesystem::exists(localRoot / "doc1.txt")); } + + TEST_F(BulkDownloadOperationTests, AReadOnlyDirectoryStillReceivesItsContent) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + using std::filesystem::perms; + const auto readOnly = perms::owner_read | perms::owner_exec | perms::group_read | perms::group_exec | + perms::others_read | perms::others_exec; + std::vector entries; + entries.push_back(SharedData::DirectoryEntry{.type = SharedData::FileType::Directory}); + entries.push_back( + SharedData::DirectoryEntry{ + .name = "Documents", + .type = SharedData::FileType::Directory, + .permissions = readOnly, + .parent = 0, + } + ); + entries.push_back( + SharedData::DirectoryEntry{ + .name = "doc1.txt", + .type = SharedData::FileType::Regular, + .size = std::string{"Document 1 content"}.size(), + .parent = 1, + } + ); + + const auto localRoot = downloadDir_.path() / "readonly"; + auto restore = Nui::ScopeExit{[&localRoot]() noexcept { + std::error_code error{}; + std::filesystem::permissions(localRoot / "Documents", perms::owner_all, error); + }}; + BulkDownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test"), + .localPath = toLocal(localRoot), + .individualOptions = {.inheritPermissions = true}, + } + }; + operation.setScanResult(std::move(entries), std::string{"Document 1 content"}.size()); + ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); + + EXPECT_TRUE(operation.getFailed().empty()); + EXPECT_EQ(readLocalFile(localRoot / "Documents" / "doc1.txt"), "Document 1 content"); + EXPECT_EQ(std::filesystem::status(localRoot / "Documents").permissions(), readOnly & ~Utility::processUmask()); + } #endif } From 664abc10887752892d1b843fff708f1e7da9828d Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:41:39 +0200 Subject: [PATCH 086/131] Tests: The tar writer gives members with unknown permissions the default mode --- tar-archive/test/tar_archive/test_header.hpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tar-archive/test/tar_archive/test_header.hpp b/tar-archive/test/tar_archive/test_header.hpp index 292e59d1..ee01884c 100644 --- a/tar-archive/test/tar_archive/test_header.hpp +++ b/tar-archive/test/tar_archive/test_header.hpp @@ -63,6 +63,26 @@ namespace TarArchive::Test EXPECT_EQ(parsed->uName, "alice"); } + TEST(TarHeader, UnknownPermissionsGetTheModeOfANewMember) + { + const auto modeOf = [](SharedData::FileType type) + { + TarArchive::MemberHeader entry{}; + entry.path = "member"; + entry.type = type; + if (type == SharedData::FileType::Symlink) + entry.linkTarget = "target"; + const auto built = buildRecords(entry); + EXPECT_TRUE(built.has_value()); + const auto parsed = parseRecord(built->records.back()); + EXPECT_TRUE(parsed.has_value()); + return static_cast(parsed->permissions); + }; + EXPECT_EQ(modeOf(SharedData::FileType::Regular), 0644u); + EXPECT_EQ(modeOf(SharedData::FileType::Directory), 0755u); + EXPECT_EQ(modeOf(SharedData::FileType::Symlink), 0777u); + } + TEST(TarHeader, BuildRecordsForDirectoryHasTrailingSlashAndNoSize) { TarArchive::MemberHeader entry{}; From eeda8206e85468db871e2f0e8dd52543ed0106b9 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 21:41:39 +0200 Subject: [PATCH 087/131] Tests: Uploaded archives never carry setuid, setgid or sticky bits --- backend/test/backend/archive_test_helpers.hpp | 2 + .../backend/test_archive_upload_operation.hpp | 45 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/backend/test/backend/archive_test_helpers.hpp b/backend/test/backend/archive_test_helpers.hpp index 352459dd..54333e2f 100644 --- a/backend/test/backend/archive_test_helpers.hpp +++ b/backend/test/backend/archive_test_helpers.hpp @@ -30,6 +30,7 @@ namespace Test std::string path{}; SharedData::FileType type{}; std::string contents{}; + std::filesystem::perms permissions{std::filesystem::perms::unknown}; }; inline SharedData::ArchiveDownloadRoot makeFileEntry(std::string_view path, std::uint64_t size) @@ -77,6 +78,7 @@ namespace Test RecoveredEntry recovered{}; recovered.path = entry.header().path; recovered.type = entry.header().type; + recovered.permissions = entry.header().permissions; if (recovered.type == SharedData::FileType::Regular) { diff --git a/backend/test/backend/test_archive_upload_operation.hpp b/backend/test/backend/test_archive_upload_operation.hpp index 5b38f90f..77026c7c 100644 --- a/backend/test/backend/test_archive_upload_operation.hpp +++ b/backend/test/backend/test_archive_upload_operation.hpp @@ -301,6 +301,51 @@ namespace Test EXPECT_EQ(deep->contents, "deep payload"); } +#ifndef _WIN32 + TEST_F(ArchiveUploadOperationTests, SpecialPermissionBitsNeverTravel) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + using std::filesystem::perms; + const auto executable = perms::owner_all | perms::group_read | perms::group_exec | perms::others_read | + perms::others_exec; + std::filesystem::permissions(srcDir_.path() / "alpha.txt", executable | perms::set_uid); + std::filesystem::permissions(srcDir_.path() / "nested" / "deep.txt", executable | perms::set_gid); + auto restore = Nui::ScopeExit{[this]() noexcept { + std::error_code error{}; + std::filesystem::permissions(srcDir_.path() / "alpha.txt", perms::owner_read | perms::owner_write, error); + std::filesystem::permissions( + srcDir_.path() / "nested" / "deep.txt", perms::owner_read | perms::owner_write, error + ); + }}; + if ((std::filesystem::status(srcDir_.path() / "alpha.txt").permissions() & perms::set_uid) == perms::none) + GTEST_SKIP() << "the filesystem drops setuid"; + + auto scan = runLocalScan(srcDir_.path() / "nested"); + const std::filesystem::path remoteArchive = "/home/test/special.tar"; + ArchiveUploadOperation operation{ + sftp, + { + .localPaths = {toLocal(srcDir_.path() / "alpha.txt"), toLocal(srcDir_.path() / "nested")}, + .remoteArchivePath = remote(remoteArchive.generic_string()), + .compression = TarArchive::Compression::None, + .mayOverwrite = true, + } + }; + operation.setScanResultForRoot(toLocal(srcDir_.path() / "nested"), std::move(scan.entries), scan.totalBytes); + ASSERT_TRUE(runToCompletion(operation).has_value()); + + const auto entries = recoverArchiveEntries(pullRemoteArchive(*sftp, remoteArchive, "special.tar")); + const auto* alpha = findEntry(entries, "alpha.txt"); + ASSERT_NE(alpha, nullptr); + EXPECT_EQ(alpha->permissions, executable); + const auto* deep = findEntry(entries, "nested/deep.txt"); + ASSERT_NE(deep, nullptr); + EXPECT_EQ(deep->permissions, executable); + } +#endif + TEST_F(ArchiveUploadOperationTests, DirectoryRootWithoutPrescanErrors) { CREATE_SERVER_AND_JOINER(sftpServer); From e1dac5aa2f34f670007ac2d16875a81f1779acfc Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:02:28 +0200 Subject: [PATCH 088/131] Tests: PathError::toString always yields UTF-8; one splitter for path segments --- paths/test/paths/test_path_segment.hpp | 78 +++++++++++++++++++++++++- paths/test/paths/test_remote_path.hpp | 31 ++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/paths/test/paths/test_path_segment.hpp b/paths/test/paths/test_path_segment.hpp index c78cd828..8ef7d558 100644 --- a/paths/test/paths/test_path_segment.hpp +++ b/paths/test/paths/test_path_segment.hpp @@ -5,6 +5,8 @@ #include #include +#include +#include namespace Paths::Test { @@ -66,11 +68,69 @@ namespace Paths::Test EXPECT_TRUE(LocalPathSegment::make(name, LocalPlatform::Windows).has_value()) << name; } + TEST(Utf8Tests, AcceptsOnlyWellFormedSequences) + { + for (auto const* text : {"", "ascii", "\xC3\xA4", "\xE2\x82\xAC", "\xF0\x9F\x9A\x80", "\xF4\x8F\xBF\xBF"}) + EXPECT_TRUE(isValidUtf8(text)) << text; + for (auto const* text : { + "\x80", + "\xBF", + "\xC3", + "\xE2\x82", + "\xF0\x9F\x9A", + "\xF4\x90\x80\x80", + "\xF5\x80\x80\x80", + "\xF8\x88\x80\x80\x80", + "\xFC\x84\x80\x80\x80\x80", + "\xFF", + "\xE0\x80\xAF", + "\xF0\x80\x80\xAF", + "\xED\xBF\xBF", + "a\xC3(b", + }) + EXPECT_FALSE(isValidUtf8(text)) << text; + } + + TEST(Utf8Tests, CountsUtf16Units) + { + EXPECT_EQ(utf16Length(""), 0u); + EXPECT_EQ(utf16Length("abc"), 3u); + EXPECT_EQ(utf16Length("\xC3\xA4\xE2\x82\xAC"), 2u); + EXPECT_EQ(utf16Length("\xF0\x9F\x9A\x80"), 2u); + } + + TEST(PathErrorTests, EscapesControlBytesAndInvalidUtf8) + { + const auto printable = [](std::string input) + { + return PathError{.reason = PathRejection::InvalidUtf8, .input = std::move(input)}.toString(); + }; + EXPECT_EQ(printable("plain"), "'plain' is not valid UTF-8"); + EXPECT_EQ(printable("tab\there\x7F"), "'tab\\x09here\\x7F' is not valid UTF-8"); + EXPECT_EQ(printable(std::string{"a\0b", 3}), "'a\\x00b' is not valid UTF-8"); + EXPECT_EQ(printable("Gr\xC3\xBC\xC3\x9F\xF0\x9F\x9A\x80"), "'Gr\xC3\xBC\xC3\x9F\xF0\x9F\x9A\x80' is not valid UTF-8"); + EXPECT_EQ(printable("a\xC3(\xFF\xE2\x82"), "'a\\xC3(\\xFF\\xE2\\x82' is not valid UTF-8"); + EXPECT_EQ(printable("\xED\xA0\x80"), "'\\xED\\xA0\\x80' is not valid UTF-8"); + EXPECT_TRUE(isValidUtf8(printable("\xC0\xAF\xF8\x88\x80\x80\x80"))); + } + TEST(RelativePathTests, ParsesAndJoinsSegments) { const auto path = RelativePath::make("a/b/c.txt"); ASSERT_TRUE(path.has_value()); - EXPECT_EQ(path->segments().size(), 3u); + ASSERT_EQ(path->segments().size(), 3u); + EXPECT_EQ(path->segments()[1].value(), "b"); + auto views = path->segmentViews(); + EXPECT_EQ( + std::vector(views.begin(), views.end()), (std::vector{"a", "b", "c.txt"}) + ); + EXPECT_TRUE(RelativePath{}.segments().empty()); + EXPECT_EQ((RelativePath{} / PathSegment::make("x").value()).value(), "x"); + EXPECT_EQ((*path / RelativePath{}).value(), "a/b/c.txt"); + EXPECT_EQ((RelativePath{} / *path).value(), "a/b/c.txt"); + EXPECT_EQ((*RelativePath::make("a") / *RelativePath::make("b/c")).value(), "a/b/c"); + EXPECT_FALSE(RelativePath{}.parent().has_value()); + EXPECT_EQ(RelativePath::make("a")->parent()->value(), ""); EXPECT_EQ(path->leaf()->value(), "c.txt"); EXPECT_EQ(path->parent()->value(), "a/b"); EXPECT_EQ(RelativePath{}.value(), ""); @@ -80,7 +140,21 @@ namespace Paths::Test TEST(RelativePathTests, RefusesAnythingThatCouldLeaveItsRoot) { - for (auto const* text : {"/a", "..", "a/../b", "a//b", "a/", "./a", "x/.."}) + for (auto const* text : {"/a", "..", "a/../b", "a//b", "a/", "./a", "x/..", "a/\xFF"}) EXPECT_FALSE(RelativePath::make(text).has_value()) << text; + EXPECT_EQ(RelativePath::make("/a").error().reason, PathRejection::EscapesRoot); + EXPECT_EQ(RelativePath::make(std::string{"a/b\0c", 5}).error().reason, PathRejection::ContainsNul); + } + + TEST(PathSplitTests, SplitsAtEverySlash) + { + const auto parts = [](std::string_view text) + { + auto split = Detail::splitAtSlashes(text); + return std::vector(split.begin(), split.end()); + }; + EXPECT_TRUE(parts("").empty()); + EXPECT_EQ(parts("a"), (std::vector{"a"})); + EXPECT_EQ(parts("/a//b/"), (std::vector{"", "a", "", "b", ""})); } } diff --git a/paths/test/paths/test_remote_path.hpp b/paths/test/paths/test_remote_path.hpp index 300632dc..542cc86f 100644 --- a/paths/test/paths/test_remote_path.hpp +++ b/paths/test/paths/test_remote_path.hpp @@ -21,6 +21,37 @@ namespace Paths::Test EXPECT_EQ(RemotePath::make("/a/../b").error().reason, PathRejection::DotOrDotDot); } + TEST(RemotePathTests, RefusesNulAndInvalidUtf8) + { + EXPECT_EQ(RemotePath::make(std::string{"/a\0b", 4}).error().reason, PathRejection::ContainsNul); + EXPECT_EQ(RemotePath::make("/home/\xC3").error().reason, PathRejection::InvalidUtf8); + EXPECT_EQ(RemotePath::make("/\xED\xA0\x80").error().reason, PathRejection::InvalidUtf8); + EXPECT_EQ(RemotePath::make("/home/t\xC3\xA4st")->value(), "/home/t\xC3\xA4st"); + } + + TEST(RemotePathTests, JoiningOntoAnUnsetPathStaysUnset) + { + const RemotePath unset{}; + EXPECT_TRUE((unset / PathSegment::make("a").value()).empty()); + EXPECT_TRUE((unset / RelativePath::make("a/b").value()).empty()); + EXPECT_TRUE((unset / RelativePath{}).empty()); + EXPECT_EQ((RemotePath::root() / RelativePath{}).value(), "/"); + EXPECT_EQ((RemotePath::root() / PathSegment::make("a").value()).value(), "/a"); + } + + TEST(RemotePathTests, SegmentsAndRelativePathsRoundTripThroughJson) + { + const nlohmann::json segment = PathSegment::make("a\\b").value(); + EXPECT_EQ(segment.get().value(), "a\\b"); + EXPECT_THROW(nlohmann::json{"a/b"}.at(0).get(), std::invalid_argument); + + const nlohmann::json relative = RelativePath::make("a/b").value(); + EXPECT_EQ(relative.get(), "a/b"); + EXPECT_EQ(relative.get().value(), "a/b"); + EXPECT_EQ(nlohmann::json{""}.at(0).get().value(), ""); + EXPECT_THROW(nlohmann::json{"a/../b"}.at(0).get(), std::invalid_argument); + } + TEST(RemotePathTests, BackslashesAndColonsAreOrdinaryCharacters) { const auto base = RemotePath::make("/home/test").value(); From 23c5cf07e3e97c0c8c2ee92051f4b530ca0ef067 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:02:43 +0200 Subject: [PATCH 089/131] Tests: Windows device names and name length follow the Windows rules --- paths/test/paths/test_path_segment.hpp | 41 +++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/paths/test/paths/test_path_segment.hpp b/paths/test/paths/test_path_segment.hpp index 8ef7d558..00ab7ae4 100644 --- a/paths/test/paths/test_path_segment.hpp +++ b/paths/test/paths/test_path_segment.hpp @@ -58,16 +58,55 @@ namespace Paths::Test {"nul.txt", PathRejection::ReservedDeviceName}, {"Lpt9", PathRejection::ReservedDeviceName}, {"com1.tar.gz", PathRejection::ReservedDeviceName}, + {"COM0", PathRejection::ReservedDeviceName}, + {"lpt0.txt", PathRejection::ReservedDeviceName}, + {"COM\xC2\xB9", PathRejection::ReservedDeviceName}, + {"lpt\xC2\xB3.log", PathRejection::ReservedDeviceName}, + {"CONIN$", PathRejection::ReservedDeviceName}, + {"conout$.txt", PathRejection::ReservedDeviceName}, + {"CON .txt", PathRejection::ReservedDeviceName}, + {"nul .tar.gz", PathRejection::ReservedDeviceName}, }) { const auto segment = LocalPathSegment::make(name, LocalPlatform::Windows); ASSERT_FALSE(segment.has_value()) << name; EXPECT_EQ(segment.error().reason, reason) << name; } - for (auto const* name : {"console", "com10", "lpt", "a.b", "nul_"}) + for (auto const* name : {"console", "com10", "lpt", "a.b", "nul_", "COM\xC2\xB4", "conin", " CON"}) EXPECT_TRUE(LocalPathSegment::make(name, LocalPlatform::Windows).has_value()) << name; } + TEST(LocalPathSegmentTests, WindowsLimitsNamesTo255Utf16Units) + { + EXPECT_TRUE(LocalPathSegment::make(std::string(255, 'a'), LocalPlatform::Windows).has_value()); + EXPECT_EQ( + LocalPathSegment::make(std::string(256, 'a'), LocalPlatform::Windows).error().reason, + PathRejection::NameTooLong + ); + + // A code point above U+FFFF takes two UTF-16 units, so 128 of them are 256 units. + const auto rockets = [](std::size_t count) + { + std::string name; + for (std::size_t index = 0; index < count; ++index) + name += "\xF0\x9F\x9A\x80"; + return name; + }; + EXPECT_TRUE(LocalPathSegment::make(rockets(127), LocalPlatform::Windows).has_value()); + EXPECT_EQ( + LocalPathSegment::make(rockets(128), LocalPlatform::Windows).error().reason, PathRejection::NameTooLong + ); + EXPECT_TRUE(LocalPathSegment::make(rockets(128), LocalPlatform::Posix).has_value()); + } + + TEST(LocalPathSegmentTests, RejectionOfAppliesTheSegmentRulesFirst) + { + EXPECT_EQ(LocalPathSegment::rejectionOf("..", LocalPlatform::Windows), PathRejection::DotOrDotDot); + EXPECT_EQ(LocalPathSegment::rejectionOf("a/b", LocalPlatform::Posix), PathRejection::ContainsSeparator); + EXPECT_EQ(LocalPathSegment::rejectionOf("a:b", LocalPlatform::Windows), PathRejection::ReservedCharacter); + EXPECT_FALSE(LocalPathSegment::rejectionOf("a:b", LocalPlatform::Posix).has_value()); + } + TEST(Utf8Tests, AcceptsOnlyWellFormedSequences) { for (auto const* text : {"", "ascii", "\xC3\xA4", "\xE2\x82\xAC", "\xF0\x9F\x9A\x80", "\xF4\x8F\xBF\xBF"}) From f54e675982f67fda63d05a970683710358585eec Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:02:53 +0200 Subject: [PATCH 090/131] Tests: A Windows LocalPath is normalized and device paths are refused --- paths/test/paths/test_local_path.hpp | 118 ++++++++++++++++++++++++++- 1 file changed, 116 insertions(+), 2 deletions(-) diff --git a/paths/test/paths/test_local_path.hpp b/paths/test/paths/test_local_path.hpp index 1c5fd92a..963c2b00 100644 --- a/paths/test/paths/test_local_path.hpp +++ b/paths/test/paths/test_local_path.hpp @@ -1,9 +1,13 @@ #pragma once +#include #include #include +#include +#include + namespace Paths::Test { TEST(LocalPathTests, ParsesPosixPaths) @@ -22,7 +26,8 @@ namespace Paths::Test { EXPECT_EQ(LocalPath::fromUtf8("C:\\Users\\tim\\", LocalPlatform::Windows)->toUtf8(), "C:/Users/tim"); EXPECT_EQ(LocalPath::fromUtf8("C:/", LocalPlatform::Windows)->toUtf8(), "C:/"); - EXPECT_EQ(LocalPath::fromUtf8("c:", LocalPlatform::Windows)->toUtf8(), "c:/"); + EXPECT_EQ(LocalPath::fromUtf8("c:", LocalPlatform::Windows)->toUtf8(), "C:/"); + EXPECT_EQ(LocalPath::fromUtf8("d:\\Data", LocalPlatform::Windows)->toUtf8(), "D:/Data"); EXPECT_EQ(LocalPath::fromUtf8("\\\\server\\share\\dir", LocalPlatform::Windows)->toUtf8(), "//server/share/dir"); EXPECT_EQ(LocalPath::fromUtf8("/x", LocalPlatform::Windows).error().reason, PathRejection::NotAbsolute); EXPECT_EQ(LocalPath::fromUtf8("C:x", LocalPlatform::Windows).error().reason, PathRejection::NotAbsolute); @@ -64,13 +69,122 @@ namespace Paths::Test EXPECT_EQ(base.relativeTo(root)->value(), "home/tim"); } + TEST(LocalPathTests, Win32PrefixesAreDroppedAndDevicePathsRefused) + { + const auto windows = [](std::string_view text) + { + return LocalPath::fromUtf8(text, LocalPlatform::Windows); + }; + EXPECT_EQ(windows("\\\\?\\c:\\Users\\tim")->toUtf8(), "C:/Users/tim"); + EXPECT_EQ(windows("\\\\?\\C:")->toUtf8(), "C:/"); + EXPECT_EQ(windows("\\\\?\\UNC\\server\\share\\dir")->toUtf8(), "//server/share/dir"); + EXPECT_EQ(windows("//?/unc/server/share")->toUtf8(), "//server/share/"); + EXPECT_EQ(*windows("\\\\?\\C:\\Users"), *windows("C:/Users")); + for (auto const* text : { + "\\\\.\\PhysicalDrive0", + "\\\\.\\C:\\x", + "\\\\?\\Volume{12345678-1234-1234-1234-123456789abc}\\x", + "\\\\?\\GLOBALROOT\\Device\\x", + "//?/x", + "//../share/x", + "//./share", + }) + { + const auto path = windows(text); + ASSERT_FALSE(path.has_value()) << text; + EXPECT_EQ(path.error().reason, PathRejection::DevicePath) << text; + } + EXPECT_EQ(windows("\\\\?\\UNC\\server").error().reason, PathRejection::NotAbsolute); + EXPECT_EQ(windows("\\\\?\\UNC\\..\\share").error().reason, PathRejection::DevicePath); + + // On POSIX these are ordinary names below the root. + EXPECT_EQ(LocalPath::fromUtf8("//?/x", LocalPlatform::Posix)->toUtf8(), "/?/x"); + } + + TEST(LocalPathTests, RootLengthCoversDrivesAndShares) + { + using Detail::rootLengthOf; + EXPECT_EQ(rootLengthOf("/a", LocalPlatform::Posix), 1u); + EXPECT_EQ(rootLengthOf("a", LocalPlatform::Posix), 0u); + EXPECT_EQ(rootLengthOf("C:/a", LocalPlatform::Windows), 3u); + EXPECT_EQ(rootLengthOf("C:", LocalPlatform::Windows), 2u); + EXPECT_EQ(rootLengthOf("C:a", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("1:/a", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("//server/share/a", LocalPlatform::Windows), 14u); + EXPECT_EQ(rootLengthOf("//server/share", LocalPlatform::Windows), 14u); + EXPECT_EQ(rootLengthOf("//server", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("//server/", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("///share", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("/a", LocalPlatform::Windows), 0u); + } + + TEST(LocalPathTests, RefusesNulAndInvalidUtf8) + { + EXPECT_EQ(LocalPath::fromUtf8(std::string{"/a\0b", 4}, LocalPlatform::Posix).error().reason, PathRejection::ContainsNul); + EXPECT_EQ( + LocalPath::fromUtf8(std::string{"//ser\0ver/share", 16}, LocalPlatform::Windows).error().reason, + PathRejection::ContainsNul + ); + EXPECT_EQ(LocalPath::fromUtf8("/a/\xFF", LocalPlatform::Posix).error().reason, PathRejection::InvalidUtf8); + EXPECT_EQ(LocalPath::fromUtf8("", LocalPlatform::Posix).error().reason, PathRejection::Empty); + } + + TEST(LocalPathTests, WithinComparesThePlatformAndOptionallyIgnoresCase) + { + const auto windowsHome = LocalPath::fromUtf8("C:/Users/Me", LocalPlatform::Windows).value(); + const auto windowsUsers = LocalPath::fromUtf8("c:/USERS", LocalPlatform::Windows).value(); + EXPECT_FALSE(windowsHome.isWithin(windowsUsers)); + EXPECT_TRUE(windowsHome.isWithinIgnoringCase(windowsUsers)); + EXPECT_TRUE(windowsHome.isWithinIgnoringCase(windowsHome)); + EXPECT_TRUE(windowsHome.isWithinIgnoringCase(LocalPath::fromUtf8("C:/", LocalPlatform::Windows).value())); + EXPECT_FALSE(windowsHome.isWithinIgnoringCase(LocalPath::fromUtf8("C:/Use", LocalPlatform::Windows).value())); + EXPECT_FALSE(windowsHome.isWithinIgnoringCase(LocalPath::fromUtf8("D:/", LocalPlatform::Windows).value())); + + const auto posix = LocalPath::fromUtf8("/C:/Users/Me", LocalPlatform::Posix).value(); + EXPECT_FALSE(posix.isWithin(LocalPath::fromUtf8("/", LocalPlatform::Windows).value_or(LocalPath{}))); + EXPECT_FALSE(windowsHome.isWithin(LocalPath::fromUtf8("/", LocalPlatform::Posix).value())); + EXPECT_FALSE(windowsHome.isWithinIgnoringCase(LocalPath::fromUtf8("/", LocalPlatform::Posix).value())); + EXPECT_TRUE(posix.isWithin(LocalPath::fromUtf8("/", LocalPlatform::Posix).value())); + EXPECT_FALSE(LocalPath{}.isWithin(LocalPath{})); + + const auto share = LocalPath::fromUtf8("//server/share/a", LocalPlatform::Windows).value(); + EXPECT_TRUE(share.isWithin(LocalPath::fromUtf8("//server/share", LocalPlatform::Windows).value())); + EXPECT_FALSE(share.isWithin(LocalPath::fromUtf8("//server/sha", LocalPlatform::Windows).value())); + } + + TEST(LocalPathTests, JoinAndAppendOnAnUnsetPathStayUnset) + { + const LocalPath unset{}; + EXPECT_EQ(unset.join(RelativePath::make("a").value()).error().reason, PathRejection::Empty); + EXPECT_EQ(unset.join(RelativePath{}).error().reason, PathRejection::Empty); + EXPECT_TRUE((unset / LocalPathSegment::make("a", LocalPlatform::Posix).value()).empty()); + EXPECT_FALSE(unset.parent().has_value()); + EXPECT_FALSE(unset.filename().has_value()); + + const auto root = LocalPath::fromUtf8("/", LocalPlatform::Posix).value(); + EXPECT_EQ(root.join(RelativePath{})->toUtf8(), "/"); + EXPECT_EQ(root.join(RelativePath::make("a/b").value())->toUtf8(), "/a/b"); + const auto drive = LocalPath::fromUtf8("C:/", LocalPlatform::Windows).value(); + EXPECT_EQ(drive.join(RelativePath::make("a").value())->toUtf8(), "C:/a"); + EXPECT_EQ(drive.join(RelativePath::make("a/x.").value()).error().reason, PathRejection::TrailingDotOrSpace); + } + + TEST(LocalPathTests, RoundTripsThroughJson) + { + const nlohmann::json json = LocalPath::fromUtf8("/home/t\xC3\xA4st").value(); + EXPECT_EQ(json.get(), "/home/t\xC3\xA4st"); + EXPECT_EQ(json.get().toUtf8(), "/home/t\xC3\xA4st"); + EXPECT_THROW(nlohmann::json{"relative"}.at(0).get(), std::invalid_argument); + } + #ifndef __EMSCRIPTEN__ TEST(LocalPathTests, ConvertsToAndFromNativePaths) { - const auto native = std::filesystem::current_path() / "t\xC3\xA4st"; + const auto native = std::filesystem::current_path() / nativeFromUtf8("t\xC3\xA4st \xE6\x9D\xB1\xE4\xBA\xAC"); const auto path = LocalPath::make(native); ASSERT_TRUE(path.has_value()) << path.error().toString(); EXPECT_EQ(path->native(), native.lexically_normal()); + EXPECT_EQ(nativeToUtf8(path->native().filename()), "t\xC3\xA4st \xE6\x9D\xB1\xE4\xBA\xAC"); EXPECT_FALSE(LocalPath::make(std::filesystem::path{"relative"}).has_value()); } #endif From b139c9f6505503ebf17542124932e2880c99b25f Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:05:28 +0200 Subject: [PATCH 091/131] Tests: The local delete guard folds case and compares homes by file identity --- .../test/backend/test_local_delete_guard.hpp | 85 +++++++++++++++++-- backend/test/backend/test_operation_queue.hpp | 6 +- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/backend/test/backend/test_local_delete_guard.hpp b/backend/test/backend/test_local_delete_guard.hpp index 6dbabe27..4a7276ee 100644 --- a/backend/test/backend/test_local_delete_guard.hpp +++ b/backend/test/backend/test_local_delete_guard.hpp @@ -1,15 +1,20 @@ #pragma once #include +#include #include +#include +#include + +extern std::filesystem::path temporaryRoot; + namespace Test { class LocalDeleteGuardTests : public ::testing::Test { protected: - // POSIX rules on every host, the Windows rules are covered by the paths tests. void SetUp() override { previous_ = Paths::defaultLocalPlatform(); @@ -30,7 +35,7 @@ namespace Test ASSERT_TRUE(home.has_value()); const auto check = [&home](std::string const& path) { - return localDeleteTarget(path, *home).has_value(); + return localDeleteTarget(path, {*home}).has_value(); }; EXPECT_FALSE(check("/")); EXPECT_FALSE(check("/home")); @@ -41,6 +46,18 @@ namespace Test EXPECT_TRUE(check("/home/user/file.txt")); EXPECT_TRUE(check("/home/username")); EXPECT_TRUE(check("/tmp/x")); + EXPECT_TRUE(check("/tmp/a\\..\\b")) << "a backslash is a character on POSIX"; + } + + TEST_F(LocalDeleteGuardTests, EveryHomeIsProtected) + { + const std::vector homes{ + Paths::LocalPath::fromUtf8("/home/user", Paths::LocalPlatform::Posix).value(), + Paths::LocalPath::fromUtf8("/var/home/user", Paths::LocalPlatform::Posix).value(), + }; + EXPECT_FALSE(localDeleteTarget("/var/home", homes).has_value()); + EXPECT_FALSE(localDeleteTarget("/home", homes).has_value()); + EXPECT_TRUE(localDeleteTarget("/var/tmp", homes).has_value()); } TEST_F(LocalDeleteGuardTests, WindowsRootsHomeAndBackslashParentsAreRefused) @@ -50,7 +67,7 @@ namespace Test ASSERT_TRUE(home.has_value()); const auto check = [&home](std::string const& path) { - return localDeleteTarget(path, *home).has_value(); + return localDeleteTarget(path, {*home}).has_value(); }; EXPECT_FALSE(check("C:/")); EXPECT_FALSE(check("C:/Users")); @@ -61,9 +78,67 @@ namespace Test EXPECT_TRUE(check("D:/data")); } + TEST_F(LocalDeleteGuardTests, WindowsSpellingsOfTheHomeAreRefused) + { + Paths::defaultLocalPlatform() = Paths::LocalPlatform::Windows; + const auto home = Paths::LocalPath::fromUtf8("C:/Users/me").value(); + for (auto const* path : {"c:/users", "C:/USERS/ME", "\\\\?\\C:\\Users", "\\\\?\\c:\\users\\Me", "c:\\"}) + EXPECT_FALSE(localDeleteTarget(path, {home}).has_value()) << path; + EXPECT_EQ(localDeleteTarget("\\\\.\\C:\\Users", {home}).error(), "'\\\\.\\C:\\Users' is a device path"); + EXPECT_TRUE(localDeleteTarget("c:/users/me/x", {home}).has_value()); + EXPECT_TRUE(localDeleteTarget("c:/userss", {home}).has_value()); + } + + TEST_F(LocalDeleteGuardTests, RefusalsNameTheReason) + { + const auto home = Paths::LocalPath::fromUtf8("/home/user", Paths::LocalPlatform::Posix).value(); + EXPECT_EQ(localDeleteTarget("relative", {home}).error(), "'relative' is not absolute"); + EXPECT_EQ(localDeleteTarget("/a/\xFF", {home}).error(), "'/a/\\xFF' is not valid UTF-8"); + EXPECT_EQ(localDeleteTarget("/a/../b", {home}).error(), "'/a/../b' contains '..'"); + EXPECT_EQ(localDeleteTarget("/", {home}).error(), "'/' is a root directory"); + EXPECT_EQ( + localDeleteTarget("/home", {home}).error(), "'/home' is or contains the home directory '/home/user'" + ); + } + TEST_F(LocalDeleteGuardTests, WithoutAKnownHomeOnlyRootsAreRefused) { - EXPECT_FALSE(localDeleteTarget("/", std::nullopt).has_value()); - EXPECT_TRUE(localDeleteTarget("/home/user", std::nullopt).has_value()); + EXPECT_FALSE(localDeleteTarget("/", {}).has_value()); + EXPECT_TRUE(localDeleteTarget("/home/user", {}).has_value()); + } + +#ifndef _WIN32 + TEST_F(LocalDeleteGuardTests, TheSameDirectoryThroughASymlinkIsRefused) + { + const Utility::TemporaryDirectory directory{temporaryRoot / "localDeleteGuard"}; + const auto root = directory.path(); + std::filesystem::create_directories(root / "real" / "user" / "work"); + std::filesystem::create_directory_symlink(root / "real", root / "link"); + const auto home = Paths::LocalPath::make(root / "link" / "user").value(); + const auto check = [&home](std::filesystem::path const& path) + { + return refuseProtectedLocalDelete(Paths::LocalPath::make(path).value(), {home}).has_value(); + }; + + EXPECT_FALSE(check(root / "real")) << "an ancestor of the home reached without the link"; + EXPECT_FALSE(check(root / "real" / "user")); + EXPECT_FALSE(check(root / "link")); + EXPECT_TRUE(check(root / "real" / "user" / "work")); + EXPECT_TRUE(check(root / "missing")); + + // Deleting a link removes the link, never what it points to. + std::filesystem::create_directory_symlink(root / "real" / "user", root / "toHome"); + EXPECT_TRUE(check(root / "toHome")); + } + + TEST_F(LocalDeleteGuardTests, HomeDirectoriesComeFromTheEnvironmentAndPasswd) + { + const auto homes = localHomeDirectories(); + ASSERT_FALSE(homes.empty()); + if (auto const* home = std::getenv("HOME"); home != nullptr && *home != '\0') + EXPECT_EQ(homes.front().native(), std::filesystem::path{home}.lexically_normal()); + for (auto const& home : homes) + EXPECT_EQ(std::ranges::count(homes, home), 1); } +#endif } diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 790f255a..595a31d3 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -932,10 +932,10 @@ namespace Test const auto file = localDirectory_.path() / "survivor.txt"; std::ofstream{file} << "survivor"; const auto root = Paths::LocalPath::make(std::filesystem::path{file}.root_path()).value(); - const auto home = localHomeDirectory(); - ASSERT_TRUE(home.has_value()); + const auto homes = localHomeDirectories(); + ASSERT_FALSE(homes.empty()); - for (auto const& target : {root, *home}) + for (auto const& target : {root, homes.front()}) { const auto result = runOnStrand([&]() { return queue->addLocalDeleteOperation( From 506ff06188e2e92a9f4c074fba66a2a4498b3b6d Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:16:48 +0200 Subject: [PATCH 092/131] Tests: SftpSession canonicalizes paths and the start directory on the server --- .../ssh/test_ssh2_servers/sftp_server.mjs | 34 ++++++++++++++++--- ssh/test/ssh/test_ssh2_servers/source/cli.mjs | 2 +- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs index 205c1791..1f69ad22 100644 --- a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs +++ b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs @@ -244,6 +244,31 @@ const advertiseExtensions = (sftpStream) => { }; }; +// Resolves every symlink on the way like realpath, undefined when a component does not exist. +const canonicalize = (path, depth = 0) => { + if (depth > 40) + return undefined; + const parts = path.split('/').filter(part => part.length > 0 && part !== '.'); + let current = '/'; + for (let index = 0; index < parts.length; ++index) { + if (parts[index] === '..') { + current = pathUtil.posix.dirname(current); + continue; + } + const candidate = pathUtil.posix.join(current, parts[index]); + const entry = fakeFilesystem.find(candidate); + if (entry === undefined) + return undefined; + if (entry.type === 'symlink') { + const target = entry.target.startsWith('/') ? entry.target : pathUtil.posix.join(current, entry.target); + const rest = parts.slice(index + 1).join('/'); + return canonicalize(rest.length > 0 ? `${target}/${rest}` : target, depth + 1); + } + current = candidate; + } + return current; +}; + const resolveFileName = (path) => { if (path === '.') { return defaultPath; @@ -481,19 +506,18 @@ const server = new Server({ return sftpStream.name(reqid, [{ filename, longname: filename, attrs: {} }]); } - path = resolveFileName(path); - - const result = fakeFilesystem.find(path); + path = canonicalize(resolveFileName(path)); + const result = path === undefined ? undefined : fakeFilesystem.find(path); if (result === undefined) { logMessage(`Path not found: ${path}`); return sftpStream.status(reqid, STATUS_CODE.NO_SUCH_FILE); } - logMessage('REALPATH', result.path(), result); + logMessage('REALPATH', path, result); const name = [{ - filename: result.path(), + filename: path, longname: makeLongName(result), attrs: result.stat }]; diff --git a/ssh/test/ssh/test_ssh2_servers/source/cli.mjs b/ssh/test/ssh/test_ssh2_servers/source/cli.mjs index 2f176b4e..3aa70557 100644 --- a/ssh/test/ssh/test_ssh2_servers/source/cli.mjs +++ b/ssh/test/ssh/test_ssh2_servers/source/cli.mjs @@ -18,7 +18,7 @@ class CommandLineInterface { const command = JSON.parse(input); if (Object.hasOwnProperty.call(command, 'command')) { - // Exit runs once: a second run wrote to the closed log and the crash waited for stdin on Windows. + // Exit runs its handler once and closes stdin. if (command.command === 'exit') { if (this.eventMap.has('exit')) { this.eventMap.get('exit')(); From a4c735c0666c5b7488f0a033c193907221e3cae2 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:19:09 +0200 Subject: [PATCH 093/131] Tests: Remote deletes are checked by a RemoteDeleteGuard --- backend/test/backend/main.cpp | 1 + .../test/backend/test_remote_delete_guard.hpp | 163 ++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 backend/test/backend/test_remote_delete_guard.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index d405f5a2..48724cbe 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -5,6 +5,7 @@ #include "test_upload_operation.hpp" #include "test_scan_operation.hpp" #include "test_local_delete_guard.hpp" +#include "test_remote_delete_guard.hpp" #include "test_local_scan_operation.hpp" #include "test_bulk_download_operation.hpp" #include "test_bulk_upload_operation.hpp" diff --git a/backend/test/backend/test_remote_delete_guard.hpp b/backend/test/backend/test_remote_delete_guard.hpp new file mode 100644 index 00000000..6ed7c316 --- /dev/null +++ b/backend/test/backend/test_remote_delete_guard.hpp @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include "real_server_tests.hpp" + +#include + +#include +#include + +namespace Test +{ + TEST(RemoteDeleteGuardDetailTests, RecognizesWindowsServerPaths) + { + using RemoteDeleteGuardDetail::looksLikeWindowsPath; + EXPECT_TRUE(looksLikeWindowsPath(remote("/C:/Users/me"))); + EXPECT_TRUE(looksLikeWindowsPath(remote("/d:"))); + EXPECT_FALSE(looksLikeWindowsPath(remote("/home/C:"))); + EXPECT_FALSE(looksLikeWindowsPath(remote("/CD:/x"))); + EXPECT_FALSE(looksLikeWindowsPath(remote("/C:x"))); + EXPECT_FALSE(looksLikeWindowsPath(Paths::RemotePath::root())); + } + + TEST(RemoteDeleteGuardDetailTests, RecognizesPathsOfCaseFoldingServers) + { + using RemoteDeleteGuardDetail::looksCaseFolding; + EXPECT_TRUE(looksCaseFolding(remote("/C:/Users/me"))); + EXPECT_TRUE(looksCaseFolding(remote("/Users/me/project"))); + EXPECT_TRUE(looksCaseFolding(remote("/Users"))); + EXPECT_FALSE(looksCaseFolding(remote("/Usersx/me"))); + EXPECT_FALSE(looksCaseFolding(remote("/home/me"))); + EXPECT_FALSE(looksCaseFolding(remote("/srv/Users"))); + EXPECT_FALSE(looksCaseFolding(Paths::RemotePath::root())); + } + + TEST(RemoteDeleteGuardDetailTests, WithinIgnoringCaseStopsAtSegments) + { + using RemoteDeleteGuardDetail::isWithinIgnoringCase; + EXPECT_TRUE(isWithinIgnoringCase(remote("/home/Test/a"), remote("/HOME/test"))); + EXPECT_TRUE(isWithinIgnoringCase(remote("/home/test"), remote("/Home/Test"))); + EXPECT_TRUE(isWithinIgnoringCase(remote("/a"), Paths::RemotePath::root())); + EXPECT_FALSE(isWithinIgnoringCase(remote("/home/tests"), remote("/home/test"))); + EXPECT_FALSE(isWithinIgnoringCase(remote("/home"), remote("/home/test"))); + EXPECT_FALSE(isWithinIgnoringCase(Paths::RemotePath{}, Paths::RemotePath::root())); + } + + TEST(RemoteDeleteGuardDetailTests, TextChecksRefuseRootHomeCaseAndWindowsNames) + { + using Reason = RemoteDeleteRefusal::Reason; + using RemoteDeleteGuardDetail::refusalByText; + const auto reasonOf = [](Paths::RemotePath const& target, std::optional const& home) + { + const auto refusal = refusalByText(target, home); + return refusal ? std::optional{refusal->reason} : std::nullopt; + }; + const auto home = remote("/home/test"); + EXPECT_EQ(reasonOf(Paths::RemotePath{}, home), Reason::NoPath); + EXPECT_EQ(reasonOf(Paths::RemotePath::root(), std::nullopt), Reason::Root); + EXPECT_EQ(reasonOf(remote("/home"), home), Reason::ContainsHome); + EXPECT_EQ(reasonOf(remote("/home/test"), home), Reason::ContainsHome); + EXPECT_FALSE(reasonOf(remote("/HOME"), home).has_value()) << "a Linux server tells names apart by case"; + EXPECT_FALSE(reasonOf(remote("/home/Test"), home).has_value()); + EXPECT_FALSE(reasonOf(remote("/home/test/a"), home).has_value()); + EXPECT_FALSE(reasonOf(remote("/home/testing"), home).has_value()); + EXPECT_FALSE(reasonOf(remote("/home"), std::nullopt).has_value()); + EXPECT_FALSE(reasonOf(remote("/home/a\\b"), home).has_value()) << "a backslash is a character on POSIX"; + + const auto windowsHome = remote("/C:/Users/me"); + EXPECT_EQ(reasonOf(remote("/c:/users"), windowsHome), Reason::ContainsHome); + EXPECT_EQ(reasonOf(remote("/C:/USERS/ME"), windowsHome), Reason::ContainsHome); + EXPECT_EQ(reasonOf(remote("/C:/Users."), windowsHome), Reason::WindowsName); + EXPECT_EQ(reasonOf(remote("/C:/Users "), windowsHome), Reason::WindowsName); + EXPECT_EQ(reasonOf(remote("/C:/Users\\me"), windowsHome), Reason::WindowsName); + EXPECT_EQ(reasonOf(remote("/D:/data\\x"), std::nullopt), Reason::WindowsName); + EXPECT_FALSE(reasonOf(remote("/C:/Users/me/file.txt"), windowsHome).has_value()); + EXPECT_FALSE(reasonOf(remote("/D:/data/a.b"), windowsHome).has_value()); + + const auto macHome = remote("/Users/me"); + EXPECT_EQ(reasonOf(remote("/users"), macHome), Reason::ContainsHome); + EXPECT_EQ(reasonOf(remote("/USERS/ME"), macHome), Reason::ContainsHome); + EXPECT_FALSE(reasonOf(remote("/users/me/file.txt"), macHome).has_value()); + EXPECT_EQ(reasonOf(remote("/Users/ME"), home), std::nullopt) << "the home of a Linux server is exact"; + EXPECT_EQ(reasonOf(remote("/Users"), remote("/users/me")), Reason::ContainsHome) + << "a target below /Users looks like a server that folds case"; + } + + TEST(RemoteDeleteGuardDetailTests, RefusalsAreLocalizedAndSayNothingWasDeleted) + { + const RemoteDeleteRefusal refusal{ + .reason = RemoteDeleteRefusal::Reason::ContainsHome, + .target = "/home", + .home = "/home/test", + }; + const auto parsed = Utility::parseLocalizedMessage(refusal.toLocalizedMessage()); + EXPECT_EQ(parsed.key, "backend.deleteGuard.containsHome"); + EXPECT_EQ(parsed.arguments, (std::vector{"/home", "/home/test"})); + EXPECT_EQ(refusal.toString(), "'/home' is or contains the home directory '/home/test'"); + } + + class RemoteDeleteGuardTests : public RealServerTests + {}; + + TEST_F(RemoteDeleteGuardTests, ParentsAreCanonicalizedSoSymlinksDoNotHideTheHome) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/alias"}, {"target", "/home"}})); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/home/test/up"}, {"target", ".."}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + std::optional home{}; + RemoteDeleteGuard guard{*sftp, home, 5s}; + const auto reasonOf = [&guard](std::string_view target) -> std::optional + { + const auto checked = guard.check(remote(target)); + return checked ? std::nullopt : std::optional{checked.error().reason}; + }; + + EXPECT_EQ(reasonOf("/alias/test"), RemoteDeleteRefusal::Reason::ContainsHome); + ASSERT_TRUE(home.has_value()) << "the home is looked up once and shared"; + EXPECT_EQ(home->value(), "/home/test"); + EXPECT_EQ(reasonOf("/home/test/up/test"), RemoteDeleteRefusal::Reason::ContainsHome); + EXPECT_FALSE(reasonOf("/HOME").has_value()) << "the mock server tells names apart by case"; + EXPECT_EQ(reasonOf("/"), RemoteDeleteRefusal::Reason::Root); + EXPECT_FALSE(reasonOf("/alias/test/file2.txt").has_value()); + EXPECT_FALSE(reasonOf("/alias").has_value()) << "deleting the link removes the link"; + EXPECT_FALSE(reasonOf("/home/test/up").has_value()); + EXPECT_FALSE(reasonOf("/missing/directory/x").has_value()); + + const std::vector targets{remote("/home/test/file2.txt"), remote("/alias/test"), remote("/tmp")}; + const auto refusal = refuseAnyProtected(guard, targets); + ASSERT_TRUE(refusal.has_value()); + EXPECT_EQ(refusal->target, "/alias/test"); + EXPECT_FALSE(refuseAnyProtected(guard, std::vector{remote("/home/test/file2.txt")}).has_value()); + } + + TEST_F(RemoteDeleteGuardTests, ScannedRootsAreCanonicalizedOnce) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/alias"}, {"target", "/home"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + std::optional home{}; + RemoteDeleteGuard guard{*sftp, home, 5s}; + guard.trustScannedBelow(remote("/alias")); + EXPECT_FALSE(guard.check(remote("/alias/test")).has_value()); + EXPECT_TRUE(guard.check(remote("/alias/test/Documents/notThere")).has_value()); + EXPECT_TRUE(guard.check(remote("/alias/other/x")).has_value()); + } + + TEST_F(RemoteDeleteGuardTests, AnUnknownHomeOnlyProtectsTheRoot) + { + CREATE_SERVER_AND_JOINER(sftpServer); + serverStartResult->command("realpathDotFailure"); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + std::optional home{}; + RemoteDeleteGuard guard{*sftp, home, 5s}; + EXPECT_EQ(guard.check(Paths::RemotePath::root()).error().reason, RemoteDeleteRefusal::Reason::Root); + EXPECT_TRUE(guard.check(remote("/home")).has_value()); + EXPECT_FALSE(home.has_value()); + } +} From ee98b35c9e4d97421909d02c2928c9df8b78216c Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:19:50 +0200 Subject: [PATCH 094/131] Tests: A sync delete of a protected directory refuses everything below it --- backend/test/backend/test_operation_queue.hpp | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 595a31d3..3c3f4760 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -1037,6 +1037,96 @@ namespace Test EXPECT_FALSE(std::filesystem::exists(localRoot / "localGone.txt")); } +#ifndef _WIN32 + TEST_F(OperationQueueTests, SyncRefusesEveryDeleteBelowAProtectedDirectory) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/other.txt"}, {"content", "other"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "syncProtected"; + std::filesystem::create_directories(localRoot / "home" / "inner"); + std::ofstream{localRoot / "home" / "keep.txt"} << "keep"; + std::ofstream{localRoot / "home" / "inner" / "keep.txt"} << "keep"; + std::ofstream{localRoot / "loose.txt"} << "loose"; + + auto const* previousHome = std::getenv("HOME"); + const std::optional savedHome = + previousHome ? std::optional{previousHome} : std::nullopt; + ASSERT_EQ(setenv("HOME", (localRoot / "home").c_str(), 1), 0); + const auto restoreHome = Nui::ScopeExit{[&savedHome]() noexcept + { + if (savedHome) + setenv("HOME", savedHome->c_str(), 1); + else + unsetenv("HOME"); + }}; + + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home"), + }, + context.get_executor() + }; + using SharedData::Sync::Action; + const auto entry = [&session, &sftp](std::string relKey, Action action, bool isDirectory) + { + return plannedNow(session, *sftp, std::move(relKey), action, isDirectory); + }; + SyncSession::ResolvedPlan plan{ + .entries = + { + entry("test", Action::DeleteRemote, true), + entry("test/file2.txt", Action::DeleteRemote, false), + entry("test/Documents", Action::DeleteRemote, true), + entry("test/Documents/doc1.txt", Action::DeleteRemote, false), + entry("other.txt", Action::DeleteRemote, false), + entry("home", Action::DeleteLocal, true), + entry("home/inner", Action::DeleteLocal, true), + entry("home/inner/keep.txt", Action::DeleteLocal, false), + entry("home/keep.txt", Action::DeleteLocal, false), + entry("loose.txt", Action::DeleteLocal, false), + }, + }; + const SharedData::Sync::SyncExecutionRequest request{ + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + + const auto result = runOnStrand([&]() { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + }); + EXPECT_EQ(result.remoteDeletes, (std::vector{"other.txt"})); + EXPECT_EQ(result.localDeletes, (std::vector{"loose.txt"})); + EXPECT_EQ( + result.failed, + (std::vector{ + "test", + "test/file2.txt", + "test/Documents", + "test/Documents/doc1.txt", + "home", + "home/inner", + "home/inner/keep.txt", + "home/keep.txt", + }) + ); + + setPaused(false); + workUntilCompletions(2); + EXPECT_TRUE(remoteNames(*sftp, "/home/test").contains("file2.txt")); + EXPECT_FALSE(remoteNames(*sftp, "/home").contains("other.txt")); + EXPECT_TRUE(std::filesystem::exists(localRoot / "home" / "keep.txt")); + EXPECT_TRUE(std::filesystem::exists(localRoot / "home" / "inner" / "keep.txt")); + EXPECT_FALSE(std::filesystem::exists(localRoot / "loose.txt")); + } +#endif + TEST_F(OperationQueueTests, CancelledDirectoryDownloadDuringScanLeavesNextUnitIntact) { CREATE_SERVER_AND_JOINER(sftpServer); From fc2e9131651b59a3bed0914173d8d1da71346aa6 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:34:46 +0200 Subject: [PATCH 095/131] Tests: paths/utf8.hpp is the one UTF-8 module --- paths/test/paths/main.cpp | 1 + paths/test/paths/test_utf8.hpp | 65 ++++++++++++++++++++++++++++++ utility/test/utility/test_utf8.hpp | 4 +- 3 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 paths/test/paths/test_utf8.hpp diff --git a/paths/test/paths/main.cpp b/paths/test/paths/main.cpp index 2fbe76bb..11ab01e4 100644 --- a/paths/test/paths/main.cpp +++ b/paths/test/paths/main.cpp @@ -2,6 +2,7 @@ #include "test_remote_path.hpp" #include "test_local_path.hpp" #include "test_native.hpp" +#include "test_utf8.hpp" #include diff --git a/paths/test/paths/test_utf8.hpp b/paths/test/paths/test_utf8.hpp new file mode 100644 index 00000000..7b681e87 --- /dev/null +++ b/paths/test/paths/test_utf8.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace Paths::Test +{ + TEST(Utf8Tests, SanitizedKeepsValidText) + { + for (const std::string_view text : {"", "plain", "Grüße 東京 \xF0\x9F\x9A\x80"}) + EXPECT_EQ(sanitizedUtf8(text), text); + } + + TEST(Utf8Tests, SanitizedReplacesEveryInvalidByte) + { + EXPECT_EQ(sanitizedUtf8("a\xFF" + "b"), + "a\xEF\xBF\xBD" + "b"); + EXPECT_EQ(sanitizedUtf8("\xC0\xAF"), "\xEF\xBF\xBD\xEF\xBF\xBD"); + EXPECT_EQ(sanitizedUtf8("x\xC3"), "x\xEF\xBF\xBD"); + EXPECT_EQ(sanitizedUtf8("Fran\xE7ois"), "Fran\xEF\xBF\xBDois"); + EXPECT_TRUE(isValidUtf8(sanitizedUtf8("\xED\xA0\x80\xF4\x90\x80\x80\xF8\x88\x80\x80\x80"))); + } + + TEST(Utf8Tests, TruncatedNeverEndsInsideASequence) + { + EXPECT_EQ(truncatedUtf8("abc", 5u), "abc"); + EXPECT_EQ(truncatedUtf8("abc", 2u), "ab"); + EXPECT_EQ(truncatedUtf8("a\xC3\xA4", 2u), "a"); + EXPECT_EQ(truncatedUtf8("a\xC3\xA4", 3u), "a\xC3\xA4"); + EXPECT_EQ(truncatedUtf8("\xF0\x9F\x9A\x80x", 3u), ""); + EXPECT_EQ(truncatedUtf8("a\xFF\xFF", 2u), "a\xFF"); + EXPECT_EQ(truncatedUtf8("", 0u), ""); + } + + TEST(Utf8Tests, Utf16RoundTrip) + { + const std::string text = "Grüße 東京 \xF0\x9F\x9A\x80"; + const auto utf16 = utf8ToUtf16(text); + EXPECT_EQ(utf16, u"Grüße 東京 \U0001F680"); + EXPECT_TRUE(isValidUtf16(std::u16string_view{utf16})); + EXPECT_EQ(utf16ToUtf8(std::u16string_view{utf16}), text); + } + + TEST(Utf8Tests, InvalidUtf8BecomesReplacementInUtf16) + { + EXPECT_EQ(utf8ToUtf16("a\xFF"), u"a�"); + } + + TEST(Utf8Tests, UnpairedSurrogatesBecomeReplacement) + { + const std::u16string lone{u'a', char16_t{0xD800u}, u'b', char16_t{0xDC00u}}; + EXPECT_FALSE(isValidUtf16(std::u16string_view{lone})); + EXPECT_EQ(utf16ToUtf8(std::u16string_view{lone}), "a\xEF\xBF\xBD" + "b\xEF\xBF\xBD"); + const std::u16string trailingHigh{u'x', char16_t{0xDBFFu}}; + EXPECT_FALSE(isValidUtf16(std::u16string_view{trailingHigh})); + EXPECT_EQ(utf16ToUtf8(std::u16string_view{trailingHigh}), "x\xEF\xBF\xBD"); + } +} diff --git a/utility/test/utility/test_utf8.hpp b/utility/test/utility/test_utf8.hpp index ad17e593..b059201d 100644 --- a/utility/test/utility/test_utf8.hpp +++ b/utility/test/utility/test_utf8.hpp @@ -37,8 +37,8 @@ namespace Utility::Tests ), replacement + U"a" ); - EXPECT_EQ(Utf8::decode("\xC0\xAF"), replacement); - EXPECT_EQ(Utf8::decode("\xED\xA0\x80"), replacement); + EXPECT_EQ(Utf8::decode("\xC0\xAF"), replacement + replacement); + EXPECT_EQ(Utf8::decode("\xED\xA0\x80"), replacement + replacement + replacement); } TEST_F(Utf8Tests, EncodeRoundTripsDecode) From a85a74d2a3174f41c272fb68cee3c211eb8ae2c6 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:43:01 +0200 Subject: [PATCH 096/131] Tests: Native paths convert to UTF-8 without throwing --- paths/test/paths/test_native.hpp | 47 ++++++++++++++ utility/test/utility/main.cpp | 1 + utility/test/utility/test_error_message.hpp | 69 +++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 utility/test/utility/test_error_message.hpp diff --git a/paths/test/paths/test_native.hpp b/paths/test/paths/test_native.hpp index 3d239379..a6552220 100644 --- a/paths/test/paths/test_native.hpp +++ b/paths/test/paths/test_native.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -22,6 +23,52 @@ namespace Paths::Test EXPECT_EQ(native.filename().extension(), nativeFromUtf8(".txt")); } + TEST(NativePathTests, ValidTextConvertsLosslessly) + { + const auto converted = convertNativeToUtf8(nativeFromUtf8("/home/Jürgen")); + EXPECT_EQ(converted.utf8, "/home/Jürgen"); + EXPECT_TRUE(converted.lossless); + EXPECT_EQ(checkedNativeToUtf8(nativeFromUtf8("/home/Jürgen")), "/home/Jürgen"); + } + +#ifndef _WIN32 + TEST(NativePathTests, BytesThatAreNotUtf8AreReplacedAndReported) + { + const std::filesystem::path latin1{std::string{"/home/Fran\xE7ois"}}; + const auto converted = convertNativeToUtf8(latin1); + EXPECT_EQ(converted.utf8, "/home/Fran\xEF\xBF\xBDois"); + EXPECT_FALSE(converted.lossless); + EXPECT_EQ(nativeToUtf8(latin1), converted.utf8); + + const auto checked = checkedNativeToUtf8(latin1); + ASSERT_FALSE(checked.has_value()); + EXPECT_EQ(checked.error().reason, PathRejection::InvalidUtf8); + } + + TEST(NativePathTests, LocalPathRefusesANativePathThatIsNotUtf8) + { + const auto made = LocalPath::make(std::filesystem::path{std::string{"/home/Fran\xE7ois"}}); + ASSERT_FALSE(made.has_value()); + EXPECT_EQ(made.error().reason, PathRejection::InvalidUtf8); + } +#else + TEST(NativePathTests, UnpairedSurrogatesAreReplacedAndReported) + { + const std::wstring wide{L'C', L':', L'/', L'a', static_cast(0xD800u)}; + const auto converted = convertNativeToUtf8(std::filesystem::path{wide}); + EXPECT_EQ(converted.utf8, "C:/a\xEF\xBF\xBD"); + EXPECT_FALSE(converted.lossless); + const auto made = LocalPath::make(std::filesystem::path{wide}); + ASSERT_FALSE(made.has_value()); + EXPECT_EQ(made.error().reason, PathRejection::InvalidUtf8); + } + + TEST(NativePathTests, TextThatIsNotUtf8BecomesReplacementCharacters) + { + EXPECT_EQ(nativeFromUtf8("a\xFF").native(), std::wstring{L"a\uFFFD"}); + } +#endif + #ifdef _WIN32 TEST(NativePathTests, BackslashesBecomeForwardSlashes) { diff --git a/utility/test/utility/main.cpp b/utility/test/utility/main.cpp index 445e8490..bca0e155 100644 --- a/utility/test/utility/main.cpp +++ b/utility/test/utility/main.cpp @@ -2,6 +2,7 @@ #include "test_command_template.hpp" #include "test_directory_traversal.hpp" #include "test_echo_suppressor.hpp" +#include "test_error_message.hpp" #include "test_fuzzy_search.hpp" #include "test_keyed_diff.hpp" #include "test_localized_message.hpp" diff --git a/utility/test/utility/test_error_message.hpp b/utility/test/utility/test_error_message.hpp new file mode 100644 index 00000000..5e78e642 --- /dev/null +++ b/utility/test/utility/test_error_message.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace Utility::Test +{ + TEST(ErrorMessageTests, GenericErrorsKeepTheirMessage) + { + const auto error = std::make_error_code(std::errc::no_such_file_or_directory); + EXPECT_EQ(errorMessage(error), Paths::sanitizedUtf8(error.message())); + EXPECT_TRUE(Paths::isValidUtf8(errorMessage(error))); + } + + TEST(ErrorMessageTests, SystemErrorsAreUtf8) + { + const std::error_code error{5, std::system_category()}; + const auto message = errorMessage(error); + EXPECT_FALSE(message.empty()); + EXPECT_TRUE(Paths::isValidUtf8(message)); + } + + TEST(ErrorMessageTests, FilesystemErrorsNameTheirPathsAsUtf8) + { + const auto code = std::make_error_code(std::errc::permission_denied); + const std::filesystem::filesystem_error error{ + "remove", Paths::nativeFromUtf8("/x/Grüße"), Paths::nativeFromUtf8("/y/東京"), code + }; + EXPECT_EQ(errorMessage(error), fmt::format("{}: '/x/Grüße', '/y/東京'", errorMessage(code))); + + const std::filesystem::filesystem_error single{"remove", Paths::nativeFromUtf8("/x/a"), code}; + EXPECT_EQ(errorMessage(single), fmt::format("{}: '/x/a'", errorMessage(code))); + + const std::filesystem::filesystem_error none{"remove", code}; + EXPECT_EQ(errorMessage(none), errorMessage(code)); + } + + TEST(ErrorMessageTests, ExceptionsAreUtf8) + { + const std::runtime_error plain{"bad \xFF"}; + EXPECT_EQ(errorMessage(plain), "bad \xEF\xBF\xBD"); + + const auto code = std::make_error_code(std::errc::permission_denied); + const std::filesystem::filesystem_error filesystemError{"remove", Paths::nativeFromUtf8("/x/ü"), code}; + std::exception const& asException = filesystemError; + EXPECT_EQ(errorMessage(asException), errorMessage(filesystemError)); + } + +#ifndef _WIN32 + TEST(ErrorMessageTests, FilesystemErrorsReplaceBytesThatAreNotUtf8) + { + const std::filesystem::filesystem_error error{ + "remove", std::filesystem::path{std::string{"/x/Fran\xE7ois"}}, std::make_error_code(std::errc::io_error) + }; + const auto message = errorMessage(error); + EXPECT_TRUE(Paths::isValidUtf8(message)); + EXPECT_NE(message.find("/x/Fran\xEF\xBF\xBDois"), std::string::npos); + } +#endif +} From 6047fcdfc07135e260686478f59d573e0cd59d7d Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:46:12 +0200 Subject: [PATCH 097/131] Tests: A listed name that is not UTF-8 is shown sanitized and never acted on --- backend/test/backend/test_operation_queue.hpp | 156 +++++++++++++++++- .../test_tree_directory_walker.hpp | 17 ++ ssh/test/ssh/test_sftp.hpp | 36 ++++ .../ssh/test_ssh2_servers/sftp_server.mjs | 44 ++++- 4 files changed, 244 insertions(+), 9 deletions(-) diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 3c3f4760..d3ea0b9f 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -130,9 +130,9 @@ namespace Test return completions_; } - static std::set remoteNames(SecureShell::SftpSession& sftp, std::filesystem::path const& directory) + static std::set remoteNames(SecureShell::SftpSession& sftp, std::string const& directory) { - auto future = sftp.listDirectory(remote(directory.generic_string())); + auto future = sftp.listDirectory(remote(directory)); if (future.wait_for(5s) != std::future_status::ready) return {""}; auto result = future.get(); @@ -219,7 +219,7 @@ namespace Test std::set names; std::error_code errorCode{}; for (auto const& entry : std::filesystem::directory_iterator{directory, errorCode}) - names.insert(entry.path().filename().generic_string()); + names.insert(Paths::nativeToUtf8(entry.path().filename())); return names; } @@ -1037,6 +1037,63 @@ namespace Test EXPECT_FALSE(std::filesystem::exists(localRoot / "localGone.txt")); } + TEST_F(OperationQueueTests, SyncTransfersNonAsciiNames) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/Größe"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/Größe/東京.txt"}, {"content", "down"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / Paths::nativeFromUtf8("Jürgen"); + std::filesystem::create_directories(localRoot / Paths::nativeFromUtf8("Ärger")); + std::ofstream{localRoot / Paths::nativeFromUtf8("Ärger") / Paths::nativeFromUtf8("Ölung.txt")} << "up"; + + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home/test/Größe"), + }, + context.get_executor() + }; + using SharedData::Sync::Action; + const auto entry = [&session](std::string relKey, Action action, std::uint64_t size) + { + auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); + return SyncSession::ResolvedPlanEntry{ + .entry = {.relKey = std::move(relKey), .action = action, .sizeBytes = size, .isDirectory = false}, + .paths = std::move(paths), + }; + }; + SyncSession::ResolvedPlan plan{ + .entries = + { + entry("Ärger/Ölung.txt", Action::Upload, 2), + entry("東京.txt", Action::Download, 4), + }, + }; + const SharedData::Sync::SyncExecutionRequest request{ + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + + const auto result = runOnStrand([&]() { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + }); + EXPECT_EQ(result.uploads, (std::vector{"Ärger/Ölung.txt"})); + EXPECT_EQ(result.downloads, (std::vector{"東京.txt"})); + + setPaused(false); + workUntilCompletions(2); + for (auto const& completed : completions()) + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Completed) << completed.operationId.value(); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Größe/Ärger"), (std::set{"Ölung.txt"})); + EXPECT_EQ(localNames(localRoot), (std::set{"Ärger", "東京.txt"})); + } + #ifndef _WIN32 TEST_F(OperationQueueTests, SyncRefusesEveryDeleteBelowAProtectedDirectory) { @@ -1454,6 +1511,99 @@ namespace Test #endif } + TEST_F(OperationQueueTests, NamesThatAreNotUtf8AreRefusedAndTheCompletionStillSerializes) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command( + "rawName", {{"directory", "/home/test/Documents"}, {"nameHex", "4672616ee76f6973"}, {"content", "latin1"}} + )); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "Documents"; + const auto download = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, download, toLocal(localPath), remote("/home/test/Documents"), false, false, false, false + ); + } + ).has_value()); + + workUntilCompletions(1); + const auto finished = completions().front(); + ASSERT_EQ(finished.failedEntries.size(), 1u); + for (auto const& [path, error] : finished.failedEntries) + { + EXPECT_EQ(error.type, SharedData::OperationErrorType::InvalidName) << path; + EXPECT_TRUE(Paths::isValidUtf8(path)); + EXPECT_NE(path.find("Fran\xEF\xBF\xBDois"), std::string::npos) << path; + } + const nlohmann::json serialized = finished; + EXPECT_NO_THROW(std::ignore = serialized.dump()); + EXPECT_EQ(localNames(localPath), (std::set{"doc1.txt", "doc2.txt"})); + } + + TEST_F(OperationQueueTests, NonAsciiNamesSurviveDownloadAndUpload) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/Grüße"}})); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/Grüße/東京 \xF0\x9F\x9A\x80.txt"}, {"content", "tokyo"}} + )); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto downloaded = localDirectory_.path() / Paths::nativeFromUtf8("Grüße"); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, + Ids::generateOperationId(), + toLocal(downloaded), + remote("/home/test/Grüße"), + false, + false, + false, + false + ); + } + ).has_value()); + workUntilCompletions(1); + EXPECT_EQ(completions().front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(localNames(downloaded), (std::set{"東京 \xF0\x9F\x9A\x80.txt"})); + std::ifstream downloadedFile{downloaded / Paths::nativeFromUtf8("東京 \xF0\x9F\x9A\x80.txt")}; + const std::string downloadedContent{std::istreambuf_iterator{downloadedFile}, {}}; + EXPECT_EQ(downloadedContent, "tokyo"); + + const auto toUpload = localDirectory_.path() / Paths::nativeFromUtf8("Jürgen"); + std::filesystem::create_directories(toUpload / Paths::nativeFromUtf8("Ärger")); + std::ofstream{toUpload / Paths::nativeFromUtf8("Ärger") / Paths::nativeFromUtf8("Größe.txt")} << "size"; + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addUploadOperation( + *sftp, + Ids::generateOperationId(), + toLocal(toUpload), + remote("/home/test/Jürgen"), + false, + false, + false, + false + ); + } + ).has_value()); + workUntilCompletions(2); + EXPECT_EQ(completions().back().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Jürgen"), (std::set{"Ärger"})); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Jürgen/Ärger"), (std::set{"Größe.txt"})); + } + TEST_F(OperationQueueTests, RecursiveDeleteNeverFollowsHostileNamesOut) { CREATE_SERVER_AND_JOINER(sftpServer); diff --git a/shared_data/test/shared_data/test_tree_directory_walker.hpp b/shared_data/test/shared_data/test_tree_directory_walker.hpp index e2de0a34..da8f6cdb 100644 --- a/shared_data/test/shared_data/test_tree_directory_walker.hpp +++ b/shared_data/test/shared_data/test_tree_directory_walker.hpp @@ -156,6 +156,23 @@ namespace SharedData::Sync::Test EXPECT_EQ(tree->children[0].children[0].children[0].children[0].name, "deep.txt"); } + TEST(TreeDirectoryWalkerTests, NamesThatWereNotUtf8AreInvalidAndNeverEntered) + { + FakeFilesystem filesystem; + filesystem.directories["/root"].push_back( + DirectoryEntry{.name = "Fran\xEF\xBF\xBDois", .nameIsValidUtf8 = false, .type = FileType::Directory} + ); + filesystem.directories["/root"].push_back( + DirectoryEntry{.name = "x\xEF\xBF\xBD", .nameIsValidUtf8 = false, .type = FileType::Regular, .size = 3} + ); + const auto tree = walkFake(filesystem); + ASSERT_TRUE(tree.has_value()) << tree.error(); + ASSERT_EQ(tree->children.size(), 2u); + for (auto const& child : tree->children) + EXPECT_EQ(child.availability, Availability::InvalidName) << child.name; + EXPECT_EQ(filesystem.listed, (std::vector{"/root"})); + } + TEST(TreeDirectoryWalkerTests, ScannerErrorsAbortTheWalk) { FakeFilesystem filesystem; diff --git a/ssh/test/ssh/test_sftp.hpp b/ssh/test/ssh/test_sftp.hpp index 6a20527d..d1747c5e 100644 --- a/ssh/test/ssh/test_sftp.hpp +++ b/ssh/test/ssh/test_sftp.hpp @@ -908,6 +908,42 @@ namespace SecureShell::Test EXPECT_NE(iter, listResult->end()); } + TEST_F(SftpTests, ListedNamesThatAreNotUtf8AreShownButNeverNamed) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("rawName", {{"directory", "/home/test"}, {"nameHex", "61ff62"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto listFut = sftp->listDirectory(remote("/home/test")); + ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); + const auto listResult = listFut.get(); + ASSERT_TRUE(listResult.has_value()); + const auto iter = std::find_if(listResult->begin(), listResult->end(), [](auto const& entry) { + return entry.name == "a\xEF\xBF\xBD" + "b"; + }); + ASSERT_NE(iter, listResult->end()); + EXPECT_FALSE(iter->nameIsValidUtf8); + const auto segment = iter->nameSegment(); + ASSERT_FALSE(segment.has_value()); + EXPECT_EQ(segment.error().reason, Paths::PathRejection::InvalidUtf8); + + const nlohmann::json listing = *listResult; + const auto dumped = listing.dump(); + const auto parsed = nlohmann::json::parse(dumped).get>(); + const auto parsedIter = std::find_if(parsed.begin(), parsed.end(), [](auto const& entry) { + return !entry.nameIsValidUtf8; + }); + ASSERT_NE(parsedIter, parsed.end()); + EXPECT_EQ(parsedIter->name, iter->name); + EXPECT_EQ( + std::count_if(parsed.begin(), parsed.end(), [](auto const& entry) { + return entry.nameIsValidUtf8; + }), + static_cast(parsed.size() - 1u) + ); + } + TEST_F(SftpTests, MockServerPosixRenameExtensionOverwrites) { CREATE_SERVER_AND_JOINER(Sftp); diff --git a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs index 1f69ad22..fff6dbef 100644 --- a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs +++ b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs @@ -600,7 +600,8 @@ const server = new Server({ return { filename: child.name, longname: makeLongName(child), - attrs: child.stat + attrs: child.stat, + rawName: child.rawName }; }); } @@ -610,7 +611,7 @@ const server = new Server({ logMessage(`Sending ${entries.length} entries`); logMessage(entries); logMessage(`Remaining entries: ${handle.remainingReadDirEntries.length}`); - sftpStream.name(reqid, entries); + sendNames(sftpStream, reqid, entries); return; } @@ -980,14 +981,45 @@ cli.on('advertise', (command) => { advertisedExtensions[command.name] = command.data ?? '1'; }); // Inserts an entry under a raw name that may be invalid as a path segment, like '..' or 'a/b'. +/** + * ssh2 writes names as UTF-8 strings, so a name whose bytes are no UTF-8 is listed under its ASCII placeholder of the + * same length and its bytes are put into the NAME packet in place of the placeholder. + */ +function sendNames(sftpStream, reqid, entries) { + const rawEntries = entries.filter((entry) => entry.rawName !== undefined); + if (rawEntries.length === 0) + return sftpStream.name(reqid, entries); + const protocol = sftpStream._protocol; + const chunks = []; + sftpStream._protocol = { channelData: (_id, data) => chunks.push(Buffer.from(data)) }; + try { + sftpStream.name(reqid, entries); + } finally { + sftpStream._protocol = protocol; + } + const packet = Buffer.concat(chunks); + for (const entry of rawEntries) { + const at = packet.indexOf(Buffer.from(entry.filename, 'latin1')); + if (at >= 0) + entry.rawName.copy(packet, at); + } + protocol.channelData(sftpStream.outgoing.id, packet); +} + cli.on('rawName', (command) => { const parent = fakeFilesystem.find(command.directory); if (parent === undefined || parent.type !== 'directory') return; - if (command.type === 'directory') - parent.insert(directory({ name: command.name }, [])); - else - parent.insert(file(command.name, command.content ?? '')); + // nameHex gives bytes that need not be UTF-8, they are listed through a placeholder of the same length. + const rawName = command.nameHex === undefined ? undefined : Buffer.from(command.nameHex, 'hex'); + const name = rawName === undefined + ? command.name + : Array.from({ length: rawName.length }, () => String.fromCharCode(65 + Math.floor(Math.random() * 26))).join(''); + const node = command.type === 'directory' + ? directory({ name }, []) + : file(name, command.content ?? ''); + node.rawName = rawName; + parent.insert(node); }); cli.on('createFile', (command) => { const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); From 96d268be42c0e36d373b9b9b90593222ff5e55e5 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 22:51:11 +0200 Subject: [PATCH 098/131] Tests: Calls into the frontend never throw on text that is not UTF-8 --- backend/test/backend/main.cpp | 1 + backend/test/backend/test_rpc_helper.hpp | 38 ++++++++++++++++++++++++ utility/test/utility/main.cpp | 1 + utility/test/utility/test_json_utf8.hpp | 36 ++++++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 backend/test/backend/test_rpc_helper.hpp create mode 100644 utility/test/utility/test_json_utf8.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index 48724cbe..42a160e4 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -28,6 +28,7 @@ #include "test_operation_queue.hpp" #include "test_existence_check.hpp" #include "test_opener_risk.hpp" +#include "test_rpc_helper.hpp" #include diff --git a/backend/test/backend/test_rpc_helper.hpp b/backend/test/backend/test_rpc_helper.hpp new file mode 100644 index 00000000..56e90f7a --- /dev/null +++ b/backend/test/backend/test_rpc_helper.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include "nui_env.hpp" + +#include +#include + +#include +#include +#include +#include + +namespace Test +{ + TEST(RpcHelperTests, ArgumentsOfTheSecondTryAreValidUtf8) + { + const auto text = RpcHelper::Detail::withValidUtf8Argument(std::string{"a\xFF"}); + EXPECT_EQ(text, nlohmann::json("a\xEF\xBF\xBD")); + + const std::uint64_t large = 1ull << 60u; + const auto number = RpcHelper::Detail::withValidUtf8Argument(large); + static_assert(std::is_same_v, std::uint64_t>); + EXPECT_EQ(number, large); + } + + TEST(RpcHelperTests, CallsCarryingTextThatIsNotUtf8NeverThrow) + { + if (!NuiEnv::instance().available()) + GTEST_SKIP() << "No display available"; + auto& hub = *NuiEnv::instance().hub; + const nlohmann::json invalid{{"name", "Fran\xE7ois"}}; + EXPECT_THROW(hub.callRemote("RpcHelperTests::nobody", invalid), nlohmann::json::type_error); + RpcHelper::callRemoteSafely(hub, "RpcHelperTests::nobody", invalid); + RpcHelper::callRemoteSafely(hub, "RpcHelperTests::nobody", std::string{"Fran\xE7ois"}, 42, true); + RpcHelper::callRemoteSafely(hub, "RpcHelperTests::nobody", {{"error", "Fran\xE7ois"}}); + } +} diff --git a/utility/test/utility/main.cpp b/utility/test/utility/main.cpp index bca0e155..fbd74a28 100644 --- a/utility/test/utility/main.cpp +++ b/utility/test/utility/main.cpp @@ -4,6 +4,7 @@ #include "test_echo_suppressor.hpp" #include "test_error_message.hpp" #include "test_fuzzy_search.hpp" +#include "test_json_utf8.hpp" #include "test_keyed_diff.hpp" #include "test_localized_message.hpp" #include "test_move_to_front.hpp" diff --git a/utility/test/utility/test_json_utf8.hpp b/utility/test/utility/test_json_utf8.hpp new file mode 100644 index 00000000..c09c0eeb --- /dev/null +++ b/utility/test/utility/test_json_utf8.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace Utility::Test +{ + TEST(JsonUtf8Tests, StringsAndKeysThatAreNotUtf8BecomeDumpable) + { + const nlohmann::json json{ + {"name", "a\xFF"}, + {"list", {"ok", "b\xC3"}}, + {"nested", {{"c\xFE", 1}}}, + {"number", 42}, + }; + EXPECT_THROW(std::ignore = json.dump(), nlohmann::json::type_error); + + const auto valid = withValidUtf8(json); + EXPECT_NO_THROW(std::ignore = valid.dump()); + EXPECT_EQ(valid["name"], "a\xEF\xBF\xBD"); + EXPECT_EQ(valid["list"][0], "ok"); + EXPECT_EQ(valid["list"][1], "b\xEF\xBF\xBD"); + EXPECT_EQ(valid["nested"]["c\xEF\xBF\xBD"], 1); + EXPECT_EQ(valid["number"], 42); + } + + TEST(JsonUtf8Tests, ValidJsonStaysTheSame) + { + const nlohmann::json json{{"name", "Grüße"}, {"values", {1, 2, 3}}, {"flag", true}}; + EXPECT_EQ(withValidUtf8(json), json); + } +} From 53f4d067d766238206360a73189b6ecc00372ab9 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:05:25 +0200 Subject: [PATCH 099/131] Tests: Key files are read by their native path and imported from memory --- ssh/test/ssh/main.cpp | 1 + ssh/test/ssh/test_key_file.hpp | 72 ++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 ssh/test/ssh/test_key_file.hpp diff --git a/ssh/test/ssh/main.cpp b/ssh/test/ssh/main.cpp index 8930f777..089072f2 100644 --- a/ssh/test/ssh/main.cpp +++ b/ssh/test/ssh/main.cpp @@ -5,6 +5,7 @@ #include "test_ssh_session.hpp" #include "test_sftp.hpp" #include "test_sftp_in_strand.hpp" +#include "test_key_file.hpp" #include diff --git a/ssh/test/ssh/test_key_file.hpp b/ssh/test/ssh/test_key_file.hpp new file mode 100644 index 00000000..8b45bbd0 --- /dev/null +++ b/ssh/test/ssh/test_key_file.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include "common_fixture.hpp" +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace SecureShell::Test +{ + TEST(KeyFileTests, SplitsAnOpenSshPublicKeyLine) + { + const auto line = Detail::splitPublicKeyLine("ssh-ed25519 AAAAC3Nza comment with spaces\n"); + ASSERT_TRUE(line.has_value()); + EXPECT_EQ(line->first, "ssh-ed25519"); + EXPECT_EQ(line->second, "AAAAC3Nza"); + EXPECT_FALSE(Detail::splitPublicKeyLine("-----BEGIN PUBLIC KEY-----\nMIIC\n").has_value()); + EXPECT_FALSE(Detail::splitPublicKeyLine("unknown-type AAAA").has_value()); + EXPECT_FALSE(Detail::splitPublicKeyLine("ssh-rsa").has_value()); + EXPECT_FALSE(Detail::splitPublicKeyLine("").has_value()); + } + + TEST(KeyFileTests, ImportsKeysFromFilesWithNonAsciiNames) + { + const auto directory = temporaryRoot / Paths::nativeFromUtf8("Schlüssel 東京"); + std::filesystem::create_directories(directory); + const auto privateFile = directory / Paths::nativeFromUtf8("ключ"); + { + std::ofstream stream{privateFile, std::ios::binary}; + stream.write(privateKey, static_cast(std::strlen(privateKey))); + } + + SshKey imported{}; + ASSERT_EQ( + importPrivateKeyFile(privateFile, nullptr, nullptr, nullptr, &imported.underlyingReferenced()), SSH_OK + ); + ASSERT_TRUE(imported); + + char* blob = nullptr; + ASSERT_EQ(ssh_pki_export_pubkey_base64(imported.underlying(), &blob), SSH_OK); + const std::unique_ptr ownedBlob{blob, &ssh_string_free_char}; + const auto publicFile = directory / Paths::nativeFromUtf8("ключ.pub"); + { + std::ofstream stream{publicFile, std::ios::binary}; + stream << fmt::format( + "{} {} test@host\n", ssh_key_type_to_char(ssh_key_type(imported.underlying())), ownedBlob.get() + ); + } + SshKey importedPublic{}; + ASSERT_EQ(importPublicKeyFile(publicFile, &importedPublic.underlyingReferenced()), SSH_OK); + EXPECT_EQ(ssh_key_cmp(importedPublic.underlying(), imported.underlying(), SSH_KEY_CMP_PUBLIC), 0); + } + + TEST(KeyFileTests, AMissingKeyFileIsReportedLikeLibssh) + { + SshKey key{}; + const auto missing = temporaryRoot / Paths::nativeFromUtf8("fehlt-ü"); + EXPECT_EQ(importPrivateKeyFile(missing, nullptr, nullptr, nullptr, &key.underlyingReferenced()), SSH_EOF); + EXPECT_EQ(importPublicKeyFile(missing, &key.underlyingReferenced()), SSH_EOF); + } +} From a590bf11a0ffbc160b5941bc372eaa458bae4a17 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:11:13 +0200 Subject: [PATCH 100/131] Tests: Case conversion changes ASCII letters only --- utility/test/utility/main.cpp | 1 + utility/test/utility/test_case_convert.hpp | 34 ++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 utility/test/utility/test_case_convert.hpp diff --git a/utility/test/utility/main.cpp b/utility/test/utility/main.cpp index fbd74a28..6f25c589 100644 --- a/utility/test/utility/main.cpp +++ b/utility/test/utility/main.cpp @@ -1,4 +1,5 @@ #include +#include "test_case_convert.hpp" #include "test_command_template.hpp" #include "test_directory_traversal.hpp" #include "test_echo_suppressor.hpp" diff --git a/utility/test/utility/test_case_convert.hpp b/utility/test/utility/test_case_convert.hpp new file mode 100644 index 00000000..e17c7765 --- /dev/null +++ b/utility/test/utility/test_case_convert.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include + +#include + +namespace Utility::Test +{ + TEST(CaseConvertTests, OnlyAsciiLettersChangeCase) + { + EXPECT_EQ(Algorithm::toLowerCase("ÄBC Größe"), "Äbc größe"); + EXPECT_EQ(Algorithm::toUpperCase("äbc größe"), "äBC GRößE"); + std::string text = "\xC3\x84X\xFF"; + Algorithm::toLowerCaseInplace(text); + EXPECT_EQ(text, "\xC3\x84x\xFF"); + } + + TEST(CaseConvertTests, CompareIgnoresAsciiCaseOnly) + { + EXPECT_EQ(Algorithm::caseInsensitiveCompare("ABC", "abc"), 0); + EXPECT_NE(Algorithm::caseInsensitiveCompare("Ä", "ä"), 0); + EXPECT_LT(Algorithm::caseInsensitiveCompare("a", "\xC3\xA4"), 0); + } + + TEST(CaseConvertTests, NamingConventionsKeepUtf8Intact) + { + const SegmentedString segments{.normalizedSegments = {"größe", "übung"}}; + EXPECT_EQ(segments.camelCase(), "größeübung"); + EXPECT_EQ(segments.screamingSnakeCase(), "größe_üBUNG"); + } +} From f7b2bdf32c99718b3de4250528dbd8efb915ec9e Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:13:10 +0200 Subject: [PATCH 101/131] Tests: The tar writer stores non-ASCII names in PAX records --- .../test_archive_download_operation.hpp | 26 ++++++++ .../backend/test_archive_upload_operation.hpp | 31 ++++++++++ tar-archive/test/tar_archive/test_header.hpp | 61 +++++++++++++++++++ tar-archive/test/tar_archive/test_pax.hpp | 28 +++++++++ 4 files changed, 146 insertions(+) diff --git a/backend/test/backend/test_archive_download_operation.hpp b/backend/test/backend/test_archive_download_operation.hpp index ee2599cc..63689e3b 100644 --- a/backend/test/backend/test_archive_download_operation.hpp +++ b/backend/test/backend/test_archive_download_operation.hpp @@ -4,6 +4,7 @@ #include #include "real_server_tests.hpp" #include "archive_test_helpers.hpp" +#include #include #include @@ -164,6 +165,31 @@ namespace Test EXPECT_EQ(entries[0].contents, "Fake file content"); } + TEST_F(ArchiveDownloadOperationTests, NonAsciiNamesRoundTrip) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/Größe.txt"}, {"content", "size"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto archivePath = archiveDir_.path() / Paths::nativeFromUtf8("Ärger.tar"); + ArchiveDownloadOperation operation{ + sftp, + { + .roots = {makeFileEntry("/home/test/Größe.txt", 4u)}, + .localArchivePath = toLocal(archivePath), + .compression = TarArchive::Compression::None, + } + }; + auto result = runToCompletion(operation); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(*result, ArchiveDownloadOperation::WorkStatus::Complete); + + const auto entries = recoverArchiveEntries(archivePath); + ASSERT_EQ(entries.size(), 1u); + EXPECT_EQ(entries[0].path, "Größe.txt"); + EXPECT_EQ(entries[0].contents, "size"); + } + TEST_F(ArchiveDownloadOperationTests, SingleFileGzipRoundTrip) { CREATE_SERVER_AND_JOINER(sftpServer); diff --git a/backend/test/backend/test_archive_upload_operation.hpp b/backend/test/backend/test_archive_upload_operation.hpp index 77026c7c..6ccdac02 100644 --- a/backend/test/backend/test_archive_upload_operation.hpp +++ b/backend/test/backend/test_archive_upload_operation.hpp @@ -5,6 +5,7 @@ #include #include "real_server_tests.hpp" #include "archive_test_helpers.hpp" +#include #include @@ -192,6 +193,36 @@ namespace Test EXPECT_EQ(entries[0].contents, "alpha payload"); } + TEST_F(ArchiveUploadOperationTests, NonAsciiNamesRoundTrip) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto source = srcDir_.path() / Paths::nativeFromUtf8("Grüße"); + std::filesystem::create_directories(source); + std::ofstream{source / Paths::nativeFromUtf8("東京.txt"), std::ios::binary} << "tokyo"; + auto scan = runLocalScan(source); + + const std::filesystem::path remoteArchive = "/home/test/nonascii.tar"; + ArchiveUploadOperation operation{ + sftp, + { + .localPaths = {toLocal(source)}, + .remoteArchivePath = remote(remoteArchive.generic_string()), + .compression = TarArchive::Compression::None, + .mayOverwrite = true, + } + }; + operation.setScanResultForRoot(toLocal(source), std::move(scan.entries), scan.totalBytes); + ASSERT_TRUE(runToCompletion(operation).has_value()); + + const auto pulled = pullRemoteArchive(*sftp, remoteArchive, "nonascii.tar"); + const auto entries = recoverArchiveEntries(pulled); + const auto* file = findEntry(entries, "Grüße/東京.txt"); + ASSERT_NE(file, nullptr); + EXPECT_EQ(file->contents, "tokyo"); + } + TEST_F(ArchiveUploadOperationTests, SingleFileGzipRoundTrip) { CREATE_SERVER_AND_JOINER(sftpServer); diff --git a/tar-archive/test/tar_archive/test_header.hpp b/tar-archive/test/tar_archive/test_header.hpp index ee01884c..c3460b12 100644 --- a/tar-archive/test/tar_archive/test_header.hpp +++ b/tar-archive/test/tar_archive/test_header.hpp @@ -2,12 +2,14 @@ #include #include +#include #include #include #include #include +#include #include namespace TarArchive::Test @@ -169,6 +171,65 @@ namespace TarArchive::Test EXPECT_EQ(fileTypeFromTypeFlag(TypeFlag::FifoSpecial), FileType::Fifo); } + TEST(TarHeader, NonAsciiNamesGetPaxRecords) + { + const std::string name = "Grüße/東京.txt"; + const auto built = buildRecords(makeRegularEntry(name, 4u)); + ASSERT_TRUE(built.has_value()) << built.error().toString(); + ASSERT_EQ(built->records.size(), 3u); + + const auto paxHeader = parseRecord(built->records[0]); + ASSERT_TRUE(paxHeader.has_value()); + EXPECT_EQ(paxHeader->typeflag, TypeFlag::ExtendedHeader); + const auto overrides = parsePaxPayload( + std::span{built->records[1]}.first(static_cast(paxHeader->size)) + ); + ASSERT_TRUE(overrides.has_value()) << overrides.error().toString(); + EXPECT_EQ(overrides->path, name); + + const auto main = parseRecord(built->records[2]); + ASSERT_TRUE(main.has_value()); + EXPECT_EQ(main->fullName(), name); + } + + TEST(TarHeader, NonAsciiLinkTargetsGetPaxRecords) + { + TarArchive::MemberHeader entry{}; + entry.path = "link"; + entry.type = SharedData::FileType::Symlink; + entry.linkTarget = "Ärger.txt"; + entry.permissions = static_cast(0777u); + + const auto built = buildRecords(entry); + ASSERT_TRUE(built.has_value()); + ASSERT_EQ(built->records.size(), 3u); + const auto paxHeader = parseRecord(built->records[0]); + ASSERT_TRUE(paxHeader.has_value()); + const auto overrides = parsePaxPayload( + std::span{built->records[1]}.first(static_cast(paxHeader->size)) + ); + ASSERT_TRUE(overrides.has_value()); + EXPECT_EQ(overrides->linkPath, "Ärger.txt"); + EXPECT_FALSE(overrides->path.has_value()); + } + + TEST(TarHeader, TruncatedUstarNamesEndAtACodePointBoundary) + { + std::string name; + for (int index = 0; index < 120; ++index) + name += "ü"; + const auto built = buildRecords(makeRegularEntry(name, 1u)); + ASSERT_TRUE(built.has_value()); + const auto main = parseRecord(built->records.back()); + ASSERT_TRUE(main.has_value()); + EXPECT_EQ(main->fullName().size(), 100u); + EXPECT_TRUE(Paths::isValidUtf8(main->fullName())); + + const auto paxHeader = parseRecord(built->records.front()); + ASSERT_TRUE(paxHeader.has_value()); + EXPECT_TRUE(Paths::isValidUtf8(paxHeader->fullName())); + } + TEST(TarHeader, LargeSizeTriggersPaxExtendedHeader) { TarArchive::MemberHeader entry{}; diff --git a/tar-archive/test/tar_archive/test_pax.hpp b/tar-archive/test/tar_archive/test_pax.hpp index dceb3caf..9515fba0 100644 --- a/tar-archive/test/tar_archive/test_pax.hpp +++ b/tar-archive/test/tar_archive/test_pax.hpp @@ -63,6 +63,34 @@ namespace TarArchive::Test EXPECT_EQ(entry.header().size, contents.size()); } + TEST(TarPax, NonAsciiPathRoundTrips) + { + const std::string name = "Grüße/東京 \xF0\x9F\x9A\x80.txt"; + TarArchive::MemberHeader meta{}; + meta.path = name; + meta.type = SharedData::FileType::Regular; + meta.size = 0u; + meta.permissions = static_cast(0644u); + + const auto path = paxDirectory() / "nonascii.tar"; + Archive archive{path}; + { + auto writer = archive.openWriter(); + ASSERT_TRUE(writer.has_value()); + auto entry = writer->beginEntry(meta); + ASSERT_TRUE(entry.has_value()); + ASSERT_TRUE(std::move(*entry).close().has_value()); + ASSERT_TRUE(writer->finalize().has_value()); + } + + auto reader = archive.openReader(); + ASSERT_TRUE(reader.has_value()); + auto next = reader->nextEntry(); + ASSERT_TRUE(next.has_value()); + ASSERT_TRUE(next->has_value()); + EXPECT_EQ((**next).header().path, name); + } + TEST(TarPax, LongSymlinkTargetRoundTripsViaExtendedHeader) { const std::string linkText = std::string(250u, 'x'); From f6b3c67ac02e2dffa736adc4caa4b7c566e20784 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:14:11 +0200 Subject: [PATCH 102/131] Tests: Add the NarrowPathConversions ctest A ctest (NarrowPathConversions) fails when native first-party sources call string() or generic_string(), which convert paths through the ANSI code page on Windows --- CMakeLists.txt | 4 +++ _cmake/check_narrow_path_text.cmake | 45 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 _cmake/check_narrow_path_text.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 177f81af..27440be4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,10 @@ else() add_subdirectory(shared_data/benchmark/shared_data) add_subdirectory(tar-archive/test/tar_archive) add_subdirectory(command-store/test/command-store) + add_test( + NAME NarrowPathConversions + COMMAND "${CMAKE_COMMAND}" "-DSOURCE_DIR=${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/_cmake/check_narrow_path_text.cmake" + ) endif() if (NUI_SFT_ENABLE_LANGUAGE_TOOLING) diff --git a/_cmake/check_narrow_path_text.cmake b/_cmake/check_narrow_path_text.cmake new file mode 100644 index 00000000..f22cf7ab --- /dev/null +++ b/_cmake/check_narrow_path_text.cmake @@ -0,0 +1,45 @@ +# Fails when native first-party sources call string() or generic_string(), which convert a path through the ANSI code +# page on Windows and throw outside it. Paths::nativeToUtf8 and Paths::nativeFromUtf8 convert explicitly. Frontend +# sources are not checked, a path there is UTF-8 already. Run with -DSOURCE_DIR=. + +set(roots + backend + command-store/include + command-store/source + log + nui-file-explorer/source/nui-file-explorer/support + paths/include + persistence/include + persistence/source + shared_data/include + shared_data/source + ssh/include + ssh/source + tar-archive/include + tar-archive/source + utility/include + utility/source +) + +set(found "") +foreach(root IN LISTS roots) + file(GLOB_RECURSE sources "${SOURCE_DIR}/${root}/*.cpp" "${SOURCE_DIR}/${root}/*.hpp" "${SOURCE_DIR}/${root}/*.mm") + foreach(source IN LISTS sources) + file(RELATIVE_PATH relative "${SOURCE_DIR}" "${source}") + if (relative MATCHES "(^|/)test/") + continue() + endif() + file(STRINGS "${source}" lines REGEX "\\.(generic_)?string\\(\\)") + foreach(line IN LISTS lines) + if (NOT line MATCHES "^[ \t]*//") + string(STRIP "${line}" stripped) + string(APPEND found "\n ${relative}: ${stripped}") + endif() + endforeach() + endforeach() +endforeach() + +if (found) + message(FATAL_ERROR "Narrow path conversions, use Paths::nativeToUtf8 or Paths::nativeFromUtf8:${found}") +endif() +message(STATUS "No narrow path conversions in native sources.") From 5ab42d2656674c04e6feee726a84f4eb7267ebb0 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:16:52 +0200 Subject: [PATCH 103/131] Tests: Local names that are not UTF-8 and non-ASCII file tracking paths Tests for a local name that is not UTF-8 (shown with U+FFFD, rejected by flat and tree scans), a watched non-ASCII directory and non-ASCII names in the file tracking change filter --- .../test/backend/test_file_change_filter.hpp | 2 ++ backend/test/backend/test_file_tracking.hpp | 13 +++++++++ .../backend/test_local_scan_operation.hpp | 28 +++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/backend/test/backend/test_file_change_filter.hpp b/backend/test/backend/test_file_change_filter.hpp index a4844ba9..42223bdf 100644 --- a/backend/test/backend/test_file_change_filter.hpp +++ b/backend/test/backend/test_file_change_filter.hpp @@ -38,6 +38,8 @@ namespace Test "archive.sb-12345678-short", "DS_Store", "filepart.txt", + "Ärger.txt", + "東京/報告 \xF0\x9F\x9A\x80.pdf", } ) EXPECT_FALSE(isTransientFile(name, filepart)) << name; diff --git a/backend/test/backend/test_file_tracking.hpp b/backend/test/backend/test_file_tracking.hpp index 6ba8d6ac..4b09a684 100644 --- a/backend/test/backend/test_file_tracking.hpp +++ b/backend/test/backend/test_file_tracking.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -430,6 +431,18 @@ namespace Test EXPECT_EQ(watch->path(), subdir); } + TEST_F(TemporaryDirectoryInstanceTest, AddWatchReturnsValidHandleForANonAsciiSubdir) + { + auto inst = makeInstance(); + auto subdir = inst->instanceDir() / Paths::nativeFromUtf8("Größe 東京"); + fs::create_directory(subdir); + + auto watch = inst->addWatch(subdir); + ASSERT_TRUE(watch.has_value()); + EXPECT_TRUE(watch->isValid()); + EXPECT_EQ(watch->path(), subdir); + } + TEST_F(TemporaryDirectoryInstanceTest, AddWatchReturnsValidHandleForInstanceDirItself) { auto inst = makeInstance(); diff --git a/backend/test/backend/test_local_scan_operation.hpp b/backend/test/backend/test_local_scan_operation.hpp index 5c524ac9..cfed736e 100644 --- a/backend/test/backend/test_local_scan_operation.hpp +++ b/backend/test/backend/test_local_scan_operation.hpp @@ -378,6 +378,34 @@ namespace Test EXPECT_EQ(operation.strand(), nullptr); } +#if !defined(_WIN32) && !defined(__APPLE__) + TEST_F(LocalScanOperationTests, NamesThatAreNotUtf8AreShownButRejected) + { + const auto latin1 = scanRoot_.path() / std::filesystem::path{std::string{"Fran\xE7ois.txt"}}; + writeFile(latin1, "latin1"); + std::error_code error{}; + if (!std::filesystem::exists(latin1, error)) + GTEST_SKIP() << "The filesystem refuses names that are not UTF-8"; + + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = false}}; + ASSERT_EQ(runScanToCompletion(operation), LocalScanOperation::WorkStatus::Complete); + const auto rejected = operation.rejectedPaths(); + ASSERT_EQ(rejected.size(), 1u); + EXPECT_TRUE(Paths::isValidUtf8(rejected.front())); + EXPECT_TRUE(rejected.front().ends_with("/Fran\xEF\xBF\xBDois.txt")) << rejected.front(); + const auto entries = operation.ejectEntries(); + EXPECT_FALSE(hasEntryNamed(entries, "Fran\xEF\xBF\xBDois.txt")); + EXPECT_TRUE(hasEntryNamed(entries, "file1.txt")); + + LocalScanOperation treeScan{{.localPath = toLocal(scanRoot_.path()), .recursive = false, .buildTree = true}}; + ASSERT_EQ(runScanToCompletion(treeScan), LocalScanOperation::WorkStatus::Complete); + const auto tree = treeScan.ejectScanTree(); + auto const* node = SharedData::Sync::findScanNode(tree, "Fran\xEF\xBF\xBDois.txt"); + ASSERT_NE(node, nullptr); + EXPECT_EQ(node->availability, SharedData::Sync::Availability::InvalidName); + } +#endif + TEST_F(LocalScanOperationTests, BuildTreeNestsAndSortsEntries) { LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = true, .buildTree = true}}; From ee3c4b070da6ca4c766c1137ac1117fdd7c2a3fc Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:42:59 +0200 Subject: [PATCH 104/131] Tests: Link-safe removal goes on past failures and names the failed item --- .../test/utility/test_link_safe_remove.hpp | 225 ++++++++++++++++++ 1 file changed, 225 insertions(+) diff --git a/utility/test/utility/test_link_safe_remove.hpp b/utility/test/utility/test_link_safe_remove.hpp index 1dfd4a34..6e1da5eb 100644 --- a/utility/test/utility/test_link_safe_remove.hpp +++ b/utility/test/utility/test_link_safe_remove.hpp @@ -1,15 +1,22 @@ #pragma once #include +#include #include #include +#include #include #include #include +#include #include +#ifndef _WIN32 +# include +#endif + extern std::filesystem::path programDirectory; extern std::filesystem::path temporaryRoot; @@ -33,9 +40,47 @@ namespace Utility::Test #endif } + /** + * @brief A tree with content, a nested directory link out of it and a file link: tree/a.txt (1 byte), + * tree/sub/b.txt (2 bytes), tree/sub/deeper/c.txt (3 bytes), tree/sub/deeper/out -> outside and + * tree/fileLink -> outside/keep.txt. 8 entries counting tree itself. + */ + std::filesystem::path makeTreeWithLinks() + { + const auto outside = root_.path() / "outside"; + std::filesystem::create_directories(outside); + std::ofstream{outside / "keep.txt"} << "keep"; + const auto tree = root_.path() / "tree"; + std::filesystem::create_directories(tree / "sub" / "deeper"); + std::ofstream{tree / "a.txt"} << "a"; + std::ofstream{tree / "sub" / "b.txt"} << "bb"; + std::ofstream{tree / "sub" / "deeper" / "c.txt"} << "ccc"; + EXPECT_TRUE(makeDirectoryLink(tree / "sub" / "deeper" / "out", outside)); + std::error_code error; + std::filesystem::create_symlink(outside / "keep.txt", tree / "fileLink", error); + EXPECT_FALSE(error) << error.message(); + return tree; + } + TemporaryDirectory root_{temporaryRoot, true}; }; + TEST(LinkReparseTagTests, OnlyNameSurrogatesAreLinks) + { + // IO_REPARSE_TAG_SYMLINK, IO_REPARSE_TAG_MOUNT_POINT (junction), IO_REPARSE_TAG_LX_SYMLINK + EXPECT_TRUE(Detail::isLinkReparseTag(0xA000000Cu)); + EXPECT_TRUE(Detail::isLinkReparseTag(0xA0000003u)); + EXPECT_TRUE(Detail::isLinkReparseTag(0xA000001Du)); + // IO_REPARSE_TAG_CLOUD_6 (OneDrive placeholder), IO_REPARSE_TAG_DEDUP, IO_REPARSE_TAG_AF_UNIX, + // IO_REPARSE_TAG_APPEXECLINK, IO_REPARSE_TAG_WOF + EXPECT_FALSE(Detail::isLinkReparseTag(0x9000601Au)); + EXPECT_FALSE(Detail::isLinkReparseTag(0x80000013u)); + EXPECT_FALSE(Detail::isLinkReparseTag(0x80000023u)); + EXPECT_FALSE(Detail::isLinkReparseTag(0x8000001Bu)); + EXPECT_FALSE(Detail::isLinkReparseTag(0x80000017u)); + EXPECT_FALSE(Detail::isLinkReparseTag(0u)); + } + TEST_F(LinkSafeRemoveTests, DirectoryLinkIsALinkAndADirectoryIsNot) { const auto target = root_.path() / "target"; @@ -129,4 +174,184 @@ namespace Utility::Test EXPECT_EQ(removeAllNoFollow(root_.path() / "missing", error), 0u); EXPECT_FALSE(error); } + + TEST_F(LinkSafeRemoveTests, KindsAreInspectedWithoutFollowing) + { + const auto target = root_.path() / "target"; + std::filesystem::create_directories(target); + std::ofstream{root_.path() / "file.txt"} << "file"; + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + + std::error_code error; + EXPECT_EQ(noFollowKindOf(target, error), NoFollowKind::Directory); + EXPECT_EQ(noFollowKindOf(root_.path() / "file.txt", error), NoFollowKind::Other); + EXPECT_EQ(noFollowKindOf(root_.path() / "link", error), NoFollowKind::Link); + EXPECT_EQ(noFollowKindOf(root_.path() / "missing", error), NoFollowKind::Missing); + EXPECT_FALSE(error) << error.message(); + } + + TEST_F(LinkSafeRemoveTests, RemoveNoFollowRemovesOneItemOnly) + { + const auto target = root_.path() / "target"; + std::filesystem::create_directories(target / "full"); + std::ofstream{target / "full" / "keep.txt"} << "keep"; + std::filesystem::create_directories(target / "empty"); + std::ofstream{target / "file.txt"} << "file"; + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + + std::error_code error; + EXPECT_TRUE(removeNoFollow(root_.path() / "link", error)); + EXPECT_FALSE(error) << error.message(); + EXPECT_TRUE(std::filesystem::exists(target / "full" / "keep.txt")) << "the link target must survive"; + + EXPECT_TRUE(removeNoFollow(target / "file.txt", error)); + EXPECT_TRUE(removeNoFollow(target / "empty", error)); + EXPECT_FALSE(removeNoFollow(target / "full", error)); + EXPECT_TRUE(error) << "a directory with content is not removed"; + EXPECT_TRUE(std::filesystem::exists(target / "full" / "keep.txt")); + + EXPECT_FALSE(removeNoFollow(root_.path() / "missing", error)); + EXPECT_FALSE(error) << "a missing item is no error"; + } + +#ifdef _WIN32 + TEST_F(LinkSafeRemoveTests, ReadOnlyItemsAreRemoved) + { + const auto tree = root_.path() / "readOnly"; + std::filesystem::create_directories(tree); + std::ofstream{tree / "locked.txt"} << "locked"; + std::filesystem::permissions(tree / "locked.txt", std::filesystem::perms::owner_write, std::filesystem::perm_options::remove); + + const auto result = removeAllNoFollow(tree); + EXPECT_TRUE(result.failures.empty()); + EXPECT_EQ(result.removed, 2u); + EXPECT_FALSE(std::filesystem::exists(tree)); + } +#endif + +#ifndef _WIN32 + TEST_F(LinkSafeRemoveTests, AnUndeletableEntryKeepsOnlyItsDirectories) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root removes entries of read-only directories"; + + const auto tree = root_.path() / "stuck"; + std::filesystem::create_directories(tree / "b-locked"); + std::filesystem::create_directories(tree / "c-free"); + std::ofstream{tree / "a.txt"} << "a"; + std::ofstream{tree / "b-locked" / "inner.txt"} << "inner"; + std::ofstream{tree / "c-free" / "free.txt"} << "free"; + std::ofstream{tree / "d.txt"} << "d"; + std::filesystem::permissions(tree / "b-locked", std::filesystem::perms::owner_write, std::filesystem::perm_options::remove); + + const auto result = removeAllNoFollow(tree); + std::filesystem::permissions(tree / "b-locked", std::filesystem::perms::owner_write, std::filesystem::perm_options::add); + + ASSERT_EQ(result.failures.size(), 1u); + EXPECT_EQ(result.failures.front().path, tree / "b-locked" / "inner.txt") << "the real path that failed"; + EXPECT_TRUE(result.failures.front().error); + EXPECT_EQ(result.removed, 4u) << "a.txt, c-free, free.txt and d.txt"; + EXPECT_TRUE(std::filesystem::exists(tree / "b-locked" / "inner.txt")); + EXPECT_FALSE(std::filesystem::exists(tree / "a.txt")); + EXPECT_FALSE(std::filesystem::exists(tree / "c-free")); + EXPECT_FALSE(std::filesystem::exists(tree / "d.txt")); + + std::error_code error; + removeAllNoFollow(tree, error); + EXPECT_FALSE(error) << "the rest goes once it can"; + } + + TEST_F(LinkSafeRemoveTests, AnUnreadableDirectoryIsAFailureOfItsOwn) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root reads every directory"; + + const auto tree = root_.path() / "unreadable"; + std::filesystem::create_directories(tree / "closed"); + std::ofstream{tree / "closed" / "inner.txt"} << "inner"; + std::ofstream{tree / "open.txt"} << "open"; + std::filesystem::permissions(tree / "closed", std::filesystem::perms::owner_read, std::filesystem::perm_options::remove); + + const auto result = removeAllNoFollow(tree); + std::filesystem::permissions(tree / "closed", std::filesystem::perms::owner_read, std::filesystem::perm_options::add); + + ASSERT_EQ(result.failures.size(), 1u); + EXPECT_EQ(result.failures.front().path, tree / "closed"); + EXPECT_FALSE(std::filesystem::exists(tree / "open.txt")); + EXPECT_TRUE(std::filesystem::exists(tree / "closed" / "inner.txt")); + } +#endif + + TEST_F(LinkSafeRemoveTests, CountingWalksTheTreeWithoutFollowingLinks) + { + const auto tree = makeTreeWithLinks(); + + NoFollowTreeWalk walk{tree, NoFollowTreeWalk::Mode::Count}; + EXPECT_TRUE(walk.step(std::chrono::steady_clock::time_point::max())); + EXPECT_TRUE(walk.done()); + EXPECT_EQ(walk.entries(), 8u) << "tree, a.txt, sub, b.txt, deeper, c.txt, out and fileLink"; + EXPECT_EQ(walk.bytes(), 6u) << "links count no bytes and are not followed"; + EXPECT_TRUE(walk.failures().empty()); + EXPECT_TRUE(std::filesystem::exists(tree / "sub" / "deeper" / "c.txt")) << "counting removes nothing"; + } + + TEST_F(LinkSafeRemoveTests, RemovingInSmallStepsDeletesTheTreeAndKeepsLinkTargets) + { + const auto tree = makeTreeWithLinks(); + + NoFollowTreeWalk walk{tree, NoFollowTreeWalk::Mode::Remove}; + int steps = 0; + std::set seen; + while (!walk.step(std::chrono::steady_clock::time_point::min())) + { + ++steps; + seen.insert(walk.lastPath()); + ASSERT_LT(steps, 100); + } + EXPECT_GE(steps, 7) << "an expired deadline still handles one entry per step"; + EXPECT_TRUE(seen.contains(tree / "sub" / "deeper" / "out")); + EXPECT_EQ(walk.entries(), 8u); + EXPECT_TRUE(walk.failures().empty()); + EXPECT_FALSE(std::filesystem::exists(tree)); + EXPECT_TRUE(std::filesystem::exists(root_.path() / "outside" / "keep.txt")); + } + + TEST_F(LinkSafeRemoveTests, WalkingALinkRootTouchesOnlyTheLink) + { + const auto target = root_.path() / "target"; + std::filesystem::create_directories(target); + std::ofstream{target / "keep.txt"} << "keep"; + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + + NoFollowTreeWalk count{root_.path() / "link", NoFollowTreeWalk::Mode::Count}; + count.step(std::chrono::steady_clock::time_point::max()); + EXPECT_EQ(count.entries(), 1u); + + NoFollowTreeWalk walk{root_.path() / "missing", NoFollowTreeWalk::Mode::Remove}; + EXPECT_TRUE(walk.step(std::chrono::steady_clock::time_point::max())); + EXPECT_EQ(walk.entries(), 0u); + EXPECT_TRUE(walk.failures().empty()); + } + + TEST_F(LinkSafeRemoveTests, VerifiedDirectoriesAreInspectedOnce) + { + const auto outside = root_.path() / "outside"; + std::filesystem::create_directories(outside); + const auto tree = root_.path() / "tree"; + std::filesystem::create_directories(tree / "real" / "deeper"); + + VerifiedDirectories verified; + std::error_code error; + EXPECT_FALSE(firstLinkBelow(tree, tree / "real" / "deeper" / "missing" / "file.txt", error, &verified)); + EXPECT_FALSE(error) << error.message(); + EXPECT_TRUE(verified.contains(tree / "real")); + EXPECT_TRUE(verified.contains(tree / "real" / "deeper")); + EXPECT_FALSE(verified.contains(tree / "real" / "deeper" / "missing")) << "a missing component is not verified"; + + ASSERT_TRUE(makeDirectoryLink(tree / "real" / "deeper" / "missing", outside)); + EXPECT_EQ( + firstLinkBelow(tree, tree / "real" / "deeper" / "missing" / "file.txt", error, &verified), + tree / "real" / "deeper" / "missing" + ) << "a component created later is inspected"; + } } From b412f435852ce03b9072160db7c153b61a10d367 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:50:21 +0200 Subject: [PATCH 105/131] Tests: The queued local delete can delete whole trees --- backend/test/backend/main.cpp | 1 + .../backend/test_local_delete_operation.hpp | 264 ++++++++++++++++++ backend/test/backend/test_operation_queue.hpp | 85 ------ 3 files changed, 265 insertions(+), 85 deletions(-) create mode 100644 backend/test/backend/test_local_delete_operation.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index 42a160e4..0cc8568d 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -5,6 +5,7 @@ #include "test_upload_operation.hpp" #include "test_scan_operation.hpp" #include "test_local_delete_guard.hpp" +#include "test_local_delete_operation.hpp" #include "test_remote_delete_guard.hpp" #include "test_local_scan_operation.hpp" #include "test_bulk_download_operation.hpp" diff --git a/backend/test/backend/test_local_delete_operation.hpp b/backend/test/backend/test_local_delete_operation.hpp new file mode 100644 index 00000000..4e5701b1 --- /dev/null +++ b/backend/test/backend/test_local_delete_operation.hpp @@ -0,0 +1,264 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +# include +#endif + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class LocalDeleteOperationTests : public ::testing::Test + { + protected: + struct Progress + { + std::string item; + std::uint64_t current; + std::uint64_t total; + }; + + static Paths::LocalPath local(std::filesystem::path const& path) + { + return Paths::LocalPath::make(path).value(); + } + + LocalDeleteOperation::LocalDeleteOperationOptions options(std::vector targets, bool recursive) + { + return { + .itemsRemovedProgress = + [this](std::string const& item, std::uint64_t current, std::uint64_t total) + { + progress_.push_back({item, current, total}); + }, + .scanProgress = + [this](std::uint64_t, std::uint64_t entries) + { + scannedEntries_ = entries; + }, + .targets = std::move(targets), + .recursive = recursive, + }; + } + + /** + * @brief Works @p operation until it ends, at most @p maximumSteps times. + */ + static std::expected + workToEnd(LocalDeleteOperation& operation, int maximumSteps = 1000) + { + for (int step = 0; step < maximumSteps; ++step) + { + auto result = operation.work(); + if (!result || *result == Operation::WorkStatus::Complete) + return result; + } + return Operation::WorkStatus::MoreWork; + } + + void makeFiles(std::filesystem::path const& directory, int count) + { + std::filesystem::create_directories(directory); + for (int index = 0; index < count; ++index) + std::ofstream{directory / ("file" + std::to_string(index) + ".txt")} << index; + } + + static bool makeDirectoryLink(std::filesystem::path const& link, std::filesystem::path const& target) + { + std::error_code error; + std::filesystem::create_directory_symlink(target, link, error); + return !error; + } + + Utility::TemporaryDirectory root_{temporaryRoot, true}; + std::vector progress_{}; + std::uint64_t scannedEntries_{0}; + }; + + TEST_F(LocalDeleteOperationTests, CommonParentIsTheDeepestSharedDirectory) + { + const auto path = [](std::string const& text) + { + return Paths::LocalPath::fromUtf8(text, Paths::LocalPlatform::Posix).value(); + }; + const std::vector siblings{path("/a/b/c"), path("/a/b/d/e")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(siblings), path("/a/b")); + const std::vector single{path("/a/b")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(single), path("/a")); + const std::vector nested{path("/a/b/c"), path("/a/b")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(nested), path("/a")); + const std::vector apart{path("/a/x"), path("/b/y")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(apart), path("/")); + const std::vector root{path("/")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(root), path("/")); + EXPECT_FALSE(LocalDeleteOperation::commonParentOf({})); + } + + TEST_F(LocalDeleteOperationTests, ALinkTargetIsRemovedAsTheLinkOnly) + { + const auto target = root_.path() / "target"; + makeFiles(target, 2); + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + std::filesystem::create_symlink(target / "file0.txt", root_.path() / "fileLink"); + + for (bool const recursive : {false, true}) + { + if (recursive) + { + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + std::filesystem::create_symlink(target / "file0.txt", root_.path() / "fileLink"); + } + LocalDeleteOperation operation{ + options({local(root_.path() / "link"), local(root_.path() / "fileLink")}, recursive) + }; + const auto result = workToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(*result, Operation::WorkStatus::Complete); + EXPECT_TRUE(operation.getFailed().empty()); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::symlink_status(root_.path() / "link"))); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::symlink_status(root_.path() / "fileLink"))); + EXPECT_TRUE(std::filesystem::exists(target / "file0.txt")) << "recursive: " << recursive; + EXPECT_TRUE(std::filesystem::exists(target / "file1.txt")) << "recursive: " << recursive; + } + } + + TEST_F(LocalDeleteOperationTests, AnItemAlreadyGoneCountsAsDeleted) + { + std::ofstream{root_.path() / "present.txt"} << "present"; + LocalDeleteOperation operation{ + options({local(root_.path() / "gone.txt"), local(root_.path() / "present.txt")}, false) + }; + + const auto result = workToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_TRUE(operation.getFailed().empty()); + EXPECT_EQ(operation.state(), SharedData::OperationState::Completed); + EXPECT_FALSE(std::filesystem::exists(root_.path() / "present.txt")); + ASSERT_FALSE(progress_.empty()); + EXPECT_EQ(progress_.back().current, 2u); + EXPECT_EQ(progress_.back().total, 2u); + } + + TEST_F(LocalDeleteOperationTests, EveryTargetFailingFailsTheOperation) + { + makeFiles(root_.path() / "first", 1); + makeFiles(root_.path() / "second", 1); + LocalDeleteOperation operation{options({local(root_.path() / "first"), local(root_.path() / "second")}, false)}; + + const auto result = workToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::DeleteFailed); + EXPECT_EQ(operation.state(), SharedData::OperationState::Failed); + ASSERT_EQ(operation.getFailed().size(), 2u); + for (auto const& [path, error] : operation.getFailed()) + EXPECT_EQ(error.extraInfo.value_or("").find(path), std::string::npos) << "the path is named once"; + EXPECT_TRUE(std::filesystem::exists(root_.path() / "first" / "file0.txt")); + } + + TEST_F(LocalDeleteOperationTests, CancelStopsBetweenEntries) + { + makeFiles(root_.path() / "many", 20); + auto settings = options({local(root_.path() / "many")}, true); + settings.stepBudget = std::chrono::milliseconds{0}; + LocalDeleteOperation operation{std::move(settings)}; + + // One entry per step: the count walks into the directory and over the first file first. + for (int step = 0; step < 25; ++step) + ASSERT_EQ(operation.work(), Operation::WorkStatus::MoreWork); + ASSERT_TRUE(operation.cancel(true).has_value()); + const auto afterCancel = operation.work(); + ASSERT_FALSE(afterCancel.has_value()); + EXPECT_EQ(afterCancel.error().type, Operation::ErrorType::CannotWorkCanceledOperation); + + std::size_t left = 0; + for ([[maybe_unused]] auto const& entry : std::filesystem::directory_iterator{root_.path() / "many"}) + ++left; + EXPECT_GT(left, 0u); + EXPECT_LT(left, 20u) << "some files went before the cancel"; + } + + TEST_F(LocalDeleteOperationTests, AnUnworkedDeleteHoldsAndThenContinues) + { + makeFiles(root_.path() / "paused" / "inner", 10); + makeFiles(root_.path() / "paused", 10); + auto settings = options({local(root_.path() / "paused")}, true); + settings.stepBudget = std::chrono::milliseconds{0}; + settings.progressInterval = std::chrono::hours{1}; + LocalDeleteOperation operation{std::move(settings)}; + + for (int step = 0; step < 30; ++step) + ASSERT_EQ(operation.work(), Operation::WorkStatus::MoreWork); + EXPECT_EQ(scannedEntries_, 22u) << "two directories and twenty files"; + EXPECT_TRUE(std::filesystem::exists(root_.path() / "paused")) << "a paused queue does not work it further"; + + const auto result = workToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_FALSE(std::filesystem::exists(root_.path() / "paused")); + EXPECT_LE(progress_.size(), 2u) << "progress is throttled, plus the final update"; + ASSERT_FALSE(progress_.empty()); + EXPECT_EQ(progress_.back().current, 22u); + EXPECT_EQ(progress_.back().total, 22u); + } + + TEST_F(LocalDeleteOperationTests, RecursiveDeleteRemovesNestedLinksWithoutFollowing) + { + const auto outside = root_.path() / "outside"; + makeFiles(outside, 3); + const auto tree = root_.path() / "tree"; + makeFiles(tree / "a" / "b", 2); + ASSERT_TRUE(makeDirectoryLink(tree / "a" / "b" / "out", outside)); + ASSERT_TRUE(makeDirectoryLink(tree / "top", outside)); + std::filesystem::create_symlink(outside / "file0.txt", tree / "a" / "fileLink"); + + LocalDeleteOperation operation{options({local(tree)}, true)}; + const auto result = workToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_TRUE(operation.getFailed().empty()); + EXPECT_FALSE(std::filesystem::exists(tree)); + for (int index = 0; index < 3; ++index) + EXPECT_TRUE(std::filesystem::exists(outside / ("file" + std::to_string(index) + ".txt"))); + EXPECT_EQ(scannedEntries_, 8u) << "tree, a, b, two files and three links"; + ASSERT_FALSE(progress_.empty()); + EXPECT_EQ(progress_.back().current, 8u); + } + +#ifndef _WIN32 + TEST_F(LocalDeleteOperationTests, AnUndeletableItemIsListedByItsOwnPath) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root removes entries of read-only directories"; + + const auto tree = root_.path() / "stuck"; + makeFiles(tree / "locked", 1); + makeFiles(tree, 3); + std::filesystem::permissions( + tree / "locked", std::filesystem::perms::owner_write, std::filesystem::perm_options::remove + ); + + LocalDeleteOperation operation{options({local(tree)}, true)}; + const auto result = workToEnd(operation); + std::filesystem::permissions( + tree / "locked", std::filesystem::perms::owner_write, std::filesystem::perm_options::add + ); + + ASSERT_TRUE(result.has_value()) << "a partial success completes"; + ASSERT_EQ(operation.getFailed().size(), 1u); + EXPECT_EQ(operation.getFailed().front().first, Paths::nativeToUtf8(tree / "locked" / "file0.txt")); + EXPECT_FALSE(std::filesystem::exists(tree / "file0.txt")); + EXPECT_TRUE(std::filesystem::exists(tree / "locked" / "file0.txt")); + } +#endif +} diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index d3ea0b9f..a054c1b5 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -863,91 +863,6 @@ namespace Test EXPECT_FALSE(remoteNames(*sftp, "/home/test").contains("file2.txt")) << "a failure does not stop the rest"; } - TEST_F(OperationQueueTests, LocalDeleteIsOneQueuedUnit) - { - const auto root = localDirectory_.path() / "localDelete"; - std::filesystem::create_directories(root / "directory" / "inner"); - std::ofstream{root / "directory" / "inner" / "a.txt"} << "a"; - std::ofstream{root / "file.txt"} << "file"; - std::ofstream{root / "keep.txt"} << "keep"; - auto queue = makeQueue(); - - const auto remove = Ids::generateOperationId(); - ASSERT_TRUE(runOnStrand([&]() { - return queue->addLocalDeleteOperation( - remove, - {Paths::LocalPath::make(root / "directory").value(), - Paths::LocalPath::make(root / "directory" / "inner").value(), - Paths::LocalPath::make(root / "directory" / "inner" / "a.txt").value(), - Paths::LocalPath::make(root / "file.txt").value()} - ); - }).has_value()); - - // Paused queues hold local deletes like any other operation. - EXPECT_TRUE(std::filesystem::exists(root / "file.txt")); - setPaused(false); - workUntilCompletions(1); - ASSERT_EQ(completions().size(), 1u); - EXPECT_EQ(completions().front().operationId, remove); - EXPECT_EQ(completions().front().reason, SharedData::OperationCompletionReason::Completed); - EXPECT_FALSE(std::filesystem::exists(root / "directory")); - EXPECT_FALSE(std::filesystem::exists(root / "file.txt")); - EXPECT_TRUE(std::filesystem::exists(root / "keep.txt")); - } - - TEST_F(OperationQueueTests, LocalDeleteKeepsUnlistedContentAndItsDirectories) - { - const auto root = localDirectory_.path() / "localDeleteUnlisted"; - std::filesystem::create_directories(root / "directory" / "inner"); - std::ofstream{root / "directory" / "listed.txt"} << "listed"; - std::ofstream{root / "directory" / "inner" / "unlisted.txt"} << "unlisted"; - std::ofstream{root / "other.txt"} << "other"; - auto queue = makeQueue(); - - const auto remove = Ids::generateOperationId(); - ASSERT_TRUE(runOnStrand([&]() { - return queue->addLocalDeleteOperation( - remove, - {Paths::LocalPath::make(root / "directory").value(), - Paths::LocalPath::make(root / "directory" / "inner").value(), - Paths::LocalPath::make(root / "directory" / "listed.txt").value(), - Paths::LocalPath::make(root / "other.txt").value()} - ); - }).has_value()); - setPaused(false); - workUntilCompletions(1); - - ASSERT_EQ(completions().size(), 1u); - const auto completion = completions().front(); - EXPECT_EQ(completion.reason, SharedData::OperationCompletionReason::Completed); - EXPECT_EQ(completion.failedEntries.size(), 2u) << "inner and directory are not empty"; - EXPECT_TRUE(std::filesystem::exists(root / "directory" / "inner" / "unlisted.txt")); - EXPECT_FALSE(std::filesystem::exists(root / "directory" / "listed.txt")); - EXPECT_FALSE(std::filesystem::exists(root / "other.txt")) << "a failure does not stop the rest"; - } - - TEST_F(OperationQueueTests, LocalDeleteOfARootOrTheHomeDirectoryIsRefused) - { - auto queue = makeQueue(); - const auto file = localDirectory_.path() / "survivor.txt"; - std::ofstream{file} << "survivor"; - const auto root = Paths::LocalPath::make(std::filesystem::path{file}.root_path()).value(); - const auto homes = localHomeDirectories(); - ASSERT_FALSE(homes.empty()); - - for (auto const& target : {root, homes.front()}) - { - const auto result = runOnStrand([&]() { - return queue->addLocalDeleteOperation( - Ids::generateOperationId(), {Paths::LocalPath::make(file).value(), target} - ); - }); - ASSERT_FALSE(result.has_value()) << target.toUtf8(); - EXPECT_EQ(result.error().type, Operation::ErrorType::InvalidPath); - } - EXPECT_TRUE(std::filesystem::exists(file)); - } - TEST_F(OperationQueueTests, SyncPlanQueuesEveryPartUnderTheFrontendIds) { CREATE_SERVER_AND_JOINER(sftpServer); From 301430d485ed76b4f424abc3a7522bb22ad8f253 Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:52:28 +0200 Subject: [PATCH 106/131] Tests: RpcFilesystem::removeSome removes items without following links --- backend/test/backend/main.cpp | 1 + backend/test/backend/test_local_removal.hpp | 85 +++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 backend/test/backend/test_local_removal.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index 0cc8568d..56e3e56d 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -6,6 +6,7 @@ #include "test_scan_operation.hpp" #include "test_local_delete_guard.hpp" #include "test_local_delete_operation.hpp" +#include "test_local_removal.hpp" #include "test_remote_delete_guard.hpp" #include "test_local_scan_operation.hpp" #include "test_bulk_download_operation.hpp" diff --git a/backend/test/backend/test_local_removal.hpp b/backend/test/backend/test_local_removal.hpp new file mode 100644 index 00000000..60db55df --- /dev/null +++ b/backend/test/backend/test_local_removal.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include +#include + + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class LocalRemovalTests : public ::testing::Test + { + protected: + std::string utf8(std::filesystem::path const& path) const + { + return Paths::nativeToUtf8(path); + } + + std::vector homes() const + { + return {Paths::LocalPath::make(root_.path() / "home").value()}; + } + + Utility::TemporaryDirectory root_{temporaryRoot, true}; + }; + + TEST_F(LocalRemovalTests, ARefusedPathRemovesNothing) + { + std::filesystem::create_directories(root_.path() / "home" / "user"); + std::ofstream{root_.path() / "file.txt"} << "file"; + const std::vector paths{utf8(root_.path() / "file.txt"), utf8(root_.path() / "home")}; + + const auto removal = removeLocalItems(paths, homes()); + ASSERT_FALSE(removal.has_value()); + EXPECT_EQ(removal.error().path, utf8(root_.path() / "home")); + EXPECT_FALSE(removal.error().reason.empty()); + EXPECT_TRUE(std::filesystem::exists(root_.path() / "file.txt")); + + const std::vector relative{"relative/path"}; + EXPECT_FALSE(removeLocalItems(relative, homes()).has_value()); + } + + TEST_F(LocalRemovalTests, AFailureDoesNotStopTheOthers) + { + std::filesystem::create_directories(root_.path() / "full"); + std::ofstream{root_.path() / "full" / "keep.txt"} << "keep"; + std::ofstream{root_.path() / "a.txt"} << "a"; + std::ofstream{root_.path() / "b.txt"} << "b"; + const std::vector paths{ + utf8(root_.path() / "a.txt"), utf8(root_.path() / "full"), utf8(root_.path() / "b.txt") + }; + + const auto removal = removeLocalItems(paths, homes()); + ASSERT_TRUE(removal.has_value()); + EXPECT_EQ(removal->removed, 2u); + ASSERT_EQ(removal->failures.size(), 1u) << "a directory with content is not removed without recursion"; + const auto& failure = removal->failures.front(); + const auto path = utf8(root_.path() / "full"); + EXPECT_TRUE(failure.starts_with(path + ": ")); + EXPECT_EQ(failure.find(path, path.size()), std::string::npos) << "the path is named once"; + EXPECT_FALSE(std::filesystem::exists(root_.path() / "a.txt")); + EXPECT_FALSE(std::filesystem::exists(root_.path() / "b.txt")); + EXPECT_TRUE(std::filesystem::exists(root_.path() / "full" / "keep.txt")); + } + + TEST_F(LocalRemovalTests, ALinkToADirectoryIsRemovedItself) + { + std::filesystem::create_directories(root_.path() / "target"); + std::ofstream{root_.path() / "target" / "keep.txt"} << "keep"; + std::filesystem::create_directory_symlink(root_.path() / "target", root_.path() / "link"); + + const std::vector link{utf8(root_.path() / "link")}; + EXPECT_EQ(removeLocalItems(link, homes()).value().removed, 1u); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::symlink_status(root_.path() / "link"))); + EXPECT_TRUE(std::filesystem::exists(root_.path() / "target" / "keep.txt")); + } +} From 7865d5d2100c7de5e84aeec701b724c87a9617bb Mon Sep 17 00:00:00 2001 From: 5cript Date: Wed, 14 Oct 2026 23:53:10 +0200 Subject: [PATCH 107/131] Tests: Add Utility::LifetimeToken for callbacks that outlive their owner --- utility/test/utility/main.cpp | 1 + utility/test/utility/test_lifetime_token.hpp | 76 ++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 utility/test/utility/test_lifetime_token.hpp diff --git a/utility/test/utility/main.cpp b/utility/test/utility/main.cpp index 6f25c589..d3195ad3 100644 --- a/utility/test/utility/main.cpp +++ b/utility/test/utility/main.cpp @@ -7,6 +7,7 @@ #include "test_fuzzy_search.hpp" #include "test_json_utf8.hpp" #include "test_keyed_diff.hpp" +#include "test_lifetime_token.hpp" #include "test_localized_message.hpp" #include "test_move_to_front.hpp" #include "test_permissions.hpp" diff --git a/utility/test/utility/test_lifetime_token.hpp b/utility/test/utility/test_lifetime_token.hpp new file mode 100644 index 00000000..c5135439 --- /dev/null +++ b/utility/test/utility/test_lifetime_token.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace Utility::Test +{ + TEST(LifetimeTokenTests, AGuardedCallbackRunsOnlyWhileItsTokenExists) + { + int calls = 0; + std::function callback; + { + const LifetimeToken token; + callback = token.guarded( + [&calls](int amount) + { + calls += amount; + } + ); + callback(2); + EXPECT_EQ(calls, 2); + using Guarded = decltype(token.guarded([](int) {})); + static_assert(std::is_invocable_v); + static_assert(!std::is_invocable_v, "the wrapper keeps the parameter types"); + } + callback(5); + EXPECT_EQ(calls, 2) << "the owner is gone"; + } + + TEST(LifetimeTokenTests, ACopiedOrMovedOwnerHasATokenOfItsOwn) + { + int calls = 0; + auto original = std::make_optional(); + auto callback = original->guarded( + [&calls]() + { + ++calls; + } + ); + const LifetimeToken copy{*original}; + const LifetimeToken moved{std::move(*original)}; + LifetimeToken assigned; + assigned = copy; + callback(); + EXPECT_EQ(calls, 1) << "the original still exists after being moved from"; + + original.reset(); + auto stillCalls = callback; + stillCalls(); + EXPECT_EQ(calls, 1) << "neither the copy, the moved-to nor the assigned token keeps it alive"; + EXPECT_TRUE(copy.watch().lock()); + EXPECT_TRUE(moved.watch().lock()); + } + + TEST(LifetimeTokenTests, GuardedCallablesKeepTheirArguments) + { + const LifetimeToken token; + std::string received; + auto callback = token.guarded( + [&received](std::string text, std::unique_ptr number) + { + received = text + std::to_string(*number); + } + ); + callback(std::string{"value "}, std::make_unique(7)); + EXPECT_EQ(received, "value 7"); + } +} From 63ad8c1bd68f3c95c209ba2b0029a6044e12a8a5 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:15:15 +0200 Subject: [PATCH 108/131] Tests: The sync diff compares each side with a per-pair baseline --- shared_data/test/shared_data/test_diff.hpp | 64 ---------------------- 1 file changed, 64 deletions(-) diff --git a/shared_data/test/shared_data/test_diff.hpp b/shared_data/test/shared_data/test_diff.hpp index d24557cc..62522510 100644 --- a/shared_data/test/shared_data/test_diff.hpp +++ b/shared_data/test/shared_data/test_diff.hpp @@ -796,70 +796,6 @@ namespace SharedData::Sync::Test EXPECT_EQ(row.descendantByteTotal, 0u); } - TEST(SyncDiffTests, WithoutALastSyncTheNewestSideWins) - { - CollectingSink sink; - diffScanTrees( - makeDir("", {makeFile("a.txt", 1, 300)}), - makeDir("", {makeFile("a.txt", 2, 200)}), - {.direction = Direction::Both}, - sink - ); - EXPECT_EQ(sink.uploads.size(), 1u); - EXPECT_TRUE(sink.blocked.empty()); - } - - TEST(SyncDiffTests, BothSidesChangedSinceTheLastSyncIsBlocked) - { - CollectingSink sink; - diffScanTrees( - makeDir("", {makeFile("both.txt", 1, 300), makeFile("local.txt", 1, 300)}), - makeDir("", {makeFile("both.txt", 2, 250), makeFile("local.txt", 2, 50)}), - {.direction = Direction::Both, .lastSyncTime = 100}, - sink - ); - ASSERT_EQ(sink.blocked.size(), 1u); - EXPECT_EQ(sink.blocked.front().relKey, "both.txt"); - EXPECT_EQ(sink.blocked.front().reason, BlockReason::ChangedOnBothSides); - ASSERT_EQ(sink.uploads.size(), 1u) << "only the local side changed, it wins"; - EXPECT_EQ(sink.uploads.front().node.relKey, "local.txt"); - } - - TEST(SyncDiffTests, UploadNeverReplacesWhatChangedOnTheServerSinceTheLastSync) - { - CollectingSink sink; - diffScanTrees( - makeDir("", {makeFile("a.txt", 1, 50), makeFile("b.txt", 1, 300)}), - makeDir("", {makeFile("a.txt", 2, 200), makeFile("b.txt", 2, 50)}), - {.direction = Direction::Upload, .lastSyncTime = 100}, - sink - ); - ASSERT_EQ(sink.blocked.size(), 1u); - EXPECT_EQ(sink.blocked.front().relKey, "a.txt"); - EXPECT_EQ(sink.blocked.front().reason, BlockReason::ChangedOnDestination); - ASSERT_EQ(sink.uploads.size(), 1u); - EXPECT_EQ(sink.uploads.front().node.relKey, "b.txt"); - } - - TEST(SyncDiffTests, DeleteNeverRemovesWhatAppearedSinceTheLastSync) - { - CollectingSink sink; - diffScanTrees( - makeDir("", {}), - makeDir( - "", - {makeFile("old.txt", 1, 50), - makeFile("new.txt", 1, 200), - makeDir("dir", {makeFile("old", 1, 50), makeFile("fresh", 1, 200)})} - ), - {.direction = Direction::Upload, .actionDelete = true, .lastSyncTime = 100}, - sink - ); - ASSERT_EQ(sink.deletes.size(), 1u); - EXPECT_EQ(sink.deletes.front().node.relKey, "old.txt"); - EXPECT_EQ(sink.blocked.size(), 2u) << "new.txt and the directory holding a new file"; - } - TEST(SyncDiffTests, NamesDifferingOnlyInCaseAcrossSidesAreBlocked) { // On a case insensitive server "Foo" replaces "foo", the delete of "foo" would then remove the upload. From 748dea8bb57c1483e43e951f93b031c8649e6da4 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:23:38 +0200 Subject: [PATCH 109/131] Tests: A prescanned bulk works below its request's roots --- backend/test/backend/test_operation_queue.hpp | 147 +++++++++++++++++- paths/test/paths/test_remote_path.hpp | 18 +++ .../test_bulk_transfer_request.hpp | 14 +- 3 files changed, 175 insertions(+), 4 deletions(-) diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index a054c1b5..6e4cdf19 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -916,8 +916,8 @@ namespace Test const auto result = runOnStrand([&]() { return queue->executeSyncPlan(*sftp, session, std::move(plan), request); }); - EXPECT_EQ(result.uploads, (std::vector{"up.txt"})); - EXPECT_EQ(result.downloads, (std::vector{"down.txt"})); + EXPECT_EQ(result.uploads, (std::vector{"up.txt", "emptyA/emptyB"})); + EXPECT_EQ(result.downloads, (std::vector{"down.txt", "newLocal/dir"})); EXPECT_EQ(result.remoteDeletes, (std::vector{"gone.txt"})); EXPECT_EQ(result.localDeletes, (std::vector{"localGone.txt"})); EXPECT_EQ(result.remoteDeleteSources, (std::vector{"/home/test/syncRemote/gone.txt"})); @@ -925,7 +925,6 @@ namespace Test result.localDeleteSources, (std::vector{Paths::LocalPath::make(localRoot / "localGone.txt").value().toUtf8()}) ); - EXPECT_EQ(result.createdDirectories, (std::vector{"emptyA/emptyB", "newLocal/dir"})); EXPECT_EQ(result.failed, (std::vector{"CON"})); setPaused(false); @@ -952,6 +951,148 @@ namespace Test EXPECT_FALSE(std::filesystem::exists(localRoot / "localGone.txt")); } + TEST_F(OperationQueueTests, SyncTransfersFilesOfSiblingDirectoriesBelowTheRoots) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/siblings"}})); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/siblings/src"}})); + ASSERT_TRUE( + serverStartResult->command("createFile", {{"path", "/home/test/siblings/src/a.cpp"}, {"content", "a"}}) + ); + ASSERT_TRUE( + serverStartResult->command("createFile", {{"path", "/home/test/siblings/README.md"}, {"content", "readme"}}) + ); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "siblings"; + std::filesystem::create_directories(localRoot / "lib"); + std::ofstream{localRoot / "lib" / "b.cpp"} << "b"; + std::ofstream{localRoot / "NOTES.md"} << "notes"; + + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home/test/siblings"), + }, + context.get_executor() + }; + using SharedData::Sync::Action; + const auto entry = [&session](std::string relKey, Action action, std::uint64_t size) + { + auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); + return SyncSession::ResolvedPlanEntry{ + .entry = {.relKey = std::move(relKey), .action = action, .sizeBytes = size}, + .paths = std::move(paths), + }; + }; + // The first file of each direction lies in a subdirectory, its parent is not the root of the others. + SyncSession::ResolvedPlan plan{ + .entries = { + entry("src/a.cpp", Action::Download, 1), + entry("README.md", Action::Download, 6), + entry("lib/b.cpp", Action::Upload, 1), + entry("NOTES.md", Action::Upload, 5), + }, + }; + const SharedData::Sync::SyncExecutionRequest request{ + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + runOnStrand( + [&]() + { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + } + ); + + setPaused(false); + workUntilCompletions(2); + for (auto const& completed : completions()) + { + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Completed) + << completed.operationId.value(); + EXPECT_TRUE(completed.failedEntries.empty()) << completed.operationId.value(); + } + EXPECT_TRUE(std::filesystem::exists(localRoot / "src" / "a.cpp")); + EXPECT_TRUE(std::filesystem::exists(localRoot / "README.md")); + EXPECT_EQ(remoteNames(*sftp, "/home/test/siblings/lib"), (std::set{"b.cpp"})); + EXPECT_TRUE(remoteNames(*sftp, "/home/test/siblings").contains("NOTES.md")); + } + +#ifndef _WIN32 + TEST_F(OperationQueueTests, SyncCreatesEmptyDirectoriesInItsBulksButNeverThroughALink) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/emptyDirs"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "emptyDirs"; + const auto outside = localDirectory_.path() / "emptyDirsOutside"; + std::filesystem::create_directories(localRoot); + std::filesystem::create_directories(outside); + std::filesystem::create_directory_symlink(outside, localRoot / "linked"); + + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home/test/emptyDirs"), + }, + context.get_executor() + }; + using SharedData::Sync::Action; + const auto entry = [&session](std::string relKey, Action action) + { + auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); + return SyncSession::ResolvedPlanEntry{ + .entry = {.relKey = std::move(relKey), .action = action, .isDirectory = true}, + .paths = std::move(paths), + }; + }; + SyncSession::ResolvedPlan plan{ + .entries = { + entry("linked/inside", Action::Download), + entry("fresh/deep", Action::Download), + entry("up/deep", Action::Upload), + }, + }; + const SharedData::Sync::SyncExecutionRequest request{ + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + const auto result = runOnStrand( + [&]() + { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + } + ); + EXPECT_EQ(result.downloads, (std::vector{"linked/inside", "fresh/deep"})); + EXPECT_EQ(result.uploads, (std::vector{"up/deep"})); + + setPaused(false); + workUntilCompletions(2); + for (auto const& completed : completions()) + { + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Completed) + << completed.operationId.value(); + if (completed.operationId == request.downloadId) + EXPECT_EQ(completed.failedEntries.size(), 1u); + else + EXPECT_TRUE(completed.failedEntries.empty()); + } + EXPECT_FALSE(std::filesystem::exists(outside / "inside")); + EXPECT_TRUE(std::filesystem::is_directory(localRoot / "fresh" / "deep")); + EXPECT_EQ(remoteNames(*sftp, "/home/test/emptyDirs/up"), (std::set{"deep"})); + } +#endif + TEST_F(OperationQueueTests, SyncTransfersNonAsciiNames) { CREATE_SERVER_AND_JOINER(sftpServer); diff --git a/paths/test/paths/test_remote_path.hpp b/paths/test/paths/test_remote_path.hpp index 542cc86f..7f92e6d4 100644 --- a/paths/test/paths/test_remote_path.hpp +++ b/paths/test/paths/test_remote_path.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -127,4 +128,21 @@ namespace Paths::Test EXPECT_FALSE(unset.isWithin(RemotePath::root())); EXPECT_FALSE(RemotePath::root().isWithin(unset)); } + + TEST(RemotePathTests, CommonParentHoldsEveryPathBelowIt) + { + const auto path = [](std::string_view text) + { + return RemotePath::make(text).value(); + }; + const std::vector siblings{path("/r/src/a.cpp"), path("/r/README.md")}; + EXPECT_EQ(commonParentOf(std::span{siblings}), path("/r")); + const std::vector nested{path("/r/a/b/c"), path("/r/a/d")}; + EXPECT_EQ(commonParentOf(std::span{nested}), path("/r/a")); + const std::vector single{path("/r/x")}; + EXPECT_EQ(commonParentOf(std::span{single}), path("/r")); + const std::vector root{RemotePath::root()}; + EXPECT_EQ(commonParentOf(std::span{root}), RemotePath::root()); + EXPECT_FALSE(commonParentOf(std::span{})); + } } diff --git a/shared_data/test/shared_data/test_bulk_transfer_request.hpp b/shared_data/test/shared_data/test_bulk_transfer_request.hpp index 77deef0a..35999add 100644 --- a/shared_data/test/shared_data/test_bulk_transfer_request.hpp +++ b/shared_data/test/shared_data/test_bulk_transfer_request.hpp @@ -36,18 +36,27 @@ namespace SharedData::Test { {.source = local("a.txt"), .destination = remote("/srv/a\\b"), .sizeBytes = 3}, {.source = local("dir"), .destination = remote("/srv/dir"), .isDirectory = true}, + {.source = local("empty"), + .destination = remote("/srv/empty"), + .isDirectory = true, + .withoutContent = true}, }, .allowOverwrite = true, .insertRefresh = true, .mode = OperationMode::PriorityQueued, + .remoteRoot = remote("/srv"), }; const nlohmann::json json = request; const auto read = json.get(); - ASSERT_EQ(read.entries.size(), 2u); + ASSERT_EQ(read.entries.size(), 3u); EXPECT_EQ(read.entries[0].source, local("a.txt")); EXPECT_EQ(read.entries[0].destination.value(), "/srv/a\\b"); EXPECT_EQ(read.entries[0].sizeBytes, 3u); EXPECT_TRUE(read.entries[1].isDirectory); + EXPECT_FALSE(read.entries[1].withoutContent); + EXPECT_TRUE(read.entries[2].withoutContent); + EXPECT_EQ(read.remoteRoot, remote("/srv")); + EXPECT_FALSE(read.localRoot); EXPECT_TRUE(read.allowOverwrite); EXPECT_TRUE(read.insertRefresh); EXPECT_EQ(read.mode, OperationMode::PriorityQueued); @@ -64,6 +73,7 @@ namespace SharedData::Test .mtime = 1700000000u, .mtimeNsec = 5u}, }, + .localRoot = local(""), }; const nlohmann::json json = request; const auto read = json.get(); @@ -73,6 +83,8 @@ namespace SharedData::Test EXPECT_EQ(read.entries[0].sizeBytes, 7u); EXPECT_EQ(read.entries[0].mtime, 1700000000u); EXPECT_EQ(read.entries[0].mtimeNsec, 5u); + EXPECT_EQ(read.localRoot, local("")); + EXPECT_FALSE(read.remoteRoot); } TEST(BulkTransferRequestTests, InvalidPathsAreRefusedOnRead) From cf6c5601162e67d001411fa7da269397c58fe538 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:31:05 +0200 Subject: [PATCH 110/131] Tests: Sync keeps a baseline per pair of a saved session --- backend/test/backend/main.cpp | 1 + .../test/backend/test_sync_baseline_store.hpp | 128 ++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 backend/test/backend/test_sync_baseline_store.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index 56e3e56d..54636c9a 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -26,6 +26,7 @@ #include "test_file_change_filter.hpp" #include "test_password_request.hpp" #include "test_sync_session.hpp" +#include "test_sync_baseline_store.hpp" #include "test_transfer_failures.hpp" #include "test_operation_queue.hpp" #include "test_existence_check.hpp" diff --git a/backend/test/backend/test_sync_baseline_store.hpp b/backend/test/backend/test_sync_baseline_store.hpp new file mode 100644 index 00000000..46ff76e8 --- /dev/null +++ b/backend/test/backend/test_sync_baseline_store.hpp @@ -0,0 +1,128 @@ +#pragma once + +#include +#include + +#include + +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class SyncBaselineStoreTests : public ::testing::Test + { + protected: + static SharedData::Sync::SyncRecord record(std::uint64_t size) + { + const SharedData::Sync::RecordedSide side{.type = SharedData::FileType::Regular, .size = size, .mtime = 7}; + return {.local = side, .remote = side}; + } + + Utility::TemporaryDirectory directory_{temporaryRoot, true}; + SyncBaselineStore store_{directory_.path() / "baselines"}; + std::string pairKey_{SyncBaselineStore::pairKey( + "saved", + Paths::LocalPath::fromUtf8("/local/root", Paths::LocalPlatform::Posix).value(), + Paths::RemotePath::make("/remote/root").value() + )}; + }; + + TEST_F(SyncBaselineStoreTests, APairWithoutFileHasNoBaseline) + { + EXPECT_FALSE(store_.load(pairKey_)); + } + + TEST_F(SyncBaselineStoreTests, UpdatesAddToWhatIsStored) + { + ASSERT_TRUE(store_.update( + pairKey_, + [](SharedData::Sync::SyncBaseline& baseline) + { + baseline.records.insert_or_assign("a.txt", record(1)); + } + )); + ASSERT_TRUE( + SyncBaselineStore{directory_.path() / "baselines"}.update( + pairKey_, + [](SharedData::Sync::SyncBaseline& baseline) + { + EXPECT_TRUE(baseline.records.contains("a.txt")); + baseline.records.insert_or_assign("b.txt", record(2)); + } + ) + ); + const auto loaded = store_.load(pairKey_); + ASSERT_TRUE(loaded); + EXPECT_EQ(loaded->records.size(), 2u); + EXPECT_EQ(loaded->records.at("b.txt"), record(2)); + } + + TEST_F(SyncBaselineStoreTests, AnEmptyUpdateStillGivesThePairAHistory) + { + ASSERT_TRUE(store_.update(pairKey_, [](SharedData::Sync::SyncBaseline&) {})); + const auto loaded = store_.load(pairKey_); + ASSERT_TRUE(loaded); + EXPECT_TRUE(loaded->records.empty()); + } + + TEST_F(SyncBaselineStoreTests, ADamagedFileIsAHistoryWithoutRecords) + { + std::filesystem::create_directories(directory_.path() / "baselines"); + std::ofstream{SyncBaselineStore::fileOf(directory_.path() / "baselines", pairKey_)} << "{not json"; + const auto loaded = store_.load(pairKey_); + ASSERT_TRUE(loaded) << "a pair whose records are lost protects every delete"; + EXPECT_TRUE(loaded->records.empty()); + } + + TEST_F(SyncBaselineStoreTests, AFileOfAnotherPairIsNeitherReadNorReplaced) + { + const auto otherKey = std::string{"other"}; + ASSERT_TRUE(store_.update( + otherKey, + [](SharedData::Sync::SyncBaseline& baseline) + { + baseline.records.insert_or_assign("other.txt", record(3)); + } + )); + // Collide on purpose by moving the other pair's file to this pair's name. + std::filesystem::rename( + SyncBaselineStore::fileOf(directory_.path() / "baselines", otherKey), + SyncBaselineStore::fileOf(directory_.path() / "baselines", pairKey_) + ); + EXPECT_FALSE(store_.load(pairKey_)); + EXPECT_FALSE(store_.update(pairKey_, [](SharedData::Sync::SyncBaseline&) {})); + } + + TEST_F(SyncBaselineStoreTests, PairKeysNameTheSessionAndBothRoots) + { + EXPECT_EQ(pairKey_, "saved\n/local/root\n/remote/root"); + EXPECT_NE( + SyncBaselineStore::fileOf(directory_.path(), pairKey_), + SyncBaselineStore::fileOf(directory_.path(), "saved\n/local/root\n/remote/other") + ); + } + + TEST_F(SyncBaselineStoreTests, PruningRemovesThePairsOfSessionsThatAreGone) + { + const auto local = Paths::LocalPath::fromUtf8("/local", Paths::LocalPlatform::Posix).value(); + const auto remote = Paths::RemotePath::make("/remote").value(); + const auto kept = SyncBaselineStore::pairKey("kept", local, remote); + const auto gone = SyncBaselineStore::pairKey("gone", local, remote); + ASSERT_TRUE(store_.update(kept, [](SharedData::Sync::SyncBaseline&) {})); + ASSERT_TRUE(store_.update(gone, [](SharedData::Sync::SyncBaseline&) {})); + std::ofstream{directory_.path() / "baselines" / "unrelated.txt"} << "x"; + + EXPECT_EQ(store_.pruneAllBut({"kept", "other"}), 1u); + EXPECT_TRUE(store_.load(kept)); + EXPECT_FALSE(store_.load(gone)); + EXPECT_TRUE(std::filesystem::exists(directory_.path() / "baselines" / "unrelated.txt")); + } + + TEST_F(SyncBaselineStoreTests, PruningWithoutADirectoryRemovesNothing) + { + EXPECT_EQ(SyncBaselineStore{directory_.path() / "none"}.pruneAllBut({}), 0u); + } +} From bbb26bb850486bf7546f0b2045b0c989257e7e2c Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:40:42 +0200 Subject: [PATCH 111/131] Tests: A bulk delete goes on past directories it cannot scan --- backend/test/backend/test_operation_queue.hpp | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 6e4cdf19..2185ea1c 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -2,6 +2,7 @@ #include #include +#include #include "nui_env.hpp" #include "real_server_tests.hpp" @@ -780,6 +781,64 @@ namespace Test EXPECT_EQ(finished.front().error->type, SharedData::OperationErrorType::FileExists); } + TEST_F(OperationQueueTests, BulkDeleteGoesOnPastARootItCannotScan) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/doomed"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/doomed/a.txt"}, {"content", "a"}})); + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto bulk = Ids::generateOperationId(); + const SharedData::BulkDeleteRequest request{ + .entries = { + {.target = remote("/home/test/Documents"), .isDirectory = true}, + {.target = remote("/home/test/doomed"), .isDirectory = true}, + {.target = remote("/home/test/file2.txt")}, + }, + }; + EXPECT_EQ( + runOnStrand( + [&]() + { + return queue->addBulkDeleteOperation(*sftp, request, bulk); + } + ), + 3u + ); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + ASSERT_EQ(finished.front().failedEntries.size(), 1u); + EXPECT_EQ(finished.front().failedEntries.front().first, "/home/test/Documents"); + const auto home = remoteNames(*sftp, "/home/test"); + EXPECT_TRUE(home.contains("Documents")); + EXPECT_FALSE(home.contains("doomed")); + EXPECT_FALSE(home.contains("file2.txt")); + } + + TEST(SequentialScanProgressTests, CountsOfLaterScansAddToTheFinishedOnes) + { + std::vector> reports; + const auto next = Detail::sequentialScanProgress( + [&reports](std::uint64_t bytes, std::uint64_t index, std::uint64_t scanned) + { + reports.push_back({bytes, index, scanned}); + } + ); + auto first = next(); + auto second = next(); + first(10, 1, 1); + first(30, 2, 2); + second(5, 1, 1); + second(7, 3, 3); + EXPECT_EQ(reports.back(), (std::array{37, 5, 5})); + } + TEST_F(OperationQueueTests, BulkDeleteOfAnAlreadyDeletedItemSucceeds) { // A resumed bulk delete meets the items it deleted before the interruption. From 91e37aa861c2a08e1a3a0a705c9229225b7737a7 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:43:44 +0200 Subject: [PATCH 112/131] Tests: When one scan of a sync fails, the other one is cancelled --- backend/test/backend/test_operation_queue.hpp | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 2185ea1c..24b749ff 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -1576,6 +1576,48 @@ namespace Test ); } + TEST_F(OperationQueueTests, AFailedSyncScanCancelsTheOtherOne) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "syncSibling"; + for (int directory = 0; directory < 50; ++directory) + { + const auto path = localRoot / fmt::format("d{}", directory); + std::filesystem::create_directories(path); + for (int file = 0; file < 20; ++file) + std::ofstream{path / fmt::format("f{}", file)} << "x"; + } + + const auto remoteScan = Ids::generateOperationId(); + const auto localScan = Ids::generateOperationId(); + runOnStrand( + [&]() + { + queue->addSyncScanOperation( + *sftp, + Ids::generateSyncSessionId(), + remoteScan, + localScan, + remote("/home/test/Documents"), + Paths::LocalPath::make(localRoot).value(), + false, + true, + false + ); + } + ); + workUntilCompletions(2); + std::map reasons; + for (auto const& completed : completions()) + reasons.emplace(completed.operationId.value(), completed.reason); + EXPECT_EQ(reasons.at(remoteScan.value()), SharedData::OperationCompletionReason::Failed); + EXPECT_EQ(reasons.at(localScan.value()), SharedData::OperationCompletionReason::Canceled); + } + TEST_F(OperationQueueTests, HostileNamesNeverBecomeLocalPaths) { CREATE_SERVER_AND_JOINER(sftpServer); From 3db6bd2178ee69b6f79af89c6384d50cef3394d8 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:48:28 +0200 Subject: [PATCH 113/131] Tests: Sync matches names through one case and normalization fold --- backend/test/backend/test_sync_session.hpp | 29 ++++++++++++++ shared_data/test/shared_data/test_diff.hpp | 44 ++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/backend/test/backend/test_sync_session.hpp b/backend/test/backend/test_sync_session.hpp index 2edea1ec..e6b33493 100644 --- a/backend/test/backend/test_sync_session.hpp +++ b/backend/test/backend/test_sync_session.hpp @@ -6,9 +6,12 @@ #include #include +#include #include #include +extern std::filesystem::path temporaryRoot; + namespace Test { namespace SyncSessionTestDetail @@ -186,4 +189,30 @@ namespace Test EXPECT_FALSE(session.resolve(Paths::RelativePath::make("dir/CON").value()).has_value()); EXPECT_EQ(session.resolve(Paths::RelativePath{})->remote.value(), "/srv/sync"); } + + TEST(SyncSessionCaseTests, CaseVariantsBlockOnlyTowardsASideThatFoldsCase) + { + using namespace SyncSessionTestDetail; + boost::asio::io_context context{}; + const auto blockedWith = [&](bool localFoldsCase, bool remoteFoldsCase, Direction direction) + { + SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::fromUtf8("/local", Paths::LocalPlatform::Posix).value(), + .remoteRoot = remote("/remote"), + .localFoldsCase = localFoldsCase, + .remoteFoldsCase = remoteFoldsCase, + }, + context.get_executor() + }; + session.setLocalTreeInStrand(directory("", {file("Readme", 1, 200)})); + session.setRemoteTreeInStrand(directory("", {file("readme", 1, 100)})); + return session.recomputeDiffInStrand({.direction = direction}, {}).blockedCount; + }; + EXPECT_EQ(blockedWith(false, false, Direction::Both), 0u); + EXPECT_EQ(blockedWith(true, false, Direction::Upload), 0u); + EXPECT_EQ(blockedWith(true, false, Direction::Download), 2u); + EXPECT_EQ(blockedWith(false, true, Direction::Upload), 2u); + EXPECT_EQ(blockedWith(false, true, Direction::Download), 0u); + } } diff --git a/shared_data/test/shared_data/test_diff.hpp b/shared_data/test/shared_data/test_diff.hpp index 62522510..2ccadcba 100644 --- a/shared_data/test/shared_data/test_diff.hpp +++ b/shared_data/test/shared_data/test_diff.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -814,6 +815,49 @@ namespace SharedData::Sync::Test EXPECT_EQ(blocked.reason, BlockReason::CaseConflict) << blocked.relKey; } + TEST(SyncCaseFoldTests, LettersOfEveryCoveredScriptFoldToOneKey) + { + EXPECT_EQ(caseFoldedName("README.md"), "readme.md"); + EXPECT_EQ(caseFoldedName("\xC3\x84PFEL"), caseFoldedName("\xC3\xA4pfel")) << "Ä and ä"; + EXPECT_EQ(caseFoldedName("ŁÓDŹ"), caseFoldedName("łódź")) << "Latin Extended-A"; + EXPECT_EQ( + caseFoldedName("\xCE\xA3\xCE\x9F\xCE\xA6\xCE\x9F\xCE\xA3"), + caseFoldedName("\xCF\x83\xCE\xBF\xCF\x86\xCE\xBF\xCF\x82") + ) << "ΣΟΦΟΣ and σοφος with a final sigma"; + EXPECT_EQ(caseFoldedName("\xD0\x81\xD0\x96"), caseFoldedName("\xD1\x91\xD0\xB6")) << "ЁЖ and ёж"; + EXPECT_NE(caseFoldedName("a"), caseFoldedName("b")); + EXPECT_EQ(caseFoldedName("caf\xFF"), "caf\xFF") << "text that is not UTF-8 is its own key"; + } + + TEST(SyncCaseFoldTests, ComposedAndDecomposedSpellingsFoldToOneKey) + { + EXPECT_EQ(caseFoldedName("caf\xC3\xA9"), caseFoldedName("cafe\xCC\x81")) << "é as NFC and NFD"; + EXPECT_EQ(caseFoldedName("CAF\xC3\x89"), caseFoldedName("cafe\xCC\x81")) << "É folds to e and the accent"; + EXPECT_EQ(caseFoldedName("\xCE\x90"), caseFoldedName("\xCE\xB9\xCC\x88\xCC\x81")) << "ΐ decomposes twice"; + EXPECT_FALSE(foldingChanges("plain-name_1.txt")); + EXPECT_TRUE(foldingChanges("Name")); + EXPECT_TRUE(foldingChanges("caf\xC3\xA9")); + } + + TEST(SyncDiffTests, NonAsciiNamesDifferingOnlyInCaseOrNormalizationAreBlocked) + { + CollectingSink sink; + diffScanTrees( + makeDir( + "", {makeFile("\xC3\x84pfel.txt", 1, 100), makeFile("cafe\xCC\x81", 1, 100), makeFile("other", 1, 100)} + ), + makeDir("", {makeFile("\xC3\xA4pfel.txt", 2, 100), makeFile("caf\xC3\xA9", 2, 100)}), + {.direction = Direction::Upload, .actionDelete = true, .remoteCaseInsensitive = true}, + sink + ); + ASSERT_EQ(sink.uploads.size(), 1u); + EXPECT_EQ(sink.uploads.front().node.relKey, "other"); + EXPECT_TRUE(sink.deletes.empty()); + EXPECT_EQ(sink.blocked.size(), 4u); + for (auto const& blocked : sink.blocked) + EXPECT_EQ(blocked.reason, BlockReason::CaseConflict) << blocked.relKey; + } + TEST(SyncDiffTests, RemoteCaseVariantsAreBlockedOnlyForACaseFoldingLocalFilesystem) { const auto local = makeDir("", {}); From 0f82b3f80ed14f1f5d0adcaa055ce40ae25a92fe Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:50:30 +0200 Subject: [PATCH 114/131] Tests: The remote existence check lists a directory only for ten children --- backend/test/backend/test_existence_check.hpp | 52 +++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/backend/test/backend/test_existence_check.hpp b/backend/test/backend/test_existence_check.hpp index a0aec4f6..5f6d1247 100644 --- a/backend/test/backend/test_existence_check.hpp +++ b/backend/test/backend/test_existence_check.hpp @@ -131,19 +131,65 @@ namespace Test EXPECT_EQ(*result, expected); } - TEST_F(ExistenceCheckTests, RemoteListingFailureFailsTheCheck) + TEST_F(ExistenceCheckTests, RemoteDirectoryThatCannotBeListedIsCheckedPathByPath) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); + // A drop box: writable, but not listable. ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); - std::vector paths; + std::vector paths{"/home/test/Documents/doc1.txt"}; + std::vector expected{true}; for (int index = 0; index < 12; ++index) + { paths.push_back("/home/test/Documents/new" + std::to_string(index) + ".txt"); + expected.push_back(false); + } const auto result = remotePathsExist(*sftp, paths, timeout); - EXPECT_FALSE(result.has_value()) << "an unknown destination must never count as absent"; + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(*result, expected); + } + + TEST_F(ExistenceCheckTests, RemoteNameListedOnlyInAnotherCaseIsLookedUp) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + // The lookup of the variant fails here, which shows it happens: a listed exact name needs none. + ASSERT_TRUE(serverStartResult->command("denyStat", {{"path", "/home/test/FILE1.txt"}})); + + std::vector paths{"/home/test/file1.txt"}; + for (int index = 0; index < 10; ++index) + paths.push_back("/home/test/new" + std::to_string(index) + ".txt"); + const auto exact = remotePathsExist(*sftp, paths, timeout); + ASSERT_TRUE(exact.has_value()) << exact.error(); + EXPECT_TRUE(exact->front()); + + paths.front() = "/home/test/FILE1.txt"; + EXPECT_FALSE(remotePathsExist(*sftp, paths, timeout).has_value()) + << "the server may fold case, so the variant is looked up instead of taken for absent"; + } + + TEST_F(ExistenceCheckTests, RemoteFewChildrenOfAParentAreStatEvenAmongManyPaths) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + // Listing it would fail, a stat of its one queried child does not. + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Pictures"}})); + + std::vector paths{"/home/test/Pictures/image1.png"}; + std::vector expected{true}; + for (int index = 0; index < 12; ++index) + { + paths.push_back("/home/test/new" + std::to_string(index) + ".txt"); + expected.push_back(false); + } + + const auto result = remotePathsExist(*sftp, paths, timeout); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(*result, expected); } TEST_F(ExistenceCheckTests, RemoteInvalidPathFailsTheCheck) From 2214b3f348bcf4d4cd5144c69f630a19deef804c Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:51:24 +0200 Subject: [PATCH 115/131] Tests: Sync scans exclude the asides of replacing moves --- backend/test/backend/test_operation_queue.hpp | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 24b749ff..8b7e68e2 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -1576,6 +1576,66 @@ namespace Test ); } + TEST_F(OperationQueueTests, SyncScansExcludeTempFilesAndTheirAsides) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto suffix = Persistence::effectiveTempFileSuffix(std::nullopt); + const auto localRoot = localDirectory_.path() / "syncAsides"; + std::filesystem::create_directories(localRoot); + std::ofstream{localRoot / "keep.txt"} << "keep"; + std::ofstream{localRoot / fmt::format("part.txt{}", suffix)} << "part"; + std::ofstream{localRoot / fmt::format("old.txt{}", Persistence::asideFileSuffix(suffix))} << "old"; + + const auto syncSessionId = Ids::generateSyncSessionId(); + runOnStrand( + [&]() + { + queue->addSyncScanOperation( + *sftp, + syncSessionId, + Ids::generateOperationId(), + Ids::generateOperationId(), + remote("/home/test/Music"), + Paths::LocalPath::make(localRoot).value(), + false, + true, + false + ); + } + ); + workUntilCompletions(2); + auto session = queue->syncSession(syncSessionId); + ASSERT_TRUE(session); + std::promise summary; + boost::asio::post( + session->strand(), + [&]() + { + summary.set_value( + session->recomputeDiffInStrand({.direction = SharedData::Sync::Direction::Upload}, {}) + ); + } + ); + const auto result = summary.get_future().get(); + std::set excluded; + for (auto const& blocked : result.blockedSamples) + { + if (blocked.reason == SharedData::Sync::BlockReason::Excluded) + excluded.insert(blocked.relKey); + } + EXPECT_EQ( + excluded, + (std::set{ + fmt::format("part.txt{}", suffix), fmt::format("old.txt{}", Persistence::asideFileSuffix(suffix)) + }) + ); + EXPECT_EQ(result.uploads.itemCount, 1u) << "keep.txt"; + } + TEST_F(OperationQueueTests, AFailedSyncScanCancelsTheOtherOne) { CREATE_SERVER_AND_JOINER(sftpServer); From 8160d5a977c2cf4afd82392a54ee02aaeffdf21e Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:53:20 +0200 Subject: [PATCH 116/131] Tests: A directory upload never creates an unreadable subdirectory empty --- backend/test/backend/test_operation_queue.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 8b7e68e2..3f4d1fb4 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -326,7 +326,8 @@ namespace Test ASSERT_EQ(finished.front().failedEntries.size(), 1u) << "an unreadable directory is never a silent success"; EXPECT_EQ(finished.front().failedEntries.front().first, Paths::nativeToUtf8(localPath / "locked")); EXPECT_EQ(finished.front().failedEntries.front().second.type, SharedData::OperationErrorType::DirectoryUnreadable); - EXPECT_TRUE(remoteNames(*sftp, "/home/test/withLocked").contains("a.txt")); + EXPECT_EQ(remoteNames(*sftp, "/home/test/withLocked"), (std::set{"a.txt"})) + << "the unreadable directory is not created empty"; } TEST_F(OperationQueueTests, ArchiveUploadOfAnUnreadableSubdirectoryFails) From 4b196bfc7803afed19c88f30ae412b0b31e2c614 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 00:54:42 +0200 Subject: [PATCH 117/131] Tests: A dropped link or junction is described as the link itself --- backend/test/backend/main.cpp | 1 + backend/test/backend/test_dropped_entry.hpp | 45 +++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 backend/test/backend/test_dropped_entry.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index 54636c9a..4e4cc085 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -27,6 +27,7 @@ #include "test_password_request.hpp" #include "test_sync_session.hpp" #include "test_sync_baseline_store.hpp" +#include "test_dropped_entry.hpp" #include "test_transfer_failures.hpp" #include "test_operation_queue.hpp" #include "test_existence_check.hpp" diff --git a/backend/test/backend/test_dropped_entry.hpp b/backend/test/backend/test_dropped_entry.hpp new file mode 100644 index 00000000..5db8c017 --- /dev/null +++ b/backend/test/backend/test_dropped_entry.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include + +#include + +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + TEST(DroppedEntryTests, DescribesWhatLiesAtThePathItself) + { + const Utility::TemporaryDirectory directory{temporaryRoot, true}; + std::filesystem::create_directories(directory.path() / "folder"); + std::ofstream{directory.path() / "file.txt"} << "x"; + + const auto file = droppedItemFromPath(directory.path() / "file.txt"); + ASSERT_TRUE(file); + EXPECT_EQ(file->entry.type, SharedData::FileType::Regular); + EXPECT_EQ(file->entry.name, "file.txt"); + const auto folder = droppedItemFromPath(directory.path() / "folder"); + ASSERT_TRUE(folder); + EXPECT_EQ(folder->entry.type, SharedData::FileType::Directory); + + std::error_code linkError{}; + std::filesystem::create_directory_symlink(directory.path() / "folder", directory.path() / "link", linkError); + if (!linkError) + { + const auto link = droppedItemFromPath(directory.path() / "link"); + ASSERT_TRUE(link); + EXPECT_EQ(link->entry.type, SharedData::FileType::Symlink) << "a dropped link is not followed"; + } + } + + TEST(DroppedEntryTests, RefusesWhatCannotBeDescribed) + { + const Utility::TemporaryDirectory directory{temporaryRoot, true}; + EXPECT_FALSE(droppedItemFromPath(directory.path() / "missing")); + EXPECT_FALSE(droppedItemFromPath("relative/path")); + } +} From 8cc9c2d10e18bf0a267c9c35630c7d8469a76b36 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:04:30 +0200 Subject: [PATCH 118/131] Tests: Format the sync, existence check and bulk changes with clang-format --- .../test/shared_data/test_bulk_transfer_request.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shared_data/test/shared_data/test_bulk_transfer_request.hpp b/shared_data/test/shared_data/test_bulk_transfer_request.hpp index 35999add..b39f0882 100644 --- a/shared_data/test/shared_data/test_bulk_transfer_request.hpp +++ b/shared_data/test/shared_data/test_bulk_transfer_request.hpp @@ -68,10 +68,10 @@ namespace SharedData::Test .entries = { {.source = remote("/srv/a.txt"), - .destination = local("a.txt"), - .sizeBytes = 7, - .mtime = 1700000000u, - .mtimeNsec = 5u}, + .destination = local("a.txt"), + .sizeBytes = 7, + .mtime = 1700000000u, + .mtimeNsec = 5u}, }, .localRoot = local(""), }; From 4ea159d2bca893e9b513425515e789a977252ac0 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:16:52 +0200 Subject: [PATCH 119/131] Tests: Operations hold their SFTP session weakly and lock it per work step --- .../test_archive_download_operation.hpp | 51 ++++----- .../backend/test_archive_upload_operation.hpp | 32 +++--- .../backend/test_bulk_download_operation.hpp | 68 ++++++------ .../backend/test_bulk_upload_operation.hpp | 38 +++---- .../test/backend/test_download_operation.hpp | 104 +++++++++++++----- backend/test/backend/test_operation_queue.hpp | 53 ++++++++- backend/test/backend/test_scan_operation.hpp | 81 ++++++++------ .../test/backend/test_transfer_failures.hpp | 32 +++--- .../test/backend/test_upload_operation.hpp | 68 ++++++------ 9 files changed, 319 insertions(+), 208 deletions(-) diff --git a/backend/test/backend/test_archive_download_operation.hpp b/backend/test/backend/test_archive_download_operation.hpp index 63689e3b..53745d9a 100644 --- a/backend/test/backend/test_archive_download_operation.hpp +++ b/backend/test/backend/test_archive_download_operation.hpp @@ -59,7 +59,7 @@ namespace Test static ScanBundle runScan(SecureShell::SftpSession& sftp, std::filesystem::path const& path) { ScanOperation scan{ - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{.remotePath = remote(path.generic_string())}, }; while (true) @@ -83,7 +83,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ArchiveDownloadOperation operation{*sftp, {}}; + ArchiveDownloadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::ArchiveDownload); } @@ -93,11 +93,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = {}, - }}; + } + }; auto result = operation.work(); ASSERT_FALSE(result.has_value()); EXPECT_EQ(result.error().type, ArchiveDownloadOperation::ErrorType::InvalidPath); @@ -109,7 +110,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry("/")}, .localArchivePath = toLocal(archiveDir_.path() / "root.tar"), @@ -128,7 +129,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {}, .localArchivePath = toLocal(archiveDir_.path() / "empty.tar"), @@ -146,7 +147,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "one.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -197,7 +198,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "one.tar.gz"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -220,7 +221,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "one.tar.bz2"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -240,7 +241,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "one.tar.zst"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -260,7 +261,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "pair.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = { @@ -289,7 +290,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "documents.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry(rootPath.generic_string())}, .localArchivePath = toLocal(archivePath), @@ -324,7 +325,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "short.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry(rootPath.generic_string())}, .localArchivePath = toLocal(archivePath), @@ -345,7 +346,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "missing_scan.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry("/home/test/Documents")}, .localArchivePath = toLocal(archivePath), @@ -371,7 +372,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "wrong_key.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry("/home/test/Documents")}, .localArchivePath = toLocal(archivePath), @@ -396,7 +397,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "two_roots.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = { @@ -431,7 +432,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "mixed_roots.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = { @@ -471,7 +472,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "var_tree.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry("/var")}, .localArchivePath = toLocal(archivePath), @@ -497,7 +498,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "large.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/large.txt", 65536u)}, .localArchivePath = toLocal(archivePath), @@ -519,7 +520,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "tmp_check.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -544,7 +545,7 @@ namespace Test } ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -568,7 +569,7 @@ namespace Test } ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -589,7 +590,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archiveDir_.path() / "finished.tar"), @@ -612,7 +613,7 @@ namespace Test std::uint64_t lastMax{0u}; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .progressCallback = [&](std::uint64_t, std::uint64_t max, std::uint64_t current, std::make_signed_t) @@ -640,7 +641,7 @@ namespace Test ASSERT_FALSE(std::filesystem::exists(archivePath.parent_path())); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), diff --git a/backend/test/backend/test_archive_upload_operation.hpp b/backend/test/backend/test_archive_upload_operation.hpp index 6ccdac02..a91fc52b 100644 --- a/backend/test/backend/test_archive_upload_operation.hpp +++ b/backend/test/backend/test_archive_upload_operation.hpp @@ -102,7 +102,7 @@ namespace Test { const auto localPath = pulledDir_.path() / localFilename; DownloadOperation download{ - sftp, + sftp.weak_from_this(), { .remotePath = remote(remotePath.generic_string()), .localPath = toLocal(localPath), @@ -131,7 +131,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ArchiveUploadOperation operation{*sftp, {}}; + ArchiveUploadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::ArchiveUpload); } @@ -141,7 +141,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = {}, @@ -158,7 +158,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {}, .remoteArchivePath = remote("/home/test/nothing.tar"), @@ -176,7 +176,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/single.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -230,7 +230,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/alpha.tar.gz"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -252,7 +252,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/alpha.tar.zst"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -274,7 +274,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/pair.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = { @@ -308,7 +308,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/tree.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path())}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -383,7 +383,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path())}, .remoteArchivePath = remote("/home/test/missing_scan.tar"), @@ -411,7 +411,7 @@ namespace Test auto otherScan = runLocalScan(otherDir.path()); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path())}, .remoteArchivePath = remote("/home/test/wrong_key.tar"), @@ -442,7 +442,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/two_roots_up.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(firstDir.path()), toLocal(secondDir.path())}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -476,7 +476,7 @@ namespace Test const std::filesystem::path topFile = srcDir_.path() / "alpha.txt"; const std::filesystem::path remoteArchive = "/home/test/mixed_up.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { // A regular-file root (no scan needed) alongside a directory // root (which does need a scan) — the op must succeed and @@ -510,7 +510,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/nested_up.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path())}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -536,7 +536,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = remote("/home/test/done.tar"), @@ -560,7 +560,7 @@ namespace Test std::uint64_t lastMax{0u}; ArchiveUploadOperation operation{ - *sftp, + sftp, { .progressCallback = [&](std::uint64_t, std::uint64_t max, std::uint64_t current, std::make_signed_t) diff --git a/backend/test/backend/test_bulk_download_operation.hpp b/backend/test/backend/test_bulk_download_operation.hpp index 2fee4c9e..99f5c1cc 100644 --- a/backend/test/backend/test_bulk_download_operation.hpp +++ b/backend/test/backend/test_bulk_download_operation.hpp @@ -43,7 +43,9 @@ namespace Test ScanBundle runScan(SecureShell::SftpSession& sftp, std::filesystem::path const& remotePath, bool recursive = true) { - ScanOperation scan{sftp, {.remotePath = remote(remotePath.generic_string()), .recursive = recursive}}; + ScanOperation scan{ + sftp.weak_from_this(), {.remotePath = remote(remotePath.generic_string()), .recursive = recursive} + }; for (int iteration = 0; iteration < 500; ++iteration) { auto workResult = scan.work(); @@ -82,14 +84,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {}}; + BulkDownloadOperation operation{sftp, {}}; } TEST_F(BulkDownloadOperationTests, BulkDownloadOperationTypeIsBulkDownload) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {}}; + BulkDownloadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::BulkDownload); } @@ -97,7 +99,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {}}; + BulkDownloadOperation operation{sftp, {}}; EXPECT_FALSE(operation.isBarrier()); } @@ -106,7 +108,7 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(downloadDir_.path() / "empty")}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(downloadDir_.path() / "empty")}}; // A scanned tree always holds its root, an empty one means the scan never delivered. operation.setScanResult({}, 0); @@ -120,7 +122,7 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {}}; + BulkDownloadOperation operation{sftp, {}}; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); EXPECT_EQ(workResult.error().type, SharedData::OperationErrorType::ImplementationError); @@ -135,7 +137,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "bad")} + sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "bad")} }; std::vector entries; @@ -164,7 +166,7 @@ namespace Test const auto localRoot = downloadDir_.path() / "Documents"; BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} + sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -189,7 +191,7 @@ namespace Test ASSERT_FALSE(std::filesystem::exists(localRoot)); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} + sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -229,7 +231,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), std::string{"Document 1 content"}.size()); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -272,7 +274,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = false} + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = false} }; operation.setScanResult(std::move(entries), 0); @@ -293,7 +295,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); const auto localRoot = downloadDir_.path() / "rootless"; - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot)}}; auto scan = runScan(*sftp, "/home/test/Documents"); operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -323,7 +325,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -378,7 +380,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -421,7 +423,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = true} + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = true} }; operation.setScanResult(std::move(entries), 0); @@ -472,7 +474,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -501,7 +503,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -519,7 +521,7 @@ namespace Test int callbackInvocations = 0; BulkDownloadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&callbackInvocations](SharedData::BulkProgress const&) @@ -544,7 +546,7 @@ namespace Test auto scan = runScan(*sftp, "/home/test/Documents"); BulkDownloadOperation operation{ - *sftp, + sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(downloadDir_.path() / "completed")} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -562,7 +564,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "failed")} + sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "failed")} }; std::vector entries; @@ -606,7 +608,7 @@ namespace Test .sizeBytes = std::string{"Document 2 content"}.size(), }); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot)}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -642,7 +644,7 @@ namespace Test .mayOverwrite = true, }); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot)}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -684,7 +686,7 @@ namespace Test .mtimeNsec = 0, }); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot)}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -720,7 +722,7 @@ namespace Test BulkDownloadOperation::BulkDownloadOperationOptions options{.localPath = toLocal(localRoot)}; options.individualOptions.createMissingDirectories = true; - BulkDownloadOperation operation{*sftp, std::move(options)}; + BulkDownloadOperation operation{sftp, std::move(options)}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -758,7 +760,7 @@ namespace Test std::filesystem::remove(localRoot / "doc2.txt"); BulkDownloadOperation::BulkDownloadOperationOptions options{.localPath = toLocal(localRoot)}; options.individualOptions.createMissingDirectories = createMissing; - BulkDownloadOperation operation{*sftp, std::move(options)}; + BulkDownloadOperation operation{sftp, std::move(options)}; operation.setPrescannedFileList(files); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -775,7 +777,7 @@ namespace Test auto scan = runScan(*sftp, "/home/test/Documents"); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(downloadDir_.path() / "cancel")} + sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(downloadDir_.path() / "cancel")} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -817,7 +819,7 @@ namespace Test .sizeBytes = std::string{"Fake file content"}.size(), }); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot), .concurrency = concurrency}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot), .concurrency = concurrency}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -879,7 +881,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .concurrency = 3} + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .concurrency = 3} }; operation.setScanResult(std::move(entries), 0); @@ -923,7 +925,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = false, .concurrency = 2} }; operation.setScanResult(std::move(entries), 0); @@ -969,7 +971,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = true, .concurrency = 2} }; operation.setScanResult(std::move(entries), 0); @@ -1027,7 +1029,7 @@ namespace Test bool monotonic = true; SharedData::BulkProgress last{}; BulkDownloadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&](SharedData::BulkProgress const& update) @@ -1078,7 +1080,7 @@ namespace Test std::size_t maxInFlight = 0; BulkDownloadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&maxInFlight](SharedData::BulkProgress const& update) @@ -1112,7 +1114,7 @@ namespace Test // The scan root carries no permissions, applying "unknown" used to chmod it to 07777. const auto localRoot = downloadDir_.path() / "inherited"; BulkDownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot), diff --git a/backend/test/backend/test_bulk_upload_operation.hpp b/backend/test/backend/test_bulk_upload_operation.hpp index e56fa64e..94f597d5 100644 --- a/backend/test/backend/test_bulk_upload_operation.hpp +++ b/backend/test/backend/test_bulk_upload_operation.hpp @@ -106,14 +106,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {}}; + BulkUploadOperation operation{sftp, {}}; } TEST_F(BulkUploadOperationTests, BulkUploadOperationTypeIsBulkUpload) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {}}; + BulkUploadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::BulkUpload); } @@ -121,7 +121,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {}}; + BulkUploadOperation operation{sftp, {}}; EXPECT_FALSE(operation.isBarrier()); } @@ -131,7 +131,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/empty_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/empty_bulk"), .localPath = toLocal(uploadSrc_.path())} }; // A scanned tree always holds its root, an empty one means the scan never delivered. operation.setScanResult({}, 0); @@ -146,7 +146,7 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {}}; + BulkUploadOperation operation{sftp, {}}; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); EXPECT_EQ(workResult.error().type, SharedData::OperationErrorType::ImplementationError); @@ -158,7 +158,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/bad_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/bad_bulk"), .localPath = toLocal(uploadSrc_.path())} }; std::vector entries; @@ -186,7 +186,7 @@ namespace Test const auto remoteRoot = std::filesystem::path{"/home/test/bulk_uploaded"}; BulkUploadOperation operation{ - *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -231,7 +231,7 @@ namespace Test writeFile(uploadSrc_.path() / "subdir" / "file1.txt", "Bulk upload file one"); BulkUploadOperation operation{ - *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(entries), std::string{"Bulk upload file one"}.size()); @@ -272,7 +272,7 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path()), .failFast = false} @@ -320,7 +320,7 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path()), .failFast = true} @@ -373,7 +373,7 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(entries), 0); @@ -414,7 +414,7 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(entries), 0); @@ -436,7 +436,7 @@ namespace Test int callbackInvocations = 0; BulkUploadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&callbackInvocations](SharedData::BulkProgress const&) @@ -461,7 +461,7 @@ namespace Test auto scan = runLocalScan(uploadSrc_.path()); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/completed_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/completed_bulk"), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -478,7 +478,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/failed_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/failed_bulk"), .localPath = toLocal(uploadSrc_.path())} }; std::vector entries; @@ -506,7 +506,7 @@ namespace Test auto scan = runLocalScan(uploadSrc_.path()); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/cancel_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/cancel_bulk"), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -545,7 +545,7 @@ namespace Test std::size_t maxInFlight = 0; BulkUploadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&maxInFlight](SharedData::BulkProgress const& update) @@ -589,7 +589,7 @@ namespace Test const std::filesystem::path remoteRoot = "/home/test/tree_concurrent"; BulkUploadOperation operation{ - *sftp, + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path()), .concurrency = 3} @@ -628,7 +628,7 @@ namespace Test }); BulkUploadOperation operation{ - *sftp, + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(uploadSrc_.path()), .failFast = false, diff --git a/backend/test/backend/test_download_operation.hpp b/backend/test/backend/test_download_operation.hpp index 0fcb286b..6e81ee16 100644 --- a/backend/test/backend/test_download_operation.hpp +++ b/backend/test/backend/test_download_operation.hpp @@ -41,14 +41,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - DownloadOperation operation{*sftp, {}}; + DownloadOperation operation{sftp, {}}; } TEST_F(DownloadOperationTests, DownloadOperationTypeIsDownload) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - DownloadOperation operation{*sftp, {}}; + DownloadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::Download); } @@ -60,7 +60,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -82,7 +82,7 @@ namespace Test for (auto const& remotePath : {Paths::RemotePath{}, Paths::RemotePath::root()}) { DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remotePath, .localPath = toLocal(localPath), @@ -103,7 +103,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -148,7 +148,7 @@ namespace Test entry.mtimeNsec = expectedMtimeNsec; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -185,7 +185,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -217,7 +217,7 @@ namespace Test } DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -240,7 +240,7 @@ namespace Test // First download { DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -262,7 +262,7 @@ namespace Test // Second download with overwrite enabled { DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -292,7 +292,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/does_not_exist.txt"), .localPath = toLocal(downloadDir_.path() / "does_not_exist.txt"), @@ -310,7 +310,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = {}, @@ -328,7 +328,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "file1.txt"), @@ -357,7 +357,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/does_not_exist.txt"), .localPath = toLocal(downloadDir_.path() / "does_not_exist.txt"), @@ -380,7 +380,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -404,7 +404,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -435,7 +435,7 @@ namespace Test } DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -473,7 +473,7 @@ namespace Test } DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -511,7 +511,7 @@ namespace Test } DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -542,7 +542,7 @@ namespace Test const auto localPath = downloadDir_.path() / "large.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -565,6 +565,52 @@ namespace Test EXPECT_EQ(std::filesystem::file_size(localPath), 65536u); } + TEST_F(DownloadOperationTests, ADownloadWhoseChannelClosesFailsWithChannelClosed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = downloadDir_.path() / "large.txt"; + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + } + }; + const auto firstStep = operation.work(); + ASSERT_TRUE(firstStep.has_value()) << firstStep.error().toString(); + ASSERT_EQ(*firstStep, DownloadOperation::WorkStatus::MoreWork); + + sftp->close(); + const auto afterClose = operation.work(); + ASSERT_FALSE(afterClose.has_value()); + EXPECT_EQ(afterClose.error().type, Operation::ErrorType::ChannelClosed); + EXPECT_EQ(operation.state(), Operation::OperationState::Failed); + EXPECT_FALSE(std::filesystem::exists(localPath)); + } + + TEST_F(DownloadOperationTests, ADownloadOfAGoneSessionFailsWithChannelClosed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(downloadDir_.path() / "file1.txt"), + } + }; + sftp->close(); + sftp.reset(); + + EXPECT_EQ(operation.strand(), nullptr); + const auto result = operation.workInStrand(); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::ChannelClosed); + } + // ---- Symlink handling -------------------------------------------------- // // Remote symlinks must be recreated as local symlinks, not downloaded as files. @@ -601,7 +647,7 @@ namespace Test const auto localPath = downloadDir_.path() / "link_to_file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_file1.txt"), .localPath = toLocal(localPath), @@ -661,7 +707,7 @@ namespace Test const auto localPath = downloadDir_.path() / "link_to_documents"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_documents"), .localPath = toLocal(localPath), @@ -683,7 +729,7 @@ namespace Test const auto localPath = downloadDir_.path() / "skipped_link"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_file1.txt"), .localPath = toLocal(localPath), @@ -716,7 +762,7 @@ namespace Test ASSERT_FALSE(std::filesystem::exists(localPath.parent_path())); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -738,7 +784,7 @@ namespace Test const auto localPath = downloadDir_.path() / "a" / "b" / "c" / "d" / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -763,7 +809,7 @@ namespace Test const auto localPath = downloadDir_.path() / "nonexistent" / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -789,7 +835,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -809,7 +855,7 @@ namespace Test const auto localPath = downloadDir_.path() / "nested" / "deep" / "link_to_file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_file1.txt"), .localPath = toLocal(localPath), @@ -832,7 +878,7 @@ namespace Test const auto localPath = downloadDir_.path() / "dangling_link"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/dangling_link"), .localPath = toLocal(localPath), diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 3f4d1fb4..244dd5e7 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -782,6 +782,54 @@ namespace Test EXPECT_EQ(finished.front().error->type, SharedData::OperationErrorType::FileExists); } + TEST_F(OperationQueueTests, QueuedOperationsOfAClosedChannelFailWithChannelClosed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto download = Ids::generateOperationId(); + const auto rename = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, + download, + toLocal(localDirectory_.path() / "large.txt"), + remote("/home/test/large.txt"), + false, + false, + false, + false + ); + } + ).has_value()); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addRenameOperation( + *sftp, rename, remote("/home/test/file1.txt"), remote("/home/test/moved.txt") + ); + } + ).has_value()); + + sftp->close(); + sftp.reset(); + setPaused(false); + workUntilCompletions(2); + + const auto finished = completions(); + ASSERT_EQ(finished.size(), 2u); + for (auto const& completed : finished) + { + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Failed) << completed.operationId.value(); + ASSERT_TRUE(completed.error.has_value()) << completed.operationId.value(); + EXPECT_EQ(completed.error->type, SharedData::OperationErrorType::ChannelClosed); + } + EXPECT_FALSE(std::filesystem::exists(localDirectory_.path() / "large.txt")); + } + TEST_F(OperationQueueTests, BulkDeleteGoesOnPastARootItCannotScan) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -1504,11 +1552,12 @@ namespace Test for (auto& entries : hostileScans) { DeleteOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = true, - }}; + } + }; operation.setScanResult(std::move(entries), 0); const auto result = operation.work(); diff --git a/backend/test/backend/test_scan_operation.hpp b/backend/test/backend/test_scan_operation.hpp index 749572a7..a92972f8 100644 --- a/backend/test/backend/test_scan_operation.hpp +++ b/backend/test/backend/test_scan_operation.hpp @@ -62,14 +62,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation operation{*sftp, {}}; + ScanOperation operation{sftp, {}}; } TEST_F(ScanOperationTests, ScanOperationTypeIsScan) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation operation{*sftp, {}}; + ScanOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::Scan); } @@ -77,7 +77,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation operation{*sftp, {}}; + ScanOperation operation{sftp, {}}; EXPECT_TRUE(operation.isBarrier()); } @@ -87,11 +87,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -106,11 +107,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -128,11 +130,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test"), .recursive = true, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -151,11 +154,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -186,11 +190,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = true, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -207,15 +212,17 @@ namespace Test int callbackInvocations = 0; ScanOperation operation{ - *sftp, + sftp, { .progressCallback = - [&callbackInvocations](std::uint64_t, std::uint64_t, std::uint64_t) { - ++callbackInvocations; - }, + [&callbackInvocations](std::uint64_t, std::uint64_t, std::uint64_t) + { + ++callbackInvocations; + }, .remotePath = remote("/home/test/Documents"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); EXPECT_GT(callbackInvocations, 0); @@ -227,11 +234,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/does_not_exist_dir"), .recursive = false, - }}; + } + }; std::expected workResult{ScanOperation::WorkStatus::MoreWork}; for (int iteration = 0; iteration < 100; ++iteration) @@ -255,11 +263,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .recursive = false, - }}; + } + }; std::expected workResult{ScanOperation::WorkStatus::MoreWork}; for (int iteration = 0; iteration < 100; ++iteration) @@ -281,11 +290,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -300,11 +310,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/does_not_exist_dir"), .recursive = false, - }}; + } + }; // Drive the state machine until it fails. for (int iteration = 0; iteration < 10; ++iteration) @@ -325,11 +336,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test"), .recursive = true, - }}; + } + }; // Kick into Running state. auto firstWork = operation.work(); @@ -350,12 +362,13 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = false, .ignoreHidden = true, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -370,7 +383,7 @@ namespace Test ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation operation{*sftp, {.remotePath = remote("/home/test"), .buildTree = true}}; + ScanOperation operation{sftp, {.remotePath = remote("/home/test"), .buildTree = true}}; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); const auto tree = operation.ejectScanTree(); @@ -390,7 +403,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); // Transfers and deletes use flat scans, an incomplete listing must fail them. - ScanOperation operation{*sftp, {.remotePath = remote("/home/test")}}; + ScanOperation operation{sftp, {.remotePath = remote("/home/test")}}; std::expected result{}; for (int iteration = 0; iteration < 100 && result.has_value() && result.value() != ScanOperation::WorkStatus::Complete; @@ -406,7 +419,7 @@ namespace Test ASSERT_TRUE(serverStartResult->command("rawName", {{"directory", "/home/test/Documents"}, {"name", ".."}})); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation flat{*sftp, {.remotePath = remote("/home/test/Documents")}}; + ScanOperation flat{sftp, {.remotePath = remote("/home/test/Documents")}}; ASSERT_EQ(runScanToCompletion(flat), ScanOperation::WorkStatus::Complete); const auto rejected = flat.rejectedPaths(); ASSERT_EQ(rejected.size(), 1u); @@ -414,7 +427,7 @@ namespace Test for (auto const& entry : flat.ejectEntries()) EXPECT_NE(entry.name, "x/.."); - ScanOperation tree{*sftp, {.remotePath = remote("/home/test/Documents"), .buildTree = true}}; + ScanOperation tree{sftp, {.remotePath = remote("/home/test/Documents"), .buildTree = true}}; ASSERT_EQ(runScanToCompletion(tree), ScanOperation::WorkStatus::Complete); const auto root = tree.ejectScanTree(); const auto hostile = std::ranges::find_if(root.children, [](auto const& child) { diff --git a/backend/test/backend/test_transfer_failures.hpp b/backend/test/backend/test_transfer_failures.hpp index 3a1a80ae..09375095 100644 --- a/backend/test/backend/test_transfer_failures.hpp +++ b/backend/test/backend/test_transfer_failures.hpp @@ -197,7 +197,7 @@ namespace Test writeLocalFile(localPath, "original content"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -222,7 +222,7 @@ namespace Test writeLocalFile(localPath, "original content"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -247,7 +247,7 @@ namespace Test writeLocalFile(localPath, "original content"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -276,7 +276,7 @@ namespace Test entry.size = 70000u; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -386,7 +386,7 @@ namespace Test writeLocalFile(localPath.generic_string() + ".filepart", alphabet(65536).substr(0, 20000)); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -409,7 +409,7 @@ namespace Test writeLocalFile(localPath.generic_string() + ".filepart", "Fake file content"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -429,7 +429,7 @@ namespace Test const auto localPath = localDirectory_.path() / "followed.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_file1.txt"), .localPath = toLocal(localPath), @@ -453,7 +453,7 @@ namespace Test writeLocalFile(localPath, alphabet(65536)); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -477,7 +477,7 @@ namespace Test writeLocalFile(localPath, "new content"); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -503,7 +503,7 @@ namespace Test writeLocalFile(localPath, content); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/resumed.txt"), .localPath = toLocal(localPath), @@ -529,7 +529,7 @@ namespace Test std::filesystem::create_symlink(outside, localPath.generic_string() + ".filepart"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -555,7 +555,7 @@ namespace Test writeLocalFile(localPath, "new content"); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -584,7 +584,7 @@ namespace Test writeLocalFile(localPath, "new content"); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -610,7 +610,7 @@ namespace Test writeLocalFile(localPath, alphabet(4096)); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(localPath)}, .remoteArchivePath = remote("/home/test/backup.tar"), @@ -636,7 +636,7 @@ namespace Test writeLocalFile(localPath, alphabet(4096)); ArchiveUploadOperation refused{ - *sftp, + sftp, { .localPaths = {toLocal(localPath)}, .remoteArchivePath = remote("/home/test/backup.tar"), @@ -649,7 +649,7 @@ namespace Test EXPECT_EQ(readRemoteFile(*sftp, "/home/test/backup.tar"), "old archive"); ArchiveUploadOperation replacing{ - *sftp, + sftp, { .localPaths = {toLocal(localPath)}, .remoteArchivePath = remote("/home/test/backup.tar"), diff --git a/backend/test/backend/test_upload_operation.hpp b/backend/test/backend/test_upload_operation.hpp index ace9d501..9317b138 100644 --- a/backend/test/backend/test_upload_operation.hpp +++ b/backend/test/backend/test_upload_operation.hpp @@ -33,14 +33,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - UploadOperation operation{*sftp, {}}; + UploadOperation operation{sftp, {}}; } TEST_F(UploadOperationTests, UploadOperationTypeIsUpload) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - UploadOperation operation{*sftp, {}}; + UploadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::Upload); } @@ -50,7 +50,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -79,7 +79,7 @@ namespace Test for (auto const& remotePath : {Paths::RemotePath{}, Paths::RemotePath::root()}) { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remotePath, .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -109,7 +109,7 @@ namespace Test ASSERT_TRUE(createResult.has_value()); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -139,7 +139,7 @@ namespace Test std::ofstream{temporaryRoot / "windows.txt"} << "text"; UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/windows.txt"), .localPath = toLocal(temporaryRoot / "windows.txt"), @@ -172,7 +172,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -230,7 +230,7 @@ namespace Test // First upload { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -255,7 +255,7 @@ namespace Test // Second upload without overwrite { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -277,7 +277,7 @@ namespace Test // First upload { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -302,7 +302,7 @@ namespace Test // Second upload with overwrite { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -354,7 +354,7 @@ namespace Test // Second upload continuing the partial file { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -431,7 +431,7 @@ namespace Test // Second upload continuing the partial file { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -488,7 +488,7 @@ namespace Test emptyFile.close(); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/emptyfile.txt"), .localPath = toLocal(temporaryRoot / "emptyfile.txt"), @@ -522,7 +522,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -557,7 +557,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -586,7 +586,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "does_not_exist.txt"), @@ -607,7 +607,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/nonexistent_dir/file.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -625,7 +625,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "does_not_exist.txt"), @@ -643,7 +643,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(programDirectory), @@ -668,7 +668,7 @@ namespace Test } UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/largefile.txt"), .localPath = toLocal(temporaryRoot / "largefile.txt"), @@ -751,7 +751,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/uploaded_sym_to_file"), .localPath = toLocal(linkPath), @@ -788,7 +788,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/uploaded_sym_to_dir"), .localPath = toLocal(linkPath), @@ -820,7 +820,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/uploaded_dangling"), .localPath = toLocal(linkPath), @@ -851,7 +851,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/uploaded_sym_skip"), .localPath = toLocal(linkPath), @@ -880,7 +880,7 @@ namespace Test // First put a regular file in place. { UploadOperation first{ - *sftp, + sftp, { .remotePath = remote("/home/test/sym_overwrite_target"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -896,7 +896,7 @@ namespace Test temporaryRoot / "testfile.txt" ); UploadOperation symOp{ - *sftp, + sftp, { .remotePath = remote("/home/test/sym_overwrite_target"), .localPath = toLocal(linkPath), @@ -928,7 +928,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/sym_prepare_check"), .localPath = toLocal(linkPath), @@ -961,7 +961,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/newly_created/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -990,7 +990,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/a/b/c/d/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -1023,7 +1023,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -1043,7 +1043,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/still_missing_dir/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -1072,7 +1072,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/missing/for/symlink/uploaded_link"), .localPath = toLocal(linkPath), @@ -1119,7 +1119,7 @@ namespace Test // Second upload attempting to continue the partial file { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), From 94b2388b69013f6da22b10eef58309468d77bb4d Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:25:50 +0200 Subject: [PATCH 120/131] Tests: Every SFTP request on a channel is answered --- backend/test/backend/main.cpp | 1 + .../test/backend/test_rpc_path_parameters.hpp | 102 ++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 backend/test/backend/test_rpc_path_parameters.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index 4e4cc085..c81ecc2e 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -33,6 +33,7 @@ #include "test_existence_check.hpp" #include "test_opener_risk.hpp" #include "test_rpc_helper.hpp" +#include "test_rpc_path_parameters.hpp" #include diff --git a/backend/test/backend/test_rpc_path_parameters.hpp b/backend/test/backend/test_rpc_path_parameters.hpp new file mode 100644 index 00000000..45efb0f2 --- /dev/null +++ b/backend/test/backend/test_rpc_path_parameters.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include +#include "nui_env.hpp" + +#include +#include + +#include +#include +#include + +namespace Test +{ + TEST(RpcPathParametersTests, AbsoluteRemotePathsParse) + { + const auto parsed = RpcHelper::parseRemotePath("/home/test/a.txt"); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->value(), "/home/test/a.txt"); + } + + TEST(RpcPathParametersTests, AnInvalidRemotePathIsNamedInTheMessage) + { + for (auto const& invalid : {std::string{"relative/a.txt"}, std::string{}, std::string{"/a/../b"}}) + { + const auto parsed = RpcHelper::parseRemotePath(invalid); + ASSERT_FALSE(parsed.has_value()) << invalid; + EXPECT_TRUE(parsed.error().starts_with("Invalid remote path: ")) << parsed.error(); + } + } + + TEST(RpcPathParametersTests, RemotePathsStopAtTheFirstInvalidOne) + { + const auto valid = RpcHelper::parseRemotePaths({"/a", "/b/c"}); + ASSERT_TRUE(valid.has_value()) << valid.error(); + ASSERT_EQ(valid->size(), 2u); + EXPECT_EQ((*valid)[1].value(), "/b/c"); + + const auto invalid = RpcHelper::parseRemotePaths({"/a", "b", "/c"}); + ASSERT_FALSE(invalid.has_value()); + EXPECT_TRUE(invalid.error().starts_with("Invalid remote path: ")) << invalid.error(); + + const auto none = RpcHelper::parseRemotePaths({}); + ASSERT_TRUE(none.has_value()); + EXPECT_TRUE(none->empty()); + } + + TEST(RpcPathParametersTests, LocalPathsMustBeAbsolute) + { +#ifdef _WIN32 + const std::string absolute = "C:/Users/test/a.txt"; +#else + const std::string absolute = "/home/test/a.txt"; +#endif + const auto parsed = RpcHelper::parseLocalPath(absolute); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->toUtf8(), absolute); + + const auto relative = RpcHelper::parseLocalPath("relative/a.txt"); + ASSERT_FALSE(relative.has_value()); + EXPECT_TRUE(relative.error().starts_with("Invalid local path: ")) << relative.error(); + + const auto all = RpcHelper::parseLocalPaths({absolute, "relative"}); + ASSERT_FALSE(all.has_value()); + EXPECT_TRUE(all.error().starts_with("Invalid local path: ")) << all.error(); + } + + TEST(RpcPathParametersTests, ARequestThatDoesNotParseNamesWhatItIs) + { + const auto invalidTarget = + nlohmann::json{{"entries", {{{"target", "relative"}, {"isDirectory", false}, {"sizeBytes", 0}}}}}; + const auto parsed = RpcHelper::parseRequest(invalidTarget, "bulk delete"); + ASSERT_FALSE(parsed.has_value()); + EXPECT_TRUE(parsed.error().starts_with("Invalid bulk delete: ")) << parsed.error(); + + const auto notJson = RpcHelper::parseRequest>( + nlohmann::json::parse("{not json", nullptr, false), "archive roots" + ); + ASSERT_FALSE(notJson.has_value()); + EXPECT_TRUE(notJson.error().starts_with("Invalid archive roots: ")) << notJson.error(); + } + + TEST(RpcPathParametersTests, AMovedReplyCountsAsAnswered) + { + if (!NuiEnv::instance().available()) + GTEST_SKIP() << "No display available"; + auto& environment = NuiEnv::instance(); + RpcHelper::RpcOnce original{*environment.window, *environment.hub, "RpcPathParametersTests::reply"}; + EXPECT_FALSE(original.answered()); + + RpcHelper::RpcOnce moved{std::move(original)}; + EXPECT_TRUE(original.answered()); + EXPECT_FALSE(moved.answered()); + + RpcHelper::RpcOnce assigned{*environment.window, *environment.hub, "RpcPathParametersTests::other"}; + assigned = std::move(moved); + EXPECT_TRUE(moved.answered()); + EXPECT_FALSE(assigned.answered()); + } +} From 47db53d317a5f847c336a0e8cb79f40307aed0e6 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:33:20 +0200 Subject: [PATCH 121/131] Tests: A bulk resumed after a reconnect leaves out what it finished --- backend/test/backend/test_operation_queue.hpp | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 244dd5e7..2ae6923b 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -527,6 +527,227 @@ namespace Test EXPECT_TRUE(names.contains("adopted_b.txt")); } + TEST_F(OperationQueueTests, AResumedBulkUploadLeavesOutTheFilesItFinished) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "partly"; + std::filesystem::create_directories(localPath); + SharedData::BulkUploadRequest request{}; + for (auto const* name : {"partly_a.txt", "partly_b.txt", "partly_c.txt"}) + { + std::ofstream{localPath / name} << name; + request.entries.push_back( + {.source = toLocal(localPath / name), + .destination = remote(fmt::format("/home/test/{}", name)), + .sizeBytes = std::string_view{name}.size()} + ); + } + const auto card = Ids::generateOperationId(); + runOnStrand( + [&]() + { + return queue->addBulkUploadOperation( + *sftp, + request, + [](std::size_t) + { + return Ids::generateOperationId(); + }, + card + ); + } + ); + + for (int step = 0; step < 50 && !remoteNames(*sftp, "/home/test").contains("partly_a.txt"); ++step) + runOnStrand( + [&]() + { + return queue->work(); + } + ); + ASSERT_TRUE(remoteNames(*sftp, "/home/test").contains("partly_a.txt")); + ASSERT_FALSE(remoteNames(*sftp, "/home/test").contains("partly_c.txt")) << "the bulk ran to its end"; + + BulkResumeRegistry registry; + tearDownInto(registry); + auto backup = registry.take(card); + ASSERT_TRUE(backup.has_value()); + auto const& resumed = std::get(backup->request); + ASSERT_FALSE(resumed.entries.empty()); + EXPECT_LT(resumed.entries.size(), 3u); + for (auto const& entry : resumed.entries) + EXPECT_NE(entry.destination.value(), "/home/test/partly_a.txt"); + EXPECT_EQ(backup->finished.entries, 3u - resumed.entries.size()) << "the card counts them as done"; + EXPECT_GE(backup->finished.bytes, std::string_view{"partly_a.txt"}.size()); + registry.store(card, *std::move(backup)); + + adoptInto(*sftp, registry, {card}); + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, card); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(finished.front().failedEntries.empty()) << finished.front().failedEntries.front().first; + const auto names = remoteNames(*sftp, "/home/test"); + for (auto const* name : {"partly_a.txt", "partly_b.txt", "partly_c.txt"}) + EXPECT_TRUE(names.contains(name)) << name; + } + + TEST_F(OperationQueueTests, AResumedSyncUploadKeepsWhatTheCompareSawAtItsDestinations) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "compared"; + std::filesystem::create_directories(localPath); + SharedData::BulkUploadRequest request{}; + for (auto const* name : {"compared_a.txt", "compared_b.txt", "compared_c.txt"}) + { + std::ofstream{localPath / name} << name; + request.entries.push_back( + {.source = toLocal(localPath / name), + .destination = remote(fmt::format("/home/test/{}", name)), + .sizeBytes = std::string_view{name}.size(), + .mayOverwrite = true, + .comparedDestination = SharedData::ComparedState{}} + ); + } + const auto card = Ids::generateOperationId(); + runOnStrand( + [&]() + { + return queue->addBulkUploadOperation( + *sftp, + request, + [](std::size_t) + { + return Ids::generateOperationId(); + }, + card + ); + } + ); + for (int step = 0; step < 50 && !remoteNames(*sftp, "/home/test").contains("compared_a.txt"); ++step) + runOnStrand( + [&]() + { + return queue->work(); + } + ); + ASSERT_TRUE(remoteNames(*sftp, "/home/test").contains("compared_a.txt")); + + BulkResumeRegistry registry; + tearDownInto(registry); + auto backup = registry.take(card); + ASSERT_TRUE(backup.has_value()); + auto const& resumed = std::get(backup->request); + ASSERT_FALSE(resumed.entries.empty()); + for (auto const& entry : resumed.entries) + { + EXPECT_EQ(entry.comparedDestination, SharedData::ComparedState{}) << entry.destination.value(); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", entry.destination.value()}, {"content", "appeared meanwhile"}} + )); + } + const auto appeared = resumed.entries.size(); + registry.store(card, *std::move(backup)); + + adoptInto(*sftp, registry, {card}); + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + ASSERT_EQ(finished.front().failedEntries.size(), appeared) << "the resumed entries replace nothing"; + for (auto const& [path, error] : finished.front().failedEntries) + EXPECT_EQ(error.type, Operation::ErrorType::ChangedSinceCompare) << path; + } + + TEST_F(OperationQueueTests, AResumedDirectoryDownloadLeavesOutTheFilesItFinished) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "Documents"; + const auto operationId = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, operationId, toLocal(localPath), remote("/home/test/Documents"), false, false, false, false + ); + } + ).has_value()); + for (int step = 0; step < 100 && localNames(localPath).empty(); ++step) + runOnStrand( + [&]() + { + return queue->work(); + } + ); + ASSERT_EQ(localNames(localPath).size(), 1u); + + BulkResumeRegistry registry; + tearDownInto(registry); + auto backup = registry.take(operationId); + ASSERT_TRUE(backup.has_value()); + auto const& resumed = std::get(backup->request); + ASSERT_EQ(resumed.entries.size(), 1u) << "only the file that is not there yet, as a flat list"; + EXPECT_FALSE(resumed.entries.front().isDirectory); + EXPECT_FALSE( + localNames(localPath).contains(Paths::nativeToUtf8(resumed.entries.front().destination.native().filename())) + ); + ASSERT_TRUE(resumed.localRoot.has_value()); + EXPECT_EQ(resumed.localRoot->native(), localPath); + EXPECT_EQ(backup->finished.entries, 1u); + registry.store(operationId, *std::move(backup)); + + adoptInto(*sftp, registry, {operationId}); + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, operationId); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(finished.front().failedEntries.empty()) << finished.front().failedEntries.front().first; + EXPECT_EQ(localNames(localPath), (std::set{"doc1.txt", "doc2.txt"})); + } + + TEST_F(OperationQueueTests, ABackupOfSeveralDirectoriesIsRefused) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + makeQueue(); + setPaused(false); + + BulkResumeEntry backup{ + .request = SharedData::BulkDownloadRequest{ + .entries = { + {.source = remote("/home/test/Documents"), + .destination = toLocal(localDirectory_.path() / "Documents"), + .isDirectory = true}, + {.source = remote("/home/test/Pictures"), + .destination = toLocal(localDirectory_.path() / "Pictures"), + .isDirectory = true}, + }, + }, + }; + const auto operationId = Ids::generateOperationId(); + runOnStrand( + [&]() + { + queue_->adoptBulkResume(*sftp, operationId, std::move(backup)); + } + ); + EXPECT_TRUE(completedIdsAfterSettling().empty()); + EXPECT_TRUE(localNames(localDirectory_.path()).empty()); + } + TEST_F(OperationQueueTests, QueuedRenameMovesTheFileButNeverReplacesTheDestination) { CREATE_SERVER_AND_JOINER(sftpServer); From 60dad1395a2b710e20fa9c9a5664f01d6c2234a6 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:38:04 +0200 Subject: [PATCH 122/131] Tests: A resumed bulk card shows the whole bulk --- .../backend/test_bulk_upload_operation.hpp | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/backend/test/backend/test_bulk_upload_operation.hpp b/backend/test/backend/test_bulk_upload_operation.hpp index 94f597d5..d46b6718 100644 --- a/backend/test/backend/test_bulk_upload_operation.hpp +++ b/backend/test/backend/test_bulk_upload_operation.hpp @@ -570,6 +570,46 @@ namespace Test EXPECT_EQ(maxInFlight, 3u); } + TEST_F(BulkUploadOperationTests, AResumedBulkCountsWhatAnEarlierConnectionFinished) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + writeFile(uploadSrc_.path() / "rest.txt", "rest"); + std::vector updates; + BulkUploadOperation operation{ + sftp, + { + .overallProgressCallback = + [&updates](SharedData::BulkProgress const& update) + { + updates.push_back(update); + }, + .remotePath = remote("/home/test"), + .localPath = toLocal(uploadSrc_.path()), + .progressEmitInterval = std::chrono::milliseconds{0}, + } + }; + operation.setPrescannedFileList({{ + .source = toLocal(uploadSrc_.path() / "rest.txt"), + .destination = remote("/home/test/rest.txt"), + .sizeBytes = 4, + }}); + operation.countEarlierProgress({.entries = 2, .bytes = 10}); + + ASSERT_EQ(runBulkToCompletion(operation), BulkUploadOperation::WorkStatus::Complete); + ASSERT_FALSE(updates.empty()); + EXPECT_EQ(updates.front().fileCount, 3u); + EXPECT_EQ(updates.front().fileCurrentIndex, 2u); + EXPECT_EQ(updates.front().bytesCurrent, 10u); + EXPECT_EQ(updates.front().bytesTotal, 14u); + EXPECT_EQ(updates.back().fileCurrentIndex, 3u); + EXPECT_EQ(updates.back().bytesCurrent, 14u); + const auto finished = operation.finishedProgress(); + EXPECT_EQ(finished.entries, 3u); + EXPECT_EQ(finished.bytes, 14u); + } + TEST_F(BulkUploadOperationTests, DirectoriesAreCreatedBeforeTheirFilesWithConcurrency) { // Two-level local tree scanned for real, uploaded with more files than slots. From 3fea40402eff5ff62e98fd7c177f4a0dc34f13b7 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:45:37 +0200 Subject: [PATCH 123/131] Tests: Walked trees build each directory's path once --- .../backend/test_bulk_download_operation.hpp | 40 +++++ .../benchmark/shared_data/sync_benchmark.cpp | 2 +- .../test_tree_directory_walker.hpp | 8 +- .../test/utility/test_directory_traversal.hpp | 139 +++++++++++++++--- 4 files changed, 159 insertions(+), 30 deletions(-) diff --git a/backend/test/backend/test_bulk_download_operation.hpp b/backend/test/backend/test_bulk_download_operation.hpp index 99f5c1cc..c8002b67 100644 --- a/backend/test/backend/test_bulk_download_operation.hpp +++ b/backend/test/backend/test_bulk_download_operation.hpp @@ -19,6 +19,46 @@ extern std::filesystem::path temporaryRoot; namespace Test { + TEST(TreeEntryPathsTests, EntriesAreJoinedBelowBothRoots) + { + using SharedData::DirectoryEntry; + const std::vector entries{ + DirectoryEntry{.type = DirectoryEntry::FileType::Directory}, + DirectoryEntry{.name = "sub", .type = DirectoryEntry::FileType::Directory, .parent = 0}, + DirectoryEntry{.name = "file.txt", .type = DirectoryEntry::FileType::Regular, .parent = 1}, + DirectoryEntry{.name = "..", .type = DirectoryEntry::FileType::Regular, .parent = 1}, + DirectoryEntry{.name = "lost.txt", .type = DirectoryEntry::FileType::Regular, .parent = 7}, + }; + const auto remoteRoot = remote("/home/test/tree"); + const auto localRoot = toLocal(temporaryRoot / "tree"); + + const auto root = Detail::treeEntryPaths(entries, 0, remoteRoot, localRoot); + ASSERT_TRUE(root.has_value()) << root.error().toString(); + EXPECT_EQ(root->remote, remoteRoot); + EXPECT_EQ(root->local, localRoot); + + const auto file = Detail::treeEntryPaths(entries, 2, remoteRoot, localRoot); + ASSERT_TRUE(file.has_value()) << file.error().toString(); + EXPECT_EQ(file->remote.value(), "/home/test/tree/sub/file.txt"); + EXPECT_EQ(file->local.native(), (temporaryRoot / "tree" / "sub" / "file.txt")); + + const auto invalid = Detail::treeEntryPaths(entries, 3, remoteRoot, localRoot); + ASSERT_FALSE(invalid.has_value()); + EXPECT_EQ(invalid.error().type, Operation::ErrorType::InvalidName); + + const auto lost = Detail::treeEntryPaths(entries, 4, remoteRoot, localRoot); + ASSERT_FALSE(lost.has_value()); + EXPECT_EQ(lost.error().type, Operation::ErrorType::ImplementationError); + + const auto outside = Detail::treeEntryPaths(entries, 5, remoteRoot, localRoot); + ASSERT_FALSE(outside.has_value()); + EXPECT_EQ(outside.error().type, Operation::ErrorType::ImplementationError); + + const auto noRoot = Detail::treeEntryPaths(entries, 2, Paths::RemotePath{}, localRoot); + ASSERT_FALSE(noRoot.has_value()); + EXPECT_EQ(noRoot.error().type, Operation::ErrorType::ImplementationError); + } + class BulkDownloadOperationTests : public RealServerTests { public: diff --git a/shared_data/benchmark/shared_data/sync_benchmark.cpp b/shared_data/benchmark/shared_data/sync_benchmark.cpp index c9fdfb3f..4c1536e6 100644 --- a/shared_data/benchmark/shared_data/sync_benchmark.cpp +++ b/shared_data/benchmark/shared_data/sync_benchmark.cpp @@ -177,7 +177,7 @@ int main(int argc, char** argv) { return listing.at(relative.empty() ? std::string{"/root"} : "/root/" + relative.value()); }; - TreeDirectoryWalker walker{std::move(scanner)}; + TreeDirectoryWalker walker{std::move(scanner)}; if (!walker.walkAll()) std::abort(); auto tree = std::move(walker).ejectTree(); diff --git a/shared_data/test/shared_data/test_tree_directory_walker.hpp b/shared_data/test/shared_data/test_tree_directory_walker.hpp index da8f6cdb..f14cef19 100644 --- a/shared_data/test/shared_data/test_tree_directory_walker.hpp +++ b/shared_data/test/shared_data/test_tree_directory_walker.hpp @@ -53,12 +53,10 @@ namespace SharedData::Sync::Test }; } - template inline std::expected walkFake(FakeFilesystem& filesystem, bool recursive = true) { auto scanner = makeScanner(filesystem); - TreeDirectoryWalker walker{std::move(scanner), recursive - }; + TreeDirectoryWalker walker{std::move(scanner), recursive}; auto result = walker.walkAll(); if (!result) return std::unexpected(result.error()); @@ -117,7 +115,7 @@ namespace SharedData::Sync::Test filesystem.addFile("/root", ".hidden", 1); filesystem.addFile("/root", "file", 1); - const auto tree = walkFake(filesystem); + const auto tree = walkFake(filesystem); ASSERT_TRUE(tree.has_value()); EXPECT_EQ(childNames(*tree), (std::vector{".hidden", "file"})); } @@ -191,7 +189,7 @@ namespace SharedData::Sync::Test filesystem.addFile("/root/dir", "b", 50); auto scanner = makeScanner(filesystem); - TreeDirectoryWalker walker{std::move(scanner)}; + TreeDirectoryWalker walker{std::move(scanner)}; ASSERT_TRUE(walker.walkAll().has_value()); EXPECT_TRUE(walker.completed()); EXPECT_EQ(walker.totalEntries(), 4u) << "root, a, dir, b"; diff --git a/utility/test/utility/test_directory_traversal.hpp b/utility/test/utility/test_directory_traversal.hpp index 76e666d2..13fb24b4 100644 --- a/utility/test/utility/test_directory_traversal.hpp +++ b/utility/test/utility/test_directory_traversal.hpp @@ -20,7 +20,6 @@ namespace Utility::Test class DirectoryTraversalTests : public ::testing::Test { public: - template auto withWalkerDo(auto&& fn) { auto scan = [this](Paths::RelativePath const& relative) @@ -30,8 +29,7 @@ namespace Utility::Test else return std::unexpected("No Scanner Installed In Test"s); }; - using WalkerType = - DeepDirectoryWalker; + using WalkerType = DeepDirectoryWalker; if (!walker_) walker_ = std::make_unique(std::move(scan)); return fn(static_cast(*walker_)); @@ -347,9 +345,15 @@ namespace Utility::Test const auto walkResult = withWalkerDo([this, &paths](auto& walker) { auto res = walker.walkAll(); auto const& entries = walker.entries(); - std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [this, &walker](auto const& entry) { - return absolutePath(walker.relativePath(entry)); - }); + std::transform( + entries.begin(), + entries.end(), + std::back_inserter(paths), + [this, &walker](auto const& entry) + { + return absolutePath(*walker.relativePath(entry)); + } + ); return res; }); @@ -431,14 +435,23 @@ namespace Utility::Test }; std::vector paths{}; - const auto walkResult = withWalkerDo([this, &paths](auto& walker) { - auto res = walker.walk(); - auto const& entries = walker.entries(); - std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [this, &walker](auto const& entry) { - return absolutePath(walker.relativePath(entry)); - }); - return res; - }); + const auto walkResult = withWalkerDo( + [this, &paths](auto& walker) + { + auto res = walker.walk(); + auto const& entries = walker.entries(); + std::transform( + entries.begin(), + entries.end(), + std::back_inserter(paths), + [this, &walker](auto const& entry) + { + return absolutePath(*walker.relativePath(entry)); + } + ); + return res; + } + ); ASSERT_TRUE(walkResult.has_value()) << "Error during walk: " << (walkResult.has_value() ? "" : walkResult.error()); @@ -488,15 +501,24 @@ namespace Utility::Test auto doTheWalk = [this, &paths]() { paths.clear(); - const auto walkResult = withWalkerDo([this, &paths](auto& walker) { - walker.reset(); - auto res = walker.walk(); - auto const& entries = walker.entries(); - std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [this, &walker](auto const& entry) { - return absolutePath(walker.relativePath(entry)); - }); - return res; - }); + const auto walkResult = withWalkerDo( + [this, &paths](auto& walker) + { + walker.reset(); + auto res = walker.walk(); + auto const& entries = walker.entries(); + std::transform( + entries.begin(), + entries.end(), + std::back_inserter(paths), + [this, &walker](auto const& entry) + { + return absolutePath(*walker.relativePath(entry)); + } + ); + return res; + } + ); ASSERT_TRUE(walkResult.has_value()) << "Error during walk: " << (walkResult.has_value() ? "" : walkResult.error()); }; @@ -522,4 +544,73 @@ namespace Utility::Test doTheWalk(); EXPECT_THAT(paths, ::testing::UnorderedElementsAreArray(expectedPaths)); } -} \ No newline at end of file + + namespace + { + std::vector smallTree() + { + using SharedData::DirectoryEntry; + return { + DirectoryEntry{.type = DirectoryEntry::FileType::Directory}, + DirectoryEntry{.name = "sub", .type = DirectoryEntry::FileType::Directory, .parent = 0}, + DirectoryEntry{.name = "file.txt", .type = DirectoryEntry::FileType::Regular, .parent = 1}, + DirectoryEntry{.name = "a/b", .type = DirectoryEntry::FileType::Regular, .parent = 1}, + DirectoryEntry{.name = "lost.txt", .type = DirectoryEntry::FileType::Regular, .parent = 9}, + }; + } + } + + TEST(WalkedEntryTests, RelativePathsFollowTheParents) + { + const auto entries = smallTree(); + const auto root = walkedRelativePath(entries, entries[0]); + ASSERT_TRUE(root.has_value()); + EXPECT_TRUE(root->empty()); + const auto file = walkedRelativePath(entries, entries[2]); + ASSERT_TRUE(file.has_value()) << file.error().toString(); + EXPECT_EQ(file->value(), "sub/file.txt"); + } + + TEST(WalkedEntryTests, RelativePathsRefuseInvalidNamesAndUnknownParentsWithoutThrowing) + { + const auto entries = smallTree(); + const auto invalid = walkedRelativePath(entries, entries[3]); + ASSERT_FALSE(invalid.has_value()); + EXPECT_EQ(invalid.error().reason, Paths::PathRejection::ContainsSeparator); + const auto lost = walkedRelativePath(entries, entries[4]); + ASSERT_FALSE(lost.has_value()); + EXPECT_EQ(lost.error().reason, Paths::PathRejection::UnknownParent); + const auto shared = SharedData::relativePath(entries, entries[4]); + ASSERT_FALSE(shared.has_value()); + EXPECT_EQ(shared.error().reason, Paths::PathRejection::UnknownParent); + } + + TEST(WalkedEntryTests, DescriptionsUseTheNamesAsListed) + { + const auto entries = smallTree(); + EXPECT_EQ(describeWalkedEntry("/home/test", entries, entries[3]), "/home/test/sub/a/b"); + EXPECT_EQ(describeWalkedEntry("/", entries, entries[2]), "/sub/file.txt"); + EXPECT_EQ(describeWalkedEntry("", entries, entries[2]), "sub/file.txt"); + EXPECT_EQ(describeWalkedEntry("/home/test", entries, entries[0]), "/home/test"); + EXPECT_EQ(describeWalkedEntry("/home/test", entries, entries[4]), "/home/test/lost.txt"); + } + + TEST_F(DirectoryTraversalTests, ACompletedWalkReportsItsEnd) + { + createSomeFilesIn(isolateDirectory_.path(), 2); + makeDefaultScan(); + withWalkerDo( + [](auto& walker) + { + ASSERT_TRUE(walker.walkAll().has_value()); + ASSERT_TRUE(walker.completed()); + const auto again = walker.walk(); + ASSERT_TRUE(again.has_value()); + EXPECT_TRUE(*again) << "a walk that is done says so"; + EXPECT_EQ(walker.entries().size(), 3u); + for (auto const& entry : walker.entries()) + EXPECT_TRUE(walker.relativePath(entry).has_value()) << entry.name; + } + ); + } +} From 8cd3e1d222806bcf40daa692fcabe73257ce40c3 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:48:43 +0200 Subject: [PATCH 124/131] Tests: An archive download looks up each file root again before packing it --- .../test_archive_download_operation.hpp | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/backend/test/backend/test_archive_download_operation.hpp b/backend/test/backend/test_archive_download_operation.hpp index 53745d9a..36058f45 100644 --- a/backend/test/backend/test_archive_download_operation.hpp +++ b/backend/test/backend/test_archive_download_operation.hpp @@ -513,6 +513,48 @@ namespace Test EXPECT_EQ(entries[0].contents.size(), 65536u); } + TEST_F(ArchiveDownloadOperationTests, AFileRootIsArchivedWithItsCurrentSize) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + // The listing the root came from is older than the file. + const auto archivePath = archiveDir_.path() / "fresh.tar"; + ArchiveDownloadOperation operation{ + sftp, + { + .roots = {makeFileEntry("/home/test/large.txt", 100u)}, + .localArchivePath = toLocal(archivePath), + .compression = TarArchive::Compression::None, + } + }; + ASSERT_TRUE(runToCompletion(operation, 2000).has_value()); + + const auto entries = recoverArchiveEntries(archivePath); + ASSERT_EQ(entries.size(), 1u); + EXPECT_EQ(entries[0].contents.size(), 65536u); + } + + TEST_F(ArchiveDownloadOperationTests, AFileRootThatIsGoneFailsTheArchive) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto archivePath = archiveDir_.path() / "gone.tar"; + ArchiveDownloadOperation operation{ + sftp, + { + .roots = {makeFileEntry("/home/test/not_there.txt", 10u)}, + .localArchivePath = toLocal(archivePath), + .compression = TarArchive::Compression::None, + } + }; + const auto result = runToCompletion(operation, 2000); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, ArchiveDownloadOperation::ErrorType::FileStatFailed); + EXPECT_FALSE(std::filesystem::exists(archivePath)); + } + TEST_F(ArchiveDownloadOperationTests, FilepartAbsentAfterCompletion) { CREATE_SERVER_AND_JOINER(sftpServer); From 3bfda248829b71f005572c0710a2e129c93b3901 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 01:51:02 +0200 Subject: [PATCH 125/131] Tests: Uploads and local scans use the error code filesystem overloads --- backend/test/backend/test_upload_operation.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/test/backend/test_upload_operation.hpp b/backend/test/backend/test_upload_operation.hpp index 9317b138..25512cbd 100644 --- a/backend/test/backend/test_upload_operation.hpp +++ b/backend/test/backend/test_upload_operation.hpp @@ -634,7 +634,7 @@ namespace Test auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); - EXPECT_EQ(workResult.error().type, UploadOperation::ErrorType::NotAFile); + EXPECT_EQ(workResult.error().type, UploadOperation::ErrorType::FileNotFound); } TEST_F(UploadOperationTests, UploadLocalDirectoryFails) From 323e51b09c0bf68845cb8d335f1773c6451f37db Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:07:58 +0200 Subject: [PATCH 126/131] Tests: Sync remembers nothing across runs --- backend/test/backend/main.cpp | 1 - .../test/backend/test_sync_baseline_store.hpp | 128 ------------------ 2 files changed, 129 deletions(-) delete mode 100644 backend/test/backend/test_sync_baseline_store.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index c81ecc2e..a79232d2 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -26,7 +26,6 @@ #include "test_file_change_filter.hpp" #include "test_password_request.hpp" #include "test_sync_session.hpp" -#include "test_sync_baseline_store.hpp" #include "test_dropped_entry.hpp" #include "test_transfer_failures.hpp" #include "test_operation_queue.hpp" diff --git a/backend/test/backend/test_sync_baseline_store.hpp b/backend/test/backend/test_sync_baseline_store.hpp deleted file mode 100644 index 46ff76e8..00000000 --- a/backend/test/backend/test_sync_baseline_store.hpp +++ /dev/null @@ -1,128 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include -#include - -extern std::filesystem::path temporaryRoot; - -namespace Test -{ - class SyncBaselineStoreTests : public ::testing::Test - { - protected: - static SharedData::Sync::SyncRecord record(std::uint64_t size) - { - const SharedData::Sync::RecordedSide side{.type = SharedData::FileType::Regular, .size = size, .mtime = 7}; - return {.local = side, .remote = side}; - } - - Utility::TemporaryDirectory directory_{temporaryRoot, true}; - SyncBaselineStore store_{directory_.path() / "baselines"}; - std::string pairKey_{SyncBaselineStore::pairKey( - "saved", - Paths::LocalPath::fromUtf8("/local/root", Paths::LocalPlatform::Posix).value(), - Paths::RemotePath::make("/remote/root").value() - )}; - }; - - TEST_F(SyncBaselineStoreTests, APairWithoutFileHasNoBaseline) - { - EXPECT_FALSE(store_.load(pairKey_)); - } - - TEST_F(SyncBaselineStoreTests, UpdatesAddToWhatIsStored) - { - ASSERT_TRUE(store_.update( - pairKey_, - [](SharedData::Sync::SyncBaseline& baseline) - { - baseline.records.insert_or_assign("a.txt", record(1)); - } - )); - ASSERT_TRUE( - SyncBaselineStore{directory_.path() / "baselines"}.update( - pairKey_, - [](SharedData::Sync::SyncBaseline& baseline) - { - EXPECT_TRUE(baseline.records.contains("a.txt")); - baseline.records.insert_or_assign("b.txt", record(2)); - } - ) - ); - const auto loaded = store_.load(pairKey_); - ASSERT_TRUE(loaded); - EXPECT_EQ(loaded->records.size(), 2u); - EXPECT_EQ(loaded->records.at("b.txt"), record(2)); - } - - TEST_F(SyncBaselineStoreTests, AnEmptyUpdateStillGivesThePairAHistory) - { - ASSERT_TRUE(store_.update(pairKey_, [](SharedData::Sync::SyncBaseline&) {})); - const auto loaded = store_.load(pairKey_); - ASSERT_TRUE(loaded); - EXPECT_TRUE(loaded->records.empty()); - } - - TEST_F(SyncBaselineStoreTests, ADamagedFileIsAHistoryWithoutRecords) - { - std::filesystem::create_directories(directory_.path() / "baselines"); - std::ofstream{SyncBaselineStore::fileOf(directory_.path() / "baselines", pairKey_)} << "{not json"; - const auto loaded = store_.load(pairKey_); - ASSERT_TRUE(loaded) << "a pair whose records are lost protects every delete"; - EXPECT_TRUE(loaded->records.empty()); - } - - TEST_F(SyncBaselineStoreTests, AFileOfAnotherPairIsNeitherReadNorReplaced) - { - const auto otherKey = std::string{"other"}; - ASSERT_TRUE(store_.update( - otherKey, - [](SharedData::Sync::SyncBaseline& baseline) - { - baseline.records.insert_or_assign("other.txt", record(3)); - } - )); - // Collide on purpose by moving the other pair's file to this pair's name. - std::filesystem::rename( - SyncBaselineStore::fileOf(directory_.path() / "baselines", otherKey), - SyncBaselineStore::fileOf(directory_.path() / "baselines", pairKey_) - ); - EXPECT_FALSE(store_.load(pairKey_)); - EXPECT_FALSE(store_.update(pairKey_, [](SharedData::Sync::SyncBaseline&) {})); - } - - TEST_F(SyncBaselineStoreTests, PairKeysNameTheSessionAndBothRoots) - { - EXPECT_EQ(pairKey_, "saved\n/local/root\n/remote/root"); - EXPECT_NE( - SyncBaselineStore::fileOf(directory_.path(), pairKey_), - SyncBaselineStore::fileOf(directory_.path(), "saved\n/local/root\n/remote/other") - ); - } - - TEST_F(SyncBaselineStoreTests, PruningRemovesThePairsOfSessionsThatAreGone) - { - const auto local = Paths::LocalPath::fromUtf8("/local", Paths::LocalPlatform::Posix).value(); - const auto remote = Paths::RemotePath::make("/remote").value(); - const auto kept = SyncBaselineStore::pairKey("kept", local, remote); - const auto gone = SyncBaselineStore::pairKey("gone", local, remote); - ASSERT_TRUE(store_.update(kept, [](SharedData::Sync::SyncBaseline&) {})); - ASSERT_TRUE(store_.update(gone, [](SharedData::Sync::SyncBaseline&) {})); - std::ofstream{directory_.path() / "baselines" / "unrelated.txt"} << "x"; - - EXPECT_EQ(store_.pruneAllBut({"kept", "other"}), 1u); - EXPECT_TRUE(store_.load(kept)); - EXPECT_FALSE(store_.load(gone)); - EXPECT_TRUE(std::filesystem::exists(directory_.path() / "baselines" / "unrelated.txt")); - } - - TEST_F(SyncBaselineStoreTests, PruningWithoutADirectoryRemovesNothing) - { - EXPECT_EQ(SyncBaselineStore{directory_.path() / "none"}.pruneAllBut({}), 0u); - } -} From 307bb92bdb16884ea94229a72d95e1b25070b3a0 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:11:04 +0200 Subject: [PATCH 127/131] Tests: Case variants block a sync only towards a side that folds case --- shared_data/test/shared_data/test_diff.hpp | 61 ++++++++++++++++++- .../test/shared_data/test_diff_tree_store.hpp | 14 +++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/shared_data/test/shared_data/test_diff.hpp b/shared_data/test/shared_data/test_diff.hpp index 2ccadcba..d19bfaa7 100644 --- a/shared_data/test/shared_data/test_diff.hpp +++ b/shared_data/test/shared_data/test_diff.hpp @@ -797,14 +797,14 @@ namespace SharedData::Sync::Test EXPECT_EQ(row.descendantByteTotal, 0u); } - TEST(SyncDiffTests, NamesDifferingOnlyInCaseAcrossSidesAreBlocked) + TEST(SyncDiffTests, NamesDifferingOnlyInCaseAcrossSidesAreBlockedForACaseFoldingServer) { // On a case insensitive server "Foo" replaces "foo", the delete of "foo" would then remove the upload. CollectingSink sink; diffScanTrees( makeDir("", {makeFile("Foo", 3, 200), makeFile("other", 1, 100)}), makeDir("", {makeFile("foo", 3, 100)}), - {.direction = Direction::Upload, .actionDelete = true}, + {.direction = Direction::Upload, .actionDelete = true, .remoteCaseInsensitive = true}, sink ); ASSERT_EQ(sink.uploads.size(), 1u); @@ -876,6 +876,63 @@ namespace SharedData::Sync::Test } } + TEST(SyncDiffTests, LocalCaseVariantsAreBlockedOnlyForACaseFoldingServer) + { + const auto local = makeDir("", {makeFile("Readme", 1, 100), makeFile("readme", 1, 100)}); + const auto remote = makeDir("", {}); + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Upload, .localCaseInsensitive = true}, sink); + EXPECT_EQ(sink.uploads.size(), 2u) << "only the destination folding case counts"; + EXPECT_TRUE(sink.blocked.empty()); + } + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Upload, .remoteCaseInsensitive = true}, sink); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_EQ(sink.blocked.size(), 2u); + } + } + + TEST(SyncDiffTests, CaseSensitiveSidesNeverBlockCaseOrNormalizationDifferences) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("Foo", 3, 200), makeFile("cafe\xCC\x81", 1, 100), makeFile("Readme", 1, 100)}), + makeDir("", {makeFile("foo", 3, 100), makeFile("caf\xC3\xA9", 2, 100), makeFile("README", 1, 100)}), + {.direction = Direction::Both}, + sink + ); + EXPECT_TRUE(sink.blocked.empty()); + EXPECT_EQ(sink.uploads.size(), 3u); + EXPECT_EQ(sink.downloads.size(), 3u); + } + + TEST(SyncDiffTests, CaseVariantsAcrossSidesCountOnlyInTheDirectionsWhoseDestinationFolds) + { + const auto local = makeDir("", {makeFile("Foo", 3, 200)}); + const auto remote = makeDir("", {makeFile("foo", 3, 100)}); + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Download, .remoteCaseInsensitive = true}, sink); + EXPECT_TRUE(sink.blocked.empty()) << "a download onto a case sensitive filesystem keeps both"; + EXPECT_EQ(sink.downloads.size(), 1u); + } + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Download, .localCaseInsensitive = true}, sink); + EXPECT_EQ(sink.blocked.size(), 2u); + EXPECT_TRUE(sink.downloads.empty()); + } + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Both, .localCaseInsensitive = true}, sink); + EXPECT_EQ(sink.blocked.size(), 2u); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + } + } + TEST(SyncDiffTests, UnreadableLocalDirectoryNeverDeletesItsRemoteContent) { CollectingSink sink; diff --git a/shared_data/test/shared_data/test_diff_tree_store.hpp b/shared_data/test/shared_data/test_diff_tree_store.hpp index df25dc31..ceb1383f 100644 --- a/shared_data/test/shared_data/test_diff_tree_store.hpp +++ b/shared_data/test/shared_data/test_diff_tree_store.hpp @@ -342,6 +342,20 @@ namespace SharedData::Sync::Test EXPECT_EQ(relKeys(stores.downloads.buildPlan({"new"}, stores.trees())), (std::vector{"new/b"})); } + TEST(DiffTreeStoreTests, UploadPlanSkipsCaseVariantsOnlyForACaseFoldingServer) + { + const auto local = + makeDir("", {makeDir("new", {makeFile("A.txt", 1, 100), makeFile("a.txt", 1, 100), makeFile("b", 1, 100)})}); + const DiffStores folding{local, makeDir("", {}), {.direction = Direction::Upload, .remoteCaseInsensitive = true}}; + EXPECT_EQ(relKeys(folding.uploads.buildPlan({"new"}, folding.trees())), (std::vector{"new/b"})); + + const DiffStores sensitive{local, makeDir("", {}), {.direction = Direction::Upload, .localCaseInsensitive = true}}; + EXPECT_EQ( + relKeys(sensitive.uploads.buildPlan({"new"}, sensitive.trees())), + (std::vector{"new/A.txt", "new/a.txt", "new/b"}) + ); + } + TEST(DiffTreeStoreTests, PlanForAPartlySelectedExpandedDirectory) { DiffStores stores{ From 38497c4b81749fb5a954ffe9a8314a8b3a8951ad Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:12:34 +0200 Subject: [PATCH 128/131] Tests: Add ComparedState and the ChangedSinceCompare error --- shared_data/test/shared_data/main.cpp | 1 + .../test/shared_data/test_compared_state.hpp | 82 +++++++++++++++++++ .../test/shared_data/test_diff_tree_store.hpp | 18 ++++ 3 files changed, 101 insertions(+) create mode 100644 shared_data/test/shared_data/test_compared_state.hpp diff --git a/shared_data/test/shared_data/main.cpp b/shared_data/test/shared_data/main.cpp index e83708e2..7c47bdb9 100644 --- a/shared_data/test/shared_data/main.cpp +++ b/shared_data/test/shared_data/main.cpp @@ -1,5 +1,6 @@ #include "test_bulk_delete_request.hpp" #include "test_bulk_transfer_request.hpp" +#include "test_compared_state.hpp" #include "test_operation_added.hpp" #include "test_diff.hpp" #include "test_diff_tree_store.hpp" diff --git a/shared_data/test/shared_data/test_compared_state.hpp b/shared_data/test/shared_data/test_compared_state.hpp new file mode 100644 index 00000000..82dded67 --- /dev/null +++ b/shared_data/test/shared_data/test_compared_state.hpp @@ -0,0 +1,82 @@ +#pragma once + +#include + +#include + +namespace SharedData::Test +{ + namespace ComparedStateTestDetail + { + inline ComparedState file(std::uint64_t size, std::uint64_t mtime) + { + return ComparedState{.exists = true, .type = FileType::Regular, .size = size, .mtime = mtime}; + } + + inline ComparedState link(std::string target) + { + return ComparedState{.exists = true, .type = FileType::Symlink, .mtime = 5, .linkTarget = std::move(target)}; + } + } + + TEST(ComparedStateTests, AnUnchangedFilePasses) + { + using namespace ComparedStateTestDetail; + EXPECT_TRUE(unchangedSinceCompare(file(3, 100), file(3, 100))); + } + + TEST(ComparedStateTests, AFileWithAnotherSizeOrTimeChanged) + { + using namespace ComparedStateTestDetail; + EXPECT_FALSE(unchangedSinceCompare(file(3, 100), file(4, 100))); + EXPECT_FALSE(unchangedSinceCompare(file(3, 100), file(3, 101))); + } + + TEST(ComparedStateTests, SomethingThatAppearedWhereNothingWasChanged) + { + using namespace ComparedStateTestDetail; + EXPECT_TRUE(unchangedSinceCompare(ComparedState{}, ComparedState{})); + EXPECT_FALSE(unchangedSinceCompare(ComparedState{}, file(3, 100))); + EXPECT_FALSE(unchangedSinceCompare(ComparedState{}, ComparedState{.exists = true, .type = FileType::Directory})); + } + + TEST(ComparedStateTests, SomethingThatIsGoneLosesNothing) + { + using namespace ComparedStateTestDetail; + EXPECT_TRUE(unchangedSinceCompare(file(3, 100), ComparedState{})); + } + + TEST(ComparedStateTests, AnotherTypeChanged) + { + using namespace ComparedStateTestDetail; + EXPECT_FALSE(unchangedSinceCompare(file(0, 5), link("target"))); + EXPECT_FALSE(unchangedSinceCompare(ComparedState{.exists = true, .type = FileType::Directory}, file(0, 0))); + } + + TEST(ComparedStateTests, ASymlinkChangesByItsLiteralOnly) + { + using namespace ComparedStateTestDetail; + auto later = link("target"); + later.mtime = 99; + EXPECT_TRUE(unchangedSinceCompare(link("target"), later)); + EXPECT_FALSE(unchangedSinceCompare(link("target"), link("other"))); + } + + TEST(ComparedStateTests, ADirectoryChangesByItsTypeOnly) + { + const ComparedState compared{.exists = true, .type = FileType::Directory, .size = 4096, .mtime = 1}; + const ComparedState current{.exists = true, .type = FileType::Directory, .size = 8192, .mtime = 2}; + EXPECT_TRUE(unchangedSinceCompare(compared, current)); + } + + TEST(ComparedStateTests, AnEntryDescribesItsState) + { + const DirectoryEntry entry{.type = FileType::Symlink, .size = 7, .mtime = 9, .linkTarget = "a/b"}; + EXPECT_EQ( + comparedStateOf(entry), + (ComparedState{.exists = true, .type = FileType::Symlink, .size = 7, .mtime = 9, .linkTarget = "a/b"}) + ); + const DirectoryEntry regular{.type = FileType::Regular, .size = 7, .mtime = 9, .linkTarget = "ignored"}; + EXPECT_FALSE(comparedStateOf(regular).linkTarget); + } +} diff --git a/shared_data/test/shared_data/test_diff_tree_store.hpp b/shared_data/test/shared_data/test_diff_tree_store.hpp index ceb1383f..9d5264df 100644 --- a/shared_data/test/shared_data/test_diff_tree_store.hpp +++ b/shared_data/test/shared_data/test_diff_tree_store.hpp @@ -356,6 +356,24 @@ namespace SharedData::Sync::Test ); } + TEST(DiffTreeStoreTests, ComparedStateAtNamesWhatTheScanSaw) + { + const auto tree = makeDir( + "", {makeDir("dir", {makeFile("a.txt", 3, 100), makeSymlink("link", "../x", 7)}), makeFile("b", 1, 2)} + ); + EXPECT_EQ( + comparedStateAt(tree, "dir/a.txt"), + (ComparedState{.exists = true, .type = FileType::Regular, .size = 3, .mtime = 100}) + ); + EXPECT_EQ( + comparedStateAt(tree, "dir/link"), + (ComparedState{.exists = true, .type = FileType::Symlink, .mtime = 7, .linkTarget = "../x"}) + ); + EXPECT_EQ(comparedStateAt(tree, "dir").type, FileType::Directory); + EXPECT_FALSE(comparedStateAt(tree, "dir/missing").exists); + EXPECT_FALSE(comparedStateAt(tree, "b/below").exists); + } + TEST(DiffTreeStoreTests, PlanForAPartlySelectedExpandedDirectory) { DiffStores stores{ From 2d7e601ec2d43b8346d748298262b02d4f892ceb Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:28:31 +0200 Subject: [PATCH 129/131] Tests: Transfers and deletes can be held to what a sync compare saw --- backend/test/backend/main.cpp | 1 + .../backend/test_compared_state_check.hpp | 293 ++++++++++++++++++ 2 files changed, 294 insertions(+) create mode 100644 backend/test/backend/test_compared_state_check.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index a79232d2..70907029 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -28,6 +28,7 @@ #include "test_sync_session.hpp" #include "test_dropped_entry.hpp" #include "test_transfer_failures.hpp" +#include "test_compared_state_check.hpp" #include "test_operation_queue.hpp" #include "test_existence_check.hpp" #include "test_opener_risk.hpp" diff --git a/backend/test/backend/test_compared_state_check.hpp b/backend/test/backend/test_compared_state_check.hpp new file mode 100644 index 00000000..684eb909 --- /dev/null +++ b/backend/test/backend/test_compared_state_check.hpp @@ -0,0 +1,293 @@ +#pragma once + +#include +#include +#include +#include "test_transfer_failures.hpp" + +#include + +#include +#include +#include + +namespace Test +{ + class ComparedStateCheckTests : public TransferFailureTests + { + public: + static SharedData::ComparedState localNow(std::filesystem::path const& path) + { + return ComparedStateCheck::localStateOf(path).value(); + } + + static SharedData::ComparedState remoteNow(SecureShell::SftpSession& sftp, std::string const& path) + { + return sftp + .performPromise( + [&sftp, path = remote(path)]() + { + return ComparedStateCheck::remoteStateOfInStrand(sftp, path); + } + ) + .get() + .value(); + } + + static std::expected checkRemote( + SecureShell::SftpSession& sftp, + std::string const& path, + SharedData::ComparedState const& compared + ) + { + return sftp + .performPromise( + [&sftp, path = remote(path), &compared]() + { + return ComparedStateCheck::requireUnchangedRemoteInStrand(sftp, path, compared); + } + ) + .get(); + } + + static void touchLater(std::filesystem::path const& path) + { + std::filesystem::last_write_time(path, std::filesystem::last_write_time(path) + std::chrono::hours{1}); + } + }; + + TEST_F(ComparedStateCheckTests, LocalStatesDescribeFilesDirectoriesAndAbsence) + { + const auto file = localDirectory_.path() / "file.txt"; + writeLocalFile(file, "12345"); + const auto state = localNow(file); + EXPECT_TRUE(state.exists); + EXPECT_EQ(state.type, SharedData::FileType::Regular); + EXPECT_EQ(state.size, 5u); + EXPECT_EQ(state.mtime, ComparedStateCheck::secondsSinceEpoch(std::filesystem::last_write_time(file))); + + EXPECT_EQ(localNow(localDirectory_.path()).type, SharedData::FileType::Directory); + EXPECT_FALSE(localNow(localDirectory_.path() / "missing").exists); + EXPECT_FALSE(localNow(localDirectory_.path() / "missing" / "below").exists); + } + +#ifndef _WIN32 + TEST_F(ComparedStateCheckTests, ALocalSymlinkIsDescribedByItsLiteral) + { + const auto link = localDirectory_.path() / "link"; + std::filesystem::create_symlink("does/not/exist", link); + const auto state = localNow(link); + EXPECT_EQ(state.type, SharedData::FileType::Symlink); + EXPECT_EQ(state.linkTarget, "does/not/exist"); + + const auto compared = state; + std::filesystem::remove(link); + std::filesystem::create_symlink("elsewhere", link); + const auto checked = ComparedStateCheck::requireUnchangedLocal(link, compared); + ASSERT_FALSE(checked); + EXPECT_EQ(checked.error().type, Operation::ErrorType::ChangedSinceCompare); + } +#endif + + TEST_F(ComparedStateCheckTests, TheLocalCheckPassesOnlyWhatTheCompareSaw) + { + const auto file = localDirectory_.path() / "file.txt"; + writeLocalFile(file, "12345"); + const auto compared = localNow(file); + EXPECT_TRUE(ComparedStateCheck::requireUnchangedLocal(file, compared)) << "unchanged"; + EXPECT_TRUE(ComparedStateCheck::requireUnchangedLocal(file, std::nullopt)) << "nothing to check"; + + const auto expectChanged = [&](SharedData::ComparedState const& expected, std::string const& what) + { + const auto checked = ComparedStateCheck::requireUnchangedLocal(file, expected); + ASSERT_FALSE(checked) << what; + EXPECT_EQ(checked.error().type, Operation::ErrorType::ChangedSinceCompare) << what; + }; + const auto time = std::filesystem::last_write_time(file); + writeLocalFile(file, "123456"); + std::filesystem::last_write_time(file, time); + expectChanged(compared, "another size"); + + writeLocalFile(file, "12345"); + std::filesystem::last_write_time(file, time); + EXPECT_TRUE(ComparedStateCheck::requireUnchangedLocal(file, compared)) << "restored"; + touchLater(file); + expectChanged(compared, "another time"); + + expectChanged(SharedData::ComparedState{}, "appeared where the compare saw nothing"); + std::filesystem::remove(file); + EXPECT_TRUE(ComparedStateCheck::requireUnchangedLocal(file, compared)) << "gone loses nothing"; + std::filesystem::create_directory(file); + expectChanged(compared, "replaced by a directory"); + } + + TEST_F(ComparedStateCheckTests, TheRemoteCheckPassesOnlyWhatTheCompareSaw) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/home/test/link"}, {"target", "file1.txt"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto compared = remoteNow(*sftp, "/home/test/file1.txt"); + EXPECT_EQ(compared.type, SharedData::FileType::Regular); + EXPECT_EQ(compared.size, std::string{"Fake file content"}.size()); + EXPECT_TRUE(checkRemote(*sftp, "/home/test/file1.txt", compared)); + + const auto link = remoteNow(*sftp, "/home/test/link"); + EXPECT_EQ(link.type, SharedData::FileType::Symlink); + EXPECT_EQ(link.linkTarget, "file1.txt"); + EXPECT_FALSE(remoteNow(*sftp, "/home/test/missing").exists); + + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/file1.txt"}, {"content", "changed"}})); + const auto changed = checkRemote(*sftp, "/home/test/file1.txt", compared); + ASSERT_FALSE(changed); + EXPECT_EQ(changed.error().type, Operation::ErrorType::ChangedSinceCompare); + + const auto appeared = checkRemote(*sftp, "/home/test/file1.txt", SharedData::ComparedState{}); + ASSERT_FALSE(appeared); + EXPECT_EQ(appeared.error().type, Operation::ErrorType::ChangedSinceCompare); + EXPECT_TRUE(checkRemote(*sftp, "/home/test/missing", compared)) << "gone loses nothing"; + } + + TEST_F(ComparedStateCheckTests, ADownloadNeverReplacesADestinationThatChangedAfterTheCompare) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto localPath = localDirectory_.path() / "file1.txt"; + writeLocalFile(localPath, "old local"); + const auto compared = localNow(localPath); + writeLocalFile(localPath, "edited locally meanwhile"); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .comparedDestination = compared, + } + }; + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::ChangedSinceCompare); + EXPECT_EQ(readLocalFile(localPath), "edited locally meanwhile"); + } + + TEST_F(ComparedStateCheckTests, ADownloadReplacesADestinationThatIsAsTheCompareSawIt) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto localPath = localDirectory_.path() / "file1.txt"; + writeLocalFile(localPath, "old local"); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .comparedDestination = localNow(localPath), + } + }; + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), "Fake file content"); + } + + TEST_F(ComparedStateCheckTests, AnUploadNeverReplacesADestinationThatChangedAfterTheCompare) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto localPath = localDirectory_.path() / "upload.txt"; + writeLocalFile(localPath, "new content"); + const auto compared = remoteNow(*sftp, "/home/test/file1.txt"); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/file1.txt"}, {"content", "edited"}})); + + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .comparedDestination = compared, + } + }; + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::ChangedSinceCompare); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "edited"); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt.filepart"), "") << "the temp file is removed"; + } + + TEST_F(ComparedStateCheckTests, AnUploadReplacesADestinationThatIsAsTheCompareSawIt) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto localPath = localDirectory_.path() / "upload.txt"; + writeLocalFile(localPath, "new content"); + + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .comparedDestination = remoteNow(*sftp, "/home/test/file1.txt"), + } + }; + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "new content"); + } + + TEST_F(ComparedStateCheckTests, ALocalDeleteLeavesATargetThatChangedAfterTheCompare) + { + const auto changed = localDirectory_.path() / "changed.txt"; + const auto unchanged = localDirectory_.path() / "unchanged.txt"; + writeLocalFile(changed, "a"); + writeLocalFile(unchanged, "b"); + const auto comparedChanged = localNow(changed); + const auto comparedUnchanged = localNow(unchanged); + writeLocalFile(changed, "edited"); + + LocalDeleteOperation operation{{ + .targets = {toLocal(changed), toLocal(unchanged)}, + .comparedStates = {comparedChanged, comparedUnchanged}, + }}; + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_TRUE(std::filesystem::exists(changed)); + EXPECT_FALSE(std::filesystem::exists(unchanged)); + ASSERT_EQ(operation.getFailed().size(), 1u); + EXPECT_EQ(operation.getFailed().front().second.type, Operation::ErrorType::ChangedSinceCompare); + } + + TEST_F(ComparedStateCheckTests, ARemoteDeleteLeavesATargetThatChangedAfterTheCompare) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/changed.txt"}, {"content", "a"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/unchanged.txt"}, {"content", "b"}})); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/home/test/link"}, {"target", "a"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto comparedChanged = remoteNow(*sftp, "/home/test/changed.txt"); + const auto comparedUnchanged = remoteNow(*sftp, "/home/test/unchanged.txt"); + auto comparedLink = remoteNow(*sftp, "/home/test/link"); + comparedLink.linkTarget = "b"; + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/changed.txt"}, {"content", "edited"}})); + + DeleteOperation operation{sftp, {.recursive = true}}; + operation.setTargets({ + {.path = remote("/home/test/changed.txt"), .compared = comparedChanged}, + {.path = remote("/home/test/unchanged.txt"), .compared = comparedUnchanged}, + {.path = remote("/home/test/link"), .compared = comparedLink}, + {.path = remote("/home/test/gone.txt"), .compared = comparedUnchanged}, + }); + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/changed.txt"), "edited"); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/unchanged.txt"), ""); + EXPECT_TRUE(remoteNow(*sftp, "/home/test/link").exists); + ASSERT_EQ(operation.getFailed().size(), 2u) << "a gone target counts as deleted"; + for (auto const& [path, error] : operation.getFailed()) + EXPECT_EQ(error.type, Operation::ErrorType::ChangedSinceCompare) << path; + } +} From 1d316b143439a9f8a8e534151f022a5d50bf276f Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:29:17 +0200 Subject: [PATCH 130/131] Tests: A sync entry replaces or deletes only what its compare saw --- backend/test/backend/test_operation_queue.hpp | 190 +++++++++++++++++- 1 file changed, 184 insertions(+), 6 deletions(-) diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 2ae6923b..f8dbd037 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include "nui_env.hpp" @@ -12,6 +13,7 @@ #include #include #include +#include #include #include @@ -20,6 +22,7 @@ #include #include #include +#include #include #ifndef _WIN32 @@ -131,6 +134,44 @@ namespace Test return completions_; } + /** + * @brief What is at @p path on the server now, as a compare that ran right now would have seen it. + */ + static SharedData::ComparedState remoteStateNow(SecureShell::SftpSession& sftp, std::string const& path) + { + auto future = sftp.performPromise( + [&sftp, path = remote(path)]() + { + return ComparedStateCheck::remoteStateOfInStrand(sftp, path); + } + ); + return future.get().value(); + } + + /** + * @brief The resolved plan entry @p relKey of @p session, which the compare saw as what is at its target now. + */ + static SyncSession::ResolvedPlanEntry plannedNow( + SyncSession const& session, + SecureShell::SftpSession& sftp, + std::string relKey, + SharedData::Sync::Action action, + bool isDirectory, + std::uint64_t size = 0 + ) + { + using SharedData::Sync::Action; + auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); + const bool onLocalSide = action == Action::Download || action == Action::DeleteLocal; + auto compared = onLocalSide ? ComparedStateCheck::localStateOf(paths.local.native()).value() + : remoteStateNow(sftp, paths.remote.value()); + return SyncSession::ResolvedPlanEntry{ + .entry = {.relKey = std::move(relKey), .action = action, .sizeBytes = size, .isDirectory = isDirectory}, + .paths = std::move(paths), + .compared = std::move(compared), + }; + } + static std::set remoteNames(SecureShell::SftpSession& sftp, std::string const& directory) { auto future = sftp.listDirectory(remote(directory)); @@ -1215,13 +1256,9 @@ namespace Test context.get_executor() }; using SharedData::Sync::Action; - const auto entry = [&session](std::string relKey, Action action, bool isDirectory, std::uint64_t size = 0) + const auto entry = [&session, &sftp](std::string relKey, Action action, bool isDirectory, std::uint64_t size = 0) { - auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); - return SyncSession::ResolvedPlanEntry{ - .entry = {.relKey = std::move(relKey), .action = action, .sizeBytes = size, .isDirectory = isDirectory}, - .paths = std::move(paths), - }; + return plannedNow(session, *sftp, std::move(relKey), action, isDirectory, size); }; SyncSession::ResolvedPlan plan{ .entries = @@ -1422,6 +1459,147 @@ namespace Test } #endif + TEST_F(OperationQueueTests, SyncLeavesWhatChangedBetweenTheCompareAndItsExecution) + { + using namespace SharedData::Sync; + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/changing"}})); + for (auto const& [name, content] : + {std::pair{"edited.txt", "old"}, std::pair{"gone.txt", "gone"}, std::pair{"kept.txt", "kept"}}) + { + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", fmt::format("/home/test/changing/{}", name)}, {"content", content}} + )); + } + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "changing"; + std::filesystem::create_directories(localRoot); + std::ofstream{localRoot / "edited.txt"} << "new local content"; + std::ofstream{localRoot / "up.txt"} << "up"; + std::ofstream{localRoot / "appears.txt"} << "local"; + + // The compare sees both sides as they are now. + const auto nodeOf = [](std::string name, SharedData::ComparedState const& state) + { + return ScanNode{.name = std::move(name), .type = state.type, .size = state.size, .mtime = state.mtime}; + }; + const auto treeOf = [&](std::vector const& names, auto const& stateOf) + { + std::vector children; + for (auto const& name : names) + children.push_back(nodeOf(name, stateOf(name))); + std::ranges::sort(children, {}, &ScanNode::name); + return ScanNode{.name = "", .type = SharedData::FileType::Directory, .children = std::move(children)}; + }; + const auto localTree = [&]() + { + return treeOf( + {"appears.txt", "edited.txt", "up.txt"}, + [&](std::string const& name) + { + return ComparedStateCheck::localStateOf(localRoot / name).value(); + } + ); + }; + const auto remoteTree = [&](std::vector const& names) + { + return treeOf( + names, + [&](std::string const& name) + { + return remoteStateNow(*sftp, fmt::format("/home/test/changing/{}", name)); + } + ); + }; + + boost::asio::io_context context{}; + SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home/test/changing"), + }, + context.get_executor() + }; + const DiffOptions options{.direction = Direction::Upload, .actionDelete = true}; + session.setLocalTreeInStrand(localTree()); + session.setRemoteTreeInStrand(remoteTree({"edited.txt", "gone.txt", "kept.txt"})); + const auto summary = session.recomputeDiffInStrand(options, {}); + EXPECT_EQ(summary.uploads.itemCount, 3u); + EXPECT_EQ(summary.deletes.itemCount, 2u); + + const auto planOf = [&](DiffSection section, std::unordered_set const& selected) + { + return session.resolvePlanInStrand(section, selected); + }; + auto uploadPlan = planOf(DiffSection::Upload, {"appears.txt", "edited.txt", "up.txt"}); + auto deletePlan = planOf(DiffSection::Delete, {"gone.txt", "kept.txt"}); + + // Meanwhile on the server: one destination is edited, one appears and one delete target changes. + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/changing/edited.txt"}, {"content", "edited on the server"}} + )); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/changing/appears.txt"}, {"content", "remote"}} + )); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/changing/kept.txt"}, {"content", "kept and edited"}} + )); + + const auto execute = [&](DiffSection section, SyncSession::ResolvedPlan plan) + { + const SyncExecutionRequest request{ + .section = section, + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + return runOnStrand( + [&]() + { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + } + ); + }; + const auto uploads = execute(DiffSection::Upload, std::move(uploadPlan)); + EXPECT_EQ(uploads.uploads.size(), 3u); + const auto deletes = execute(DiffSection::Delete, std::move(deletePlan)); + EXPECT_EQ(deletes.remoteDeletes.size(), 2u); + + setPaused(false); + workUntilCompletions(2); + std::map failures; + for (auto const& completed : completions()) + { + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Completed) + << completed.operationId.value(); + for (auto const& [path, error] : completed.failedEntries) + failures.emplace(path.substr(path.rfind('/') + 1), error.type); + } + EXPECT_EQ( + failures, + (std::map{ + {"appears.txt", Operation::ErrorType::FileExists}, + {"edited.txt", Operation::ErrorType::ChangedSinceCompare}, + {"kept.txt", Operation::ErrorType::ChangedSinceCompare}, + }) + ); + EXPECT_EQ( + remoteNames(*sftp, "/home/test/changing"), + (std::set{"appears.txt", "edited.txt", "kept.txt", "up.txt"}) + ); + EXPECT_EQ(remoteStateNow(*sftp, "/home/test/changing/edited.txt").size, std::string{"edited on the server"}.size()); + EXPECT_EQ(remoteStateNow(*sftp, "/home/test/changing/appears.txt").size, std::string{"remote"}.size()); + + // A recompare offers the entries that were left alone again. + session.setRemoteTreeInStrand(remoteTree({"appears.txt", "edited.txt", "kept.txt", "up.txt"})); + const auto again = session.recomputeDiffInStrand(options, {}); + EXPECT_EQ(again.uploads.itemCount, 2u); + EXPECT_EQ(again.deletes.itemCount, 1u); + } + TEST_F(OperationQueueTests, SyncTransfersNonAsciiNames) { CREATE_SERVER_AND_JOINER(sftpServer); From 6b6f8df12dcf17b69518f762840b46f9cee6c153 Mon Sep 17 00:00:00 2001 From: 5cript Date: Thu, 15 Oct 2026 02:49:25 +0200 Subject: [PATCH 131/131] Tests: Local deletes run in a LocalOperationQueue of their own --- backend/test/backend/main.cpp | 1 + .../backend/test_local_operation_queue.hpp | 368 ++++++++++++++++++ backend/test/backend/test_operation_queue.hpp | 42 +- 3 files changed, 401 insertions(+), 10 deletions(-) create mode 100644 backend/test/backend/test_local_operation_queue.hpp diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index 70907029..3e5c05d0 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -7,6 +7,7 @@ #include "test_local_delete_guard.hpp" #include "test_local_delete_operation.hpp" #include "test_local_removal.hpp" +#include "test_local_operation_queue.hpp" #include "test_remote_delete_guard.hpp" #include "test_local_scan_operation.hpp" #include "test_bulk_download_operation.hpp" diff --git a/backend/test/backend/test_local_operation_queue.hpp b/backend/test/backend/test_local_operation_queue.hpp new file mode 100644 index 00000000..2cff746d --- /dev/null +++ b/backend/test/backend/test_local_operation_queue.hpp @@ -0,0 +1,368 @@ +#pragma once + +#include +#include +#include +#include +#include +#include "nui_env.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class LocalOperationQueueTests : public ::testing::Test + { + public: + void SetUp() override + { + if (!NuiEnv::instance().available()) + GTEST_SKIP() << "No display available; skipping LocalOperationQueue tests"; + auto& env = NuiEnv::instance(); + queue_ = std::make_shared( + pool_.get_executor(), strand_, *env.window, *env.hub, deletionPrevented_ + ); + queue_->setCompletionObserver( + [this](SharedData::OperationCompleted const& completed) + { + std::scoped_lock lock{completionsMutex_}; + completions_.push_back(completed); + } + ); + } + + void TearDown() override + { + queue_.reset(); + } + + template + auto runOnStrand(FunctionT&& function) -> std::invoke_result_t + { + using ResultT = std::invoke_result_t; + auto promise = std::make_shared>(); + auto future = promise->get_future(); + boost::asio::post( + *strand_, + [promise, function = std::forward(function)]() mutable + { + if constexpr (std::is_void_v) + { + function(); + promise->set_value(); + } + else + promise->set_value(function()); + } + ); + return future.get(); + } + + std::expected add(LocalOperationQueue::DeleteRequest request) + { + return runOnStrand( + [this, &request]() + { + return queue_->addDelete(std::move(request)); + } + ); + } + + /** + * @brief The completions once @p expected arrived, the queue working on its own; fails after a while. + */ + std::vector waitForCompletions(std::size_t expected) + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{20}; + while (std::chrono::steady_clock::now() < deadline) + { + if (auto finished = completions(); finished.size() >= expected) + return finished; + std::this_thread::sleep_for(std::chrono::milliseconds{2}); + } + ADD_FAILURE() << "Timed out waiting for " << expected << " completions"; + return completions(); + } + + /** + * @brief The completions after the queue had time to finish what it may run. + */ + std::vector completionsAfterSettling() + { + for (int cycle = 0; cycle < 20; ++cycle) + { + runOnStrand([]() {}); + std::this_thread::sleep_for(std::chrono::milliseconds{5}); + } + return completions(); + } + + std::vector completions() + { + std::scoped_lock lock{completionsMutex_}; + return completions_; + } + + static Paths::LocalPath local(std::filesystem::path const& path) + { + return Paths::LocalPath::make(path).value(); + } + + std::filesystem::path makeTree(std::string const& name) + { + const auto tree = root_.path() / name; + std::filesystem::create_directories(tree / "inner"); + std::ofstream{tree / "inner" / "a.txt"} << "a"; + std::ofstream{tree / "b.txt"} << "b"; + return tree; + } + + protected: + boost::asio::thread_pool pool_{2}; + std::shared_ptr> strand_{ + std::make_shared>(pool_.get_executor()) + }; + std::shared_ptr deletionPrevented_{std::make_shared(false)}; + std::shared_ptr queue_{}; + Ids::SessionId owner_{Ids::makeSessionId("local-operation-queue-test")}; + std::mutex completionsMutex_{}; + std::vector completions_{}; + Utility::TemporaryDirectory root_{temporaryRoot, true}; + }; + + TEST_F(LocalOperationQueueTests, AFlatDeleteRemovesExactlyItsTargets) + { + const auto root = root_.path() / "flat"; + std::filesystem::create_directories(root / "directory" / "inner"); + std::ofstream{root / "directory" / "listed.txt"} << "listed"; + std::ofstream{root / "directory" / "inner" / "unlisted.txt"} << "unlisted"; + std::ofstream{root / "file.txt"} << "file"; + std::ofstream{root / "keep.txt"} << "keep"; + + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE(add({ + .operationId = remove, + .targets = + {local(root / "directory"), + local(root / "directory" / "inner"), + local(root / "directory" / "listed.txt"), + local(root / "file.txt")}, + }) + .has_value()); + + const auto finished = waitForCompletions(1); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, remove); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(finished.front().failedEntries.size(), 2u) << "inner and directory are not empty"; + EXPECT_TRUE(std::filesystem::exists(root / "directory" / "inner" / "unlisted.txt")); + EXPECT_FALSE(std::filesystem::exists(root / "directory" / "listed.txt")); + EXPECT_FALSE(std::filesystem::exists(root / "file.txt")); + EXPECT_TRUE(std::filesystem::exists(root / "keep.txt")); + } + +#ifndef _WIN32 + TEST_F(LocalOperationQueueTests, ARecursiveDeleteRemovesTheTreeWithoutFollowingLinks) + { + const auto outside = root_.path() / "outside"; + std::filesystem::create_directories(outside); + std::ofstream{outside / "keep.txt"} << "keep"; + const auto tree = makeTree("recursive"); + std::filesystem::create_directory_symlink(outside, tree / "inner" / "out"); + + ASSERT_TRUE( + add({.operationId = Ids::generateOperationId(), .targets = {local(tree)}, .recursive = true}).has_value() + ); + + const auto finished = waitForCompletions(1); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(finished.front().failedEntries.empty()); + EXPECT_FALSE(std::filesystem::exists(tree)); + EXPECT_TRUE(std::filesystem::exists(outside / "keep.txt")); + } +#endif + + TEST_F(LocalOperationQueueTests, AHeldDeleteWaitsUntilItsOwnerContinuesOrEnds) + { + const auto first = makeTree("first"); + const auto second = makeTree("second"); + const auto unowned = makeTree("unowned"); + const auto firstId = Ids::generateOperationId(); + const auto secondId = Ids::generateOperationId(); + const auto unownedId = Ids::generateOperationId(); + ASSERT_TRUE( + add({.operationId = firstId, .targets = {local(first)}, .recursive = true, .owner = owner_, .held = true}) + .has_value() + ); + ASSERT_TRUE( + add({.operationId = secondId, .targets = {local(second)}, .recursive = true, .owner = owner_, .held = true}) + .has_value() + ); + ASSERT_TRUE(add({.operationId = unownedId, .targets = {local(unowned)}, .recursive = true}).has_value()); + + auto finished = waitForCompletions(1); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, unownedId) << "a held delete does not stop the others"; + EXPECT_TRUE(std::filesystem::exists(first / "b.txt")); + + runOnStrand( + [this]() + { + queue_->hold(owner_, false); + queue_->hold(owner_, true); + } + ); + EXPECT_EQ(completionsAfterSettling().size(), 1u); + + runOnStrand( + [this]() + { + queue_->hold(owner_, false); + } + ); + finished = waitForCompletions(2); + ASSERT_GE(finished.size(), 2u); + EXPECT_EQ(finished[1].operationId, firstId); + + // A delete of a session that ended runs on its own and is not held again. + ASSERT_TRUE(add({.operationId = Ids::generateOperationId(), + .targets = {local(makeTree("third"))}, + .recursive = true, + .owner = owner_, + .held = true}) + .has_value()); + runOnStrand( + [this]() + { + queue_->hold(owner_, true); + queue_->release(owner_); + queue_->hold(owner_, true); + } + ); + finished = waitForCompletions(4); + EXPECT_EQ(finished.size(), 4u); + EXPECT_FALSE(std::filesystem::exists(second)); + EXPECT_FALSE(std::filesystem::exists(root_.path() / "third")); + } + + TEST_F(LocalOperationQueueTests, APriorityDeleteIsNeverHeldAndRunsFirst) + { + const auto queued = makeTree("queued"); + const auto priority = makeTree("priority"); + const auto queuedId = Ids::generateOperationId(); + const auto priorityId = Ids::generateOperationId(); + runOnStrand( + [&]() + { + EXPECT_TRUE(queue_->addDelete({.operationId = queuedId, .targets = {local(queued)}, .recursive = true}) + .has_value()); + EXPECT_TRUE(queue_ + ->addDelete({ + .operationId = priorityId, + .targets = {local(priority)}, + .recursive = true, + .mode = SharedData::OperationMode::PriorityQueued, + .owner = owner_, + .held = true, + }) + .has_value()); + } + ); + + const auto finished = waitForCompletions(2); + ASSERT_EQ(finished.size(), 2u); + EXPECT_EQ(finished[0].operationId, priorityId); + EXPECT_EQ(finished[1].operationId, queuedId); + } + + TEST_F(LocalOperationQueueTests, ACanceledDeleteEndsAsCanceledAndKeepsWhatIsLeft) + { + const auto tree = makeTree("canceled"); + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE( + add({.operationId = remove, .targets = {local(tree)}, .recursive = true, .owner = owner_, .held = true}) + .has_value() + ); + runOnStrand( + [&]() + { + queue_->cancel(remove); + queue_->cancel(remove); + } + ); + + const auto finished = completionsAfterSettling(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, remove); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Canceled); + EXPECT_TRUE(std::filesystem::exists(tree / "b.txt")); + } + + TEST_F(LocalOperationQueueTests, ADeleteOfARootOrTheHomeDirectoryIsRefused) + { + const auto file = root_.path() / "survivor.txt"; + std::ofstream{file} << "survivor"; + const auto root = local(std::filesystem::path{file}.root_path()); + const auto homes = localHomeDirectories(); + ASSERT_FALSE(homes.empty()); + + for (auto const& target : {root, homes.front()}) + { + const auto result = add({.operationId = Ids::generateOperationId(), .targets = {local(file), target}}); + ASSERT_FALSE(result.has_value()) << target.toUtf8(); + EXPECT_EQ(result.error().type, Operation::ErrorType::InvalidPath); + } + EXPECT_FALSE(add({.operationId = Ids::generateOperationId()}).has_value()) << "no targets"; + EXPECT_TRUE(completionsAfterSettling().empty()); + EXPECT_TRUE(std::filesystem::exists(file)); + } + + TEST_F(LocalOperationQueueTests, AnExplorerDeleteGoesWithTheContentUnlessRefused) + { + const auto tree = makeTree("explorer"); + const auto explorerDelete = [this](std::vector const& paths) + { + return runOnStrand( + [this, &paths]() + { + return queue_->addExplorerDelete(Ids::generateOperationId(), paths, std::nullopt, false); + } + ); + }; + + deletionPrevented_->store(true); + const auto prevented = explorerDelete({Paths::nativeToUtf8(tree)}); + ASSERT_FALSE(prevented.has_value()); + EXPECT_EQ(prevented.error(), "backend.rpcFilesystem.deletionPrevented"); + deletionPrevented_->store(false); + + const auto relative = explorerDelete({Paths::nativeToUtf8(tree), "relative/path"}); + ASSERT_FALSE(relative.has_value()); + EXPECT_EQ(Utility::parseLocalizedMessage(relative.error()).key, "backend.rpcFilesystem.removeRefused"); + EXPECT_TRUE(completionsAfterSettling().empty()); + EXPECT_TRUE(std::filesystem::exists(tree / "inner" / "a.txt")); + + ASSERT_TRUE(explorerDelete({Paths::nativeToUtf8(tree)}).has_value()); + const auto finished = waitForCompletions(1); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_FALSE(std::filesystem::exists(tree)); + } +} diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index f8dbd037..266494bb 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -50,6 +50,7 @@ namespace Test queue_->cancelAll(); }); queue_.reset(); + localQueue_.reset(); } std::shared_ptr makeQueue() @@ -61,16 +62,26 @@ namespace Test *env.window, *env.hub, Persistence::SftpOptions{}, - Ids::makeSessionId("operation-queue-test"), + sessionId_, 2 ); - queue_->setCompletionObserver( - [this](SharedData::OperationCompleted const& completed) - { - std::scoped_lock lock{completionsMutex_}; - completions_.push_back(completed); - } - ); + const auto observe = [this](SharedData::OperationCompleted const& completed) + { + std::scoped_lock lock{completionsMutex_}; + completions_.push_back(completed); + }; + queue_->setCompletionObserver(observe); + if (!localQueue_) + { + localQueue_ = std::make_shared( + queuePool_.get_executor(), + strand_, + *env.window, + *env.hub, + std::make_shared(false) + ); + localQueue_->setCompletionObserver(observe); + } return queue_; } @@ -100,10 +111,15 @@ namespace Test return future.get(); } + /** + * @brief Pauses the queue and holds its local deletes, as the session does. + */ void setPaused(bool pause) { queue_->paused(pause); - runOnStrand([]() {}); + runOnStrand([this, pause]() { + localQueue_->hold(sessionId_, pause); + }); } /** @@ -269,7 +285,10 @@ namespace Test boost::asio::thread_pool queuePool_{2}; std::shared_ptr> strand_{ std::make_shared>(queuePool_.get_executor())}; + Ids::SessionId sessionId_{Ids::makeSessionId("operation-queue-test")}; std::shared_ptr queue_{}; + /** @brief Runs the local deletes of sync plans, on the strand of the queue. */ + std::shared_ptr localQueue_{}; std::mutex completionsMutex_{}; std::vector completions_{}; Utility::TemporaryDirectory localDirectory_{temporaryRoot, true}; @@ -1280,7 +1299,7 @@ namespace Test }; const auto result = runOnStrand([&]() { - return queue->executeSyncPlan(*sftp, session, std::move(plan), request); + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); }); EXPECT_EQ(result.uploads, (std::vector{"up.txt", "emptyA/emptyB"})); EXPECT_EQ(result.downloads, (std::vector{"down.txt", "newLocal/dir"})); @@ -1293,6 +1312,9 @@ namespace Test ); EXPECT_EQ(result.failed, (std::vector{"CON"})); + for (int cycle = 0; cycle < 10; ++cycle) + runOnStrand([]() {}); + EXPECT_TRUE(std::filesystem::exists(localRoot / "localGone.txt")) << "the paused queue holds the local delete"; setPaused(false); workUntilCompletions(4); std::set finishedIds;