diff --git a/CMakeLists.txt b/CMakeLists.txt index cfa040cf..27440be4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,9 @@ project(nui-sftp VERSION "${PROJECT_VERSION}" LANGUAGES CXX) set(SPDLOG_FMT_EXTERNAL ON CACHE BOOL "Use an external fmt library (provide it manually)" FORCE) set(SPDLOG_BUILD_SHARED OFF CACHE BOOL "Dont build shared spdlog" FORCE) +if (WIN32) + set(SPDLOG_WCHAR_FILENAMES ON CACHE BOOL "Open log files by their wide name" FORCE) +endif() set(NUI_FILE_EXPLORER_MEMORY64 ON CACHE BOOL "Enable MEMORY64 for nui-file-explorer" FORCE) set(JSON_Diagnostics ON CACHE BOOL "Enable json diagnostics" FORCE) set(NUI_JSON_DIAGNOSTICS ON CACHE BOOL "Enable json diagnostics for nlohmann_json through nui" FORCE) @@ -88,6 +91,9 @@ if (WIN32) set_target_properties(${PROJECT_NAME} PROPERTIES WIN32_EXECUTABLE TRUE) include("${CMAKE_CURRENT_LIST_DIR}/_cmake/windows/icon.cmake") file(TO_CMAKE_PATH "${NUI_SFTP_ICO}" NUI_SFTP_ICO_FORWARD) + # activeCodePage UTF-8: narrow Windows APIs in third-party code (libssh, yaml-cpp, spdlog) read UTF-8. + set(NUI_SFTP_MANIFEST "${CMAKE_CURRENT_LIST_DIR}/_cmake/windows/nui-sftp.manifest") + file(TO_CMAKE_PATH "${NUI_SFTP_MANIFEST}" NUI_SFTP_MANIFEST_FORWARD) configure_file( "${CMAKE_CURRENT_LIST_DIR}/_cmake/windows/nui-sftp.rc.in" "${CMAKE_BINARY_DIR}/generated/nui-sftp.rc" @@ -96,7 +102,7 @@ if (WIN32) target_sources(${PROJECT_NAME} PRIVATE "${CMAKE_BINARY_DIR}/generated/nui-sftp.rc") set_source_files_properties( "${CMAKE_BINARY_DIR}/generated/nui-sftp.rc" - PROPERTIES OBJECT_DEPENDS "${NUI_SFTP_ICO}" + PROPERTIES OBJECT_DEPENDS "${NUI_SFTP_ICO};${NUI_SFTP_MANIFEST}" ) add_dependencies(${PROJECT_NAME} nui-sftp-icon) elseif (APPLE AND NOT EMSCRIPTEN) @@ -199,6 +205,10 @@ else() add_subdirectory(shared_data/benchmark/shared_data) add_subdirectory(tar-archive/test/tar_archive) add_subdirectory(command-store/test/command-store) + add_test( + NAME NarrowPathConversions + COMMAND "${CMAKE_COMMAND}" "-DSOURCE_DIR=${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/_cmake/check_narrow_path_text.cmake" + ) endif() if (NUI_SFT_ENABLE_LANGUAGE_TOOLING) diff --git a/_cmake/check_narrow_path_text.cmake b/_cmake/check_narrow_path_text.cmake new file mode 100644 index 00000000..f22cf7ab --- /dev/null +++ b/_cmake/check_narrow_path_text.cmake @@ -0,0 +1,45 @@ +# Fails when native first-party sources call string() or generic_string(), which convert a path through the ANSI code +# page on Windows and throw outside it. Paths::nativeToUtf8 and Paths::nativeFromUtf8 convert explicitly. Frontend +# sources are not checked, a path there is UTF-8 already. Run with -DSOURCE_DIR=. + +set(roots + backend + command-store/include + command-store/source + log + nui-file-explorer/source/nui-file-explorer/support + paths/include + persistence/include + persistence/source + shared_data/include + shared_data/source + ssh/include + ssh/source + tar-archive/include + tar-archive/source + utility/include + utility/source +) + +set(found "") +foreach(root IN LISTS roots) + file(GLOB_RECURSE sources "${SOURCE_DIR}/${root}/*.cpp" "${SOURCE_DIR}/${root}/*.hpp" "${SOURCE_DIR}/${root}/*.mm") + foreach(source IN LISTS sources) + file(RELATIVE_PATH relative "${SOURCE_DIR}" "${source}") + if (relative MATCHES "(^|/)test/") + continue() + endif() + file(STRINGS "${source}" lines REGEX "\\.(generic_)?string\\(\\)") + foreach(line IN LISTS lines) + if (NOT line MATCHES "^[ \t]*//") + string(STRIP "${line}" stripped) + string(APPEND found "\n ${relative}: ${stripped}") + endif() + endforeach() + endforeach() +endforeach() + +if (found) + message(FATAL_ERROR "Narrow path conversions, use Paths::nativeToUtf8 or Paths::nativeFromUtf8:${found}") +endif() +message(STATUS "No narrow path conversions in native sources.") diff --git a/_cmake/windows/nui-sftp.manifest b/_cmake/windows/nui-sftp.manifest new file mode 100644 index 00000000..f2fff698 --- /dev/null +++ b/_cmake/windows/nui-sftp.manifest @@ -0,0 +1,8 @@ + + + + + UTF-8 + + + diff --git a/_cmake/windows/nui-sftp.rc.in b/_cmake/windows/nui-sftp.rc.in index d975a860..715a13a9 100644 --- a/_cmake/windows/nui-sftp.rc.in +++ b/_cmake/windows/nui-sftp.rc.in @@ -4,6 +4,9 @@ // (Windows picks the lowest-numbered ICON resource). 1 ICON "@NUI_SFTP_ICO_FORWARD@" +// The application manifest: CREATEPROCESS_MANIFEST_RESOURCE_ID (1) of type RT_MANIFEST (24). +1 24 "@NUI_SFTP_MANIFEST_FORWARD@" + VS_VERSION_INFO VERSIONINFO FILEVERSION @SEMVER_MAJOR@,@SEMVER_MINOR@,@SEMVER_PATCH@,0 PRODUCTVERSION @SEMVER_MAJOR@,@SEMVER_MINOR@,@SEMVER_PATCH@,0 diff --git a/backend/include/backend/dropped_entry.hpp b/backend/include/backend/dropped_entry.hpp index 20dc1653..cc60630b 100644 --- a/backend/include/backend/dropped_entry.hpp +++ b/backend/include/backend/dropped_entry.hpp @@ -6,7 +6,9 @@ #include /** - * @brief Describes a file dropped onto the window from the desktop, with just enough detail for the transfer. - * @param path Local path of the dropped file or directory, nothing when it is not an absolute path. + * @brief Describes a file dropped onto the window from the desktop, with just enough detail for the transfer. A + * dropped link or junction is a symlink, never what it points to. + * @param path Local path of the dropped item. Nothing, and a log line, when it is not an absolute path or cannot be + * inspected. */ std::optional droppedItemFromPath(std::filesystem::path const& path); diff --git a/backend/include/backend/existence_check.hpp b/backend/include/backend/existence_check.hpp index d985d39a..3c906c36 100644 --- a/backend/include/backend/existence_check.hpp +++ b/backend/include/backend/existence_check.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -14,8 +15,15 @@ std::expected, std::string> localPathsExist(std::vector const& paths); /** - * @brief Whether each of @p paths exists on the server, with the same rules as localPathsExist. Few paths are - * checked with one lstat each, more with one listing per parent directory. + * @brief How many queried children of one remote directory are checked by listing it instead of an lstat each. + */ +constexpr std::size_t remoteListingThreshold = 10; + +/** + * @brief Whether each of @p paths exists on the server, with the same rules as localPathsExist. The children of a + * directory with at least @ref remoteListingThreshold of them queried are checked with one listing of it + * (names only), the others with one lstat each. A directory that cannot be listed for its permissions has its + * children looked up one by one, and so has a name only a case or normalization variant of is listed. */ std::expected, std::string> remotePathsExist( SecureShell::SftpSession& session, diff --git a/backend/include/backend/local_delete_guard.hpp b/backend/include/backend/local_delete_guard.hpp index 55120717..7f33e7a5 100644 --- a/backend/include/backend/local_delete_guard.hpp +++ b/backend/include/backend/local_delete_guard.hpp @@ -2,55 +2,33 @@ #include -#include - #include -#include -#include #include -#include +#include /** - * @brief Refuses a root directory and @p home or a directory containing it. + * @brief Refuses an unset path, a root directory, and each of @p homes or a directory containing one. + * + * Paths are compared as text, and without ASCII case for Windows paths and on macOS, whose filesystems fold case. + * A target that is an existing directory (not a link to one) is also compared by file identity with every home and + * each of its ancestors, which catches other spellings of the same directory: case beyond ASCII, 8.3 names, NFD + * names and ancestors reached through a symlink. */ -inline std::expected -refuseProtectedLocalDelete(Paths::LocalPath const& target, std::optional const& home) -{ - if (target.empty()) - return std::unexpected(std::string{"no path"}); - if (target.isRoot()) - return std::unexpected(fmt::format("'{}' is a root directory", target.toUtf8())); - if (home && home->isWithin(target)) - return std::unexpected(fmt::format("'{}' contains the home directory", target.toUtf8())); - return {}; -} +std::expected +refuseProtectedLocalDelete(Paths::LocalPath const& target, std::vector const& homes); /** - * @brief A local path that may be deleted, or the reason to refuse it: not absolute, containing '..', a root - * directory, or @p home or a directory containing it. + * @brief A local path that may be deleted, or the reason to refuse it: not a valid absolute path, containing '..' + * (also as '\\..' on Windows) or refused by @ref refuseProtectedLocalDelete. */ -inline std::expected -localDeleteTarget(std::string const& utf8, std::optional const& home) -{ - // LocalPath resolves ".." lexically, which goes elsewhere when a directory on the way is a symlink. - for (auto const segment : std::views::split(std::string_view{utf8}, '/')) - { - for (auto const part : std::views::split(segment, '\\')) - { - if (std::string_view{part.begin(), part.end()} == "..") - return std::unexpected(fmt::format("'{}' contains '..'", utf8)); - } - } - - auto target = Paths::LocalPath::fromUtf8(utf8); - if (!target) - return std::unexpected(fmt::format("'{}' {}", utf8, Paths::toString(target.error().reason))); - if (auto allowed = refuseProtectedLocalDelete(*target, home); !allowed) - return std::unexpected(std::move(allowed).error()); - return *std::move(target); -} +std::expected +localDeleteTarget(std::string const& utf8, std::vector const& homes); /** - * @brief The home directory of the user running the backend, empty when it cannot be determined. + * @brief The home directories of the user running the backend, which the delete guards protect, each once. + * + * On Windows the profile folder (FOLDERID_Profile) and the USERPROFILE and HOME variables, elsewhere the HOME + * variable and the home of the passwd entry. When none can be determined a warning is logged and only roots stay + * protected. Never throws. */ -std::optional localHomeDirectory(); +std::vector localHomeDirectories(); diff --git a/backend/include/backend/local_operation_queue.hpp b/backend/include/backend/local_operation_queue.hpp new file mode 100644 index 00000000..d5df04e9 --- /dev/null +++ b/backend/include/backend/local_operation_queue.hpp @@ -0,0 +1,152 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @brief Runs local deletes apart from the SSH sessions, so closing a tab or losing a connection never stops one. + * + * The deletes run one after another on the strand the queue shares with the sessions, a work step at a time, and + * report to the frontend under "LocalOperationQueue::...". A delete a session started waits while the queue of that + * session is paused, for as long as the session exists. + */ +class LocalOperationQueue + : public RpcHelper::StrandRpc + , public std::enable_shared_from_this +{ + public: + using OperationCompleted = SharedData::OperationCompleted; + + struct DeleteRequest + { + Ids::OperationId operationId{}; + std::vector targets{}; + /** @brief See LocalDeleteOperation::LocalDeleteOperationOptions::recursive. */ + bool recursive{false}; + /** @brief The card refreshes the local listing when the delete ends. */ + bool insertRefresh{false}; + /** @brief A priority delete runs before the others and is never held. */ + SharedData::OperationMode mode{SharedData::OperationMode::Queued}; + /** @brief The session whose paused queue holds the delete, none for a delete without a session. */ + std::optional owner{}; + /** @brief The queue of @ref owner is paused, so the delete starts held. */ + bool held{false}; + /** @brief See LocalDeleteOperation::LocalDeleteOperationOptions::comparedStates. */ + std::vector comparedStates{}; + }; + + LocalOperationQueue( + boost::asio::any_io_executor executor, + std::shared_ptr> strand, + Nui::Window& wnd, + Nui::RpcHub& hub, + std::shared_ptr localDeletionPrevented + ); + ~LocalOperationQueue() override = default; + LocalOperationQueue(LocalOperationQueue const&) = delete; + LocalOperationQueue& operator=(LocalOperationQueue const&) = delete; + LocalOperationQueue(LocalOperationQueue&&) = delete; + LocalOperationQueue& operator=(LocalOperationQueue&&) = delete; + + /** + * @brief Handles "LocalOperationQueue::addDelete" (operation id, local paths), the delete of explorer items + * without a session, and "LocalOperationQueue::cancel" (operation id). + */ + void registerRpc(); + + /** + * @brief Queues a delete with one card, labelled with the directory its targets lie in. Refused as a whole when + * one target is a root directory or contains a home directory. Assumed in strand. + */ + std::expected addDelete(DeleteRequest request); + + /** + * @brief Queues the delete of the explorer items @p pathsUtf8 with their content, as @ref addDelete with a card + * that refreshes the local listing. Refused when deletion is prevented in the settings or a path is not + * an item that may be deleted. Assumed in strand. + * + * @param owner The session whose explorer asks, none without one. + * @param held The queue of @p owner is paused. + * @return A message for Utility::localizedMessage or the language file saying why nothing was queued. + */ + std::expected addExplorerDelete( + Ids::OperationId operationId, + std::vector const& pathsUtf8, + std::optional owner, + bool held + ); + + /** + * @brief Holds the deletes of @p owner, or lets them run again. Assumed in strand. + */ + void hold(Ids::SessionId const& owner, bool held); + + /** + * @brief The deletes of @p owner run on their own from now on, never held again. Assumed in strand. + */ + void release(Ids::SessionId const& owner); + + /** + * @brief Ends the delete @p operationId as canceled and tells the frontend. Assumed in strand. + */ + void cancel(Ids::OperationId const& operationId); + + /** + * @brief Works the first delete that is not held for one step and tells the frontend when it ends. Assumed in + * strand; the queue drives itself, this is public for tests. + * + * @return True while a delete that is not held is left. + */ + bool work(); + + /** + * @brief Test hook: @p observer sees every completion right before it is sent to the frontend. + */ + void setCompletionObserver(std::function observer); + + private: + struct QueuedDelete + { + Ids::OperationId operationId; + std::unique_ptr operation; + std::optional owner; + bool held; + bool priority; + }; + + /** + * @brief Posts the next work step unless one is posted already. + */ + void schedule(); + + void complete( + QueuedDelete const& queued, + SharedData::OperationCompletionReason reason, + std::optional error + ); + std::string rpcName(std::string_view event) const; + + std::shared_ptr localDeletionPrevented_; + std::deque deletes_{}; + bool scheduled_{false}; + std::function completionObserver_{}; +}; diff --git a/backend/include/backend/local_removal.hpp b/backend/include/backend/local_removal.hpp new file mode 100644 index 00000000..62cc55e7 --- /dev/null +++ b/backend/include/backend/local_removal.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +/** + * @brief What @ref removeLocalItems removed. + */ +struct LocalRemoval +{ + std::uintmax_t removed{0}; + /** @brief "path: reason" for every item that could not be removed, by the item's own path. */ + std::vector failures{}; +}; + +/** + * @brief A path that @ref removeLocalItems refused, so it removed nothing. + */ +struct LocalRemovalRefusal +{ + std::string path; + std::string reason; +}; + +/** + * @brief Removes each of the local items @p pathsUtf8 by itself, so a directory only when it is empty, without + * following links or junctions. An item that cannot be removed does not stop the others. Nothing is removed + * when one of the paths is refused by localDeleteTarget. A directory with its content goes through the + * LocalOperationQueue instead. + */ +std::expected +removeLocalItems(std::span pathsUtf8, std::vector const& homes); diff --git a/backend/include/backend/process/json_process_io.hpp b/backend/include/backend/process/json_process_io.hpp index f67ecafd..8d6ba5ee 100644 --- a/backend/include/backend/process/json_process_io.hpp +++ b/backend/include/backend/process/json_process_io.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -90,7 +91,7 @@ class JsonFraming } catch (std::exception const& exc) { - onResult(std::unexpected(std::string{"JSON parse error: "} + exc.what())); + onResult(std::unexpected(fmt::format("JSON parse error: {}", Utility::errorMessage(exc)))); return false; } } @@ -168,7 +169,7 @@ class JsonProcessIo : public JsonFraming boost::asio::buffer(*frame), [this, frame](boost::system::error_code err, std::size_t bytes) { - spdlog::debug("Sent {} byte(s), error: {}", bytes, err ? err.message() : "none"); + spdlog::debug("Sent {} byte(s), error: {}", bytes, err ? Utility::errorMessage(err) : "none"); if (err) { // The callbacks may own this object, so nothing here is touched after they ran. @@ -194,8 +195,8 @@ class JsonProcessIo : public JsonFraming { if (ec) { - spdlog::error("Error reading from input: {}", ec.message()); - onResult_(std::unexpected(ec.message())); + spdlog::error("Error reading from input: {}", Utility::errorMessage(ec)); + onResult_(std::unexpected(Utility::errorMessage(ec))); return; } readBufferAccum_.append(readBufferCurrent_, 0, bytes_transferred); diff --git a/backend/include/backend/rpc_filesystem.hpp b/backend/include/backend/rpc_filesystem.hpp index 2fed420a..c50a2dc9 100644 --- a/backend/include/backend/rpc_filesystem.hpp +++ b/backend/include/backend/rpc_filesystem.hpp @@ -19,7 +19,6 @@ class RpcFilesystem : public RpcHelper::StrandRpc ); private: - void registerRemove(); void registerRemoveMultiple(); void registerRename(); void registerListFiles(); diff --git a/backend/include/backend/rpc_helper.hpp b/backend/include/backend/rpc_helper.hpp index fd57b0bc..a70de7d6 100644 --- a/backend/include/backend/rpc_helper.hpp +++ b/backend/include/backend/rpc_helper.hpp @@ -1,9 +1,12 @@ #pragma once +#include +#include +#include + #include #include #include -#include #include #include #include @@ -14,7 +17,10 @@ #include #include #include +#include #include +#include +#include // TODO: split and document this file! namespace RpcHelper @@ -56,6 +62,71 @@ namespace RpcHelper template using pop_front_t = typename pop_front::type; + + /** + * @brief @p argument for a second try of a call: numbers as they are, everything else as JSON with U+FFFD + * for every byte that is not UTF-8. + */ + template + auto withValidUtf8Argument(T const& argument) + { + if constexpr (std::is_arithmetic_v) + return argument; + else + return Utility::withValidUtf8(nlohmann::json(argument)); + } + } + + /** + * @brief Calls @p name in the frontend like Nui::RpcHub::callRemote, but never throws. Text that is not UTF-8, + * which the JSON dump refuses, is sent again with U+FFFD for every invalid byte; any other failure is + * logged. + */ + template + void callRemoteSafely(Nui::RpcHub const& hub, std::string const& name, Args const&... arguments) noexcept + { + const auto logFailure = [&name](std::exception const& error) noexcept + { + try + { + Log::error("Failed to call '{}' in the frontend: {}", name, Utility::errorMessage(error)); + } + catch (...) + {} + }; + constexpr int invalidUtf8 = 316; + try + { + hub.callRemote(name, arguments...); + return; + } + catch (nlohmann::json::type_error const& error) + { + if (error.id != invalidUtf8) + return logFailure(error); + } + catch (std::exception const& error) + { + return logFailure(error); + } + + try + { + Log::warn("The call of '{}' in the frontend carries text that is not UTF-8, it is sent replaced.", name); + hub.callRemote(name, Detail::withValidUtf8Argument(arguments)...); + } + catch (std::exception const& error) + { + logFailure(error); + } + } + + /** + * @brief @ref callRemoteSafely for a JSON value, which may be written as a braced list. + */ + inline void callRemoteSafely(Nui::RpcHub const& hub, std::string const& name, nlohmann::json const& json) noexcept + { + callRemoteSafely(hub, name, json); } class RpcInCorrectThread @@ -77,14 +148,7 @@ namespace RpcHelper wnd_->runInJavascriptThread( [this, json = std::move(json)]() { - try - { - hub_->callRemote(responseId_, json); - } - catch (const std::exception& e) - { - Log::error("Failed to call rpc respond '{}': {}", responseId_, e.what()); - } + callRemoteSafely(*hub_, responseId_, json); } ); } @@ -103,17 +167,48 @@ namespace RpcHelper , hub_{&hub} , responseId_{responseId} {} - RpcOnce(RpcOnce&&) = default; - RpcOnce& operator=(RpcOnce&&) = default; + /** + * @brief Takes over the reply of @p other, which counts as answered from then on, so a reply through it is + * dropped with a warning instead of going out under an empty id. + */ + RpcOnce(RpcOnce&& other) noexcept + : wnd_{other.wnd_} + , hub_{other.hub_} + , responseId_{std::move(other.responseId_)} + , called_{std::exchange(other.called_, true)} + {} + RpcOnce& operator=(RpcOnce&& other) noexcept + { + if (this != &other) + { + wnd_ = other.wnd_; + hub_ = other.hub_; + responseId_ = std::move(other.responseId_); + called_ = std::exchange(other.called_, true); + } + return *this; + } RpcOnce(RpcOnce const&) = delete; RpcOnce& operator=(RpcOnce const&) = delete; ~RpcOnce() = default; + /** + * @brief The reply was sent, or handed to another RpcOnce. + */ + bool answered() const noexcept + { + return called_; + } + void operator()(nlohmann::json&& json) const { if (called_) { - Log::warn("RPC response with id '{}' already sent. Not sending: {}", responseId_, json.dump(4)); + Log::warn( + "RPC response with id '{}' already sent. Not sending: {}", + responseId_, + json.dump(4, ' ', false, nlohmann::json::error_handler_t::replace) + ); return; } called_ = true; @@ -121,14 +216,7 @@ namespace RpcHelper wnd_->runInJavascriptThread( [hub = hub_, responseId = std::move(responseId_), json = std::move(json)]() { - try - { - hub->callRemote(responseId, json); - } - catch (const std::exception& e) - { - Log::error("Failed to call rpc respond '{}': {}", responseId, e.what()); - } + callRemoteSafely(*hub, responseId, json); } ); } @@ -145,14 +233,7 @@ namespace RpcHelper wnd_->runInJavascriptThread( [hub = hub_, responseId = std::move(responseId_), message]() { - try - { - hub->callRemote(responseId, {{"success", false}, {"error", message}}); - } - catch (const std::exception& e) - { - Log::error("Failed to call rpc respond '{}': {}", responseId, e.what()); - } + callRemoteSafely(*hub, responseId, {{"success", false}, {"error", message}}); } ); } @@ -247,7 +328,7 @@ namespace RpcHelper } catch (const std::exception& e) { - (*rpcOnce)({{"error", e.what()}}); + (*rpcOnce)({{"error", Utility::errorMessage(e)}}); } }; } @@ -365,7 +446,7 @@ namespace RpcHelper [func = std::forward(func)](auto const& ec) { if (ec) - Log::info("Timer canceled in within_strand_do_delayed: {}", ec.message()); + Log::info("Timer canceled in within_strand_do_delayed: {}", Utility::errorMessage(ec)); // Run func anyway, if its because of a shutdown, it will stop early. func(); diff --git a/backend/include/backend/rpc_path_parameters.hpp b/backend/include/backend/rpc_path_parameters.hpp new file mode 100644 index 00000000..d0804189 --- /dev/null +++ b/backend/include/backend/rpc_path_parameters.hpp @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace RpcHelper +{ + /** + * @brief A remote path from the frontend, or the message to reply with when it is not a valid absolute path. + */ + std::expected parseRemotePath(std::string const& path); + + /** + * @brief The remote paths from the frontend, or the message about the first one that is not a valid absolute path. + */ + std::expected, std::string> parseRemotePaths(std::vector const& paths); + + /** + * @brief A local path from the frontend, or the message to reply with when it is not a valid absolute path. + */ + std::expected parseLocalPath(std::string const& path); + + /** + * @brief The local paths from the frontend, or the message about the first one that is not a valid absolute path. + */ + std::expected, std::string> parseLocalPaths(std::vector const& paths); + + /** + * @brief A JSON request from the frontend, or the message to reply with when it does not parse. Paths are + * validated while parsing. + * + * @param what Names the request in the message, e.g. "bulk download". + */ + template + std::expected parseRequest(nlohmann::json const& json, std::string_view what) + { + try + { + return json.get(); + } + catch (std::exception const& exception) + { + return std::unexpected(fmt::format("Invalid {}: {}", what, Utility::errorMessage(exception))); + } + } +} diff --git a/backend/include/backend/session.hpp b/backend/include/backend/session.hpp index 35d5571b..df21957e 100644 --- a/backend/include/backend/session.hpp +++ b/backend/include/backend/session.hpp @@ -4,14 +4,19 @@ #include #include #include +#include #include #include #include -#include -#include #include +#include +#include +#include #include +#include +#include +#include /** * @brief This session is the implementation equivalent of one tab in the UI. @@ -36,7 +41,8 @@ class Session Nui::Window& wnd, Nui::RpcHub& hub, Persistence::SftpOptions const& sftpOptions, - BulkResumeRegistry* bulkResumeRegistry + BulkResumeRegistry* bulkResumeRegistry, + std::shared_ptr localOperationQueue ); Session(const Session&) = delete; @@ -205,6 +211,13 @@ class Session void registerRpcSftpAddBulkDownloadOperation(); void registerRpcSftpAddBulkUploadOperation(); void registerRpcSftpAddBulkDeleteOperation(); + + /** + * @brief Handles "Session::{id}::addLocalDelete" (operation id, local paths): queues the recursive delete of the + * local items in the local operation queue, held while the queue of this session is paused, see + * LocalOperationQueue::addExplorerDelete. + */ + void registerRpcAddLocalDeleteOperation(); void registerRpcSftpExistsBatch(); void registerRpcSftpOpenSyncSession(); void registerRpcSftpRecomputeSyncDiff(); @@ -218,65 +231,121 @@ class Session void removeSftpChannel(Ids::ChannelId channelId); - template - void withChannelDo(Ids::ChannelId channelId, FunctionT&& func, ReplyCallable&& reply) + /** + * @brief Runs @p func on the strand with the channel @p channelId of @p channels, as (reply, channel) or + * (reply, session, channel). @p reply is always answered: when the session is gone, when there is no such + * channel, and with the message of an exception @p func throws. + */ + template + void withLockedChannelDo( + std::unordered_map, Ids::IdHash> Session::* channels, + void (Session::*remove)(Ids::ChannelId), + Ids::ChannelId channelId, + FunctionT&& func, + RpcHelper::RpcOnce&& reply + ) { within_strand_do( - [this, + [weak = weak_from_this(), + channels, + remove, channelId = std::move(channelId), func = std::forward(func), - reply = std::forward(reply)]() mutable + reply = std::move(reply)]() mutable { - if (auto iter = channels_.find(channelId); iter != channels_.end()) + auto self = weak.lock(); + if (!self) + return reply.error("Session no longer exists"); + auto& map = (*self).*channels; + const auto iter = map.find(channelId); + if (iter == map.end()) { - if (auto channel = iter->second.lock(); channel) - { - func(std::move(reply), std::move(channel)); - } + Log::error("No channel found with id: {}", channelId.value()); + return reply.error("No channel found with id"); + } + auto channel = iter->second.lock(); + if (!channel) + { + Log::error("Failed to lock channel with id: {}", channelId.value()); + ((*self).*remove)(channelId); + return reply.error("Failed to lock channel"); + } + try + { + if constexpr ( + std::is_invocable_v> + ) + func(std::move(reply), *self, std::move(channel)); else - { - Log::error("Failed to lock channel with id: {}", channelId.value()); - removeChannel(channelId); - return reply({{"error", "Failed to lock channel"}}); - } + func(std::move(reply), std::move(channel)); } - else + catch (std::exception const& exception) { - Log::error("No channel found with id: {}", channelId.value()); - return reply({{"error", "No channel found with id"}}); + Log::error( + "A call on channel '{}' failed: {}", channelId.value(), Utility::errorMessage(exception) + ); + reply.error(Utility::errorMessage(exception)); + } + catch (...) + { + Log::error("A call on channel '{}' failed with an unknown exception", channelId.value()); + reply.error("Unknown error"); } } ); } - template - void withSftpChannelDo(Ids::ChannelId channelId, FunctionT&& func, ReplyCallable&& reply) + template + void withChannelDo(Ids::ChannelId channelId, FunctionT&& func, RpcHelper::RpcOnce&& reply) { - within_strand_do( - [this, - channelId = std::move(channelId), - func = std::forward(func), - reply = std::forward(reply)]() mutable + withLockedChannelDo( + &Session::channels_, + &Session::removeChannel, + std::move(channelId), + std::forward(func), + std::move(reply) + ); + } + + template + void withSftpChannelDo(Ids::ChannelId channelId, FunctionT&& func, RpcHelper::RpcOnce&& reply) + { + withLockedChannelDo( + &Session::sftpChannels_, + &Session::removeSftpChannel, + std::move(channelId), + std::forward(func), + std::move(reply) + ); + } + + /** + * @brief Queues operations on the SFTP channel @p channelId with @p queue, called on the strand with the operation + * queue and the channel, and answers @p reply with the JSON it returns or with the message it fails with, + * which the log names as the failure to queue @p description. A queued request resets the queue throttle. + */ + template + void queueOnChannel(RpcHelper::RpcOnce&& reply, Ids::ChannelId channelId, std::string description, QueueT&& queue) + { + withSftpChannelDo( + std::move(channelId), + [description = std::move(description), queue = std::forward(queue)]( + RpcHelper::RpcOnce&& reply, Session& self, std::shared_ptr const& channel + ) mutable { - if (auto iter = sftpChannels_.find(channelId); iter != sftpChannels_.end()) + if (!self.operationQueue_) + return reply.error("Session no longer exists"); + std::expected queued = queue(*self.operationQueue_, *channel); + if (!queued) { - if (auto channel = iter->second.lock(); channel) - { - func(std::move(reply), std::move(channel)); - } - else - { - Log::error("Failed to lock channel with id: {}", channelId.value()); - removeSftpChannel(channelId); - return reply({{"error", "Failed to lock channel"}}); - } + Log::error("Failed to queue {}: {}", description, queued.error()); + return reply.error(queued.error()); } - else - { - Log::error("No channel found with id: {}", channelId.value()); - return reply({{"error", "No channel found with id"}}); - } - } + Log::info("Queued {}", description); + self.resetQueueThrottle(); + reply(*std::move(queued)); + }, + std::move(reply) ); } @@ -297,4 +366,8 @@ class Session BulkResumeRegistry* bulkResumeRegistry_{}; std::unordered_map, Ids::IdHash> channels_{}; std::unordered_map, Ids::IdHash> sftpChannels_{}; + /** @brief The temp file suffix of uploads, which names where a replaced item is kept during a rename. */ + std::string remoteTempFileSuffix_; + /** @brief Runs the local deletes of this session, which go on when it ends. */ + std::shared_ptr localOperationQueue_; }; \ No newline at end of file diff --git a/backend/include/backend/session_manager.hpp b/backend/include/backend/session_manager.hpp index 708a313c..430c7282 100644 --- a/backend/include/backend/session_manager.hpp +++ b/backend/include/backend/session_manager.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include +#include #include #include #include @@ -39,6 +41,11 @@ class SessionManager void registerRpc(); + /** + * @brief Whether the sessions refuse local deletes, as the local filesystem options say. Refused until set. + */ + void preventLocalDeletion(bool prevent); + void addPasswordProvider(int priority, PasswordProvider* provider); void joinSessionAdder(); void addSession( @@ -113,6 +120,10 @@ class SessionManager std::atomic_bool updateDispatchRunning_{false}; Persistence::StateHolder* stateHolder_{}; std::unordered_map, Ids::IdHash> sessions_{}; + /** @brief Read by the local operation queue on each local delete the explorer asks for. */ + std::shared_ptr localDeletionPrevented_{std::make_shared(true)}; + /** @brief Shared with every session, outlives them all, so a local delete never ends with its session. */ + std::shared_ptr localOperationQueue_; std::map passwordProviders_{}; std::mutex passwordRequestMutex_{}; diff --git a/backend/include/backend/sftp/archive_download_operation.hpp b/backend/include/backend/sftp/archive_download_operation.hpp index a7305ed4..9acda6a7 100644 --- a/backend/include/backend/sftp/archive_download_operation.hpp +++ b/backend/include/backend/sftp/archive_download_operation.hpp @@ -94,7 +94,7 @@ class ArchiveDownloadOperation : public Operation std::chrono::seconds futureTimeout{5}; }; - ArchiveDownloadOperation(SecureShell::SftpSession& sftp, Options options); + ArchiveDownloadOperation(std::weak_ptr sftp, Options options); ~ArchiveDownloadOperation() override; ArchiveDownloadOperation(ArchiveDownloadOperation const&) = delete; ArchiveDownloadOperation& operator=(ArchiveDownloadOperation const&) = delete; @@ -137,7 +137,7 @@ class ArchiveDownloadOperation : public Operation SecureShell::ProcessingStrand* strand() const override { - return sftp_ ? sftp_->strand() : nullptr; + return strandOf(sftp_); } std::expected cancel(bool adoptCancelState) override; @@ -177,8 +177,11 @@ class ArchiveDownloadOperation : public Operation Paths::RemotePath source{}; }; - std::expected prepareInStrand(); - std::expected flattenRootsInStrand(); + std::expected prepareInStrand(SecureShell::SftpSession& sftp); + /** + * @brief Resolves the roots into archive members: a file root by a fresh lstat, a directory root from its scan. + */ + std::expected flattenRootsInStrand(SecureShell::SftpSession& sftp); /** * @brief Expand a preceding ScanOperation's flat entry vector into the * op's ResolvedEntry list, prefixing each entry's tar path with @@ -192,7 +195,7 @@ class ArchiveDownloadOperation : public Operation Paths::RemotePath const& remoteRoot, Paths::RelativePath const& archivePrefix ); - std::expected openNextEntryInStrand(); + std::expected openNextEntryInStrand(SecureShell::SftpSession& sftp); std::expected readChunkInStrand(); std::expected finaliseEntryInStrand(); std::expected finaliseArchiveInStrand(); @@ -201,7 +204,7 @@ class ArchiveDownloadOperation : public Operation void cleanup(); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Options options_; std::vector resolvedEntries_{}; std::optional writer_{std::nullopt}; diff --git a/backend/include/backend/sftp/archive_upload_operation.hpp b/backend/include/backend/sftp/archive_upload_operation.hpp index c0eddf13..6e13ab0d 100644 --- a/backend/include/backend/sftp/archive_upload_operation.hpp +++ b/backend/include/backend/sftp/archive_upload_operation.hpp @@ -75,7 +75,7 @@ class ArchiveUploadOperation : public Operation std::chrono::seconds futureTimeout{5}; }; - ArchiveUploadOperation(SecureShell::SftpSession& sftp, Options options); + ArchiveUploadOperation(std::weak_ptr sftp, Options options); ~ArchiveUploadOperation() override; ArchiveUploadOperation(ArchiveUploadOperation const&) = delete; ArchiveUploadOperation& operator=(ArchiveUploadOperation const&) = delete; @@ -118,7 +118,7 @@ class ArchiveUploadOperation : public Operation SecureShell::ProcessingStrand* strand() const override { - return sftp_ ? sftp_->strand() : nullptr; + return strandOf(sftp_); } std::expected cancel(bool adoptCancelState) override; @@ -153,7 +153,7 @@ class ArchiveUploadOperation : public Operation Paths::LocalPath source{}; }; - std::expected prepareInStrand(); + std::expected prepareInStrand(SecureShell::SftpSession& sftp); std::expected flattenRoots(); /** * @brief Expand a preceding LocalScanOperation's flat entry vector into @@ -170,13 +170,13 @@ class ArchiveUploadOperation : public Operation std::expected openNextEntryInStrand(); std::expected readChunkInStrand(); std::expected finaliseEntryInStrand(); - std::expected finaliseArchiveInStrand(); + std::expected finaliseArchiveInStrand(SecureShell::SftpSession& sftp); bool isPayloadEntry(ResolvedEntry const& entry) const noexcept; void cleanup(); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Options options_; std::vector resolvedEntries_{}; std::optional writer_{std::nullopt}; diff --git a/backend/include/backend/sftp/bulk_download_operation.hpp b/backend/include/backend/sftp/bulk_download_operation.hpp index d782adc5..cb66cf42 100644 --- a/backend/include/backend/sftp/bulk_download_operation.hpp +++ b/backend/include/backend/sftp/bulk_download_operation.hpp @@ -2,9 +2,11 @@ #include #include +#include #include #include +#include #include class BulkDownloadOperation : public BulkTransferOperation @@ -28,7 +30,7 @@ class BulkDownloadOperation : public BulkTransferOperation std::chrono::milliseconds progressEmitInterval{100}; }; - BulkDownloadOperation(SecureShell::SftpSession& sftp, BulkDownloadOperationOptions options); + BulkDownloadOperation(std::weak_ptr sftp, BulkDownloadOperationOptions options); ~BulkDownloadOperation() override; BulkDownloadOperation(BulkDownloadOperation const&) = delete; BulkDownloadOperation(BulkDownloadOperation&&) = delete; @@ -51,12 +53,26 @@ class BulkDownloadOperation : public BulkTransferOperation std::uint64_t mtime{0}; std::uint32_t mtimeNsec{0}; bool mayOverwrite{false}; + /** @brief A directory created without its content, with the link check of every local directory. */ + bool isDirectory{false}; + /** @brief See SharedData::BulkDownloadEntry::comparedDestination. */ + std::optional comparedDestination{}; }; void setPrescannedFileList(std::vector files); + /** + * @brief The request that resumes this bulk on another connection: the entries it has not finished, as a flat + * list below its roots, with the options of @p request. None while the entries are not known yet, the + * backup then stays as it is. Also none when a remote symlink is among them, which a flat list could only + * follow. + */ + std::optional resumeRequest(SharedData::BulkDownloadRequest request) const; + protected: - std::expected prepareRootInStrand(SharedData::DirectoryEntry const& root) override; - std::expected createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) override; + std::expected + prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) override; + std::expected + createDirectoryInStrand(SecureShell::SftpSession& sftp, std::size_t entryIndex, EntryPaths const& paths) override; std::unique_ptr makeChild( std::size_t entryIndex, EntryPaths const& paths, @@ -64,6 +80,7 @@ class BulkDownloadOperation : public BulkTransferOperation ) override; bool isSkippableError(Error const& error) const override; bool isSkippableDirectoryError(Error const& error) const override; + std::expected finishInStrand() override; bool sourceIsRemote() const noexcept override { @@ -78,9 +95,14 @@ class BulkDownloadOperation : public BulkTransferOperation private: std::expected createLocalDirectory(Paths::LocalPath const& path, SharedData::DirectoryEntry const& entry); - std::expected - applyPermsToDirectory(std::filesystem::path const& path, SharedData::DirectoryEntry const& entryToInheritFrom); + /** + * @brief Remembers the permissions @p path gets once the bulk is done, so a directory without write + * permission still receives its content. + */ + void deferDirectoryPermissions(std::filesystem::path const& path, SharedData::DirectoryEntry const& entryToInheritFrom); private: DownloadOperation::DownloadOperationOptions individualOptions_; + /** @brief Created directories with the permissions they get at the end, parents before their children. */ + std::vector> deferredDirectoryPermissions_{}; }; diff --git a/backend/include/backend/sftp/bulk_resume_registry.hpp b/backend/include/backend/sftp/bulk_resume_registry.hpp index 25a5b3af..83e49e73 100644 --- a/backend/include/backend/sftp/bulk_resume_registry.hpp +++ b/backend/include/backend/sftp/bulk_resume_registry.hpp @@ -6,23 +6,41 @@ #include #include +#include #include #include #include #include +/** + * @brief The request a backup re-issues, its type picks the addBulk* path. + */ +using BulkResumeRequest = + std::variant; + +/** + * @brief Entries and bytes of a bulk finished on an earlier connection. Its resumed card counts them as done, so it + * shows the whole bulk instead of a smaller one. + */ +struct BulkResumeProgress +{ + std::uint64_t entries{0}; + std::uint64_t bytes{0}; +}; + /** * @brief Carries the state needed to re-enqueue a bulk operation against a * replacement Session after a seamless reconnect. * - * One entry per frontend card: the request's type picks the addBulk* - * path. A transfer entry holds either the files of one bulk card or a - * single directory, so adopting it re-issues exactly that card under - * its own id. + * One entry per frontend card. A transfer entry holds either the files + * of one bulk card or a single directory, so adopting it re-issues + * exactly that card under its own id. */ struct BulkResumeEntry { - std::variant request; + BulkResumeRequest request; + /** @brief What the bulk finished before the backup, left out of @ref request. */ + BulkResumeProgress finished{}; std::chrono::steady_clock::time_point storedAt{std::chrono::steady_clock::now()}; }; diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index a5a44c4e..a530dbde 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -1,10 +1,13 @@ #pragma once #include +#include #include #include #include +#include #include +#include #include #include @@ -58,6 +61,8 @@ class BulkTransferOperation : public Operation Paths::LocalPath local{}; /** @brief This entry's destination may be replaced even when the bulk's options do not allow it. */ bool mayOverwrite{false}; + /** @brief See SharedData::BulkUploadEntry::comparedDestination, kept by a resume. */ + std::optional comparedDestination{}; }; ~BulkTransferOperation() override; @@ -114,6 +119,8 @@ class BulkTransferOperation : public Operation addRejectedEntries(finished.rejectedPaths()); if constexpr (requires { finished.unreadableDirectories(); }) addUnreadableDirectories(finished.unreadableDirectories()); + if constexpr (requires { finished.skippedLinks(); }) + addSkippedLinks(finished.skippedLinks()); setScanResult(finished.ejectEntries(), scannedBytes); } ); @@ -131,6 +138,11 @@ class BulkTransferOperation : public Operation */ void addRejectedEntries(std::vector const& paths); + /** + * @brief Records links the scan left out as failed, they are not transferred. + */ + void addSkippedLinks(std::vector const& paths); + /** * @brief Records directories the scan could not list as failed, their content is not transferred. */ @@ -141,11 +153,53 @@ class BulkTransferOperation : public Operation */ std::vector> getFailed() const; + /** + * @brief The entries that were skipped by their index, and why. + */ + std::vector const& indexedFailures() const noexcept + { + return indexedFailures_; + } + std::size_t inFlightCount() const noexcept { return slots_.size(); } + /** + * @brief The entries are known: the scan delivered them or a flat list was set. + */ + bool hasEntries() const noexcept + { + return entrySource_ != EntrySource::None; + } + + /** + * @brief The indices of the entries a resume has to do again: every entry that was neither transferred, nor + * created as a directory, nor skipped for its type. Failed and in flight entries are among them. + */ + std::vector unfinishedEntryIndices() const; + + /** + * @brief Counts @p progress, what an earlier connection finished of this bulk, as done: its entries and bytes + * are part of the totals and of the progress from the start. Call it after the entries are set. + */ + void countEarlierProgress(BulkResumeProgress progress); + + /** + * @brief What a resume of this bulk leaves out: the entries it finished, without the root of a tree, and the + * bytes of their files, with what an earlier connection finished. + */ + BulkResumeProgress finishedProgress() const; + + /** + * @brief What an earlier connection finished of this bulk, see @ref countEarlierProgress. + */ + BulkResumeProgress earlierProgress() const noexcept + { + return earlierProgress_; + } + std::uint64_t completedCount() const noexcept { return completedEntries_; @@ -157,10 +211,11 @@ class BulkTransferOperation : public Operation } protected: - BulkTransferOperation(SecureShell::SftpSession& sftp, CommonOptions common); + BulkTransferOperation(std::weak_ptr sftp, CommonOptions common); /** - * @brief Flat-list mode: every entry is a file, @p paths holds both of its ends (index aligned). + * @brief Flat-list mode: every entry is a file or a directory created without content, @p paths holds both of + * its ends (index aligned). A directory that cannot be created fails only its own entry. */ void setPrescannedEntries( std::vector entries, @@ -174,8 +229,8 @@ class BulkTransferOperation : public Operation } /** - * @brief Both ends of the entry at @p entryIndex. In tree mode the entry's names are joined below the roots, - * so an entry whose name the local side cannot hold fails with InvalidName. + * @brief Both ends of the entry at @p entryIndex. In tree mode the entry's names are joined below the roots + * with @ref Detail::treeEntryPaths. */ std::expected entryPaths(std::size_t entryIndex) const; @@ -189,6 +244,11 @@ class BulkTransferOperation : public Operation */ std::string failureSource(std::size_t entryIndex) const; + /** + * @brief The source of @p paths, the key a failure is reported under. + */ + std::string failureSource(EntryPaths const& paths) const; + /** * @brief The entry below the source root by its names as listed, for entries without valid paths. */ @@ -199,9 +259,12 @@ class BulkTransferOperation : public Operation return entries_; } - SecureShell::SftpSession& sftp() const noexcept + /** + * @brief The session the children are given, held weakly like the bulk's own. + */ + std::weak_ptr const& sftp() const noexcept { - return *sftp_; + return sftp_; } CommonOptions const& common() const noexcept @@ -212,12 +275,14 @@ class BulkTransferOperation : public Operation /** * @brief Tree mode only: creates the bulk's root directory from entries()[0]. */ - virtual std::expected prepareRootInStrand(SharedData::DirectoryEntry const& root) = 0; + virtual std::expected + prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) = 0; /** * @brief Creates the directory described by the entry at @p entryIndex. */ - virtual std::expected createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) = 0; + virtual std::expected + createDirectoryInStrand(SecureShell::SftpSession& sftp, std::size_t entryIndex, EntryPaths const& paths) = 0; /** * @brief Builds the child transfer for the file at @p entryIndex. @@ -244,6 +309,14 @@ class BulkTransferOperation : public Operation return false; } + /** + * @brief Runs once every entry is done, before the bulk completes. A failure fails the bulk. + */ + virtual std::expected finishInStrand() + { + return {}; + } + /** * @brief Log prefix, e.g. "BulkDownloadOperation". */ @@ -258,17 +331,22 @@ class BulkTransferOperation : public Operation struct Slot { std::size_t entryIndex; + /** @brief Both ends of the entry, built once when it starts. */ + EntryPaths paths; std::unique_ptr child; std::uint64_t bytes{0}; std::uint64_t totalBytes{0}; bool finished{false}; }; - std::expected start(); - std::expected step(); - std::expected fillSlots(); + std::expected start(SecureShell::SftpSession& sftp); + std::expected step(SecureShell::SftpSession& sftp); + std::expected fillSlots(SecureShell::SftpSession& sftp); std::expected stepSlots(); - void finishSlot(Slot& slot); + /** + * @brief Counts the entry of @p slot as done, as @p transferred or as a failure recorded for the user. + */ + void finishSlot(Slot& slot, bool transferred); void cancelChildrenInStrand(bool adoptCancelState); Slot* slotFor(std::size_t entryIndex) noexcept; std::uint64_t fileCount() const noexcept; @@ -296,7 +374,7 @@ class BulkTransferOperation : public Operation Flat, }; - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; CommonOptions common_; std::unique_ptr scanPhase_{}; EntrySource entrySource_{EntrySource::None}; @@ -313,8 +391,11 @@ class BulkTransferOperation : public Operation std::uint64_t completedBytes_{0}; std::uint64_t totalBytes_{0}; int budget_{1}; + BulkResumeProgress earlierProgress_{}; + /** @brief Per entry: transferred, created or skipped for its type, so a resume leaves it out. */ + std::vector finishedEntries_{}; std::vector> failedEntries_{}; - std::vector failedEntryIndices_{}; + std::vector indexedFailures_{}; // Rolling bulk-level throughput. Per-file rates reset between files, which made the number // flicker; sampling the cumulative byte count keeps one coherent rate per bulk. std::chrono::steady_clock::time_point lastBpsSampleTime_{}; @@ -322,3 +403,18 @@ class BulkTransferOperation : public Operation std::int64_t bulkBytesPerSecond_{0}; std::chrono::steady_clock::time_point lastEmit_{}; }; + +namespace Detail +{ + /** + * @brief Both ends of the entry @p entryIndex of a scanned tree: its names on the way from the root joined below + * @p remoteRoot and @p localRoot. InvalidName when a name is no path segment or the local side cannot hold + * it, ImplementationError for a missing root or an index out of range. + */ + std::expected treeEntryPaths( + std::vector const& entries, + std::size_t entryIndex, + Paths::RemotePath const& remoteRoot, + Paths::LocalPath const& localRoot + ); +} diff --git a/backend/include/backend/sftp/bulk_upload_operation.hpp b/backend/include/backend/sftp/bulk_upload_operation.hpp index 4eb22e1e..44151e98 100644 --- a/backend/include/backend/sftp/bulk_upload_operation.hpp +++ b/backend/include/backend/sftp/bulk_upload_operation.hpp @@ -2,9 +2,11 @@ #include #include +#include #include #include +#include #include class BulkUploadOperation : public BulkTransferOperation @@ -24,7 +26,7 @@ class BulkUploadOperation : public BulkTransferOperation std::chrono::milliseconds progressEmitInterval{100}; }; - BulkUploadOperation(SecureShell::SftpSession& sftp, BulkUploadOperationOptions options); + BulkUploadOperation(std::weak_ptr sftp, BulkUploadOperationOptions options); ~BulkUploadOperation() override; BulkUploadOperation(BulkUploadOperation const&) = delete; BulkUploadOperation(BulkUploadOperation&&) = delete; @@ -40,12 +42,24 @@ class BulkUploadOperation : public BulkTransferOperation Paths::RemotePath destination; std::uint64_t sizeBytes; bool mayOverwrite{false}; + /** @brief A directory created without its content, its missing parents below the bulk's root with it. */ + bool isDirectory{false}; + std::optional comparedDestination{}; }; void setPrescannedFileList(std::vector files); + /** + * @brief The request that resumes this bulk on another connection: the entries it has not finished, as a flat + * list below its roots, with the options of @p request. None while the entries are not known yet, the + * backup then stays as it is. + */ + std::optional resumeRequest(SharedData::BulkUploadRequest request) const; + protected: - std::expected prepareRootInStrand(SharedData::DirectoryEntry const& root) override; - std::expected createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) override; + std::expected + prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) override; + std::expected + createDirectoryInStrand(SecureShell::SftpSession& sftp, std::size_t entryIndex, EntryPaths const& paths) override; std::unique_ptr makeChild( std::size_t entryIndex, EntryPaths const& paths, @@ -65,8 +79,16 @@ class BulkUploadOperation : public BulkTransferOperation private: std::filesystem::perms determinePerms(SharedData::DirectoryEntry const& entry) const; + /** + * @brief Creates the missing directories between the bulk's remote root and @p path, not @p path itself. + */ std::expected - createRemoteDirectoryInStrand(Paths::RemotePath const& path, SharedData::DirectoryEntry const& entry); + createMissingParentsInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& path); + std::expected createRemoteDirectoryInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& path, + SharedData::DirectoryEntry const& entry + ); private: UploadOperation::UploadOperationOptions individualOptions_; diff --git a/backend/include/backend/sftp/compared_state_check.hpp b/backend/include/backend/sftp/compared_state_check.hpp new file mode 100644 index 00000000..a4997815 --- /dev/null +++ b/backend/include/backend/sftp/compared_state_check.hpp @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/** + * @brief The checks that keep a sync entry from replacing or deleting what changed after its compare. The path is + * inspected right before the move or the removal, a change in the moment between the two goes unnoticed. + */ +namespace ComparedStateCheck +{ + /** + * @brief Whole seconds since the epoch of @p time, how a local scan records modification times. + */ + std::uint64_t secondsSinceEpoch(std::filesystem::file_time_type time); + + /** + * @brief What is at the local @p path now, without following a link, as a local scan sees it. A link or junction + * on Windows has an unknown type, the scan excludes it. + */ + std::expected localStateOf(std::filesystem::path const& path); + + /** + * @brief What is at the remote @p path now, without following a link. + */ + std::expected + remoteStateOfInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& path); + + /** + * @brief Fails with ChangedSinceCompare when the local @p path no longer holds @p compared (see + * SharedData::unchangedSinceCompare), and with FileStatFailed when it cannot be inspected. Without + * @p compared there is nothing to check. + */ + std::expected + requireUnchangedLocal(std::filesystem::path const& path, std::optional const& compared); + + /** + * @brief The remote counterpart of @ref requireUnchangedLocal, failing with SftpError when the server cannot + * tell. + */ + std::expected requireUnchangedRemoteInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& path, + std::optional const& compared + ); + + /** + * @brief Utility::moveIntoPlace of a finished local transfer, once @ref requireUnchangedLocal passed for @p to. + */ + std::expected moveLocalIntoPlace( + std::filesystem::path const& from, + std::filesystem::path const& to, + bool replace, + std::optional const& compared + ); + + /** + * @brief SftpSession::moveIntoPlaceInStrand of a finished remote transfer, once + * @ref requireUnchangedRemoteInStrand passed for @p to. + */ + std::expected moveRemoteIntoPlaceInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& from, + Paths::RemotePath const& to, + bool replace, + std::string const& asideSuffix, + SharedData::FileType sourceType, + std::optional const& compared + ); +} diff --git a/backend/include/backend/sftp/delete_operation.hpp b/backend/include/backend/sftp/delete_operation.hpp index 8c907408..1875d65c 100644 --- a/backend/include/backend/sftp/delete_operation.hpp +++ b/backend/include/backend/sftp/delete_operation.hpp @@ -6,9 +6,11 @@ #include #include #include +#include #include #include +#include #include class DeleteOperation : public Operation @@ -33,7 +35,20 @@ class DeleteOperation : public Operation return false; } - DeleteOperation(SecureShell::SftpSession& sftp, DeleteOperationOptions options); + /** + * @brief One item to delete. + */ + struct Target + { + Paths::RemotePath path{}; + /** + * @brief What a sync compare saw there. An item that no longer is what it saw is not deleted but fails with + * ChangedSinceCompare. Unset outside of sync. + */ + std::optional compared{}; + }; + + DeleteOperation(std::weak_ptr sftp, DeleteOperationOptions options); ~DeleteOperation() override; DeleteOperation(DeleteOperation const&) = delete; DeleteOperation(DeleteOperation&&) = delete; @@ -48,24 +63,27 @@ class DeleteOperation : public Operation void setScanResult(std::vector&& entries, std::uint64_t /* totalBytes */); /** - * @brief Adds the scanned content of @p root, which has to lie below it, otherwise nothing is deleted. + * @brief Adds the entries of a scan of @p root as targets. The first entry is @p root itself, every other one + * lies below it through its parents. A second root, an invalid name or the root directory refuses the + * scan, then nothing is deleted. */ void appendScanResult(Paths::RemotePath const& root, std::vector const& entries); /** * @brief Deletes exactly @p targets, last first, without a scan. For the files of a bulk delete. */ - void setTargets(std::vector targets); + void setTargets(std::vector targets); /** - * @brief Scans with @p scan before deleting; every scanned entry is deleted, deepest first. + * @brief Scans with @p scan before deleting; every scanned entry is deleted, deepest first. One operation can + * take several roots. A root whose scan fails is a failed entry and the other roots are still deleted. */ - void scanFirst(std::unique_ptr scan); + void addScanRoot(std::unique_ptr scan); /** - * @brief Like @ref scanFirst for one more directory, so one operation deletes several directories. + * @brief Counts @p target as an item that could not be deleted, like one refused before it was queued. */ - void addScanRoot(std::unique_ptr scan); + void addFailedTarget(Paths::RemotePath const& target, Error error); bool scanning() const noexcept { @@ -109,18 +127,20 @@ class DeleteOperation : public Operation } private: - std::expected removeOnce(Paths::RemotePath const& path); + std::expected removeOnce(Target const& target); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Paths::RemotePath remotePath_; std::function filesRemovedProgress_; std::chrono::seconds futureTimeout_; std::size_t deletedThusFar_{0}; std::size_t totalToDelete_{0}; + /** @brief Targets whose removal failed, the operation fails when that is all of them and nothing else. */ + std::size_t removalFailures_{0}; /** @brief Deleted from the back, so a scanned directory goes after its content. */ - std::vector targets_{}; - /** @brief A scanned entry was not a valid path within remotePath_, nothing may be deleted. */ + std::vector targets_{}; + /** @brief Why a scan result was refused, nothing may be deleted then. */ std::optional invalidScanEntry_{}; std::vector> failedEntries_{}; std::unique_ptr> scanPhase_{}; diff --git a/backend/include/backend/sftp/download_operation.hpp b/backend/include/backend/sftp/download_operation.hpp index 1f0296da..3ced0e97 100644 --- a/backend/include/backend/sftp/download_operation.hpp +++ b/backend/include/backend/sftp/download_operation.hpp @@ -8,9 +8,12 @@ #include #include #include +#include +#include #include #include +#include #include class DownloadOperation : public Operation @@ -51,16 +54,24 @@ class DownloadOperation : public Operation * a link planted inside a downloaded tree cannot redirect the download out of it. */ Paths::LocalPath linkFreeRoot{}; + /** @brief Directories below @ref linkFreeRoot already found to be no links, shared by a bulk download. */ + std::shared_ptr verifiedDirectories{}; + /** + * @brief What a sync compare saw at the destination. The transfer then fails with ChangedSinceCompare + * instead of replacing a destination that no longer holds it. A source that changed since is still + * transferred as it is now. Unset outside of sync. + */ + std::optional comparedDestination{std::nullopt}; }; SecureShell::ProcessingStrand* strand() const override { if (auto stream = fileStream_.lock(); stream) return stream->strand(); - return sftp_ ? sftp_->strand() : nullptr; + return strandOf(sftp_); } - DownloadOperation(SecureShell::SftpSession& sftp, DownloadOperationOptions options); + DownloadOperation(std::weak_ptr sftp, DownloadOperationOptions options); ~DownloadOperation() override; DownloadOperation(DownloadOperation const&) = delete; DownloadOperation(DownloadOperation&&) = delete; @@ -122,12 +133,12 @@ class DownloadOperation : public Operation std::expected readOnceInStrand(); std::expected handleSymlink(); - std::expected handleSymlinkInStrand(); + std::expected handleSymlinkInStrand(SecureShell::SftpSession& sftp); bool commitBufferToFile(SecureShell::IFileStream::SignedSizeType bytesRead); /// Runs the remote-heavy portion of prepare() under a single strand umbrella. - std::expected prepareInStrand(); + std::expected prepareInStrand(SecureShell::SftpSession& sftp); std::expected openOrAdoptFileInStrand(SecureShell::IFileStream& stream); @@ -138,24 +149,46 @@ class DownloadOperation : public Operation std::expected refuseLinkedPath(std::filesystem::path const& path) const; std::expected - readSymlinkInStrand(Paths::RemotePath const& remoteFullPath); + readSymlinkInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& remoteFullPath); + + /** + * @brief The error of a file that ended before its listed size arrived. + */ + Error sizeMismatch() const; + + /** + * @brief Creates the temp file, or limits an existing one, to its owner until the final permissions are set. + */ + std::expected restrictTempFile() const; + + /** + * @brief The permissions of the finished file: inherited, configured, or those of a new file. + */ + std::expected finalPermissionsInStrand(); + + /** + * @brief Gives @p path the modification time of the remote file. A failure is only logged. + */ + void applyModificationTime(std::filesystem::path const& path) const; void cleanup(); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; std::weak_ptr fileStream_; DownloadOperationOptions options_; /** @brief options_.remotePath, unset when none was given. */ Paths::RemotePath remotePath_{}; /** @brief options_.localPath for the filesystem calls, empty when none was given. */ std::filesystem::path localNativePath_{}; + /** @brief Where the download is written before it replaces localNativePath_, empty when no local path was given. */ + std::filesystem::path tempNativePath_{}; std::ofstream localFile_; SecureShell::BufferLease buffer_; std::shared_ptr asyncTransferContext_; /** @brief Bytes already present in an adopted partial file. */ std::uint64_t resumeOffset_{0u}; - /** @brief Bytes in the temp file so far, counted instead of tellp, which append mode misreports on Windows. */ + /** @brief Bytes of the content in the temp file so far, an adopted partial file included. */ std::uint64_t bytesInFile_{0u}; /** @brief An adopted partial file already holds the whole content. */ bool transferComplete_{false}; diff --git a/backend/include/backend/sftp/local_delete_operation.hpp b/backend/include/backend/sftp/local_delete_operation.hpp index a60c00d9..fd1cb766 100644 --- a/backend/include/backend/sftp/local_delete_operation.hpp +++ b/backend/include/backend/sftp/local_delete_operation.hpp @@ -2,28 +2,65 @@ #include #include +#include +#include #include #include #include +#include +#include #include #include /** - * @brief Deletes local items through the queue, so they honour pause and show progress like remote deletes. - * Exactly the given items are removed: a directory only once empty, a link as the link itself. + * @brief Deletes local items in the LocalOperationQueue, so they can be held and canceled and show progress like + * remote deletes. Each work step deletes until its time budget is used. Links and junctions are removed + * themselves and never followed. The queue checks the items with the local delete guard before it adds the + * operation. */ class LocalDeleteOperation : public Operation { public: struct LocalDeleteOperationOptions { + /** + * @brief The item handled last, the entries handled so far and the total. Called at most every + * @ref progressInterval and once more when the deletion ends. + */ std::function itemsRemovedProgress = [](std::string const&, auto, auto) {}; - /** @brief Deleted from the back, so a directory goes before its content in this list. */ + /** + * @brief Recursive only: the bytes and entries counted before the deletion starts, at most every + * @ref progressInterval and once when the count is done. + */ + std::function scanProgress = [](auto, auto) {}; + /** @brief Deleted from the back, so without @ref recursive a directory goes before its content in this list. */ std::vector targets{}; + /** + * @brief Every target goes with everything below it, counted first. Without, exactly the targets are removed, + * a directory only once it is empty. + */ + bool recursive{false}; + /** + * @brief Empty, or what a sync compare saw at each target, index aligned with @ref targets. A target that no + * longer holds it is not deleted and fails with ChangedSinceCompare. + */ + std::vector comparedStates{}; + /** @brief How long one work step deletes before it hands back to the queue. */ + std::chrono::milliseconds stepBudget{20}; + std::chrono::milliseconds progressInterval{100}; }; + /** @brief At most this many failed items are listed, the others are only counted and logged. */ + constexpr static std::size_t maximumListedFailures = 1000; + + /** + * @brief The deepest directory that every one of @p targets lies below, the first target itself when it is a + * root. Unset when there are no targets or they share no directory. + */ + static std::optional commonParentOf(std::span targets); + explicit LocalDeleteOperation(LocalDeleteOperationOptions options); ~LocalDeleteOperation() override = default; LocalDeleteOperation(LocalDeleteOperation const&) = delete; @@ -56,19 +93,42 @@ class LocalDeleteOperation : public Operation return 1; } + /** + * @brief Deletes until the step budget is used. Completes as a partial success when some items could not be + * deleted, and fails only when not a single target could be deleted. An item that is already gone counts + * as deleted. + */ std::expected work() override; std::expected cancel(bool adoptCancelState) override; /** - * @brief Items that could not be deleted, with why. The rest was still deleted. + * @brief Items that could not be deleted, with why, at most @ref maximumListedFailures. The rest was still deleted. */ std::vector> const& getFailed() const noexcept { return failedEntries_; } + private: + bool workFlat(std::chrono::steady_clock::time_point deadline); + bool workRecursive(std::chrono::steady_clock::time_point deadline); + void finishWalk(); + void recordFailure(std::filesystem::path const& path, std::error_code const& error); + void recordFailure(std::string const& path, Error error); + void reportProgress(bool final); + std::uint64_t handledEntries() const; + private: LocalDeleteOperationOptions options_; std::vector> failedEntries_{}; - std::size_t deletedThusFar_{0}; + std::size_t failureCount_{0}; + std::size_t targetIndex_{0}; + std::size_t completedTargets_{0}; + bool counting_{false}; + std::optional walk_{}; + std::uint64_t countedEntries_{0}; + std::uint64_t countedBytes_{0}; + std::uint64_t removedEntries_{0}; + std::string lastItem_{}; + std::chrono::steady_clock::time_point lastProgress_{}; }; diff --git a/backend/include/backend/sftp/local_scan_operation.hpp b/backend/include/backend/sftp/local_scan_operation.hpp index cf4d8da8..8377d7c3 100644 --- a/backend/include/backend/sftp/local_scan_operation.hpp +++ b/backend/include/backend/sftp/local_scan_operation.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include @@ -79,14 +81,7 @@ class LocalScanOperation : public Operation auto withWalkerDo(auto&& fn) { - auto scan = [this](Paths::RelativePath const& relative) - { - return scanner(relative); - }; - using WalkerType = Utility::DeepDirectoryWalker; - if (!walker_) - walker_ = std::make_unique(std::move(scan)); - return fn(static_cast(*walker_)); + return ScanWalkers::withFlatWalkerDo(*this, walker_, std::forward(fn)); } /** @@ -95,37 +90,26 @@ class LocalScanOperation : public Operation */ auto withTreeWalkerDo(auto&& fn) { - auto scan = [this](Paths::RelativePath const& relative) - { - return scanner(relative); - }; - using WalkerType = SharedData::Sync::TreeDirectoryWalker; - if (!walker_) - walker_ = std::make_unique( - std::move(scan), - recursive_, - [](Error const& error) - { - return error.type == ErrorType::DirectoryUnreadable; - }, - /*requireLocalNames=*/false, - excludedSuffixes_ - ); - return fn(static_cast(*walker_)); + return ScanWalkers::withTreeWalkerDo( + *this, walker_, recursive_, /*requireLocalNames=*/false, excludedSuffixes_, std::forward(fn) + ); } /** * @brief Eject the scanned directory entries. Careful!: The internal list is moved out. - * Only valid when @ref buildsTree() is false. + * Only valid when @ref buildsTree() is false. A directory that could not be listed is kept as an entry + * of unknown type, which no consumer creates, see @ref unreadableDirectories. */ std::vector ejectEntries() { - return withWalkerDo( + auto entries = withWalkerDo( [](auto& walker) { return std::move(walker).ejectEntries(); } ); + markUnreadable(entries); + return entries; } /** @@ -137,26 +121,28 @@ class LocalScanOperation : public Operation return withWalkerDo( [this](auto& walker) { - std::vector paths; - auto root = localPath_.toUtf8(); - if (!root.ends_with('/')) - root.push_back('/'); - for (auto const& entry : walker.rejectedEntries()) - paths.push_back(root + walker.describeRejected(entry)); - return paths; + return ScanWalkers::rejectedPaths(walker, localPath_.toUtf8()); } ); } /** - * @brief Full paths of directories below the root that could not be listed. Flat scans only, they are left out - * of the entries; the tree of a sync scan keeps them as unreadable nodes. + * @brief Full paths of directories below the root that could not be listed. Flat scans only, their entries are + * never created; the tree of a sync scan keeps them as unreadable nodes. */ std::vector const& unreadableDirectories() const noexcept { return unreadableDirectories_; } + /** + * @brief Full paths of links and junctions a flat scan on Windows leaves out, they are not uploaded from there. + */ + std::vector const& skippedLinks() const noexcept + { + return skippedLinks_; + } + /** * @brief Eject the scanned node tree. Only valid when @ref buildsTree() is true. */ @@ -193,8 +179,16 @@ class LocalScanOperation : public Operation std::error_code const& errorCode ); + /** + * @brief Gives the entries of @ref unreadableDirectories an unknown type. + */ + void markUnreadable(std::vector& entries) const; + private: std::vector unreadableDirectories_{}; + /** @brief The unreadable directories below the root, as relative paths. */ + std::unordered_set unreadableRelatives_{}; + std::vector skippedLinks_{}; Paths::LocalPath localPath_; std::function progressCallback_; diff --git a/backend/include/backend/sftp/operation.hpp b/backend/include/backend/sftp/operation.hpp index 683d2f28..55abc65a 100644 --- a/backend/include/backend/sftp/operation.hpp +++ b/backend/include/backend/sftp/operation.hpp @@ -8,12 +8,17 @@ #include #include #include +#include #include #include -#include +#include + #include +#include +#include +#include class Operation { @@ -159,12 +164,84 @@ class Operation return std::unexpected(error_.value()); } + public: + /** + * @brief The error of a finished transfer that could not be moved into place on the server: FileExists when the + * destination exists and may not be replaced. + */ + static Error remoteMoveError(SecureShell::SftpError error) + { + if (error.sftpError == SSH_FX_FILE_ALREADY_EXISTS) + return Error{.type = ErrorType::FileExists}; + return Error{.type = ErrorType::SftpError, .sftpError = std::move(error)}; + } + + /** + * @brief The error of a finished transfer that could not be moved into place on this machine: FileExists when the + * destination exists and may not be replaced. + */ + static Error localMoveError(std::error_code error) + { + if (error == std::errc::file_exists) + return Error{.type = ErrorType::FileExists}; + return Error{.type = ErrorType::RenameFailure}; + } + protected: void enterState(OperationState newState) { state_ = newState; } + /** + * @brief The operation completed, failed or was canceled. + */ + bool hasEnded() const noexcept + { + return state_ >= OperationState::Completed; + } + + /** + * @brief The session behind @p session for one work step, none once its channel was closed or is closing. + * Operations hold their session weakly and lock it per step, so a closed channel ends them instead of + * leaving them a dangling session. + */ + static std::shared_ptr openSession(std::weak_ptr const& session) + { + auto locked = session.lock(); + if (!locked || !locked->strand() || locked->strand()->isFinalized()) + return nullptr; + return locked; + } + + /** + * @brief The processing strand of @p session, none once its channel was closed or is closing. It stays valid + * until the channel is closed, which happens on the queue's strand. + */ + static SecureShell::ProcessingStrand* strandOf(std::weak_ptr const& session) + { + const auto locked = openSession(session); + return locked ? locked->strand() : nullptr; + } + + /** + * @brief The error of an operation whose SFTP channel was closed before it ended. + */ + static Error channelClosedError() + { + return Error{.type = ErrorType::ChannelClosed, .extraInfo = "The SFTP channel was closed"}; + } + + /** + * @brief Waits until every task queued on @p strand so far has run, so none of them outlives the operation that + * queued it. Returns at once on the processing thread itself and on a finalized strand. + */ + static void waitForStrandTasks(SecureShell::ProcessingStrand* strand) + { + if (strand && !strand->withinProcessingThread()) + strand->pushPromiseTask([]() {}).wait(); + } + /** * @brief The error of an async transfer that ended without completing. */ diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index 6af445b5..34760186 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -1,7 +1,9 @@ #pragma once #include +#include #include +#include #include #include #include @@ -151,8 +153,8 @@ class OperationQueue ); /** - * @brief Queues the delete of @p remotePath, refused for the root and for the home directory or any directory - * that contains it. + * @brief Queues the delete of @p remotePath, refused by a @ref remoteDeleteGuard for the root and for the home + * directory or any directory that contains it. */ std::expected addDeleteOperation( SecureShell::SftpSession& sftp, @@ -164,54 +166,61 @@ class OperationQueue ); /** - * @brief Bulk-add download operations from a pre-known entry list. - * Files are taken at face value (no lstat — sizes come from the - * caller) and turned into individual DownloadOperation cards. - * Directory entries fall back to the existing Scan+BulkDownload - * flow. Everything happens inside a single strand dispatch so - * only one SSH-thread context switch is paid for the batch. + * @brief @ref addDeleteOperation checking with @p guard, so a caller that checked the target with it already + * costs no further round trip. + */ + std::expected addDeleteOperation( + RemoteDeleteGuard& guard, + SecureShell::SftpSession& sftp, + Ids::OperationId operationId, + Paths::RemotePath const& remotePath, + bool recursive, + bool insertRefresh, + SharedData::OperationMode mode = SharedData::OperationMode::Queued + ); + + /** + * @brief Queues the entries of @p request: each directory with its content as a bulk of its own that scans it + * first, under the id @p operationIdFor gives its index; all files, and directories created without their + * content, as one bulk under @p bulkCardId. Sizes and times of files are taken as given, without an lstat. + * Each queued part keeps a resume backup under its id. * - * @param sftp SFTP session to use. - * @param request Full bulk request (file + directory entries). - * @param operationIdFor Callback producing a fresh OperationId per entry - * index (frontend reserves them so it can register - * completion callbacks before the RPC returns). - * @return Number of entries successfully enqueued. + * @param sftp SFTP session to use. + * @param request The entries; the card of the files works below its roots, unset the closest directories + * holding every source and every destination. + * @param operationIdFor The id of the entry at an index, reserved by the frontend so it can register callbacks + * before the reply. + * @param bulkCardId The id of the card of the files, reserved by the frontend apart from the entry ids. + * @return Number of entries queued: the files and directories of the card and the directories queued on their + * own. A request without any entry still queues the card, which then completes at once. */ std::size_t addBulkDownloadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkDownloadRequest const& request, - std::function operationIdFor, - Ids::OperationId bulkCardId + SharedData::BulkDownloadRequest request, + std::function const& operationIdFor, + Ids::OperationId const& bulkCardId ); - /** @brief Upload analogue of addBulkDownloadOperation. File entries skip - * the per-file local lstat + RPC round-trip; directories fall - * back to the existing LocalScan+BulkUpload flow. - * - * @param bulkCardId Separate id reserved by the frontend for the aggregate - * BulkUpload card that covers all file entries. Must not - * collide with any of the per-entry ids produced by - * @p operationIdFor. */ + /** + * @brief Upload analogue of @ref addBulkDownloadOperation. + */ std::size_t addBulkUploadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkUploadRequest const& request, - std::function operationIdFor, - Ids::OperationId bulkCardId + SharedData::BulkUploadRequest request, + std::function const& operationIdFor, + Ids::OperationId const& bulkCardId ); /** - * @brief Bulk-delete: collapse N file entries (and any directory entries) - * into a single delete operation card on the frontend. File - * entries are pre-known so no SFTP stat is needed. Directory - * entries get the existing Scan+Delete pair so descendants are - * recursively removed. + * @brief Queues the remote items of @p request as one delete operation with one card, labelled with the + * directory they lie in. Entries a @ref remoteDeleteGuard refuses, and directories that could not be + * looked up in time, are failed entries of that card. * - * @param sftp SFTP session to use. - * @param request The remote items; directories are scanned and deleted with their content. - * @param bulkOperationId Single OperationId for the aggregate file-bulk - * card. Per-directory ids are generated locally. - * @return Number of entries successfully enqueued. + * @param sftp SFTP session to use. + * @param request The remote items; directories are scanned and deleted with their content, unless the request + * deletes listed content only. + * @param bulkOperationId Operation id of the one card. + * @return Number of entries queued for deletion; zero queues nothing, not even the card. */ std::size_t addBulkDeleteOperation( SecureShell::SftpSession& sftp, @@ -220,32 +229,34 @@ class OperationQueue ); /** - * @brief Queues a resolved sync plan under the operation ids of @p request: files of each direction as one - * bulk transfer, remote and local deletes as one operation each. Directories without content are created - * right away. Every path comes from @p plan, which resolved them below the sync roots. + * @brief @ref addBulkDeleteOperation checking with @p guard, so a caller that checked the targets with it + * already costs no further round trip. */ - SharedData::Sync::SyncExecutionResult executeSyncPlan( + std::size_t addBulkDeleteOperation( + RemoteDeleteGuard& guard, SecureShell::SftpSession& sftp, - SyncSession const& session, - SyncSession::ResolvedPlan plan, - SharedData::Sync::SyncExecutionRequest const& request + SharedData::BulkDeleteRequest const& request, + Ids::OperationId bulkOperationId ); /** - * @brief Queues the delete of local items as one operation with one card. Refused as a whole when one of them - * is a root directory or contains the home directory. + * @brief Queues a resolved sync plan under the operation ids of @p request: files and directories without + * content of each direction as one bulk transfer below the sync roots, remote deletes as one operation + * and local deletes as one operation of @p localDeleteQueue, held while this queue is paused. Every path + * comes from @p plan, which resolved them below the sync roots. */ - std::expected addLocalDeleteOperation( - Ids::OperationId operationId, - std::vector targets, - SharedData::OperationMode mode = SharedData::OperationMode::Queued + SharedData::Sync::SyncExecutionResult executeSyncPlan( + SecureShell::SftpSession& sftp, + SyncSession const& session, + SyncSession::ResolvedPlan plan, + SharedData::Sync::SyncExecutionRequest const& request, + LocalOperationQueue& localDeleteQueue ); /** - * @brief Refuses an unset delete target, the root, and the home directory or any directory that contains it. + * @brief A guard for the remote deletes of one request, sharing the home directory this queue looked up. */ - std::expected - refuseProtectedDeleteTarget(SecureShell::SftpSession& sftp, Paths::RemotePath const& target); + RemoteDeleteGuard remoteDeleteGuard(SecureShell::SftpSession& sftp); std::expected addRenameOperation( SecureShell::SftpSession& sftp, @@ -255,8 +266,10 @@ class OperationQueue SharedData::OperationMode mode = SharedData::OperationMode::Queued ); - /** @brief Queues a remote scan and a local scan as priority operations for sync comparison. - * When each scan completes the results are sent to the frontend via onSyncScanResult. + /** + * @brief Opens a @ref SyncSession and kicks off both scans as priority operations. The scans build + * ScanNode trees directly and feed them into the session on completion. + * Emits @c onSyncScanPhaseDone(sessionId, isLocal) for each side. * * @param sftp SFTP session to use for the remote scan. * @param remoteScanId Pre-assigned operation ID for the remote scan. @@ -264,11 +277,6 @@ class OperationQueue * @param remotePath Remote directory root to scan. * @param localPath Local directory root to scan. */ - /** - * @brief Opens a @ref SyncSession and kicks off both scans. The scans build - * ScanNode trees directly and feed them into the session on completion. - * Emits @c onSyncScanPhaseDone(sessionId, isLocal) for each side. - */ void addSyncScanOperation( SecureShell::SftpSession& sftp, Ids::SyncSessionId syncSessionId, @@ -301,31 +309,29 @@ class OperationQueue void paused(bool pause); /** - * @brief Observes every completion right before it is sent to the frontend. Used by tests. + * @brief Test hook: @p observer sees every completion right before it is sent to the frontend. The application + * never sets one. */ void setCompletionObserver(std::function observer); bool hasPriorityWork() const; /** - * @brief Moves every still-tracked bulk-operation backup into @p - * registry, keyed by the bulk's OperationId. Called by Session - * during teardown so a seamless reconnect can adopt them. + * @brief Moves every resume backup into @p registry, keyed by the id of its card. Called by Session during + * teardown so a seamless reconnect can adopt them. * - * Covers both queued-not-started bulks and bulks already running - * — for the latter we re-issue the full original entry list and - * rely on the per-file tryContinue path to skip whole files - * already present (the in-progress file's bytes are not - * byte-checkpointed here). + * A bulk transfer that has its entries already leaves only the entries it has not finished, as a flat + * list below its roots, and records how many entries and bytes it finished, which the resumed card counts + * as done; a running file in it starts over unless its partial file can be continued. A bulk that did not + * start yet or is still scanning keeps its request. */ void dumpBulkResumes(BulkResumeRegistry& registry); /** - * @brief Re-enqueues the bulk operation described by @p entry against - * this queue, reusing @p operationId as the bulk-card id so the - * frontend's existing card surface stays attached. Called from - * Session::adoptBulkResumes after pulling @p entry out of the - * registry. + * @brief Re-enqueues the backup @p entry under @p operationId as the id of every part, so the frontend's card + * stays attached. Refuses a backup of more than one directory, or of a directory and other entries, + * which would complete the card more than once. Called from Session::adoptBulkResumes after pulling + * @p entry out of the registry. */ void adoptBulkResume(SecureShell::SftpSession& sftp, Ids::OperationId const& operationId, BulkResumeEntry entry); @@ -340,19 +346,76 @@ class OperationQueue auto makeBulkProgressCallback(std::string_view eventName, Ids::OperationId operationId) -> BulkProgressCallback; + /** + * @brief Sends the delete progress of @p operationId (current item, items handled, items in all) to the frontend. + */ + auto makeDeleteProgressCallback(Ids::OperationId operationId) + -> std::function; + private: bool workQueue(std::deque>>& queue); + /** + * @brief The queue an operation of @p mode goes to. + */ + auto queueFor(SharedData::OperationMode mode) + -> std::deque>>&; + + /** + * @brief Shared part of the bulk transfer requests: each directory with content goes to @p queueDirectory as + * (index, entry, request options) and counts when it returns true, every other entry into one request that + * @p queueFiles queues as the card @p bulkCardId and sets the roots of. That request is the card's backup. + * + * @return Number of entries queued. + */ + template + std::size_t splitBulkRequest( + RequestT request, + Ids::OperationId const& bulkCardId, + QueueDirectoryT&& queueDirectory, + QueueFilesT&& queueFiles + ); + + /** + * @brief The queued or running operation @p operationId, none when there is none. + */ + Operation* findOperation(Ids::OperationId const& operationId); + + /** + * @brief Reduces the backup @p entry of @p operation to what the operation has not finished yet and records + * what it left out in @p entry, which the resumed card counts as done. + */ + static void leaveOutFinished(Operation const& operation, BulkResumeEntry& entry); + /** * @brief Tells the sync session waiting for the scan @p id, if any, that it failed or was cancelled. + * @param reason A localized message, see Utility::localizedMessage. */ void failSyncScan(Ids::OperationId const& id, std::string const& reason); + /** + * @brief Drops the scan @p id of a sync whose other scan failed, without telling its session again. + */ + void cancelSiblingScan(Ids::OperationId const& id); + /** * @brief Hands the tree of a completed standalone scan to the sync session registered for its id. */ void deliverSyncScan(Ids::OperationId const& id, Operation& operation); + /** + * @brief The callback of the scan of one side of a sync: hands the tree to @p session through @p setTree on the + * session's strand and tells the frontend the side is done, or cancels the other scan @p siblingScanId and + * reports the failure. + */ + auto makeSyncScanCallback( + Ids::SyncSessionId syncSessionId, + std::weak_ptr session, + Ids::OperationId siblingScanId, + bool isLocal, + void (SyncSession::*setTree)(SharedData::Sync::ScanNode) + ) -> std::function)>; + private: Persistence::SftpOptions sftpOpts_{}; Ids::SessionId sessionId_{}; @@ -370,9 +433,7 @@ class OperationQueue std::map)>> syncScanCallbacks_{}; - // SyncSessions active on this channel. Keyed by SyncSessionId.value() because - // Ids::IdHash isn't specialized on the strongly-typed id; the existing - // bulkResumeStash_ does the specialization — we stay string-keyed for simplicity. + // SyncSessions active on this channel, keyed by SyncSessionId.value(). std::map> syncSessions_{}; // Per-bulk backup snapshot, populated when a bulk-add path enqueues an @@ -380,6 +441,6 @@ class OperationQueue // the bulk's OperationId. Read by dumpBulkResumes() during teardown. std::unordered_map bulkResumeStash_{}; - /** @brief The server's home directory, asked once for the delete guard. */ + /** @brief The canonical home directory of the server, asked once for the delete guards. */ std::optional remoteHome_{}; }; diff --git a/backend/include/backend/sftp/remote_delete_guard.hpp b/backend/include/backend/sftp/remote_delete_guard.hpp new file mode 100644 index 00000000..31a3368b --- /dev/null +++ b/backend/include/backend/sftp/remote_delete_guard.hpp @@ -0,0 +1,154 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SecureShell +{ + class SftpSession; +} + +/** + * @brief Why a remote delete target is protected. + */ +struct RemoteDeleteRefusal +{ + enum class Reason : std::uint8_t + { + NoPath, + Root, + ContainsHome, + /** + * @brief On a server with Windows paths a backslash, or a trailing dot or space, can name another item. + */ + WindowsName, + /** + * @brief The server did not answer in time, so the target could not be compared with the home directory. + */ + NotVerified, + }; + + Reason reason{Reason::NoPath}; + std::string target{}; + std::string home{}; + + /** + * @brief English text for the log. + */ + std::string toString() const; + + /** + * @brief A message built with Utility::localizedMessage that also says nothing was deleted. + */ + std::string toLocalizedMessage() const; +}; + +namespace RemoteDeleteGuardDetail +{ + /** + * @brief True when @p path starts with a drive like "/C:", as OpenSSH for Windows reports paths. + */ + bool looksLikeWindowsPath(Paths::RemotePath const& path); + + /** + * @brief True when @p path looks like it lies on a server whose filesystem folds case: a Windows drive path or + * one below the /Users of macOS. + */ + bool looksCaseFolding(Paths::RemotePath const& path); + + /** + * @brief @ref Paths::RemotePath::isWithin with ASCII letters compared without case. + */ + bool isWithinIgnoringCase(Paths::RemotePath const& path, Paths::RemotePath const& base); + + /** + * @brief The checks that need no server: an unset path, the root, a name a Windows server could read as another + * one, and @p home within @p target, without ASCII case when the target or the home looks like it lies on a + * server that folds case (@ref looksCaseFolding). + */ + std::optional + refusalByText(Paths::RemotePath const& target, std::optional const& home); +} + +/** + * @brief Refuses remote delete targets that are the root, the home directory or a directory containing it, also when + * reached through a symlink: the parent of each target is canonicalized by the server, once per parent, and + * the target's name appended unresolved, because deleting a link removes the link. + * + * One guard serves one request. It runs on the session strand and waits for the SFTP strand. + */ +class RemoteDeleteGuard +{ + public: + /** + * @param knownHome The canonical home of the session. Looked up on the first check that needs it while unset, + * and stored there once known. While it is unknown the guard logs it and only refuses by text. + */ + RemoteDeleteGuard( + SecureShell::SftpSession& sftp, + std::optional& knownHome, + std::chrono::milliseconds timeout + ); + + std::expected check(Paths::RemotePath const& target); + + /** + * @brief The canonical home of the session, looked up like for a check. Nothing when it is unknown or the + * server did not answer in time. + */ + std::optional homeDirectory(); + + /** + * @brief Treats every directory below @p root as no symlink, so the canonical form of a path below it is the + * canonical @p root with the rest appended. For targets that come from an lstat scan of @p root, which + * does not descend into links. + */ + void trustScannedBelow(Paths::RemotePath root); + + private: + std::expected, RemoteDeleteRefusal> home(Paths::RemotePath const& target); + std::expected, RemoteDeleteRefusal> + canonicalDirectory(Paths::RemotePath const& directory, Paths::RemotePath const& target); + std::expected, RemoteDeleteRefusal> + lookUpCanonical(Paths::RemotePath const& directory, Paths::RemotePath const& target); + + SecureShell::SftpSession* sftp_; + std::optional* knownHome_; + std::chrono::milliseconds timeout_; + bool homeUnknown_{false}; + std::vector scannedRoots_{}; + std::unordered_map> canonicalDirectories_{}; +}; + +/** + * @brief The refusal of the first protected target in @p targets, nothing when every one may be deleted. + */ +template +std::optional +refuseAnyProtected(RemoteDeleteGuard& guard, RangeT&& targets, ProjectionT projection = {}) +{ + std::optional refusal{}; + const auto protectedTarget = std::ranges::find_if( + targets, + [&guard, &refusal](Paths::RemotePath const& target) + { + auto checked = guard.check(target); + if (!checked) + refusal = std::move(checked).error(); + return refusal.has_value(); + }, + projection + ); + if (protectedTarget == std::ranges::end(targets)) + return std::nullopt; + return refusal; +} diff --git a/backend/include/backend/sftp/rename_operation.hpp b/backend/include/backend/sftp/rename_operation.hpp index d573c0fe..91c4a3dd 100644 --- a/backend/include/backend/sftp/rename_operation.hpp +++ b/backend/include/backend/sftp/rename_operation.hpp @@ -6,6 +6,7 @@ #include #include +#include class RenameOperation : public Operation { @@ -17,7 +18,7 @@ class RenameOperation : public Operation std::chrono::seconds futureTimeout{5}; }; - RenameOperation(SecureShell::SftpSession& sftp, RenameOperationOptions options); + RenameOperation(std::weak_ptr sftp, RenameOperationOptions options); ~RenameOperation() override = default; RenameOperation(RenameOperation const&) = delete; @@ -63,7 +64,7 @@ class RenameOperation : public Operation } private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Paths::RemotePath sourcePath_; Paths::RemotePath destinationPath_; std::chrono::seconds futureTimeout_; diff --git a/backend/include/backend/sftp/scan_operation.hpp b/backend/include/backend/sftp/scan_operation.hpp index 2d85228a..719e5831 100644 --- a/backend/include/backend/sftp/scan_operation.hpp +++ b/backend/include/backend/sftp/scan_operation.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -11,6 +12,7 @@ #include #include +#include #include #include @@ -46,7 +48,7 @@ class ScanOperation : public Operation return false; } - ScanOperation(SecureShell::SftpSession& sftp, ScanOperationOptions options); + ScanOperation(std::weak_ptr sftp, ScanOperationOptions options); ~ScanOperation() override; ScanOperation(ScanOperation const&) = delete; ScanOperation(ScanOperation&&) = delete; @@ -79,14 +81,7 @@ class ScanOperation : public Operation auto withWalkerDo(auto&& fn) { - auto scan = [this](Paths::RelativePath const& relative) - { - return scanner(relative); - }; - using WalkerType = Utility::DeepDirectoryWalker; - if (!walker_) - walker_ = std::make_unique(std::move(scan)); - return fn(static_cast(*walker_)); + return ScanWalkers::withFlatWalkerDo(*this, walker_, std::forward(fn)); } /** @@ -95,23 +90,9 @@ class ScanOperation : public Operation */ auto withTreeWalkerDo(auto&& fn) { - auto scan = [this](Paths::RelativePath const& relative) - { - return scanner(relative); - }; - using WalkerType = SharedData::Sync::TreeDirectoryWalker; - if (!walker_) - walker_ = std::make_unique( - std::move(scan), - recursive_, - [](Error const& error) - { - return error.type == ErrorType::DirectoryUnreadable; - }, - /*requireLocalNames=*/true, - excludedSuffixes_ - ); - return fn(static_cast(*walker_)); + return ScanWalkers::withTreeWalkerDo( + *this, walker_, recursive_, /*requireLocalNames=*/true, excludedSuffixes_, std::forward(fn) + ); } /** @@ -137,11 +118,7 @@ class ScanOperation : public Operation return withWalkerDo( [this](auto& walker) { - std::vector paths; - const auto root = remotePath_.isRoot() ? std::string{} : remotePath_.value(); - for (auto const& entry : walker.rejectedEntries()) - paths.push_back(root + "/" + walker.describeRejected(entry)); - return paths; + return ScanWalkers::rejectedPaths(walker, remotePath_.value()); } ); } @@ -172,7 +149,7 @@ class ScanOperation : public Operation std::expected, Error> scanner(Paths::RelativePath const& relative); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; Paths::RemotePath remotePath_; std::function progressCallback_; diff --git a/backend/include/backend/sftp/scan_phase.hpp b/backend/include/backend/sftp/scan_phase.hpp index ad5a3a82..92e9d39d 100644 --- a/backend/include/backend/sftp/scan_phase.hpp +++ b/backend/include/backend/sftp/scan_phase.hpp @@ -39,8 +39,17 @@ class ScanPhase : public ScanPhaseDriver */ using DeliverFunction = std::function; - explicit ScanPhase(DeliverFunction deliver) + /** + * @brief Receives a scan that failed, the phase then goes on with the next root. + */ + using FailFunction = std::function; + + /** + * @param onFailed Without it, a failed scan ends the phase with its error. + */ + explicit ScanPhase(DeliverFunction deliver, FailFunction onFailed = {}) : deliver_{std::move(deliver)} + , onFailed_{std::move(onFailed)} {} void addRoot(std::unique_ptr scan) @@ -56,11 +65,15 @@ class ScanPhase : public ScanPhaseDriver auto& current = *scans_.front(); auto result = current.work(); if (!result.has_value()) - return std::unexpected(std::move(result).error()); - if (result.value() != Operation::WorkStatus::Complete) + { + if (!onFailed_) + return std::unexpected(std::move(result).error()); + onFailed_(current, result.error()); + } + else if (result.value() != Operation::WorkStatus::Complete) return Operation::WorkStatus::MoreWork; - - deliver_(current); + else + deliver_(current); scans_.pop_front(); return scans_.empty() ? Operation::WorkStatus::Complete : Operation::WorkStatus::MoreWork; } @@ -73,6 +86,7 @@ class ScanPhase : public ScanPhaseDriver private: DeliverFunction deliver_; + FailFunction onFailed_; std::deque> scans_{}; }; diff --git a/backend/include/backend/sftp/scan_walkers.hpp b/backend/include/backend/sftp/scan_walkers.hpp new file mode 100644 index 00000000..f65b1e95 --- /dev/null +++ b/backend/include/backend/sftp/scan_walkers.hpp @@ -0,0 +1,99 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/** + * @brief The walkers the remote and the local scan share: one walker per scan, created on first use as a flat + * DeepDirectoryWalker or a sync TreeDirectoryWalker, both listing through the scan's scanner. + */ +namespace ScanWalkers +{ + /** + * @brief Lists a directory through @p OperationT::scanner. A named type, so every use of a scan's walker names + * the same walker type. + */ + template + struct ScannerOf + { + OperationT* operation; + + auto operator()(Paths::RelativePath const& relative) const + { + return operation->scanner(relative); + } + }; + + template + using FlatWalker = + Utility::DeepDirectoryWalker>; + + template + using TreeWalker = SharedData::Sync::TreeDirectoryWalker>; + + /** + * @brief Calls @p function with the flat walker of @p operation in @p walker, which it creates when there is none. + */ + template + decltype(auto) + withFlatWalkerDo(OperationT& operation, std::unique_ptr& walker, FunctionT&& function) + { + if (!walker) + walker = std::make_unique>(ScannerOf{&operation}); + return std::forward(function)(static_cast&>(*walker)); + } + + /** + * @brief Calls @p function with the tree walker of @p operation in @p walker, which it creates when there is + * none. A directory below the root that cannot be listed stays in the tree as unreadable. + * + * @param requireLocalNames Names must be creatable on this machine, for remote trees. + */ + template + decltype(auto) withTreeWalkerDo( + OperationT& operation, + std::unique_ptr& walker, + bool recursive, + bool requireLocalNames, + std::vector const& excludedSuffixes, + FunctionT&& function + ) + { + if (!walker) + { + walker = std::make_unique>( + ScannerOf{&operation}, + recursive, + [](Operation::Error const& error) + { + return error.type == Operation::ErrorType::DirectoryUnreadable; + }, + requireLocalNames, + excludedSuffixes + ); + } + return std::forward(function)(static_cast&>(*walker)); + } + + /** + * @brief The entries a flat walk refused for their name, below @p root by their names as listed, for display. + */ + template + std::vector rejectedPaths(WalkerT const& walker, std::string_view root) + { + std::vector paths; + paths.reserve(walker.rejectedEntries().size()); + for (auto const& entry : walker.rejectedEntries()) + paths.push_back(Utility::describeWalkedEntry(root, walker.entries(), entry)); + return paths; + } +} diff --git a/backend/include/backend/sftp/sequential_scan_progress.hpp b/backend/include/backend/sftp/sequential_scan_progress.hpp new file mode 100644 index 00000000..d0298310 --- /dev/null +++ b/backend/include/backend/sftp/sequential_scan_progress.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace Detail +{ + using ScanProgressFunction = std::function; + + /** + * @brief Hands out a progress callback per scan of scans that run one after another, each reporting the running + * total of all of them (bytes, index, entries scanned) to @p report. + */ + inline std::function sequentialScanProgress(ScanProgressFunction report) + { + struct Totals + { + std::size_t roots{0}; + std::size_t current{0}; + std::array finished{}; + std::array running{}; + }; + auto totals = std::make_shared(); + return [totals, report = std::move(report)]() -> ScanProgressFunction + { + const auto root = totals->roots++; + return [totals, report, root](std::uint64_t bytes, std::uint64_t index, std::uint64_t scanned) + { + if (root != totals->current) + { + std::ranges::transform(totals->finished, totals->running, totals->finished.begin(), std::plus{}); + totals->current = root; + } + totals->running = {bytes, index, scanned}; + report(totals->finished[0] + bytes, totals->finished[1] + index, totals->finished[2] + scanned); + }; + }; + } +} diff --git a/backend/include/backend/sftp/upload_operation.hpp b/backend/include/backend/sftp/upload_operation.hpp index 618e5251..8ffdc1e2 100644 --- a/backend/include/backend/sftp/upload_operation.hpp +++ b/backend/include/backend/sftp/upload_operation.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -42,11 +43,17 @@ class UploadOperation : public Operation * subtrees whose remote structure may not exist yet. */ bool createMissingDirectories{false}; + /** + * @brief What a sync compare saw at the destination. The transfer then fails with ChangedSinceCompare + * instead of replacing a destination that no longer holds it. A source that changed since is still + * transferred as it is now. Unset outside of sync. + */ + std::optional comparedDestination{std::nullopt}; }; SecureShell::ProcessingStrand* strand() const override; - UploadOperation(SecureShell::SftpSession& sftp, UploadOperationOptions options); + UploadOperation(std::weak_ptr sftp, UploadOperationOptions options); ~UploadOperation() override; UploadOperation(UploadOperation const&) = delete; UploadOperation(UploadOperation&&) = delete; @@ -107,20 +114,26 @@ class UploadOperation : public Operation SecureShell::IFileStream::SignedSizeType commitFileToBuffer(SecureShell::IFileStream::SignedSizeType bytes); std::expected handleSymlink(); - std::expected handleSymlinkInStrand(); + std::expected handleSymlinkInStrand(SecureShell::SftpSession& sftp); + + /** + * @brief Checks the local source and opens it, or for a link that is not followed only notes it is one: a + * regular file or a link, no other type, that exists. + */ + std::expected openLocalSource(); - std::expected openOrAdoptFileInStrand(); + std::expected openOrAdoptFileInStrand(SecureShell::SftpSession& sftp); /// Remote-heavy portion of prepare() — runs under a single strand umbrella. - std::expected prepareInStrand(); + std::expected prepareInStrand(SecureShell::SftpSession& sftp); /// Remote-heavy portion of finalize() — runs under a single strand umbrella. - std::expected finalizeInStrand(); + std::expected finalizeInStrand(SecureShell::SftpSession& sftp); /** * @brief Whether the existing remote temp file of @p tempSize bytes ends like the same range of the source. */ - bool tempBelongsToSourceInStrand(std::uint64_t tempSize); + bool tempBelongsToSourceInStrand(SecureShell::SftpSession& sftp, std::uint64_t tempSize); /** @brief Ensures every ancestor directory of @p dir exists on the remote, creating * any missing ones. Idempotent: existing directories are left alone. @@ -129,14 +142,20 @@ class UploadOperation : public Operation * the parent, so deeply-nested missing paths are created in one pass. * Must be called from within the processing thread. */ - std::expected ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& dir); + std::expected + ensureRemoteDirectoryExistsInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& dir); std::filesystem::perms determinePerms(std::filesystem::perms localPerms) const; + /** + * @brief Gives the remote temp file the modification time of the local file. A failure is only logged. + */ + void applyModificationTimeInStrand(SecureShell::SftpSession& sftp); + void cleanup(); private: - SecureShell::SftpSession* sftp_; + std::weak_ptr sftp_; std::weak_ptr fileStream_; UploadOperationOptions options_; /** @brief options_.remotePath, unset when none was given. */ diff --git a/backend/include/backend/sync/sync_session.hpp b/backend/include/backend/sync/sync_session.hpp index 8f4a4f51..f58bd5a9 100644 --- a/backend/include/backend/sync/sync_session.hpp +++ b/backend/include/backend/sync/sync_session.hpp @@ -58,6 +58,14 @@ class SyncSession : public std::enable_shared_from_this /// Number of scan entries (sum of both sides) above which /// @ref SharedData::Sync::DiffSummary::heavyCompare is set to true. std::uint64_t heavyCompareThreshold{5000}; + /** @brief The local filesystem folds case, the platform default: Windows and macOS do, others do not. */ +#if defined(_WIN32) || defined(__APPLE__) + bool localFoldsCase{true}; +#else + bool localFoldsCase{false}; +#endif + /** @brief The server looks like it folds case, see RemoteDeleteGuardDetail::looksCaseFolding. */ + bool remoteFoldsCase{false}; }; SyncSession(Options options, boost::asio::any_io_executor executor); @@ -160,20 +168,16 @@ class SyncSession : public std::enable_shared_from_this std::string const& parentRelKey ); - /** - * @brief Computes the enqueue plan from a SPARSE selection set: an entry X in - * @p selectedRelKeys means "X and every descendant is selected". See - * @ref SharedData::Sync::DiffTreeStore::buildPlan. Precondition: running on @ref strand(). - */ - std::vector buildEnqueuePlanInStrand( - SharedData::Sync::DiffSection section, - std::unordered_set const& selectedRelKeys - ) const; - struct ResolvedPlanEntry { SharedData::Sync::EnqueuePlanEntry entry; ResolvedPaths paths; + /** + * @brief What the compare saw at the path the entry replaces (the destination of a transfer) or deletes, + * which it may then only replace or delete while the path still holds it. A changed source is + * transferred as it is. + */ + SharedData::ComparedState compared{}; }; struct ResolvedPlan @@ -184,7 +188,8 @@ class SyncSession : public std::enable_shared_from_this }; /** - * @brief @ref buildEnqueuePlanInStrand with every entry resolved below the session roots. + * @brief The plan of a sparse selection (an entry in @p selectedRelKeys selects it and everything below it, see + * @ref SharedData::Sync::DiffTreeStore::buildPlan) with every entry resolved below the session roots. * Precondition: running on @ref strand(). */ ResolvedPlan resolvePlanInStrand( @@ -193,6 +198,10 @@ class SyncSession : public std::enable_shared_from_this ) const; private: + std::vector buildEnqueuePlanInStrand( + SharedData::Sync::DiffSection section, + std::unordered_set const& selectedRelKeys + ) const; SharedData::Sync::DiffTreeStore& storeFor(SharedData::Sync::DiffSection section); SharedData::Sync::DiffTreeStore const& storeFor(SharedData::Sync::DiffSection section) const; SharedData::Sync::ScanTrees scanTrees() const; diff --git a/backend/include/backend/windows/cotask_mem_string.hpp b/backend/include/backend/windows/cotask_mem_string.hpp index 68db1c9a..aa6bc28c 100644 --- a/backend/include/backend/windows/cotask_mem_string.hpp +++ b/backend/include/backend/windows/cotask_mem_string.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -55,7 +55,7 @@ class CoTaskMemString std::string toUtf8String() const { - return Nui::utf16ToUtf8(toStandardString()); + return Paths::wideToUtf8(toStandardString()); } private: diff --git a/backend/include/backend/windows/main_windows.hpp b/backend/include/backend/windows/main_windows.hpp index fca772ad..3638c871 100644 --- a/backend/include/backend/windows/main_windows.hpp +++ b/backend/include/backend/windows/main_windows.hpp @@ -10,8 +10,35 @@ struct Main::PlatformSpecifics EnableWindowsDragDrop enableDragDrop; }; +/** + * @brief The console the standard streams were pointed at. It writes and reads UTF-8 while this lives and gets its + * previous code pages back when this is destroyed, since the console belongs to the parent as well. + */ +class ConsoleAttachment +{ + public: + ConsoleAttachment() = default; + ~ConsoleAttachment(); + ConsoleAttachment(ConsoleAttachment&& other) noexcept; + ConsoleAttachment& operator=(ConsoleAttachment&& other) noexcept; + ConsoleAttachment(ConsoleAttachment const&) = delete; + ConsoleAttachment& operator=(ConsoleAttachment const&) = delete; + + private: + friend ConsoleAttachment attachParentConsole(); + + /** + * @brief The output code page to restore, 0 when it was not changed. + */ + unsigned int previousOutputCodePage_{0}; + /** + * @brief The input code page to restore, 0 when it was not changed. + */ + unsigned int previousInputCodePage_{0}; +}; + /** * @brief Attaches to the console of the process that started this one, if there is one, and points the standard - * streams at it. The program is a GUI program, without this its output goes nowhere. + * streams without a file or pipe at it. The program is a GUI program, without this its output goes nowhere. */ -void attachParentConsole(); \ No newline at end of file +ConsoleAttachment attachParentConsole(); \ No newline at end of file diff --git a/backend/source/backend/CMakeLists.txt b/backend/source/backend/CMakeLists.txt index c2071970..7d7b8500 100644 --- a/backend/source/backend/CMakeLists.txt +++ b/backend/source/backend/CMakeLists.txt @@ -5,7 +5,10 @@ add_library( session.cpp rpc_filesystem.cpp local_delete_guard.cpp + local_removal.cpp + local_operation_queue.cpp existence_check.cpp + rpc_path_parameters.cpp rpc_system.cpp theme_finder.cpp dropped_entry.cpp @@ -31,8 +34,10 @@ add_library( sftp/bulk_download_operation.cpp sftp/bulk_upload_operation.cpp sftp/bulk_resume_registry.cpp + sftp/compared_state_check.cpp sftp/delete_operation.cpp sftp/local_delete_operation.cpp + sftp/remote_delete_guard.cpp sftp/rename_operation.cpp sftp/archive_download_operation.cpp sftp/archive_upload_operation.cpp diff --git a/backend/source/backend/dropped_entry.cpp b/backend/source/backend/dropped_entry.cpp index 1e76b3e7..f15b97c1 100644 --- a/backend/source/backend/dropped_entry.cpp +++ b/backend/source/backend/dropped_entry.cpp @@ -1,36 +1,62 @@ #include +#include +#include +#include +#include -std::optional droppedItemFromPath(std::filesystem::path const& path) +#include + +namespace { - using FileType = SharedData::DirectoryEntry::FileType; + /** + * @brief The type of what lies at @p path itself: a link or junction is a symlink, never what it points to, + * like every upload from the explorer sees it. + */ + SharedData::FileType droppedTypeOf(std::filesystem::path const& path, std::filesystem::file_status const& status) + { + using FileType = SharedData::FileType; + std::error_code linkError{}; + if (std::filesystem::is_symlink(status) || Utility::isLinkOrJunction(path, linkError)) + return FileType::Symlink; + if (std::filesystem::is_directory(status)) + return FileType::Directory; + if (std::filesystem::is_regular_file(status)) + return FileType::Regular; + if (std::filesystem::is_socket(status)) + return FileType::Socket; + if (std::filesystem::is_character_file(status)) + return FileType::CharDevice; + if (std::filesystem::is_block_file(status)) + return FileType::BlockDevice; + if (std::filesystem::is_other(status)) + return FileType::Special; + return FileType::Unknown; + } +} +std::optional droppedItemFromPath(std::filesystem::path const& path) +{ auto localPath = Paths::LocalPath::make(path); if (!localPath) + { + Log::warn("Drop: '{}' is ignored: {}", Paths::nativeToUtf8(path), localPath.error().toString()); + return std::nullopt; + } + std::error_code statusError{}; + const auto status = std::filesystem::symlink_status(path, statusError); + if (status.type() == std::filesystem::file_type::not_found || statusError) + { + Log::warn( + "Drop: '{}' is ignored, it cannot be inspected: {}", + localPath->toUtf8(), + statusError ? Utility::errorMessage(statusError) : std::string{"not found"} + ); return std::nullopt; + } const auto name = localPath->filename(); auto entry = SharedData::DirectoryEntry{ .name = name ? name->value() : std::string{}, - .type = [&path]() - { - std::error_code statusError; - const auto status = std::filesystem::status(path, statusError); - if (std::filesystem::is_directory(status)) - return FileType::Directory; - else if (std::filesystem::is_regular_file(status)) - return FileType::Regular; - else if (std::filesystem::is_symlink(status)) - return FileType::Symlink; - else if (std::filesystem::is_socket(status)) - return FileType::Socket; - else if (std::filesystem::is_character_file(status)) - return FileType::CharDevice; - else if (std::filesystem::is_block_file(status)) - return FileType::BlockDevice; - else if (std::filesystem::is_other(status)) - return FileType::Special; - else - return FileType::Unknown; - }() + .type = droppedTypeOf(path, status), }; return SharedData::DroppedItem{.path = *std::move(localPath), .entry = std::move(entry)}; } diff --git a/backend/source/backend/existence_check.cpp b/backend/source/backend/existence_check.cpp index 85b293ea..2585d2a5 100644 --- a/backend/source/backend/existence_check.cpp +++ b/backend/source/backend/existence_check.cpp @@ -1,32 +1,36 @@ #include +#include +#include +#include #include #include #include #include +#include #include #include #include std::expected, std::string> localPathsExist(std::vector const& paths) { + const auto localPaths = RpcHelper::parseLocalPaths(paths); + if (!localPaths) + return std::unexpected(localPaths.error()); std::vector results; - results.reserve(paths.size()); - for (auto const& pathUtf8 : paths) + results.reserve(localPaths->size()); + for (auto const& path : *localPaths) { - const auto path = Paths::LocalPath::fromUtf8(pathUtf8); - if (!path) - return std::unexpected(path.error().toString()); std::error_code ec; - const auto status = std::filesystem::symlink_status(path->native(), ec); + const auto status = std::filesystem::symlink_status(path.native(), ec); if (status.type() == std::filesystem::file_type::not_found) { results.push_back(false); continue; } if (ec) - return std::unexpected(fmt::format("'{}': {}", pathUtf8, ec.message())); + return std::unexpected(fmt::format("'{}': {}", path.toUtf8(), Utility::errorMessage(ec))); results.push_back(true); } return results; @@ -34,79 +38,104 @@ std::expected, std::string> localPathsExist(std::vector, std::string> remotePathsExistByStat( + /** + * @brief Whether @p path exists, nothing but "no such file" counts as absent. + */ + std::expected remotePathExists( SecureShell::SftpSession& session, - std::vector const& paths, + Paths::RemotePath const& path, std::chrono::milliseconds timeout ) { - std::vector results; - results.reserve(paths.size()); - for (auto const& path : paths) + auto future = session.lstat(path); + if (future.wait_for(timeout) != std::future_status::ready) + return std::unexpected(fmt::format("'{}': stat timed out", path.value())); + const auto result = future.get(); + if (!result && result.error().sftpError != SSH_FX_NO_SUCH_FILE) + return std::unexpected(fmt::format("'{}': {}", path.value(), result.error().toString())); + return result.has_value(); + } + + /** + * @brief The names in @p parent, without resolving symlinks. Nothing for a directory that cannot be listed + * because of its permissions, like a drop box; an empty set for one that does not exist yet. + */ + std::expected>, std::string> + listNames(SecureShell::SftpSession& session, Paths::RemotePath const& parent, std::chrono::milliseconds timeout) + { + auto future = session.listDirectory(parent, /*resolveLinks=*/false); + if (future.wait_for(timeout) != std::future_status::ready) + return std::unexpected(fmt::format("'{}': listing timed out", parent.value())); + const auto result = future.get(); + if (!result) { - auto future = session.lstat(path); - if (future.wait_for(timeout) != std::future_status::ready) - return std::unexpected(fmt::format("'{}': stat timed out", path.value())); - const auto result = future.get(); - if (!result && result.error().sftpError != SSH_FX_NO_SUCH_FILE) - return std::unexpected(fmt::format("'{}': {}", path.value(), result.error().toString())); - results.push_back(result.has_value()); + if (result.error().sftpError == SSH_FX_NO_SUCH_FILE) + return std::unordered_set{}; + if (result.error().sftpError == SSH_FX_PERMISSION_DENIED) + return std::nullopt; + return std::unexpected(fmt::format("'{}': {}", parent.value(), result.error().toString())); } - return results; + std::unordered_set names; + names.reserve(result->size()); + for (auto const& entry : *result) + names.insert(entry.name); + return names; } /** - * @brief One listing per parent directory instead of a stat per path. A parent that does not exist yet holds - * nothing. + * @brief Whether the queried children of one parent exist, from one listing of it. A name the listing lacks + * while it holds a case or normalization variant is looked up, since the server may fold names. */ - std::expected, std::string> remotePathsExistByListing( + std::expected checkByListing( SecureShell::SftpSession& session, + Paths::RemotePath const& parent, std::vector const& paths, - std::chrono::milliseconds timeout + std::vector const& indices, + std::chrono::milliseconds timeout, + std::vector& results ) { - const auto listParent = - [&session, timeout](Paths::RemotePath const& parent) -> std::expected, std::string> + auto names = listNames(session, parent, timeout); + if (!names) + return std::unexpected(names.error()); + if (!*names) { - auto future = session.listDirectory(parent); - if (future.wait_for(timeout) != std::future_status::ready) - return std::unexpected(fmt::format("'{}': listing timed out", parent.value())); - const auto result = future.get(); - if (!result) + for (const auto index : indices) { - if (result.error().sftpError == SSH_FX_NO_SUCH_FILE) - return std::unordered_set{}; - return std::unexpected(fmt::format("'{}': {}", parent.value(), result.error().toString())); + auto exists = remotePathExists(session, paths[index], timeout); + if (!exists) + return std::unexpected(exists.error()); + results[index] = *exists; } - std::unordered_set names; - names.reserve(result->size()); - for (auto const& entry : *result) - names.insert(entry.name); - return names; - }; + return {}; + } - std::unordered_map> listingByParent; - std::vector results; - results.reserve(paths.size()); - for (auto const& path : paths) + std::optional> foldedNames{}; + for (const auto index : indices) { - const auto parent = path.parent(); - if (!parent) - return std::unexpected(fmt::format("'{}' has no parent directory", path.value())); - auto iter = listingByParent.find(parent->value()); - if (iter == listingByParent.end()) + const auto name = std::string{paths[index].filename()->view()}; + if ((*names)->contains(name)) + { + results[index] = true; + continue; + } + if (!foldedNames) + { + foldedNames.emplace(); + for (auto const& listed : **names) + foldedNames->insert(SharedData::Sync::caseFoldedName(listed)); + } + if (!foldedNames->contains(SharedData::Sync::caseFoldedName(name))) { - auto listing = listParent(*parent); - if (!listing) - return std::unexpected(listing.error()); - iter = listingByParent.emplace(parent->value(), *std::move(listing)).first; + results[index] = false; + continue; } - results.push_back(iter->second.contains(std::string{path.filename()->view()})); + auto exists = remotePathExists(session, paths[index], timeout); + if (!exists) + return std::unexpected(exists.error()); + results[index] = *exists; } - return results; + return {}; } } @@ -116,16 +145,40 @@ std::expected, std::string> remotePathsExist( std::chrono::milliseconds timeout ) { - std::vector remotePaths; - remotePaths.reserve(paths.size()); - for (auto const& path : paths) + const auto parsed = RpcHelper::parseRemotePaths(paths); + if (!parsed) + return std::unexpected(parsed.error()); + auto const& remotePaths = *parsed; + // Parents in the order they first appear, with the indices of their queried children. + std::vector>> byParent; + std::unordered_map parentIndex; + for (std::size_t index = 0; index < remotePaths.size(); ++index) + { + const auto parent = remotePaths[index].parent(); + if (!parent) + return std::unexpected(fmt::format("'{}' has no parent directory", remotePaths[index].value())); + const auto [found, inserted] = parentIndex.try_emplace(parent->value(), byParent.size()); + if (inserted) + byParent.emplace_back(*parent, std::vector{}); + byParent[found->second].second.push_back(index); + } + + std::vector results(remotePaths.size(), false); + for (auto const& [parent, indices] : byParent) { - auto remotePath = Paths::RemotePath::make(path); - if (!remotePath) - return std::unexpected(remotePath.error().toString()); - remotePaths.push_back(*std::move(remotePath)); + if (indices.size() >= remoteListingThreshold) + { + if (auto checked = checkByListing(session, parent, remotePaths, indices, timeout, results); !checked) + return std::unexpected(checked.error()); + continue; + } + for (const auto index : indices) + { + auto exists = remotePathExists(session, remotePaths[index], timeout); + if (!exists) + return std::unexpected(exists.error()); + results[index] = *exists; + } } - if (remotePaths.size() < listingThreshold) - return remotePathsExistByStat(session, remotePaths, timeout); - return remotePathsExistByListing(session, remotePaths, timeout); + return results; } diff --git a/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp b/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp index 1191f3d5..448935bc 100644 --- a/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp +++ b/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -63,7 +64,7 @@ namespace FileTracking { namespace fs = std::filesystem; - auto const dirName = dirPath.filename().string(); + auto const dirName = Paths::nativeToUtf8(dirPath.filename()); for (auto const& sid : skipIds) if (sid == dirName) return false; @@ -151,11 +152,18 @@ namespace FileTracking if (!isOrphanedDir(dirPath, skipIds)) continue; - Log::info("forceCleanOrphanedInstanceDirs: removing orphaned instance '{}'", dirPath.filename().string()); + Log::info( + "forceCleanOrphanedInstanceDirs: removing orphaned instance '{}'", + Paths::nativeToUtf8(dirPath.filename()) + ); std::error_code rmErr; Utility::removeAllNoFollow(dirPath, rmErr); if (rmErr) - Log::warn("forceCleanOrphanedInstanceDirs: failed to remove '{}': {}", dirPath.filename().string(), rmErr.message()); + Log::warn( + "forceCleanOrphanedInstanceDirs: failed to remove '{}': {}", + Paths::nativeToUtf8(dirPath.filename()), + Utility::errorMessage(rmErr) + ); else ++removed; } @@ -182,7 +190,7 @@ namespace FileTracking continue; auto const& dirPath = entry.path(); - auto const dirName = dirPath.filename().string(); + auto const dirName = Paths::nativeToUtf8(dirPath.filename()); bool skip = false; for (auto const& sid : skipIds) @@ -228,7 +236,7 @@ namespace FileTracking Log::warn( "scanAndCleanDeadInstanceDirs: could not write deadAt for '{}': {}", dirName, - exc.what() + Utility::errorMessage(exc) ); } }; @@ -254,7 +262,7 @@ namespace FileTracking Log::warn( "scanAndCleanDeadInstanceDirs: error reading metadata for '{}': {}", dirName, - exc.what() + Utility::errorMessage(exc) ); } @@ -272,7 +280,7 @@ namespace FileTracking std::error_code rmErr; Utility::removeAllNoFollow(dirPath, rmErr); if (rmErr) - Log::warn("Failed to remove dead instance '{}': {}", dirName, rmErr.message()); + Log::warn("Failed to remove dead instance '{}': {}", dirName, Utility::errorMessage(rmErr)); } } } // namespace @@ -434,7 +442,11 @@ namespace FileTracking }); } if (walkError) - Log::warn("FileTracking::fileStates: walking '{}' failed: {}", instanceId, walkError.message()); + Log::warn( + "FileTracking::fileStates: walking '{}' failed: {}", + instanceId, + Utility::errorMessage(walkError) + ); reply({{"success", true}, {"files", std::move(files)}}); } ); @@ -483,7 +495,7 @@ namespace FileTracking } catch (std::exception const& exc) { - Log::error("TempDirInstanceManager::createInstance failed: {}", exc.what()); + Log::error("TempDirInstanceManager::createInstance failed: {}", Utility::errorMessage(exc)); return nullptr; } } diff --git a/backend/source/backend/file_tracking/temp_dir_instancing.cpp b/backend/source/backend/file_tracking/temp_dir_instancing.cpp index 4133f8c9..a29c0088 100644 --- a/backend/source/backend/file_tracking/temp_dir_instancing.cpp +++ b/backend/source/backend/file_tracking/temp_dir_instancing.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -36,7 +38,9 @@ namespace FileTracking std::error_code err; std::filesystem::create_directories(dir, err); if (err) - throw std::runtime_error{"Failed to create instance directory: " + err.message()}; + throw std::runtime_error{ + fmt::format("Failed to create instance directory: {}", Utility::errorMessage(err)) + }; return dir; } @@ -111,7 +115,7 @@ namespace FileTracking strand_, [hub = hub_, name = eventName_, pld = std::move(payload)]() mutable { - hub->callRemote(name, pld); + RpcHelper::callRemoteSafely(*hub, name, pld); } ); } @@ -264,7 +268,11 @@ namespace FileTracking std::error_code err; Utility::removeAllNoFollow(impl_->instanceDir, err); if (err) - Log::warn("cleanupNow: failed to remove '{}': {}", impl_->instanceDir.string(), err.message()); + Log::warn( + "cleanupNow: failed to remove '{}': {}", + Paths::nativeToUtf8(impl_->instanceDir), + Utility::errorMessage(err) + ); impl_->valid = false; } @@ -305,7 +313,9 @@ namespace FileTracking } catch (std::exception const& exc) { - Log::error("writeDeadTimestamp: failed for instance '{}': {}", impl_->instanceId, exc.what()); + Log::error( + "writeDeadTimestamp: failed for instance '{}': {}", impl_->instanceId, Utility::errorMessage(exc) + ); } } } diff --git a/backend/source/backend/linux/opener_linux.cpp b/backend/source/backend/linux/opener_linux.cpp index 19e87821..f33be552 100644 --- a/backend/source/backend/linux/opener_linux.cpp +++ b/backend/source/backend/linux/opener_linux.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -419,7 +420,7 @@ std::optional Opener::riskOf(std::filesystem::path const& path) con return std::string{"backend.opener.risk.program"}; // Types a desktop starts as programs instead of showing them. static constexpr std::string_view runnable[] = {".desktop", ".appimage", ".run", ".jar"}; - auto extension = path.extension().string(); + auto extension = Paths::nativeToUtf8(path.extension()); std::ranges::transform(extension, extension.begin(), [](unsigned char character) { return std::tolower(character); }); if (std::ranges::find(runnable, std::string_view{extension}) != std::end(runnable)) return std::string{"backend.opener.risk.dangerousType"}; @@ -428,7 +429,12 @@ std::optional Opener::riskOf(std::filesystem::path const& path) con std::expected Opener::openFile(std::filesystem::path const& path, bool openWith) { - Log::info("Opener: openFile path='{}' openWith={} parentWindow='{}'", path.string(), openWith, impl_->parentWindow); + Log::info( + "Opener: openFile path='{}' openWith={} parentWindow='{}'", + Paths::nativeToUtf8(path), + openWith, + impl_->parentWindow + ); if (!impl_->connection) { @@ -439,7 +445,7 @@ std::expected Opener::openFile(std::filesystem::path const& p Utility::FdGuard fd{::open(path.c_str(), O_RDONLY)}; if (!fd.valid()) { - Log::error("Opener: failed to open fd for '{}': {}", path.string(), std::strerror(errno)); + Log::error("Opener: failed to open fd for '{}': {}", Paths::nativeToUtf8(path), std::strerror(errno)); return std::unexpected{Utility::localizedMessage("backend.opener.openFileFailed", std::strerror(errno))}; } @@ -496,8 +502,7 @@ SharedData::OpenerCapabilities Opener::capabilities() const std::expected Opener::openInFileManager(std::filesystem::path const& path) { - Log::info( - "Opener: openInFileManager path='{}' parentWindow='{}'", path.string(), impl_->parentWindow); + Log::info("Opener: openInFileManager path='{}' parentWindow='{}'", Paths::nativeToUtf8(path), impl_->parentWindow); if (!impl_->connection) { @@ -512,7 +517,11 @@ std::expected Opener::openInFileManager(std::filesystem::path { auto result = launchDefaultForPath(path); if (!result) - Log::error("Opener: g_app_info_launch_default_for_uri failed for '{}': {}", path.string(), result.error()); + Log::error( + "Opener: g_app_info_launch_default_for_uri failed for '{}': {}", + Paths::nativeToUtf8(path), + result.error() + ); return result; } @@ -522,7 +531,7 @@ std::expected Opener::openInFileManager(std::filesystem::path Utility::FdGuard fd{::open(path.c_str(), O_RDONLY)}; if (!fd.valid()) { - Log::error("Opener: failed to open fd for '{}': {}", path.string(), std::strerror(errno)); + Log::error("Opener: failed to open fd for '{}': {}", Paths::nativeToUtf8(path), std::strerror(errno)); return std::unexpected{Utility::localizedMessage("backend.opener.openFileFailed", std::strerror(errno))}; } diff --git a/backend/source/backend/local_delete_guard.cpp b/backend/source/backend/local_delete_guard.cpp index cd786edc..51212eab 100644 --- a/backend/source/backend/local_delete_guard.cpp +++ b/backend/source/backend/local_delete_guard.cpp @@ -1,11 +1,173 @@ #include +#include +#include -#include +#include -std::optional localHomeDirectory() +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# include +#else +# include +# include +#endif + +namespace +{ + bool foldsCase([[maybe_unused]] Paths::LocalPath const& path) + { +#ifdef __APPLE__ + return true; +#else + return path.platform() == Paths::LocalPlatform::Windows; +#endif + } + + bool containsAsText(Paths::LocalPath const& target, Paths::LocalPath const& home) + { + return home.isWithin(target) || (foldsCase(target) && home.isWithinIgnoringCase(target)); + } + + /** + * @brief True when @p target is an existing directory that is @p home or one of its ancestors on disk. + */ + bool isHomeOrAncestorOnDisk(Paths::LocalPath const& target, Paths::LocalPath const& home) + { + if (target.platform() != Paths::compiledLocalPlatform() || home.platform() != target.platform()) + return false; + const auto native = target.native(); + std::error_code error{}; + if (!std::filesystem::is_directory(std::filesystem::symlink_status(native, error))) + return false; + for (std::optional ancestor = home; ancestor; ancestor = ancestor->parent()) + { + if (std::filesystem::equivalent(ancestor->native(), native, error) && !error) + return true; + } + return false; + } + + void addHome(std::vector& homes, std::filesystem::path const& native) + { + if (native.empty()) + return; + try + { + auto home = Paths::LocalPath::make(native); + if (!home) + { + Log::warn("Ignoring the home directory candidate {}", home.error().toString()); + return; + } + if (std::ranges::find(homes, *home) == homes.end()) + homes.push_back(*std::move(home)); + } + catch (std::exception const& exception) + { + Log::warn( + "Ignoring a home directory candidate that is not representable: {}", Utility::errorMessage(exception) + ); + } + } + +#ifdef _WIN32 + struct CoTaskMemoryDeleter + { + void operator()(wchar_t* memory) const noexcept + { + CoTaskMemFree(memory); + } + }; + + void addEnvironmentHome(std::vector& homes, wchar_t const* name) + { + if (auto const* value = _wgetenv(name); value != nullptr && *value != L'\0') + addHome(homes, std::filesystem::path{value}); + } +#endif +} + +std::expected +refuseProtectedLocalDelete(Paths::LocalPath const& target, std::vector const& homes) { - auto home = Paths::LocalPath::make(Nui::resolvePath("%userprofile%")); - if (!home) - return std::nullopt; - return *std::move(home); + if (target.empty()) + return std::unexpected(std::string{"no path"}); + if (target.isRoot()) + return std::unexpected(fmt::format("'{}' is a root directory", target.toUtf8())); + const auto protectedHome = std::ranges::find_if( + homes, + [&target](Paths::LocalPath const& home) + { + return containsAsText(target, home) || isHomeOrAncestorOnDisk(target, home); + } + ); + if (protectedHome != homes.end()) + return std::unexpected( + fmt::format("'{}' is or contains the home directory '{}'", target.toUtf8(), protectedHome->toUtf8()) + ); + return {}; +} + +std::expected +localDeleteTarget(std::string const& utf8, std::vector const& homes) +{ + auto target = Paths::LocalPath::fromUtf8(utf8); + if (!target) + return std::unexpected(target.error().toString()); + + // LocalPath resolves ".." lexically, which goes elsewhere when a directory on the way is a symlink. + std::string generic{utf8}; + if (target->platform() == Paths::LocalPlatform::Windows) + std::ranges::replace(generic, '\\', '/'); + const auto isParentReference = [](std::string_view name) + { + return name == ".."; + }; + if (std::ranges::any_of(Paths::Detail::splitAtSlashes(generic), isParentReference)) + return std::unexpected(fmt::format("'{}' contains '..'", utf8)); + + if (auto allowed = refuseProtectedLocalDelete(*target, homes); !allowed) + return std::unexpected(std::move(allowed).error()); + return *std::move(target); +} + +std::vector localHomeDirectories() +{ + std::vector homes; +#ifdef _WIN32 + PWSTR rawProfile = nullptr; + const auto result = SHGetKnownFolderPath(FOLDERID_Profile, KF_FLAG_DEFAULT, nullptr, &rawProfile); + const std::unique_ptr profile{rawProfile}; + if (SUCCEEDED(result) && profile != nullptr) + addHome(homes, std::filesystem::path{profile.get()}); + addEnvironmentHome(homes, L"USERPROFILE"); + addEnvironmentHome(homes, L"HOME"); +#else + if (auto const* home = std::getenv("HOME"); home != nullptr && *home != '\0') + addHome(homes, std::filesystem::path{home}); + + const auto suggestedSize = sysconf(_SC_GETPW_R_SIZE_MAX); + std::vector buffer(suggestedSize > 0 ? static_cast(suggestedSize) : 16384u); + passwd entry{}; + passwd* found = nullptr; + if (getpwuid_r(getuid(), &entry, buffer.data(), buffer.size(), &found) == 0 && found != nullptr && + found->pw_dir != nullptr) + addHome(homes, std::filesystem::path{found->pw_dir}); +#endif + if (homes.empty()) + Log::warn("The home directory is unknown, local deletes only refuse root directories."); + return homes; } diff --git a/backend/source/backend/local_operation_queue.cpp b/backend/source/backend/local_operation_queue.cpp new file mode 100644 index 00000000..95fd5ee6 --- /dev/null +++ b/backend/source/backend/local_operation_queue.cpp @@ -0,0 +1,306 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +LocalOperationQueue::LocalOperationQueue( + boost::asio::any_io_executor executor, + std::shared_ptr> strand, + Nui::Window& wnd, + Nui::RpcHub& hub, + std::shared_ptr localDeletionPrevented +) + : RpcHelper::StrandRpc{executor, std::move(strand), wnd, hub} + , localDeletionPrevented_{std::move(localDeletionPrevented)} +{} + +std::string LocalOperationQueue::rpcName(std::string_view event) const +{ + return fmt::format("LocalOperationQueue::{}", event); +} + +void LocalOperationQueue::registerRpc() +{ + on(rpcName("addDelete")) + .perform( + [weak = weak_from_this()]( + RpcHelper::RpcOnce&& reply, std::string const& operationIdString, std::vector const& paths + ) + { + auto self = weak.lock(); + if (!self) + return reply.error("The local operation queue no longer exists"); + const auto added = + self->addExplorerDelete(Ids::makeOperationId(operationIdString), paths, std::nullopt, false); + if (!added) + return reply.error(added.error()); + reply(SharedData::success()); + } + ); + + on(rpcName("cancel")) + .perform( + [weak = weak_from_this()](RpcHelper::RpcOnce&& reply, Ids::OperationId operationId) + { + auto self = weak.lock(); + if (!self) + return reply.error("The local operation queue no longer exists"); + self->cancel(operationId); + reply(SharedData::success()); + } + ); +} + +std::expected LocalOperationQueue::addDelete(DeleteRequest request) +{ + if (request.targets.empty()) + return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = "no targets"}); + const auto homes = localHomeDirectories(); + for (auto const& target : request.targets) + { + if (auto allowed = refuseProtectedLocalDelete(target, homes); !allowed) + { + Log::error("Refusing to delete locally: {}", allowed.error()); + return std::unexpected( + Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = target.toUtf8()} + ); + } + } + + auto parent = LocalDeleteOperation::commonParentOf(request.targets); + const auto priority = request.mode == SharedData::OperationMode::PriorityQueued; + auto operation = std::make_unique(LocalDeleteOperation::LocalDeleteOperationOptions{ + .itemsRemovedProgress = + [weak = weak_from_this(), operationId = request.operationId]( + std::string const& path, std::uint64_t itemsDeleted, std::uint64_t totalItems + ) + { + if (auto self = weak.lock(); self) + { + RpcHelper::callRemoteSafely( + *self->hub_, + self->rpcName("onDeleteProgress"), + SharedData::BulkDeleteProgress{ + .operationId = operationId, + .currentFile = path, + .filesDeleted = itemsDeleted, + .totalFiles = totalItems, + } + ); + } + }, + .scanProgress = + [weak = weak_from_this(), operationId = request.operationId](std::uint64_t bytes, std::uint64_t entries) + { + if (auto self = weak.lock(); self) + { + RpcHelper::callRemoteSafely( + *self->hub_, + self->rpcName("onScanProgress"), + SharedData::ScanProgress{ + .operationId = operationId, + .totalBytes = bytes, + .currentIndex = entries, + .totalScanned = entries, + } + ); + } + }, + .targets = std::move(request.targets), + .recursive = request.recursive, + .comparedStates = std::move(request.comparedStates), + }); + + const auto position = priority ? std::ranges::find_if( + deletes_, + [](QueuedDelete const& queued) + { + return !queued.priority; + } + ) + : deletes_.end(); + deletes_.insert( + position, + QueuedDelete{ + .operationId = request.operationId, + .operation = std::move(operation), + .owner = std::move(request.owner), + .held = request.held && !priority, + .priority = priority, + } + ); + + RpcHelper::callRemoteSafely( + *hub_, + rpcName("onOperationAdded"), + SharedData::OperationAdded{ + .operationId = request.operationId, + .type = SharedData::OperationType::LocalDelete, + .mode = request.mode, + .insertRefresh = request.insertRefresh, + .localPath = std::move(parent), + .scanPhase = request.recursive, + } + ); + schedule(); + return {}; +} + +std::expected LocalOperationQueue::addExplorerDelete( + Ids::OperationId operationId, + std::vector const& pathsUtf8, + std::optional owner, + bool held +) +{ + if (!localDeletionPrevented_ || localDeletionPrevented_->load()) + { + Log::warn("A local delete was asked for, but deletion is prevented by configuration."); + return std::unexpected(std::string{"backend.rpcFilesystem.deletionPrevented"}); + } + + std::vector targets; + targets.reserve(pathsUtf8.size()); + const auto homes = localHomeDirectories(); + for (auto const& path : pathsUtf8) + { + auto target = localDeleteTarget(path, homes); + if (!target) + { + Log::error("Refusing to delete locally: {}", target.error()); + return std::unexpected(Utility::localizedMessage("backend.rpcFilesystem.removeRefused", path)); + } + targets.push_back(*std::move(target)); + } + + return addDelete({ + .operationId = std::move(operationId), + .targets = std::move(targets), + .recursive = true, + .insertRefresh = true, + .owner = std::move(owner), + .held = held, + }) + .transform_error( + [](Operation::Error const& error) + { + return error.toString(); + } + ); +} + +void LocalOperationQueue::hold(Ids::SessionId const& owner, bool held) +{ + for (auto& queued : deletes_) + { + if (queued.owner == owner && !queued.priority) + queued.held = held; + } + if (!held) + schedule(); +} + +void LocalOperationQueue::release(Ids::SessionId const& owner) +{ + for (auto& queued : deletes_) + { + if (queued.owner != owner) + continue; + queued.owner.reset(); + queued.held = false; + } + schedule(); +} + +void LocalOperationQueue::cancel(Ids::OperationId const& operationId) +{ + const auto found = std::ranges::find(deletes_, operationId, &QueuedDelete::operationId); + if (found == deletes_.end()) + { + Log::info("LocalOperationQueue: '{}' is not queued, nothing to cancel.", operationId.value()); + return; + } + std::ignore = found->operation->cancel(true); + complete(*found, SharedData::OperationCompletionReason::Canceled, std::nullopt); + deletes_.erase(found); +} + +bool LocalOperationQueue::work() +{ + const auto runnable = [](QueuedDelete const& queued) + { + return !queued.held; + }; + const auto next = std::ranges::find_if(deletes_, runnable); + if (next == deletes_.end()) + return false; + + const auto result = next->operation->work(); + if (!result) + { + complete(*next, SharedData::OperationCompletionReason::Failed, result.error()); + deletes_.erase(next); + } + else if (*result == Operation::WorkStatus::Complete) + { + complete(*next, SharedData::OperationCompletionReason::Completed, std::nullopt); + deletes_.erase(next); + } + return std::ranges::any_of(deletes_, runnable); +} + +void LocalOperationQueue::schedule() +{ + if (scheduled_) + return; + scheduled_ = true; + boost::asio::post( + *strand_, + [weak = weak_from_this()]() + { + auto self = weak.lock(); + if (!self) + return; + self->scheduled_ = false; + if (self->work()) + self->schedule(); + } + ); +} + +void LocalOperationQueue::complete( + QueuedDelete const& queued, + SharedData::OperationCompletionReason reason, + std::optional error +) +{ + if (error) + Log::error("Local delete failed: {}", error->toString()); + OperationCompleted completed{ + .reason = reason, + .operationId = queued.operationId, + .completionTime = std::chrono::system_clock::now(), + .error = std::move(error), + .failedEntries = queued.operation->getFailed(), + }; + if (completionObserver_) + completionObserver_(completed); + RpcHelper::callRemoteSafely(*hub_, rpcName("onOperationCompleted"), completed); +} + +void LocalOperationQueue::setCompletionObserver(std::function observer) +{ + completionObserver_ = std::move(observer); +} diff --git a/backend/source/backend/local_removal.cpp b/backend/source/backend/local_removal.cpp new file mode 100644 index 00000000..98827881 --- /dev/null +++ b/backend/source/backend/local_removal.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +std::expected +removeLocalItems(std::span pathsUtf8, std::vector const& homes) +{ + std::vector targets; + targets.reserve(pathsUtf8.size()); + for (auto const& pathUtf8 : pathsUtf8) + { + auto target = localDeleteTarget(pathUtf8, homes); + if (!target) + return std::unexpected(LocalRemovalRefusal{.path = pathUtf8, .reason = std::move(target).error()}); + targets.push_back(*std::move(target)); + } + + LocalRemoval removal{}; + const auto addFailure = [&removal](std::filesystem::path const& path, std::error_code const& error) + { + const auto pathUtf8 = Paths::nativeToUtf8(path); + const auto message = Utility::errorMessage(error); + Log::error("Failed to remove local item '{}': {}", pathUtf8, message); + removal.failures.push_back(fmt::format("{}: {}", pathUtf8, message)); + }; + for (auto const& target : targets) + { + std::error_code error{}; + if (Utility::removeNoFollow(target.native(), error)) + ++removal.removed; + if (error) + addFailure(target.native(), error); + } + return removal; +} diff --git a/backend/source/backend/mac/opener_mac.mm b/backend/source/backend/mac/opener_mac.mm index 215d7e96..b6d44451 100644 --- a/backend/source/backend/mac/opener_mac.mm +++ b/backend/source/backend/mac/opener_mac.mm @@ -1,4 +1,5 @@ #include +#include #include #include @@ -82,7 +83,7 @@ void chooseApplicationAndOpen(NSURL* url) static constexpr std::string_view runnable[] = { ".command", ".tool", ".terminal", ".pkg", ".mpkg", ".workflow", ".scpt", ".applescript", ".jar", ".webloc", }; - auto extension = path.extension().string(); + auto extension = Paths::nativeToUtf8(path.extension()); std::ranges::transform(extension, extension.begin(), [](unsigned char character) { return std::tolower(character); }); if (std::ranges::find(runnable, std::string_view{extension}) != std::end(runnable)) return std::string{"backend.opener.risk.dangerousType"}; @@ -104,7 +105,7 @@ void chooseApplicationAndOpen(NSURL* url) std::expected Opener::openFile(std::filesystem::path const& path, bool openWith) { - Log::info("Opener: openFile path='{}' openWith={}", path.string(), openWith); + Log::info("Opener: openFile path='{}' openWith={}", Paths::nativeToUtf8(path), openWith); std::error_code existsError; if (!path.is_absolute() || !std::filesystem::exists(path, existsError)) @@ -123,7 +124,7 @@ void chooseApplicationAndOpen(NSURL* url) } if (![[NSWorkspace sharedWorkspace] openURL:url]) - return std::unexpected{Utility::localizedMessage("backend.opener.mac.openFailed", path.string())}; + return std::unexpected{Utility::localizedMessage("backend.opener.mac.openFailed", Paths::nativeToUtf8(path))}; } return {}; } @@ -136,7 +137,7 @@ void chooseApplicationAndOpen(NSURL* url) std::expected Opener::openInFileManager(std::filesystem::path const& path) { - Log::info("Opener: openInFileManager path='{}'", path.string()); + Log::info("Opener: openInFileManager path='{}'", Paths::nativeToUtf8(path)); std::error_code directoryCheckError; const bool isDirectory = std::filesystem::is_directory(path, directoryCheckError); @@ -148,7 +149,7 @@ void chooseApplicationAndOpen(NSURL* url) { NSString* const pathString = [NSString stringWithUTF8String:path.c_str()]; if (![workspace selectFile:nil inFileViewerRootedAtPath:pathString]) - return std::unexpected{Utility::localizedMessage("backend.opener.mac.openFailed", path.string())}; + return std::unexpected{Utility::localizedMessage("backend.opener.mac.openFailed", Paths::nativeToUtf8(path))}; return {}; } [workspace activateFileViewerSelectingURLs:@[fileUrl(path, false)]]; diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index 1e05f773..5b34c19e 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -1,4 +1,5 @@ #include +#include #ifdef _WIN32 # include @@ -14,9 +15,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -35,6 +38,9 @@ #include #include #include +#include +#include +#include #include #include @@ -138,7 +144,7 @@ namespace // Check if file exists and return 404 if not if (!std::filesystem::exists(file)) { - Log::error("File not found: '{}'", file.string()); + Log::error("File not found: '{}'", Paths::nativeToUtf8(file)); return CustomSchemeResponse{ .statusCode = 404, .reasonPhrase = "Not Found", @@ -148,11 +154,11 @@ namespace // Do not forget to allow CORS {"Access-Control-Allow-Origin"s, "*"s}, }, - .body = "Not Found: "s + file.string(), + .body = fmt::format("Not Found: {}", Paths::nativeToUtf8(file)), }; } - Log::debug("Serving file: '{}'", file.string()); + Log::debug("Serving file: '{}'", Paths::nativeToUtf8(file)); // Stream the file body via bodyFile — avoids loading the whole file into memory // (matters for large assets like DWARF blobs). Empty files yield a 204. @@ -163,7 +169,7 @@ namespace // Roar's MIME table has no entry for .wasm; emit application/wasm explicitly so that // WebAssembly.instantiateStreaming() accepts the response and compiles the module as // bytes arrive (otherwise the browser falls back to buffering the full binary first). - const auto ext = file.extension().string(); + const auto ext = Paths::nativeToUtf8(file.extension()); const auto mime = (ext == ".wasm") ? std::string{"application/wasm"} : Roar::extensionToMime(ext).value_or("application/octet-stream"); @@ -309,9 +315,11 @@ void Main::registerRpc() }, *opener_ ); + sshSessionManager_->preventLocalDeletion(true); return; } + sshSessionManager_->preventLocalDeletion(holder.stateCache().localFilesystemOptions.preventDeletion); rpcFilesystem_ = std::make_unique( window_.getExecutor(), window_, @@ -377,7 +385,8 @@ void Main::registerInitialWarningGetter() false; #endif - hub_.callRemote( + RpcHelper::callRemoteSafely( + hub_, responseId, nlohmann::json{ {"warnings", initialPersistenceLoadWarnings_}, @@ -394,7 +403,8 @@ void Main::registerVersionGetter() "Main::getVersion", [this](std::string responseId) { - hub_.callRemote( + RpcHelper::callRemoteSafely( + hub_, responseId, nlohmann::json{ {"version", VERSION}, @@ -443,17 +453,20 @@ void Main::startChildSignalTimer() int main(int const argc, char const* const* argv) { + // Reading the umask changes it for a moment on macOS, so it is read before any thread creates files. + static_cast(Utility::processUmask()); #ifdef _WIN32 // A GUI program has no console; help and --console write to the one it was started from. - for (int index = 1; index < argc; ++index) - { - const std::string_view argument{argv[index]}; - if (argument == "--help" || argument == "--console") - { - attachParentConsole(); - break; - } - } + const auto arguments = std::span{argv, static_cast(argc)} | std::views::drop(1); + const auto console = std::ranges::any_of( + arguments, + [](std::string_view argument) + { + return argument == "--help" || argument == "--console"; + } + ) + ? attachParentConsole() + : ConsoleAttachment{}; #endif auto options = parseProgramOptions(argc, argv); if (!options) diff --git a/backend/source/backend/process/environment.cpp b/backend/source/backend/process/environment.cpp index 3d4b4286..feb94a32 100644 --- a/backend/source/backend/process/environment.cpp +++ b/backend/source/backend/process/environment.cpp @@ -1,8 +1,10 @@ #include #include +#include #include +#include #include #include #include @@ -13,6 +15,20 @@ namespace { constexpr std::string_view originalValuePrefix = "NUI_SFTP_APPIMAGE_ORIGINAL_"; constexpr std::string_view unsetMarkerPrefix = "NUI_SFTP_APPIMAGE_UNSET_"; + + /** + * @brief The text of an environment key or value as UTF-8, converted from the wide text on Windows. + */ + template + std::string utf8Of(ViewT const& view) + { + const auto native = view.native(); +#ifdef _WIN32 + return Paths::wideToUtf8(std::wstring_view{native.data(), native.size()}); +#else + return std::string{native.data(), native.size()}; +#endif + } } Environment::Environment( @@ -38,17 +54,7 @@ void Environment::loadFromCurrent() for (auto iter = currentEnv.begin(); iter != currentEnv.end(); ++iter) { auto deref = *iter; - environment_.emplace( - [&]() { - if (deref.key().size() > 0) - return deref.key().string(); - return std::string{}; - }(), - [&]() { - if (deref.value().size() > 0) - return deref.value().string(); - return std::string{}; - }()); + environment_.emplace(utf8Of(deref.key()), utf8Of(deref.value())); } restoreHostEnvironment(); } diff --git a/backend/source/backend/process/fork_pool.cpp b/backend/source/backend/process/fork_pool.cpp index 4d9336f9..4f410472 100644 --- a/backend/source/backend/process/fork_pool.cpp +++ b/backend/source/backend/process/fork_pool.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -249,8 +250,10 @@ namespace } catch (std::exception const& exc) { - spdlog::error("[worker] command exception: cmd='{}' id='{}' what='{}'", cmd, procId, exc.what()); - sendJson({{"id", procId}, {"type", "error"}, {"message", exc.what()}}); + spdlog::error( + "[worker] command exception: cmd='{}' id='{}' what='{}'", cmd, procId, Utility::errorMessage(exc) + ); + sendJson({{"id", procId}, {"type", "error"}, {"message", Utility::errorMessage(exc)}}); } } @@ -584,9 +587,7 @@ namespace auto logPath = Nui::resolvePath("%state_home2%/nui-sftp/logs/worker.log"); std::error_code mkdirEc; std::filesystem::create_directories(logPath.parent_path(), mkdirEc); - return std::make_shared( - logPath.string(), 2 * 1024 * 1024, 3, true - ); + return std::make_shared(logPath.native(), 2 * 1024 * 1024, 3, true); } catch (std::exception const&) { diff --git a/backend/source/backend/process/process.cpp b/backend/source/backend/process/process.cpp index 041612d8..1dd5439b 100644 --- a/backend/source/backend/process/process.cpp +++ b/backend/source/backend/process/process.cpp @@ -1,6 +1,9 @@ #include #include +#include +#include + #include #include @@ -8,6 +11,9 @@ #include #include #include +#include +#include +#include namespace bp2 = boost::process::v2; @@ -303,12 +309,17 @@ void Process::spawn( { if (key.empty() || value.empty()) continue; +#ifdef _WIN32 + env.emplace(Paths::utf8ToWide(key), Paths::utf8ToWide(value)); +#else env.emplace(key, value); +#endif } - auto executable = bp2::environment::find_executable(processName, env); + const boost::filesystem::path name{Paths::nativeFromUtf8(processName).native()}; + auto executable = bp2::environment::find_executable(name, env); if (executable.empty()) - executable = processName; + executable = name; #ifdef _WIN32 // cmd.exe reads its own path as given: in "C:/Windows/System32/cmd.exe" it takes "/c" as a switch and runs // "md.exe" instead of starting a shell. @@ -326,10 +337,18 @@ void Process::spawn( // The app is a GUI program without a console, a console child would open a window of its own. bp2::windows::default_launcher windowlessLauncher{}; windowlessLauncher.creation_flags |= CREATE_NO_WINDOW; + const auto wideArguments = arguments | + std::views::transform( + [](std::string const& argument) + { + return Paths::utf8ToWide(argument); + } + ) | + std::ranges::to(); impl_->child = std::make_unique(windowlessLauncher( impl_->executor, executable, - arguments, + wideArguments, bp2::process_environment{env}, bp2::process_stdio(impl_->stdinPipe, impl_->stdoutPipe, impl_->stderrPipe) )); diff --git a/backend/source/backend/process/process_store.cpp b/backend/source/backend/process/process_store.cpp index 888ec0b5..34b89523 100644 --- a/backend/source/backend/process/process_store.cpp +++ b/backend/source/backend/process/process_store.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -171,7 +173,9 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) wnd_->runInJavascriptThread( [this, procId, msg]() { - hub_->callRemote(msg["responseId"].get(), nlohmann::json{{"id", procId}}); + RpcHelper::callRemoteSafely( + *hub_, msg["responseId"].get(), nlohmann::json{{"id", procId}} + ); } ); } @@ -185,7 +189,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) wnd_->runInJavascriptThread( [this, receptacle, procId, data]() { - hub_->callRemote(receptacle, nlohmann::json{{"id", procId}, {"data", data}}); + RpcHelper::callRemoteSafely(*hub_, receptacle, nlohmann::json{{"id", procId}, {"data", data}}); } ); } @@ -198,7 +202,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) wnd_->runInJavascriptThread( [this, procId]() { - hub_->callRemote("execTerminalExit_" + procId, nlohmann::json{{"id", procId}}); + RpcHelper::callRemoteSafely(*hub_, "execTerminalExit_" + procId, nlohmann::json{{"id", procId}}); } ); } @@ -211,7 +215,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) wnd_->runInJavascriptThread( [this, responseId, errMsg]() { - hub_->callRemote(responseId, nlohmann::json{{"error", errMsg}}); + RpcHelper::callRemoteSafely(*hub_, responseId, nlohmann::json{{"error", errMsg}}); } ); return; @@ -222,7 +226,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) { if (procsList.empty()) { - hub_->callRemote(responseId, nlohmann::json{{"error", "No processes found"}}); + RpcHelper::callRemoteSafely(*hub_, responseId, nlohmann::json{{"error", "No processes found"}}); return; } nlohmann::json j = nlohmann::json::object(); @@ -231,7 +235,7 @@ void ProcessStore::handleWorkerMessage(nlohmann::json const& msg) {"cmdline", procsList.front()["cmdline"]}, }; j["all"] = procsList; - hub_->callRemote(responseId, j); + RpcHelper::callRemoteSafely(*hub_, responseId, j); } ); } @@ -268,7 +272,7 @@ void ProcessStore::notifyChildExit(Nui::RpcHub& hub, std::string const& id) process->second->exit(); processes_.erase(process); - hub.callRemote("execTerminalExit_" + id, nlohmann::json{{"id", id}}); + RpcHelper::callRemoteSafely(hub, "execTerminalExit_" + id, nlohmann::json{{"id", id}}); } void ProcessStore::notifyChildExit(Nui::RpcHub& hub, long long pid) @@ -280,7 +284,9 @@ void ProcessStore::notifyChildExit(Nui::RpcHub& hub, long long pid) process->exit(); const auto idCopy = id; processes_.erase(id); - hub.callRemote("execTerminalExit_" + idCopy, nlohmann::json{{"id", idCopy}, {"pid", pid}}); + RpcHelper::callRemoteSafely( + hub, "execTerminalExit_" + idCopy, nlohmann::json{{"id", idCopy}, {"pid", pid}} + ); return; } } @@ -368,7 +374,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) using PtyType = PTY::PseudoTerminal; #endif - hub->callRemote(responseId, nlohmann::json{{"id", processId}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"id", processId}}); if (processes_[processId]->getState(ProcessAttachedState::ProcessInfo).isPty) { @@ -380,8 +386,10 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) wnd->runInJavascriptThread( [hub, stdoutReceptacle, id, msg = std::string{message}]() { - hub->callRemote( - stdoutReceptacle, nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} + RpcHelper::callRemoteSafely( + *hub, + stdoutReceptacle, + nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} ); } ); @@ -391,8 +399,10 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) wnd->runInJavascriptThread( [hub, stderrReceptacle, id, msg = std::string{message}]() { - hub->callRemote( - stderrReceptacle, nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} + RpcHelper::callRemoteSafely( + *hub, + stderrReceptacle, + nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} ); } ); @@ -408,8 +418,10 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) wnd->runInJavascriptThread( [hub, stdoutReceptacle, id, msg = std::string{message}]() { - hub->callRemote( - stdoutReceptacle, nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} + RpcHelper::callRemoteSafely( + *hub, + stdoutReceptacle, + nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} ); } ); @@ -420,8 +432,10 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) wnd->runInJavascriptThread( [hub, stderrReceptacle, id, msg = std::string{message}]() { - hub->callRemote( - stderrReceptacle, nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} + RpcHelper::callRemoteSafely( + *hub, + stderrReceptacle, + nlohmann::json{{"id", *id}, {"data", Roar::base64Encode(msg)}} ); } ); @@ -432,8 +446,8 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) } catch (std::exception const& e) { - Log::error("Failed to spawn process: {}", e.what()); - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + Log::error("Failed to spawn process: {}", Utility::errorMessage(e)); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -456,7 +470,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) if (forkPool_) forkPool_->send(nlohmann::json{{"id", id}, {"command", "kill"}}); forkPoolProcesses_.erase(iter); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); return; } } @@ -464,7 +478,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } auto& proc = process->second; @@ -475,12 +489,12 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) process->second->exit(0s); processes_.erase(process); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); #endif } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -502,7 +516,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) if (forkPool_) forkPool_->send(nlohmann::json{{"id", id}, {"command", "kill"}}); forkPoolProcesses_.erase(iter); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); return; } } @@ -510,7 +524,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } auto& proc = process->second; @@ -521,12 +535,12 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) process->second->exit(); processes_.erase(process); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); #endif } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -551,7 +565,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) {"payload", {{"data", data}}}, } ); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); return; } } @@ -560,7 +574,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } @@ -578,11 +592,11 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) else process->second->write(Roar::base64Decode(data) + "\r"); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -608,7 +622,9 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) } ); else - hub->callRemote(responseId, nlohmann::json{{"error", "No fork pool available"}}); + RpcHelper::callRemoteSafely( + *hub, responseId, nlohmann::json{{"error", "No fork pool available"}} + ); return; } } @@ -616,7 +632,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } @@ -629,13 +645,13 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) if (process->second->getState(ProcessAttachedState::ProcessInfo).isPty) { #ifdef _WIN32 - hub->callRemote(responseId, nlohmann::json{{"error", "Not implemented"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Not implemented"}}); #else auto& pty = process->second->getState(ProcessAttachedState::PseudoConsole); const auto procs = pty.listProcessesUnderPty(); if (procs.empty()) { - hub->callRemote(responseId, nlohmann::json{{"error", "No processes found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "No processes found"}}); return; } nlohmann::json j = nlohmann::json::object(); @@ -651,17 +667,17 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) {"cmdline", p.cmdline}, }); } - hub->callRemote(responseId, j); + RpcHelper::callRemoteSafely(*hub, responseId, j); #endif } else { - hub->callRemote(responseId, nlohmann::json{{"error", "Process is not a PTY"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process is not a PTY"}}); } } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } @@ -688,7 +704,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) {"payload", {{"cols", cols}, {"rows", rows}}}, } ); - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); return; } } @@ -697,7 +713,7 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) auto process = processes_.find(id); if (process == processes_.end()) { - hub->callRemote(responseId, nlohmann::json{{"error", "Process not found"}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", "Process not found"}}); return; } @@ -718,11 +734,11 @@ void ProcessStore::registerRpc(Nui::Window& wnd, Nui::RpcHub& hub) process->second->signal(SIGWINCH); #endif - hub->callRemote(responseId, nlohmann::json{{"success", true}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"success", true}}); } catch (std::exception const& e) { - hub->callRemote(responseId, nlohmann::json{{"error", e.what()}}); + RpcHelper::callRemoteSafely(*hub, responseId, nlohmann::json{{"error", Utility::errorMessage(e)}}); return; } } diff --git a/backend/source/backend/program_options.cpp b/backend/source/backend/program_options.cpp index 77b858eb..3e1cc18f 100644 --- a/backend/source/backend/program_options.cpp +++ b/backend/source/backend/program_options.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -34,7 +35,7 @@ std::optional parseProgramOptions(int argc, char const* const* a } catch (std::exception& e) { - std::cerr << "Error parsing program options: " << e.what() << "\n"; + std::cerr << "Error parsing program options: " << Utility::errorMessage(e) << "\n"; return std::nullopt; } } \ No newline at end of file diff --git a/backend/source/backend/pty/posix/process_list.cpp b/backend/source/backend/pty/posix/process_list.cpp index f3701cf9..7ebbdeb2 100644 --- a/backend/source/backend/pty/posix/process_list.cpp +++ b/backend/source/backend/pty/posix/process_list.cpp @@ -93,7 +93,7 @@ namespace PTY iterator.increment(iterationError)) { auto const& entry = *iterator; - const auto id = entry.path().filename().string(); + const auto id = entry.path().filename().native(); if (id.empty() || !std::all_of( id.begin(), @@ -110,7 +110,7 @@ namespace PTY // Unreadable entries (other users' processes) fail here and are skipped. std::error_code linkError; const auto target = std::filesystem::read_symlink(entry.path() / "fd" / "0", linkError); - if (linkError || target.string() != terminalName) + if (linkError || target.native() != terminalName) continue; std::ifstream cmdlineFile{entry.path() / "cmdline"}; diff --git a/backend/source/backend/pty/posix/pty.cpp b/backend/source/backend/pty/posix/pty.cpp index e385e74a..df49b8bf 100644 --- a/backend/source/backend/pty/posix/pty.cpp +++ b/backend/source/backend/pty/posix/pty.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -347,7 +348,7 @@ namespace PTY const boost::system::error_code& error ) { - Log::error("Failed to launch the process on the pty: {}", error.message()); + Log::error("Failed to launch the process on the pty: {}", Utility::errorMessage(error)); terminal_->impl_->closeInput(); close(terminal_->impl_->master); diff --git a/backend/source/backend/rpc_filesystem.cpp b/backend/source/backend/rpc_filesystem.cpp index bdf0f912..a050f112 100644 --- a/backend/source/backend/rpc_filesystem.cpp +++ b/backend/source/backend/rpc_filesystem.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #include @@ -33,24 +35,12 @@ namespace */ std::expected localPathParameter(std::string const& utf8) { - return Paths::LocalPath::fromUtf8(utf8) - .transform( - [](Paths::LocalPath const& path) - { - return path.native(); - } - ) - .transform_error( - [](Paths::PathError const& error) - { - return error.toString(); - } - ); - } - - std::expected deletableLocalPath(std::string const& utf8) - { - return localDeleteTarget(utf8, localHomeDirectory()); + return RpcHelper::parseLocalPath(utf8).transform( + [](Paths::LocalPath const& path) + { + return path.native(); + } + ); } #ifndef _WIN32 @@ -73,7 +63,7 @@ namespace struct passwd* result = nullptr; std::string name; if (::getpwuid_r(uid, &pwd, buffer, sizeof(buffer), &result) == 0 && result) - name = result->pw_name; + name = Paths::sanitizedUtf8(result->pw_name); return users.emplace(uid, std::move(name)).first->second; } @@ -87,7 +77,7 @@ namespace struct group* result = nullptr; std::string name; if (::getgrgid_r(gid, &grp, buffer, sizeof(buffer), &result) == 0 && result) - name = result->gr_name; + name = Paths::sanitizedUtf8(result->gr_name); return groups.emplace(gid, std::move(name)).first->second; } }; @@ -149,13 +139,13 @@ namespace struct passwd pwd{}; struct passwd* pwdResult = nullptr; if (::getpwuid_r(st.st_uid, &pwd, pwdBuf, sizeof(pwdBuf), &pwdResult) == 0 && pwdResult) - entry.owner = pwdResult->pw_name; + entry.owner = Paths::sanitizedUtf8(pwdResult->pw_name); char grpBuf[1024]; struct group grp{}; struct group* grpResult = nullptr; if (::getgrgid_r(st.st_gid, &grp, grpBuf, sizeof(grpBuf), &grpResult) == 0 && grpResult) - entry.group = grpResult->gr_name; + entry.group = Paths::sanitizedUtf8(grpResult->gr_name); } #endif @@ -185,7 +175,6 @@ RpcFilesystem::RpcFilesystem( , opener_{&opener} , riskyFilePolicy_{std::move(riskyFilePolicy)} { - registerRemove(); registerRemoveMultiple(); registerRename(); registerListFiles(); @@ -201,66 +190,6 @@ RpcFilesystem::RpcFilesystem( registerOpenerCapabilities(); } -void RpcFilesystem::registerRemove() -{ - on("RpcFilesystem::remove") - .perform( - [this](RpcHelper::RpcOnce&& reply, nlohmann::json const& parameters) - { - if (options_.preventDeletion) - { - Log::warn("RpcFilesystem::remove called but deletion is prevented by configuration."); - return reply.error("backend.rpcFilesystem.deletionPrevented"); - } - - Log::info("RpcFilesystem::remove called with parameters: {}", parameters.dump(4)); - - if (!RpcHelper::ParameterVerifyView{reply, "RpcFilesystem::remove", parameters}.hasValueDeep("path")) - { - Log::error("RpcFilesystem::remove missing required parameter 'path'."); - return reply.error("Missing required parameter 'path'."); - } - - const bool recursive = parameters.value("recursive", false); - const auto pathUtf8 = parameters["path"].get(); - const auto target = deletableLocalPath(pathUtf8); - if (!target) - { - Log::error("RpcFilesystem::remove refused: {}", target.error()); - return reply.error(Utility::localizedMessage("backend.rpcFilesystem.removeRefused", pathUtf8)); - } - const auto path = target->native(); - - if (!recursive) - { - std::error_code ec; - if (std::filesystem::remove(path, ec)) - { - Log::info("Successfully removed file: {}", pathUtf8); - return reply({{"success", true}}); - } - - Log::error("Failed to remove file: {}: {}", pathUtf8, ec.message()); - return reply.error(ec.message()); - } - - std::error_code ec; - const auto filesRemoved = Utility::removeAllNoFollow(path, ec); - if (ec) - { - Log::error("Failed to recursively remove path: {}: {}", pathUtf8, ec.message()); - return reply.error(ec.message()); - } - Log::info( - "Successfully recursively removed path: {} ({} files/directories removed)", - pathUtf8, - filesRemoved - ); - return reply({{"success", true}, {"removedCount", filesRemoved}}); - } - ); -} - void RpcFilesystem::registerRemoveMultiple() { on("RpcFilesystem::removeSome") @@ -269,7 +198,7 @@ void RpcFilesystem::registerRemoveMultiple() { if (options_.preventDeletion) { - Log::warn("RpcFilesystem::remove called but deletion is prevented by configuration."); + Log::warn("RpcFilesystem::removeSome called but deletion is prevented by configuration."); return reply.error("backend.rpcFilesystem.deletionPrevented"); } @@ -281,59 +210,27 @@ void RpcFilesystem::registerRemoveMultiple() return reply.error("Missing required parameter 'paths'."); } - const bool recursive = parameters.value("recursive", false); const auto pathsUtf8 = parameters["paths"].get>(); - - // Nothing is removed when one of the paths is refused. - std::vector targets; - targets.reserve(pathsUtf8.size()); - for (const auto& pathUtf8 : pathsUtf8) - { - auto target = deletableLocalPath(pathUtf8); - if (!target) - { - Log::error("RpcFilesystem::removeSome refused: {}", target.error()); - return reply.error(Utility::localizedMessage("backend.rpcFilesystem.removeRefused", pathUtf8)); - } - targets.push_back(*std::move(target)); - } - - // One item that cannot be removed does not keep the others, every failure is reported. - size_t removedCount = 0; - std::vector failures; - for (const auto& target : targets) + const auto removal = removeLocalItems(pathsUtf8, localHomeDirectories()); + if (!removal) { - const auto path = target.native(); - const auto& pathUtf8 = target.toUtf8(); - std::error_code ec; - if (recursive) - { - removedCount += Utility::removeAllNoFollow(path, ec); - } - else - { - if (std::filesystem::remove(path, ec)) - ++removedCount; - } - - if (ec) - { - Log::error("Failed to remove path '{}': {}", pathUtf8, ec.message()); - failures.push_back(fmt::format("{}: {}", pathUtf8, ec.message())); - } + Log::error("RpcFilesystem::removeSome refused: {}", removal.error().reason); + return reply.error( + Utility::localizedMessage("backend.rpcFilesystem.removeRefused", removal.error().path) + ); } - if (!failures.empty()) + if (!removal->failures.empty()) { return reply.error( Utility::localizedMessage( "backend.rpcFilesystem.removeSomeFailed", - failures.size(), - fmt::format("{}", fmt::join(failures, "\n")) + removal->failures.size(), + fmt::format("{}", fmt::join(removal->failures, "\n")) ) ); } - Log::info("Successfully removed all specified paths."); - return reply({{"success", true}, {"removedCount", removedCount}}); + Log::info("RpcFilesystem::removeSome removed {} entries.", removal->removed); + return reply({{"success", true}, {"removedCount", removal->removed}}); } ); } @@ -376,18 +273,17 @@ void RpcFilesystem::registerRename() return reply.error("backend.rename.invalidName"); } - // Never replaces an existing item unless the user confirmed it. - std::error_code ec; - if (overwrite) - std::filesystem::rename(oldPath, newPath, ec); - else if (auto renamed = Utility::renameNoReplace(oldPath, newPath); !renamed.has_value()) - ec = renamed.error(); - if (ec == std::errc::file_exists) - return reply.error("backend.rename.targetExists"); - if (ec) + if (const auto moved = Utility::moveIntoPlace(oldPath, newPath, overwrite); !moved) { - Log::error("Failed to rename from '{}' to '{}': {}", oldPathUtf8, newPathUtf8, ec.message()); - return reply.error(ec.message()); + if (moved.error() == std::errc::file_exists) + return reply.error("backend.rename.targetExists"); + Log::error( + "Failed to rename from '{}' to '{}': {}", + oldPathUtf8, + newPathUtf8, + Utility::errorMessage(moved.error()) + ); + return reply.error(Utility::errorMessage(moved.error())); } Log::info("Successfully renamed from '{}' to '{}'", oldPathUtf8, newPathUtf8); @@ -424,9 +320,11 @@ void RpcFilesystem::registerListFiles() if (ec) { Log::error( - "Failed to list files in directory '{}': {}", Paths::nativeToUtf8(directoryPath), ec.message() + "Failed to list files in directory '{}': {}", + Paths::nativeToUtf8(directoryPath), + Utility::errorMessage(ec) ); - return reply.error(ec.message()); + return reply.error(Utility::errorMessage(ec)); } #ifndef _WIN32 @@ -437,7 +335,9 @@ void RpcFilesystem::registerListFiles() const auto entryPath = entry.path(); SharedData::DirectoryEntry dirEntry{}; - dirEntry.name = Paths::nativeToUtf8(entryPath.filename()); + auto name = Paths::convertNativeToUtf8(entryPath.filename()); + dirEntry.name = std::move(name.utf8); + dirEntry.nameIsValidUtf8 = name.lossless; dirEntry.type = SharedData::fileTypeFromStdFilesystemType(entry.symlink_status().type()); dirEntry.permissions = entry.symlink_status().permissions(); @@ -505,8 +405,8 @@ void RpcFilesystem::registerCreateFile() std::ofstream fileStream(filePath, std::ios_base::noreplace | std::ios_base::binary); if (!fileStream) { - Log::error("Failed to create file '{}': {}", filePathUtf8, ec.message()); - return reply.error(ec.message()); + Log::error("Failed to create file '{}': {}", filePathUtf8, Utility::errorMessage(ec)); + return reply.error(Utility::errorMessage(ec)); } Log::info("Successfully created file '{}'", filePathUtf8); @@ -548,8 +448,8 @@ void RpcFilesystem::registerCreateDirectory() std::filesystem::create_directory(directoryPath, ec); if (ec) { - Log::error("Failed to create directory '{}': {}", directoryPathUtf8, ec.message()); - return reply.error(ec.message()); + Log::error("Failed to create directory '{}': {}", directoryPathUtf8, Utility::errorMessage(ec)); + return reply.error(Utility::errorMessage(ec)); } Log::info("Successfully created directory '{}'", directoryPathUtf8); @@ -582,15 +482,21 @@ void RpcFilesystem::registerProperties() auto const& path = *pathParameter; SharedData::DirectoryEntry entry{}; - entry.name = Paths::nativeToUtf8(path.filename()); + auto name = Paths::convertNativeToUtf8(path.filename()); + entry.name = std::move(name.utf8); + entry.nameIsValidUtf8 = name.lossless; std::string resolvedPath{}; std::error_code symEc; const auto symStatus = std::filesystem::symlink_status(path, symEc); if (symEc) { - Log::error("Failed to get properties for path '{}': {}", Paths::nativeToUtf8(path), symEc.message()); - return reply.error(symEc.message()); + Log::error( + "Failed to get properties for path '{}': {}", + Paths::nativeToUtf8(path), + Utility::errorMessage(symEc) + ); + return reply.error(Utility::errorMessage(symEc)); } entry.type = SharedData::fileTypeFromStdFilesystemType(symStatus.type()); entry.permissions = symStatus.permissions(); @@ -698,8 +604,8 @@ void RpcFilesystem::registerDoesExist() const auto status = std::filesystem::exists(*path, ec); if (ec) { - Log::error("Failed to get properties for path '{}': {}", pathUtf8, ec.message()); - return reply.error(ec.message()); + Log::error("Failed to get properties for path '{}': {}", pathUtf8, Utility::errorMessage(ec)); + return reply.error(Utility::errorMessage(ec)); } Log::info("Successfully retrieved existence for path '{}'", pathUtf8); @@ -710,10 +616,7 @@ void RpcFilesystem::registerDoesExist() void RpcFilesystem::registerDoesExistBatch() { - // Batched form of RpcFilesystem::exists. Lets the frontend ask about a - // whole transfer's worth of destination paths in a single round-trip - // instead of N — eliminating the per-file network latency that made - // multi-file drag/drop downloads feel slow. + // Whether each of N local paths exists, in one RPC. on("RpcFilesystem::existsBatch") .perform( [](RpcHelper::RpcOnce&& reply, std::vector const& paths) diff --git a/backend/source/backend/rpc_path_parameters.cpp b/backend/source/backend/rpc_path_parameters.cpp new file mode 100644 index 00000000..a10044b8 --- /dev/null +++ b/backend/source/backend/rpc_path_parameters.cpp @@ -0,0 +1,56 @@ +#include + +#include + +#include + +namespace RpcHelper +{ + namespace + { + template + std::expected, std::string> parseAll(std::vector const& paths, ParseT parse) + { + std::vector parsed; + parsed.reserve(paths.size()); + for (auto const& path : paths) + { + auto one = parse(path); + if (!one) + return std::unexpected(std::move(one).error()); + parsed.push_back(*std::move(one)); + } + return parsed; + } + } + + std::expected parseRemotePath(std::string const& path) + { + return Paths::RemotePath::make(path).transform_error( + [](Paths::PathError const& error) + { + return fmt::format("Invalid remote path: {}", error.toString()); + } + ); + } + + std::expected, std::string> parseRemotePaths(std::vector const& paths) + { + return parseAll(paths, parseRemotePath); + } + + std::expected parseLocalPath(std::string const& path) + { + return Paths::LocalPath::fromUtf8(path).transform_error( + [](Paths::PathError const& error) + { + return fmt::format("Invalid local path: {}", error.toString()); + } + ); + } + + std::expected, std::string> parseLocalPaths(std::vector const& paths) + { + return parseAll(paths, parseLocalPath); + } +} diff --git a/backend/source/backend/rpc_system.cpp b/backend/source/backend/rpc_system.cpp index aba3024c..ecc983c5 100644 --- a/backend/source/backend/rpc_system.cpp +++ b/backend/source/backend/rpc_system.cpp @@ -1,5 +1,6 @@ #include #include +#include #ifdef _WIN32 # include @@ -48,7 +49,7 @@ std::string RpcSystem::username() const DWORD username_len = UNLEN + 1; if (GetUserNameW(username, &username_len)) { - usernameMemo_ = Nui::utf16ToUtf8(std::wstring(username)); + usernameMemo_ = Paths::wideToUtf8(std::wstring_view{username}); return usernameMemo_; } else @@ -60,14 +61,14 @@ std::string RpcSystem::username() const char username[loginNameMaximum]; if (getlogin_r(username, sizeof(username)) == 0) { - usernameMemo_ = std::string(username); + usernameMemo_ = Paths::sanitizedUtf8(username); return usernameMemo_; } else { struct passwd* pwd = getpwuid(getuid()); if (pwd) - return pwd->pw_name; + return Paths::sanitizedUtf8(pwd->pw_name); Log::error("RpcSystem::username: getlogin_r failed"); return "unknown"; } diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index 38e4752e..09b21c70 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -1,5 +1,9 @@ #include +#include +#include +#include #include +#include #include #include @@ -22,60 +26,40 @@ using namespace std::chrono_literals; namespace { /** - * @brief A remote path from the frontend, or the message to reply with when it is not a valid absolute path. + * @brief The reply to a request that queued one operation, or why it could not be queued. */ - std::expected parseRemotePath(std::string const& path) + std::expected queuedReply(std::expected const& queued) { - return Paths::RemotePath::make(path).transform_error( - [](Paths::PathError const& error) - { - return fmt::format("Invalid remote path: {}", error.toString()); - } - ); + return queued + .transform( + []() + { + return nlohmann::json{{"success", true}}; + } + ) + .transform_error( + [](Operation::Error const& error) + { + return error.toString(); + } + ); } /** - * @brief A local path from the frontend, or the message to reply with when it is not a valid absolute path. + * @brief The reply to a request that queued @p enqueued entries. */ - std::expected parseLocalPath(std::string const& path) + nlohmann::json enqueuedReply(std::size_t enqueued) { - return Paths::LocalPath::fromUtf8(path).transform_error( - [](Paths::PathError const& error) - { - return fmt::format("Invalid local path: {}", error.toString()); - } - ); + return {{"success", true}, {"enqueued", enqueued}}; } /** - * @brief A JSON request from the frontend, or the message to reply with when it does not parse. Paths are - * validated while parsing. + * @brief Answers a delete request with the localized reason why nothing was deleted. */ - template - std::expected parseRequest(nlohmann::json const& json, std::string_view what) - { - try - { - return json.get(); - } - catch (std::exception const& exception) - { - return std::unexpected(fmt::format("Invalid {}: {}", what, exception.what())); - } - } - - std::expected, std::string> parseRemotePaths(std::vector const& paths) + void replyRefusal(RpcHelper::RpcOnce const& reply, RemoteDeleteRefusal const& refusal) { - std::vector result; - result.reserve(paths.size()); - for (auto const& path : paths) - { - auto parsed = parseRemotePath(path); - if (!parsed) - return std::unexpected(std::move(parsed).error()); - result.push_back(*std::move(parsed)); - } - return result; + Log::error("Refusing to delete: {}", refusal.toString()); + reply.error(refusal.toLocalizedMessage()); } } @@ -87,14 +71,26 @@ Session::Session( Nui::Window& wnd, Nui::RpcHub& hub, Persistence::SftpOptions const& sftpOptions, - BulkResumeRegistry* bulkResumeRegistry + BulkResumeRegistry* bulkResumeRegistry, + std::shared_ptr localOperationQueue ) : RpcHelper::StrandRpc{executor, std::move(strand), wnd, hub} , id_{std::move(id)} , session_{std::move(session)} - , operationQueue_{std::make_shared< - OperationQueue>(executor_, strand_, wnd, hub, sftpOptions, id_, sftpOptions.concurrency.value_or(Persistence::defaultSftpConcurrency))} + , operationQueue_{std::make_shared( + executor_, + strand_, + wnd, + hub, + sftpOptions, + id_, + sftpOptions.concurrency.value_or(Persistence::defaultSftpConcurrency) + )} , bulkResumeRegistry_{bulkResumeRegistry} + , remoteTempFileSuffix_{Persistence::effectiveTempFileSuffix( + sftpOptions.uploadOptions ? sftpOptions.uploadOptions->tempFileSuffix : std::nullopt + )} + , localOperationQueue_{std::move(localOperationQueue)} {} Session::~Session() @@ -136,6 +132,7 @@ void Session::start() self->registerRpcSftpAddBulkDownloadOperation(); self->registerRpcSftpAddBulkUploadOperation(); self->registerRpcSftpAddBulkDeleteOperation(); + self->registerRpcAddLocalDeleteOperation(); self->registerRpcSftpExistsBatch(); self->registerRpcSftpOpenSyncSession(); self->registerRpcSftpRecomputeSyncDiff(); @@ -189,6 +186,7 @@ void Session::stop() // registry TTL) handle cleanup if no resume occurs. if (self->bulkResumeRegistry_) self->operationQueue_->dumpBulkResumes(*self->bulkResumeRegistry_); + self->localOperationQueue_->release(self->id_); self->operationQueue_->cancelAll(); self->operationQueue_.reset(); @@ -637,7 +635,7 @@ void Session::registerRpcSftpListDirectory() Ids::makeChannelId(channelIdString), [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - const auto remotePath = parseRemotePath(path); + const auto remotePath = RpcHelper::parseRemotePath(path); if (!remotePath) return reply.error(remotePath.error()); auto fut = channel->listDirectory(*remotePath); @@ -712,7 +710,7 @@ void Session::registerRpcSftpUserDirectories() Ids::makeChannelId(channelIdString), [home](RpcHelper::RpcOnce&& reply, auto&& channel) { - const auto homePath = parseRemotePath(home); + const auto homePath = RpcHelper::parseRemotePath(home); if (!homePath) return reply.error(homePath.error()); auto fut = channel->userDirectories(*homePath); @@ -756,33 +754,31 @@ void Session::registerRpcQueuedRemoves() if (!self) return reply.error("Session no longer exists"); - auto parsed = parseRemotePaths(pathStrings); + auto parsed = RpcHelper::parseRemotePaths(pathStrings); if (!parsed) return reply.error(parsed.error()); self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), paths = *std::move(parsed), recursive, insertRefresh, mode]( - RpcHelper::RpcOnce&& reply, auto&& channel + [paths = *std::move(parsed), recursive, insertRefresh, mode]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel ) mutable { - auto self = weak.lock(); - if (!self) + if (!self.operationQueue_) return reply.error("Session no longer exists"); - // Nothing is queued when one of the targets is protected. - for (auto const& path : paths) - { - if (auto allowed = self->operationQueue_->refuseProtectedDeleteTarget(*channel, path); - !allowed) - return reply.error(fmt::format("Refusing to delete '{}'", path.value())); - } + auto guard = self.operationQueue_->remoteDeleteGuard(*channel); + if (const auto refusal = refuseAnyProtected(guard, paths); refusal) + return replyRefusal(reply, *refusal); std::vector operationIds; for (auto const& path : paths) { const auto newOperationId = Ids::generateOperationId(); - const auto result = self->operationQueue_->addDeleteOperation( + const auto result = self.operationQueue_->addDeleteOperation( + guard, *channel, newOperationId, path, @@ -822,7 +818,7 @@ void Session::registerRpcSftpCreateDirectory() Ids::makeChannelId(channelIdString), [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - const auto remotePath = parseRemotePath(path); + const auto remotePath = RpcHelper::parseRemotePath(path); if (!remotePath) return reply.error(remotePath.error()); auto fut = channel->createDirectory(*remotePath); @@ -860,7 +856,7 @@ void Session::registerRpcSftpCreateFile() Ids::makeChannelId(channelIdString), [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - const auto remotePath = parseRemotePath(path); + const auto remotePath = RpcHelper::parseRemotePath(path); if (!remotePath) return reply.error(remotePath.error()); auto fut = channel->createFile(*remotePath); @@ -899,34 +895,34 @@ void Session::registerRpcSftpAddDownloadOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - auto remote = parseRemotePath(remotePath); + auto remote = RpcHelper::parseRemotePath(remotePath); if (!remote) return reply.error(remote.error()); - auto local = parseLocalPath(localPath); + auto local = RpcHelper::parseLocalPath(localPath); if (!local) return reply.error(local.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the download '{}' of '{}' to '{}'", newOperationIdString, remote->value(), local->toUtf8() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), local = *std::move(local), remote = *std::move(remote), allowOverwrite, bigFile, insertRefresh, createMissingDirectories, - mode](RpcHelper::RpcOnce&& reply, auto&& channel) + mode](OperationQueue& queue, SecureShell::SftpSession& channel) { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto result = self->operationQueue_->addDownloadOperation( - *channel, - Ids::makeOperationId(newOperationIdString), + return queuedReply(queue.addDownloadOperation( + channel, + operationId, local, remote, allowOverwrite, @@ -934,30 +930,8 @@ void Session::registerRpcSftpAddDownloadOperation() insertRefresh, createMissingDirectories, static_cast(mode) - ); - - if (!result.has_value()) - { - Log::error( - "Failed to add download operation for file '{}' to '{}': {}", - remote.value(), - local.toUtf8(), - result.error().toString() - ); - return reply.error("Failed to add download operation: " + result.error().toString()); - } - - Log::info( - "Added download operation with id '{}' for file '{}' to '{}'", - newOperationIdString, - remote.value(), - local.toUtf8() - ); - - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + )); + } ); } ); @@ -966,8 +940,6 @@ void Session::registerRpcSftpAddDownloadOperation() void Session::registerRpcSftpAddArchiveDownloadOperation() { // One RPC to pack N remote files into a single local tar[.ext] archive. - // The entries list is trusted (no per-entry lstat) and the operation runs - // sequentially on the SFTP strand like a regular DownloadOperation. on(fmt::format("Session::{}::sftp::addArchiveDownload", id_.value())) .perform( [weak = weak_from_this()]( @@ -984,74 +956,46 @@ void Session::registerRpcSftpAddArchiveDownloadOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - // An invalid remote path fails the parse. - std::vector roots; - try - { - roots = nlohmann::json::parse(rootsJson).get>(); - } - catch (std::exception const& exc) - { - Log::error("addArchiveDownload: failed to parse roots JSON: {}", exc.what()); - return reply({{"error", std::string{"Invalid roots payload: "} + exc.what()}}); - } - auto archivePath = parseLocalPath(localArchivePath); + auto roots = RpcHelper::parseRequest>( + nlohmann::json::parse(rootsJson, nullptr, false), "archive roots" + ); + if (!roots) + return reply.error(roots.error()); + auto archivePath = RpcHelper::parseLocalPath(localArchivePath); if (!archivePath) return reply.error(archivePath.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the archive download '{}' of {} roots to '{}'", + newOperationIdString, + roots->size(), + archivePath->toUtf8() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, - roots = std::move(roots), - localArchivePath = *std::move(archivePath), + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), + roots = *std::move(roots), + archivePath = *std::move(archivePath), compressionCodec, compressionLevel, mayOverwrite, - mode](RpcHelper::RpcOnce&& reply, auto&& channel) mutable + mode](OperationQueue& queue, SecureShell::SftpSession& channel) mutable { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto codec = static_cast(compressionCodec); - - const auto result = self->operationQueue_->addArchiveDownloadOperation( - *channel, - Ids::makeOperationId(newOperationIdString), + return queuedReply(queue.addArchiveDownloadOperation( + channel, + operationId, std::move(roots), - localArchivePath, - codec, + archivePath, + static_cast(compressionCodec), compressionLevel, mayOverwrite, static_cast(mode) - ); - - if (!result.has_value()) - { - Log::error( - "Failed to add archive-download operation to '{}': {}", - localArchivePath.toUtf8(), - result.error().toString() - ); - return reply.error( - "Failed to add archive-download operation: " + result.error().toString() - ); - } - - Log::info( - "Added archive-download operation '{}' → '{}' ({} roots)", - newOperationIdString, - localArchivePath.toUtf8(), - roots.size() - ); - - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + )); + } ); } ); @@ -1075,65 +1019,44 @@ void Session::registerRpcSftpAddArchiveUploadOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - std::vector localPaths; - localPaths.reserve(localPathStrings.size()); - for (auto const& pathString : localPathStrings) - { - auto localPath = parseLocalPath(pathString); - if (!localPath) - return reply.error(localPath.error()); - localPaths.push_back(*std::move(localPath)); - } - auto archivePath = parseRemotePath(remoteArchivePath); + auto localPaths = RpcHelper::parseLocalPaths(localPathStrings); + if (!localPaths) + return reply.error(localPaths.error()); + auto archivePath = RpcHelper::parseRemotePath(remoteArchivePath); if (!archivePath) return reply.error(archivePath.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the archive upload '{}' of {} paths to '{}'", + newOperationIdString, + localPaths->size(), + archivePath->value() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, - localPaths = std::move(localPaths), - remoteArchivePath = *std::move(archivePath), + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), + localPaths = *std::move(localPaths), + archivePath = *std::move(archivePath), compressionCodec, compressionLevel, mayOverwrite, - mode](RpcHelper::RpcOnce&& reply, auto&& channel) mutable + mode](OperationQueue& queue, SecureShell::SftpSession& channel) mutable { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto codec = static_cast(compressionCodec); - const auto result = self->operationQueue_->addArchiveUploadOperation( - *channel, - Ids::makeOperationId(newOperationIdString), + return queuedReply(queue.addArchiveUploadOperation( + channel, + operationId, std::move(localPaths), - remoteArchivePath, - codec, + archivePath, + static_cast(compressionCodec), compressionLevel, mayOverwrite, static_cast(mode) - ); - if (!result.has_value()) - { - Log::error( - "Failed to add archive-upload operation to '{}': {}", - remoteArchivePath.value(), - result.error().toString() - ); - return reply.error("Failed to add archive-upload operation: " + result.error().toString()); - } - Log::info( - "Added archive-upload operation '{}' → '{}'", - newOperationIdString, - remoteArchivePath.value() - ); - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + )); + } ); } ); @@ -1141,59 +1064,47 @@ void Session::registerRpcSftpAddArchiveUploadOperation() void Session::registerRpcSftpAddBulkDownloadOperation() { - // One RPC for a whole batch. The handler performs all queueing inside a - // single strand dispatch so the SSH thread-hop cost is paid once per - // batch instead of once per file. Files are trusted (no lstat per entry). + // One RPC and one strand dispatch for a whole batch. Files are trusted, there is no lstat per entry. on(fmt::format("Session::{}::sftp::addBulkDownload", id_.value())) .perform( [weak = weak_from_this()]( RpcHelper::RpcOnce&& reply, std::string const& channelIdString, - std::vector const& operationIdStrings, + std::vector&& operationIdStrings, nlohmann::json const& requestJson ) { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); - - // Parsed here, so an invalid path is answered with an error instead of no reply at all. - auto parsed = parseRequest(requestJson, "bulk download"); - if (!parsed) - return reply.error(parsed.error()); - auto request = *std::move(parsed); + return reply.error("Session no longer exists"); - // Expect N per-entry ids + 1 dedicated aggregate-bulk-card id at the end. - if (operationIdStrings.size() != request.entries.size() + 1) + auto request = RpcHelper::parseRequest(requestJson, "bulk download"); + if (!request) + return reply.error(request.error()); + // An id per entry and the id of the card of the files at the end. + if (operationIdStrings.size() != request->entries.size() + 1) return reply.error("addBulkDownload: operationIds and entries length mismatch"); - self->withSftpChannelDo( + auto description = fmt::format("the bulk download of {} entries", request->entries.size()); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - operationIdStrings, - request](RpcHelper::RpcOnce&& reply, auto&& channel) + std::move(description), + [operationIds = std::move(operationIdStrings), request = *std::move(request)]( + OperationQueue& queue, SecureShell::SftpSession& channel + ) mutable -> std::expected { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto bulkCardId = Ids::makeOperationId(operationIdStrings.back()); - const auto enqueued = self->operationQueue_->addBulkDownloadOperation( - *channel, - request, - [&operationIdStrings](std::size_t idx) + const auto bulkCardId = Ids::makeOperationId(operationIds.back()); + return enqueuedReply(queue.addBulkDownloadOperation( + channel, + std::move(request), + [&operationIds](std::size_t index) { - return Ids::makeOperationId(operationIdStrings[idx]); + return Ids::makeOperationId(operationIds[index]); }, bulkCardId - ); - - Log::info("addBulkDownload: queued {}/{} entries", enqueued, request.entries.size()); - - self->resetQueueThrottle(); - reply({{"success", true}, {"enqueued", enqueued}}); - }, - std::move(reply) + )); + } ); } ); @@ -1201,55 +1112,46 @@ void Session::registerRpcSftpAddBulkDownloadOperation() void Session::registerRpcSftpAddBulkUploadOperation() { - // See registerRpcSftpAddBulkDownloadOperation — same contract for upload. + // Same contract as addBulkDownload. on(fmt::format("Session::{}::sftp::addBulkUpload", id_.value())) .perform( [weak = weak_from_this()]( RpcHelper::RpcOnce&& reply, std::string const& channelIdString, - std::vector const& operationIdStrings, + std::vector&& operationIdStrings, nlohmann::json const& requestJson ) { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); - - // Parsed here, so an invalid path is answered with an error instead of no reply at all. - auto parsed = parseRequest(requestJson, "bulk upload"); - if (!parsed) - return reply.error(parsed.error()); - auto request = *std::move(parsed); + return reply.error("Session no longer exists"); - if (operationIdStrings.size() != request.entries.size() + 1) + auto request = RpcHelper::parseRequest(requestJson, "bulk upload"); + if (!request) + return reply.error(request.error()); + if (operationIdStrings.size() != request->entries.size() + 1) return reply.error("addBulkUpload: operationIds and entries length mismatch"); - self->withSftpChannelDo( + auto description = fmt::format("the bulk upload of {} entries", request->entries.size()); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - operationIdStrings, - request](RpcHelper::RpcOnce&& reply, auto&& channel) + std::move(description), + [operationIds = std::move(operationIdStrings), request = *std::move(request)]( + OperationQueue& queue, SecureShell::SftpSession& channel + ) mutable -> std::expected { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto bulkCardId = Ids::makeOperationId(operationIdStrings.back()); - const auto enqueued = self->operationQueue_->addBulkUploadOperation( - *channel, - request, - [&operationIdStrings](std::size_t idx) + const auto bulkCardId = Ids::makeOperationId(operationIds.back()); + return enqueuedReply(queue.addBulkUploadOperation( + channel, + std::move(request), + [&operationIds](std::size_t index) { - return Ids::makeOperationId(operationIdStrings[idx]); + return Ids::makeOperationId(operationIds[index]); }, bulkCardId - ); - - Log::info("addBulkUpload: queued {}/{} entries", enqueued, request.entries.size()); - self->resetQueueThrottle(); - reply({{"success", true}, {"enqueued", enqueued}}); - }, - std::move(reply) + )); + } ); } ); @@ -1274,34 +1176,34 @@ void Session::registerRpcSftpAddUploadOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - auto local = parseLocalPath(localPath); + auto local = RpcHelper::parseLocalPath(localPath); if (!local) return reply.error(local.error()); - auto remote = parseRemotePath(remotePath); + auto remote = RpcHelper::parseRemotePath(remotePath); if (!remote) return reply.error(remote.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the upload '{}' of '{}' to '{}'", newOperationIdString, local->toUtf8(), remote->value() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), local = *std::move(local), remote = *std::move(remote), allowOverwrite, bigFile, insertRefresh, createMissingDirectories, - mode](RpcHelper::RpcOnce&& reply, auto&& channel) + mode](OperationQueue& queue, SecureShell::SftpSession& channel) { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto result = self->operationQueue_->addUploadOperation( - *channel, - Ids::makeOperationId(newOperationIdString), + return queuedReply(queue.addUploadOperation( + channel, + operationId, local, remote, allowOverwrite, @@ -1309,30 +1211,8 @@ void Session::registerRpcSftpAddUploadOperation() insertRefresh, createMissingDirectories, static_cast(mode) - ); - - if (!result.has_value()) - { - Log::error( - "Failed to add upload operation for file '{}' to '{}': {}", - local.toUtf8(), - remote.value(), - result.error().toString() - ); - return reply.error("Failed to add upload operation: " + result.error().toString()); - } - - Log::info( - "Added upload operation with id '{}' for file '{}' to '{}'", - newOperationIdString, - local.toUtf8(), - remote.value() - ); - - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + )); + } ); } ); @@ -1353,57 +1233,31 @@ void Session::registerRpcSftpAddRenameOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - auto source = parseRemotePath(sourcePath); + auto source = RpcHelper::parseRemotePath(sourcePath); if (!source) return reply.error(source.error()); - auto destination = parseRemotePath(destinationPath); + auto destination = RpcHelper::parseRemotePath(destinationPath); if (!destination) return reply.error(destination.error()); - self->withSftpChannelDo( + auto description = fmt::format( + "the rename '{}' of '{}' to '{}'", newOperationIdString, source->value(), destination->value() + ); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - newOperationIdString, + std::move(description), + [operationId = Ids::makeOperationId(newOperationIdString), source = *std::move(source), destination = *std::move(destination), - mode](RpcHelper::RpcOnce&& reply, auto&& channel) + mode](OperationQueue& queue, SecureShell::SftpSession& channel) { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - const auto result = self->operationQueue_->addRenameOperation( - *channel, - Ids::makeOperationId(newOperationIdString), - source, - destination, - static_cast(mode) - ); - - if (!result.has_value()) - { - Log::error( - "Failed to add rename operation for '{}' to '{}': {}", - source.value(), - destination.value(), - result.error().toString() - ); - return reply.error("Failed to add rename operation: " + result.error().toString()); - } - - Log::info( - "Added rename operation with id '{}' for '{}' to '{}'", - newOperationIdString, - source.value(), - destination.value() - ); - - self->resetQueueThrottle(); - reply({{"success", true}}); - }, - std::move(reply) + return queuedReply(queue.addRenameOperation( + channel, operationId, source, destination, static_cast(mode) + )); + } ); } ); @@ -1429,6 +1283,7 @@ void Session::registerOperationQueuePauseUnpause() } self->operationQueue_->paused(pause); + self->localOperationQueue_->hold(self->id_, pause); self->resetQueueThrottle(); return reply(SharedData::success()); } @@ -1456,30 +1311,32 @@ void Session::registerRpcSftpOpenSyncSession() return reply({{"error", "Session no longer exists"}}); // The roots every synchronized path is resolved against, refused here when they are not absolute. - auto remoteRoot = parseRemotePath(remotePath); + auto remoteRoot = RpcHelper::parseRemotePath(remotePath); if (!remoteRoot) return reply.error(remoteRoot.error()); - auto localRoot = Paths::LocalPath::fromUtf8(localPath); + auto localRoot = RpcHelper::parseLocalPath(localPath); if (!localRoot) - return reply.error(fmt::format("Invalid local path: {}", localRoot.error().toString())); + return reply.error(localRoot.error()); self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - syncSessionIdString, + [syncSessionIdString, remoteScanIdString, localScanIdString, remotePath = *std::move(remoteRoot), localPath = *std::move(localRoot), respectIgnoreFiles, recursive, - ignoreHidden](RpcHelper::RpcOnce&& reply, auto&& channel) + ignoreHidden]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel + ) { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); + if (!self.operationQueue_) + return reply.error("Session no longer exists"); - self->operationQueue_->addSyncScanOperation( + self.operationQueue_->addSyncScanOperation( *channel, Ids::makeSyncSessionId(syncSessionIdString), Ids::makeOperationId(remoteScanIdString), @@ -1491,7 +1348,7 @@ void Session::registerRpcSftpOpenSyncSession() ignoreHidden ); - self->resetQueueThrottle(); + self.resetQueueThrottle(); reply({{"success", true}}); }, std::move(reply) @@ -1539,7 +1396,8 @@ void Session::registerRpcSftpRecomputeSyncDiff() { // Matches the OperationQueue::rpcName scheme // the frontend OperationQueue listens on. - parent->hub_->callRemote( + RpcHelper::callRemoteSafely( + *parent->hub_, fmt::format("OperationQueue::{}::onSyncDiffProgress", parent->id_.value()), syncSessionId, std::to_string(compared) @@ -1608,8 +1466,7 @@ void Session::registerRpcSftpLoadSyncDiffChildren() void Session::registerRpcSftpExecuteSyncSelection() { - // The backend plans the selection and resolves every row below its own sync roots, so no absolute path - // round-trips through the frontend. + // Plans the selection and resolves every row below the sync roots of the session. on(fmt::format("Session::{}::sftp::executeSyncSelection", id_.value())) .perform( [weak = weak_from_this()]( @@ -1620,15 +1477,11 @@ void Session::registerRpcSftpExecuteSyncSelection() if (!self || !self->operationQueue_) return reply.error("Session no longer exists"); - SharedData::Sync::SyncExecutionRequest request{}; - try - { - request = requestJson.get(); - } - catch (std::exception const& exception) - { - return reply.error(fmt::format("Invalid sync request: {}", exception.what())); - } + auto parsed = + RpcHelper::parseRequest(requestJson, "sync request"); + if (!parsed) + return reply.error(parsed.error()); + auto request = *std::move(parsed); auto session = self->operationQueue_->syncSession(request.syncSessionId); if (!session) @@ -1637,24 +1490,32 @@ void Session::registerRpcSftpExecuteSyncSelection() auto replyShared = std::make_shared(std::move(reply)); boost::asio::post( session->strand(), - [self, session, request = std::move(request), channelIdString, replyShared]() mutable + [weak, session, request = std::move(request), channelIdString, replyShared]() mutable { if (session->generation() != request.generation) - return replyShared->error("stale generation"); + return replyShared->error(Utility::localizedMessage("backend.sync.staleComparison")); + auto self = weak.lock(); + if (!self) + return replyShared->error("Session no longer exists"); auto plan = session->resolvePlanInStrand( request.section, std::unordered_set{request.relKeys.begin(), request.relKeys.end()} ); self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [self, session, plan = std::move(plan), request = std::move(request)]( - RpcHelper::RpcOnce&& reply, auto&& channel + [session, plan = std::move(plan), request = std::move(request)]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel ) mutable { - auto result = self->operationQueue_->executeSyncPlan( - *channel, *session, std::move(plan), request + // A stopped session has no queue any more. + if (!self.operationQueue_) + return reply.error("Session no longer exists"); + auto result = self.operationQueue_->executeSyncPlan( + *channel, *session, std::move(plan), request, *self.localOperationQueue_ ); - self->resetQueueThrottle(); + self.resetQueueThrottle(); nlohmann::json json; SharedData::to_json(json, result); reply(std::move(json)); @@ -1717,13 +1578,9 @@ void Session::registerRpcSftpPreDeleteChecks() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), directories](RpcHelper::RpcOnce&& reply, auto&& channel) + [directories](RpcHelper::RpcOnce&& reply, auto&& channel) { - auto self = weak.lock(); - if (!self) - return reply.error("Session no longer exists"); - - auto remotePaths = parseRemotePaths(directories); + auto remotePaths = RpcHelper::parseRemotePaths(directories); if (!remotePaths) return reply.error(remotePaths.error()); auto fut = channel->filterOutEmptyDirectories(*std::move(remotePaths)); @@ -1754,21 +1611,20 @@ void Session::registerRpcSftpDeleteFiles() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), paths](RpcHelper::RpcOnce&& reply, auto&& channel) + [paths]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel + ) { - auto self = weak.lock(); - if (!self) + if (!self.operationQueue_) return reply.error("Session no longer exists"); - - auto remotePaths = parseRemotePaths(paths); + auto remotePaths = RpcHelper::parseRemotePaths(paths); if (!remotePaths) return reply.error(remotePaths.error()); - for (auto const& path : *remotePaths) - { - if (auto allowed = self->operationQueue_->refuseProtectedDeleteTarget(*channel, path); - !allowed) - return reply.error(fmt::format("Refusing to delete '{}'", path.value())); - } + auto guard = self.operationQueue_->remoteDeleteGuard(*channel); + if (const auto refusal = refuseAnyProtected(guard, *remotePaths); refusal) + return replyRefusal(reply, *refusal); auto fut = channel->removeAll(*std::move(remotePaths)); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error("Failed to delete files: timeout"); @@ -1802,25 +1658,31 @@ void Session::registerRpcSftpRename() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), oldName, newName, overwrite]( - RpcHelper::RpcOnce&& reply, auto&& channel + [oldName, newName, overwrite]( + RpcHelper::RpcOnce&& reply, + Session& self, + std::shared_ptr const& channel ) { - auto self = weak.lock(); - if (!self) - return reply.error("Session no longer exists"); - const auto oldPath = Paths::RemotePath::make(oldName); const auto newPath = Paths::RemotePath::make(newName); if (!oldPath || !newPath || !oldPath->parent() || oldPath->parent() != newPath->parent()) return reply.error("backend.rename.invalidName"); - // Never replaces an existing item unless the user confirmed it. auto fut = channel->performPromise( - [channel, oldPath = *oldPath, newPath = *newPath, overwrite]() + [session = std::weak_ptr{channel}, + oldPath = *oldPath, + newPath = *newPath, + overwrite, + asideSuffix = Persistence::asideFileSuffix(self.remoteTempFileSuffix_)]() + -> std::expected { - return overwrite ? channel->renameReplaceInStrand(oldPath, newPath, ".nui-sftp-old") - : channel->renameNoReplaceInStrand(oldPath, newPath); + const auto sftp = session.lock(); + if (!sftp) + return std::unexpected( + SecureShell::SftpError{.message = "The SFTP channel was closed"} + ); + return sftp->moveIntoPlaceInStrand(oldPath, newPath, overwrite, asideSuffix); } ); if (fut.wait_for(futureTimeout) != std::future_status::ready) @@ -1843,9 +1705,7 @@ void Session::registerRpcSftpRename() void Session::registerRpcSftpAddBulkDeleteOperation() { - // One RPC for the whole bulk delete. Frontend pre-allocates a single - // OperationId for the aggregate file-bulk card; the backend may also - // create extra single-Delete cards for any directory entries. + // One RPC and one card for the whole bulk delete, under the operation id the frontend reserved. on(fmt::format("Session::{}::sftp::addBulkDelete", id_.value())) .perform( [weak = weak_from_this()]( @@ -1857,58 +1717,58 @@ void Session::registerRpcSftpAddBulkDeleteOperation() { auto self = weak.lock(); if (!self) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - // Parsed here, so an invalid target is answered with an error instead of no reply at all. - SharedData::BulkDeleteRequest request{}; - try - { - request = requestJson.get(); - } - catch (std::exception const& exception) - { - return reply.error(fmt::format("Invalid bulk delete: {}", exception.what())); - } + auto request = RpcHelper::parseRequest(requestJson, "bulk delete"); + if (!request) + return reply.error(request.error()); - self->withSftpChannelDo( + auto description = fmt::format("the bulk delete of {} entries", request->entries.size()); + self->queueOnChannel( + std::move(reply), Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), - bulkOperationIdString, - request](RpcHelper::RpcOnce&& reply, auto&& channel) + std::move(description), + [operationId = Ids::makeOperationId(bulkOperationIdString), request = *std::move(request)]( + OperationQueue& queue, SecureShell::SftpSession& channel + ) -> std::expected { - auto self = weak.lock(); - if (!self) - return reply({{"error", "Session no longer exists"}}); - - // Nothing is queued when one of the targets is protected. - for (auto const& entry : request.entries) - { - if (auto allowed = self->operationQueue_->refuseProtectedDeleteTarget(*channel, entry.target); - !allowed) - return reply.error(fmt::format("Refusing to delete '{}'", entry.target.value())); - } - - const auto enqueued = self->operationQueue_->addBulkDeleteOperation( - *channel, request, Ids::makeOperationId(bulkOperationIdString) - ); + auto guard = queue.remoteDeleteGuard(channel); + if (const auto refusal = + refuseAnyProtected(guard, request.entries, &SharedData::BulkDeleteEntry::target); + refusal) + return std::unexpected(refusal->toLocalizedMessage()); + return enqueuedReply(queue.addBulkDeleteOperation(guard, channel, request, operationId)); + } + ); + } + ); +} - Log::info("addBulkDelete: queued {} entries", enqueued); - self->resetQueueThrottle(); - reply({{"success", true}, {"enqueued", enqueued}}); - }, - std::move(reply) +void Session::registerRpcAddLocalDeleteOperation() +{ + on(fmt::format("Session::{}::addLocalDelete", id_.value())) + .perform( + [weak = weak_from_this()]( + RpcHelper::RpcOnce&& reply, std::string const& operationIdString, std::vector const& paths + ) + { + auto self = weak.lock(); + if (!self || !self->operationQueue_) + return reply.error("Session no longer exists"); + const auto added = self->localOperationQueue_->addExplorerDelete( + Ids::makeOperationId(operationIdString), paths, self->id_, self->operationQueue_->paused() ); + if (!added) + return reply.error(added.error()); + reply(SharedData::success()); } ); } void Session::registerRpcSftpExistsBatch() { - // Batched existence probe over SFTP. Lets the frontend ask whether N - // remote paths exist in a single round-trip — eliminates the per-file - // SFTP stat latency that made multi-file uploads feel slow. Only "no such - // file" is absent, any other failure fails the whole batch: the frontend - // overwrites what it takes for absent. + // Whether each of N remote paths exists, in one RPC. Only "no such file" is absent, any other failure fails the + // whole batch: the frontend overwrites what it takes for absent. on(fmt::format("Session::{}::sftp::existsBatch", id_.value())) .perform( [weak = weak_from_this()]( @@ -1952,13 +1812,9 @@ void Session::registerRpcSftpStat() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), path](RpcHelper::RpcOnce&& reply, auto&& channel) + [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - auto self = weak.lock(); - if (!self) - return reply.error("Session no longer exists"); - - const auto remotePath = parseRemotePath(path); + const auto remotePath = RpcHelper::parseRemotePath(path); if (!remotePath) return reply.error(remotePath.error()); auto fut = channel->stat(*remotePath); diff --git a/backend/source/backend/session_manager.cpp b/backend/source/backend/session_manager.cpp index e8005fcb..f572a535 100644 --- a/backend/source/backend/session_manager.cpp +++ b/backend/source/backend/session_manager.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -103,8 +105,14 @@ SessionManager::SessionManager( ) : RpcHelper::StrandRpc{executor, wnd, hub} , stateHolder_{&stateHolder} + , localOperationQueue_{std::make_shared(executor_, strand_, wnd, hub, localDeletionPrevented_)} {} +void SessionManager::preventLocalDeletion(bool prevent) +{ + localDeletionPrevented_->store(prevent); +} + SessionManager::~SessionManager() { for (auto& [id, session] : sessions_) @@ -193,14 +201,18 @@ void SessionManager::addSession( { try { - hub_->callRemote(fmt::format("Session::{}::onDisconnect", sessionId.value())); + RpcHelper::callRemoteSafely( + *hub_, fmt::format("Session::{}::onDisconnect", sessionId.value()) + ); } catch (const std::exception& e) { // There should never anything throw here, but lets log just in case and dont let an // exception escape out of this lambda. Log::error( - "Failed to call Session::{}::onDisconnect: {}.", sessionId.value(), e.what() + "Failed to call Session::{}::onDisconnect: {}.", + sessionId.value(), + Utility::errorMessage(e) ); } } @@ -218,7 +230,8 @@ void SessionManager::addSession( *wnd_, *hub_, sessionOptions.sftpOptions.value(), - &bulkResumeRegistry_ + &bulkResumeRegistry_, + localOperationQueue_ ); const auto emplaced = sessions_.emplace(sessionId, session); if (!emplaced.second) @@ -265,6 +278,7 @@ void SessionManager::registerRpc() registerRpcSessionDisconnect(); registerRpcAdoptBulkResume(); registerRpcDiscardBulkResume(); + localOperationQueue_->registerRpc(); } void SessionManager::registerRpcSessionConnect() @@ -325,8 +339,8 @@ void SessionManager::registerRpcSessionDisconnect() } catch (std::exception const& e) { - Log::error("Error disconnecting to ssh server: {}", e.what()); - return reply({{"error", e.what()}}); + Log::error("Error disconnecting to ssh server: {}", Utility::errorMessage(e)); + return reply({{"error", Utility::errorMessage(e)}}); } } ); @@ -376,8 +390,8 @@ void SessionManager::registerRpcAdoptBulkResume() } catch (std::exception const& e) { - Log::error("Error in adoptBulkResume: {}", e.what()); - return reply({{"error", e.what()}}); + Log::error("Error in adoptBulkResume: {}", Utility::errorMessage(e)); + return reply({{"error", Utility::errorMessage(e)}}); } } ); @@ -401,8 +415,8 @@ void SessionManager::registerRpcDiscardBulkResume() } catch (std::exception const& e) { - Log::error("Error in discardBulkResume: {}", e.what()); - return reply({{"error", e.what()}}); + Log::error("Error in discardBulkResume: {}", Utility::errorMessage(e)); + return reply({{"error", Utility::errorMessage(e)}}); } } ); diff --git a/backend/source/backend/sftp/archive_download_operation.cpp b/backend/source/backend/sftp/archive_download_operation.cpp index 53986882..7e64c669 100644 --- a/backend/source/backend/sftp/archive_download_operation.cpp +++ b/backend/source/backend/sftp/archive_download_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -118,9 +119,9 @@ bool ArchiveDownloadOperation::isPayloadEntry(ResolvedEntry const& entry) const return isRegularFile(entry.header); } -ArchiveDownloadOperation::ArchiveDownloadOperation(SecureShell::SftpSession& sftp, Options options) +ArchiveDownloadOperation::ArchiveDownloadOperation(std::weak_ptr sftp, Options options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , options_{std::move(options)} { options_.tempFileSuffix = Persistence::effectiveTempFileSuffix(options_.tempFileSuffix); @@ -136,11 +137,7 @@ ArchiveDownloadOperation::ArchiveDownloadOperation(SecureShell::SftpSession& sft ArchiveDownloadOperation::~ArchiveDownloadOperation() { std::ignore = cancel(false); - - if (auto stream = currentStream_.lock(); stream) - { - stream->strand()->pushPromiseTask([]() {}).get(); - } + waitForStrandTasks(strand()); } std::expected ArchiveDownloadOperation::work() @@ -153,7 +150,15 @@ std::expected ArchiveDownloadOperation: return result; } - auto future = sftp_->performPromise([this]() { return workInStrand(); }); + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto future = sftp->performPromise( + [this]() + { + return workInStrand(); + } + ); if (future.wait_for(options_.futureTimeout) != std::future_status::ready) { Log::error("ArchiveDownloadOperation: work umbrella timed out."); @@ -175,6 +180,10 @@ ArchiveDownloadOperation::workInStrand() ); } + const auto sftp = openSession(sftp_); + if (!sftp && !hasEnded()) + return enterErrorState(channelClosedError()); + switch (state_) { case NotStarted: @@ -190,10 +199,10 @@ ArchiveDownloadOperation::workInStrand() return enterErrorState( {.type = ErrorType::ImplementationError, .extraInfo = "archive has no roots"} ); - if (std::filesystem::exists(localArchiveNativePath_) && !options_.mayOverwrite) + if (std::error_code existsError{}; std::filesystem::exists(localArchiveNativePath_, existsError) && !options_.mayOverwrite) return enterErrorState({.type = ErrorType::FileExists}); - const auto prepareResult = prepareInStrand(); + const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) return enterErrorState(prepareResult.error()); state_ = Prepared; @@ -222,7 +231,7 @@ ArchiveDownloadOperation::workInStrand() } else { - const auto openResult = openNextEntryInStrand(); + const auto openResult = openNextEntryInStrand(*sftp); if (!openResult.has_value()) return enterErrorState(openResult.error()); return WorkStatus::MoreWork; @@ -265,7 +274,7 @@ ArchiveDownloadOperation::workInStrand() return enterErrorState({.type = ErrorType::UnknownWorkState}); } -std::expected ArchiveDownloadOperation::prepareInStrand() +std::expected ArchiveDownloadOperation::prepareInStrand(SecureShell::SftpSession& sftp) { if (options_.createMissingDirectories) { @@ -275,14 +284,14 @@ std::expected ArchiveDownloadOperation::prepareInStrand( { Log::error( "ArchiveDownloadOperation: cannot create parents of '{}': {}", - tempPath_.generic_string(), - mkErr.message() + Paths::nativeToUtf8(tempPath_), + Utility::errorMessage(mkErr) ); return std::unexpected(Error{.type = ErrorType::CannotCreateDirectory}); } } - const auto flattenResult = flattenRootsInStrand(); + const auto flattenResult = flattenRootsInStrand(sftp); if (!flattenResult.has_value()) return std::unexpected(flattenResult.error()); @@ -295,7 +304,7 @@ std::expected ArchiveDownloadOperation::prepareInStrand( // Opening for writing follows a symlink planted at the temp path, out of the target directory. if (std::error_code linkError{}; std::filesystem::is_symlink(std::filesystem::symlink_status(tempPath_, linkError))) { - Log::error("ArchiveDownloadOperation: temp path '{}' is a symlink.", tempPath_.generic_string()); + Log::error("ArchiveDownloadOperation: temp path '{}' is a symlink.", Paths::nativeToUtf8(tempPath_)); return std::unexpected(Error{.type = ErrorType::OpenFailure, .extraInfo = "The temporary file path is a symlink"}); } @@ -311,7 +320,7 @@ std::expected ArchiveDownloadOperation::prepareInStrand( } writer_ = TarArchive::Writer::makeFromSink(std::move(*sinkOrError)); - buffer_ = sftp_->bufferProvider().lease(totalPayloadBytes_); + buffer_ = sftp.bufferProvider().lease(totalPayloadBytes_); if (buffer_.empty()) { Log::error("ArchiveDownloadOperation: No transfer buffer available from pool."); @@ -326,7 +335,7 @@ std::expected ArchiveDownloadOperation::prepareInStrand( return {}; } -std::expected ArchiveDownloadOperation::flattenRootsInStrand() +std::expected ArchiveDownloadOperation::flattenRootsInStrand(SecureShell::SftpSession& sftp) { for (auto const& root : options_.roots) { @@ -343,8 +352,30 @@ std::expected ArchiveDownloadOperation::flattenRootsInSt if (root.entry.type == SharedData::FileType::Regular) { + // The listing the root came from may be old, the header must carry the size that is read. + const auto current = sftp.lstatInStrand(root.path); + if (!current) + { + Log::error( + "ArchiveDownloadOperation: cannot look up root '{}': {}", + root.path.value(), + current.error().toString() + ); + return std::unexpected( + Error{ + .type = ErrorType::FileStatFailed, + .sftpError = current.error(), + .extraInfo = root.path.value(), + } + ); + } + if (!current->isRegularFile()) + { + Log::error("ArchiveDownloadOperation: root '{}' is no longer a regular file", root.path.value()); + return std::unexpected(Error{.type = ErrorType::NotAFile, .extraInfo = root.path.value()}); + } ResolvedEntry resolved{ - .header = TarArchive::memberHeaderOf(archivePrefix.value(), root.entry), + .header = TarArchive::memberHeaderOf(archivePrefix.value(), *current), .source = root.path, }; totalPayloadBytes_ += resolved.header.size; @@ -432,7 +463,7 @@ void ArchiveDownloadOperation::appendPrescannedInStrand( } } -std::expected ArchiveDownloadOperation::openNextEntryInStrand() +std::expected ArchiveDownloadOperation::openNextEntryInStrand(SecureShell::SftpSession& sftp) { if (!writer_) return std::unexpected(Error{ @@ -463,10 +494,8 @@ std::expected ArchiveDownloadOperation::openNextEntryInS return {}; } - const auto streamOpenResult = sftp_->openFileInStrand( - resolved.source, - SecureShell::SftpSession::OpenType::Read, - std::filesystem::perms::unknown + const auto streamOpenResult = sftp.openFileInStrand( + resolved.source, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown ); if (!streamOpenResult.has_value()) { @@ -662,21 +691,17 @@ std::expected ArchiveDownloadOperation::finaliseArchiveI if (options_.markAsDownloaded) { if (const auto marked = Utility::markAsDownloaded(tempPath_); marked) - Log::warn("ArchiveDownloadOperation: Cannot mark the archive as downloaded: {}", marked.message()); + Log::warn( + "ArchiveDownloadOperation: Cannot mark the archive as downloaded: {}", Utility::errorMessage(marked) + ); } - // Without permission to overwrite, the rename itself refuses an existing archive: no check-then-rename race. - std::error_code renameErr{}; - if (options_.mayOverwrite) - std::filesystem::rename(tempPath_, localArchiveNativePath_, renameErr); - else if (auto renamed = Utility::renameNoReplace(tempPath_, localArchiveNativePath_); !renamed.has_value()) - renameErr = renamed.error(); - if (renameErr == std::errc::file_exists) - return std::unexpected(Error{.type = ErrorType::FileExists}); - if (renameErr) + if (const auto moved = Utility::moveIntoPlace(tempPath_, localArchiveNativePath_, options_.mayOverwrite); !moved) { - Log::error("ArchiveDownloadOperation: rename failed: {}", renameErr.message()); - return std::unexpected(Error{.type = ErrorType::RenameFailure}); + Log::error( + "ArchiveDownloadOperation: Failed to move the archive into place: {}", Utility::errorMessage(moved.error()) + ); + return std::unexpected(localMoveError(moved.error())); } return {}; } @@ -717,10 +742,16 @@ void ArchiveDownloadOperation::cleanup() currentStream_.reset(); writer_.reset(); - if (!tempPath_.empty() && std::filesystem::exists(tempPath_)) + if (!tempPath_.empty()) { - std::error_code removeErr{}; - std::filesystem::remove(tempPath_, removeErr); + std::error_code removeError{}; + std::filesystem::remove(tempPath_, removeError); + if (removeError) + Log::warn( + "ArchiveDownloadOperation: Cannot remove '{}': {}", + Paths::nativeToUtf8(tempPath_), + Utility::errorMessage(removeError) + ); } } diff --git a/backend/source/backend/sftp/archive_upload_operation.cpp b/backend/source/backend/sftp/archive_upload_operation.cpp index c406cfce..ce25476d 100644 --- a/backend/source/backend/sftp/archive_upload_operation.cpp +++ b/backend/source/backend/sftp/archive_upload_operation.cpp @@ -1,9 +1,11 @@ #include +#include #include #include #include +#include #include #include @@ -69,6 +71,17 @@ namespace }; } + /** + * @brief The tar header of @p entry under @p path. Special bits never travel, and unknown permissions, as on + * Windows, get the defaults of the tar writer. + */ + TarArchive::MemberHeader memberHeaderFor(std::string path, SharedData::DirectoryEntry const& entry) + { + auto header = TarArchive::memberHeaderOf(std::move(path), entry); + header.permissions = Utility::sanitizePermissions(header.permissions).value_or(std::filesystem::perms::unknown); + return header; + } + /** * @brief The metadata of @p localPath for its tar header. */ @@ -80,14 +93,14 @@ namespace { Log::warn( "ArchiveUploadOperation: cannot stat '{}': {}", - localPath.generic_string(), - statErr.message() + Paths::nativeToUtf8(localPath), + Utility::errorMessage(statErr) ); return std::nullopt; } SharedData::DirectoryEntry entry{}; - entry.permissions = status.permissions(); + entry.permissions = Utility::localPermissions(status); entry.type = SharedData::fileTypeFromStdFilesystemType(status.type()); if (entry.type == SharedData::FileType::Regular) @@ -118,9 +131,9 @@ bool ArchiveUploadOperation::isPayloadEntry(ResolvedEntry const& entry) const no return entry.header.type == SharedData::FileType::Regular; } -ArchiveUploadOperation::ArchiveUploadOperation(SecureShell::SftpSession& sftp, Options options) +ArchiveUploadOperation::ArchiveUploadOperation(std::weak_ptr sftp, Options options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , options_{std::move(options)} { remoteArchivePath_ = options_.remoteArchivePath; @@ -130,8 +143,7 @@ ArchiveUploadOperation::ArchiveUploadOperation(SecureShell::SftpSession& sftp, O ArchiveUploadOperation::~ArchiveUploadOperation() { std::ignore = cancel(false); - if (auto stream = remoteStream_.lock(); stream) - stream->strand()->pushPromiseTask([]() {}).get(); + waitForStrandTasks(strand()); } std::expected ArchiveUploadOperation::work() @@ -144,7 +156,15 @@ std::expected ArchiveUploadOperation::w return result; } - auto future = sftp_->performPromise([this]() { return workInStrand(); }); + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto future = sftp->performPromise( + [this]() + { + return workInStrand(); + } + ); if (future.wait_for(options_.futureTimeout) != std::future_status::ready) { Log::error("ArchiveUploadOperation: work umbrella timed out."); @@ -170,6 +190,10 @@ std::expected ArchiveUploadOperation::w return enterErrorState(*incompleteScan_); } + const auto sftp = openSession(sftp_); + if (!sftp && !hasEnded()) + return enterErrorState(channelClosedError()); + switch (state_) { case NotStarted: @@ -186,7 +210,7 @@ std::expected ArchiveUploadOperation::w {.type = ErrorType::InvalidPath, .extraInfo = "archive has no source paths"} ); - const auto prepareResult = prepareInStrand(); + const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) return enterErrorState(prepareResult.error()); state_ = Prepared; @@ -231,7 +255,7 @@ std::expected ArchiveUploadOperation::w } case Finalizing: { - const auto finaliseResult = finaliseArchiveInStrand(); + const auto finaliseResult = finaliseArchiveInStrand(*sftp); if (!finaliseResult.has_value()) return enterErrorState(finaliseResult.error()); state_ = Completed; @@ -283,7 +307,7 @@ std::expected ArchiveUploadOperation::flattenRoots() totalPayloadBytes_ += rootEntry->size; resolvedEntries_.push_back( ResolvedEntry{ - .header = TarArchive::memberHeaderOf(archivePrefix.value(), *rootEntry), + .header = memberHeaderFor(archivePrefix.value(), *rootEntry), .source = localRoot, } ); @@ -292,7 +316,7 @@ std::expected ArchiveUploadOperation::flattenRoots() { rootEntry->size = 0u; resolvedEntries_.push_back( - ResolvedEntry{.header = TarArchive::memberHeaderOf(archivePrefix.value(), *rootEntry)} + ResolvedEntry{.header = memberHeaderFor(archivePrefix.value(), *rootEntry)} ); // Directory roots are expanded exclusively from a preceding @@ -331,38 +355,38 @@ void ArchiveUploadOperation::appendPrescannedFromScan( { // Index 0 is the synthetic scan root (the directory itself); the caller // emitted its tar header already, so we start at 1 and consume the - // walker's flat depth-first entry list — mirrors the download op's + // walker's flat depth-first entry list, like the download op's // appendPrescannedInStrand. for (std::size_t idx = 1u; idx < entries.size(); ++idx) { auto const& child = entries[idx]; - auto source = SharedData::relativePath(entries, child) - .and_then( - [&localRoot](Paths::RelativePath const& relative) - { - return localRoot.join(relative); - } - ); + const auto relative = SharedData::relativePath(entries, child); + auto source = relative.and_then( + [&localRoot](Paths::RelativePath const& path) + { + return localRoot.join(path); + } + ); if (!source) { Log::warn("ArchiveUploadOperation: skipping pre-scanned child: {}", source.error().toString()); continue; } - const auto memberPath = archivePrefix / *SharedData::relativePath(entries, child); + const auto memberPath = archivePrefix / *relative; if (child.type == SharedData::FileType::Regular) { totalPayloadBytes_ += child.size; resolvedEntries_.push_back( ResolvedEntry{ - .header = TarArchive::memberHeaderOf(memberPath.value(), child), + .header = memberHeaderFor(memberPath.value(), child), .source = *std::move(source), } ); } else if (child.type == SharedData::FileType::Directory) { - ResolvedEntry dirEntry{.header = TarArchive::memberHeaderOf(memberPath.value(), child)}; + ResolvedEntry dirEntry{.header = memberHeaderFor(memberPath.value(), child)}; dirEntry.header.size = 0u; resolvedEntries_.push_back(std::move(dirEntry)); } @@ -377,7 +401,7 @@ void ArchiveUploadOperation::appendPrescannedFromScan( } } -std::expected ArchiveUploadOperation::prepareInStrand() +std::expected ArchiveUploadOperation::prepareInStrand(SecureShell::SftpSession& sftp) { if (remoteArchivePath_.empty() || remoteArchivePath_.isRoot() || tempPath_.empty()) { @@ -400,12 +424,12 @@ std::expected ArchiveUploadOperation::prepareInStrand() }); // Fail early instead of packing everything first, the final rename refuses to replace it anyway. - if (!options_.mayOverwrite && sftp_->lstatInStrand(remoteArchivePath_).has_value()) + if (!options_.mayOverwrite && sftp.lstatInStrand(remoteArchivePath_).has_value()) return std::unexpected(Error{.type = ErrorType::FileExists}); // Written under the temp name, so a failed or cancelled upload never leaves a truncated destination. using OpenType = SecureShell::SftpSession::OpenType; - const auto openResult = sftp_->openFileInStrand( + const auto openResult = sftp.openFileInStrand( tempPath_, OpenType::Write | OpenType::Create | OpenType::Truncate, std::filesystem::perms{0644u} ); if (!openResult.has_value()) @@ -449,7 +473,7 @@ std::expected ArchiveUploadOperation::prepareInStrand() writer_ = TarArchive::Writer::makeFromSink(std::move(*wrappedSink)); - buffer_ = sftp_->bufferProvider().lease(totalPayloadBytes_); + buffer_ = sftp.bufferProvider().lease(totalPayloadBytes_); if (buffer_.empty()) { Log::error("ArchiveUploadOperation: No transfer buffer available from pool."); @@ -623,7 +647,7 @@ std::expected ArchiveUploadOperation::finaliseEntryInStr return {}; } -std::expected ArchiveUploadOperation::finaliseArchiveInStrand() +std::expected ArchiveUploadOperation::finaliseArchiveInStrand(SecureShell::SftpSession& sftp) { if (!writer_) return std::unexpected(Error{ @@ -646,19 +670,21 @@ std::expected ArchiveUploadOperation::finaliseArchiveInS stream->close(false); remoteStream_.reset(); - const auto renameResult = options_.mayOverwrite - ? sftp_->renameReplaceInStrand(tempPath_, remoteArchivePath_, options_.tempFileSuffix + ".old") - : sftp_->renameNoReplaceInStrand(tempPath_, remoteArchivePath_); - if (!renameResult.has_value()) + const auto moved = sftp.moveIntoPlaceInStrand( + tempPath_, + remoteArchivePath_, + options_.mayOverwrite, + Persistence::asideFileSuffix(options_.tempFileSuffix), + SharedData::FileType::Regular + ); + if (!moved.has_value()) { Log::error( "ArchiveUploadOperation: cannot move the archive to '{}': {}", options_.remoteArchivePath.value(), - renameResult.error().message + moved.error().message ); - if (renameResult.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) - return std::unexpected(Error{.type = ErrorType::FileExists}); - return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = renameResult.error()}); + return std::unexpected(remoteMoveError(moved.error())); } tempCreated_ = false; return {}; @@ -704,10 +730,13 @@ void ArchiveUploadOperation::cleanup() if (tempCreated_) { tempCreated_ = false; - if (auto* processingStrand = sftp_->strand(); processingStrand && processingStrand->withinProcessingThread()) - std::ignore = sftp_->removeFileInStrand(tempPath_); + const auto sftp = openSession(sftp_); + if (!sftp) + Log::warn("ArchiveUploadOperation: The channel is closed, '{}' stays on the server.", tempPath_.value()); + else if (sftp->strand()->withinProcessingThread()) + std::ignore = sftp->removeFileInStrand(tempPath_); else - std::ignore = sftp_->removeFile(tempPath_).wait_for(options_.futureTimeout); + std::ignore = sftp->removeFile(tempPath_).wait_for(options_.futureTimeout); } } diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index d3d54d70..b5938ee6 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -1,10 +1,14 @@ #include +#include #include #include #include #include #include +#include +#include + namespace { BulkTransferOperation::CommonOptions toCommon(BulkDownloadOperation::BulkDownloadOperationOptions const& options) @@ -21,10 +25,15 @@ namespace } } -BulkDownloadOperation::BulkDownloadOperation(SecureShell::SftpSession& sftp, BulkDownloadOperationOptions options) - : BulkTransferOperation{sftp, toCommon(options)} +BulkDownloadOperation::BulkDownloadOperation( + std::weak_ptr sftp, + BulkDownloadOperationOptions options +) + : BulkTransferOperation{std::move(sftp), toCommon(options)} , individualOptions_{std::move(options.individualOptions)} -{} +{ + individualOptions_.verifiedDirectories = std::make_shared(); +} BulkDownloadOperation::~BulkDownloadOperation() = default; @@ -34,24 +43,33 @@ SharedData::OperationType BulkDownloadOperation::type() const } std::expected -BulkDownloadOperation::prepareRootInStrand(SharedData::DirectoryEntry const& root) +BulkDownloadOperation::prepareRootInStrand(SecureShell::SftpSession&, SharedData::DirectoryEntry const& root) { // Base directory of the download, everything after this will be relative to this: return createLocalDirectory(common().localPath, root); } -std::expected -BulkDownloadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) +std::expected BulkDownloadOperation::createDirectoryInStrand( + SecureShell::SftpSession&, + std::size_t entryIndex, + EntryPaths const& paths +) { // create_directories follows a link planted inside the tree and would create and fill directories outside it. std::error_code error{}; - const auto link = Utility::firstLinkBelow(common().localPath.native(), paths.local.native(), error); + const auto link = Utility::firstLinkBelow( + common().localPath.native(), paths.local.native(), error, individualOptions_.verifiedDirectories.get() + ); if (error) { - return std::unexpected(Error{ - .type = ErrorType::FileStatFailed, - .extraInfo = fmt::format("Inspecting local directory: {}: {}", paths.local.toUtf8(), error.message()) - }); + return std::unexpected( + Error{ + .type = ErrorType::FileStatFailed, + .extraInfo = fmt::format( + "Inspecting local directory: {}: {}", paths.local.toUtf8(), Utility::errorMessage(error) + ) + } + ); } if (link) { @@ -74,15 +92,22 @@ BulkDownloadOperation::createLocalDirectory(Paths::LocalPath const& localPath, S std::filesystem::create_directories(path, ec); if (ec) { - Log::error("BulkDownloadOperation: Failed to create local directory: {}: {}", path.string(), ec.message()); + Log::error( + "BulkDownloadOperation: Failed to create local directory: {}: {}", + Paths::nativeToUtf8(path), + Utility::errorMessage(ec) + ); return std::unexpected( Error{ .type = ErrorType::CannotCreateDirectory, - .extraInfo = fmt::format("Creating local directory: {}: {}", path.string(), ec.message()) + .extraInfo = fmt::format( + "Creating local directory: {}: {}", Paths::nativeToUtf8(path), Utility::errorMessage(ec) + ) } ); } - return applyPermsToDirectory(path, entry); + deferDirectoryPermissions(path, entry); + return {}; } std::unique_ptr BulkDownloadOperation::makeChild( @@ -97,6 +122,7 @@ std::unique_ptr BulkDownloadOperation::makeChild( downloadOptions.remotePath = paths.remote; downloadOptions.localPath = paths.local; downloadOptions.mayOverwrite = downloadOptions.mayOverwrite || paths.mayOverwrite; + downloadOptions.comparedDestination = paths.comparedDestination; downloadOptions.bigFileOptimized = entry.size >= Constants::bigFileCutOff; downloadOptions.progressCallback = [onProgress = std::move(onProgress)](auto, auto max, auto current, auto) { @@ -112,9 +138,11 @@ bool BulkDownloadOperation::isSkippableError(Error const& error) const { // Missing or unreadable single files, and local files that cannot be written, do not compromise // the rest of the bulk. - // A destination that appeared after the existence check and may not be replaced fails only its own entry. + // A destination that appeared after the existence check and may not be replaced, or one that changed after a + // sync compared it, fails only its own entry. if (error.type == ErrorType::OpenFailure || error.type == ErrorType::CannotCreateDirectory || - error.type == ErrorType::LinkInLocalPath || error.type == ErrorType::FileExists) + error.type == ErrorType::LinkInLocalPath || error.type == ErrorType::FileExists || + error.type == ErrorType::ChangedSinceCompare) return true; return error.sftpError && (error.sftpError->sftpError == SSH_FX_NO_SUCH_FILE || error.sftpError->sftpError == SSH_FX_PERMISSION_DENIED || @@ -126,35 +154,46 @@ bool BulkDownloadOperation::isSkippableDirectoryError(Error const& error) const return error.type == ErrorType::LinkInLocalPath; } -std::expected BulkDownloadOperation::applyPermsToDirectory( +void BulkDownloadOperation::deferDirectoryPermissions( std::filesystem::path const& path, SharedData::DirectoryEntry const& entryToInheritFrom ) { // Unknown permissions (the scan root carries none) leave the directory as created. - std::optional toApplyPerms{std::nullopt}; - if (individualOptions_.inheritPermissions) - toApplyPerms = Utility::inheritablePermissions(entryToInheritFrom.permissions); - else if (individualOptions_.directoryPermissions) - toApplyPerms = Utility::sanitizePermissions(*individualOptions_.directoryPermissions); - - if (!toApplyPerms) - return {}; + const auto permissions = individualOptions_.inheritPermissions + ? Utility::inheritablePermissions(entryToInheritFrom.permissions) + : individualOptions_.directoryPermissions.and_then(Utility::sanitizePermissions); + if (permissions) + deferredDirectoryPermissions_.emplace_back(path, *permissions); +} - std::error_code ec; - std::filesystem::permissions(path, *toApplyPerms, ec); - if (ec) +std::expected BulkDownloadOperation::finishInStrand() +{ + // Children first, a parent that loses its search permission would stop them. + for (auto const& [path, permissions] : deferredDirectoryPermissions_ | std::views::reverse) { - Log::error( - "BulkDownloadOperation: Failed to set permissions on local directory: {}: {}", path.string(), ec.message() - ); - return std::unexpected( - Error{ - .type = ErrorType::CannotSetFilePermissions, - .extraInfo = fmt::format("Setting permissions on local directory: {}: {}", path.string(), ec.message()) - } - ); + std::error_code error{}; + std::filesystem::permissions(path, permissions, error); + if (error) + { + Log::error( + "BulkDownloadOperation: Failed to set permissions on local directory: {}: {}", + Paths::nativeToUtf8(path), + Utility::errorMessage(error) + ); + return std::unexpected( + Error{ + .type = ErrorType::CannotSetFilePermissions, + .extraInfo = fmt::format( + "Setting permissions on local directory: {}: {}", + Paths::nativeToUtf8(path), + Utility::errorMessage(error) + ), + } + ); + } } + deferredDirectoryPermissions_.clear(); return {}; } @@ -171,17 +210,57 @@ void BulkDownloadOperation::setPrescannedFileList(std::vector fi // Flat entries carry their name only, the paths come from the prescanned paths. if (const auto name = file.source.filename(); name) entry.name = name->value(); - entry.type = SharedData::FileType::Regular; - entry.size = file.sizeBytes; + entry.type = file.isDirectory ? SharedData::FileType::Directory : SharedData::FileType::Regular; + entry.size = file.isDirectory ? 0u : file.sizeBytes; entry.mtime = file.mtime; entry.mtimeNsec = file.mtimeNsec; - totalBytes += file.sizeBytes; + totalBytes += entry.size; entries.push_back(std::move(entry)); paths.push_back( EntryPaths{ - .remote = std::move(file.source), .local = std::move(file.destination), .mayOverwrite = file.mayOverwrite + .remote = std::move(file.source), + .local = std::move(file.destination), + .mayOverwrite = file.mayOverwrite, + .comparedDestination = std::move(file.comparedDestination), } ); } setPrescannedEntries(std::move(entries), std::move(paths), totalBytes); } + +std::optional +BulkDownloadOperation::resumeRequest(SharedData::BulkDownloadRequest request) const +{ + if (!hasEntries()) + return std::nullopt; + const auto unfinished = unfinishedEntryIndices(); + const auto isSymlink = [this](std::size_t index) + { + return entries()[index].isSymlink(); + }; + if (std::ranges::any_of(unfinished, isSymlink)) + return std::nullopt; + + request.entries.clear(); + for (const auto index : unfinished) + { + auto const& entry = entries()[index]; + const auto paths = entryPaths(index); + if (!paths || (!entry.isRegularFile() && !entry.isDirectory())) + continue; + request.entries.push_back({ + .source = paths->remote, + .destination = paths->local, + .sizeBytes = entry.isDirectory() ? 0u : entry.size, + .isDirectory = entry.isDirectory(), + .withoutContent = entry.isDirectory(), + .mtime = entry.mtime, + .mtimeNsec = entry.mtimeNsec, + .mayOverwrite = paths->mayOverwrite, + .comparedDestination = paths->comparedDestination, + }); + } + request.remoteRoot = common().remotePath; + request.localRoot = common().localPath; + return request; +} diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 19fc4572..c8b17055 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -1,14 +1,16 @@ #include #include #include +#include #include +#include using namespace std::chrono_literals; -BulkTransferOperation::BulkTransferOperation(SecureShell::SftpSession& sftp, CommonOptions common) +BulkTransferOperation::BulkTransferOperation(std::weak_ptr sftp, CommonOptions common) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , common_{std::move(common)} , budget_{std::max(1, common_.concurrency)} {} @@ -16,6 +18,7 @@ BulkTransferOperation::BulkTransferOperation(SecureShell::SftpSession& sftp, Com BulkTransferOperation::~BulkTransferOperation() { std::ignore = cancel(false); + waitForStrandTasks(strand()); } std::expected BulkTransferOperation::work() @@ -28,7 +31,10 @@ std::expected B return result; } - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return workInStrand(); @@ -57,13 +63,15 @@ std::expected B switch (state()) { case (NotStarted): - return start(); case (Preparing): - [[fallthrough]]; case (Prepared): - [[fallthrough]]; case (Running): - return step(); + { + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + return state() == NotStarted ? start(*sftp) : step(*sftp); + } case (Finalizing): { enterState(Completed); @@ -96,7 +104,8 @@ std::expected B return enterErrorState({.type = ErrorType::UnknownWorkState}); } -std::expected BulkTransferOperation::start() +std::expected +BulkTransferOperation::start(SecureShell::SftpSession& sftp) { if (entrySource_ == EntrySource::None) { @@ -140,9 +149,10 @@ std::expected B Error{.type = ErrorType::ImplementationError, .extraInfo = "A tree needs both roots."} ); } - const auto result = prepareRootInStrand(root); + const auto result = prepareRootInStrand(sftp, root); if (!result.has_value()) return enterErrorState(result.error()); + finishedEntries_[0] = true; cursor_ = 1; } @@ -151,7 +161,8 @@ std::expected B return WorkStatus::MoreWork; } -std::expected BulkTransferOperation::step() +std::expected +BulkTransferOperation::step(SecureShell::SftpSession& sftp) { if (cursor_ > entries_.size()) { @@ -163,7 +174,7 @@ std::expected B } const auto slotsBefore = slots_.size(); - if (const auto filled = fillSlots(); !filled.has_value()) + if (const auto filled = fillSlots(sftp); !filled.has_value()) return enterErrorState(filled.error()); // Report newly started files before stepping them: small files finish within this very // step and would otherwise never be seen in flight. @@ -183,6 +194,8 @@ std::expected B if (slots_.empty() && deferred_.empty() && cursor_ == entries_.size()) { + if (const auto finished = finishInStrand(); !finished.has_value()) + return enterErrorState(finished.error()); emitProgress(true); Log::info("{}: Bulk transfer completed.", logName()); enterState(OperationState::Completed); @@ -193,7 +206,7 @@ std::expected B return WorkStatus::MoreWork; } -std::expected BulkTransferOperation::fillSlots() +std::expected BulkTransferOperation::fillSlots(SecureShell::SftpSession& sftp) { // Directories are created inline and each blocks the strand (a round trip or two on // upload), so a step creates only a few before yielding to the queue. @@ -226,7 +239,7 @@ std::expected BulkTransferOperation::fillSlo { Log::warn("{}: Skipping '{}': {}", logName(), displayPath(index), paths.error().toString()); failedEntries_.emplace_back(failureSource(index), paths.error()); - failedEntryIndices_.push_back(static_cast(index)); + indexedFailures_.push_back({.index = static_cast(index), .type = paths.error().type}); completedBytes_ += entry.isRegularFile() ? entry.size : 0u; ++completedEntries_; continue; @@ -235,14 +248,19 @@ std::expected BulkTransferOperation::fillSlo { // Created inline while the cursor passes it, so files listed after it only start once // it exists. Scan order lists parents before children. - const auto result = createDirectoryInStrand(index, *paths); + const auto result = createDirectoryInStrand(sftp, index, *paths); if (!result.has_value()) { - if (!isSkippableDirectoryError(result.error())) + // Without a tree, nothing else depends on a directory entry. + if (!prescanned() && !isSkippableDirectoryError(result.error())) return std::unexpected(result.error()); Log::warn("{}: Skipping directory '{}': {}", logName(), displayPath(index), result.error().toString()); failedEntries_.emplace_back(failureSource(index), result.error()); - failedEntryIndices_.push_back(static_cast(index)); + indexedFailures_.push_back({.index = static_cast(index), .type = result.error().type}); + } + else + { + finishedEntries_[index] = true; } ++completedEntries_; ++directoriesCreated; @@ -262,10 +280,13 @@ std::expected BulkTransferOperation::fillSlo } } ); - slots_.push_back(Slot{.entryIndex = index, .child = std::move(child), .totalBytes = entry.size}); + slots_.push_back( + Slot{.entryIndex = index, .paths = *paths, .child = std::move(child), .totalBytes = entry.size} + ); continue; } Log::warn("{}: Skipping unsupported file type for entry: {}.", logName(), displayPath(index)); + finishedEntries_[index] = true; ++completedEntries_; } return {}; @@ -279,7 +300,7 @@ std::expected BulkTransferOperation::stepSlo if (result.has_value()) { if (result.value() == WorkStatus::Complete) - finishSlot(slot); + finishSlot(slot, true); continue; } @@ -300,13 +321,13 @@ std::expected BulkTransferOperation::stepSlo continue; } - Log::error("{}: Transfer failed for: {}: {}", logName(), displayPath(slot.entryIndex), error.toString()); + Log::error("{}: Transfer failed for: {}: {}", logName(), slot.paths.remote.value(), error.toString()); if (isSkippableError(error)) { // Not critical for the bulk as a whole: record the entry for the user and move on. - failedEntries_.emplace_back(failureSource(slot.entryIndex), error); - failedEntryIndices_.push_back(static_cast(slot.entryIndex)); - finishSlot(slot); + failedEntries_.emplace_back(failureSource(slot.paths), error); + indexedFailures_.push_back({.index = static_cast(slot.entryIndex), .type = error.type}); + finishSlot(slot, false); if (common_.failFast) { Log::error("{}: failFast is enabled, failing the whole operation due to the error.", logName()); @@ -319,8 +340,10 @@ std::expected BulkTransferOperation::stepSlo return {}; } -void BulkTransferOperation::finishSlot(Slot& slot) +void BulkTransferOperation::finishSlot(Slot& slot, bool transferred) { + if (transferred) + finishedEntries_[slot.entryIndex] = true; // Advance by the entry's reported size, not the child's measured size: totalBytes_ is // summed from the same reported sizes, so the two converge exactly. completedBytes_ += entries_[slot.entryIndex].size; @@ -348,8 +371,8 @@ std::uint64_t BulkTransferOperation::fileCount() const noexcept // Tree mode treats entries_[0] as the synthetic root (not counted toward the user-visible // total); prescanned mode has no root, so every entry counts. if (entries_.empty()) - return 0; - return entries_.size() - (prescanned() ? 0 : 1); + return earlierProgress_.entries; + return earlierProgress_.entries + entries_.size() - (prescanned() ? 0 : 1); } std::uint64_t BulkTransferOperation::bytesNow() const noexcept @@ -377,8 +400,12 @@ void BulkTransferOperation::emitProgress(bool force) .bytesTotal = totalBytes_, .bytesPerSecond = 0, .inFlight = {}, - .failedEntryIndices = failedEntryIndices_, + .failedEntryIndices = {}, }; + update.failedEntryIndices.reserve(indexedFailures_.size()); + std::ranges::transform( + indexedFailures_, std::back_inserter(update.failedEntryIndices), &SharedData::IndexedFailure::index + ); update.bytesPerSecond = updateBulkBytesPerSecond(update.bytesCurrent); update.inFlight.reserve(slots_.size() + deferred_.size()); for (auto const& slot : slots_) @@ -387,7 +414,7 @@ void BulkTransferOperation::emitProgress(bool force) continue; update.inFlight.push_back( SharedData::BulkFileProgress{ - .file = displayPath(slot.entryIndex), + .file = slot.paths.remote.value(), .entryIndex = slot.entryIndex, .bytes = slot.bytes, .totalBytes = slot.totalBytes, @@ -460,7 +487,8 @@ void BulkTransferOperation::setParallelBudget(int slots) void BulkTransferOperation::setScanResult(std::vector&& entries, std::uint64_t totalBytes) { entries_ = std::move(entries); - totalBytes_ = totalBytes; + finishedEntries_.assign(entries_.size(), false); + totalBytes_ = totalBytes + earlierProgress_.bytes; prescannedPaths_.clear(); entrySource_ = EntrySource::Tree; } @@ -472,8 +500,9 @@ void BulkTransferOperation::setPrescannedEntries( ) { entries_ = std::move(entries); + finishedEntries_.assign(entries_.size(), false); prescannedPaths_ = std::move(paths); - totalBytes_ = totalBytes; + totalBytes_ = totalBytes + earlierProgress_.bytes; entrySource_ = EntrySource::Flat; } @@ -488,6 +517,21 @@ void BulkTransferOperation::addRejectedEntries(std::vector const& p } } +void BulkTransferOperation::addSkippedLinks(std::vector const& paths) +{ + for (auto const& path : paths) + { + Log::warn("{}: Skipping the link '{}', links are not transferred from here.", logName(), path); + failedEntries_.emplace_back( + path, + Error{ + .type = ErrorType::OperationNotPossibleOnFileType, + .extraInfo = "Links and junctions are not transferred" + } + ); + } +} + void BulkTransferOperation::addUnreadableDirectories(std::vector const& paths) { for (auto const& path : paths) @@ -504,45 +548,87 @@ std::vector> BulkTransferOp return failedEntries_; } -std::expected -BulkTransferOperation::entryPaths(std::size_t entryIndex) const +void BulkTransferOperation::countEarlierProgress(BulkResumeProgress progress) { - if (prescanned()) - return prescannedPaths_[entryIndex]; - if (common_.remotePath.empty() || common_.localPath.empty()) - return std::unexpected(Error{.type = ErrorType::ImplementationError, .extraInfo = "A tree needs both roots"}); + earlierProgress_ = progress; + completedEntries_ += progress.entries; + completedBytes_ += progress.bytes; + totalBytes_ += progress.bytes; +} - const auto invalidName = [](Paths::PathError const& error) +BulkResumeProgress BulkTransferOperation::finishedProgress() const +{ + auto progress = earlierProgress_; + for (std::size_t index = prescanned() ? 0 : 1; index < finishedEntries_.size(); ++index) { - return Error{.type = ErrorType::InvalidName, .extraInfo = error.toString()}; - }; + if (!finishedEntries_[index]) + continue; + ++progress.entries; + progress.bytes += entries_[index].isRegularFile() ? entries_[index].size : 0u; + } + return progress; +} - // The names from the entry up to, but not including, the root. - std::vector names; - for (auto const* current = &entries_[entryIndex]; current->parent; current = &entries_[*current->parent]) +std::vector BulkTransferOperation::unfinishedEntryIndices() const +{ + std::vector unfinished; + for (std::size_t index = 0; index < finishedEntries_.size(); ++index) { - if (*current->parent >= entries_.size()) - return std::unexpected(Error{.type = ErrorType::ImplementationError, .extraInfo = "Parent out of range"}); - names.push_back(¤t->name); + if (!finishedEntries_[index]) + unfinished.push_back(index); } + return unfinished; +} - Paths::RelativePath relative{}; - for (auto name = names.rbegin(); name != names.rend(); ++name) +std::expected +BulkTransferOperation::entryPaths(std::size_t entryIndex) const +{ + if (prescanned()) + return prescannedPaths_[entryIndex]; + return Detail::treeEntryPaths(entries_, entryIndex, common_.remotePath, common_.localPath); +} + +namespace Detail +{ + std::expected treeEntryPaths( + std::vector const& entries, + std::size_t entryIndex, + Paths::RemotePath const& remoteRoot, + Paths::LocalPath const& localRoot + ) { - auto segment = Paths::PathSegment::make(**name); - if (!segment) - return std::unexpected(invalidName(segment.error())); - relative = relative / *segment; - } + using ErrorType = Operation::ErrorType; + if (remoteRoot.empty() || localRoot.empty()) + return std::unexpected( + Operation::Error{.type = ErrorType::ImplementationError, .extraInfo = "A tree needs both roots"} + ); + if (entryIndex >= entries.size()) + return std::unexpected( + Operation::Error{.type = ErrorType::ImplementationError, .extraInfo = "Entry out of range"} + ); - return common_.localPath.join(relative) - .transform( - [this, &relative](Paths::LocalPath local) - { - return EntryPaths{.remote = common_.remotePath / relative, .local = std::move(local)}; - } - ) - .transform_error(invalidName); + const auto refused = [](Paths::PathError const& error) + { + if (error.reason == Paths::PathRejection::UnknownParent) + return Operation::Error{.type = ErrorType::ImplementationError, .extraInfo = "Parent out of range"}; + return Operation::Error{.type = ErrorType::InvalidName, .extraInfo = error.toString()}; + }; + return Utility::walkedRelativePath(entries, entries[entryIndex]) + .and_then( + [&](Paths::RelativePath const& relative) + { + return localRoot.join(relative).transform( + [&](Paths::LocalPath local) + { + return BulkTransferOperation::EntryPaths{ + .remote = remoteRoot / relative, .local = std::move(local) + }; + } + ); + } + ) + .transform_error(refused); + } } std::string BulkTransferOperation::displayPath(std::size_t entryIndex) const @@ -555,27 +641,21 @@ std::string BulkTransferOperation::displayPath(std::size_t entryIndex) const std::string BulkTransferOperation::failureSource(std::size_t entryIndex) const { if (auto paths = entryPaths(entryIndex); paths) - return sourceIsRemote() ? paths->remote.value() : paths->local.toUtf8(); + return failureSource(*paths); return describeEntry(entryIndex); } +std::string BulkTransferOperation::failureSource(EntryPaths const& paths) const +{ + return sourceIsRemote() ? paths.remote.value() : paths.local.toUtf8(); +} + std::string BulkTransferOperation::describeEntry(std::size_t entryIndex) const { // The names as listed, one of them is no path segment or the entry would have paths. - std::vector names; - for (auto const* current = &entries_[entryIndex]; current->parent && *current->parent < entries_.size(); - current = &entries_[*current->parent]) - { - names.push_back(current->name); - } - auto root = sourceIsRemote() ? common_.remotePath.value() : common_.localPath.toUtf8(); - for (auto name = names.rbegin(); name != names.rend(); ++name) - { - if (!root.ends_with('/')) - root.push_back('/'); - root += *name; - } - return root; + return Utility::describeWalkedEntry( + sourceIsRemote() ? common_.remotePath.value() : common_.localPath.toUtf8(), entries_, entries_[entryIndex] + ); } void BulkTransferOperation::cancelChildrenInStrand(bool adoptCancelState) @@ -596,12 +676,12 @@ std::expected BulkTransferOperation::cancel( // Each child's cancel and destructor would block on the processing thread separately; // hop once and do all of them there. - auto* processingStrand = sftp_->strand(); - if (!slots_.empty() && processingStrand && !processingStrand->withinProcessingThread()) + const auto sftp = openSession(sftp_); + if (!slots_.empty() && sftp && !sftp->strand()->withinProcessingThread()) { // Only wait, never get(): a finalized strand hands back an exceptional future and this // runs from the destructor too. - auto fut = sftp_->performPromise( + auto fut = sftp->performPromise( [this, adoptCancelState]() -> bool { cancelChildrenInStrand(adoptCancelState); @@ -631,5 +711,5 @@ void BulkTransferOperation::pause(bool doPause) SecureShell::ProcessingStrand* BulkTransferOperation::strand() const { - return sftp_->strand(); + return strandOf(sftp_); } diff --git a/backend/source/backend/sftp/bulk_upload_operation.cpp b/backend/source/backend/sftp/bulk_upload_operation.cpp index c09678ae..a54eaacf 100644 --- a/backend/source/backend/sftp/bulk_upload_operation.cpp +++ b/backend/source/backend/sftp/bulk_upload_operation.cpp @@ -4,6 +4,9 @@ #include #include +#include +#include + namespace { BulkTransferOperation::CommonOptions toCommon(BulkUploadOperation::BulkUploadOperationOptions const& options) @@ -20,8 +23,11 @@ namespace } } -BulkUploadOperation::BulkUploadOperation(SecureShell::SftpSession& sftp, BulkUploadOperationOptions options) - : BulkTransferOperation{sftp, toCommon(options)} +BulkUploadOperation::BulkUploadOperation( + std::weak_ptr sftp, + BulkUploadOperationOptions options +) + : BulkTransferOperation{std::move(sftp), toCommon(options)} , individualOptions_{std::move(options.individualOptions)} {} @@ -33,24 +39,65 @@ SharedData::OperationType BulkUploadOperation::type() const } std::expected -BulkUploadOperation::prepareRootInStrand(SharedData::DirectoryEntry const& root) +BulkUploadOperation::prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) { // Base directory of the upload, everything after this will be relative to this: - return createRemoteDirectoryInStrand(common().remotePath, root); + return createRemoteDirectoryInStrand(sftp, common().remotePath, root); +} + +std::expected BulkUploadOperation::createDirectoryInStrand( + SecureShell::SftpSession& sftp, + std::size_t entryIndex, + EntryPaths const& paths +) +{ + // A scanned directory comes after its parent, a prescanned one may be the first entry below it. + if (prescanned() && individualOptions_.createMissingDirectories) + { + if (auto parents = createMissingParentsInStrand(sftp, paths.remote); !parents) + return parents; + } + return createRemoteDirectoryInStrand(sftp, paths.remote, entries()[entryIndex]); } std::expected -BulkUploadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) +BulkUploadOperation::createMissingParentsInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& path) { - return createRemoteDirectoryInStrand(paths.remote, entries()[entryIndex]); + auto const& root = common().remotePath; + std::vector missing; + for (auto parent = path.parent(); parent && *parent != root && parent->isWithin(root); parent = parent->parent()) + { + const auto stat = sftp.lstatInStrand(*parent); + if (stat) + break; + if (stat.error().sftpError != SSH_FX_NO_SUCH_FILE) + { + return std::unexpected( + Error{ + .type = ErrorType::SftpError, + .sftpError = stat.error(), + .extraInfo = fmt::format("Inspecting remote directory: {}", parent->value()) + } + ); + } + missing.push_back(*parent); + } + const SharedData::DirectoryEntry directory{.type = SharedData::FileType::Directory}; + for (auto const& parent : missing | std::views::reverse) + { + if (auto created = createRemoteDirectoryInStrand(sftp, parent, directory); !created) + return created; + } + return {}; } std::expected BulkUploadOperation::createRemoteDirectoryInStrand( + SecureShell::SftpSession& sftp, Paths::RemotePath const& path, SharedData::DirectoryEntry const& entry ) { - const auto result = sftp().createDirectoryIfItDoesntExistInStrand(path, determinePerms(entry)); + const auto result = sftp.createDirectoryIfItDoesntExistInStrand(path, determinePerms(entry)); if (!result.has_value()) { Log::error("BulkUploadOperation: Failed to create remote sftp directory: {}.", result.error().message); @@ -77,6 +124,7 @@ std::unique_ptr BulkUploadOperation::makeChild( uploadOptions.remotePath = paths.remote; uploadOptions.localPath = paths.local; uploadOptions.mayOverwrite = uploadOptions.mayOverwrite || paths.mayOverwrite; + uploadOptions.comparedDestination = paths.comparedDestination; uploadOptions.bigFileOptimized = entry.size >= Constants::bigFileCutOff; uploadOptions.progressCallback = [onProgress = std::move(onProgress)](auto, auto max, auto current, auto) { @@ -139,15 +187,46 @@ void BulkUploadOperation::setPrescannedFileList(std::vector file // Flat entries carry their name only, the paths come from the prescanned paths. if (const auto name = file.source.filename(); name) entry.name = name->value(); - entry.type = SharedData::FileType::Regular; - entry.size = file.sizeBytes; - totalBytes += file.sizeBytes; + entry.type = file.isDirectory ? SharedData::FileType::Directory : SharedData::FileType::Regular; + entry.size = file.isDirectory ? 0u : file.sizeBytes; + totalBytes += entry.size; entries.push_back(std::move(entry)); paths.push_back( EntryPaths{ - .remote = std::move(file.destination), .local = std::move(file.source), .mayOverwrite = file.mayOverwrite + .remote = std::move(file.destination), + .local = std::move(file.source), + .mayOverwrite = file.mayOverwrite, + .comparedDestination = std::move(file.comparedDestination), } ); } setPrescannedEntries(std::move(entries), std::move(paths), totalBytes); } + +std::optional +BulkUploadOperation::resumeRequest(SharedData::BulkUploadRequest request) const +{ + if (!hasEntries()) + return std::nullopt; + + request.entries.clear(); + for (const auto index : unfinishedEntryIndices()) + { + auto const& entry = entries()[index]; + const auto paths = entryPaths(index); + if (!paths || (!entry.isRegularFile() && !entry.isSymlink() && !entry.isDirectory())) + continue; + request.entries.push_back({ + .source = paths->local, + .destination = paths->remote, + .sizeBytes = entry.isDirectory() ? 0u : entry.size, + .isDirectory = entry.isDirectory(), + .withoutContent = entry.isDirectory(), + .mayOverwrite = paths->mayOverwrite, + .comparedDestination = paths->comparedDestination, + }); + } + request.remoteRoot = common().remotePath; + request.localRoot = common().localPath; + return request; +} diff --git a/backend/source/backend/sftp/compared_state_check.cpp b/backend/source/backend/sftp/compared_state_check.cpp new file mode 100644 index 00000000..b33e922f --- /dev/null +++ b/backend/source/backend/sftp/compared_state_check.cpp @@ -0,0 +1,197 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace ComparedStateCheck +{ + namespace + { + Operation::Error changedError(std::string const& path) + { + Log::warn("'{}' changed after the sync compared it, it is neither replaced nor deleted.", path); + return Operation::Error{ + .type = Operation::ErrorType::ChangedSinceCompare, + .extraInfo = fmt::format("'{}' changed after the compare", path), + }; + } + + SharedData::FileType localTypeOf( + [[maybe_unused]] std::filesystem::path const& path, + std::filesystem::file_status status, + [[maybe_unused]] std::error_code& error + ) + { +#ifdef _WIN32 + if (std::filesystem::is_symlink(status) || Utility::isLinkOrJunction(path, error)) + return SharedData::FileType::Unknown; +#else + if (std::filesystem::is_symlink(status)) + return SharedData::FileType::Symlink; +#endif + return SharedData::fileTypeFromStdFilesystemType(status.type()); + } + } + + std::uint64_t secondsSinceEpoch(std::filesystem::file_time_type time) + { + return static_cast( + std::chrono::duration_cast(std::chrono::file_clock::to_sys(time).time_since_epoch()) + .count() + ); + } + + std::expected localStateOf(std::filesystem::path const& path) + { + std::error_code error{}; + const auto status = std::filesystem::symlink_status(path, error); + if (error == std::errc::no_such_file_or_directory || status.type() == std::filesystem::file_type::not_found) + return SharedData::ComparedState{}; + if (error) + return std::unexpected(error); + + SharedData::ComparedState state{.exists = true, .type = localTypeOf(path, status, error)}; + if (error) + return std::unexpected(error); + switch (state.type) + { + case SharedData::FileType::Symlink: + { + const auto literal = std::filesystem::read_symlink(path, error); + if (error) + return std::unexpected(error); + state.linkTarget = Paths::nativeToUtf8(literal); + return state; + } + case SharedData::FileType::Regular: + { + state.size = std::filesystem::file_size(path, error); + if (error) + return std::unexpected(error); + const auto modified = std::filesystem::last_write_time(path, error); + if (error) + return std::unexpected(error); + state.mtime = secondsSinceEpoch(modified); + return state; + } + default: + return state; + } + } + + std::expected + remoteStateOfInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& path) + { + const auto information = sftp.lstatInStrand(path); + if (!information) + { + if (information.error().sftpError == SSH_FX_NO_SUCH_FILE) + return SharedData::ComparedState{}; + return std::unexpected(information.error()); + } + auto state = SharedData::comparedStateOf(*information); + if (state.type != SharedData::FileType::Symlink) + return state; + return sftp.readLinkInStrand(path).transform( + [&state](std::string literal) + { + state.linkTarget = std::move(literal); + return state; + } + ); + } + + std::expected + requireUnchangedLocal(std::filesystem::path const& path, std::optional const& compared) + { + if (!compared) + return {}; + const auto current = localStateOf(path); + if (!current) + { + return std::unexpected( + Operation::Error{ + .type = Operation::ErrorType::FileStatFailed, + .extraInfo = fmt::format( + "Inspecting '{}': {}", Paths::nativeToUtf8(path), Utility::errorMessage(current.error()) + ), + } + ); + } + if (!SharedData::unchangedSinceCompare(*compared, *current)) + return std::unexpected(changedError(Paths::nativeToUtf8(path))); + return {}; + } + + std::expected requireUnchangedRemoteInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& path, + std::optional const& compared + ) + { + if (!compared) + return {}; + const auto current = remoteStateOfInStrand(sftp, path); + if (!current) + return std::unexpected(Operation::Error{.type = Operation::ErrorType::SftpError, .sftpError = current.error()}); + if (!SharedData::unchangedSinceCompare(*compared, *current)) + return std::unexpected(changedError(path.value())); + return {}; + } + + std::expected moveLocalIntoPlace( + std::filesystem::path const& from, + std::filesystem::path const& to, + bool replace, + std::optional const& compared + ) + { + return requireUnchangedLocal(to, compared) + .and_then( + [&]() -> std::expected + { + if (const auto moved = Utility::moveIntoPlace(from, to, replace); !moved) + { + Log::error( + "Failed to move '{}' into place: {}", + Paths::nativeToUtf8(to), + Utility::errorMessage(moved.error()) + ); + return std::unexpected(Operation::localMoveError(moved.error())); + } + return {}; + } + ); + } + + std::expected moveRemoteIntoPlaceInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& from, + Paths::RemotePath const& to, + bool replace, + std::string const& asideSuffix, + SharedData::FileType sourceType, + std::optional const& compared + ) + { + return requireUnchangedRemoteInStrand(sftp, to, compared) + .and_then( + [&]() -> std::expected + { + const auto moved = sftp.moveIntoPlaceInStrand(from, to, replace, asideSuffix, sourceType); + if (!moved) + { + Log::error("Failed to move '{}' into place: {}", to.value(), moved.error().message); + return std::unexpected(Operation::remoteMoveError(moved.error())); + } + return {}; + } + ); + } +} diff --git a/backend/source/backend/sftp/delete_operation.cpp b/backend/source/backend/sftp/delete_operation.cpp index 55af74a3..6f049a50 100644 --- a/backend/source/backend/sftp/delete_operation.cpp +++ b/backend/source/backend/sftp/delete_operation.cpp @@ -1,11 +1,35 @@ +#include #include #include #include -DeleteOperation::DeleteOperation(SecureShell::SftpSession& sftp, DeleteOperationOptions options) +namespace +{ + /** + * @brief Removes @p path by itself, a directory only when it is empty. Its type is looked up unless given. + */ + std::expected removeOneInStrand( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& path, + std::optional type + ) + { + if (!type) + { + const auto information = sftp.lstatInStrand(path); + if (!information) + return std::unexpected(information.error()); + type = information->type; + } + return *type == SharedData::FileType::Directory ? sftp.removeDirectoryInStrand(path) + : sftp.removeFileInStrand(path); + } +} + +DeleteOperation::DeleteOperation(std::weak_ptr sftp, DeleteOperationOptions options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , remotePath_{std::move(options.remotePath)} , filesRemovedProgress_{std::move(options.filesRemovedProgress)} , futureTimeout_{options.futureTimeout} @@ -16,7 +40,7 @@ DeleteOperation::~DeleteOperation() SecureShell::ProcessingStrand* DeleteOperation::strand() const { - return sftp_->strand(); + return strandOf(sftp_); } std::expected DeleteOperation::work() @@ -47,7 +71,7 @@ std::expected DeleteOperati Log::error("DeleteOperation: Refusing to delete '{}'.", remotePath_.value()); return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = remotePath_.value()}); } - targets_ = {remotePath_}; + targets_ = {Target{.path = remotePath_}}; totalToDelete_ = 1; filesRemovedProgress_ = {}; } @@ -86,7 +110,7 @@ std::expected DeleteOperati return WorkStatus::MoreWork; } // Nothing deleted at all is a failure, some items left behind a partial success. - if (totalToDelete_ > 0 && failedEntries_.size() == totalToDelete_) + if (!failedEntries_.empty() && removalFailures_ == totalToDelete_) return enterErrorState(failedEntries_.back().second); state_ = Completed; return WorkStatus::Complete; @@ -125,20 +149,49 @@ std::expected DeleteOperati return enterErrorState({.type = ErrorType::UnknownWorkState}); } -std::expected DeleteOperation::removeOnce(Paths::RemotePath const& path) +std::expected DeleteOperation::removeOnce(Target const& target) { - auto fut = sftp_->removeAll({path}); + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto const& path = target.path; + auto fut = sftp->performPromise( + [weak = std::weak_ptr{sftp}, path, compared = target.compared]() -> std::expected + { + const auto session = weak.lock(); + if (!session) + return std::unexpected(channelClosedError()); + return ComparedStateCheck::requireUnchangedRemoteInStrand(*session, path, compared) + .and_then( + [&]() -> std::expected + { + const auto knownType = compared ? std::optional{compared->type} : std::nullopt; + // An item that is already gone, after a resume for one, counts as deleted. + if (auto removed = removeOneInStrand(*session, path, knownType); + !removed && removed.error().sftpError != SSH_FX_NO_SUCH_FILE) + { + return std::unexpected( + Error{ + .type = ErrorType::DeleteFailed, + .sftpError = std::move(removed).error(), + .extraInfo = path.value(), + } + ); + } + return {}; + } + ); + } + ); if (fut.wait_for(futureTimeout_) != std::future_status::ready) return enterErrorState({.type = ErrorType::FutureTimeout}); - // One item that cannot be deleted does not keep the others, its directories then stay as well. An item that is - // already gone, after a resume for one, counts as deleted. - if (const auto result = fut.get(); !result.has_value() && result.error().sftpError != SSH_FX_NO_SUCH_FILE) + // One item that cannot be deleted does not keep the others, its directories then stay as well. + if (auto result = fut.get(); !result.has_value()) { Log::error("DeleteOperation: Failed to delete '{}': {}", path.value(), result.error().toString()); - failedEntries_.emplace_back( - path.value(), Error{.type = ErrorType::DeleteFailed, .sftpError = result.error(), .extraInfo = path.value()} - ); + ++removalFailures_; + failedEntries_.emplace_back(path.value(), std::move(result).error()); } ++deletedThusFar_; @@ -176,21 +229,21 @@ void DeleteOperation::appendScanResult(Paths::RemotePath const& root, std::vecto invalidScanEntry_ = target.value(); break; } - targets_.push_back(std::move(target)); + targets_.push_back(Target{.path = std::move(target)}); } if (invalidScanEntry_) targets_.clear(); totalToDelete_ = targets_.size(); } -void DeleteOperation::setTargets(std::vector targets) +void DeleteOperation::setTargets(std::vector targets) { scanDelivered_ = true; for (auto& target : targets) { - if (target.empty() || target.isRoot()) + if (target.path.empty() || target.path.isRoot()) { - invalidScanEntry_ = target.value(); + invalidScanEntry_ = target.path.value(); break; } targets_.push_back(std::move(target)); @@ -200,11 +253,6 @@ void DeleteOperation::setTargets(std::vector targets) totalToDelete_ = targets_.size(); } -void DeleteOperation::scanFirst(std::unique_ptr scan) -{ - addScanRoot(std::move(scan)); -} - void DeleteOperation::addScanRoot(std::unique_ptr scan) { if (!scanPhase_) @@ -213,12 +261,30 @@ void DeleteOperation::addScanRoot(std::unique_ptr scan) [this](ScanOperation& finished) { appendScanResult(finished.remotePath(), finished.ejectEntries()); + }, + [this](ScanOperation& failed, Error const& error) + { + Log::error( + "DeleteOperation: Scan of '{}' failed, nothing of it is deleted: {}", + failed.remotePath().value(), + error.toString() + ); + // The other roots are still deleted, the failure alone does not mean the scans are incomplete. + scanDelivered_ = true; + addFailedTarget(failed.remotePath(), error); } ); } scanPhase_->addRoot(std::move(scan)); } +void DeleteOperation::addFailedTarget(Paths::RemotePath const& target, Error error) +{ + if (!error.extraInfo) + error.extraInfo = target.value(); + failedEntries_.emplace_back(target.value(), std::move(error)); +} + std::expected DeleteOperation::cancel(bool adoptCancelState) { if (scanPhase_) diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index 0302e2fd..2d38959f 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -7,34 +9,31 @@ #include #include #include -#include #include using namespace std::chrono_literals; -DownloadOperation::DownloadOperation(SecureShell::SftpSession& sftp, DownloadOperationOptions options) +DownloadOperation::DownloadOperation(std::weak_ptr sftp, DownloadOperationOptions options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , options_{std::move(options)} , localFile_{} { options_.tempFileSuffix = Persistence::effectiveTempFileSuffix(options_.tempFileSuffix); remotePath_ = options_.remotePath; if (!options_.localPath.empty()) + { localNativePath_ = options_.localPath.native(); + tempNativePath_ = localNativePath_; + tempNativePath_ += options_.tempFileSuffix; + } } DownloadOperation::~DownloadOperation() { std::ignore = cancel(false); - - if (auto stream = fileStream_.lock(); stream && !stream->strand()->withinProcessingThread()) - { - // Wait for all tasks of the operation to finish. Skipped on the processing thread itself, - // where the wait could never be satisfied. - stream->strand()->pushPromiseTask([]() {}).get(); - } + waitForStrandTasks(strand()); } std::expected DownloadOperation::work() @@ -42,7 +41,10 @@ std::expected DownloadO // Drive one work() step from outside the strand. OperationQueue normally calls // workInStrand directly inside its batched umbrella, so this wrapper is only used by // callers that aren't already on the strand (standalone ops, tests). - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return workInStrand(); @@ -60,6 +62,10 @@ std::expected DownloadO { using enum OperationState; + const auto sftp = openSession(sftp_); + if (!sftp && !hasEnded()) + return enterErrorState(channelClosedError()); + switch (state_) { case (NotStarted): @@ -76,7 +82,7 @@ std::expected DownloadO Log::error("DownloadOperation: Invalid local path."); return enterErrorState({.type = ErrorType::InvalidPath}); } - if (std::filesystem::exists(localNativePath_) && !options_.mayOverwrite) + if (std::error_code existsError{}; std::filesystem::exists(localNativePath_, existsError) && !options_.mayOverwrite) { Log::error( "DownloadOperation: File '{}' already exists and may not be overwritten.", @@ -84,7 +90,7 @@ std::expected DownloadO ); return enterErrorState({.type = ErrorType::FileExists}); } - const auto prepareResult = prepareInStrand(); + const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) { Log::error("DownloadOperation: Failed to prepare operation: {}", prepareResult.error().toString()); @@ -98,7 +104,7 @@ std::expected DownloadO state_ = Running; if (options_.entry->isSymlink() && options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) { - const auto result = handleSymlinkInStrand(); + const auto result = handleSymlinkInStrand(*sftp); if (!result.has_value()) { Log::error("DownloadOperation: Failed to handle symlink: {}", result.error().toString()); @@ -230,10 +236,16 @@ std::expected DownloadO std::expected DownloadOperation::handleSymlink() { - auto fut = sftp_->performPromise( - [this]() + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + auto fut = sftp->performPromise( + [this]() -> std::expected { - return handleSymlinkInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return handleSymlinkInStrand(*sftp); } ); if (fut.wait_for(options_.futureTimeout) != std::future_status::ready) @@ -244,7 +256,7 @@ std::expected DownloadOperation::handleSymlink() return fut.get(); } -std::expected DownloadOperation::handleSymlinkInStrand() +std::expected DownloadOperation::handleSymlinkInStrand(SecureShell::SftpSession& sftp) { if (options_.symlinkHandling == Persistence::SymlinkHandling::SkipSymlink) { @@ -252,7 +264,7 @@ std::expected DownloadOperation::handleSymlinkIn return {}; } - const auto readResult = readSymlinkInStrand(remotePath_); + const auto readResult = readSymlinkInStrand(sftp, remotePath_); if (!readResult.has_value()) return std::unexpected(readResult.error()); auto const& linkEntry = readResult.value().targetInfo; @@ -273,30 +285,55 @@ std::expected DownloadOperation::handleSymlinkIn Log::error( "DownloadOperation: Failed to create parent dirs for symlink '{}': {}.", options_.localPath.toUtf8(), - ec.message() + Utility::errorMessage(ec) ); return std::unexpected(Error{.type = ErrorType::CannotCreateDirectory}); } } - // The copy keeps the link's own content, so a relative link still points into the downloaded tree. - const auto linkContent = Paths::nativeFromUtf8(readResult.value().linkContent); + // The link is created under the temp name and moved into place like a file. Its copy keeps the link's own + // content, so a relative link still points into the downloaded tree. + auto const& linkText = readResult.value().linkContent; + if (!Paths::isValidUtf8(linkText)) + { + Log::error("DownloadOperation: The link '{}' points to a name that is not UTF-8.", options_.remotePath.value()); + return std::unexpected( + Error{.type = ErrorType::InvalidName, .extraInfo = Paths::sanitizedUtf8(linkText)} + ); + } + std::filesystem::remove(tempNativePath_, ec); + ec.clear(); + const auto linkContent = Paths::nativeFromUtf8(linkText); if (linkEntry && linkEntry->isDirectory()) - std::filesystem::create_directory_symlink(linkContent, localNativePath_, ec); + std::filesystem::create_directory_symlink(linkContent, tempNativePath_, ec); else - std::filesystem::create_symlink(linkContent, localNativePath_, ec); + std::filesystem::create_symlink(linkContent, tempNativePath_, ec); if (ec) { Log::error( - "DownloadOperation: Failed to create symlink at '{}': {}.", options_.localPath.toUtf8(), ec.message() + "DownloadOperation: Failed to create symlink at '{}': {}.", + Paths::nativeToUtf8(tempNativePath_), + Utility::errorMessage(ec) ); return std::unexpected(Error{.type = ErrorType::CannotCreateSymlink}); } + + if (auto moved = ComparedStateCheck::moveLocalIntoPlace( + tempNativePath_, localNativePath_, options_.mayOverwrite, options_.comparedDestination + ); + !moved) + { + std::filesystem::remove(tempNativePath_, ec); + return std::unexpected(std::move(moved).error()); + } return {}; } std::expected DownloadOperation::readOnce() { - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return readOnceInStrand(); @@ -346,21 +383,20 @@ std::expected DownloadOperation::readOnceInStran } const auto readAmount = result.value(); + const auto fileSize = options_.entry->size; if (readAmount == 0) { - return false; + if (bytesInFile_ >= fileSize) + return false; + Log::error("DownloadOperation: '{}' ended after {} of {} bytes.", options_.remotePath.value(), bytesInFile_, fileSize); + return enterErrorState(sizeMismatch()); } - std::uint64_t fileSize = 0; - bool good = true; - { - localFile_.write(buffer_.data(), static_cast(readAmount)); - bytesInFile_ += static_cast(readAmount); - fileSize = options_.entry->size; - good = localFile_.good(); - options_.progressCallback(0ull, fileSize, bytesInFile_, 0); - } + localFile_.write(buffer_.data(), static_cast(readAmount)); + bytesInFile_ += static_cast(readAmount); + const bool good = localFile_.good(); + options_.progressCallback(0ull, fileSize, bytesInFile_, 0); if (!good) { Log::error("DownloadOperation read cycle stopped: localFile_.good() == false"); @@ -369,12 +405,23 @@ std::expected DownloadOperation::readOnceInStran }); return false; } - return good && bytesInFile_ < fileSize; + return bytesInFile_ < fileSize; +} + +DownloadOperation::Error DownloadOperation::sizeMismatch() const +{ + return Error{ + .type = ErrorType::SizeMismatch, + .extraInfo = fmt::format( + "Received {} of {} bytes for {}", bytesInFile_, options_.entry->size, options_.remotePath.value() + ), + }; } bool DownloadOperation::commitBufferToFile(SecureShell::IFileStream::SignedSizeType bytesRead) { localFile_.write(buffer_.data(), static_cast(bytesRead)); + bytesInFile_ += static_cast(bytesRead); if (!localFile_.good()) { Log::error("DownloadOperation read cycle stopped: localFile_.good() == false"); @@ -392,12 +439,16 @@ DownloadOperation::refuseLinkedPath(std::filesystem::path const& path) const std::error_code error{}; std::optional link{}; if (!options_.linkFreeRoot.empty()) - link = Utility::firstLinkBelow(options_.linkFreeRoot.native(), path, error); + link = Utility::firstLinkBelow( + options_.linkFreeRoot.native(), path, error, options_.verifiedDirectories.get() + ); else if (Utility::isLinkOrJunction(path, error)) link = path; if (error) { - Log::error("DownloadOperation: Cannot inspect '{}': {}.", Paths::nativeToUtf8(path), error.message()); + Log::error( + "DownloadOperation: Cannot inspect '{}': {}.", Paths::nativeToUtf8(path), Utility::errorMessage(error) + ); return std::unexpected(Error{.type = ErrorType::FileStatFailed, .extraInfo = Paths::nativeToUtf8(path)}); } if (link) @@ -416,8 +467,7 @@ DownloadOperation::refuseLinkedPath(std::filesystem::path const& path) const std::expected DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) { - auto tempPath = localNativePath_; - tempPath += options_.tempFileSuffix; + auto const& tempPath = tempNativePath_; // Opening for writing follows a link planted at the temp path or above it, out of the target directory. if (const auto refused = refuseLinkedPath(tempPath); !refused) @@ -431,60 +481,75 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) { Log::error( "DownloadOperation: Failed to create parent directories for '{}': {}.", - tempPath.generic_string(), - mkErr.message() + Paths::nativeToUtf8(tempPath), + Utility::errorMessage(mkErr) ); return std::unexpected(Error{.type = ErrorType::CannotCreateDirectory}); } } - if (options_.tryContinue && std::filesystem::exists(tempPath)) + std::error_code existsError{}; + if (options_.tryContinue && std::filesystem::exists(tempPath, existsError)) { - // The size on disk, tellp right after opening in append mode reports 0 on Windows. std::error_code sizeError{}; const auto existingSize = std::filesystem::file_size(tempPath, sizeError); if (sizeError) { - Log::error("DownloadOperation: Cannot read the size of '{}': {}", tempPath.generic_string(), sizeError.message()); + Log::error( + "DownloadOperation: Cannot read the size of '{}': {}", + Paths::nativeToUtf8(tempPath), + Utility::errorMessage(sizeError) + ); return std::unexpected(Error{.type = ErrorType::FileStatFailed}); } // A leftover of another program or of an older version of the source must not be continued. - bool belongsToSource = existingSize > 0 && existingSize <= options_.entry->size; - if (belongsToSource) + const auto belongsToSource = [&]() -> std::expected { - const auto matches = ResumeCheck::tailsMatchInStrand(stream, tempPath, existingSize); - if (!matches) - return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = matches.error()}); - belongsToSource = *matches; - } + if (existingSize == 0 || existingSize > options_.entry->size) + return false; + return ResumeCheck::tailsMatchInStrand(stream, tempPath, existingSize) + .transform_error( + [](SecureShell::SftpError error) + { + return Error{.type = ErrorType::SftpError, .sftpError = std::move(error)}; + } + ); + }(); + if (!belongsToSource) + return std::unexpected(belongsToSource.error()); - if (belongsToSource && existingSize == options_.entry->size) + if (*belongsToSource && existingSize == options_.entry->size) { Log::info( "DownloadOperation: File '{}' already complete, will be renamed in finalize() step.", - tempPath.generic_string() + Paths::nativeToUtf8(tempPath) ); + bytesInFile_ = existingSize; transferComplete_ = true; return {}; } - else if (!belongsToSource) + else if (!*belongsToSource) { Log::info( "DownloadOperation: File '{}' is not a part of this source, discarding and starting over.", - tempPath.generic_string() + Paths::nativeToUtf8(tempPath) ); if (auto rewound = stream.seekInStrand(0); !rewound) return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = rewound.error()}); + if (const auto restricted = restrictTempFile(); !restricted) + return restricted; localFile_.open(tempPath, std::ios::binary | std::ios::trunc); } else { - Log::info("DownloadOperation: File '{}' is incomplete, continuing download.", tempPath.generic_string()); + Log::info("DownloadOperation: File '{}' is incomplete, continuing download.", Paths::nativeToUtf8(tempPath)); + if (const auto restricted = restrictTempFile(); !restricted) + return restricted; localFile_.open(tempPath, std::ios::binary | std::ios::app); if (!localFile_.is_open()) { - Log::error("DownloadOperation: Failed to open file for appending: {}", tempPath.generic_string()); + Log::error("DownloadOperation: Failed to open file for appending: {}", Paths::nativeToUtf8(tempPath)); return std::unexpected(Error{.type = ErrorType::OpenFailure}); } resumeOffset_ = existingSize; @@ -496,22 +561,38 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) } else { + if (const auto restricted = restrictTempFile(); !restricted) + return restricted; localFile_.open(tempPath, std::ios::binary | std::ios::trunc); } if (!localFile_.is_open()) { - Log::error("DownloadOperation: Failed to open file: {}", tempPath.generic_string()); + Log::error("DownloadOperation: Failed to open file: {}", Paths::nativeToUtf8(tempPath)); return std::unexpected(Error{.type = ErrorType::OpenFailure}); } return {}; } +std::expected DownloadOperation::restrictTempFile() const +{ + if (const auto restricted = Utility::restrictToOwner(tempNativePath_); !restricted) + { + Log::error( + "DownloadOperation: Cannot create '{}': {}", + Paths::nativeToUtf8(tempNativePath_), + Utility::errorMessage(restricted.error()) + ); + return std::unexpected(Error{.type = ErrorType::OpenFailure}); + } + return {}; +} + std::expected -DownloadOperation::readSymlinkInStrand(Paths::RemotePath const& remoteFullPath) +DownloadOperation::readSymlinkInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& remoteFullPath) { - auto deepLinkResult = sftp_->readLinkDeepInStrand(remoteFullPath); + auto deepLinkResult = sftp.readLinkDeepInStrand(remoteFullPath); if (!deepLinkResult.has_value()) { Log::error( @@ -538,8 +619,7 @@ std::expected DownloadOperation::prepare() return enterErrorState({.type = ErrorType::InvalidPath}); } - // Initial check. Check again later before rename. - if (std::filesystem::exists(localNativePath_)) + if (std::error_code existsError{}; std::filesystem::exists(localNativePath_, existsError)) { if (!options_.mayOverwrite) { @@ -550,10 +630,16 @@ std::expected DownloadOperation::prepare() } } - auto fut = sftp_->performPromise( - [this]() + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( + [this]() -> std::expected { - return prepareInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return prepareInStrand(*sftp); } ); if (fut.wait_for(options_.futureTimeout) != std::future_status::ready) @@ -567,7 +653,7 @@ std::expected DownloadOperation::prepare() return {}; } -std::expected DownloadOperation::prepareInStrand() +std::expected DownloadOperation::prepareInStrand(SecureShell::SftpSession& sftp) { if (remotePath_.empty() || remotePath_.isRoot()) { @@ -577,7 +663,7 @@ std::expected DownloadOperation::prepareInStrand if (!options_.entry) { - const auto fileInfo = sftp_->lstatInStrand(remotePath_); + const auto fileInfo = sftp.lstatInStrand(remotePath_); if (!fileInfo.has_value()) { Log::error("DownloadOperation: Failed to stat file."); @@ -590,9 +676,8 @@ std::expected DownloadOperation::prepareInStrand if (options_.entry->isSymlink() && options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) return {}; - const auto streamOpenResult = sftp_->openFileInStrand( - remotePath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown - ); + const auto streamOpenResult = + sftp.openFileInStrand(remotePath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown); if (!streamOpenResult.has_value()) { Log::error("DownloadOperation: Failed to open remote sftp file: {}.", streamOpenResult.error().message); @@ -638,7 +723,7 @@ std::expected DownloadOperation::prepareInStrand return std::unexpected(openResult.error()); } - buffer_ = sftp_->bufferProvider().leaseForTransfer(options_.entry->size, stream->readLengthLimit()); + buffer_ = sftp.bufferProvider().leaseForTransfer(options_.entry->size, stream->readLengthLimit()); if (buffer_.empty()) { Log::error("DownloadOperation: No transfer buffer available from pool."); @@ -653,7 +738,8 @@ std::expected DownloadOperation::prepareInStrand ); } - if (options_.reserveSpace && options_.entry->size != 0) + // Appending to a partial file or finishing a complete one reserves nothing. + if (options_.reserveSpace && options_.entry->size != 0 && resumeOffset_ == 0 && !transferComplete_) { const auto pos = localFile_.tellp(); localFile_.seekp(options_.entry->size - 1); @@ -697,10 +783,17 @@ void DownloadOperation::cleanup() localFile_.close(); - auto tempPath = localNativePath_; - tempPath += options_.tempFileSuffix; - if (options_.doCleanup && std::filesystem::exists(tempPath)) - std::filesystem::remove(tempPath); + if (options_.doCleanup && !tempNativePath_.empty()) + { + std::error_code removeError{}; + std::filesystem::remove(tempNativePath_, removeError); + if (removeError) + Log::warn( + "DownloadOperation: Cannot remove '{}': {}", + Paths::nativeToUtf8(tempNativePath_), + Utility::errorMessage(removeError) + ); + } } void DownloadOperation::pause(bool doPause) @@ -711,7 +804,10 @@ void DownloadOperation::pause(bool doPause) std::expected DownloadOperation::finalize() { - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return finalizeInStrand(); @@ -735,31 +831,22 @@ std::expected DownloadOperation::finalizeInStran localFile_.close(); - auto tempPath = localNativePath_; - tempPath += options_.tempFileSuffix; + auto const& tempPath = tempNativePath_; - // A short file must never replace the destination. + // A short file must never replace the destination. A reserved file has its full size from the start, so the + // bytes written count. std::error_code sizeError{}; - const auto writtenSize = std::filesystem::file_size(tempPath, sizeError); - if (sizeError || writtenSize != options_.entry->size) + const auto sizeOnDisk = std::filesystem::file_size(tempPath, sizeError); + if (sizeError || sizeOnDisk != options_.entry->size || bytesInFile_ != options_.entry->size) { Log::error( - "DownloadOperation: '{}' holds {} bytes instead of {}.", - tempPath.generic_string(), - sizeError ? 0u : writtenSize, + "DownloadOperation: '{}' received {} bytes and holds {} instead of {}.", + Paths::nativeToUtf8(tempPath), + bytesInFile_, + sizeError ? 0u : sizeOnDisk, options_.entry->size ); - return std::unexpected( - Error{ - .type = ErrorType::SizeMismatch, - .extraInfo = fmt::format( - "Received {} of {} bytes for {}", - sizeError ? 0u : writtenSize, - options_.entry->size, - options_.remotePath.value() - ), - } - ); + return std::unexpected(sizeMismatch()); } // Marked before the rename, so the final file never exists without the mark. A filesystem without streams or @@ -767,85 +854,68 @@ std::expected DownloadOperation::finalizeInStran if (options_.markAsDownloaded) { if (const auto marked = Utility::markAsDownloaded(tempPath); marked) - Log::warn("DownloadOperation: Cannot mark '{}' as downloaded: {}", Paths::nativeToUtf8(tempPath), marked.message()); + Log::warn( + "DownloadOperation: Cannot mark '{}' as downloaded: {}", + Paths::nativeToUtf8(tempPath), + Utility::errorMessage(marked) + ); } - // Without permission to overwrite, the rename itself refuses an existing file: no check-then-rename race. - std::error_code ec{}; - if (options_.mayOverwrite) - std::filesystem::rename(tempPath, localNativePath_, ec); - else if (auto renamed = Utility::renameNoReplace(tempPath, localNativePath_); !renamed.has_value()) - ec = renamed.error(); - if (ec == std::errc::file_exists) + // Times and permissions go on the temp file, so nothing can fail once the destination is replaced. + applyModificationTime(tempPath); + const auto permissions = finalPermissionsInStrand(); + if (!permissions) + return std::unexpected(permissions.error()); + std::error_code permissionsError{}; + std::filesystem::permissions(tempPath, *permissions, permissionsError); + if (permissionsError) { - Log::error( - "DownloadOperation: File '{}' already exists and may not be overwritten.", options_.localPath.toUtf8() - ); - return std::unexpected(Error{.type = ErrorType::FileExists}); + Log::error("DownloadOperation: Failed to set permissions: {}", Utility::errorMessage(permissionsError)); + return std::unexpected(Error{.type = ErrorType::CannotSetFilePermissions}); } - if (ec) + + return ComparedStateCheck::moveLocalIntoPlace( + tempPath, localNativePath_, options_.mayOverwrite, options_.comparedDestination + ); +} + +std::expected DownloadOperation::finalPermissionsInStrand() +{ + if (!options_.inheritPermissions) { - Log::error("DownloadOperation: Failed to rename file: {}", ec.message()); - return std::unexpected(Error{.type = ErrorType::RenameFailure}); + return options_.filePermissions.and_then(Utility::sanitizePermissions).value_or(Utility::newFilePermissions()); } - if (options_.inheritPermissions) + auto stream = fileStream_.lock(); + if (!stream) { - auto stream = fileStream_.lock(); - if (!stream) - { - Log::error("DownloadOperation: File stream expired."); - return std::unexpected(Error{.type = ErrorType::FileStreamExpired}); - } - - const auto fileInfo = stream->statInStrand(); - if (!fileInfo.has_value()) - { - Log::error("DownloadOperation: Failed to stat file."); - return std::unexpected(Error{.type = ErrorType::FileStatFailed, .sftpError = fileInfo.error()}); - } - - // Special bits never travel and the umask applies, like for a file created locally. - std::error_code permissionsError{}; - if (const auto inherited = Utility::inheritablePermissions(fileInfo->permissions); inherited) - std::filesystem::permissions(localNativePath_, *inherited, permissionsError); - if (permissionsError) - { - Log::error("DownloadOperation: Failed to set permissions: {}", permissionsError.message()); - return std::unexpected(Error{.type = ErrorType::CannotSetFilePermissions}); - } + Log::error("DownloadOperation: File stream expired."); + return std::unexpected(Error{.type = ErrorType::FileStreamExpired}); } - else if (options_.filePermissions) + const auto fileInfo = stream->statInStrand(); + if (!fileInfo.has_value()) { - std::error_code permissionsError{}; - if (const auto configured = Utility::sanitizePermissions(*options_.filePermissions); configured) - std::filesystem::permissions(localNativePath_, *configured, permissionsError); - if (permissionsError) - { - Log::error("DownloadOperation: Failed to set permissions: {}", permissionsError.message()); - return std::unexpected(Error{.type = ErrorType::CannotSetFilePermissions}); - } + Log::error("DownloadOperation: Failed to stat file."); + return std::unexpected(Error{.type = ErrorType::FileStatFailed, .sftpError = fileInfo.error()}); } - /* else keep default */ - - // Preserve the remote file's mtime on the downloaded local file so - // subsequent syncs don't see the file as modified (best-effort; a failure - // is only a warning). Mirrors UploadOperation's post-transfer setstat. - if (options_.entry) - { - using namespace std::chrono; - const auto sysTime = system_clock::time_point{duration_cast( - seconds{options_.entry->mtime} + nanoseconds{options_.entry->mtimeNsec} - )}; - const auto fileTime = std::chrono::file_clock::from_sys(sysTime); - std::error_code mtimeError{}; - std::filesystem::last_write_time(localNativePath_, fileTime, mtimeError); - if (mtimeError) - { - Log::warn( - "DownloadOperation: Failed to set mtime on '{}': {}.", options_.localPath.toUtf8(), mtimeError.message() - ); - } + // Special bits never travel and the umask applies, like for a file created locally. + return Utility::inheritablePermissions(fileInfo->permissions).value_or(Utility::newFilePermissions()); +} + +void DownloadOperation::applyModificationTime(std::filesystem::path const& path) const +{ + using namespace std::chrono; + const auto sysTime = system_clock::time_point{ + duration_cast(seconds{options_.entry->mtime} + nanoseconds{options_.entry->mtimeNsec}) + }; + std::error_code mtimeError{}; + std::filesystem::last_write_time(path, std::chrono::file_clock::from_sys(sysTime), mtimeError); + if (mtimeError) + { + Log::warn( + "DownloadOperation: Failed to set mtime on '{}': {}.", + Paths::nativeToUtf8(path), + Utility::errorMessage(mtimeError) + ); } - return {}; } \ No newline at end of file diff --git a/backend/source/backend/sftp/local_delete_operation.cpp b/backend/source/backend/sftp/local_delete_operation.cpp index 7362f48a..80b098ea 100644 --- a/backend/source/backend/sftp/local_delete_operation.cpp +++ b/backend/source/backend/sftp/local_delete_operation.cpp @@ -1,12 +1,23 @@ +#include #include +#include +#include +#include #include +#include #include #include +std::optional LocalDeleteOperation::commonParentOf(std::span targets) +{ + return Paths::commonParentOf(targets); +} + LocalDeleteOperation::LocalDeleteOperation(LocalDeleteOperationOptions options) : Operation{} , options_{std::move(options)} + , counting_{options_.recursive} {} std::expected LocalDeleteOperation::work() @@ -19,48 +30,25 @@ std::expected Loc case Preparing: case Prepared: { - for (auto const& target : options_.targets) - { - if (target.empty() || target.isRoot()) - { - Log::error("LocalDeleteOperation: Refusing to delete '{}'.", target.toUtf8()); - return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = target.toUtf8()}); - } - } state_ = Running; [[fallthrough]]; } case Running: { - const auto total = options_.targets.size(); - if (deletedThusFar_ < total) + const auto deadline = std::chrono::steady_clock::now() + options_.stepBudget; + const bool finished = options_.recursive ? workRecursive(deadline) : workFlat(deadline); + reportProgress(finished); + if (!finished) + return WorkStatus::MoreWork; + walk_.reset(); + if (failureCount_ > 0 && removedEntries_ == 0 && completedTargets_ == 0) { - auto const& target = options_.targets[total - deletedThusFar_ - 1]; - std::error_code error{}; - if (Utility::isLinkOrJunction(target.native(), error) && !error) - Utility::removeAllNoFollow(target.native(), error); - else if (!error) - std::filesystem::remove(target.native(), error); - // One item that cannot be deleted does not keep the others, its directories then stay as well. - if (error) - { - Log::error("LocalDeleteOperation: Failed to delete '{}': {}", target.toUtf8(), error.message()); - failedEntries_.emplace_back( - target.toUtf8(), - Error{ - .type = ErrorType::DeleteFailed, - .extraInfo = fmt::format("{}: {}", target.toUtf8(), error.message()) - } - ); - } - ++deletedThusFar_; - options_.itemsRemovedProgress(target.toUtf8(), deletedThusFar_, total); - if (deletedThusFar_ < total) - return WorkStatus::MoreWork; + auto const& [path, error] = failedEntries_.front(); + return enterErrorState(Error{ + .type = ErrorType::DeleteFailed, + .extraInfo = fmt::format("{}: {}", path, error.extraInfo.value_or(std::string{})), + }); } - // Nothing deleted at all is a failure, some items left behind a partial success. - if (total > 0 && failedEntries_.size() == total) - return enterErrorState(failedEntries_.back().second); state_ = Completed; return WorkStatus::Complete; } @@ -80,8 +68,134 @@ std::expected Loc return enterErrorState({.type = ErrorType::UnknownWorkState}); } +bool LocalDeleteOperation::workFlat(std::chrono::steady_clock::time_point deadline) +{ + auto const& targets = options_.targets; + do + { + if (targetIndex_ >= targets.size()) + return true; + const auto index = targets.size() - targetIndex_ - 1; + auto const& target = targets[index]; + const auto compared = index < options_.comparedStates.size() + ? std::optional{options_.comparedStates[index]} + : std::nullopt; + if (auto unchanged = ComparedStateCheck::requireUnchangedLocal(target.native(), compared); !unchanged) + { + recordFailure(target.toUtf8(), std::move(unchanged).error()); + } + else + { + std::error_code error{}; + if (Utility::removeNoFollow(target.native(), error)) + ++removedEntries_; + if (error) + recordFailure(target.native(), error); + else + ++completedTargets_; + } + lastItem_ = target.toUtf8(); + ++targetIndex_; + } while (std::chrono::steady_clock::now() < deadline); + return targetIndex_ >= targets.size(); +} + +bool LocalDeleteOperation::workRecursive(std::chrono::steady_clock::time_point deadline) +{ + auto const& targets = options_.targets; + while (true) + { + if (!walk_) + { + if (targetIndex_ >= targets.size()) + { + if (!counting_) + return true; + counting_ = false; + targetIndex_ = 0; + options_.scanProgress(countedBytes_, countedEntries_); + continue; + } + walk_.emplace( + targets[targets.size() - targetIndex_ - 1].native(), + counting_ ? Utility::NoFollowTreeWalk::Mode::Count : Utility::NoFollowTreeWalk::Mode::Remove + ); + } + const bool walked = walk_->step(deadline); + if (!walked) + return false; + finishWalk(); + if (std::chrono::steady_clock::now() >= deadline) + return false; + } +} + +void LocalDeleteOperation::finishWalk() +{ + if (counting_) + { + countedEntries_ += walk_->entries(); + countedBytes_ += walk_->bytes(); + } + else + { + removedEntries_ += walk_->entries(); + lastItem_ = Paths::nativeToUtf8(walk_->lastPath()); + for (auto const& failure : walk_->failures()) + recordFailure(failure.path, failure.error); + if (walk_->failures().empty()) + ++completedTargets_; + } + walk_.reset(); + ++targetIndex_; +} + +void LocalDeleteOperation::recordFailure(std::filesystem::path const& path, std::error_code const& error) +{ + recordFailure(Paths::nativeToUtf8(path), Error{.type = ErrorType::DeleteFailed, .extraInfo = Utility::errorMessage(error)}); +} + +void LocalDeleteOperation::recordFailure(std::string const& path, Error error) +{ + ++failureCount_; + if (failedEntries_.size() >= maximumListedFailures) + return; + Log::error("LocalDeleteOperation: Failed to delete '{}': {}", path, error.toString()); + failedEntries_.emplace_back(path, std::move(error)); + if (failedEntries_.size() == maximumListedFailures) + Log::error("LocalDeleteOperation: Further failures are counted, not listed."); +} + +std::uint64_t LocalDeleteOperation::handledEntries() const +{ + if (!options_.recursive) + return targetIndex_; + const auto running = walk_ && !counting_ ? walk_->entries() + walk_->failures().size() : 0; + return removedEntries_ + failureCount_ + running; +} + +void LocalDeleteOperation::reportProgress(bool final) +{ + const auto now = std::chrono::steady_clock::now(); + if (!final && now - lastProgress_ < options_.progressInterval) + return; + lastProgress_ = now; + if (counting_) + { + const auto entries = countedEntries_ + (walk_ ? walk_->entries() : 0); + const auto bytes = countedBytes_ + (walk_ ? walk_->bytes() : 0); + return options_.scanProgress(bytes, entries); + } + const auto current = handledEntries(); + const auto total = options_.recursive ? std::max(countedEntries_, current) : options_.targets.size(); + if (walk_) + lastItem_ = Paths::nativeToUtf8(walk_->lastPath()); + options_.itemsRemovedProgress(lastItem_, current, total); +} + std::expected LocalDeleteOperation::cancel(bool adoptCancelState) { + walk_.reset(); if (adoptCancelState) state_ = OperationState::Canceled; return {}; diff --git a/backend/source/backend/sftp/local_scan_operation.cpp b/backend/source/backend/sftp/local_scan_operation.cpp index 8d74bdf8..461a0ce2 100644 --- a/backend/source/backend/sftp/local_scan_operation.cpp +++ b/backend/source/backend/sftp/local_scan_operation.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -6,7 +8,6 @@ #include -#include #include #include #include @@ -86,107 +87,118 @@ LocalScanOperation::scanner(Paths::RelativePath const& relative) { if (iterErrc) return listingFailed(path, relative, iterErrc); - try + auto const& entry = *iter; + const auto failed = [this, &entry](std::error_code const& error) { - auto const& entry = *iter; - const auto status = entry.symlink_status(); - SharedData::DirectoryEntry::FileType type; + Log::error( + "LocalScanOperation: Cannot inspect '{}': {}", + Paths::nativeToUtf8(entry.path()), + Utility::errorMessage(error) + ); + return enterErrorState>( + {.type = ErrorType::FilesystemError, .extraInfo = Utility::errorMessage(error)} + ); + }; + std::error_code statusError{}; + const auto status = entry.symlink_status(statusError); + if (statusError) + return failed(statusError); + SharedData::DirectoryEntry::FileType type; #ifdef _WIN32 - // A junction is a reparse point that std::filesystem reports as a directory. - std::error_code linkError{}; - const bool isSymlink = std::filesystem::is_symlink(status) || Utility::isLinkOrJunction(entry.path(), linkError); + // A junction is a reparse point that std::filesystem reports as a directory. + std::error_code linkError{}; + const bool isSymlink = + std::filesystem::is_symlink(status) || Utility::isLinkOrJunction(entry.path(), linkError); #else - const bool isSymlink = std::filesystem::is_symlink(status); + const bool isSymlink = std::filesystem::is_symlink(status); #endif - if (isSymlink) - { + if (isSymlink) + { #ifdef _WIN32 - // Windows symlinks and junctions are not transferred. A sync tree keeps them as excluded - // placeholders, so the name still counts as present and the remote side is left alone. - if (!buildTree_) - continue; - type = SharedData::DirectoryEntry::FileType::Unknown; -#else - type = SharedData::DirectoryEntry::FileType::Symlink; -#endif - } - else if (std::filesystem::is_directory(status)) - type = SharedData::DirectoryEntry::FileType::Directory; - else if (std::filesystem::is_regular_file(status)) - type = SharedData::DirectoryEntry::FileType::Regular; - else + // Windows symlinks and junctions are not transferred. A sync tree keeps them as excluded + // placeholders, so the name still counts as present and the remote side is left alone. + if (!buildTree_) { - // Sockets, devices and FIFOs are not uploadable, a sync tree keeps them as excluded placeholders. - if (!buildTree_) - continue; - type = SharedData::fileTypeFromStdFilesystemType(status.type()); - if (type == SharedData::DirectoryEntry::FileType::Regular || - type == SharedData::DirectoryEntry::FileType::Directory) - type = SharedData::DirectoryEntry::FileType::Unknown; + skippedLinks_.push_back(Paths::nativeToUtf8(entry.path())); + continue; } + type = SharedData::DirectoryEntry::FileType::Unknown; +#else + type = SharedData::DirectoryEntry::FileType::Symlink; +#endif + } + else if (std::filesystem::is_directory(status)) + type = SharedData::DirectoryEntry::FileType::Directory; + else if (std::filesystem::is_regular_file(status)) + type = SharedData::DirectoryEntry::FileType::Regular; + else + { + // Sockets, devices and FIFOs are not uploadable, a sync tree keeps them as excluded placeholders. + if (!buildTree_) + continue; + type = SharedData::fileTypeFromStdFilesystemType(status.type()); + if (type == SharedData::DirectoryEntry::FileType::Regular || + type == SharedData::DirectoryEntry::FileType::Directory) + type = SharedData::DirectoryEntry::FileType::Unknown; + } - std::uint64_t mtimeSecs = 0; - if (isSymlink) - { + std::uint64_t mtimeSecs = 0; + if (isSymlink) + { #ifndef _WIN32 - // For symlinks we want the link's own mtime (lstat), not the target's. - struct ::stat st{}; - if (::lstat(entry.path().c_str(), &st) == 0) - mtimeSecs = static_cast(st.st_mtime); + // For symlinks we want the link's own mtime (lstat), not the target's. + struct ::stat st{}; + if (::lstat(entry.path().c_str(), &st) == 0) + mtimeSecs = static_cast(st.st_mtime); #endif - } - else - { - std::error_code errc{}; - const auto ftime = entry.last_write_time(errc); - if (!errc) - { - const auto sctime = std::chrono::file_clock::to_sys(ftime); - mtimeSecs = static_cast( - std::chrono::duration_cast(sctime.time_since_epoch()).count() - ); - } - } - - std::optional linkTarget{}; - if (isSymlink) - { - std::error_code errc{}; - const auto target = std::filesystem::read_symlink(entry.path(), errc); - if (!errc) - linkTarget = Paths::nativeToUtf8(target); - } - - const auto name = Paths::nativeToUtf8(entry.path().filename()); - if (ignoreHidden_ && !name.empty() && name.front() == '.') - continue; - if (respectIgnoreFiles_ && !ignoreMatcher_.empty()) - { - // A name that is no segment cannot match a rule, the walker rejects it. - const auto segment = Paths::PathSegment::make(name); - if (segment && - ignoreMatcher_.isIgnored(relative / *segment, type == SharedData::DirectoryEntry::FileType::Directory)) - continue; - } + } + else + { + std::error_code errc{}; + const auto ftime = entry.last_write_time(errc); + if (!errc) + mtimeSecs = ComparedStateCheck::secondsSinceEpoch(ftime); + } - entries.push_back( - SharedData::DirectoryEntry{ - .name = name, - .type = type, - .size = entry.is_regular_file() ? entry.file_size() : 0, - .permissions = Utility::localPermissions(status), - .mtime = mtimeSecs, - .linkTarget = std::move(linkTarget), - } - ); + std::optional linkTarget{}; + if (isSymlink) + { + std::error_code errc{}; + const auto target = std::filesystem::read_symlink(entry.path(), errc); + if (!errc) + linkTarget = Paths::nativeToUtf8(target); } - catch (const std::filesystem::filesystem_error& e) + + auto converted = Paths::convertNativeToUtf8(entry.path().filename()); + auto const& name = converted.utf8; + if (ignoreHidden_ && !name.empty() && name.front() == '.') + continue; + if (respectIgnoreFiles_ && !ignoreMatcher_.empty()) { - Log::error("LocalScanOperation: Filesystem error while scanning '{}': {}", path.generic_string(), e.what()); - return enterErrorState>( - {.type = ErrorType::FilesystemError, .extraInfo = e.what()} - ); + // A name that is no segment cannot match a rule, the walker rejects it. + const auto segment = Paths::PathSegment::make(name); + if (segment && + ignoreMatcher_.isIgnored(relative / *segment, type == SharedData::DirectoryEntry::FileType::Directory)) + continue; } + + // The size of what a followed link points to, like a regular file's own. + std::error_code sizeError{}; + const auto size = entry.is_regular_file(sizeError) ? entry.file_size(sizeError) : 0u; + if (sizeError && !isSymlink) + return failed(sizeError); + + entries.push_back( + SharedData::DirectoryEntry{ + .name = std::move(converted.utf8), + .nameIsValidUtf8 = converted.lossless, + .type = type, + .size = sizeError ? 0u : size, + .permissions = Utility::localPermissions(status), + .mtime = mtimeSecs, + .linkTarget = std::move(linkTarget), + } + ); } return {std::move(entries)}; } @@ -203,26 +215,45 @@ LocalScanOperation::listingFailed( errorCode == std::errc::permission_denied || errorCode == std::errc::no_such_file_or_directory; if (belowRoot && deniedOrVanished) { - Log::warn("LocalScanOperation: Cannot list '{}': {}", path.generic_string(), errorCode.message()); + Log::warn( + "LocalScanOperation: Cannot list '{}': {}", Paths::nativeToUtf8(path), Utility::errorMessage(errorCode) + ); if (!buildTree_) { - // Left out of the entries, the consumer reports it instead of transferring an empty directory. + // The consumer reports it instead of transferring an empty directory. unreadableDirectories_.push_back(Paths::nativeToUtf8(path)); + unreadableRelatives_.insert(relative.value()); return std::vector{}; } return std::unexpected( Error{ .type = ErrorType::DirectoryUnreadable, - .extraInfo = fmt::format("{}: {}", path.generic_string(), errorCode.message()), + .extraInfo = fmt::format("{}: {}", Paths::nativeToUtf8(path), Utility::errorMessage(errorCode)), } ); } - Log::error("LocalScanOperation: Failed to list '{}': {}", path.generic_string(), errorCode.message()); + Log::error( + "LocalScanOperation: Failed to list '{}': {}", Paths::nativeToUtf8(path), Utility::errorMessage(errorCode) + ); return enterErrorState>( - {.type = ErrorType::FilesystemError, .extraInfo = errorCode.message()} + {.type = ErrorType::FilesystemError, .extraInfo = Utility::errorMessage(errorCode)} ); } +void LocalScanOperation::markUnreadable(std::vector& entries) const +{ + if (unreadableRelatives_.empty()) + return; + for (auto& entry : entries) + { + if (!entry.isDirectory()) + continue; + if (const auto relative = SharedData::relativePath(entries, entry); + relative && unreadableRelatives_.contains(relative->value())) + entry.type = SharedData::FileType::Unknown; + } +} + std::uint64_t LocalScanOperation::totalBytes() const { return walker_->totalBytes(); diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index baac44e8..17d4086a 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1,7 +1,12 @@ #include +#include +#include +#include +#include #include #include #include +#include #include #include #include @@ -18,8 +23,12 @@ #include #include +#include +#include #include #include +#include +#include using namespace std::chrono_literals; @@ -36,6 +45,19 @@ namespace return path; } + /** + * @brief The closest directory holding the path @p member of every file of a prescanned bulk. + */ + template + PathT commonParentOf(std::vector const& files, PathT FileT::* member) + { + std::vector paths; + paths.reserve(files.size()); + std::ranges::transform(files, std::back_inserter(paths), member); + auto parent = Paths::commonParentOf(std::span{paths}); + return parent ? *std::move(parent) : PathT{}; + } + OperationQueue::OperationCompleted makeCompletedOperation( OperationQueue::CompletionReason reason, Ids::OperationId operationId, @@ -43,129 +65,82 @@ namespace std::optional error = std::nullopt ) { - return operation.visit( + OperationQueue::OperationCompleted completed{ + .reason = reason, + .operationId = std::move(operationId), + .completionTime = std::chrono::system_clock::now(), + .error = std::move(error), + }; + const auto transferPaths = [&completed](auto const& transfer) + { + completed.localPath = unlessEmpty(transfer.localPath()); + completed.remotePath = unlessEmpty(transfer.remotePath()); + }; + const auto bulkFailures = [&completed](BulkTransferOperation const& bulk) + { + completed.failedEntries = bulk.getFailed(); + completed.indexedFailures = bulk.indexedFailures(); + }; + operation.visit( Utility::overloaded( - [reason, operationId, error](DownloadOperation const& op) + [&](DownloadOperation const& download) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .localPath = unlessEmpty(op.localPath()), - .remotePath = unlessEmpty(op.remotePath()), - .error = error, - }; - }, - [reason, operationId, error](ScanOperation const& op) - { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .remotePath = unlessEmpty(op.remotePath()), - .error = error, - }; - }, - [reason, operationId, error](LocalScanOperation const& op) - { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .localPath = unlessEmpty(op.localPath()), - .error = error, - }; - }, - [reason, operationId, error](BulkDownloadOperation const& op) - { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - .failedEntries = op.getFailed(), - }; + transferPaths(download); }, - [reason, operationId, error](UploadOperation const& op) + [&](UploadOperation const& upload) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .localPath = unlessEmpty(op.localPath()), - .remotePath = unlessEmpty(op.remotePath()), - .error = error, - }; + transferPaths(upload); }, - [reason, operationId, error](BulkUploadOperation const& op) + [&](ScanOperation const& scan) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - .failedEntries = op.getFailed(), - }; + completed.remotePath = unlessEmpty(scan.remotePath()); }, - [reason, operationId, error](DeleteOperation const& op) + [&](LocalScanOperation const& scan) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .remotePath = unlessEmpty(op.remotePath()), - .error = error, - .failedEntries = op.getFailed(), - }; + completed.localPath = unlessEmpty(scan.localPath()); }, - [reason, operationId, error](LocalDeleteOperation const& op) + [&](BulkDownloadOperation const& bulk) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - .failedEntries = op.getFailed(), - }; + bulkFailures(bulk); }, - [reason, operationId, error](RenameOperation const& op) + [&](BulkUploadOperation const& bulk) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .remotePath = unlessEmpty(op.destinationPath()), - .error = error, - }; + bulkFailures(bulk); }, - [reason, operationId, error](ArchiveDownloadOperation const&) + [&](DeleteOperation const& deletion) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - }; + completed.remotePath = unlessEmpty(deletion.remotePath()); + completed.failedEntries = deletion.getFailed(); }, - [reason, operationId, error](ArchiveUploadOperation const&) + [](LocalDeleteOperation const&) {}, + [&](RenameOperation const& rename) { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - .error = error, - }; + completed.remotePath = unlessEmpty(rename.destinationPath()); }, - [reason, operationId](std::nullopt_t) - { - return OperationQueue::OperationCompleted{ - .reason = reason, - .operationId = operationId, - .completionTime = std::chrono::system_clock::now(), - }; - } + [](ArchiveDownloadOperation const&) {}, + [](ArchiveUploadOperation const&) {}, + [](std::nullopt_t) {} ) ); + return completed; + } + + /** + * @brief The resume backup of the directory transfer @p source to @p destination: the directory alone, a resume + * scans it again. + */ + template + RequestT directoryResumeRequest( + SourceT const& source, + DestinationT const& destination, + bool allowOverwrite, + bool insertRefresh, + SharedData::OperationMode mode + ) + { + RequestT request{.allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, .mode = mode}; + request.entries.push_back({.source = source, .destination = destination, .isDirectory = true}); + return request; } DownloadOperation::DownloadOperationOptions @@ -249,7 +224,8 @@ auto OperationQueue::makeScanProgressCallback(std::string_view eventName, Ids::O auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::ScanProgress{ .operationId = operationId, @@ -263,6 +239,29 @@ auto OperationQueue::makeScanProgressCallback(std::string_view eventName, Ids::O }; } +auto OperationQueue::makeDeleteProgressCallback(Ids::OperationId operationId) + -> std::function +{ + return [weak = weak_from_this(), operationId, name = rpcName("onDeleteProgress")]( + std::string const& path, std::uint64_t itemsDeleted, std::uint64_t totalItems + ) + { + auto self = weak.lock(); + if (!self) + return; + RpcHelper::callRemoteSafely( + *self->hub_, + name, + SharedData::BulkDeleteProgress{ + .operationId = operationId, + .currentFile = path, + .filesDeleted = itemsDeleted, + .totalFiles = totalItems, + } + ); + }; +} + auto OperationQueue::makeBulkProgressCallback(std::string_view eventName, Ids::OperationId operationId) -> BulkProgressCallback { @@ -280,7 +279,7 @@ auto OperationQueue::makeBulkProgressCallback(std::string_view eventName, Ids::O auto self = weak.lock(); if (!self) return; - self->hub_->callRemote(name, progress); + RpcHelper::callRemoteSafely(*self->hub_, name, progress); } ); }; @@ -299,7 +298,7 @@ void OperationQueue::cancelAll() self->priorityOperations_.clear(); auto waiting = std::exchange(self->syncScanCallbacks_, {}); for (auto& [id, callback] : waiting) - callback(std::unexpected(std::string{"The scan was cancelled."})); + callback(std::unexpected(Utility::localizedMessage("backend.sync.scanCancelled"))); Log::info("All operations in the queue have been canceled."); } ); @@ -325,7 +324,7 @@ void OperationQueue::cancel(Ids::OperationId id) const auto erasedFromRegular = std::erase_if(self->operations_, cancelPredicate); if (erasedFromRegular == 0) std::erase_if(self->priorityOperations_, cancelPredicate); - self->failSyncScan(id, "The scan was cancelled."); + self->failSyncScan(id, Utility::localizedMessage("backend.sync.scanCancelled")); } ); } @@ -345,7 +344,8 @@ void OperationQueue::moveOperation(Ids::OperationId operationId, std::size_t new // on this to apply or discard the user's intended reorder. auto emitResolution = [&self, &operationId](std::size_t resolvedIndex, bool applied) { - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, self->rpcName("onOperationsReordered"), SharedData::OperationsReordered{ .operationId = operationId, @@ -437,7 +437,7 @@ void OperationQueue::completeOperation(SharedData::OperationCompleted&& operatio if (self->completionObserver_) self->completionObserver_(operationCompleted); - self->hub_->callRemote(self->rpcName("onOperationCompleted"), std::move(operationCompleted)); + RpcHelper::callRemoteSafely(*self->hub_, self->rpcName("onOperationCompleted"), operationCompleted); } ); } @@ -452,45 +452,117 @@ void OperationQueue::dumpBulkResumes(BulkResumeRegistry& registry) if (bulkResumeStash_.empty()) return; Log::info("OperationQueue::dumpBulkResumes: persisting {} bulk-operation backup(s)", bulkResumeStash_.size()); - for (auto& [opId, entry] : bulkResumeStash_) - registry.store(opId, std::move(entry)); + for (auto& [operationId, entry] : bulkResumeStash_) + { + if (auto* operation = findOperation(operationId); operation) + leaveOutFinished(*operation, entry); + registry.store(operationId, std::move(entry)); + } bulkResumeStash_.clear(); } +Operation* OperationQueue::findOperation(Ids::OperationId const& operationId) +{ + for (auto* queue : {&priorityOperations_, &operations_}) + { + const auto found = + std::ranges::find(*queue, operationId, &std::pair>::first); + if (found != queue->end()) + return found->second.get(); + } + return nullptr; +} + +void OperationQueue::leaveOutFinished(Operation const& operation, BulkResumeEntry& entry) +{ + const auto trim = [&entry](auto const& bulk, auto& request) + { + auto remaining = bulk.resumeRequest(request); + if (remaining) + { + request = *std::move(remaining); + entry.finished = bulk.finishedProgress(); + } + else + { + entry.finished = bulk.earlierProgress(); + } + }; + std::visit( + Utility::overloaded{ + [&](SharedData::BulkDownloadRequest& download) + { + if (operation.type() == SharedData::OperationType::BulkDownload) + trim(static_cast(operation), download); + }, + [&](SharedData::BulkUploadRequest& upload) + { + if (operation.type() == SharedData::OperationType::BulkUpload) + trim(static_cast(operation), upload); + }, + [](SharedData::BulkDeleteRequest&) + { + // A delete of an item that is gone already counts as done, the whole request can run again. + }, + }, + entry.request + ); +} + void OperationQueue::adoptBulkResume( SecureShell::SftpSession& sftp, Ids::OperationId const& operationId, BulkResumeEntry entry ) { - Log::info( - "OperationQueue::adoptBulkResume: re-issuing bulk operation '{}' ({} entries)", - operationId.value(), - std::visit( + const auto [entryCount, directoriesWithContent] = std::visit( + Utility::overloaded{ + [](SharedData::BulkDeleteRequest const& request) + { + return std::pair{request.entries.size(), std::size_t{0}}; + }, [](auto const& request) { - return request.entries.size(); + const auto directories = std::ranges::count_if( + request.entries, + [](auto const& transferEntry) + { + return transferEntry.isDirectory && !transferEntry.withoutContent; + } + ); + return std::pair{request.entries.size(), static_cast(directories)}; }, - entry.request - ) + }, + entry.request + ); + // A backup holds the files of one card or a single directory, more would complete the card more than once. + if (directoriesWithContent > 1 || (directoriesWithContent == 1 && entryCount > 1)) + { + Log::error( + "OperationQueue::adoptBulkResume: refusing the backup of '{}', it holds {} directories among {} entries", + operationId.value(), + directoriesWithContent, + entryCount + ); + return; + } + Log::info( + "OperationQueue::adoptBulkResume: re-issuing bulk operation '{}' ({} entries)", operationId.value(), entryCount ); - // A transfer backup holds either the files of one bulk card or a single directory, both resume under the - // stashed id so the surviving card re-attaches. - auto operationIdFor = [&operationId](std::size_t) + const auto operationIdFor = [&operationId](std::size_t) { return operationId; }; - std::visit( Utility::overloaded{ - [&](SharedData::BulkDownloadRequest const& request) + [&](SharedData::BulkDownloadRequest& request) { - addBulkDownloadOperation(sftp, request, operationIdFor, operationId); + addBulkDownloadOperation(sftp, std::move(request), operationIdFor, operationId); }, - [&](SharedData::BulkUploadRequest const& request) + [&](SharedData::BulkUploadRequest& request) { - addBulkUploadOperation(sftp, request, operationIdFor, operationId); + addBulkUploadOperation(sftp, std::move(request), operationIdFor, operationId); }, [&](SharedData::BulkDeleteRequest const& request) { @@ -499,6 +571,12 @@ void OperationQueue::adoptBulkResume( }, entry.request ); + + auto* adopted = findOperation(operationId); + if (adopted && + (adopted->type() == SharedData::OperationType::BulkDownload || + adopted->type() == SharedData::OperationType::BulkUpload)) + static_cast(*adopted).countEarlierProgress(entry.finished); } void OperationQueue::deepPause(bool pause) @@ -590,7 +668,14 @@ bool OperationQueue::workQueue(std::dequetype() == SharedData::OperationType::LocalScan ? "backend.sync.localScanFailed" + : "backend.sync.remoteScanFailed", + item.result.error().toString() + ) + ); completeOperation( makeCompletedOperation(OperationQueue::CompletionReason::Failed, id, *operation, item.result.error()) ); @@ -615,6 +700,39 @@ bool OperationQueue::workQueue(std::dequecancel(true); + return true; + }; + if (std::erase_if(self->priorityOperations_, matches) + std::erase_if(self->operations_, matches) > 0) + { + self->completeOperation( + SharedData::OperationCompleted{ + .reason = SharedData::OperationCompletionReason::Canceled, + .operationId = id, + .completionTime = std::chrono::system_clock::now(), + } + ); + } + } + ); +} + void OperationQueue::failSyncScan(Ids::OperationId const& id, std::string const& reason) { auto callback = syncScanCallbacks_.extract(id.value()); @@ -681,6 +799,50 @@ void OperationQueue::paused(bool pause) ); } +auto OperationQueue::queueFor(SharedData::OperationMode mode) + -> std::deque>>& +{ + return mode == SharedData::OperationMode::PriorityQueued ? priorityOperations_ : operations_; +} + +template +std::size_t OperationQueue::splitBulkRequest( + RequestT request, + Ids::OperationId const& bulkCardId, + QueueDirectoryT&& queueDirectory, + QueueFilesT&& queueFiles +) +{ + // A directory resumes as a unit of its own under its own id, so the card's backup holds the rest. + RequestT unit{ + .allowOverwrite = request.allowOverwrite, + .insertRefresh = request.insertRefresh, + .mode = request.mode, + .localRoot = std::move(request.localRoot), + .remoteRoot = std::move(request.remoteRoot), + }; + std::size_t queued = 0; + for (std::size_t index = 0; index < request.entries.size(); ++index) + { + auto& entry = request.entries[index]; + if (entry.isDirectory && !entry.withoutContent) + { + if (queueDirectory(index, entry, unit)) + ++queued; + continue; + } + unit.entries.push_back(std::move(entry)); + } + // Without any entry the card is still queued, so a resumed card that had nothing left completes. + if (unit.entries.empty() && !request.entries.empty()) + return queued; + + queueFiles(unit); + queued += unit.entries.size(); + bulkResumeStash_.insert_or_assign(bulkCardId, BulkResumeEntry{.request = std::move(unit)}); + return queued; +} + std::expected OperationQueue::addDownloadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, @@ -730,14 +892,15 @@ std::expected OperationQueue::addDownloadOperation( auto self = weak.lock(); if (!self) return; - // Invoked from the SFTP processing strand via workInStrand — marshal back. + // Invoked from the SFTP processing strand via workInStrand, marshal back. self->within_strand_do( [weak, name, operationId, min, max, current, bytesPerSecond]() { auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::TransferProgress{ .operationId = operationId, @@ -751,11 +914,14 @@ std::expected OperationQueue::addDownloadOperation( ); }; - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - targetQueue.emplace_back(operationId, std::make_unique(sftp, std::move(opts))); + auto& targetQueue = queueFor(mode); + targetQueue.emplace_back( + operationId, std::make_unique(sftp.weak_from_this(), std::move(opts)) + ); Log::info("Calling OperationQueue::{}::onOperationAdded", sessionId_.value()); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -774,7 +940,7 @@ std::expected OperationQueue::addDownloadOperation( else if (result->isDirectory()) { auto bulk = std::make_unique( - sftp, + sftp.weak_from_this(), BulkDownloadOperation::BulkDownloadOperationOptions{ .overallProgressCallback = makeBulkProgressCallback("onBulkDownloadProgress", operationId), .remotePath = remotePath, @@ -788,7 +954,7 @@ std::expected OperationQueue::addDownloadOperation( // The scan is part of the bulk, so cancelling or moving the card acts on both. bulk->scanFirst( std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ .progressCallback = makeScanProgressCallback("onScanProgress", operationId), .remotePath = remotePath, @@ -797,10 +963,11 @@ std::expected OperationQueue::addDownloadOperation( ) ); - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); targetQueue.emplace_back(operationId, std::move(bulk)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -813,18 +980,14 @@ std::expected OperationQueue::addDownloadOperation( } ); - // Synthetic single-entry resume backup: re-issuing this through - // adopt walks the same scan+bulk path, so the entry list itself - // doesn't matter — we just need the directory paths preserved. - SharedData::BulkDownloadRequest synthetic{ - .entries = {SharedData::BulkDownloadEntry{ - .source = remotePath, .destination = localPath, .isDirectory = true - }}, - .allowOverwrite = allowOverwrite, - .insertRefresh = insertRefresh, - .mode = mode, - }; - bulkResumeStash_.insert_or_assign(operationId, BulkResumeEntry{.request = std::move(synthetic)}); + bulkResumeStash_.insert_or_assign( + operationId, + BulkResumeEntry{ + .request = directoryResumeRequest( + remotePath, localPath, allowOverwrite, insertRefresh, mode + ) + } + ); return {}; } @@ -948,7 +1111,8 @@ std::expected OperationQueue::addArchiveDownloadOperatio auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::TransferProgress{ .operationId = operationId, @@ -962,15 +1126,15 @@ std::expected OperationQueue::addArchiveDownloadOperatio ); }; - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); // Directory roots are scanned by the archive operation itself before it packs. - auto archive = std::make_unique(sftp, std::move(opts)); + auto archive = std::make_unique(sftp.weak_from_this(), std::move(opts)); for (auto const& rootPath : directoryRootPaths) { archive->addScanRoot( std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ .progressCallback = makeScanProgressCallback("onScanProgress", operationId), .remotePath = rootPath, @@ -989,7 +1153,8 @@ std::expected OperationQueue::addArchiveDownloadOperatio totalPayloadBytes, directoryRootPaths.size() ); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1075,7 +1240,8 @@ std::expected OperationQueue::addArchiveUploadOperation( auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::TransferProgress{ .operationId = operationId, @@ -1089,10 +1255,10 @@ std::expected OperationQueue::addArchiveUploadOperation( ); }; - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); // Directory roots are scanned by the archive operation itself before it packs. - auto archive = std::make_unique(sftp, std::move(opts)); + auto archive = std::make_unique(sftp.weak_from_this(), std::move(opts)); for (auto const& rootPath : directoryRootPaths) { archive->addScanRoot(std::make_unique(LocalScanOperation::ScanOperationOptions{ @@ -1110,7 +1276,8 @@ std::expected OperationQueue::addArchiveUploadOperation( estimatedTotalBytes, directoryRootPaths.size() ); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1127,127 +1294,101 @@ std::expected OperationQueue::addArchiveUploadOperation( std::size_t OperationQueue::addBulkDownloadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkDownloadRequest const& request, - std::function operationIdFor, - Ids::OperationId bulkCardId + SharedData::BulkDownloadRequest request, + std::function const& operationIdFor, + Ids::OperationId const& bulkCardId ) { - // Assumed in strand. The file portion is collapsed into a SINGLE - // BulkDownloadOperation with prescanned entries — one card on the - // frontend, one SSH-thread worker for the whole batch. Directory - // entries still use the existing Scan+BulkDownload flow (one card - // per directory root). - + // Assumed in strand. auto transferOptions = sftpOpts_.downloadOptions.value_or(Persistence::DownloadOptions{}); if (request.allowOverwrite) transferOptions.mayOverwrite = true; const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); - auto& targetQueue = (request.mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + const auto allowOverwrite = request.allowOverwrite; - std::vector files; - files.reserve(request.entries.size()); - std::size_t directoryCount = 0; - // Each directory resumes as its own unit under its own id, so the card's backup holds only the files. - SharedData::BulkDownloadRequest fileUnit{ - .allowOverwrite = request.allowOverwrite, - .insertRefresh = request.insertRefresh, - .mode = request.mode, - }; - for (std::size_t idx = 0; idx < request.entries.size(); ++idx) - { - auto const& entry = request.entries[idx]; - if (entry.isDirectory) + return splitBulkRequest( + std::move(request), + bulkCardId, + [&](std::size_t index, SharedData::BulkDownloadEntry const& entry, SharedData::BulkDownloadRequest const& unit) { const auto result = addDownloadOperation( sftp, - operationIdFor(idx), + operationIdFor(index), entry.destination, entry.source, - request.allowOverwrite || entry.mayOverwrite, + allowOverwrite || entry.mayOverwrite, /*isBigFile*/ false, - // Propagate the request's flag so manual folder downloads refresh - // the local panel. Sync passes false to avoid refresh spam; a - // manual bulk download passes true and expects one refresh per - // top-level directory on completion. - /*insertRefresh*/ request.insertRefresh, + unit.insertRefresh, /*createMissingDirectories*/ true, - request.mode + unit.mode ); - if (!result.has_value()) + if (!result) { Log::error( "Bulk download: directory entry '{}' failed to queue: {}", entry.source.value(), result.error().toString() ); - continue; } - ++directoryCount; - continue; + return result.has_value(); + }, + [&](SharedData::BulkDownloadRequest& unit) + { + std::vector files; + files.reserve(unit.entries.size()); + std::ranges::transform( + unit.entries, + std::back_inserter(files), + [](SharedData::BulkDownloadEntry const& entry) + { + return BulkDownloadOperation::PrescannedFile{ + .source = entry.source, + .destination = entry.destination, + .sizeBytes = entry.sizeBytes, + .mtime = entry.mtime, + .mtimeNsec = entry.mtimeNsec, + .mayOverwrite = entry.mayOverwrite, + .isDirectory = entry.isDirectory, + .comparedDestination = entry.comparedDestination, + }; + } + ); + unit.remoteRoot = unit.remoteRoot ? *unit.remoteRoot + : commonParentOf(files, &BulkDownloadOperation::PrescannedFile::source); + unit.localRoot = unit.localRoot + ? *unit.localRoot + : commonParentOf(files, &BulkDownloadOperation::PrescannedFile::destination); + + auto downloadOptions = resolveDownloadOptions(transferOptions, resolvedTimeout); + // Sync plans list files of directories that do not exist locally yet. + downloadOptions.createMissingDirectories = true; + auto bulk = std::make_unique( + sftp.weak_from_this(), + BulkDownloadOperation::BulkDownloadOperationOptions{ + .overallProgressCallback = makeBulkProgressCallback("onBulkDownloadProgress", bulkCardId), + .remotePath = *unit.remoteRoot, + .localPath = *unit.localRoot, + .individualOptions = std::move(downloadOptions), + .failFast = transferOptions.failFast.value_or(false), + .concurrency = bulkConcurrency_, + } + ); + bulk->setPrescannedFileList(std::move(files)); + queueFor(unit.mode).emplace_back(bulkCardId, std::move(bulk)); + RpcHelper::callRemoteSafely( + *hub_, + rpcName("onOperationAdded"), + SharedData::OperationAdded{ + .operationId = bulkCardId, + .type = SharedData::OperationType::BulkDownload, + .mode = unit.mode, + .insertRefresh = unit.insertRefresh, + .localPath = *unit.localRoot, + .remotePath = *unit.remoteRoot, + } + ); } - fileUnit.entries.push_back(entry); - files.push_back( - BulkDownloadOperation::PrescannedFile{ - .source = entry.source, - .destination = entry.destination, - .sizeBytes = entry.sizeBytes, - .mtime = entry.mtime, - .mtimeNsec = entry.mtimeNsec, - .mayOverwrite = entry.mayOverwrite, - } - ); - } - - if (!files.empty()) - { - // Dedicated operationId for the aggregate bulk-download card, reserved - // by the frontend separately from the per-entry ids to avoid collisions - // (entry-0's id would clash when entry 0 is a directory). The first - // file entry drives the UI's "localPath"/"remotePath" labels for the - // card — arbitrary pick; per-file paths show up in progress events. - const auto bulkOpId = bulkCardId; - // The card's labels, a file directly below the root labels with the root itself. - const auto remoteLabel = files.front().source.parent().value_or(Paths::RemotePath::root()); - const auto localLabel = files.front().destination.parent().value_or(files.front().destination); - - auto downloadOpts = resolveDownloadOptions(transferOptions, resolvedTimeout); - // Sync plans list files of directories that do not exist locally yet. - downloadOpts.createMissingDirectories = true; - - auto bulk = std::make_unique( - sftp, - BulkDownloadOperation::BulkDownloadOperationOptions{ - .overallProgressCallback = makeBulkProgressCallback("onBulkDownloadProgress", bulkOpId), - .remotePath = remoteLabel, - .localPath = localLabel, - .individualOptions = std::move(downloadOpts), - .failFast = transferOptions.failFast.value_or(false), - .concurrency = bulkConcurrency_, - } - ); - bulk->setPrescannedFileList(std::move(files)); - - targetQueue.emplace_back(bulkOpId, std::move(bulk)); - hub_->callRemote( - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = bulkOpId, - .type = SharedData::OperationType::BulkDownload, - .mode = request.mode, - .insertRefresh = request.insertRefresh, - .localPath = localLabel, - .remotePath = remoteLabel, - } - ); - } - - Log::info("Bulk download: queued one bulk card (for files) + {} directory entries", directoryCount); - - // Stashed under the card id so a teardown can hand it to the resume registry. - if (!fileUnit.entries.empty()) - bulkResumeStash_.insert_or_assign(bulkCardId, BulkResumeEntry{.request = std::move(fileUnit)}); - - return request.entries.size(); + ); } std::expected OperationQueue::addUploadOperation( @@ -1273,7 +1414,7 @@ std::expected OperationQueue::addUploadOperation( if (ec) { - Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), ec.message()); + Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), Utility::errorMessage(ec)); return std::unexpected(Operation::Error{.type = Operation::ErrorType::FileStatFailed}); } @@ -1284,7 +1425,7 @@ std::expected OperationQueue::addUploadOperation( stat = std::filesystem::status(localNativePath, ec); if (ec) { - Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), ec.message()); + Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), Utility::errorMessage(ec)); return std::unexpected(Operation::Error{.type = Operation::ErrorType::FileStatFailed}); } } @@ -1317,7 +1458,8 @@ std::expected OperationQueue::addUploadOperation( auto self = weak.lock(); if (!self) return; - self->hub_->callRemote( + RpcHelper::callRemoteSafely( + *self->hub_, name, SharedData::TransferProgress{ .operationId = operationId, @@ -1331,10 +1473,13 @@ std::expected OperationQueue::addUploadOperation( ); }; - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - targetQueue.emplace_back(operationId, std::make_unique(sftp, std::move(opts))); + auto& targetQueue = queueFor(mode); + targetQueue.emplace_back( + operationId, std::make_unique(sftp.weak_from_this(), std::move(opts)) + ); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1352,7 +1497,7 @@ std::expected OperationQueue::addUploadOperation( else if (stat.type() == std::filesystem::file_type::directory) { auto bulk = std::make_unique( - sftp, + sftp.weak_from_this(), BulkUploadOperation::BulkUploadOperationOptions{ .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", operationId), .remotePath = remotePath, @@ -1370,10 +1515,11 @@ std::expected OperationQueue::addUploadOperation( }) ); - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); targetQueue.emplace_back(operationId, std::move(bulk)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1386,16 +1532,14 @@ std::expected OperationQueue::addUploadOperation( } ); - // Synthetic single-entry resume backup: adopting it walks the same scan and bulk path. - SharedData::BulkUploadRequest synthetic{ - .entries = {SharedData::BulkUploadEntry{ - .source = localPath, .destination = remotePath, .isDirectory = true - }}, - .allowOverwrite = allowOverwrite, - .insertRefresh = insertRefresh, - .mode = mode, - }; - bulkResumeStash_.insert_or_assign(operationId, BulkResumeEntry{.request = std::move(synthetic)}); + bulkResumeStash_.insert_or_assign( + operationId, + BulkResumeEntry{ + .request = directoryResumeRequest( + localPath, remotePath, allowOverwrite, insertRefresh, mode + ) + } + ); return {}; } @@ -1408,229 +1552,231 @@ std::expected OperationQueue::addUploadOperation( std::size_t OperationQueue::addBulkUploadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkUploadRequest const& request, - std::function operationIdFor, - Ids::OperationId bulkCardId + SharedData::BulkUploadRequest request, + std::function const& operationIdFor, + Ids::OperationId const& bulkCardId ) { - // Mirrors addBulkDownloadOperation — file portion collapses into one - // BulkUploadOperation card; directory entries reuse the existing - // LocalScan+BulkUpload flow. + // Assumed in strand. auto transferOptions = sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}); if (request.allowOverwrite) transferOptions.mayOverwrite = true; const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); - auto& targetQueue = (request.mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + const auto allowOverwrite = request.allowOverwrite; - std::vector files; - files.reserve(request.entries.size()); - std::size_t directoryCount = 0; - // Each directory resumes as its own unit under its own id, so the card's backup holds only the files. - SharedData::BulkUploadRequest fileUnit{ - .allowOverwrite = request.allowOverwrite, - .insertRefresh = request.insertRefresh, - .mode = request.mode, - }; - for (std::size_t idx = 0; idx < request.entries.size(); ++idx) - { - auto const& entry = request.entries[idx]; - if (entry.isDirectory) + return splitBulkRequest( + std::move(request), + bulkCardId, + [&](std::size_t index, SharedData::BulkUploadEntry const& entry, SharedData::BulkUploadRequest const& unit) { const auto result = addUploadOperation( sftp, - operationIdFor(idx), + operationIdFor(index), entry.source, entry.destination, - request.allowOverwrite || entry.mayOverwrite, + allowOverwrite || entry.mayOverwrite, /*isBigFile*/ false, - // Propagate the request's flag so manual folder uploads refresh - // the remote panel. Sync passes false to avoid refresh spam; a - // manual bulk upload passes true and expects one refresh per - // top-level directory on completion. - /*insertRefresh*/ request.insertRefresh, + unit.insertRefresh, /*createMissingDirectories*/ true, - request.mode + unit.mode ); - if (!result.has_value()) + if (!result) { Log::error( "Bulk upload: directory entry '{}' failed to queue: {}", entry.source.toUtf8(), result.error().toString() ); - continue; } - ++directoryCount; - continue; - } - fileUnit.entries.push_back(entry); - files.push_back( - BulkUploadOperation::PrescannedFile{ - .source = entry.source, - .destination = entry.destination, - .sizeBytes = entry.sizeBytes, - .mayOverwrite = entry.mayOverwrite, - } - ); - } - - if (!files.empty()) - { - // Separate reserved id — see addBulkDownloadOperation for rationale. - const auto bulkOpId = bulkCardId; - const auto localLabel = files.front().source.parent().value_or(files.front().source); - const auto remoteLabel = files.front().destination.parent().value_or(Paths::RemotePath::root()); - - auto uploadOpts = resolveUploadOptions(transferOptions, resolvedTimeout); - uploadOpts.createMissingDirectories = true; - - auto bulk = std::make_unique( - sftp, - BulkUploadOperation::BulkUploadOperationOptions{ - .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", bulkOpId), - .remotePath = remoteLabel, - .localPath = localLabel, - .individualOptions = std::move(uploadOpts), - .failFast = transferOptions.failFast.value_or(false), - .concurrency = bulkConcurrency_, - } - ); - bulk->setPrescannedFileList(std::move(files)); - - targetQueue.emplace_back(bulkOpId, std::move(bulk)); - hub_->callRemote( - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = bulkOpId, - .type = SharedData::OperationType::BulkUpload, - .mode = request.mode, - .insertRefresh = request.insertRefresh, - .localPath = localLabel, - .remotePath = remoteLabel, - } - ); - } - - Log::info("Bulk upload: queued one bulk card (for files) + {} directory entries", directoryCount); - - if (!fileUnit.entries.empty()) - bulkResumeStash_.insert_or_assign(bulkCardId, BulkResumeEntry{.request = std::move(fileUnit)}); - - return request.entries.size(); -} - -namespace -{ - /** - * @brief Creates @p directory and its missing parents below @p root, which has to exist. - */ - bool createRemoteDirectoryBelow( - SecureShell::SftpSession& sftp, - Paths::RemotePath const& root, - Paths::RemotePath const& directory, - std::chrono::seconds timeout - ) - { - std::vector chain; - for (auto current = std::optional{directory}; current && *current != root && current->isWithin(root); - current = current->parent()) - chain.push_back(*current); - for (auto iter = chain.rbegin(); iter != chain.rend(); ++iter) + return result.has_value(); + }, + [&](SharedData::BulkUploadRequest& unit) { - auto created = sftp.createDirectoryIfItDoesntExist(*iter); - if (created.wait_for(timeout) != std::future_status::ready || !created.get().has_value()) - { - Log::error("Sync: could not create remote directory '{}'.", iter->value()); - return false; - } + std::vector files; + files.reserve(unit.entries.size()); + std::ranges::transform( + unit.entries, + std::back_inserter(files), + [](SharedData::BulkUploadEntry const& entry) + { + return BulkUploadOperation::PrescannedFile{ + .source = entry.source, + .destination = entry.destination, + .sizeBytes = entry.sizeBytes, + .mayOverwrite = entry.mayOverwrite, + .isDirectory = entry.isDirectory, + .comparedDestination = entry.comparedDestination, + }; + } + ); + unit.localRoot = + unit.localRoot ? *unit.localRoot : commonParentOf(files, &BulkUploadOperation::PrescannedFile::source); + unit.remoteRoot = unit.remoteRoot + ? *unit.remoteRoot + : commonParentOf(files, &BulkUploadOperation::PrescannedFile::destination); + + auto uploadOptions = resolveUploadOptions(transferOptions, resolvedTimeout); + uploadOptions.createMissingDirectories = true; + auto bulk = std::make_unique( + sftp.weak_from_this(), + BulkUploadOperation::BulkUploadOperationOptions{ + .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", bulkCardId), + .remotePath = *unit.remoteRoot, + .localPath = *unit.localRoot, + .individualOptions = std::move(uploadOptions), + .failFast = transferOptions.failFast.value_or(false), + .concurrency = bulkConcurrency_, + } + ); + bulk->setPrescannedFileList(std::move(files)); + queueFor(unit.mode).emplace_back(bulkCardId, std::move(bulk)); + RpcHelper::callRemoteSafely( + *hub_, + rpcName("onOperationAdded"), + SharedData::OperationAdded{ + .operationId = bulkCardId, + .type = SharedData::OperationType::BulkUpload, + .mode = unit.mode, + .insertRefresh = unit.insertRefresh, + .localPath = *unit.localRoot, + .remotePath = *unit.remoteRoot, + } + ); } - return true; - } + ); } SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( SecureShell::SftpSession& sftp, SyncSession const& session, SyncSession::ResolvedPlan plan, - SharedData::Sync::SyncExecutionRequest const& request + SharedData::Sync::SyncExecutionRequest const& request, + LocalOperationQueue& localDeleteQueue ) { // Assumed in strand using SharedData::Sync::Action; - const auto timeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); SharedData::Sync::SyncExecutionResult result{.failed = std::move(plan.refused)}; - SharedData::BulkUploadRequest uploads{.allowOverwrite = true, .mode = request.mode}; - SharedData::BulkDownloadRequest downloads{.allowOverwrite = true, .mode = request.mode}; + // An entry replaces only a destination the compare saw, and only while it still holds what the compare saw. + SharedData::BulkUploadRequest uploads{ + .mode = request.mode, + .localRoot = session.localRoot(), + .remoteRoot = session.remoteRoot(), + }; + SharedData::BulkDownloadRequest downloads{ + .mode = request.mode, + .localRoot = session.localRoot(), + .remoteRoot = session.remoteRoot(), + }; SharedData::BulkDeleteRequest remoteDeletes{.mode = request.mode, .listedContentOnly = true}; std::vector localDeletes; - const auto localHome = localHomeDirectory(); + std::vector localDeletesCompared; + const auto localHomes = localHomeDirectories(); + auto remoteGuard = remoteDeleteGuard(sftp); + remoteGuard.trustScannedBelow(session.remoteRoot()); + + // The plan lists a directory before its content, so a refused directory refuses every entry below it. + std::vector refusedRemoteDirectories; + std::vector refusedLocalDirectories; + const auto isBelowAny = [](std::vector const& directories, std::string const& relKey) + { + return std::ranges::any_of( + directories, + [&relKey](std::string const& directory) + { + return relKey.size() > directory.size() && relKey.starts_with(directory) && + relKey[directory.size()] == '/'; + } + ); + }; + const auto refuse = [&result]( + SharedData::Sync::EnqueuePlanEntry& entry, + std::vector& refusedDirectories, + std::string_view side, + std::string const& reason + ) + { + Log::error("Sync: refusing to delete {} '{}' and what is below it: {}", side, entry.relKey, reason); + if (entry.isDirectory) + refusedDirectories.push_back(entry.relKey); + result.failed.push_back(std::move(entry.relKey)); + }; - for (auto& [entry, paths] : plan.entries) + for (auto& planned : plan.entries) { + auto& entry = planned.entry; + auto& paths = planned.paths; + const auto comparedDestination = + entry.isDirectory ? std::nullopt : std::optional{planned.compared}; switch (entry.action) { case Action::Upload: { - if (entry.isDirectory) - { - auto& target = createRemoteDirectoryBelow(sftp, session.remoteRoot(), paths.remote, timeout) - ? result.createdDirectories - : result.failed; - target.push_back(std::move(entry.relKey)); - break; - } - result.uploadSources.push_back(paths.local.toUtf8()); uploads.entries.push_back({ .source = std::move(paths.local), .destination = std::move(paths.remote), .sizeBytes = entry.sizeBytes, + .isDirectory = entry.isDirectory, + .withoutContent = entry.isDirectory, + .mayOverwrite = planned.compared.exists, + .comparedDestination = comparedDestination, }); result.uploads.push_back(std::move(entry.relKey)); break; } case Action::Download: { - if (entry.isDirectory) - { - std::error_code error{}; - std::filesystem::create_directories(paths.local.native(), error); - (error ? result.failed : result.createdDirectories).push_back(std::move(entry.relKey)); - break; - } - result.downloadSources.push_back(paths.remote.value()); downloads.entries.push_back({ .source = std::move(paths.remote), .destination = std::move(paths.local), .sizeBytes = entry.sizeBytes, + .isDirectory = entry.isDirectory, + .withoutContent = entry.isDirectory, .mtime = entry.mtime, .mtimeNsec = entry.mtimeNsec, + .mayOverwrite = planned.compared.exists, + .comparedDestination = comparedDestination, }); result.downloads.push_back(std::move(entry.relKey)); break; } case Action::DeleteRemote: { - if (!refuseProtectedDeleteTarget(sftp, paths.remote).has_value()) + if (isBelowAny(refusedRemoteDirectories, entry.relKey)) { result.failed.push_back(std::move(entry.relKey)); break; } + if (auto allowed = remoteGuard.check(paths.remote); !allowed) + { + refuse(entry, refusedRemoteDirectories, "remotely", allowed.error().toString()); + break; + } result.remoteDeleteSources.push_back(paths.remote.value()); - remoteDeletes.entries.push_back({.target = std::move(paths.remote), .isDirectory = entry.isDirectory}); + remoteDeletes.entries.push_back({ + .target = std::move(paths.remote), + .isDirectory = entry.isDirectory, + .compared = std::move(planned.compared), + }); result.remoteDeletes.push_back(std::move(entry.relKey)); break; } case Action::DeleteLocal: { - if (auto allowed = refuseProtectedLocalDelete(paths.local, localHome); !allowed) + if (isBelowAny(refusedLocalDirectories, entry.relKey)) { - Log::error("Sync: refusing to delete locally: {}", allowed.error()); result.failed.push_back(std::move(entry.relKey)); break; } + if (auto allowed = refuseProtectedLocalDelete(paths.local, localHomes); !allowed) + { + refuse(entry, refusedLocalDirectories, "locally", allowed.error()); + break; + } result.localDeleteSources.push_back(paths.local.toUtf8()); localDeletes.push_back(std::move(paths.local)); + localDeletesCompared.push_back(std::move(planned.compared)); result.localDeletes.push_back(std::move(entry.relKey)); break; } @@ -1641,7 +1787,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( { addBulkUploadOperation( sftp, - uploads, + std::move(uploads), [](std::size_t) { return Ids::generateOperationId(); @@ -1653,7 +1799,7 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( { addBulkDownloadOperation( sftp, - downloads, + std::move(downloads), [](std::size_t) { return Ids::generateOperationId(); @@ -1661,13 +1807,22 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( request.downloadId ); } - if (!remoteDeletes.entries.empty() && addBulkDeleteOperation(sftp, remoteDeletes, request.remoteDeleteId) == 0) + if (!remoteDeletes.entries.empty() && + addBulkDeleteOperation(remoteGuard, sftp, remoteDeletes, request.remoteDeleteId) == 0) { std::ranges::move(result.remoteDeletes, std::back_inserter(result.failed)); result.remoteDeletes.clear(); result.remoteDeleteSources.clear(); } - if (!localDeletes.empty() && !addLocalDeleteOperation(request.localDeleteId, std::move(localDeletes), request.mode)) + if (!localDeletes.empty() && + !localDeleteQueue.addDelete({ + .operationId = request.localDeleteId, + .targets = std::move(localDeletes), + .mode = request.mode, + .owner = sessionId_, + .held = paused_, + .comparedStates = std::move(localDeletesCompared), + })) { std::ranges::move(result.localDeletes, std::back_inserter(result.failed)); result.localDeletes.clear(); @@ -1675,104 +1830,36 @@ SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( } Log::info( - "Sync: queued {} uploads, {} downloads, {} remote and {} local deletes, created {} directories, {} failed", + "Sync: queued {} uploads, {} downloads, {} remote and {} local deletes, {} failed", result.uploads.size(), result.downloads.size(), result.remoteDeletes.size(), result.localDeletes.size(), - result.createdDirectories.size(), result.failed.size() ); return result; } -std::expected OperationQueue::addLocalDeleteOperation( - Ids::OperationId operationId, - std::vector targets, - SharedData::OperationMode mode -) +RemoteDeleteGuard OperationQueue::remoteDeleteGuard(SecureShell::SftpSession& sftp) { - // Assumed in strand - if (targets.empty()) - return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = "no targets"}); - const auto home = localHomeDirectory(); - for (auto const& target : targets) - { - if (auto allowed = refuseProtectedLocalDelete(target, home); !allowed) - { - Log::error("Refusing to delete locally: {}", allowed.error()); - return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = target.toUtf8()}); - } - } - - const auto commonParent = targets.front().parent().value_or(targets.front()); - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - targetQueue.emplace_back( - operationId, - std::make_unique(LocalDeleteOperation::LocalDeleteOperationOptions{ - .itemsRemovedProgress = - [weak = weak_from_this(), operationId, name = rpcName("onDeleteProgress")]( - std::string const& path, std::uint64_t itemsDeleted, std::uint64_t totalItems - ) - { - auto self = weak.lock(); - if (!self) - return; - self->hub_->callRemote( - name, - SharedData::BulkDeleteProgress{ - .operationId = operationId, - .currentFile = path, - .filesDeleted = itemsDeleted, - .totalFiles = totalItems, - } - ); - }, - .targets = std::move(targets), - }) - ); - - hub_->callRemote( - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = operationId, - .type = SharedData::OperationType::LocalDelete, - .mode = mode, - .localPath = commonParent, - } - ); - return {}; + return RemoteDeleteGuard{sftp, remoteHome_, sftpOpts_.operationTimeout.value_or(defaultFutureTimeout)}; } -std::expected -OperationQueue::refuseProtectedDeleteTarget(SecureShell::SftpSession& sftp, Paths::RemotePath const& target) +std::expected OperationQueue::addDeleteOperation( + SecureShell::SftpSession& sftp, + Ids::OperationId operationId, + Paths::RemotePath const& remotePath, + bool recursive, + bool insertRefresh, + SharedData::OperationMode mode +) { - const auto refuse = [&target](std::string_view why) - { - Log::error("Refusing to delete '{}': {}", target.value(), why); - return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = target.value()}); - }; - if (target.empty()) - return refuse("no path"); - if (target.isRoot()) - return refuse("it is the root directory"); - - if (!remoteHome_) - { - auto homeFuture = sftp.homeDirectory(); - if (homeFuture.wait_for(sftpOpts_.operationTimeout.value_or(defaultFutureTimeout)) != std::future_status::ready) - return refuse("the home directory could not be determined in time"); - auto home = homeFuture.get(); - if (!home) - return refuse("the home directory could not be determined"); - remoteHome_ = *std::move(home); - } - if (remoteHome_->isWithin(target)) - return refuse(fmt::format("it contains the home directory '{}'", remoteHome_->value())); - return {}; + auto guard = remoteDeleteGuard(sftp); + return addDeleteOperation(guard, sftp, std::move(operationId), remotePath, recursive, insertRefresh, mode); } std::expected OperationQueue::addDeleteOperation( + RemoteDeleteGuard& guard, SecureShell::SftpSession& sftp, Ids::OperationId operationId, Paths::RemotePath const& remotePath, @@ -1782,10 +1869,15 @@ std::expected OperationQueue::addDeleteOperation( ) { // Assumed in strand - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); - if (auto allowed = refuseProtectedDeleteTarget(sftp, remotePath); !allowed) - return allowed; + if (auto allowed = guard.check(remotePath); !allowed) + { + Log::error("Refusing to delete: {}", allowed.error().toString()); + return std::unexpected( + Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = allowed.error().toString()} + ); + } // lstat: a symlink to a directory is removed as a link, recursing would delete the target's content. auto statFut = sftp.lstat(remotePath); @@ -1799,26 +1891,9 @@ std::expected OperationQueue::addDeleteOperation( const auto isBulk = recursive && isDirectory; auto operation = std::make_unique( - sftp, + sftp.weak_from_this(), DeleteOperation::DeleteOperationOptions{ - .filesRemovedProgress = - [weak = weak_from_this(), operationId, name = rpcName("onDeleteProgress")]( - auto const& path, std::uint64_t filesDeleted, std::uint64_t totalFiles - ) - { - auto self = weak.lock(); - if (!self) - return; - self->hub_->callRemote( - name, - SharedData::BulkDeleteProgress{ - .operationId = operationId, - .currentFile = path, - .filesDeleted = filesDeleted, - .totalFiles = totalFiles, - } - ); - }, + .filesRemovedProgress = makeDeleteProgressCallback(operationId), .remotePath = remotePath, .futureTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout), .recursive = isBulk, @@ -1827,9 +1902,9 @@ std::expected OperationQueue::addDeleteOperation( if (isBulk) { // The scan is part of the delete, so cancelling or moving the card acts on both. - operation->scanFirst( + operation->addScanRoot( std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ .progressCallback = makeScanProgressCallback("onScanProgress", operationId), .remotePath = remotePath, @@ -1841,7 +1916,8 @@ std::expected OperationQueue::addDeleteOperation( targetQueue.emplace_back(operationId, std::move(operation)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -1861,44 +1937,54 @@ std::size_t OperationQueue::addBulkDeleteOperation( SharedData::BulkDeleteRequest const& request, Ids::OperationId bulkOperationId ) +{ + auto guard = remoteDeleteGuard(sftp); + return addBulkDeleteOperation(guard, sftp, request, std::move(bulkOperationId)); +} + +std::size_t OperationQueue::addBulkDeleteOperation( + RemoteDeleteGuard& guard, + SecureShell::SftpSession& sftp, + SharedData::BulkDeleteRequest const& request, + Ids::OperationId bulkOperationId +) { // Assumed in strand. One DeleteOperation, so one card and one completion for the whole request: files are // deleted as given, every real directory is scanned first and deleted with its content. - auto& targetQueue = (request.mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(request.mode); const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); auto operation = std::make_unique( - sftp, + sftp.weak_from_this(), DeleteOperation::DeleteOperationOptions{ - .filesRemovedProgress = - [weak = weak_from_this(), bulkOperationId, name = rpcName("onDeleteProgress")]( - auto const& path, std::uint64_t filesDeleted, std::uint64_t totalFiles - ) - { - auto self = weak.lock(); - if (!self) - return; - self->hub_->callRemote( - name, - SharedData::BulkDeleteProgress{ - .operationId = bulkOperationId, - .currentFile = path, - .filesDeleted = filesDeleted, - .totalFiles = totalFiles, - } - ); - }, + .filesRemovedProgress = makeDeleteProgressCallback(bulkOperationId), .futureTimeout = resolvedTimeout, .recursive = true, } ); - std::vector fileTargets; + std::vector fileTargets; std::size_t scannedDirectories = 0; + std::size_t failedTargets = 0; + const auto failTarget = [&](Paths::RemotePath const& target, Operation::Error error) + { + operation->addFailedTarget(target, std::move(error)); + ++failedTargets; + }; + // The roots are scanned one after another, their counts add up on the card. + const auto scanProgressOfNextRoot = + Detail::sequentialScanProgress(makeScanProgressCallback("onScanProgress", bulkOperationId)); for (auto const& entry : request.entries) { - if (!refuseProtectedDeleteTarget(sftp, entry.target).has_value()) + if (auto allowed = guard.check(entry.target); !allowed) + { + Log::error("Bulk delete: refusing to delete: {}", allowed.error().toString()); + failTarget( + entry.target, + Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = allowed.error().toString()} + ); continue; + } if (entry.isDirectory && !request.listedContentOnly) { // lstat: a symlink to a directory is removed as a link, recursing would delete the target's content. @@ -1906,6 +1992,7 @@ std::size_t OperationQueue::addBulkDeleteOperation( if (statFuture.wait_for(resolvedTimeout) != std::future_status::ready) { Log::error("Bulk delete: lstat timed out for '{}'", entry.target.value()); + failTarget(entry.target, Operation::Error{.type = Operation::ErrorType::FutureTimeout}); continue; } const auto information = statFuture.get(); @@ -1913,9 +2000,9 @@ std::size_t OperationQueue::addBulkDeleteOperation( { operation->addScanRoot( std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ - .progressCallback = makeScanProgressCallback("onScanProgress", bulkOperationId), + .progressCallback = scanProgressOfNextRoot(), .remotePath = entry.target, .futureTimeout = resolvedTimeout, } @@ -1925,20 +2012,23 @@ std::size_t OperationQueue::addBulkDeleteOperation( continue; } } - fileTargets.push_back(entry.target); + fileTargets.push_back(DeleteOperation::Target{.path = entry.target, .compared = entry.compared}); } const auto queued = fileTargets.size() + scannedDirectories; if (queued == 0) return 0; + if (failedTargets > 0) + Log::warn("Bulk delete: {} items were refused, they are reported as failed", failedTargets); - // Parent of the first item, so the card can read "Deleting items in " before any progress arrives. - const auto commonParent = request.entries.front().target.parent().value_or(Paths::RemotePath::root()); + // The card reads "Deleting items in " before any progress arrives. + const auto commonParent = commonParentOf(request.entries, &SharedData::BulkDeleteEntry::target); const auto fileCount = fileTargets.size(); operation->setTargets(std::move(fileTargets)); targetQueue.emplace_back(bulkOperationId, std::move(operation)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = bulkOperationId, @@ -1971,7 +2061,7 @@ std::expected OperationQueue::addRenameOperation( // Assumed in strand auto operation = std::make_unique( - sftp, + sftp.weak_from_this(), RenameOperation::RenameOperationOptions{ .sourcePath = sourcePath, .destinationPath = destinationPath, @@ -1979,10 +2069,11 @@ std::expected OperationQueue::addRenameOperation( } ); - auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + auto& targetQueue = queueFor(mode); targetQueue.emplace_back(operationId, std::move(operation)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, @@ -2010,83 +2101,49 @@ void OperationQueue::addSyncScanOperation( { // Create (or replace) the SyncSession. A fresh id from the frontend is the // norm; re-opening over an existing id is handled as "discard previous state". + const auto remoteFoldsCase = [&]() + { + if (RemoteDeleteGuardDetail::looksCaseFolding(remotePath)) + return true; + const auto home = remoteDeleteGuard(sftp).homeDirectory(); + return home && RemoteDeleteGuardDetail::looksCaseFolding(*home); + }(); auto session = std::make_shared( SyncSession::Options{ .sessionId = syncSessionId, .localRoot = localPath, .remoteRoot = remotePath, + .remoteFoldsCase = remoteFoldsCase, }, executor_ ); syncSessions_[syncSessionId.value()] = session; - // Scan completion callbacks hand the ScanNode trees into the session on its - // own strand, then emit onSyncScanPhaseDone to the frontend. - syncScanCallbacks_[remoteScanId.value()] = - [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}]( - std::expected tree - ) - { - auto self = weak.lock(); - auto sess = weakSession.lock(); - if (!self || !sess) - return; - if (!tree) - { - self->hub_->callRemote( - self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/false, tree.error() - ); - return; - } - boost::asio::post( - sess->strand(), - [sess, tree = std::move(*tree)]() mutable - { - sess->setRemoteTreeInStrand(std::move(tree)); - } - ); - self->hub_->callRemote(self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/false); - }; - - syncScanCallbacks_[localScanId.value()] = - [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}]( - std::expected tree - ) - { - auto self = weak.lock(); - auto sess = weakSession.lock(); - if (!self || !sess) - return; - if (!tree) - { - self->hub_->callRemote( - self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/true, tree.error() - ); - return; - } - boost::asio::post( - sess->strand(), - [sess, tree = std::move(*tree)]() mutable - { - sess->setLocalTreeInStrand(std::move(tree)); - } - ); - self->hub_->callRemote(self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/true); - }; + syncScanCallbacks_[remoteScanId.value()] = makeSyncScanCallback( + syncSessionId, session, localScanId, /*isLocal=*/false, &SyncSession::setRemoteTreeInStrand + ); + syncScanCallbacks_[localScanId.value()] = makeSyncScanCallback( + syncSessionId, session, remoteScanId, /*isLocal=*/true, &SyncSession::setLocalTreeInStrand + ); // Queue remote scan — build-tree mode. - // Temp files of transfers on either side, a sync neither copies nor deletes them. + // Temp files of transfers on either side and the asides of replaced destinations, a sync neither copies nor + // deletes them. + const auto uploadSuffix = Persistence::effectiveTempFileSuffix( + sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}).tempFileSuffix + ); + const auto downloadSuffix = Persistence::effectiveTempFileSuffix( + sftpOpts_.downloadOptions.value_or(Persistence::DownloadOptions{}).tempFileSuffix + ); std::vector temporarySuffixes{ - Persistence::effectiveTempFileSuffix( - sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}).tempFileSuffix - ), - Persistence::effectiveTempFileSuffix( - sftpOpts_.downloadOptions.value_or(Persistence::DownloadOptions{}).tempFileSuffix - ), + uploadSuffix, + downloadSuffix, + Persistence::asideFileSuffix(uploadSuffix), + Persistence::asideFileSuffix(downloadSuffix), }; auto remoteScan = std::make_unique( - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{ .progressCallback = makeScanProgressCallback("onScanProgress", remoteScanId), .remotePath = remotePath, @@ -2100,7 +2157,8 @@ void OperationQueue::addSyncScanOperation( ); priorityOperations_.emplace_back(remoteScanId, std::move(remoteScan)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = remoteScanId, @@ -2122,7 +2180,8 @@ void OperationQueue::addSyncScanOperation( }); priorityOperations_.emplace_back(localScanId, std::move(localScan)); - hub_->callRemote( + RpcHelper::callRemoteSafely( + *hub_, rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = localScanId, @@ -2133,6 +2192,44 @@ void OperationQueue::addSyncScanOperation( ); } +auto OperationQueue::makeSyncScanCallback( + Ids::SyncSessionId syncSessionId, + std::weak_ptr session, + Ids::OperationId siblingScanId, + bool isLocal, + void (SyncSession::*setTree)(SharedData::Sync::ScanNode) +) -> std::function)> +{ + return [weak = weak_from_this(), + syncSessionId = std::move(syncSessionId), + session = std::move(session), + siblingScanId = std::move(siblingScanId), + isLocal, + setTree](std::expected tree) + { + auto self = weak.lock(); + auto locked = session.lock(); + if (!self || !locked) + return; + if (!tree) + { + self->cancelSiblingScan(siblingScanId); + RpcHelper::callRemoteSafely( + *self->hub_, self->rpcName("onSyncScanFailed"), syncSessionId, isLocal, tree.error() + ); + return; + } + boost::asio::post( + locked->strand(), + [locked, setTree, tree = std::move(*tree)]() mutable + { + ((*locked).*setTree)(std::move(tree)); + } + ); + RpcHelper::callRemoteSafely(*self->hub_, self->rpcName("onSyncScanPhaseDone"), syncSessionId, isLocal); + }; +} + std::shared_ptr OperationQueue::syncSession(Ids::SyncSessionId const& sessionId) const { const auto iter = syncSessions_.find(sessionId.value()); diff --git a/backend/source/backend/sftp/remote_delete_guard.cpp b/backend/source/backend/sftp/remote_delete_guard.cpp new file mode 100644 index 00000000..a9ab9245 --- /dev/null +++ b/backend/source/backend/sftp/remote_delete_guard.cpp @@ -0,0 +1,267 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace +{ + char foldedCase(char character) + { + return character >= 'A' && character <= 'Z' ? static_cast(character - 'A' + 'a') : character; + } + + bool hasWindowsAmbiguousName(Paths::RemotePath const& path) + { + return std::ranges::any_of( + Paths::Detail::splitAtSlashes(path.view()), + [](std::string_view name) + { + return name.find('\\') != std::string_view::npos || + (!name.empty() && (name.back() == '.' || name.back() == ' ')); + } + ); + } + + bool containsHome(Paths::RemotePath const& target, Paths::RemotePath const& home) + { + if (home.isWithin(target)) + return true; + const auto foldsCase = + RemoteDeleteGuardDetail::looksCaseFolding(home) || RemoteDeleteGuardDetail::looksCaseFolding(target); + return foldsCase && RemoteDeleteGuardDetail::isWithinIgnoringCase(home, target); + } +} + +std::string RemoteDeleteRefusal::toString() const +{ + switch (reason) + { + case Reason::NoPath: + return "no path"; + case Reason::Root: + return fmt::format("'{}' is the root directory", target); + case Reason::ContainsHome: + return fmt::format("'{}' is or contains the home directory '{}'", target, home); + case Reason::WindowsName: + return fmt::format("'{}' could name another item on a server with Windows paths", target); + case Reason::NotVerified: + return fmt::format("'{}' could not be compared with the home directory in time", target); + } + return fmt::format("'{}' is protected", target); +} + +std::string RemoteDeleteRefusal::toLocalizedMessage() const +{ + switch (reason) + { + case Reason::NoPath: + return Utility::localizedMessage("backend.deleteGuard.noPath"); + case Reason::Root: + return Utility::localizedMessage("backend.deleteGuard.root", target); + case Reason::ContainsHome: + return Utility::localizedMessage("backend.deleteGuard.containsHome", target, home); + case Reason::WindowsName: + return Utility::localizedMessage("backend.deleteGuard.windowsName", target); + case Reason::NotVerified: + return Utility::localizedMessage("backend.deleteGuard.notVerified", target); + } + return Utility::localizedMessage("backend.deleteGuard.notVerified", target); +} + +namespace RemoteDeleteGuardDetail +{ + bool looksLikeWindowsPath(Paths::RemotePath const& path) + { + const auto text = path.view(); + const auto isLetter = [](char character) + { + return (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'); + }; + return text.size() >= 3 && text[0] == '/' && isLetter(text[1]) && text[2] == ':' && + (text.size() == 3 || text[3] == '/'); + } + + bool looksCaseFolding(Paths::RemotePath const& path) + { + const auto text = path.view(); + return looksLikeWindowsPath(path) || text == "/Users" || text.starts_with("/Users/"); + } + + bool isWithinIgnoringCase(Paths::RemotePath const& path, Paths::RemotePath const& base) + { + if (path.empty() || base.empty()) + return false; + if (base.isRoot()) + return true; + const auto text = path.view(); + const auto prefix = base.view(); + return text.size() >= prefix.size() && + std::ranges::equal(text.substr(0, prefix.size()), prefix, {}, foldedCase, foldedCase) && + (text.size() == prefix.size() || text[prefix.size()] == '/'); + } + + std::optional + refusalByText(Paths::RemotePath const& target, std::optional const& home) + { + if (target.empty()) + return RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::NoPath}; + if (target.isRoot()) + return RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::Root, .target = target.value()}; + const auto windowsServer = looksLikeWindowsPath(target) || (home && looksLikeWindowsPath(*home)); + if (windowsServer && hasWindowsAmbiguousName(target)) + return RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::WindowsName, .target = target.value()}; + if (home && containsHome(target, *home)) + { + return RemoteDeleteRefusal{ + .reason = RemoteDeleteRefusal::Reason::ContainsHome, + .target = target.value(), + .home = home->value(), + }; + } + return std::nullopt; + } +} + +RemoteDeleteGuard::RemoteDeleteGuard( + SecureShell::SftpSession& sftp, + std::optional& knownHome, + std::chrono::milliseconds timeout +) + : sftp_{&sftp} + , knownHome_{&knownHome} + , timeout_{timeout} +{} + +void RemoteDeleteGuard::trustScannedBelow(Paths::RemotePath root) +{ + scannedRoots_.push_back(std::move(root)); +} + +std::expected RemoteDeleteGuard::check(Paths::RemotePath const& target) +{ + if (const auto refusal = RemoteDeleteGuardDetail::refusalByText(target, std::nullopt); refusal) + return std::unexpected(*refusal); + + const auto refuseFor = [&target](std::optional const& home) + -> std::expected + { + if (const auto refusal = RemoteDeleteGuardDetail::refusalByText(target, home); refusal) + return std::unexpected(*refusal); + return {}; + }; + + return home(target).and_then( + [&](std::optional const& home) -> std::expected + { + if (auto refused = refuseFor(home); !refused || !home) + return refused; + return canonicalDirectory(*target.parent(), target) + .and_then( + [&](std::optional const& directory) -> std::expected + { + if (!directory || !containsHome(*directory / *target.filename(), *home)) + return {}; + return std::unexpected( + RemoteDeleteRefusal{ + .reason = RemoteDeleteRefusal::Reason::ContainsHome, + .target = target.value(), + .home = home->value(), + } + ); + } + ); + } + ); +} + +std::optional RemoteDeleteGuard::homeDirectory() +{ + return home(Paths::RemotePath{}).value_or(std::nullopt); +} + +std::expected, RemoteDeleteRefusal> +RemoteDeleteGuard::home(Paths::RemotePath const& target) +{ + if (*knownHome_ || homeUnknown_) + return *knownHome_; + + auto future = sftp_->performPromise( + [sftp = sftp_]() + { + return sftp->canonicalHomeDirectoryInStrand(); + } + ); + if (future.wait_for(timeout_) != std::future_status::ready) + return std::unexpected( + RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::NotVerified, .target = target.value()} + ); + auto home = future.get(); + if (!home) + { + Log::warn( + "The remote home directory is unknown ({}), remote deletes only refuse the root directory.", + home.error().toString() + ); + homeUnknown_ = true; + return std::nullopt; + } + *knownHome_ = *std::move(home); + return *knownHome_; +} + +std::expected, RemoteDeleteRefusal> +RemoteDeleteGuard::canonicalDirectory(Paths::RemotePath const& directory, Paths::RemotePath const& target) +{ + const auto scannedRoot = std::ranges::find_if( + scannedRoots_, + [&directory](Paths::RemotePath const& root) + { + return directory.isWithin(root) && directory != root; + } + ); + if (scannedRoot == scannedRoots_.end()) + return lookUpCanonical(directory, target); + return lookUpCanonical(*scannedRoot, target) + .transform( + [&directory, &root = *scannedRoot](std::optional const& canonicalRoot) + -> std::optional + { + if (!canonicalRoot) + return std::nullopt; + return *canonicalRoot / *directory.relativeTo(root); + } + ); +} + +std::expected, RemoteDeleteRefusal> +RemoteDeleteGuard::lookUpCanonical(Paths::RemotePath const& directory, Paths::RemotePath const& target) +{ + if (const auto cached = canonicalDirectories_.find(directory.value()); cached != canonicalDirectories_.end()) + return cached->second; + + auto future = sftp_->performPromise( + [sftp = sftp_, directory]() + { + return sftp->canonicalPathInStrand(directory); + } + ); + if (future.wait_for(timeout_) != std::future_status::ready) + return std::unexpected( + RemoteDeleteRefusal{.reason = RemoteDeleteRefusal::Reason::NotVerified, .target = target.value()} + ); + auto canonical = future.get(); + if (!canonical) + { + // A directory that does not exist holds nothing to delete, the text checks still apply. + Log::debug("Delete guard: cannot canonicalize '{}': {}", directory.value(), canonical.error().toString()); + return canonicalDirectories_.emplace(directory.value(), std::nullopt).first->second; + } + return canonicalDirectories_.emplace(directory.value(), *std::move(canonical)).first->second; +} diff --git a/backend/source/backend/sftp/rename_operation.cpp b/backend/source/backend/sftp/rename_operation.cpp index 8fac293a..35e2473c 100644 --- a/backend/source/backend/sftp/rename_operation.cpp +++ b/backend/source/backend/sftp/rename_operation.cpp @@ -2,9 +2,9 @@ #include -RenameOperation::RenameOperation(SecureShell::SftpSession& sftp, RenameOperationOptions options) +RenameOperation::RenameOperation(std::weak_ptr sftp, RenameOperationOptions options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , sourcePath_{std::move(options.sourcePath)} , destinationPath_{std::move(options.destinationPath)} , futureTimeout_{options.futureTimeout} @@ -12,7 +12,7 @@ RenameOperation::RenameOperation(SecureShell::SftpSession& sftp, RenameOperation SecureShell::ProcessingStrand* RenameOperation::strand() const { - return sftp_->strand(); + return strandOf(sftp_); } std::expected RenameOperation::work() @@ -30,11 +30,20 @@ std::expected RenameOperati } case (Running): { + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); // A queued rename mirrors a local move and must not replace what is at the destination. - auto fut = sftp_->performPromise( - [this]() + // The task outlives this operation when the wait times out. + auto fut = sftp->performPromise( + [session = sftp_, + source = sourcePath_, + destination = destinationPath_]() -> std::expected { - return sftp_->renameNoReplaceInStrand(sourcePath_, destinationPath_); + const auto sftp = session.lock(); + if (!sftp) + return std::unexpected(SecureShell::SftpError{.message = "The SFTP channel was closed"}); + return sftp->renameNoReplaceInStrand(source, destination); } ); if (fut.wait_for(futureTimeout_) != std::future_status::ready) diff --git a/backend/source/backend/sftp/scan_operation.cpp b/backend/source/backend/sftp/scan_operation.cpp index 7dc3c004..4ae1f105 100644 --- a/backend/source/backend/sftp/scan_operation.cpp +++ b/backend/source/backend/sftp/scan_operation.cpp @@ -7,8 +7,8 @@ #include #include -ScanOperation::ScanOperation(SecureShell::SftpSession& sftp, ScanOperationOptions options) - : sftp_(&sftp) +ScanOperation::ScanOperation(std::weak_ptr sftp, ScanOperationOptions options) + : sftp_{std::move(sftp)} , remotePath_{std::move(options.remotePath)} , progressCallback_{std::move(options.progressCallback)} , futureTimeout_{options.futureTimeout} @@ -62,9 +62,11 @@ ScanOperation::scanner(Paths::RelativePath const& relative) return std::vector{}; rootScanned_ = true; + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState>(channelClosedError()); const auto path = remotePath_ / relative; - auto fut = sftp_->listDirectory(path); - fut.wait_for(futureTimeout_); + auto fut = sftp->listDirectory(path); if (fut.wait_for(futureTimeout_) != std::future_status::ready) return enterErrorState>({.type = ErrorType::FutureTimeout}); @@ -109,7 +111,7 @@ ScanOperation::scanner(Paths::RelativePath const& relative) continue; if (entry.type != SharedData::DirectoryEntry::FileType::Regular) continue; - if (auto content = readRemoteFile(*sftp_, path / *Paths::PathSegment::make(name), futureTimeout_)) + if (auto content = readRemoteFile(*sftp, path / *entry.nameSegment(), futureTimeout_)) ignoreMatcher_.addFile(relative, *content); } @@ -121,7 +123,7 @@ ScanOperation::scanner(Paths::RelativePath const& relative) for (auto& entry : entries) { // A name that is no segment cannot match a rule, the walker rejects it. - const auto segment = Paths::PathSegment::make(entry.name); + const auto segment = entry.nameSegment(); const bool isDir = entry.type == SharedData::DirectoryEntry::FileType::Directory; if (segment && ignoreMatcher_.isIgnored(relative / *segment, isDir)) continue; @@ -226,5 +228,5 @@ std::expected ScanOperation::cancel(bool adoptCancel SecureShell::ProcessingStrand* ScanOperation::strand() const { - return sftp_->strand(); + return strandOf(sftp_); } \ No newline at end of file diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index c6fc4033..82348c4c 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -5,15 +6,16 @@ #include #include +#include #include #include #include #include -UploadOperation::UploadOperation(SecureShell::SftpSession& sftp, UploadOperationOptions options) +UploadOperation::UploadOperation(std::weak_ptr sftp, UploadOperationOptions options) : Operation{} - , sftp_{&sftp} + , sftp_{std::move(sftp)} , fileStream_{} , options_{std::move(options)} , localFile_{} @@ -28,17 +30,12 @@ UploadOperation::UploadOperation(SecureShell::SftpSession& sftp, UploadOperation UploadOperation::~UploadOperation() { std::ignore = cancel(false); - // Wait for in-flight strand tasks unless we already run on the processing thread, where - // waiting would deadlock (tasks only get queued, never executed inline). - if (auto* stra = strand(); stra && !stra->withinProcessingThread()) - stra->pushPromiseTask([]() {}).get(); + waitForStrandTasks(strand()); } SecureShell::ProcessingStrand* UploadOperation::strand() const { - if (!sftp_) - return nullptr; - return sftp_->strand(); + return strandOf(sftp_); } void UploadOperation::pause(bool doPause) @@ -49,7 +46,10 @@ void UploadOperation::pause(bool doPause) std::expected UploadOperation::work() { - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return workInStrand(); @@ -67,6 +67,10 @@ std::expected UploadOperati { using enum OperationState; + const auto sftp = openSession(sftp_); + if (!sftp && !hasEnded()) + return enterErrorState(channelClosedError()); + switch (state_) { case (NotStarted): @@ -76,57 +80,11 @@ std::expected UploadOperati } case (Preparing): { - // Local-only preconditions replicated from prepare(); remote work lives in - // prepareInStrand() and runs without an inner umbrella. - if (options_.localPath.empty()) - { - Log::error("UploadOperation: Invalid local path."); - return enterErrorState({.type = ErrorType::InvalidPath}); - } - const auto localStatus = std::filesystem::symlink_status(localNativePath_); - if (!std::filesystem::is_regular_file(localStatus) && !std::filesystem::is_symlink(localStatus)) + if (const auto opened = openLocalSource(); !opened) + return enterErrorState(opened.error()); + if (!isSymlink_) { - Log::error( - "UploadOperation: Local path '{}' is not a regular file or symlink.", options_.localPath.toUtf8() - ); - return enterErrorState({.type = ErrorType::NotAFile}); - } - if (std::filesystem::is_symlink(localStatus) && - options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) - { - isSymlink_ = true; - } - else - { - if (!std::filesystem::is_regular_file(std::filesystem::status(localNativePath_))) - { - Log::error( - "UploadOperation: Local path '{}' is a directory, but a file was expected.", - options_.localPath.toUtf8() - ); - return enterErrorState({.type = ErrorType::NotAFile}); - } - if (!std::filesystem::exists(localNativePath_)) - { - Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::FileNotFound}); - } - localFile_.open(localNativePath_, std::ios::binary); - if (!localFile_.is_open()) - { - Log::error("UploadOperation: Failed to open local file: {}", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::OpenFailure}); - } - localFile_.seekg(0, std::ios::end); - leftToUpload_ = static_cast(localFile_.tellg()); - totalSize_ = leftToUpload_; - localFile_.seekg(0, std::ios::beg); - if (!localFile_.good()) - { - Log::error("UploadOperation: Failed to seek in local file: {}", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::FileSeekFailure}); - } - const auto prepareResult = prepareInStrand(); + const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) { Log::error("UploadOperation: Failed to prepare operation: {}", prepareResult.error().toString()); @@ -141,7 +99,7 @@ std::expected UploadOperati state_ = Running; if (isSymlink_ && options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) { - const auto result = handleSymlinkInStrand(); + const auto result = handleSymlinkInStrand(*sftp); if (!result.has_value()) { Log::error("UploadOperation: Failed to handle symlink: {}", result.error().toString()); @@ -229,7 +187,7 @@ std::expected UploadOperati case (Finalizing): { localFile_.close(); - const auto finalizeResult = finalizeInStrand(); + const auto finalizeResult = finalizeInStrand(*sftp); if (!finalizeResult.has_value()) { Log::error("UploadOperation: Failed to finalize operation: {}", finalizeResult.error().toString()); @@ -287,10 +245,16 @@ UploadOperation::commitFileToBuffer(SecureShell::IFileStream::SignedSizeType byt std::expected UploadOperation::handleSymlink() { - auto fut = sftp_->performPromise( - [this]() + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + auto fut = sftp->performPromise( + [this]() -> std::expected { - return handleSymlinkInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return handleSymlinkInStrand(*sftp); } ); if (fut.wait_for(options_.futureTimeout) != std::future_status::ready) @@ -301,7 +265,7 @@ std::expected UploadOperation::handleSymlink() return fut.get(); } -std::expected UploadOperation::handleSymlinkInStrand() +std::expected UploadOperation::handleSymlinkInStrand(SecureShell::SftpSession& sftp) { if (options_.symlinkHandling == Persistence::SymlinkHandling::SkipSymlink) { @@ -309,18 +273,34 @@ std::expected UploadOperation::handleSymlinkInStra return {}; } - const auto target = std::filesystem::read_symlink(localNativePath_); + std::error_code readError{}; + const auto target = std::filesystem::read_symlink(localNativePath_, readError); + if (readError) + { + Log::error( + "UploadOperation: Cannot read the symlink '{}': {}.", + options_.localPath.toUtf8(), + Utility::errorMessage(readError) + ); + return std::unexpected(Error{.type = ErrorType::FileStatFailed, .extraInfo = Utility::errorMessage(readError)}); + } + const auto targetText = Paths::checkedNativeToUtf8(target); + if (!targetText) + { + Log::error("UploadOperation: The symlink '{}' points to a name that is not UTF-8.", options_.localPath.toUtf8()); + return std::unexpected(Error{.type = ErrorType::InvalidName, .extraInfo = targetText.error().toString()}); + } if (options_.createMissingDirectories) { - if (auto res = ensureRemoteDirectoryExistsInStrand(*remotePath_.parent()); !res.has_value()) + if (auto res = ensureRemoteDirectoryExistsInStrand(sftp, *remotePath_.parent()); !res.has_value()) return std::unexpected(res.error()); } // The link is created under the temp name and renamed into place, so an existing destination is // never removed before its replacement exists. - std::ignore = sftp_->removeFileInStrand(tempPath_); - const auto result = sftp_->createSymLinkInStrand(Paths::nativeToUtf8(target), tempPath_); + std::ignore = sftp.removeFileInStrand(tempPath_); + const auto result = sftp.createSymLinkInStrand(*targetText, tempPath_); if (!result.has_value()) { Log::error( @@ -329,27 +309,29 @@ std::expected UploadOperation::handleSymlinkInStra return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = result.error()}); } - const auto renameResult = options_.mayOverwrite - ? sftp_->renameReplaceInStrand(tempPath_, remotePath_, options_.tempFileSuffix + ".old") - : sftp_->renameNoReplaceInStrand(tempPath_, remotePath_); - if (!renameResult.has_value()) + auto moved = ComparedStateCheck::moveRemoteIntoPlaceInStrand( + sftp, + tempPath_, + remotePath_, + options_.mayOverwrite, + Persistence::asideFileSuffix(options_.tempFileSuffix), + SharedData::FileType::Symlink, + options_.comparedDestination + ); + if (!moved.has_value()) { - std::ignore = sftp_->removeFileInStrand(tempPath_); - if (renameResult.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) - return std::unexpected(Error{.type = ErrorType::FileExists}); - Log::error( - "UploadOperation: Failed to move symlink to '{}': {}.", - options_.remotePath.value(), - renameResult.error().message - ); - return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = renameResult.error()}); + std::ignore = sftp.removeFileInStrand(tempPath_); + return std::unexpected(std::move(moved).error()); } return {}; } std::expected UploadOperation::writeOnce() { - auto fut = sftp_->performPromise( + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto fut = sftp->performPromise( [this]() { return writeOnceInStrand(); @@ -410,9 +392,9 @@ std::expected UploadOperation::writeOnceInStrand() return leftToUpload_ > 0; } -std::expected UploadOperation::openOrAdoptFileInStrand() +std::expected UploadOperation::openOrAdoptFileInStrand(SecureShell::SftpSession& sftp) { - const auto result = sftp_->lstatInStrand(remotePath_); + const auto result = sftp.lstatInStrand(remotePath_); if (!result.has_value()) { const auto err = result.error(); @@ -435,7 +417,7 @@ std::expected UploadOperation::openOrAdoptFileInSt } } - const auto tempResult = sftp_->lstatInStrand(tempPath_); + const auto tempResult = sftp.lstatInStrand(tempPath_); // Permissions of what is uploaded: the target of a followed symlink, not the link. std::error_code statusError{}; @@ -450,7 +432,7 @@ std::expected UploadOperation::openOrAdoptFileInSt Log::debug("UploadOperation: Remote temp file is larger than local file, do not adopt file."); // Do not adopt file } - else if (options_.tryContinue && !tempBelongsToSourceInStrand(tempResult->size)) + else if (options_.tryContinue && !tempBelongsToSourceInStrand(sftp, tempResult->size)) { Log::info( "UploadOperation: Remote temp file '{}' is not a part of this source, starting over.", tempPath_.value() @@ -460,7 +442,7 @@ std::expected UploadOperation::openOrAdoptFileInSt { Log::debug("UploadOperation: Continuing upload to existing temp file."); - const auto openResult = sftp_->openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Write, perms); + const auto openResult = sftp.openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Write, perms); if (!openResult.has_value()) { Log::error("Failed to open remote sftp file for continue: {}", openResult.error().message); @@ -491,13 +473,13 @@ std::expected UploadOperation::openOrAdoptFileInSt if (options_.createMissingDirectories) { - if (auto res = ensureRemoteDirectoryExistsInStrand(*tempPath_.parent()); !res.has_value()) + if (auto res = ensureRemoteDirectoryExistsInStrand(sftp, *tempPath_.parent()); !res.has_value()) return std::unexpected(res.error()); } // Open temp file part regularly, freshly: Log::debug("UploadOperation: Starting new upload to '{}'.", tempPath_.value()); - const auto openResult = sftp_->openFileInStrand( + const auto openResult = sftp.openFileInStrand( tempPath_, SecureShell::SftpSession::OpenType::Write | SecureShell::SftpSession::OpenType::Create | SecureShell::SftpSession::OpenType::Truncate, @@ -514,13 +496,13 @@ std::expected UploadOperation::openOrAdoptFileInSt return {}; } -bool UploadOperation::tempBelongsToSourceInStrand(std::uint64_t tempSize) +bool UploadOperation::tempBelongsToSourceInStrand(SecureShell::SftpSession& sftp, std::uint64_t tempSize) { // Another program's partial file or a part of an older version of the source must not be continued. if (tempSize == 0) return false; const auto opened = - sftp_->openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::none); + sftp.openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::none); if (!opened) return false; auto stream = opened->lock(); @@ -532,12 +514,12 @@ bool UploadOperation::tempBelongsToSourceInStrand(std::uint64_t tempSize) } std::expected -UploadOperation::ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& dir) +UploadOperation::ensureRemoteDirectoryExistsInStrand(SecureShell::SftpSession& sftp, Paths::RemotePath const& dir) { if (dir.isRoot()) return {}; - const auto statResult = sftp_->lstatInStrand(dir); + const auto statResult = sftp.lstatInStrand(dir); if (statResult.has_value()) { if (statResult->type == SharedData::FileType::Directory) @@ -558,7 +540,7 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& di if (statResult.error().sftpError != SSH_FX_NO_SUCH_FILE) return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = statResult.error()}); - if (auto parent = ensureRemoteDirectoryExistsInStrand(dir.parent().value_or(Paths::RemotePath::root())); + if (auto parent = ensureRemoteDirectoryExistsInStrand(sftp, dir.parent().value_or(Paths::RemotePath::root())); !parent.has_value()) return parent; @@ -566,7 +548,7 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& di std::filesystem::perms::owner_all | std::filesystem::perms::group_read | std::filesystem::perms::group_exec | std::filesystem::perms::others_read | std::filesystem::perms::others_exec ); - const auto mkdirResult = sftp_->createDirectoryIfItDoesntExistInStrand(dir, dirPerms); + const auto mkdirResult = sftp.createDirectoryIfItDoesntExistInStrand(dir, dirPerms); if (!mkdirResult.has_value()) { Log::error( @@ -577,69 +559,99 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& di return {}; } -std::expected UploadOperation::prepare() +std::expected UploadOperation::openLocalSource() { if (options_.localPath.empty()) { Log::error("UploadOperation: Invalid local path."); - return enterErrorState({.type = ErrorType::InvalidPath}); + return std::unexpected(Error{.type = ErrorType::InvalidPath}); } - if (remotePath_.empty() || remotePath_.isRoot() || tempPath_.empty()) + std::error_code statusError{}; + const auto linkStatus = std::filesystem::symlink_status(localNativePath_, statusError); + if (linkStatus.type() == std::filesystem::file_type::not_found) { - Log::error("UploadOperation: Invalid remote path '{}'.", options_.remotePath.value()); - return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = options_.remotePath.value()}); + Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); + return std::unexpected(Error{.type = ErrorType::FileNotFound}); } - const auto localStatus = std::filesystem::symlink_status(localNativePath_); - if (!std::filesystem::is_regular_file(localStatus) && !std::filesystem::is_symlink(localStatus)) + if (statusError) + { + Log::error( + "UploadOperation: Cannot inspect '{}': {}", options_.localPath.toUtf8(), Utility::errorMessage(statusError) + ); + return std::unexpected( + Error{.type = ErrorType::FileStatFailed, .extraInfo = Utility::errorMessage(statusError)} + ); + } + if (!std::filesystem::is_regular_file(linkStatus) && !std::filesystem::is_symlink(linkStatus)) { Log::error("UploadOperation: Local path '{}' is not a regular file or symlink.", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::NotAFile}); + return std::unexpected(Error{.type = ErrorType::NotAFile}); } - - if (std::filesystem::is_symlink(localStatus) && + if (std::filesystem::is_symlink(linkStatus) && options_.symlinkHandling != Persistence::SymlinkHandling::FollowSymlink) { isSymlink_ = true; return {}; } - if (!std::filesystem::is_regular_file(std::filesystem::status(localNativePath_))) + // A followed link must point to an existing regular file. + const auto status = std::filesystem::status(localNativePath_, statusError); + if (status.type() == std::filesystem::file_type::not_found) { - Log::error( - "UploadOperation: Local path '{}' is a directory, but a file was expected.", options_.localPath.toUtf8() - ); - return enterErrorState({.type = ErrorType::NotAFile}); + Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); + return std::unexpected(Error{.type = ErrorType::FileNotFound}); } - - // Initial check. Check again later before rename - if (!std::filesystem::exists(localNativePath_)) + if (statusError || !std::filesystem::is_regular_file(status)) { - Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::FileNotFound}); + Log::error("UploadOperation: Local path '{}' is not a regular file.", options_.localPath.toUtf8()); + return std::unexpected(Error{.type = ErrorType::NotAFile}); } localFile_.open(localNativePath_, std::ios::binary); if (!localFile_.is_open()) { Log::error("UploadOperation: Failed to open local file: {}", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::OpenFailure}); + return std::unexpected(Error{.type = ErrorType::OpenFailure}); } - localFile_.seekg(0, std::ios::end); leftToUpload_ = static_cast(localFile_.tellg()); totalSize_ = leftToUpload_; localFile_.seekg(0, std::ios::beg); - if (!localFile_.good()) { Log::error("UploadOperation: Failed to seek in local file: {}", options_.localPath.toUtf8()); - return enterErrorState({.type = ErrorType::FileSeekFailure}); + return std::unexpected(Error{.type = ErrorType::FileSeekFailure}); } + return {}; +} - auto umbrellaFut = sftp_->performPromise( - [this]() +std::expected UploadOperation::prepare() +{ + if (options_.localPath.empty()) + { + Log::error("UploadOperation: Invalid local path."); + return enterErrorState({.type = ErrorType::InvalidPath}); + } + if (remotePath_.empty() || remotePath_.isRoot() || tempPath_.empty()) + { + Log::error("UploadOperation: Invalid remote path '{}'.", options_.remotePath.value()); + return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = options_.remotePath.value()}); + } + if (const auto opened = openLocalSource(); !opened) + return enterErrorState(opened.error()); + if (isSymlink_) + return {}; + + const auto sftp = openSession(sftp_); + if (!sftp) + return enterErrorState(channelClosedError()); + auto umbrellaFut = sftp->performPromise( + [this]() -> std::expected { - return prepareInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return prepareInStrand(*sftp); } ); if (umbrellaFut.wait_for(options_.futureTimeout) != std::future_status::ready) @@ -661,9 +673,9 @@ std::expected UploadOperation::prepare() return {}; } -std::expected UploadOperation::prepareInStrand() +std::expected UploadOperation::prepareInStrand(SecureShell::SftpSession& sftp) { - if (auto result = openOrAdoptFileInStrand(); !result.has_value()) + if (auto result = openOrAdoptFileInStrand(sftp); !result.has_value()) return std::unexpected(result.error()); auto stream = fileStream_.lock(); @@ -673,7 +685,7 @@ std::expected UploadOperation::prepareInStrand() return std::unexpected(Error{.type = ErrorType::FileStreamExpired}); } - buffer_ = sftp_->bufferProvider().leaseForTransfer(totalSize_, stream->writeLengthLimit()); + buffer_ = sftp.bufferProvider().leaseForTransfer(totalSize_, stream->writeLengthLimit()); if (buffer_.empty()) { Log::error("UploadOperation: No transfer buffer available from pool."); @@ -727,10 +739,16 @@ std::expected UploadOperation::finalize() localFile_.close(); - auto fut = sftp_->performPromise( - [this]() + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + auto fut = sftp->performPromise( + [this]() -> std::expected { - return finalizeInStrand(); + const auto sftp = openSession(sftp_); + if (!sftp) + return std::unexpected(channelClosedError()); + return finalizeInStrand(*sftp); } ); // finalize has more work than a single call (stat + maybe removeFile + rename + setstat), @@ -750,7 +768,7 @@ std::expected UploadOperation::finalize() return {}; } -std::expected UploadOperation::finalizeInStrand() +std::expected UploadOperation::finalizeInStrand(SecureShell::SftpSession& sftp) { auto stream = fileStream_.lock(); if (!stream) @@ -775,60 +793,52 @@ std::expected UploadOperation::finalizeInStrand() stream->closeInStrand(); - // The destination is never deleted ahead of the rename: replacing is atomic or moves it aside, and - // without permission to overwrite the rename fails instead of replacing. - const auto renameResult = options_.mayOverwrite - ? sftp_->renameReplaceInStrand(tempPath_, remotePath_, options_.tempFileSuffix + ".old") - : sftp_->renameNoReplaceInStrand(tempPath_, remotePath_); - if (!renameResult.has_value()) - { - if (renameResult.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) - { - Log::warn( - "UploadOperation: Remote file '{}' already exists and may not be overwritten.", - options_.remotePath.value() - ); - return std::unexpected(Error{.type = ErrorType::FileExists}); - } - Log::error("Failed to rename remote sftp file: {}", renameResult.error().message); - return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = renameResult.error()}); - } + // The modification time goes on the temp file, so nothing can fail once the destination is replaced. + applyModificationTimeInStrand(sftp); - // Preserve the local file's mtime on the remote file so subsequent syncs - // don't see the file as modified (best-effort; a failure is only a warning). + auto moved = ComparedStateCheck::moveRemoteIntoPlaceInStrand( + sftp, + tempPath_, + remotePath_, + options_.mayOverwrite, + Persistence::asideFileSuffix(options_.tempFileSuffix), + SharedData::FileType::Regular, + options_.comparedDestination + ); + if (!moved.has_value()) { - std::error_code errc{}; - const auto ftime = std::filesystem::last_write_time(localNativePath_, errc); - if (!errc) - { - const auto sctime = std::chrono::file_clock::to_sys(ftime); - const auto mtimeSecs = static_cast( - std::chrono::duration_cast(sctime.time_since_epoch()).count() - ); - sftp_attributes_struct attrs{}; - attrs.flags = SSH_FILEXFER_ATTR_ACMODTIME; - attrs.atime = mtimeSecs; - attrs.mtime = mtimeSecs; - const auto mtimeResult = sftp_->statInStrand(remotePath_, &attrs); - if (!mtimeResult.has_value()) - { - Log::warn( - "UploadOperation: Failed to set mtime on '{}': {}.", - options_.remotePath.value(), - mtimeResult.error().message - ); - } - } - else - { - Log::warn( - "UploadOperation: Could not read local mtime for '{}': {}.", options_.localPath.toUtf8(), errc.message() - ); - } + if (auto removed = sftp.removeFileInStrand(tempPath_); !removed.has_value()) + Log::warn("UploadOperation: Cannot remove '{}': {}", tempPath_.value(), removed.error().message); + return std::unexpected(std::move(moved).error()); } return {}; } +void UploadOperation::applyModificationTimeInStrand(SecureShell::SftpSession& sftp) +{ + std::error_code timeError{}; + const auto localTime = std::filesystem::last_write_time(localNativePath_, timeError); + if (timeError) + { + Log::warn( + "UploadOperation: Could not read local mtime for '{}': {}.", + options_.localPath.toUtf8(), + Utility::errorMessage(timeError) + ); + return; + } + const auto seconds = static_cast( + std::chrono::duration_cast(std::chrono::file_clock::to_sys(localTime).time_since_epoch()) + .count() + ); + sftp_attributes_struct attributes{}; + attributes.flags = SSH_FILEXFER_ATTR_ACMODTIME; + attributes.atime = seconds; + attributes.mtime = seconds; + if (const auto applied = sftp.statInStrand(tempPath_, &attributes); !applied.has_value()) + Log::warn("UploadOperation: Failed to set mtime on '{}': {}.", tempPath_.value(), applied.error().message); +} + std::filesystem::perms UploadOperation::determinePerms(std::filesystem::perms localPerms) const { const auto raw = [this, localPerms]() diff --git a/backend/source/backend/sync/sync_session.cpp b/backend/source/backend/sync/sync_session.cpp index 083de7b9..06352b34 100644 --- a/backend/source/backend/sync/sync_session.cpp +++ b/backend/source/backend/sync/sync_session.cpp @@ -85,12 +85,8 @@ DiffSummary SyncSession::recomputeDiffInStrand( ) { auto options = requestedOptions; -#if defined(_WIN32) || defined(__APPLE__) - // NTFS and APFS fold case by default. - options.localCaseInsensitive = true; -#else - options.localCaseInsensitive = false; -#endif + options.localCaseInsensitive = options_.localFoldsCase; + options.remoteCaseInsensitive = options_.remoteFoldsCase; uploads_.clear(options); downloads_.clear(options); deletes_.clear(options); @@ -181,6 +177,12 @@ SyncSession::ResolvedPlan SyncSession::resolvePlanInStrand(DiffSection section, std::unordered_set const& selectedRelKeys) const { ResolvedPlan result; + const auto comparedAt = [this](EnqueuePlanEntry const& entry) + { + const bool onLocalSide = entry.action == Action::Download || entry.action == Action::DeleteLocal; + auto const& tree = onLocalSide ? localTree_ : remoteTree_; + return tree ? comparedStateAt(*tree, entry.relKey) : SharedData::ComparedState{}; + }; for (auto& entry : buildEnqueuePlanInStrand(section, selectedRelKeys)) { auto paths = Paths::RelativePath::make(entry.relKey).and_then( @@ -195,7 +197,10 @@ SyncSession::resolvePlanInStrand(DiffSection section, std::unordered_set +#include +#include #include #include @@ -9,7 +11,7 @@ ThemeFinder::ThemeFinder(std::filesystem::path const& relativeRoot, AppWideEvent events.onReloadThemes, [&events, relativeRoot](bool) { - Log::info("Reloading themes... relativeRoot='{}'", relativeRoot.generic_string()); + Log::info("Reloading themes... relativeRoot='{}'", Paths::nativeToUtf8(relativeRoot)); events.availableThemes = findAvailableThemes(relativeRoot); Log::info("availableThemes updated, count={}", events.availableThemes.value().size()); events.availableThemes.eventContext().sync(); @@ -17,15 +19,15 @@ ThemeFinder::ThemeFinder(std::filesystem::path const& relativeRoot, AppWideEvent } )} { - Log::info("ThemeFinder constructed with relativeRoot='{}'", relativeRoot.generic_string()); + Log::info("ThemeFinder constructed with relativeRoot='{}'", Paths::nativeToUtf8(relativeRoot)); } std::vector ThemeFinder::findAvailableThemes(std::filesystem::path const& relativeRoot) { - Log::info("findAvailableThemes called with relativeRoot='{}'", relativeRoot.generic_string()); + Log::info("findAvailableThemes called with relativeRoot='{}'", Paths::nativeToUtf8(relativeRoot)); std::error_code ec; const bool rootExists = std::filesystem::exists(relativeRoot, ec); - Log::info("relativeRoot exists={}, ec='{}'", rootExists, ec.message()); + Log::info("relativeRoot exists={}, ec='{}'", rootExists, Utility::errorMessage(ec)); const auto files = findFilesInSearchPaths(relativeRoot, "themes/*.css"); Log::info("findFilesInSearchPaths returned {} file(s).", files.size()); @@ -33,7 +35,7 @@ std::vector ThemeFinder::findAvailableThemes(std::filesys themes.reserve(files.size()); for (const auto& file : files) { - Log::info("Found theme file: '{}', stem='{}'", file.generic_string(), file.stem().generic_string()); + Log::info("Found theme file: '{}', stem='{}'", Paths::nativeToUtf8(file), Paths::nativeToUtf8(file.stem())); themes.push_back(file.stem()); } Log::info("Theme discovery complete, found {} themes.", themes.size()); diff --git a/backend/source/backend/windows/main_windows.cpp b/backend/source/backend/windows/main_windows.cpp index 8ce15627..8c49aad5 100644 --- a/backend/source/backend/windows/main_windows.cpp +++ b/backend/source/backend/windows/main_windows.cpp @@ -1,9 +1,10 @@ #include -#include - #include #include +#include + +#include Main::PlatformSpecifics::PlatformSpecifics(Nui::Window& window, Nui::RpcHub& hub) : enableDragDrop{window, hub} @@ -19,26 +20,68 @@ namespace const auto handle = GetStdHandle(standardHandle); return handle != nullptr && handle != INVALID_HANDLE_VALUE && GetFileType(handle) != FILE_TYPE_UNKNOWN; } + + bool reopen(char const* device, char const* mode, FILE* target) + { + FILE* stream = nullptr; + return freopen_s(&stream, device, mode, target) == 0; + } +} + +ConsoleAttachment::~ConsoleAttachment() +{ + std::cout.flush(); + std::cerr.flush(); + std::fflush(stdout); + std::fflush(stderr); + if (previousOutputCodePage_ != 0) + SetConsoleOutputCP(previousOutputCodePage_); + if (previousInputCodePage_ != 0) + SetConsoleCP(previousInputCodePage_); +} + +ConsoleAttachment::ConsoleAttachment(ConsoleAttachment&& other) noexcept + : previousOutputCodePage_{std::exchange(other.previousOutputCodePage_, 0u)} + , previousInputCodePage_{std::exchange(other.previousInputCodePage_, 0u)} +{} + +ConsoleAttachment& ConsoleAttachment::operator=(ConsoleAttachment&& other) noexcept +{ + if (this != &other) + { + previousOutputCodePage_ = std::exchange(other.previousOutputCodePage_, 0u); + previousInputCodePage_ = std::exchange(other.previousInputCodePage_, 0u); + } + return *this; } -void attachParentConsole() +ConsoleAttachment attachParentConsole() { // Redirected streams keep their file or pipe, only the ones without a target go to the console. const bool outputUsable = isUsable(STD_OUTPUT_HANDLE); const bool errorUsable = isUsable(STD_ERROR_HANDLE); const bool inputUsable = isUsable(STD_INPUT_HANDLE); if (outputUsable && errorUsable && inputUsable) - return; + return {}; if (!AttachConsole(ATTACH_PARENT_PROCESS)) - return; - FILE* stream = nullptr; - if (!outputUsable) - freopen_s(&stream, "CONOUT$", "w", stdout); - if (!errorUsable) - freopen_s(&stream, "CONOUT$", "w", stderr); - if (!inputUsable) - freopen_s(&stream, "CONIN$", "r", stdin); - std::cout.clear(); - std::cerr.clear(); - std::cin.clear(); + return {}; + + ConsoleAttachment attachment{}; + const bool outputReopened = !outputUsable && reopen("CONOUT$", "w", stdout); + const bool errorReopened = !errorUsable && reopen("CONOUT$", "w", stderr); + const bool inputReopened = !inputUsable && reopen("CONIN$", "r", stdin); + if (outputReopened || errorReopened) + { + attachment.previousOutputCodePage_ = GetConsoleOutputCP(); + SetConsoleOutputCP(CP_UTF8); + std::cout.clear(); + std::cerr.clear(); + } + if (inputReopened) + { + attachment.previousInputCodePage_ = GetConsoleCP(); + SetConsoleCP(CP_UTF8); + std::cin.clear(); + } + return attachment; } diff --git a/backend/source/backend/windows/windows_drag_drop.cpp b/backend/source/backend/windows/windows_drag_drop.cpp index 2cbe6297..57b4a129 100644 --- a/backend/source/backend/windows/windows_drag_drop.cpp +++ b/backend/source/backend/windows/windows_drag_drop.cpp @@ -1,5 +1,7 @@ #include +#include +#include #include #include #include @@ -73,7 +75,7 @@ EnableWindowsDragDrop::EnableWindowsDragDrop(Nui::Window& wnd, Nui::RpcHub& hub) catch (std::exception const& exc) { std::wcout << L"Failed to parse message as JSON." << std::endl; - std::cout << exc.what(); + std::cout << Utility::errorMessage(exc); return S_OK; } @@ -141,12 +143,12 @@ EnableWindowsDragDrop::EnableWindowsDragDrop(Nui::Window& wnd, Nui::RpcHub& hub) if (messageJson.contains("subdir")) reply["subdir"] = messageJson["subdir"]; - hub.callRemote("SessionArea::onFilesDropped", reply); + RpcHelper::callRemoteSafely(hub, "SessionArea::onFilesDropped", reply); } catch (std::exception const& exc) { std::wcout << L"Failed to send drop message to RPC hub." << std::endl; - std::cout << exc.what(); + std::cout << Utility::errorMessage(exc); } return S_OK; diff --git a/backend/test/backend/archive_test_helpers.hpp b/backend/test/backend/archive_test_helpers.hpp index 352459dd..54333e2f 100644 --- a/backend/test/backend/archive_test_helpers.hpp +++ b/backend/test/backend/archive_test_helpers.hpp @@ -30,6 +30,7 @@ namespace Test std::string path{}; SharedData::FileType type{}; std::string contents{}; + std::filesystem::perms permissions{std::filesystem::perms::unknown}; }; inline SharedData::ArchiveDownloadRoot makeFileEntry(std::string_view path, std::uint64_t size) @@ -77,6 +78,7 @@ namespace Test RecoveredEntry recovered{}; recovered.path = entry.header().path; recovered.type = entry.header().type; + recovered.permissions = entry.header().permissions; if (recovered.type == SharedData::FileType::Regular) { diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index d405f5a2..3e5c05d0 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -5,6 +5,10 @@ #include "test_upload_operation.hpp" #include "test_scan_operation.hpp" #include "test_local_delete_guard.hpp" +#include "test_local_delete_operation.hpp" +#include "test_local_removal.hpp" +#include "test_local_operation_queue.hpp" +#include "test_remote_delete_guard.hpp" #include "test_local_scan_operation.hpp" #include "test_bulk_download_operation.hpp" #include "test_bulk_upload_operation.hpp" @@ -23,10 +27,14 @@ #include "test_file_change_filter.hpp" #include "test_password_request.hpp" #include "test_sync_session.hpp" +#include "test_dropped_entry.hpp" #include "test_transfer_failures.hpp" +#include "test_compared_state_check.hpp" #include "test_operation_queue.hpp" #include "test_existence_check.hpp" #include "test_opener_risk.hpp" +#include "test_rpc_helper.hpp" +#include "test_rpc_path_parameters.hpp" #include diff --git a/backend/test/backend/test_archive_download_operation.hpp b/backend/test/backend/test_archive_download_operation.hpp index ee2599cc..36058f45 100644 --- a/backend/test/backend/test_archive_download_operation.hpp +++ b/backend/test/backend/test_archive_download_operation.hpp @@ -4,6 +4,7 @@ #include #include "real_server_tests.hpp" #include "archive_test_helpers.hpp" +#include #include #include @@ -58,7 +59,7 @@ namespace Test static ScanBundle runScan(SecureShell::SftpSession& sftp, std::filesystem::path const& path) { ScanOperation scan{ - sftp, + sftp.weak_from_this(), ScanOperation::ScanOperationOptions{.remotePath = remote(path.generic_string())}, }; while (true) @@ -82,7 +83,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ArchiveDownloadOperation operation{*sftp, {}}; + ArchiveDownloadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::ArchiveDownload); } @@ -92,11 +93,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = {}, - }}; + } + }; auto result = operation.work(); ASSERT_FALSE(result.has_value()); EXPECT_EQ(result.error().type, ArchiveDownloadOperation::ErrorType::InvalidPath); @@ -108,7 +110,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry("/")}, .localArchivePath = toLocal(archiveDir_.path() / "root.tar"), @@ -127,7 +129,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {}, .localArchivePath = toLocal(archiveDir_.path() / "empty.tar"), @@ -145,7 +147,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "one.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -164,6 +166,31 @@ namespace Test EXPECT_EQ(entries[0].contents, "Fake file content"); } + TEST_F(ArchiveDownloadOperationTests, NonAsciiNamesRoundTrip) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/Größe.txt"}, {"content", "size"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto archivePath = archiveDir_.path() / Paths::nativeFromUtf8("Ärger.tar"); + ArchiveDownloadOperation operation{ + sftp, + { + .roots = {makeFileEntry("/home/test/Größe.txt", 4u)}, + .localArchivePath = toLocal(archivePath), + .compression = TarArchive::Compression::None, + } + }; + auto result = runToCompletion(operation); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(*result, ArchiveDownloadOperation::WorkStatus::Complete); + + const auto entries = recoverArchiveEntries(archivePath); + ASSERT_EQ(entries.size(), 1u); + EXPECT_EQ(entries[0].path, "Größe.txt"); + EXPECT_EQ(entries[0].contents, "size"); + } + TEST_F(ArchiveDownloadOperationTests, SingleFileGzipRoundTrip) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -171,7 +198,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "one.tar.gz"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -194,7 +221,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "one.tar.bz2"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -214,7 +241,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "one.tar.zst"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -234,7 +261,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "pair.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = { @@ -263,7 +290,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "documents.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry(rootPath.generic_string())}, .localArchivePath = toLocal(archivePath), @@ -298,7 +325,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "short.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry(rootPath.generic_string())}, .localArchivePath = toLocal(archivePath), @@ -319,7 +346,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "missing_scan.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry("/home/test/Documents")}, .localArchivePath = toLocal(archivePath), @@ -345,7 +372,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "wrong_key.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry("/home/test/Documents")}, .localArchivePath = toLocal(archivePath), @@ -370,7 +397,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "two_roots.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = { @@ -405,7 +432,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "mixed_roots.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = { @@ -445,7 +472,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "var_tree.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeDirectoryEntry("/var")}, .localArchivePath = toLocal(archivePath), @@ -471,7 +498,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "large.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/large.txt", 65536u)}, .localArchivePath = toLocal(archivePath), @@ -486,6 +513,48 @@ namespace Test EXPECT_EQ(entries[0].contents.size(), 65536u); } + TEST_F(ArchiveDownloadOperationTests, AFileRootIsArchivedWithItsCurrentSize) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + // The listing the root came from is older than the file. + const auto archivePath = archiveDir_.path() / "fresh.tar"; + ArchiveDownloadOperation operation{ + sftp, + { + .roots = {makeFileEntry("/home/test/large.txt", 100u)}, + .localArchivePath = toLocal(archivePath), + .compression = TarArchive::Compression::None, + } + }; + ASSERT_TRUE(runToCompletion(operation, 2000).has_value()); + + const auto entries = recoverArchiveEntries(archivePath); + ASSERT_EQ(entries.size(), 1u); + EXPECT_EQ(entries[0].contents.size(), 65536u); + } + + TEST_F(ArchiveDownloadOperationTests, AFileRootThatIsGoneFailsTheArchive) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto archivePath = archiveDir_.path() / "gone.tar"; + ArchiveDownloadOperation operation{ + sftp, + { + .roots = {makeFileEntry("/home/test/not_there.txt", 10u)}, + .localArchivePath = toLocal(archivePath), + .compression = TarArchive::Compression::None, + } + }; + const auto result = runToCompletion(operation, 2000); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, ArchiveDownloadOperation::ErrorType::FileStatFailed); + EXPECT_FALSE(std::filesystem::exists(archivePath)); + } + TEST_F(ArchiveDownloadOperationTests, FilepartAbsentAfterCompletion) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -493,7 +562,7 @@ namespace Test const auto archivePath = archiveDir_.path() / "tmp_check.tar"; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -518,7 +587,7 @@ namespace Test } ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -542,7 +611,7 @@ namespace Test } ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), @@ -563,7 +632,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archiveDir_.path() / "finished.tar"), @@ -586,7 +655,7 @@ namespace Test std::uint64_t lastMax{0u}; ArchiveDownloadOperation operation{ - *sftp, + sftp, { .progressCallback = [&](std::uint64_t, std::uint64_t max, std::uint64_t current, std::make_signed_t) @@ -614,7 +683,7 @@ namespace Test ASSERT_FALSE(std::filesystem::exists(archivePath.parent_path())); ArchiveDownloadOperation operation{ - *sftp, + sftp, { .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = toLocal(archivePath), diff --git a/backend/test/backend/test_archive_upload_operation.hpp b/backend/test/backend/test_archive_upload_operation.hpp index 5b38f90f..a91fc52b 100644 --- a/backend/test/backend/test_archive_upload_operation.hpp +++ b/backend/test/backend/test_archive_upload_operation.hpp @@ -5,6 +5,7 @@ #include #include "real_server_tests.hpp" #include "archive_test_helpers.hpp" +#include #include @@ -101,7 +102,7 @@ namespace Test { const auto localPath = pulledDir_.path() / localFilename; DownloadOperation download{ - sftp, + sftp.weak_from_this(), { .remotePath = remote(remotePath.generic_string()), .localPath = toLocal(localPath), @@ -130,7 +131,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ArchiveUploadOperation operation{*sftp, {}}; + ArchiveUploadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::ArchiveUpload); } @@ -140,7 +141,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = {}, @@ -157,7 +158,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {}, .remoteArchivePath = remote("/home/test/nothing.tar"), @@ -175,7 +176,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/single.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -192,6 +193,36 @@ namespace Test EXPECT_EQ(entries[0].contents, "alpha payload"); } + TEST_F(ArchiveUploadOperationTests, NonAsciiNamesRoundTrip) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto source = srcDir_.path() / Paths::nativeFromUtf8("Grüße"); + std::filesystem::create_directories(source); + std::ofstream{source / Paths::nativeFromUtf8("東京.txt"), std::ios::binary} << "tokyo"; + auto scan = runLocalScan(source); + + const std::filesystem::path remoteArchive = "/home/test/nonascii.tar"; + ArchiveUploadOperation operation{ + sftp, + { + .localPaths = {toLocal(source)}, + .remoteArchivePath = remote(remoteArchive.generic_string()), + .compression = TarArchive::Compression::None, + .mayOverwrite = true, + } + }; + operation.setScanResultForRoot(toLocal(source), std::move(scan.entries), scan.totalBytes); + ASSERT_TRUE(runToCompletion(operation).has_value()); + + const auto pulled = pullRemoteArchive(*sftp, remoteArchive, "nonascii.tar"); + const auto entries = recoverArchiveEntries(pulled); + const auto* file = findEntry(entries, "Grüße/東京.txt"); + ASSERT_NE(file, nullptr); + EXPECT_EQ(file->contents, "tokyo"); + } + TEST_F(ArchiveUploadOperationTests, SingleFileGzipRoundTrip) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -199,7 +230,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/alpha.tar.gz"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -221,7 +252,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/alpha.tar.zst"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -243,7 +274,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/pair.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = { @@ -277,7 +308,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/tree.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path())}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -301,13 +332,58 @@ namespace Test EXPECT_EQ(deep->contents, "deep payload"); } +#ifndef _WIN32 + TEST_F(ArchiveUploadOperationTests, SpecialPermissionBitsNeverTravel) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + using std::filesystem::perms; + const auto executable = perms::owner_all | perms::group_read | perms::group_exec | perms::others_read | + perms::others_exec; + std::filesystem::permissions(srcDir_.path() / "alpha.txt", executable | perms::set_uid); + std::filesystem::permissions(srcDir_.path() / "nested" / "deep.txt", executable | perms::set_gid); + auto restore = Nui::ScopeExit{[this]() noexcept { + std::error_code error{}; + std::filesystem::permissions(srcDir_.path() / "alpha.txt", perms::owner_read | perms::owner_write, error); + std::filesystem::permissions( + srcDir_.path() / "nested" / "deep.txt", perms::owner_read | perms::owner_write, error + ); + }}; + if ((std::filesystem::status(srcDir_.path() / "alpha.txt").permissions() & perms::set_uid) == perms::none) + GTEST_SKIP() << "the filesystem drops setuid"; + + auto scan = runLocalScan(srcDir_.path() / "nested"); + const std::filesystem::path remoteArchive = "/home/test/special.tar"; + ArchiveUploadOperation operation{ + sftp, + { + .localPaths = {toLocal(srcDir_.path() / "alpha.txt"), toLocal(srcDir_.path() / "nested")}, + .remoteArchivePath = remote(remoteArchive.generic_string()), + .compression = TarArchive::Compression::None, + .mayOverwrite = true, + } + }; + operation.setScanResultForRoot(toLocal(srcDir_.path() / "nested"), std::move(scan.entries), scan.totalBytes); + ASSERT_TRUE(runToCompletion(operation).has_value()); + + const auto entries = recoverArchiveEntries(pullRemoteArchive(*sftp, remoteArchive, "special.tar")); + const auto* alpha = findEntry(entries, "alpha.txt"); + ASSERT_NE(alpha, nullptr); + EXPECT_EQ(alpha->permissions, executable); + const auto* deep = findEntry(entries, "nested/deep.txt"); + ASSERT_NE(deep, nullptr); + EXPECT_EQ(deep->permissions, executable); + } +#endif + TEST_F(ArchiveUploadOperationTests, DirectoryRootWithoutPrescanErrors) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path())}, .remoteArchivePath = remote("/home/test/missing_scan.tar"), @@ -335,7 +411,7 @@ namespace Test auto otherScan = runLocalScan(otherDir.path()); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path())}, .remoteArchivePath = remote("/home/test/wrong_key.tar"), @@ -366,7 +442,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/two_roots_up.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(firstDir.path()), toLocal(secondDir.path())}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -400,7 +476,7 @@ namespace Test const std::filesystem::path topFile = srcDir_.path() / "alpha.txt"; const std::filesystem::path remoteArchive = "/home/test/mixed_up.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { // A regular-file root (no scan needed) alongside a directory // root (which does need a scan) — the op must succeed and @@ -434,7 +510,7 @@ namespace Test const std::filesystem::path remoteArchive = "/home/test/nested_up.tar"; ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path())}, .remoteArchivePath = remote(remoteArchive.generic_string()), @@ -460,7 +536,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = remote("/home/test/done.tar"), @@ -484,7 +560,7 @@ namespace Test std::uint64_t lastMax{0u}; ArchiveUploadOperation operation{ - *sftp, + sftp, { .progressCallback = [&](std::uint64_t, std::uint64_t max, std::uint64_t current, std::make_signed_t) diff --git a/backend/test/backend/test_bulk_download_operation.hpp b/backend/test/backend/test_bulk_download_operation.hpp index e3871039..c8002b67 100644 --- a/backend/test/backend/test_bulk_download_operation.hpp +++ b/backend/test/backend/test_bulk_download_operation.hpp @@ -5,6 +5,7 @@ #include "real_server_tests.hpp" #include +#include #include #include @@ -18,6 +19,46 @@ extern std::filesystem::path temporaryRoot; namespace Test { + TEST(TreeEntryPathsTests, EntriesAreJoinedBelowBothRoots) + { + using SharedData::DirectoryEntry; + const std::vector entries{ + DirectoryEntry{.type = DirectoryEntry::FileType::Directory}, + DirectoryEntry{.name = "sub", .type = DirectoryEntry::FileType::Directory, .parent = 0}, + DirectoryEntry{.name = "file.txt", .type = DirectoryEntry::FileType::Regular, .parent = 1}, + DirectoryEntry{.name = "..", .type = DirectoryEntry::FileType::Regular, .parent = 1}, + DirectoryEntry{.name = "lost.txt", .type = DirectoryEntry::FileType::Regular, .parent = 7}, + }; + const auto remoteRoot = remote("/home/test/tree"); + const auto localRoot = toLocal(temporaryRoot / "tree"); + + const auto root = Detail::treeEntryPaths(entries, 0, remoteRoot, localRoot); + ASSERT_TRUE(root.has_value()) << root.error().toString(); + EXPECT_EQ(root->remote, remoteRoot); + EXPECT_EQ(root->local, localRoot); + + const auto file = Detail::treeEntryPaths(entries, 2, remoteRoot, localRoot); + ASSERT_TRUE(file.has_value()) << file.error().toString(); + EXPECT_EQ(file->remote.value(), "/home/test/tree/sub/file.txt"); + EXPECT_EQ(file->local.native(), (temporaryRoot / "tree" / "sub" / "file.txt")); + + const auto invalid = Detail::treeEntryPaths(entries, 3, remoteRoot, localRoot); + ASSERT_FALSE(invalid.has_value()); + EXPECT_EQ(invalid.error().type, Operation::ErrorType::InvalidName); + + const auto lost = Detail::treeEntryPaths(entries, 4, remoteRoot, localRoot); + ASSERT_FALSE(lost.has_value()); + EXPECT_EQ(lost.error().type, Operation::ErrorType::ImplementationError); + + const auto outside = Detail::treeEntryPaths(entries, 5, remoteRoot, localRoot); + ASSERT_FALSE(outside.has_value()); + EXPECT_EQ(outside.error().type, Operation::ErrorType::ImplementationError); + + const auto noRoot = Detail::treeEntryPaths(entries, 2, Paths::RemotePath{}, localRoot); + ASSERT_FALSE(noRoot.has_value()); + EXPECT_EQ(noRoot.error().type, Operation::ErrorType::ImplementationError); + } + class BulkDownloadOperationTests : public RealServerTests { public: @@ -42,7 +83,9 @@ namespace Test ScanBundle runScan(SecureShell::SftpSession& sftp, std::filesystem::path const& remotePath, bool recursive = true) { - ScanOperation scan{sftp, {.remotePath = remote(remotePath.generic_string()), .recursive = recursive}}; + ScanOperation scan{ + sftp.weak_from_this(), {.remotePath = remote(remotePath.generic_string()), .recursive = recursive} + }; for (int iteration = 0; iteration < 500; ++iteration) { auto workResult = scan.work(); @@ -81,14 +124,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {}}; + BulkDownloadOperation operation{sftp, {}}; } TEST_F(BulkDownloadOperationTests, BulkDownloadOperationTypeIsBulkDownload) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {}}; + BulkDownloadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::BulkDownload); } @@ -96,7 +139,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {}}; + BulkDownloadOperation operation{sftp, {}}; EXPECT_FALSE(operation.isBarrier()); } @@ -105,7 +148,7 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(downloadDir_.path() / "empty")}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(downloadDir_.path() / "empty")}}; // A scanned tree always holds its root, an empty one means the scan never delivered. operation.setScanResult({}, 0); @@ -119,7 +162,7 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {}}; + BulkDownloadOperation operation{sftp, {}}; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); EXPECT_EQ(workResult.error().type, SharedData::OperationErrorType::ImplementationError); @@ -134,7 +177,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "bad")} + sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "bad")} }; std::vector entries; @@ -163,7 +206,7 @@ namespace Test const auto localRoot = downloadDir_.path() / "Documents"; BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} + sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -188,7 +231,7 @@ namespace Test ASSERT_FALSE(std::filesystem::exists(localRoot)); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} + sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -228,7 +271,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), std::string{"Document 1 content"}.size()); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -271,7 +314,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = false} + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = false} }; operation.setScanResult(std::move(entries), 0); @@ -292,7 +335,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); const auto localRoot = downloadDir_.path() / "rootless"; - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot)}}; auto scan = runScan(*sftp, "/home/test/Documents"); operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -322,7 +365,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -377,7 +420,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -420,7 +463,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = true} + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = true} }; operation.setScanResult(std::move(entries), 0); @@ -471,7 +514,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -500,7 +543,7 @@ namespace Test } ); - BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -518,7 +561,7 @@ namespace Test int callbackInvocations = 0; BulkDownloadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&callbackInvocations](SharedData::BulkProgress const&) @@ -543,7 +586,7 @@ namespace Test auto scan = runScan(*sftp, "/home/test/Documents"); BulkDownloadOperation operation{ - *sftp, + sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(downloadDir_.path() / "completed")} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -561,7 +604,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "failed")} + sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "failed")} }; std::vector entries; @@ -605,7 +648,7 @@ namespace Test .sizeBytes = std::string{"Document 2 content"}.size(), }); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot)}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -641,7 +684,7 @@ namespace Test .mayOverwrite = true, }); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot)}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -683,7 +726,7 @@ namespace Test .mtimeNsec = 0, }); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot)}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -719,7 +762,7 @@ namespace Test BulkDownloadOperation::BulkDownloadOperationOptions options{.localPath = toLocal(localRoot)}; options.individualOptions.createMissingDirectories = true; - BulkDownloadOperation operation{*sftp, std::move(options)}; + BulkDownloadOperation operation{sftp, std::move(options)}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -757,7 +800,7 @@ namespace Test std::filesystem::remove(localRoot / "doc2.txt"); BulkDownloadOperation::BulkDownloadOperationOptions options{.localPath = toLocal(localRoot)}; options.individualOptions.createMissingDirectories = createMissing; - BulkDownloadOperation operation{*sftp, std::move(options)}; + BulkDownloadOperation operation{sftp, std::move(options)}; operation.setPrescannedFileList(files); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -774,7 +817,7 @@ namespace Test auto scan = runScan(*sftp, "/home/test/Documents"); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(downloadDir_.path() / "cancel")} + sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(downloadDir_.path() / "cancel")} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -816,7 +859,7 @@ namespace Test .sizeBytes = std::string{"Fake file content"}.size(), }); - BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot), .concurrency = concurrency}}; + BulkDownloadOperation operation{sftp, {.localPath = toLocal(localRoot), .concurrency = concurrency}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -878,7 +921,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .concurrency = 3} + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .concurrency = 3} }; operation.setScanResult(std::move(entries), 0); @@ -922,7 +965,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = false, .concurrency = 2} }; operation.setScanResult(std::move(entries), 0); @@ -968,7 +1011,7 @@ namespace Test ); BulkDownloadOperation operation{ - *sftp, + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = true, .concurrency = 2} }; operation.setScanResult(std::move(entries), 0); @@ -1026,7 +1069,7 @@ namespace Test bool monotonic = true; SharedData::BulkProgress last{}; BulkDownloadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&](SharedData::BulkProgress const& update) @@ -1077,7 +1120,7 @@ namespace Test std::size_t maxInFlight = 0; BulkDownloadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&maxInFlight](SharedData::BulkProgress const& update) @@ -1111,7 +1154,7 @@ namespace Test // The scan root carries no permissions, applying "unknown" used to chmod it to 07777. const auto localRoot = downloadDir_.path() / "inherited"; BulkDownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot), @@ -1127,5 +1170,53 @@ namespace Test EXPECT_EQ(mode & std::filesystem::perms::others_write, std::filesystem::perms::none); EXPECT_TRUE(std::filesystem::exists(localRoot / "doc1.txt")); } + + TEST_F(BulkDownloadOperationTests, AReadOnlyDirectoryStillReceivesItsContent) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + using std::filesystem::perms; + const auto readOnly = perms::owner_read | perms::owner_exec | perms::group_read | perms::group_exec | + perms::others_read | perms::others_exec; + std::vector entries; + entries.push_back(SharedData::DirectoryEntry{.type = SharedData::FileType::Directory}); + entries.push_back( + SharedData::DirectoryEntry{ + .name = "Documents", + .type = SharedData::FileType::Directory, + .permissions = readOnly, + .parent = 0, + } + ); + entries.push_back( + SharedData::DirectoryEntry{ + .name = "doc1.txt", + .type = SharedData::FileType::Regular, + .size = std::string{"Document 1 content"}.size(), + .parent = 1, + } + ); + + const auto localRoot = downloadDir_.path() / "readonly"; + auto restore = Nui::ScopeExit{[&localRoot]() noexcept { + std::error_code error{}; + std::filesystem::permissions(localRoot / "Documents", perms::owner_all, error); + }}; + BulkDownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test"), + .localPath = toLocal(localRoot), + .individualOptions = {.inheritPermissions = true}, + } + }; + operation.setScanResult(std::move(entries), std::string{"Document 1 content"}.size()); + ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); + + EXPECT_TRUE(operation.getFailed().empty()); + EXPECT_EQ(readLocalFile(localRoot / "Documents" / "doc1.txt"), "Document 1 content"); + EXPECT_EQ(std::filesystem::status(localRoot / "Documents").permissions(), readOnly & ~Utility::processUmask()); + } #endif } diff --git a/backend/test/backend/test_bulk_upload_operation.hpp b/backend/test/backend/test_bulk_upload_operation.hpp index e56fa64e..d46b6718 100644 --- a/backend/test/backend/test_bulk_upload_operation.hpp +++ b/backend/test/backend/test_bulk_upload_operation.hpp @@ -106,14 +106,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {}}; + BulkUploadOperation operation{sftp, {}}; } TEST_F(BulkUploadOperationTests, BulkUploadOperationTypeIsBulkUpload) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {}}; + BulkUploadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::BulkUpload); } @@ -121,7 +121,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {}}; + BulkUploadOperation operation{sftp, {}}; EXPECT_FALSE(operation.isBarrier()); } @@ -131,7 +131,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/empty_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/empty_bulk"), .localPath = toLocal(uploadSrc_.path())} }; // A scanned tree always holds its root, an empty one means the scan never delivered. operation.setScanResult({}, 0); @@ -146,7 +146,7 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {}}; + BulkUploadOperation operation{sftp, {}}; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); EXPECT_EQ(workResult.error().type, SharedData::OperationErrorType::ImplementationError); @@ -158,7 +158,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/bad_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/bad_bulk"), .localPath = toLocal(uploadSrc_.path())} }; std::vector entries; @@ -186,7 +186,7 @@ namespace Test const auto remoteRoot = std::filesystem::path{"/home/test/bulk_uploaded"}; BulkUploadOperation operation{ - *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -231,7 +231,7 @@ namespace Test writeFile(uploadSrc_.path() / "subdir" / "file1.txt", "Bulk upload file one"); BulkUploadOperation operation{ - *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(entries), std::string{"Bulk upload file one"}.size()); @@ -272,7 +272,7 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path()), .failFast = false} @@ -320,7 +320,7 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path()), .failFast = true} @@ -373,7 +373,7 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(entries), 0); @@ -414,7 +414,7 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(entries), 0); @@ -436,7 +436,7 @@ namespace Test int callbackInvocations = 0; BulkUploadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&callbackInvocations](SharedData::BulkProgress const&) @@ -461,7 +461,7 @@ namespace Test auto scan = runLocalScan(uploadSrc_.path()); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/completed_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/completed_bulk"), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -478,7 +478,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/failed_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/failed_bulk"), .localPath = toLocal(uploadSrc_.path())} }; std::vector entries; @@ -506,7 +506,7 @@ namespace Test auto scan = runLocalScan(uploadSrc_.path()); BulkUploadOperation operation{ - *sftp, {.remotePath = remote("/home/test/cancel_bulk"), .localPath = toLocal(uploadSrc_.path())} + sftp, {.remotePath = remote("/home/test/cancel_bulk"), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -545,7 +545,7 @@ namespace Test std::size_t maxInFlight = 0; BulkUploadOperation operation{ - *sftp, + sftp, { .overallProgressCallback = [&maxInFlight](SharedData::BulkProgress const& update) @@ -570,6 +570,46 @@ namespace Test EXPECT_EQ(maxInFlight, 3u); } + TEST_F(BulkUploadOperationTests, AResumedBulkCountsWhatAnEarlierConnectionFinished) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + writeFile(uploadSrc_.path() / "rest.txt", "rest"); + std::vector updates; + BulkUploadOperation operation{ + sftp, + { + .overallProgressCallback = + [&updates](SharedData::BulkProgress const& update) + { + updates.push_back(update); + }, + .remotePath = remote("/home/test"), + .localPath = toLocal(uploadSrc_.path()), + .progressEmitInterval = std::chrono::milliseconds{0}, + } + }; + operation.setPrescannedFileList({{ + .source = toLocal(uploadSrc_.path() / "rest.txt"), + .destination = remote("/home/test/rest.txt"), + .sizeBytes = 4, + }}); + operation.countEarlierProgress({.entries = 2, .bytes = 10}); + + ASSERT_EQ(runBulkToCompletion(operation), BulkUploadOperation::WorkStatus::Complete); + ASSERT_FALSE(updates.empty()); + EXPECT_EQ(updates.front().fileCount, 3u); + EXPECT_EQ(updates.front().fileCurrentIndex, 2u); + EXPECT_EQ(updates.front().bytesCurrent, 10u); + EXPECT_EQ(updates.front().bytesTotal, 14u); + EXPECT_EQ(updates.back().fileCurrentIndex, 3u); + EXPECT_EQ(updates.back().bytesCurrent, 14u); + const auto finished = operation.finishedProgress(); + EXPECT_EQ(finished.entries, 3u); + EXPECT_EQ(finished.bytes, 14u); + } + TEST_F(BulkUploadOperationTests, DirectoriesAreCreatedBeforeTheirFilesWithConcurrency) { // Two-level local tree scanned for real, uploaded with more files than slots. @@ -589,7 +629,7 @@ namespace Test const std::filesystem::path remoteRoot = "/home/test/tree_concurrent"; BulkUploadOperation operation{ - *sftp, + sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path()), .concurrency = 3} @@ -628,7 +668,7 @@ namespace Test }); BulkUploadOperation operation{ - *sftp, + sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(uploadSrc_.path()), .failFast = false, diff --git a/backend/test/backend/test_compared_state_check.hpp b/backend/test/backend/test_compared_state_check.hpp new file mode 100644 index 00000000..684eb909 --- /dev/null +++ b/backend/test/backend/test_compared_state_check.hpp @@ -0,0 +1,293 @@ +#pragma once + +#include +#include +#include +#include "test_transfer_failures.hpp" + +#include + +#include +#include +#include + +namespace Test +{ + class ComparedStateCheckTests : public TransferFailureTests + { + public: + static SharedData::ComparedState localNow(std::filesystem::path const& path) + { + return ComparedStateCheck::localStateOf(path).value(); + } + + static SharedData::ComparedState remoteNow(SecureShell::SftpSession& sftp, std::string const& path) + { + return sftp + .performPromise( + [&sftp, path = remote(path)]() + { + return ComparedStateCheck::remoteStateOfInStrand(sftp, path); + } + ) + .get() + .value(); + } + + static std::expected checkRemote( + SecureShell::SftpSession& sftp, + std::string const& path, + SharedData::ComparedState const& compared + ) + { + return sftp + .performPromise( + [&sftp, path = remote(path), &compared]() + { + return ComparedStateCheck::requireUnchangedRemoteInStrand(sftp, path, compared); + } + ) + .get(); + } + + static void touchLater(std::filesystem::path const& path) + { + std::filesystem::last_write_time(path, std::filesystem::last_write_time(path) + std::chrono::hours{1}); + } + }; + + TEST_F(ComparedStateCheckTests, LocalStatesDescribeFilesDirectoriesAndAbsence) + { + const auto file = localDirectory_.path() / "file.txt"; + writeLocalFile(file, "12345"); + const auto state = localNow(file); + EXPECT_TRUE(state.exists); + EXPECT_EQ(state.type, SharedData::FileType::Regular); + EXPECT_EQ(state.size, 5u); + EXPECT_EQ(state.mtime, ComparedStateCheck::secondsSinceEpoch(std::filesystem::last_write_time(file))); + + EXPECT_EQ(localNow(localDirectory_.path()).type, SharedData::FileType::Directory); + EXPECT_FALSE(localNow(localDirectory_.path() / "missing").exists); + EXPECT_FALSE(localNow(localDirectory_.path() / "missing" / "below").exists); + } + +#ifndef _WIN32 + TEST_F(ComparedStateCheckTests, ALocalSymlinkIsDescribedByItsLiteral) + { + const auto link = localDirectory_.path() / "link"; + std::filesystem::create_symlink("does/not/exist", link); + const auto state = localNow(link); + EXPECT_EQ(state.type, SharedData::FileType::Symlink); + EXPECT_EQ(state.linkTarget, "does/not/exist"); + + const auto compared = state; + std::filesystem::remove(link); + std::filesystem::create_symlink("elsewhere", link); + const auto checked = ComparedStateCheck::requireUnchangedLocal(link, compared); + ASSERT_FALSE(checked); + EXPECT_EQ(checked.error().type, Operation::ErrorType::ChangedSinceCompare); + } +#endif + + TEST_F(ComparedStateCheckTests, TheLocalCheckPassesOnlyWhatTheCompareSaw) + { + const auto file = localDirectory_.path() / "file.txt"; + writeLocalFile(file, "12345"); + const auto compared = localNow(file); + EXPECT_TRUE(ComparedStateCheck::requireUnchangedLocal(file, compared)) << "unchanged"; + EXPECT_TRUE(ComparedStateCheck::requireUnchangedLocal(file, std::nullopt)) << "nothing to check"; + + const auto expectChanged = [&](SharedData::ComparedState const& expected, std::string const& what) + { + const auto checked = ComparedStateCheck::requireUnchangedLocal(file, expected); + ASSERT_FALSE(checked) << what; + EXPECT_EQ(checked.error().type, Operation::ErrorType::ChangedSinceCompare) << what; + }; + const auto time = std::filesystem::last_write_time(file); + writeLocalFile(file, "123456"); + std::filesystem::last_write_time(file, time); + expectChanged(compared, "another size"); + + writeLocalFile(file, "12345"); + std::filesystem::last_write_time(file, time); + EXPECT_TRUE(ComparedStateCheck::requireUnchangedLocal(file, compared)) << "restored"; + touchLater(file); + expectChanged(compared, "another time"); + + expectChanged(SharedData::ComparedState{}, "appeared where the compare saw nothing"); + std::filesystem::remove(file); + EXPECT_TRUE(ComparedStateCheck::requireUnchangedLocal(file, compared)) << "gone loses nothing"; + std::filesystem::create_directory(file); + expectChanged(compared, "replaced by a directory"); + } + + TEST_F(ComparedStateCheckTests, TheRemoteCheckPassesOnlyWhatTheCompareSaw) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/home/test/link"}, {"target", "file1.txt"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto compared = remoteNow(*sftp, "/home/test/file1.txt"); + EXPECT_EQ(compared.type, SharedData::FileType::Regular); + EXPECT_EQ(compared.size, std::string{"Fake file content"}.size()); + EXPECT_TRUE(checkRemote(*sftp, "/home/test/file1.txt", compared)); + + const auto link = remoteNow(*sftp, "/home/test/link"); + EXPECT_EQ(link.type, SharedData::FileType::Symlink); + EXPECT_EQ(link.linkTarget, "file1.txt"); + EXPECT_FALSE(remoteNow(*sftp, "/home/test/missing").exists); + + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/file1.txt"}, {"content", "changed"}})); + const auto changed = checkRemote(*sftp, "/home/test/file1.txt", compared); + ASSERT_FALSE(changed); + EXPECT_EQ(changed.error().type, Operation::ErrorType::ChangedSinceCompare); + + const auto appeared = checkRemote(*sftp, "/home/test/file1.txt", SharedData::ComparedState{}); + ASSERT_FALSE(appeared); + EXPECT_EQ(appeared.error().type, Operation::ErrorType::ChangedSinceCompare); + EXPECT_TRUE(checkRemote(*sftp, "/home/test/missing", compared)) << "gone loses nothing"; + } + + TEST_F(ComparedStateCheckTests, ADownloadNeverReplacesADestinationThatChangedAfterTheCompare) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto localPath = localDirectory_.path() / "file1.txt"; + writeLocalFile(localPath, "old local"); + const auto compared = localNow(localPath); + writeLocalFile(localPath, "edited locally meanwhile"); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .comparedDestination = compared, + } + }; + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::ChangedSinceCompare); + EXPECT_EQ(readLocalFile(localPath), "edited locally meanwhile"); + } + + TEST_F(ComparedStateCheckTests, ADownloadReplacesADestinationThatIsAsTheCompareSawIt) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto localPath = localDirectory_.path() / "file1.txt"; + writeLocalFile(localPath, "old local"); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .comparedDestination = localNow(localPath), + } + }; + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), "Fake file content"); + } + + TEST_F(ComparedStateCheckTests, AnUploadNeverReplacesADestinationThatChangedAfterTheCompare) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto localPath = localDirectory_.path() / "upload.txt"; + writeLocalFile(localPath, "new content"); + const auto compared = remoteNow(*sftp, "/home/test/file1.txt"); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/file1.txt"}, {"content", "edited"}})); + + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .comparedDestination = compared, + } + }; + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::ChangedSinceCompare); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "edited"); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt.filepart"), "") << "the temp file is removed"; + } + + TEST_F(ComparedStateCheckTests, AnUploadReplacesADestinationThatIsAsTheCompareSawIt) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto localPath = localDirectory_.path() / "upload.txt"; + writeLocalFile(localPath, "new content"); + + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .comparedDestination = remoteNow(*sftp, "/home/test/file1.txt"), + } + }; + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "new content"); + } + + TEST_F(ComparedStateCheckTests, ALocalDeleteLeavesATargetThatChangedAfterTheCompare) + { + const auto changed = localDirectory_.path() / "changed.txt"; + const auto unchanged = localDirectory_.path() / "unchanged.txt"; + writeLocalFile(changed, "a"); + writeLocalFile(unchanged, "b"); + const auto comparedChanged = localNow(changed); + const auto comparedUnchanged = localNow(unchanged); + writeLocalFile(changed, "edited"); + + LocalDeleteOperation operation{{ + .targets = {toLocal(changed), toLocal(unchanged)}, + .comparedStates = {comparedChanged, comparedUnchanged}, + }}; + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_TRUE(std::filesystem::exists(changed)); + EXPECT_FALSE(std::filesystem::exists(unchanged)); + ASSERT_EQ(operation.getFailed().size(), 1u); + EXPECT_EQ(operation.getFailed().front().second.type, Operation::ErrorType::ChangedSinceCompare); + } + + TEST_F(ComparedStateCheckTests, ARemoteDeleteLeavesATargetThatChangedAfterTheCompare) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/changed.txt"}, {"content", "a"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/unchanged.txt"}, {"content", "b"}})); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/home/test/link"}, {"target", "a"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto comparedChanged = remoteNow(*sftp, "/home/test/changed.txt"); + const auto comparedUnchanged = remoteNow(*sftp, "/home/test/unchanged.txt"); + auto comparedLink = remoteNow(*sftp, "/home/test/link"); + comparedLink.linkTarget = "b"; + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/changed.txt"}, {"content", "edited"}})); + + DeleteOperation operation{sftp, {.recursive = true}}; + operation.setTargets({ + {.path = remote("/home/test/changed.txt"), .compared = comparedChanged}, + {.path = remote("/home/test/unchanged.txt"), .compared = comparedUnchanged}, + {.path = remote("/home/test/link"), .compared = comparedLink}, + {.path = remote("/home/test/gone.txt"), .compared = comparedUnchanged}, + }); + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/changed.txt"), "edited"); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/unchanged.txt"), ""); + EXPECT_TRUE(remoteNow(*sftp, "/home/test/link").exists); + ASSERT_EQ(operation.getFailed().size(), 2u) << "a gone target counts as deleted"; + for (auto const& [path, error] : operation.getFailed()) + EXPECT_EQ(error.type, Operation::ErrorType::ChangedSinceCompare) << path; + } +} diff --git a/backend/test/backend/test_download_operation.hpp b/backend/test/backend/test_download_operation.hpp index 761f73b6..6e81ee16 100644 --- a/backend/test/backend/test_download_operation.hpp +++ b/backend/test/backend/test_download_operation.hpp @@ -41,14 +41,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - DownloadOperation operation{*sftp, {}}; + DownloadOperation operation{sftp, {}}; } TEST_F(DownloadOperationTests, DownloadOperationTypeIsDownload) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - DownloadOperation operation{*sftp, {}}; + DownloadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::Download); } @@ -60,7 +60,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -82,7 +82,7 @@ namespace Test for (auto const& remotePath : {Paths::RemotePath{}, Paths::RemotePath::root()}) { DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remotePath, .localPath = toLocal(localPath), @@ -103,7 +103,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -148,7 +148,7 @@ namespace Test entry.mtimeNsec = expectedMtimeNsec; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -185,7 +185,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -217,7 +217,7 @@ namespace Test } DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -240,7 +240,7 @@ namespace Test // First download { DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -262,7 +262,7 @@ namespace Test // Second download with overwrite enabled { DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -292,7 +292,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/does_not_exist.txt"), .localPath = toLocal(downloadDir_.path() / "does_not_exist.txt"), @@ -310,7 +310,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = {}, @@ -328,7 +328,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "file1.txt"), @@ -357,7 +357,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/does_not_exist.txt"), .localPath = toLocal(downloadDir_.path() / "does_not_exist.txt"), @@ -380,7 +380,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -404,7 +404,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -435,7 +435,7 @@ namespace Test } DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -473,7 +473,7 @@ namespace Test } DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -511,7 +511,7 @@ namespace Test } DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -542,7 +542,7 @@ namespace Test const auto localPath = downloadDir_.path() / "large.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -565,6 +565,52 @@ namespace Test EXPECT_EQ(std::filesystem::file_size(localPath), 65536u); } + TEST_F(DownloadOperationTests, ADownloadWhoseChannelClosesFailsWithChannelClosed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = downloadDir_.path() / "large.txt"; + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + } + }; + const auto firstStep = operation.work(); + ASSERT_TRUE(firstStep.has_value()) << firstStep.error().toString(); + ASSERT_EQ(*firstStep, DownloadOperation::WorkStatus::MoreWork); + + sftp->close(); + const auto afterClose = operation.work(); + ASSERT_FALSE(afterClose.has_value()); + EXPECT_EQ(afterClose.error().type, Operation::ErrorType::ChannelClosed); + EXPECT_EQ(operation.state(), Operation::OperationState::Failed); + EXPECT_FALSE(std::filesystem::exists(localPath)); + } + + TEST_F(DownloadOperationTests, ADownloadOfAGoneSessionFailsWithChannelClosed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(downloadDir_.path() / "file1.txt"), + } + }; + sftp->close(); + sftp.reset(); + + EXPECT_EQ(operation.strand(), nullptr); + const auto result = operation.workInStrand(); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::ChannelClosed); + } + // ---- Symlink handling -------------------------------------------------- // // Remote symlinks must be recreated as local symlinks, not downloaded as files. @@ -601,7 +647,7 @@ namespace Test const auto localPath = downloadDir_.path() / "link_to_file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_file1.txt"), .localPath = toLocal(localPath), @@ -616,6 +662,43 @@ namespace Test EXPECT_FALSE(std::filesystem::is_regular_file(std::filesystem::symlink_status(localPath))); } + TEST_F(DownloadOperationTests, DownloadRemoteSymlinkReplacesOnlyWhenAllowed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + for (const bool mayOverwrite : {false, true}) + { + const auto localPath = downloadDir_.path() / fmt::format("existing_{}", mayOverwrite); + std::ofstream{localPath} << "existing"; + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/link_to_file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = mayOverwrite, + } + }; + const auto result = [&operation]() + { + for (;;) + { + auto step = operation.work(); + if (!step.has_value() || step.value() == DownloadOperation::WorkStatus::Complete) + return step; + } + }(); + ASSERT_EQ(result.has_value(), mayOverwrite); + if (!mayOverwrite) + EXPECT_EQ(result.error().type, Operation::ErrorType::FileExists); + EXPECT_EQ(std::filesystem::is_symlink(std::filesystem::symlink_status(localPath)), mayOverwrite); + if (mayOverwrite) + EXPECT_EQ(std::filesystem::read_symlink(localPath), "/home/test/file1.txt"); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::symlink_status(localPath.string() + ".filepart"))); + } + } + TEST_F(DownloadOperationTests, DownloadRemoteSymlinkToDirectoryCreatesLocalSymlink) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -624,7 +707,7 @@ namespace Test const auto localPath = downloadDir_.path() / "link_to_documents"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_documents"), .localPath = toLocal(localPath), @@ -646,7 +729,7 @@ namespace Test const auto localPath = downloadDir_.path() / "skipped_link"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_file1.txt"), .localPath = toLocal(localPath), @@ -679,7 +762,7 @@ namespace Test ASSERT_FALSE(std::filesystem::exists(localPath.parent_path())); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -701,7 +784,7 @@ namespace Test const auto localPath = downloadDir_.path() / "a" / "b" / "c" / "d" / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -726,7 +809,7 @@ namespace Test const auto localPath = downloadDir_.path() / "nonexistent" / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -752,7 +835,7 @@ namespace Test const auto localPath = downloadDir_.path() / "file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -772,7 +855,7 @@ namespace Test const auto localPath = downloadDir_.path() / "nested" / "deep" / "link_to_file1.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_file1.txt"), .localPath = toLocal(localPath), @@ -795,7 +878,7 @@ namespace Test const auto localPath = downloadDir_.path() / "dangling_link"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/dangling_link"), .localPath = toLocal(localPath), diff --git a/backend/test/backend/test_dropped_entry.hpp b/backend/test/backend/test_dropped_entry.hpp new file mode 100644 index 00000000..5db8c017 --- /dev/null +++ b/backend/test/backend/test_dropped_entry.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include + +#include + +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + TEST(DroppedEntryTests, DescribesWhatLiesAtThePathItself) + { + const Utility::TemporaryDirectory directory{temporaryRoot, true}; + std::filesystem::create_directories(directory.path() / "folder"); + std::ofstream{directory.path() / "file.txt"} << "x"; + + const auto file = droppedItemFromPath(directory.path() / "file.txt"); + ASSERT_TRUE(file); + EXPECT_EQ(file->entry.type, SharedData::FileType::Regular); + EXPECT_EQ(file->entry.name, "file.txt"); + const auto folder = droppedItemFromPath(directory.path() / "folder"); + ASSERT_TRUE(folder); + EXPECT_EQ(folder->entry.type, SharedData::FileType::Directory); + + std::error_code linkError{}; + std::filesystem::create_directory_symlink(directory.path() / "folder", directory.path() / "link", linkError); + if (!linkError) + { + const auto link = droppedItemFromPath(directory.path() / "link"); + ASSERT_TRUE(link); + EXPECT_EQ(link->entry.type, SharedData::FileType::Symlink) << "a dropped link is not followed"; + } + } + + TEST(DroppedEntryTests, RefusesWhatCannotBeDescribed) + { + const Utility::TemporaryDirectory directory{temporaryRoot, true}; + EXPECT_FALSE(droppedItemFromPath(directory.path() / "missing")); + EXPECT_FALSE(droppedItemFromPath("relative/path")); + } +} diff --git a/backend/test/backend/test_existence_check.hpp b/backend/test/backend/test_existence_check.hpp index a0aec4f6..5f6d1247 100644 --- a/backend/test/backend/test_existence_check.hpp +++ b/backend/test/backend/test_existence_check.hpp @@ -131,19 +131,65 @@ namespace Test EXPECT_EQ(*result, expected); } - TEST_F(ExistenceCheckTests, RemoteListingFailureFailsTheCheck) + TEST_F(ExistenceCheckTests, RemoteDirectoryThatCannotBeListedIsCheckedPathByPath) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); + // A drop box: writable, but not listable. ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); - std::vector paths; + std::vector paths{"/home/test/Documents/doc1.txt"}; + std::vector expected{true}; for (int index = 0; index < 12; ++index) + { paths.push_back("/home/test/Documents/new" + std::to_string(index) + ".txt"); + expected.push_back(false); + } const auto result = remotePathsExist(*sftp, paths, timeout); - EXPECT_FALSE(result.has_value()) << "an unknown destination must never count as absent"; + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(*result, expected); + } + + TEST_F(ExistenceCheckTests, RemoteNameListedOnlyInAnotherCaseIsLookedUp) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + // The lookup of the variant fails here, which shows it happens: a listed exact name needs none. + ASSERT_TRUE(serverStartResult->command("denyStat", {{"path", "/home/test/FILE1.txt"}})); + + std::vector paths{"/home/test/file1.txt"}; + for (int index = 0; index < 10; ++index) + paths.push_back("/home/test/new" + std::to_string(index) + ".txt"); + const auto exact = remotePathsExist(*sftp, paths, timeout); + ASSERT_TRUE(exact.has_value()) << exact.error(); + EXPECT_TRUE(exact->front()); + + paths.front() = "/home/test/FILE1.txt"; + EXPECT_FALSE(remotePathsExist(*sftp, paths, timeout).has_value()) + << "the server may fold case, so the variant is looked up instead of taken for absent"; + } + + TEST_F(ExistenceCheckTests, RemoteFewChildrenOfAParentAreStatEvenAmongManyPaths) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + // Listing it would fail, a stat of its one queried child does not. + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Pictures"}})); + + std::vector paths{"/home/test/Pictures/image1.png"}; + std::vector expected{true}; + for (int index = 0; index < 12; ++index) + { + paths.push_back("/home/test/new" + std::to_string(index) + ".txt"); + expected.push_back(false); + } + + const auto result = remotePathsExist(*sftp, paths, timeout); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(*result, expected); } TEST_F(ExistenceCheckTests, RemoteInvalidPathFailsTheCheck) diff --git a/backend/test/backend/test_file_change_filter.hpp b/backend/test/backend/test_file_change_filter.hpp index a4844ba9..42223bdf 100644 --- a/backend/test/backend/test_file_change_filter.hpp +++ b/backend/test/backend/test_file_change_filter.hpp @@ -38,6 +38,8 @@ namespace Test "archive.sb-12345678-short", "DS_Store", "filepart.txt", + "Ärger.txt", + "東京/報告 \xF0\x9F\x9A\x80.pdf", } ) EXPECT_FALSE(isTransientFile(name, filepart)) << name; diff --git a/backend/test/backend/test_file_tracking.hpp b/backend/test/backend/test_file_tracking.hpp index 6ba8d6ac..4b09a684 100644 --- a/backend/test/backend/test_file_tracking.hpp +++ b/backend/test/backend/test_file_tracking.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -430,6 +431,18 @@ namespace Test EXPECT_EQ(watch->path(), subdir); } + TEST_F(TemporaryDirectoryInstanceTest, AddWatchReturnsValidHandleForANonAsciiSubdir) + { + auto inst = makeInstance(); + auto subdir = inst->instanceDir() / Paths::nativeFromUtf8("Größe 東京"); + fs::create_directory(subdir); + + auto watch = inst->addWatch(subdir); + ASSERT_TRUE(watch.has_value()); + EXPECT_TRUE(watch->isValid()); + EXPECT_EQ(watch->path(), subdir); + } + TEST_F(TemporaryDirectoryInstanceTest, AddWatchReturnsValidHandleForInstanceDirItself) { auto inst = makeInstance(); diff --git a/backend/test/backend/test_local_delete_guard.hpp b/backend/test/backend/test_local_delete_guard.hpp index 6dbabe27..4a7276ee 100644 --- a/backend/test/backend/test_local_delete_guard.hpp +++ b/backend/test/backend/test_local_delete_guard.hpp @@ -1,15 +1,20 @@ #pragma once #include +#include #include +#include +#include + +extern std::filesystem::path temporaryRoot; + namespace Test { class LocalDeleteGuardTests : public ::testing::Test { protected: - // POSIX rules on every host, the Windows rules are covered by the paths tests. void SetUp() override { previous_ = Paths::defaultLocalPlatform(); @@ -30,7 +35,7 @@ namespace Test ASSERT_TRUE(home.has_value()); const auto check = [&home](std::string const& path) { - return localDeleteTarget(path, *home).has_value(); + return localDeleteTarget(path, {*home}).has_value(); }; EXPECT_FALSE(check("/")); EXPECT_FALSE(check("/home")); @@ -41,6 +46,18 @@ namespace Test EXPECT_TRUE(check("/home/user/file.txt")); EXPECT_TRUE(check("/home/username")); EXPECT_TRUE(check("/tmp/x")); + EXPECT_TRUE(check("/tmp/a\\..\\b")) << "a backslash is a character on POSIX"; + } + + TEST_F(LocalDeleteGuardTests, EveryHomeIsProtected) + { + const std::vector homes{ + Paths::LocalPath::fromUtf8("/home/user", Paths::LocalPlatform::Posix).value(), + Paths::LocalPath::fromUtf8("/var/home/user", Paths::LocalPlatform::Posix).value(), + }; + EXPECT_FALSE(localDeleteTarget("/var/home", homes).has_value()); + EXPECT_FALSE(localDeleteTarget("/home", homes).has_value()); + EXPECT_TRUE(localDeleteTarget("/var/tmp", homes).has_value()); } TEST_F(LocalDeleteGuardTests, WindowsRootsHomeAndBackslashParentsAreRefused) @@ -50,7 +67,7 @@ namespace Test ASSERT_TRUE(home.has_value()); const auto check = [&home](std::string const& path) { - return localDeleteTarget(path, *home).has_value(); + return localDeleteTarget(path, {*home}).has_value(); }; EXPECT_FALSE(check("C:/")); EXPECT_FALSE(check("C:/Users")); @@ -61,9 +78,67 @@ namespace Test EXPECT_TRUE(check("D:/data")); } + TEST_F(LocalDeleteGuardTests, WindowsSpellingsOfTheHomeAreRefused) + { + Paths::defaultLocalPlatform() = Paths::LocalPlatform::Windows; + const auto home = Paths::LocalPath::fromUtf8("C:/Users/me").value(); + for (auto const* path : {"c:/users", "C:/USERS/ME", "\\\\?\\C:\\Users", "\\\\?\\c:\\users\\Me", "c:\\"}) + EXPECT_FALSE(localDeleteTarget(path, {home}).has_value()) << path; + EXPECT_EQ(localDeleteTarget("\\\\.\\C:\\Users", {home}).error(), "'\\\\.\\C:\\Users' is a device path"); + EXPECT_TRUE(localDeleteTarget("c:/users/me/x", {home}).has_value()); + EXPECT_TRUE(localDeleteTarget("c:/userss", {home}).has_value()); + } + + TEST_F(LocalDeleteGuardTests, RefusalsNameTheReason) + { + const auto home = Paths::LocalPath::fromUtf8("/home/user", Paths::LocalPlatform::Posix).value(); + EXPECT_EQ(localDeleteTarget("relative", {home}).error(), "'relative' is not absolute"); + EXPECT_EQ(localDeleteTarget("/a/\xFF", {home}).error(), "'/a/\\xFF' is not valid UTF-8"); + EXPECT_EQ(localDeleteTarget("/a/../b", {home}).error(), "'/a/../b' contains '..'"); + EXPECT_EQ(localDeleteTarget("/", {home}).error(), "'/' is a root directory"); + EXPECT_EQ( + localDeleteTarget("/home", {home}).error(), "'/home' is or contains the home directory '/home/user'" + ); + } + TEST_F(LocalDeleteGuardTests, WithoutAKnownHomeOnlyRootsAreRefused) { - EXPECT_FALSE(localDeleteTarget("/", std::nullopt).has_value()); - EXPECT_TRUE(localDeleteTarget("/home/user", std::nullopt).has_value()); + EXPECT_FALSE(localDeleteTarget("/", {}).has_value()); + EXPECT_TRUE(localDeleteTarget("/home/user", {}).has_value()); + } + +#ifndef _WIN32 + TEST_F(LocalDeleteGuardTests, TheSameDirectoryThroughASymlinkIsRefused) + { + const Utility::TemporaryDirectory directory{temporaryRoot / "localDeleteGuard"}; + const auto root = directory.path(); + std::filesystem::create_directories(root / "real" / "user" / "work"); + std::filesystem::create_directory_symlink(root / "real", root / "link"); + const auto home = Paths::LocalPath::make(root / "link" / "user").value(); + const auto check = [&home](std::filesystem::path const& path) + { + return refuseProtectedLocalDelete(Paths::LocalPath::make(path).value(), {home}).has_value(); + }; + + EXPECT_FALSE(check(root / "real")) << "an ancestor of the home reached without the link"; + EXPECT_FALSE(check(root / "real" / "user")); + EXPECT_FALSE(check(root / "link")); + EXPECT_TRUE(check(root / "real" / "user" / "work")); + EXPECT_TRUE(check(root / "missing")); + + // Deleting a link removes the link, never what it points to. + std::filesystem::create_directory_symlink(root / "real" / "user", root / "toHome"); + EXPECT_TRUE(check(root / "toHome")); + } + + TEST_F(LocalDeleteGuardTests, HomeDirectoriesComeFromTheEnvironmentAndPasswd) + { + const auto homes = localHomeDirectories(); + ASSERT_FALSE(homes.empty()); + if (auto const* home = std::getenv("HOME"); home != nullptr && *home != '\0') + EXPECT_EQ(homes.front().native(), std::filesystem::path{home}.lexically_normal()); + for (auto const& home : homes) + EXPECT_EQ(std::ranges::count(homes, home), 1); } +#endif } diff --git a/backend/test/backend/test_local_delete_operation.hpp b/backend/test/backend/test_local_delete_operation.hpp new file mode 100644 index 00000000..4e5701b1 --- /dev/null +++ b/backend/test/backend/test_local_delete_operation.hpp @@ -0,0 +1,264 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +# include +#endif + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class LocalDeleteOperationTests : public ::testing::Test + { + protected: + struct Progress + { + std::string item; + std::uint64_t current; + std::uint64_t total; + }; + + static Paths::LocalPath local(std::filesystem::path const& path) + { + return Paths::LocalPath::make(path).value(); + } + + LocalDeleteOperation::LocalDeleteOperationOptions options(std::vector targets, bool recursive) + { + return { + .itemsRemovedProgress = + [this](std::string const& item, std::uint64_t current, std::uint64_t total) + { + progress_.push_back({item, current, total}); + }, + .scanProgress = + [this](std::uint64_t, std::uint64_t entries) + { + scannedEntries_ = entries; + }, + .targets = std::move(targets), + .recursive = recursive, + }; + } + + /** + * @brief Works @p operation until it ends, at most @p maximumSteps times. + */ + static std::expected + workToEnd(LocalDeleteOperation& operation, int maximumSteps = 1000) + { + for (int step = 0; step < maximumSteps; ++step) + { + auto result = operation.work(); + if (!result || *result == Operation::WorkStatus::Complete) + return result; + } + return Operation::WorkStatus::MoreWork; + } + + void makeFiles(std::filesystem::path const& directory, int count) + { + std::filesystem::create_directories(directory); + for (int index = 0; index < count; ++index) + std::ofstream{directory / ("file" + std::to_string(index) + ".txt")} << index; + } + + static bool makeDirectoryLink(std::filesystem::path const& link, std::filesystem::path const& target) + { + std::error_code error; + std::filesystem::create_directory_symlink(target, link, error); + return !error; + } + + Utility::TemporaryDirectory root_{temporaryRoot, true}; + std::vector progress_{}; + std::uint64_t scannedEntries_{0}; + }; + + TEST_F(LocalDeleteOperationTests, CommonParentIsTheDeepestSharedDirectory) + { + const auto path = [](std::string const& text) + { + return Paths::LocalPath::fromUtf8(text, Paths::LocalPlatform::Posix).value(); + }; + const std::vector siblings{path("/a/b/c"), path("/a/b/d/e")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(siblings), path("/a/b")); + const std::vector single{path("/a/b")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(single), path("/a")); + const std::vector nested{path("/a/b/c"), path("/a/b")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(nested), path("/a")); + const std::vector apart{path("/a/x"), path("/b/y")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(apart), path("/")); + const std::vector root{path("/")}; + EXPECT_EQ(LocalDeleteOperation::commonParentOf(root), path("/")); + EXPECT_FALSE(LocalDeleteOperation::commonParentOf({})); + } + + TEST_F(LocalDeleteOperationTests, ALinkTargetIsRemovedAsTheLinkOnly) + { + const auto target = root_.path() / "target"; + makeFiles(target, 2); + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + std::filesystem::create_symlink(target / "file0.txt", root_.path() / "fileLink"); + + for (bool const recursive : {false, true}) + { + if (recursive) + { + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + std::filesystem::create_symlink(target / "file0.txt", root_.path() / "fileLink"); + } + LocalDeleteOperation operation{ + options({local(root_.path() / "link"), local(root_.path() / "fileLink")}, recursive) + }; + const auto result = workToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(*result, Operation::WorkStatus::Complete); + EXPECT_TRUE(operation.getFailed().empty()); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::symlink_status(root_.path() / "link"))); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::symlink_status(root_.path() / "fileLink"))); + EXPECT_TRUE(std::filesystem::exists(target / "file0.txt")) << "recursive: " << recursive; + EXPECT_TRUE(std::filesystem::exists(target / "file1.txt")) << "recursive: " << recursive; + } + } + + TEST_F(LocalDeleteOperationTests, AnItemAlreadyGoneCountsAsDeleted) + { + std::ofstream{root_.path() / "present.txt"} << "present"; + LocalDeleteOperation operation{ + options({local(root_.path() / "gone.txt"), local(root_.path() / "present.txt")}, false) + }; + + const auto result = workToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_TRUE(operation.getFailed().empty()); + EXPECT_EQ(operation.state(), SharedData::OperationState::Completed); + EXPECT_FALSE(std::filesystem::exists(root_.path() / "present.txt")); + ASSERT_FALSE(progress_.empty()); + EXPECT_EQ(progress_.back().current, 2u); + EXPECT_EQ(progress_.back().total, 2u); + } + + TEST_F(LocalDeleteOperationTests, EveryTargetFailingFailsTheOperation) + { + makeFiles(root_.path() / "first", 1); + makeFiles(root_.path() / "second", 1); + LocalDeleteOperation operation{options({local(root_.path() / "first"), local(root_.path() / "second")}, false)}; + + const auto result = workToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::DeleteFailed); + EXPECT_EQ(operation.state(), SharedData::OperationState::Failed); + ASSERT_EQ(operation.getFailed().size(), 2u); + for (auto const& [path, error] : operation.getFailed()) + EXPECT_EQ(error.extraInfo.value_or("").find(path), std::string::npos) << "the path is named once"; + EXPECT_TRUE(std::filesystem::exists(root_.path() / "first" / "file0.txt")); + } + + TEST_F(LocalDeleteOperationTests, CancelStopsBetweenEntries) + { + makeFiles(root_.path() / "many", 20); + auto settings = options({local(root_.path() / "many")}, true); + settings.stepBudget = std::chrono::milliseconds{0}; + LocalDeleteOperation operation{std::move(settings)}; + + // One entry per step: the count walks into the directory and over the first file first. + for (int step = 0; step < 25; ++step) + ASSERT_EQ(operation.work(), Operation::WorkStatus::MoreWork); + ASSERT_TRUE(operation.cancel(true).has_value()); + const auto afterCancel = operation.work(); + ASSERT_FALSE(afterCancel.has_value()); + EXPECT_EQ(afterCancel.error().type, Operation::ErrorType::CannotWorkCanceledOperation); + + std::size_t left = 0; + for ([[maybe_unused]] auto const& entry : std::filesystem::directory_iterator{root_.path() / "many"}) + ++left; + EXPECT_GT(left, 0u); + EXPECT_LT(left, 20u) << "some files went before the cancel"; + } + + TEST_F(LocalDeleteOperationTests, AnUnworkedDeleteHoldsAndThenContinues) + { + makeFiles(root_.path() / "paused" / "inner", 10); + makeFiles(root_.path() / "paused", 10); + auto settings = options({local(root_.path() / "paused")}, true); + settings.stepBudget = std::chrono::milliseconds{0}; + settings.progressInterval = std::chrono::hours{1}; + LocalDeleteOperation operation{std::move(settings)}; + + for (int step = 0; step < 30; ++step) + ASSERT_EQ(operation.work(), Operation::WorkStatus::MoreWork); + EXPECT_EQ(scannedEntries_, 22u) << "two directories and twenty files"; + EXPECT_TRUE(std::filesystem::exists(root_.path() / "paused")) << "a paused queue does not work it further"; + + const auto result = workToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_FALSE(std::filesystem::exists(root_.path() / "paused")); + EXPECT_LE(progress_.size(), 2u) << "progress is throttled, plus the final update"; + ASSERT_FALSE(progress_.empty()); + EXPECT_EQ(progress_.back().current, 22u); + EXPECT_EQ(progress_.back().total, 22u); + } + + TEST_F(LocalDeleteOperationTests, RecursiveDeleteRemovesNestedLinksWithoutFollowing) + { + const auto outside = root_.path() / "outside"; + makeFiles(outside, 3); + const auto tree = root_.path() / "tree"; + makeFiles(tree / "a" / "b", 2); + ASSERT_TRUE(makeDirectoryLink(tree / "a" / "b" / "out", outside)); + ASSERT_TRUE(makeDirectoryLink(tree / "top", outside)); + std::filesystem::create_symlink(outside / "file0.txt", tree / "a" / "fileLink"); + + LocalDeleteOperation operation{options({local(tree)}, true)}; + const auto result = workToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_TRUE(operation.getFailed().empty()); + EXPECT_FALSE(std::filesystem::exists(tree)); + for (int index = 0; index < 3; ++index) + EXPECT_TRUE(std::filesystem::exists(outside / ("file" + std::to_string(index) + ".txt"))); + EXPECT_EQ(scannedEntries_, 8u) << "tree, a, b, two files and three links"; + ASSERT_FALSE(progress_.empty()); + EXPECT_EQ(progress_.back().current, 8u); + } + +#ifndef _WIN32 + TEST_F(LocalDeleteOperationTests, AnUndeletableItemIsListedByItsOwnPath) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root removes entries of read-only directories"; + + const auto tree = root_.path() / "stuck"; + makeFiles(tree / "locked", 1); + makeFiles(tree, 3); + std::filesystem::permissions( + tree / "locked", std::filesystem::perms::owner_write, std::filesystem::perm_options::remove + ); + + LocalDeleteOperation operation{options({local(tree)}, true)}; + const auto result = workToEnd(operation); + std::filesystem::permissions( + tree / "locked", std::filesystem::perms::owner_write, std::filesystem::perm_options::add + ); + + ASSERT_TRUE(result.has_value()) << "a partial success completes"; + ASSERT_EQ(operation.getFailed().size(), 1u); + EXPECT_EQ(operation.getFailed().front().first, Paths::nativeToUtf8(tree / "locked" / "file0.txt")); + EXPECT_FALSE(std::filesystem::exists(tree / "file0.txt")); + EXPECT_TRUE(std::filesystem::exists(tree / "locked" / "file0.txt")); + } +#endif +} diff --git a/backend/test/backend/test_local_operation_queue.hpp b/backend/test/backend/test_local_operation_queue.hpp new file mode 100644 index 00000000..2cff746d --- /dev/null +++ b/backend/test/backend/test_local_operation_queue.hpp @@ -0,0 +1,368 @@ +#pragma once + +#include +#include +#include +#include +#include +#include "nui_env.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class LocalOperationQueueTests : public ::testing::Test + { + public: + void SetUp() override + { + if (!NuiEnv::instance().available()) + GTEST_SKIP() << "No display available; skipping LocalOperationQueue tests"; + auto& env = NuiEnv::instance(); + queue_ = std::make_shared( + pool_.get_executor(), strand_, *env.window, *env.hub, deletionPrevented_ + ); + queue_->setCompletionObserver( + [this](SharedData::OperationCompleted const& completed) + { + std::scoped_lock lock{completionsMutex_}; + completions_.push_back(completed); + } + ); + } + + void TearDown() override + { + queue_.reset(); + } + + template + auto runOnStrand(FunctionT&& function) -> std::invoke_result_t + { + using ResultT = std::invoke_result_t; + auto promise = std::make_shared>(); + auto future = promise->get_future(); + boost::asio::post( + *strand_, + [promise, function = std::forward(function)]() mutable + { + if constexpr (std::is_void_v) + { + function(); + promise->set_value(); + } + else + promise->set_value(function()); + } + ); + return future.get(); + } + + std::expected add(LocalOperationQueue::DeleteRequest request) + { + return runOnStrand( + [this, &request]() + { + return queue_->addDelete(std::move(request)); + } + ); + } + + /** + * @brief The completions once @p expected arrived, the queue working on its own; fails after a while. + */ + std::vector waitForCompletions(std::size_t expected) + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{20}; + while (std::chrono::steady_clock::now() < deadline) + { + if (auto finished = completions(); finished.size() >= expected) + return finished; + std::this_thread::sleep_for(std::chrono::milliseconds{2}); + } + ADD_FAILURE() << "Timed out waiting for " << expected << " completions"; + return completions(); + } + + /** + * @brief The completions after the queue had time to finish what it may run. + */ + std::vector completionsAfterSettling() + { + for (int cycle = 0; cycle < 20; ++cycle) + { + runOnStrand([]() {}); + std::this_thread::sleep_for(std::chrono::milliseconds{5}); + } + return completions(); + } + + std::vector completions() + { + std::scoped_lock lock{completionsMutex_}; + return completions_; + } + + static Paths::LocalPath local(std::filesystem::path const& path) + { + return Paths::LocalPath::make(path).value(); + } + + std::filesystem::path makeTree(std::string const& name) + { + const auto tree = root_.path() / name; + std::filesystem::create_directories(tree / "inner"); + std::ofstream{tree / "inner" / "a.txt"} << "a"; + std::ofstream{tree / "b.txt"} << "b"; + return tree; + } + + protected: + boost::asio::thread_pool pool_{2}; + std::shared_ptr> strand_{ + std::make_shared>(pool_.get_executor()) + }; + std::shared_ptr deletionPrevented_{std::make_shared(false)}; + std::shared_ptr queue_{}; + Ids::SessionId owner_{Ids::makeSessionId("local-operation-queue-test")}; + std::mutex completionsMutex_{}; + std::vector completions_{}; + Utility::TemporaryDirectory root_{temporaryRoot, true}; + }; + + TEST_F(LocalOperationQueueTests, AFlatDeleteRemovesExactlyItsTargets) + { + const auto root = root_.path() / "flat"; + std::filesystem::create_directories(root / "directory" / "inner"); + std::ofstream{root / "directory" / "listed.txt"} << "listed"; + std::ofstream{root / "directory" / "inner" / "unlisted.txt"} << "unlisted"; + std::ofstream{root / "file.txt"} << "file"; + std::ofstream{root / "keep.txt"} << "keep"; + + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE(add({ + .operationId = remove, + .targets = + {local(root / "directory"), + local(root / "directory" / "inner"), + local(root / "directory" / "listed.txt"), + local(root / "file.txt")}, + }) + .has_value()); + + const auto finished = waitForCompletions(1); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, remove); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(finished.front().failedEntries.size(), 2u) << "inner and directory are not empty"; + EXPECT_TRUE(std::filesystem::exists(root / "directory" / "inner" / "unlisted.txt")); + EXPECT_FALSE(std::filesystem::exists(root / "directory" / "listed.txt")); + EXPECT_FALSE(std::filesystem::exists(root / "file.txt")); + EXPECT_TRUE(std::filesystem::exists(root / "keep.txt")); + } + +#ifndef _WIN32 + TEST_F(LocalOperationQueueTests, ARecursiveDeleteRemovesTheTreeWithoutFollowingLinks) + { + const auto outside = root_.path() / "outside"; + std::filesystem::create_directories(outside); + std::ofstream{outside / "keep.txt"} << "keep"; + const auto tree = makeTree("recursive"); + std::filesystem::create_directory_symlink(outside, tree / "inner" / "out"); + + ASSERT_TRUE( + add({.operationId = Ids::generateOperationId(), .targets = {local(tree)}, .recursive = true}).has_value() + ); + + const auto finished = waitForCompletions(1); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(finished.front().failedEntries.empty()); + EXPECT_FALSE(std::filesystem::exists(tree)); + EXPECT_TRUE(std::filesystem::exists(outside / "keep.txt")); + } +#endif + + TEST_F(LocalOperationQueueTests, AHeldDeleteWaitsUntilItsOwnerContinuesOrEnds) + { + const auto first = makeTree("first"); + const auto second = makeTree("second"); + const auto unowned = makeTree("unowned"); + const auto firstId = Ids::generateOperationId(); + const auto secondId = Ids::generateOperationId(); + const auto unownedId = Ids::generateOperationId(); + ASSERT_TRUE( + add({.operationId = firstId, .targets = {local(first)}, .recursive = true, .owner = owner_, .held = true}) + .has_value() + ); + ASSERT_TRUE( + add({.operationId = secondId, .targets = {local(second)}, .recursive = true, .owner = owner_, .held = true}) + .has_value() + ); + ASSERT_TRUE(add({.operationId = unownedId, .targets = {local(unowned)}, .recursive = true}).has_value()); + + auto finished = waitForCompletions(1); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, unownedId) << "a held delete does not stop the others"; + EXPECT_TRUE(std::filesystem::exists(first / "b.txt")); + + runOnStrand( + [this]() + { + queue_->hold(owner_, false); + queue_->hold(owner_, true); + } + ); + EXPECT_EQ(completionsAfterSettling().size(), 1u); + + runOnStrand( + [this]() + { + queue_->hold(owner_, false); + } + ); + finished = waitForCompletions(2); + ASSERT_GE(finished.size(), 2u); + EXPECT_EQ(finished[1].operationId, firstId); + + // A delete of a session that ended runs on its own and is not held again. + ASSERT_TRUE(add({.operationId = Ids::generateOperationId(), + .targets = {local(makeTree("third"))}, + .recursive = true, + .owner = owner_, + .held = true}) + .has_value()); + runOnStrand( + [this]() + { + queue_->hold(owner_, true); + queue_->release(owner_); + queue_->hold(owner_, true); + } + ); + finished = waitForCompletions(4); + EXPECT_EQ(finished.size(), 4u); + EXPECT_FALSE(std::filesystem::exists(second)); + EXPECT_FALSE(std::filesystem::exists(root_.path() / "third")); + } + + TEST_F(LocalOperationQueueTests, APriorityDeleteIsNeverHeldAndRunsFirst) + { + const auto queued = makeTree("queued"); + const auto priority = makeTree("priority"); + const auto queuedId = Ids::generateOperationId(); + const auto priorityId = Ids::generateOperationId(); + runOnStrand( + [&]() + { + EXPECT_TRUE(queue_->addDelete({.operationId = queuedId, .targets = {local(queued)}, .recursive = true}) + .has_value()); + EXPECT_TRUE(queue_ + ->addDelete({ + .operationId = priorityId, + .targets = {local(priority)}, + .recursive = true, + .mode = SharedData::OperationMode::PriorityQueued, + .owner = owner_, + .held = true, + }) + .has_value()); + } + ); + + const auto finished = waitForCompletions(2); + ASSERT_EQ(finished.size(), 2u); + EXPECT_EQ(finished[0].operationId, priorityId); + EXPECT_EQ(finished[1].operationId, queuedId); + } + + TEST_F(LocalOperationQueueTests, ACanceledDeleteEndsAsCanceledAndKeepsWhatIsLeft) + { + const auto tree = makeTree("canceled"); + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE( + add({.operationId = remove, .targets = {local(tree)}, .recursive = true, .owner = owner_, .held = true}) + .has_value() + ); + runOnStrand( + [&]() + { + queue_->cancel(remove); + queue_->cancel(remove); + } + ); + + const auto finished = completionsAfterSettling(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, remove); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Canceled); + EXPECT_TRUE(std::filesystem::exists(tree / "b.txt")); + } + + TEST_F(LocalOperationQueueTests, ADeleteOfARootOrTheHomeDirectoryIsRefused) + { + const auto file = root_.path() / "survivor.txt"; + std::ofstream{file} << "survivor"; + const auto root = local(std::filesystem::path{file}.root_path()); + const auto homes = localHomeDirectories(); + ASSERT_FALSE(homes.empty()); + + for (auto const& target : {root, homes.front()}) + { + const auto result = add({.operationId = Ids::generateOperationId(), .targets = {local(file), target}}); + ASSERT_FALSE(result.has_value()) << target.toUtf8(); + EXPECT_EQ(result.error().type, Operation::ErrorType::InvalidPath); + } + EXPECT_FALSE(add({.operationId = Ids::generateOperationId()}).has_value()) << "no targets"; + EXPECT_TRUE(completionsAfterSettling().empty()); + EXPECT_TRUE(std::filesystem::exists(file)); + } + + TEST_F(LocalOperationQueueTests, AnExplorerDeleteGoesWithTheContentUnlessRefused) + { + const auto tree = makeTree("explorer"); + const auto explorerDelete = [this](std::vector const& paths) + { + return runOnStrand( + [this, &paths]() + { + return queue_->addExplorerDelete(Ids::generateOperationId(), paths, std::nullopt, false); + } + ); + }; + + deletionPrevented_->store(true); + const auto prevented = explorerDelete({Paths::nativeToUtf8(tree)}); + ASSERT_FALSE(prevented.has_value()); + EXPECT_EQ(prevented.error(), "backend.rpcFilesystem.deletionPrevented"); + deletionPrevented_->store(false); + + const auto relative = explorerDelete({Paths::nativeToUtf8(tree), "relative/path"}); + ASSERT_FALSE(relative.has_value()); + EXPECT_EQ(Utility::parseLocalizedMessage(relative.error()).key, "backend.rpcFilesystem.removeRefused"); + EXPECT_TRUE(completionsAfterSettling().empty()); + EXPECT_TRUE(std::filesystem::exists(tree / "inner" / "a.txt")); + + ASSERT_TRUE(explorerDelete({Paths::nativeToUtf8(tree)}).has_value()); + const auto finished = waitForCompletions(1); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_FALSE(std::filesystem::exists(tree)); + } +} diff --git a/backend/test/backend/test_local_removal.hpp b/backend/test/backend/test_local_removal.hpp new file mode 100644 index 00000000..60db55df --- /dev/null +++ b/backend/test/backend/test_local_removal.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include +#include + + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class LocalRemovalTests : public ::testing::Test + { + protected: + std::string utf8(std::filesystem::path const& path) const + { + return Paths::nativeToUtf8(path); + } + + std::vector homes() const + { + return {Paths::LocalPath::make(root_.path() / "home").value()}; + } + + Utility::TemporaryDirectory root_{temporaryRoot, true}; + }; + + TEST_F(LocalRemovalTests, ARefusedPathRemovesNothing) + { + std::filesystem::create_directories(root_.path() / "home" / "user"); + std::ofstream{root_.path() / "file.txt"} << "file"; + const std::vector paths{utf8(root_.path() / "file.txt"), utf8(root_.path() / "home")}; + + const auto removal = removeLocalItems(paths, homes()); + ASSERT_FALSE(removal.has_value()); + EXPECT_EQ(removal.error().path, utf8(root_.path() / "home")); + EXPECT_FALSE(removal.error().reason.empty()); + EXPECT_TRUE(std::filesystem::exists(root_.path() / "file.txt")); + + const std::vector relative{"relative/path"}; + EXPECT_FALSE(removeLocalItems(relative, homes()).has_value()); + } + + TEST_F(LocalRemovalTests, AFailureDoesNotStopTheOthers) + { + std::filesystem::create_directories(root_.path() / "full"); + std::ofstream{root_.path() / "full" / "keep.txt"} << "keep"; + std::ofstream{root_.path() / "a.txt"} << "a"; + std::ofstream{root_.path() / "b.txt"} << "b"; + const std::vector paths{ + utf8(root_.path() / "a.txt"), utf8(root_.path() / "full"), utf8(root_.path() / "b.txt") + }; + + const auto removal = removeLocalItems(paths, homes()); + ASSERT_TRUE(removal.has_value()); + EXPECT_EQ(removal->removed, 2u); + ASSERT_EQ(removal->failures.size(), 1u) << "a directory with content is not removed without recursion"; + const auto& failure = removal->failures.front(); + const auto path = utf8(root_.path() / "full"); + EXPECT_TRUE(failure.starts_with(path + ": ")); + EXPECT_EQ(failure.find(path, path.size()), std::string::npos) << "the path is named once"; + EXPECT_FALSE(std::filesystem::exists(root_.path() / "a.txt")); + EXPECT_FALSE(std::filesystem::exists(root_.path() / "b.txt")); + EXPECT_TRUE(std::filesystem::exists(root_.path() / "full" / "keep.txt")); + } + + TEST_F(LocalRemovalTests, ALinkToADirectoryIsRemovedItself) + { + std::filesystem::create_directories(root_.path() / "target"); + std::ofstream{root_.path() / "target" / "keep.txt"} << "keep"; + std::filesystem::create_directory_symlink(root_.path() / "target", root_.path() / "link"); + + const std::vector link{utf8(root_.path() / "link")}; + EXPECT_EQ(removeLocalItems(link, homes()).value().removed, 1u); + EXPECT_FALSE(std::filesystem::exists(std::filesystem::symlink_status(root_.path() / "link"))); + EXPECT_TRUE(std::filesystem::exists(root_.path() / "target" / "keep.txt")); + } +} diff --git a/backend/test/backend/test_local_scan_operation.hpp b/backend/test/backend/test_local_scan_operation.hpp index 5c524ac9..cfed736e 100644 --- a/backend/test/backend/test_local_scan_operation.hpp +++ b/backend/test/backend/test_local_scan_operation.hpp @@ -378,6 +378,34 @@ namespace Test EXPECT_EQ(operation.strand(), nullptr); } +#if !defined(_WIN32) && !defined(__APPLE__) + TEST_F(LocalScanOperationTests, NamesThatAreNotUtf8AreShownButRejected) + { + const auto latin1 = scanRoot_.path() / std::filesystem::path{std::string{"Fran\xE7ois.txt"}}; + writeFile(latin1, "latin1"); + std::error_code error{}; + if (!std::filesystem::exists(latin1, error)) + GTEST_SKIP() << "The filesystem refuses names that are not UTF-8"; + + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = false}}; + ASSERT_EQ(runScanToCompletion(operation), LocalScanOperation::WorkStatus::Complete); + const auto rejected = operation.rejectedPaths(); + ASSERT_EQ(rejected.size(), 1u); + EXPECT_TRUE(Paths::isValidUtf8(rejected.front())); + EXPECT_TRUE(rejected.front().ends_with("/Fran\xEF\xBF\xBDois.txt")) << rejected.front(); + const auto entries = operation.ejectEntries(); + EXPECT_FALSE(hasEntryNamed(entries, "Fran\xEF\xBF\xBDois.txt")); + EXPECT_TRUE(hasEntryNamed(entries, "file1.txt")); + + LocalScanOperation treeScan{{.localPath = toLocal(scanRoot_.path()), .recursive = false, .buildTree = true}}; + ASSERT_EQ(runScanToCompletion(treeScan), LocalScanOperation::WorkStatus::Complete); + const auto tree = treeScan.ejectScanTree(); + auto const* node = SharedData::Sync::findScanNode(tree, "Fran\xEF\xBF\xBDois.txt"); + ASSERT_NE(node, nullptr); + EXPECT_EQ(node->availability, SharedData::Sync::Availability::InvalidName); + } +#endif + TEST_F(LocalScanOperationTests, BuildTreeNestsAndSortsEntries) { LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = true, .buildTree = true}}; diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index 790f255a..266494bb 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -1,7 +1,9 @@ #pragma once #include +#include #include +#include #include "nui_env.hpp" #include "real_server_tests.hpp" @@ -11,6 +13,7 @@ #include #include #include +#include #include #include @@ -19,6 +22,7 @@ #include #include #include +#include #include #ifndef _WIN32 @@ -46,6 +50,7 @@ namespace Test queue_->cancelAll(); }); queue_.reset(); + localQueue_.reset(); } std::shared_ptr makeQueue() @@ -57,16 +62,26 @@ namespace Test *env.window, *env.hub, Persistence::SftpOptions{}, - Ids::makeSessionId("operation-queue-test"), + sessionId_, 2 ); - queue_->setCompletionObserver( - [this](SharedData::OperationCompleted const& completed) - { - std::scoped_lock lock{completionsMutex_}; - completions_.push_back(completed); - } - ); + const auto observe = [this](SharedData::OperationCompleted const& completed) + { + std::scoped_lock lock{completionsMutex_}; + completions_.push_back(completed); + }; + queue_->setCompletionObserver(observe); + if (!localQueue_) + { + localQueue_ = std::make_shared( + queuePool_.get_executor(), + strand_, + *env.window, + *env.hub, + std::make_shared(false) + ); + localQueue_->setCompletionObserver(observe); + } return queue_; } @@ -96,10 +111,15 @@ namespace Test return future.get(); } + /** + * @brief Pauses the queue and holds its local deletes, as the session does. + */ void setPaused(bool pause) { queue_->paused(pause); - runOnStrand([]() {}); + runOnStrand([this, pause]() { + localQueue_->hold(sessionId_, pause); + }); } /** @@ -130,9 +150,47 @@ namespace Test return completions_; } - static std::set remoteNames(SecureShell::SftpSession& sftp, std::filesystem::path const& directory) + /** + * @brief What is at @p path on the server now, as a compare that ran right now would have seen it. + */ + static SharedData::ComparedState remoteStateNow(SecureShell::SftpSession& sftp, std::string const& path) + { + auto future = sftp.performPromise( + [&sftp, path = remote(path)]() + { + return ComparedStateCheck::remoteStateOfInStrand(sftp, path); + } + ); + return future.get().value(); + } + + /** + * @brief The resolved plan entry @p relKey of @p session, which the compare saw as what is at its target now. + */ + static SyncSession::ResolvedPlanEntry plannedNow( + SyncSession const& session, + SecureShell::SftpSession& sftp, + std::string relKey, + SharedData::Sync::Action action, + bool isDirectory, + std::uint64_t size = 0 + ) + { + using SharedData::Sync::Action; + auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); + const bool onLocalSide = action == Action::Download || action == Action::DeleteLocal; + auto compared = onLocalSide ? ComparedStateCheck::localStateOf(paths.local.native()).value() + : remoteStateNow(sftp, paths.remote.value()); + return SyncSession::ResolvedPlanEntry{ + .entry = {.relKey = std::move(relKey), .action = action, .sizeBytes = size, .isDirectory = isDirectory}, + .paths = std::move(paths), + .compared = std::move(compared), + }; + } + + static std::set remoteNames(SecureShell::SftpSession& sftp, std::string const& directory) { - auto future = sftp.listDirectory(remote(directory.generic_string())); + auto future = sftp.listDirectory(remote(directory)); if (future.wait_for(5s) != std::future_status::ready) return {""}; auto result = future.get(); @@ -219,7 +277,7 @@ namespace Test std::set names; std::error_code errorCode{}; for (auto const& entry : std::filesystem::directory_iterator{directory, errorCode}) - names.insert(entry.path().filename().generic_string()); + names.insert(Paths::nativeToUtf8(entry.path().filename())); return names; } @@ -227,7 +285,10 @@ namespace Test boost::asio::thread_pool queuePool_{2}; std::shared_ptr> strand_{ std::make_shared>(queuePool_.get_executor())}; + Ids::SessionId sessionId_{Ids::makeSessionId("operation-queue-test")}; std::shared_ptr queue_{}; + /** @brief Runs the local deletes of sync plans, on the strand of the queue. */ + std::shared_ptr localQueue_{}; std::mutex completionsMutex_{}; std::vector completions_{}; Utility::TemporaryDirectory localDirectory_{temporaryRoot, true}; @@ -325,7 +386,8 @@ namespace Test ASSERT_EQ(finished.front().failedEntries.size(), 1u) << "an unreadable directory is never a silent success"; EXPECT_EQ(finished.front().failedEntries.front().first, Paths::nativeToUtf8(localPath / "locked")); EXPECT_EQ(finished.front().failedEntries.front().second.type, SharedData::OperationErrorType::DirectoryUnreadable); - EXPECT_TRUE(remoteNames(*sftp, "/home/test/withLocked").contains("a.txt")); + EXPECT_EQ(remoteNames(*sftp, "/home/test/withLocked"), (std::set{"a.txt"})) + << "the unreadable directory is not created empty"; } TEST_F(OperationQueueTests, ArchiveUploadOfAnUnreadableSubdirectoryFails) @@ -525,6 +587,227 @@ namespace Test EXPECT_TRUE(names.contains("adopted_b.txt")); } + TEST_F(OperationQueueTests, AResumedBulkUploadLeavesOutTheFilesItFinished) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "partly"; + std::filesystem::create_directories(localPath); + SharedData::BulkUploadRequest request{}; + for (auto const* name : {"partly_a.txt", "partly_b.txt", "partly_c.txt"}) + { + std::ofstream{localPath / name} << name; + request.entries.push_back( + {.source = toLocal(localPath / name), + .destination = remote(fmt::format("/home/test/{}", name)), + .sizeBytes = std::string_view{name}.size()} + ); + } + const auto card = Ids::generateOperationId(); + runOnStrand( + [&]() + { + return queue->addBulkUploadOperation( + *sftp, + request, + [](std::size_t) + { + return Ids::generateOperationId(); + }, + card + ); + } + ); + + for (int step = 0; step < 50 && !remoteNames(*sftp, "/home/test").contains("partly_a.txt"); ++step) + runOnStrand( + [&]() + { + return queue->work(); + } + ); + ASSERT_TRUE(remoteNames(*sftp, "/home/test").contains("partly_a.txt")); + ASSERT_FALSE(remoteNames(*sftp, "/home/test").contains("partly_c.txt")) << "the bulk ran to its end"; + + BulkResumeRegistry registry; + tearDownInto(registry); + auto backup = registry.take(card); + ASSERT_TRUE(backup.has_value()); + auto const& resumed = std::get(backup->request); + ASSERT_FALSE(resumed.entries.empty()); + EXPECT_LT(resumed.entries.size(), 3u); + for (auto const& entry : resumed.entries) + EXPECT_NE(entry.destination.value(), "/home/test/partly_a.txt"); + EXPECT_EQ(backup->finished.entries, 3u - resumed.entries.size()) << "the card counts them as done"; + EXPECT_GE(backup->finished.bytes, std::string_view{"partly_a.txt"}.size()); + registry.store(card, *std::move(backup)); + + adoptInto(*sftp, registry, {card}); + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, card); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(finished.front().failedEntries.empty()) << finished.front().failedEntries.front().first; + const auto names = remoteNames(*sftp, "/home/test"); + for (auto const* name : {"partly_a.txt", "partly_b.txt", "partly_c.txt"}) + EXPECT_TRUE(names.contains(name)) << name; + } + + TEST_F(OperationQueueTests, AResumedSyncUploadKeepsWhatTheCompareSawAtItsDestinations) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "compared"; + std::filesystem::create_directories(localPath); + SharedData::BulkUploadRequest request{}; + for (auto const* name : {"compared_a.txt", "compared_b.txt", "compared_c.txt"}) + { + std::ofstream{localPath / name} << name; + request.entries.push_back( + {.source = toLocal(localPath / name), + .destination = remote(fmt::format("/home/test/{}", name)), + .sizeBytes = std::string_view{name}.size(), + .mayOverwrite = true, + .comparedDestination = SharedData::ComparedState{}} + ); + } + const auto card = Ids::generateOperationId(); + runOnStrand( + [&]() + { + return queue->addBulkUploadOperation( + *sftp, + request, + [](std::size_t) + { + return Ids::generateOperationId(); + }, + card + ); + } + ); + for (int step = 0; step < 50 && !remoteNames(*sftp, "/home/test").contains("compared_a.txt"); ++step) + runOnStrand( + [&]() + { + return queue->work(); + } + ); + ASSERT_TRUE(remoteNames(*sftp, "/home/test").contains("compared_a.txt")); + + BulkResumeRegistry registry; + tearDownInto(registry); + auto backup = registry.take(card); + ASSERT_TRUE(backup.has_value()); + auto const& resumed = std::get(backup->request); + ASSERT_FALSE(resumed.entries.empty()); + for (auto const& entry : resumed.entries) + { + EXPECT_EQ(entry.comparedDestination, SharedData::ComparedState{}) << entry.destination.value(); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", entry.destination.value()}, {"content", "appeared meanwhile"}} + )); + } + const auto appeared = resumed.entries.size(); + registry.store(card, *std::move(backup)); + + adoptInto(*sftp, registry, {card}); + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + ASSERT_EQ(finished.front().failedEntries.size(), appeared) << "the resumed entries replace nothing"; + for (auto const& [path, error] : finished.front().failedEntries) + EXPECT_EQ(error.type, Operation::ErrorType::ChangedSinceCompare) << path; + } + + TEST_F(OperationQueueTests, AResumedDirectoryDownloadLeavesOutTheFilesItFinished) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "Documents"; + const auto operationId = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, operationId, toLocal(localPath), remote("/home/test/Documents"), false, false, false, false + ); + } + ).has_value()); + for (int step = 0; step < 100 && localNames(localPath).empty(); ++step) + runOnStrand( + [&]() + { + return queue->work(); + } + ); + ASSERT_EQ(localNames(localPath).size(), 1u); + + BulkResumeRegistry registry; + tearDownInto(registry); + auto backup = registry.take(operationId); + ASSERT_TRUE(backup.has_value()); + auto const& resumed = std::get(backup->request); + ASSERT_EQ(resumed.entries.size(), 1u) << "only the file that is not there yet, as a flat list"; + EXPECT_FALSE(resumed.entries.front().isDirectory); + EXPECT_FALSE( + localNames(localPath).contains(Paths::nativeToUtf8(resumed.entries.front().destination.native().filename())) + ); + ASSERT_TRUE(resumed.localRoot.has_value()); + EXPECT_EQ(resumed.localRoot->native(), localPath); + EXPECT_EQ(backup->finished.entries, 1u); + registry.store(operationId, *std::move(backup)); + + adoptInto(*sftp, registry, {operationId}); + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, operationId); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(finished.front().failedEntries.empty()) << finished.front().failedEntries.front().first; + EXPECT_EQ(localNames(localPath), (std::set{"doc1.txt", "doc2.txt"})); + } + + TEST_F(OperationQueueTests, ABackupOfSeveralDirectoriesIsRefused) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + makeQueue(); + setPaused(false); + + BulkResumeEntry backup{ + .request = SharedData::BulkDownloadRequest{ + .entries = { + {.source = remote("/home/test/Documents"), + .destination = toLocal(localDirectory_.path() / "Documents"), + .isDirectory = true}, + {.source = remote("/home/test/Pictures"), + .destination = toLocal(localDirectory_.path() / "Pictures"), + .isDirectory = true}, + }, + }, + }; + const auto operationId = Ids::generateOperationId(); + runOnStrand( + [&]() + { + queue_->adoptBulkResume(*sftp, operationId, std::move(backup)); + } + ); + EXPECT_TRUE(completedIdsAfterSettling().empty()); + EXPECT_TRUE(localNames(localDirectory_.path()).empty()); + } + TEST_F(OperationQueueTests, QueuedRenameMovesTheFileButNeverReplacesTheDestination) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -780,6 +1063,112 @@ namespace Test EXPECT_EQ(finished.front().error->type, SharedData::OperationErrorType::FileExists); } + TEST_F(OperationQueueTests, QueuedOperationsOfAClosedChannelFailWithChannelClosed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto download = Ids::generateOperationId(); + const auto rename = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, + download, + toLocal(localDirectory_.path() / "large.txt"), + remote("/home/test/large.txt"), + false, + false, + false, + false + ); + } + ).has_value()); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addRenameOperation( + *sftp, rename, remote("/home/test/file1.txt"), remote("/home/test/moved.txt") + ); + } + ).has_value()); + + sftp->close(); + sftp.reset(); + setPaused(false); + workUntilCompletions(2); + + const auto finished = completions(); + ASSERT_EQ(finished.size(), 2u); + for (auto const& completed : finished) + { + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Failed) << completed.operationId.value(); + ASSERT_TRUE(completed.error.has_value()) << completed.operationId.value(); + EXPECT_EQ(completed.error->type, SharedData::OperationErrorType::ChannelClosed); + } + EXPECT_FALSE(std::filesystem::exists(localDirectory_.path() / "large.txt")); + } + + TEST_F(OperationQueueTests, BulkDeleteGoesOnPastARootItCannotScan) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/doomed"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/doomed/a.txt"}, {"content", "a"}})); + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto bulk = Ids::generateOperationId(); + const SharedData::BulkDeleteRequest request{ + .entries = { + {.target = remote("/home/test/Documents"), .isDirectory = true}, + {.target = remote("/home/test/doomed"), .isDirectory = true}, + {.target = remote("/home/test/file2.txt")}, + }, + }; + EXPECT_EQ( + runOnStrand( + [&]() + { + return queue->addBulkDeleteOperation(*sftp, request, bulk); + } + ), + 3u + ); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + ASSERT_EQ(finished.front().failedEntries.size(), 1u); + EXPECT_EQ(finished.front().failedEntries.front().first, "/home/test/Documents"); + const auto home = remoteNames(*sftp, "/home/test"); + EXPECT_TRUE(home.contains("Documents")); + EXPECT_FALSE(home.contains("doomed")); + EXPECT_FALSE(home.contains("file2.txt")); + } + + TEST(SequentialScanProgressTests, CountsOfLaterScansAddToTheFinishedOnes) + { + std::vector> reports; + const auto next = Detail::sequentialScanProgress( + [&reports](std::uint64_t bytes, std::uint64_t index, std::uint64_t scanned) + { + reports.push_back({bytes, index, scanned}); + } + ); + auto first = next(); + auto second = next(); + first(10, 1, 1); + first(30, 2, 2); + second(5, 1, 1); + second(7, 3, 3); + EXPECT_EQ(reports.back(), (std::array{37, 5, 5})); + } + TEST_F(OperationQueueTests, BulkDeleteOfAnAlreadyDeletedItemSucceeds) { // A resumed bulk delete meets the items it deleted before the interruption. @@ -863,98 +1252,13 @@ namespace Test EXPECT_FALSE(remoteNames(*sftp, "/home/test").contains("file2.txt")) << "a failure does not stop the rest"; } - TEST_F(OperationQueueTests, LocalDeleteIsOneQueuedUnit) + TEST_F(OperationQueueTests, SyncPlanQueuesEveryPartUnderTheFrontendIds) { - const auto root = localDirectory_.path() / "localDelete"; - std::filesystem::create_directories(root / "directory" / "inner"); - std::ofstream{root / "directory" / "inner" / "a.txt"} << "a"; - std::ofstream{root / "file.txt"} << "file"; - std::ofstream{root / "keep.txt"} << "keep"; - auto queue = makeQueue(); - - const auto remove = Ids::generateOperationId(); - ASSERT_TRUE(runOnStrand([&]() { - return queue->addLocalDeleteOperation( - remove, - {Paths::LocalPath::make(root / "directory").value(), - Paths::LocalPath::make(root / "directory" / "inner").value(), - Paths::LocalPath::make(root / "directory" / "inner" / "a.txt").value(), - Paths::LocalPath::make(root / "file.txt").value()} - ); - }).has_value()); - - // Paused queues hold local deletes like any other operation. - EXPECT_TRUE(std::filesystem::exists(root / "file.txt")); - setPaused(false); - workUntilCompletions(1); - ASSERT_EQ(completions().size(), 1u); - EXPECT_EQ(completions().front().operationId, remove); - EXPECT_EQ(completions().front().reason, SharedData::OperationCompletionReason::Completed); - EXPECT_FALSE(std::filesystem::exists(root / "directory")); - EXPECT_FALSE(std::filesystem::exists(root / "file.txt")); - EXPECT_TRUE(std::filesystem::exists(root / "keep.txt")); - } - - TEST_F(OperationQueueTests, LocalDeleteKeepsUnlistedContentAndItsDirectories) - { - const auto root = localDirectory_.path() / "localDeleteUnlisted"; - std::filesystem::create_directories(root / "directory" / "inner"); - std::ofstream{root / "directory" / "listed.txt"} << "listed"; - std::ofstream{root / "directory" / "inner" / "unlisted.txt"} << "unlisted"; - std::ofstream{root / "other.txt"} << "other"; - auto queue = makeQueue(); - - const auto remove = Ids::generateOperationId(); - ASSERT_TRUE(runOnStrand([&]() { - return queue->addLocalDeleteOperation( - remove, - {Paths::LocalPath::make(root / "directory").value(), - Paths::LocalPath::make(root / "directory" / "inner").value(), - Paths::LocalPath::make(root / "directory" / "listed.txt").value(), - Paths::LocalPath::make(root / "other.txt").value()} - ); - }).has_value()); - setPaused(false); - workUntilCompletions(1); - - ASSERT_EQ(completions().size(), 1u); - const auto completion = completions().front(); - EXPECT_EQ(completion.reason, SharedData::OperationCompletionReason::Completed); - EXPECT_EQ(completion.failedEntries.size(), 2u) << "inner and directory are not empty"; - EXPECT_TRUE(std::filesystem::exists(root / "directory" / "inner" / "unlisted.txt")); - EXPECT_FALSE(std::filesystem::exists(root / "directory" / "listed.txt")); - EXPECT_FALSE(std::filesystem::exists(root / "other.txt")) << "a failure does not stop the rest"; - } - - TEST_F(OperationQueueTests, LocalDeleteOfARootOrTheHomeDirectoryIsRefused) - { - auto queue = makeQueue(); - const auto file = localDirectory_.path() / "survivor.txt"; - std::ofstream{file} << "survivor"; - const auto root = Paths::LocalPath::make(std::filesystem::path{file}.root_path()).value(); - const auto home = localHomeDirectory(); - ASSERT_TRUE(home.has_value()); - - for (auto const& target : {root, *home}) - { - const auto result = runOnStrand([&]() { - return queue->addLocalDeleteOperation( - Ids::generateOperationId(), {Paths::LocalPath::make(file).value(), target} - ); - }); - ASSERT_FALSE(result.has_value()) << target.toUtf8(); - EXPECT_EQ(result.error().type, Operation::ErrorType::InvalidPath); - } - EXPECT_TRUE(std::filesystem::exists(file)); - } - - TEST_F(OperationQueueTests, SyncPlanQueuesEveryPartUnderTheFrontendIds) - { - CREATE_SERVER_AND_JOINER(sftpServer); - ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/syncRemote"}})); - ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/syncRemote/down.txt"}, {"content", "down"}})); - ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/syncRemote/gone.txt"}, {"content", "gone"}})); - auto [_, sftp] = createSftpSession(serverStartResult->port); + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/syncRemote"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/syncRemote/down.txt"}, {"content", "down"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/syncRemote/gone.txt"}, {"content", "gone"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); auto queue = makeQueue(); const auto localRoot = localDirectory_.path() / "syncExecution"; @@ -971,13 +1275,9 @@ namespace Test context.get_executor() }; using SharedData::Sync::Action; - const auto entry = [&session](std::string relKey, Action action, bool isDirectory, std::uint64_t size = 0) + const auto entry = [&session, &sftp](std::string relKey, Action action, bool isDirectory, std::uint64_t size = 0) { - auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); - return SyncSession::ResolvedPlanEntry{ - .entry = {.relKey = std::move(relKey), .action = action, .sizeBytes = size, .isDirectory = isDirectory}, - .paths = std::move(paths), - }; + return plannedNow(session, *sftp, std::move(relKey), action, isDirectory, size); }; SyncSession::ResolvedPlan plan{ .entries = @@ -999,10 +1299,10 @@ namespace Test }; const auto result = runOnStrand([&]() { - return queue->executeSyncPlan(*sftp, session, std::move(plan), request); + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); }); - EXPECT_EQ(result.uploads, (std::vector{"up.txt"})); - EXPECT_EQ(result.downloads, (std::vector{"down.txt"})); + EXPECT_EQ(result.uploads, (std::vector{"up.txt", "emptyA/emptyB"})); + EXPECT_EQ(result.downloads, (std::vector{"down.txt", "newLocal/dir"})); EXPECT_EQ(result.remoteDeletes, (std::vector{"gone.txt"})); EXPECT_EQ(result.localDeletes, (std::vector{"localGone.txt"})); EXPECT_EQ(result.remoteDeleteSources, (std::vector{"/home/test/syncRemote/gone.txt"})); @@ -1010,9 +1310,11 @@ namespace Test result.localDeleteSources, (std::vector{Paths::LocalPath::make(localRoot / "localGone.txt").value().toUtf8()}) ); - EXPECT_EQ(result.createdDirectories, (std::vector{"emptyA/emptyB", "newLocal/dir"})); EXPECT_EQ(result.failed, (std::vector{"CON"})); + for (int cycle = 0; cycle < 10; ++cycle) + runOnStrand([]() {}); + EXPECT_TRUE(std::filesystem::exists(localRoot / "localGone.txt")) << "the paused queue holds the local delete"; setPaused(false); workUntilCompletions(4); std::set finishedIds; @@ -1037,6 +1339,436 @@ namespace Test EXPECT_FALSE(std::filesystem::exists(localRoot / "localGone.txt")); } + TEST_F(OperationQueueTests, SyncTransfersFilesOfSiblingDirectoriesBelowTheRoots) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/siblings"}})); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/siblings/src"}})); + ASSERT_TRUE( + serverStartResult->command("createFile", {{"path", "/home/test/siblings/src/a.cpp"}, {"content", "a"}}) + ); + ASSERT_TRUE( + serverStartResult->command("createFile", {{"path", "/home/test/siblings/README.md"}, {"content", "readme"}}) + ); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "siblings"; + std::filesystem::create_directories(localRoot / "lib"); + std::ofstream{localRoot / "lib" / "b.cpp"} << "b"; + std::ofstream{localRoot / "NOTES.md"} << "notes"; + + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home/test/siblings"), + }, + context.get_executor() + }; + using SharedData::Sync::Action; + const auto entry = [&session](std::string relKey, Action action, std::uint64_t size) + { + auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); + return SyncSession::ResolvedPlanEntry{ + .entry = {.relKey = std::move(relKey), .action = action, .sizeBytes = size}, + .paths = std::move(paths), + }; + }; + // The first file of each direction lies in a subdirectory, its parent is not the root of the others. + SyncSession::ResolvedPlan plan{ + .entries = { + entry("src/a.cpp", Action::Download, 1), + entry("README.md", Action::Download, 6), + entry("lib/b.cpp", Action::Upload, 1), + entry("NOTES.md", Action::Upload, 5), + }, + }; + const SharedData::Sync::SyncExecutionRequest request{ + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + runOnStrand( + [&]() + { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + } + ); + + setPaused(false); + workUntilCompletions(2); + for (auto const& completed : completions()) + { + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Completed) + << completed.operationId.value(); + EXPECT_TRUE(completed.failedEntries.empty()) << completed.operationId.value(); + } + EXPECT_TRUE(std::filesystem::exists(localRoot / "src" / "a.cpp")); + EXPECT_TRUE(std::filesystem::exists(localRoot / "README.md")); + EXPECT_EQ(remoteNames(*sftp, "/home/test/siblings/lib"), (std::set{"b.cpp"})); + EXPECT_TRUE(remoteNames(*sftp, "/home/test/siblings").contains("NOTES.md")); + } + +#ifndef _WIN32 + TEST_F(OperationQueueTests, SyncCreatesEmptyDirectoriesInItsBulksButNeverThroughALink) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/emptyDirs"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "emptyDirs"; + const auto outside = localDirectory_.path() / "emptyDirsOutside"; + std::filesystem::create_directories(localRoot); + std::filesystem::create_directories(outside); + std::filesystem::create_directory_symlink(outside, localRoot / "linked"); + + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home/test/emptyDirs"), + }, + context.get_executor() + }; + using SharedData::Sync::Action; + const auto entry = [&session](std::string relKey, Action action) + { + auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); + return SyncSession::ResolvedPlanEntry{ + .entry = {.relKey = std::move(relKey), .action = action, .isDirectory = true}, + .paths = std::move(paths), + }; + }; + SyncSession::ResolvedPlan plan{ + .entries = { + entry("linked/inside", Action::Download), + entry("fresh/deep", Action::Download), + entry("up/deep", Action::Upload), + }, + }; + const SharedData::Sync::SyncExecutionRequest request{ + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + const auto result = runOnStrand( + [&]() + { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + } + ); + EXPECT_EQ(result.downloads, (std::vector{"linked/inside", "fresh/deep"})); + EXPECT_EQ(result.uploads, (std::vector{"up/deep"})); + + setPaused(false); + workUntilCompletions(2); + for (auto const& completed : completions()) + { + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Completed) + << completed.operationId.value(); + if (completed.operationId == request.downloadId) + EXPECT_EQ(completed.failedEntries.size(), 1u); + else + EXPECT_TRUE(completed.failedEntries.empty()); + } + EXPECT_FALSE(std::filesystem::exists(outside / "inside")); + EXPECT_TRUE(std::filesystem::is_directory(localRoot / "fresh" / "deep")); + EXPECT_EQ(remoteNames(*sftp, "/home/test/emptyDirs/up"), (std::set{"deep"})); + } +#endif + + TEST_F(OperationQueueTests, SyncLeavesWhatChangedBetweenTheCompareAndItsExecution) + { + using namespace SharedData::Sync; + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/changing"}})); + for (auto const& [name, content] : + {std::pair{"edited.txt", "old"}, std::pair{"gone.txt", "gone"}, std::pair{"kept.txt", "kept"}}) + { + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", fmt::format("/home/test/changing/{}", name)}, {"content", content}} + )); + } + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "changing"; + std::filesystem::create_directories(localRoot); + std::ofstream{localRoot / "edited.txt"} << "new local content"; + std::ofstream{localRoot / "up.txt"} << "up"; + std::ofstream{localRoot / "appears.txt"} << "local"; + + // The compare sees both sides as they are now. + const auto nodeOf = [](std::string name, SharedData::ComparedState const& state) + { + return ScanNode{.name = std::move(name), .type = state.type, .size = state.size, .mtime = state.mtime}; + }; + const auto treeOf = [&](std::vector const& names, auto const& stateOf) + { + std::vector children; + for (auto const& name : names) + children.push_back(nodeOf(name, stateOf(name))); + std::ranges::sort(children, {}, &ScanNode::name); + return ScanNode{.name = "", .type = SharedData::FileType::Directory, .children = std::move(children)}; + }; + const auto localTree = [&]() + { + return treeOf( + {"appears.txt", "edited.txt", "up.txt"}, + [&](std::string const& name) + { + return ComparedStateCheck::localStateOf(localRoot / name).value(); + } + ); + }; + const auto remoteTree = [&](std::vector const& names) + { + return treeOf( + names, + [&](std::string const& name) + { + return remoteStateNow(*sftp, fmt::format("/home/test/changing/{}", name)); + } + ); + }; + + boost::asio::io_context context{}; + SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home/test/changing"), + }, + context.get_executor() + }; + const DiffOptions options{.direction = Direction::Upload, .actionDelete = true}; + session.setLocalTreeInStrand(localTree()); + session.setRemoteTreeInStrand(remoteTree({"edited.txt", "gone.txt", "kept.txt"})); + const auto summary = session.recomputeDiffInStrand(options, {}); + EXPECT_EQ(summary.uploads.itemCount, 3u); + EXPECT_EQ(summary.deletes.itemCount, 2u); + + const auto planOf = [&](DiffSection section, std::unordered_set const& selected) + { + return session.resolvePlanInStrand(section, selected); + }; + auto uploadPlan = planOf(DiffSection::Upload, {"appears.txt", "edited.txt", "up.txt"}); + auto deletePlan = planOf(DiffSection::Delete, {"gone.txt", "kept.txt"}); + + // Meanwhile on the server: one destination is edited, one appears and one delete target changes. + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/changing/edited.txt"}, {"content", "edited on the server"}} + )); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/changing/appears.txt"}, {"content", "remote"}} + )); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/changing/kept.txt"}, {"content", "kept and edited"}} + )); + + const auto execute = [&](DiffSection section, SyncSession::ResolvedPlan plan) + { + const SyncExecutionRequest request{ + .section = section, + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + return runOnStrand( + [&]() + { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + } + ); + }; + const auto uploads = execute(DiffSection::Upload, std::move(uploadPlan)); + EXPECT_EQ(uploads.uploads.size(), 3u); + const auto deletes = execute(DiffSection::Delete, std::move(deletePlan)); + EXPECT_EQ(deletes.remoteDeletes.size(), 2u); + + setPaused(false); + workUntilCompletions(2); + std::map failures; + for (auto const& completed : completions()) + { + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Completed) + << completed.operationId.value(); + for (auto const& [path, error] : completed.failedEntries) + failures.emplace(path.substr(path.rfind('/') + 1), error.type); + } + EXPECT_EQ( + failures, + (std::map{ + {"appears.txt", Operation::ErrorType::FileExists}, + {"edited.txt", Operation::ErrorType::ChangedSinceCompare}, + {"kept.txt", Operation::ErrorType::ChangedSinceCompare}, + }) + ); + EXPECT_EQ( + remoteNames(*sftp, "/home/test/changing"), + (std::set{"appears.txt", "edited.txt", "kept.txt", "up.txt"}) + ); + EXPECT_EQ(remoteStateNow(*sftp, "/home/test/changing/edited.txt").size, std::string{"edited on the server"}.size()); + EXPECT_EQ(remoteStateNow(*sftp, "/home/test/changing/appears.txt").size, std::string{"remote"}.size()); + + // A recompare offers the entries that were left alone again. + session.setRemoteTreeInStrand(remoteTree({"appears.txt", "edited.txt", "kept.txt", "up.txt"})); + const auto again = session.recomputeDiffInStrand(options, {}); + EXPECT_EQ(again.uploads.itemCount, 2u); + EXPECT_EQ(again.deletes.itemCount, 1u); + } + + TEST_F(OperationQueueTests, SyncTransfersNonAsciiNames) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/Größe"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/Größe/東京.txt"}, {"content", "down"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / Paths::nativeFromUtf8("Jürgen"); + std::filesystem::create_directories(localRoot / Paths::nativeFromUtf8("Ärger")); + std::ofstream{localRoot / Paths::nativeFromUtf8("Ärger") / Paths::nativeFromUtf8("Ölung.txt")} << "up"; + + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home/test/Größe"), + }, + context.get_executor() + }; + using SharedData::Sync::Action; + const auto entry = [&session](std::string relKey, Action action, std::uint64_t size) + { + auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); + return SyncSession::ResolvedPlanEntry{ + .entry = {.relKey = std::move(relKey), .action = action, .sizeBytes = size, .isDirectory = false}, + .paths = std::move(paths), + }; + }; + SyncSession::ResolvedPlan plan{ + .entries = + { + entry("Ärger/Ölung.txt", Action::Upload, 2), + entry("東京.txt", Action::Download, 4), + }, + }; + const SharedData::Sync::SyncExecutionRequest request{ + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + + const auto result = runOnStrand([&]() { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + }); + EXPECT_EQ(result.uploads, (std::vector{"Ärger/Ölung.txt"})); + EXPECT_EQ(result.downloads, (std::vector{"東京.txt"})); + + setPaused(false); + workUntilCompletions(2); + for (auto const& completed : completions()) + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Completed) << completed.operationId.value(); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Größe/Ärger"), (std::set{"Ölung.txt"})); + EXPECT_EQ(localNames(localRoot), (std::set{"Ärger", "東京.txt"})); + } + +#ifndef _WIN32 + TEST_F(OperationQueueTests, SyncRefusesEveryDeleteBelowAProtectedDirectory) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/other.txt"}, {"content", "other"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "syncProtected"; + std::filesystem::create_directories(localRoot / "home" / "inner"); + std::ofstream{localRoot / "home" / "keep.txt"} << "keep"; + std::ofstream{localRoot / "home" / "inner" / "keep.txt"} << "keep"; + std::ofstream{localRoot / "loose.txt"} << "loose"; + + auto const* previousHome = std::getenv("HOME"); + const std::optional savedHome = + previousHome ? std::optional{previousHome} : std::nullopt; + ASSERT_EQ(setenv("HOME", (localRoot / "home").c_str(), 1), 0); + const auto restoreHome = Nui::ScopeExit{[&savedHome]() noexcept + { + if (savedHome) + setenv("HOME", savedHome->c_str(), 1); + else + unsetenv("HOME"); + }}; + + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home"), + }, + context.get_executor() + }; + using SharedData::Sync::Action; + const auto entry = [&session, &sftp](std::string relKey, Action action, bool isDirectory) + { + return plannedNow(session, *sftp, std::move(relKey), action, isDirectory); + }; + SyncSession::ResolvedPlan plan{ + .entries = + { + entry("test", Action::DeleteRemote, true), + entry("test/file2.txt", Action::DeleteRemote, false), + entry("test/Documents", Action::DeleteRemote, true), + entry("test/Documents/doc1.txt", Action::DeleteRemote, false), + entry("other.txt", Action::DeleteRemote, false), + entry("home", Action::DeleteLocal, true), + entry("home/inner", Action::DeleteLocal, true), + entry("home/inner/keep.txt", Action::DeleteLocal, false), + entry("home/keep.txt", Action::DeleteLocal, false), + entry("loose.txt", Action::DeleteLocal, false), + }, + }; + const SharedData::Sync::SyncExecutionRequest request{ + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + + const auto result = runOnStrand([&]() { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request, *localQueue_); + }); + EXPECT_EQ(result.remoteDeletes, (std::vector{"other.txt"})); + EXPECT_EQ(result.localDeletes, (std::vector{"loose.txt"})); + EXPECT_EQ( + result.failed, + (std::vector{ + "test", + "test/file2.txt", + "test/Documents", + "test/Documents/doc1.txt", + "home", + "home/inner", + "home/inner/keep.txt", + "home/keep.txt", + }) + ); + + setPaused(false); + workUntilCompletions(2); + EXPECT_TRUE(remoteNames(*sftp, "/home/test").contains("file2.txt")); + EXPECT_FALSE(remoteNames(*sftp, "/home").contains("other.txt")); + EXPECT_TRUE(std::filesystem::exists(localRoot / "home" / "keep.txt")); + EXPECT_TRUE(std::filesystem::exists(localRoot / "home" / "inner" / "keep.txt")); + EXPECT_FALSE(std::filesystem::exists(localRoot / "loose.txt")); + } +#endif + TEST_F(OperationQueueTests, CancelledDirectoryDownloadDuringScanLeavesNextUnitIntact) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -1241,11 +1973,12 @@ namespace Test for (auto& entries : hostileScans) { DeleteOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = true, - }}; + } + }; operation.setScanResult(std::move(entries), 0); const auto result = operation.work(); @@ -1314,6 +2047,108 @@ namespace Test ); } + TEST_F(OperationQueueTests, SyncScansExcludeTempFilesAndTheirAsides) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto suffix = Persistence::effectiveTempFileSuffix(std::nullopt); + const auto localRoot = localDirectory_.path() / "syncAsides"; + std::filesystem::create_directories(localRoot); + std::ofstream{localRoot / "keep.txt"} << "keep"; + std::ofstream{localRoot / fmt::format("part.txt{}", suffix)} << "part"; + std::ofstream{localRoot / fmt::format("old.txt{}", Persistence::asideFileSuffix(suffix))} << "old"; + + const auto syncSessionId = Ids::generateSyncSessionId(); + runOnStrand( + [&]() + { + queue->addSyncScanOperation( + *sftp, + syncSessionId, + Ids::generateOperationId(), + Ids::generateOperationId(), + remote("/home/test/Music"), + Paths::LocalPath::make(localRoot).value(), + false, + true, + false + ); + } + ); + workUntilCompletions(2); + auto session = queue->syncSession(syncSessionId); + ASSERT_TRUE(session); + std::promise summary; + boost::asio::post( + session->strand(), + [&]() + { + summary.set_value( + session->recomputeDiffInStrand({.direction = SharedData::Sync::Direction::Upload}, {}) + ); + } + ); + const auto result = summary.get_future().get(); + std::set excluded; + for (auto const& blocked : result.blockedSamples) + { + if (blocked.reason == SharedData::Sync::BlockReason::Excluded) + excluded.insert(blocked.relKey); + } + EXPECT_EQ( + excluded, + (std::set{ + fmt::format("part.txt{}", suffix), fmt::format("old.txt{}", Persistence::asideFileSuffix(suffix)) + }) + ); + EXPECT_EQ(result.uploads.itemCount, 1u) << "keep.txt"; + } + + TEST_F(OperationQueueTests, AFailedSyncScanCancelsTheOtherOne) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "syncSibling"; + for (int directory = 0; directory < 50; ++directory) + { + const auto path = localRoot / fmt::format("d{}", directory); + std::filesystem::create_directories(path); + for (int file = 0; file < 20; ++file) + std::ofstream{path / fmt::format("f{}", file)} << "x"; + } + + const auto remoteScan = Ids::generateOperationId(); + const auto localScan = Ids::generateOperationId(); + runOnStrand( + [&]() + { + queue->addSyncScanOperation( + *sftp, + Ids::generateSyncSessionId(), + remoteScan, + localScan, + remote("/home/test/Documents"), + Paths::LocalPath::make(localRoot).value(), + false, + true, + false + ); + } + ); + workUntilCompletions(2); + std::map reasons; + for (auto const& completed : completions()) + reasons.emplace(completed.operationId.value(), completed.reason); + EXPECT_EQ(reasons.at(remoteScan.value()), SharedData::OperationCompletionReason::Failed); + EXPECT_EQ(reasons.at(localScan.value()), SharedData::OperationCompletionReason::Canceled); + } + TEST_F(OperationQueueTests, HostileNamesNeverBecomeLocalPaths) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -1364,6 +2199,99 @@ namespace Test #endif } + TEST_F(OperationQueueTests, NamesThatAreNotUtf8AreRefusedAndTheCompletionStillSerializes) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command( + "rawName", {{"directory", "/home/test/Documents"}, {"nameHex", "4672616ee76f6973"}, {"content", "latin1"}} + )); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "Documents"; + const auto download = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, download, toLocal(localPath), remote("/home/test/Documents"), false, false, false, false + ); + } + ).has_value()); + + workUntilCompletions(1); + const auto finished = completions().front(); + ASSERT_EQ(finished.failedEntries.size(), 1u); + for (auto const& [path, error] : finished.failedEntries) + { + EXPECT_EQ(error.type, SharedData::OperationErrorType::InvalidName) << path; + EXPECT_TRUE(Paths::isValidUtf8(path)); + EXPECT_NE(path.find("Fran\xEF\xBF\xBDois"), std::string::npos) << path; + } + const nlohmann::json serialized = finished; + EXPECT_NO_THROW(std::ignore = serialized.dump()); + EXPECT_EQ(localNames(localPath), (std::set{"doc1.txt", "doc2.txt"})); + } + + TEST_F(OperationQueueTests, NonAsciiNamesSurviveDownloadAndUpload) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/Grüße"}})); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/Grüße/東京 \xF0\x9F\x9A\x80.txt"}, {"content", "tokyo"}} + )); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto downloaded = localDirectory_.path() / Paths::nativeFromUtf8("Grüße"); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, + Ids::generateOperationId(), + toLocal(downloaded), + remote("/home/test/Grüße"), + false, + false, + false, + false + ); + } + ).has_value()); + workUntilCompletions(1); + EXPECT_EQ(completions().front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(localNames(downloaded), (std::set{"東京 \xF0\x9F\x9A\x80.txt"})); + std::ifstream downloadedFile{downloaded / Paths::nativeFromUtf8("東京 \xF0\x9F\x9A\x80.txt")}; + const std::string downloadedContent{std::istreambuf_iterator{downloadedFile}, {}}; + EXPECT_EQ(downloadedContent, "tokyo"); + + const auto toUpload = localDirectory_.path() / Paths::nativeFromUtf8("Jürgen"); + std::filesystem::create_directories(toUpload / Paths::nativeFromUtf8("Ärger")); + std::ofstream{toUpload / Paths::nativeFromUtf8("Ärger") / Paths::nativeFromUtf8("Größe.txt")} << "size"; + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addUploadOperation( + *sftp, + Ids::generateOperationId(), + toLocal(toUpload), + remote("/home/test/Jürgen"), + false, + false, + false, + false + ); + } + ).has_value()); + workUntilCompletions(2); + EXPECT_EQ(completions().back().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Jürgen"), (std::set{"Ärger"})); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Jürgen/Ärger"), (std::set{"Größe.txt"})); + } + TEST_F(OperationQueueTests, RecursiveDeleteNeverFollowsHostileNamesOut) { CREATE_SERVER_AND_JOINER(sftpServer); diff --git a/backend/test/backend/test_remote_delete_guard.hpp b/backend/test/backend/test_remote_delete_guard.hpp new file mode 100644 index 00000000..6ed7c316 --- /dev/null +++ b/backend/test/backend/test_remote_delete_guard.hpp @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include "real_server_tests.hpp" + +#include + +#include +#include + +namespace Test +{ + TEST(RemoteDeleteGuardDetailTests, RecognizesWindowsServerPaths) + { + using RemoteDeleteGuardDetail::looksLikeWindowsPath; + EXPECT_TRUE(looksLikeWindowsPath(remote("/C:/Users/me"))); + EXPECT_TRUE(looksLikeWindowsPath(remote("/d:"))); + EXPECT_FALSE(looksLikeWindowsPath(remote("/home/C:"))); + EXPECT_FALSE(looksLikeWindowsPath(remote("/CD:/x"))); + EXPECT_FALSE(looksLikeWindowsPath(remote("/C:x"))); + EXPECT_FALSE(looksLikeWindowsPath(Paths::RemotePath::root())); + } + + TEST(RemoteDeleteGuardDetailTests, RecognizesPathsOfCaseFoldingServers) + { + using RemoteDeleteGuardDetail::looksCaseFolding; + EXPECT_TRUE(looksCaseFolding(remote("/C:/Users/me"))); + EXPECT_TRUE(looksCaseFolding(remote("/Users/me/project"))); + EXPECT_TRUE(looksCaseFolding(remote("/Users"))); + EXPECT_FALSE(looksCaseFolding(remote("/Usersx/me"))); + EXPECT_FALSE(looksCaseFolding(remote("/home/me"))); + EXPECT_FALSE(looksCaseFolding(remote("/srv/Users"))); + EXPECT_FALSE(looksCaseFolding(Paths::RemotePath::root())); + } + + TEST(RemoteDeleteGuardDetailTests, WithinIgnoringCaseStopsAtSegments) + { + using RemoteDeleteGuardDetail::isWithinIgnoringCase; + EXPECT_TRUE(isWithinIgnoringCase(remote("/home/Test/a"), remote("/HOME/test"))); + EXPECT_TRUE(isWithinIgnoringCase(remote("/home/test"), remote("/Home/Test"))); + EXPECT_TRUE(isWithinIgnoringCase(remote("/a"), Paths::RemotePath::root())); + EXPECT_FALSE(isWithinIgnoringCase(remote("/home/tests"), remote("/home/test"))); + EXPECT_FALSE(isWithinIgnoringCase(remote("/home"), remote("/home/test"))); + EXPECT_FALSE(isWithinIgnoringCase(Paths::RemotePath{}, Paths::RemotePath::root())); + } + + TEST(RemoteDeleteGuardDetailTests, TextChecksRefuseRootHomeCaseAndWindowsNames) + { + using Reason = RemoteDeleteRefusal::Reason; + using RemoteDeleteGuardDetail::refusalByText; + const auto reasonOf = [](Paths::RemotePath const& target, std::optional const& home) + { + const auto refusal = refusalByText(target, home); + return refusal ? std::optional{refusal->reason} : std::nullopt; + }; + const auto home = remote("/home/test"); + EXPECT_EQ(reasonOf(Paths::RemotePath{}, home), Reason::NoPath); + EXPECT_EQ(reasonOf(Paths::RemotePath::root(), std::nullopt), Reason::Root); + EXPECT_EQ(reasonOf(remote("/home"), home), Reason::ContainsHome); + EXPECT_EQ(reasonOf(remote("/home/test"), home), Reason::ContainsHome); + EXPECT_FALSE(reasonOf(remote("/HOME"), home).has_value()) << "a Linux server tells names apart by case"; + EXPECT_FALSE(reasonOf(remote("/home/Test"), home).has_value()); + EXPECT_FALSE(reasonOf(remote("/home/test/a"), home).has_value()); + EXPECT_FALSE(reasonOf(remote("/home/testing"), home).has_value()); + EXPECT_FALSE(reasonOf(remote("/home"), std::nullopt).has_value()); + EXPECT_FALSE(reasonOf(remote("/home/a\\b"), home).has_value()) << "a backslash is a character on POSIX"; + + const auto windowsHome = remote("/C:/Users/me"); + EXPECT_EQ(reasonOf(remote("/c:/users"), windowsHome), Reason::ContainsHome); + EXPECT_EQ(reasonOf(remote("/C:/USERS/ME"), windowsHome), Reason::ContainsHome); + EXPECT_EQ(reasonOf(remote("/C:/Users."), windowsHome), Reason::WindowsName); + EXPECT_EQ(reasonOf(remote("/C:/Users "), windowsHome), Reason::WindowsName); + EXPECT_EQ(reasonOf(remote("/C:/Users\\me"), windowsHome), Reason::WindowsName); + EXPECT_EQ(reasonOf(remote("/D:/data\\x"), std::nullopt), Reason::WindowsName); + EXPECT_FALSE(reasonOf(remote("/C:/Users/me/file.txt"), windowsHome).has_value()); + EXPECT_FALSE(reasonOf(remote("/D:/data/a.b"), windowsHome).has_value()); + + const auto macHome = remote("/Users/me"); + EXPECT_EQ(reasonOf(remote("/users"), macHome), Reason::ContainsHome); + EXPECT_EQ(reasonOf(remote("/USERS/ME"), macHome), Reason::ContainsHome); + EXPECT_FALSE(reasonOf(remote("/users/me/file.txt"), macHome).has_value()); + EXPECT_EQ(reasonOf(remote("/Users/ME"), home), std::nullopt) << "the home of a Linux server is exact"; + EXPECT_EQ(reasonOf(remote("/Users"), remote("/users/me")), Reason::ContainsHome) + << "a target below /Users looks like a server that folds case"; + } + + TEST(RemoteDeleteGuardDetailTests, RefusalsAreLocalizedAndSayNothingWasDeleted) + { + const RemoteDeleteRefusal refusal{ + .reason = RemoteDeleteRefusal::Reason::ContainsHome, + .target = "/home", + .home = "/home/test", + }; + const auto parsed = Utility::parseLocalizedMessage(refusal.toLocalizedMessage()); + EXPECT_EQ(parsed.key, "backend.deleteGuard.containsHome"); + EXPECT_EQ(parsed.arguments, (std::vector{"/home", "/home/test"})); + EXPECT_EQ(refusal.toString(), "'/home' is or contains the home directory '/home/test'"); + } + + class RemoteDeleteGuardTests : public RealServerTests + {}; + + TEST_F(RemoteDeleteGuardTests, ParentsAreCanonicalizedSoSymlinksDoNotHideTheHome) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/alias"}, {"target", "/home"}})); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/home/test/up"}, {"target", ".."}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + std::optional home{}; + RemoteDeleteGuard guard{*sftp, home, 5s}; + const auto reasonOf = [&guard](std::string_view target) -> std::optional + { + const auto checked = guard.check(remote(target)); + return checked ? std::nullopt : std::optional{checked.error().reason}; + }; + + EXPECT_EQ(reasonOf("/alias/test"), RemoteDeleteRefusal::Reason::ContainsHome); + ASSERT_TRUE(home.has_value()) << "the home is looked up once and shared"; + EXPECT_EQ(home->value(), "/home/test"); + EXPECT_EQ(reasonOf("/home/test/up/test"), RemoteDeleteRefusal::Reason::ContainsHome); + EXPECT_FALSE(reasonOf("/HOME").has_value()) << "the mock server tells names apart by case"; + EXPECT_EQ(reasonOf("/"), RemoteDeleteRefusal::Reason::Root); + EXPECT_FALSE(reasonOf("/alias/test/file2.txt").has_value()); + EXPECT_FALSE(reasonOf("/alias").has_value()) << "deleting the link removes the link"; + EXPECT_FALSE(reasonOf("/home/test/up").has_value()); + EXPECT_FALSE(reasonOf("/missing/directory/x").has_value()); + + const std::vector targets{remote("/home/test/file2.txt"), remote("/alias/test"), remote("/tmp")}; + const auto refusal = refuseAnyProtected(guard, targets); + ASSERT_TRUE(refusal.has_value()); + EXPECT_EQ(refusal->target, "/alias/test"); + EXPECT_FALSE(refuseAnyProtected(guard, std::vector{remote("/home/test/file2.txt")}).has_value()); + } + + TEST_F(RemoteDeleteGuardTests, ScannedRootsAreCanonicalizedOnce) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createSymlink", {{"path", "/alias"}, {"target", "/home"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + std::optional home{}; + RemoteDeleteGuard guard{*sftp, home, 5s}; + guard.trustScannedBelow(remote("/alias")); + EXPECT_FALSE(guard.check(remote("/alias/test")).has_value()); + EXPECT_TRUE(guard.check(remote("/alias/test/Documents/notThere")).has_value()); + EXPECT_TRUE(guard.check(remote("/alias/other/x")).has_value()); + } + + TEST_F(RemoteDeleteGuardTests, AnUnknownHomeOnlyProtectsTheRoot) + { + CREATE_SERVER_AND_JOINER(sftpServer); + serverStartResult->command("realpathDotFailure"); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + std::optional home{}; + RemoteDeleteGuard guard{*sftp, home, 5s}; + EXPECT_EQ(guard.check(Paths::RemotePath::root()).error().reason, RemoteDeleteRefusal::Reason::Root); + EXPECT_TRUE(guard.check(remote("/home")).has_value()); + EXPECT_FALSE(home.has_value()); + } +} diff --git a/backend/test/backend/test_rpc_helper.hpp b/backend/test/backend/test_rpc_helper.hpp new file mode 100644 index 00000000..56e90f7a --- /dev/null +++ b/backend/test/backend/test_rpc_helper.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include "nui_env.hpp" + +#include +#include + +#include +#include +#include +#include + +namespace Test +{ + TEST(RpcHelperTests, ArgumentsOfTheSecondTryAreValidUtf8) + { + const auto text = RpcHelper::Detail::withValidUtf8Argument(std::string{"a\xFF"}); + EXPECT_EQ(text, nlohmann::json("a\xEF\xBF\xBD")); + + const std::uint64_t large = 1ull << 60u; + const auto number = RpcHelper::Detail::withValidUtf8Argument(large); + static_assert(std::is_same_v, std::uint64_t>); + EXPECT_EQ(number, large); + } + + TEST(RpcHelperTests, CallsCarryingTextThatIsNotUtf8NeverThrow) + { + if (!NuiEnv::instance().available()) + GTEST_SKIP() << "No display available"; + auto& hub = *NuiEnv::instance().hub; + const nlohmann::json invalid{{"name", "Fran\xE7ois"}}; + EXPECT_THROW(hub.callRemote("RpcHelperTests::nobody", invalid), nlohmann::json::type_error); + RpcHelper::callRemoteSafely(hub, "RpcHelperTests::nobody", invalid); + RpcHelper::callRemoteSafely(hub, "RpcHelperTests::nobody", std::string{"Fran\xE7ois"}, 42, true); + RpcHelper::callRemoteSafely(hub, "RpcHelperTests::nobody", {{"error", "Fran\xE7ois"}}); + } +} diff --git a/backend/test/backend/test_rpc_path_parameters.hpp b/backend/test/backend/test_rpc_path_parameters.hpp new file mode 100644 index 00000000..45efb0f2 --- /dev/null +++ b/backend/test/backend/test_rpc_path_parameters.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include +#include "nui_env.hpp" + +#include +#include + +#include +#include +#include + +namespace Test +{ + TEST(RpcPathParametersTests, AbsoluteRemotePathsParse) + { + const auto parsed = RpcHelper::parseRemotePath("/home/test/a.txt"); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->value(), "/home/test/a.txt"); + } + + TEST(RpcPathParametersTests, AnInvalidRemotePathIsNamedInTheMessage) + { + for (auto const& invalid : {std::string{"relative/a.txt"}, std::string{}, std::string{"/a/../b"}}) + { + const auto parsed = RpcHelper::parseRemotePath(invalid); + ASSERT_FALSE(parsed.has_value()) << invalid; + EXPECT_TRUE(parsed.error().starts_with("Invalid remote path: ")) << parsed.error(); + } + } + + TEST(RpcPathParametersTests, RemotePathsStopAtTheFirstInvalidOne) + { + const auto valid = RpcHelper::parseRemotePaths({"/a", "/b/c"}); + ASSERT_TRUE(valid.has_value()) << valid.error(); + ASSERT_EQ(valid->size(), 2u); + EXPECT_EQ((*valid)[1].value(), "/b/c"); + + const auto invalid = RpcHelper::parseRemotePaths({"/a", "b", "/c"}); + ASSERT_FALSE(invalid.has_value()); + EXPECT_TRUE(invalid.error().starts_with("Invalid remote path: ")) << invalid.error(); + + const auto none = RpcHelper::parseRemotePaths({}); + ASSERT_TRUE(none.has_value()); + EXPECT_TRUE(none->empty()); + } + + TEST(RpcPathParametersTests, LocalPathsMustBeAbsolute) + { +#ifdef _WIN32 + const std::string absolute = "C:/Users/test/a.txt"; +#else + const std::string absolute = "/home/test/a.txt"; +#endif + const auto parsed = RpcHelper::parseLocalPath(absolute); + ASSERT_TRUE(parsed.has_value()) << parsed.error(); + EXPECT_EQ(parsed->toUtf8(), absolute); + + const auto relative = RpcHelper::parseLocalPath("relative/a.txt"); + ASSERT_FALSE(relative.has_value()); + EXPECT_TRUE(relative.error().starts_with("Invalid local path: ")) << relative.error(); + + const auto all = RpcHelper::parseLocalPaths({absolute, "relative"}); + ASSERT_FALSE(all.has_value()); + EXPECT_TRUE(all.error().starts_with("Invalid local path: ")) << all.error(); + } + + TEST(RpcPathParametersTests, ARequestThatDoesNotParseNamesWhatItIs) + { + const auto invalidTarget = + nlohmann::json{{"entries", {{{"target", "relative"}, {"isDirectory", false}, {"sizeBytes", 0}}}}}; + const auto parsed = RpcHelper::parseRequest(invalidTarget, "bulk delete"); + ASSERT_FALSE(parsed.has_value()); + EXPECT_TRUE(parsed.error().starts_with("Invalid bulk delete: ")) << parsed.error(); + + const auto notJson = RpcHelper::parseRequest>( + nlohmann::json::parse("{not json", nullptr, false), "archive roots" + ); + ASSERT_FALSE(notJson.has_value()); + EXPECT_TRUE(notJson.error().starts_with("Invalid archive roots: ")) << notJson.error(); + } + + TEST(RpcPathParametersTests, AMovedReplyCountsAsAnswered) + { + if (!NuiEnv::instance().available()) + GTEST_SKIP() << "No display available"; + auto& environment = NuiEnv::instance(); + RpcHelper::RpcOnce original{*environment.window, *environment.hub, "RpcPathParametersTests::reply"}; + EXPECT_FALSE(original.answered()); + + RpcHelper::RpcOnce moved{std::move(original)}; + EXPECT_TRUE(original.answered()); + EXPECT_FALSE(moved.answered()); + + RpcHelper::RpcOnce assigned{*environment.window, *environment.hub, "RpcPathParametersTests::other"}; + assigned = std::move(moved); + EXPECT_TRUE(moved.answered()); + EXPECT_FALSE(assigned.answered()); + } +} diff --git a/backend/test/backend/test_scan_operation.hpp b/backend/test/backend/test_scan_operation.hpp index 749572a7..a92972f8 100644 --- a/backend/test/backend/test_scan_operation.hpp +++ b/backend/test/backend/test_scan_operation.hpp @@ -62,14 +62,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation operation{*sftp, {}}; + ScanOperation operation{sftp, {}}; } TEST_F(ScanOperationTests, ScanOperationTypeIsScan) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation operation{*sftp, {}}; + ScanOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::Scan); } @@ -77,7 +77,7 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation operation{*sftp, {}}; + ScanOperation operation{sftp, {}}; EXPECT_TRUE(operation.isBarrier()); } @@ -87,11 +87,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -106,11 +107,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -128,11 +130,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test"), .recursive = true, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -151,11 +154,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -186,11 +190,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = true, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -207,15 +212,17 @@ namespace Test int callbackInvocations = 0; ScanOperation operation{ - *sftp, + sftp, { .progressCallback = - [&callbackInvocations](std::uint64_t, std::uint64_t, std::uint64_t) { - ++callbackInvocations; - }, + [&callbackInvocations](std::uint64_t, std::uint64_t, std::uint64_t) + { + ++callbackInvocations; + }, .remotePath = remote("/home/test/Documents"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); EXPECT_GT(callbackInvocations, 0); @@ -227,11 +234,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/does_not_exist_dir"), .recursive = false, - }}; + } + }; std::expected workResult{ScanOperation::WorkStatus::MoreWork}; for (int iteration = 0; iteration < 100; ++iteration) @@ -255,11 +263,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .recursive = false, - }}; + } + }; std::expected workResult{ScanOperation::WorkStatus::MoreWork}; for (int iteration = 0; iteration < 100; ++iteration) @@ -281,11 +290,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = false, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -300,11 +310,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/does_not_exist_dir"), .recursive = false, - }}; + } + }; // Drive the state machine until it fails. for (int iteration = 0; iteration < 10; ++iteration) @@ -325,11 +336,12 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test"), .recursive = true, - }}; + } + }; // Kick into Running state. auto firstWork = operation.work(); @@ -350,12 +362,13 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); ScanOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/Documents"), .recursive = false, .ignoreHidden = true, - }}; + } + }; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); @@ -370,7 +383,7 @@ namespace Test ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation operation{*sftp, {.remotePath = remote("/home/test"), .buildTree = true}}; + ScanOperation operation{sftp, {.remotePath = remote("/home/test"), .buildTree = true}}; ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); const auto tree = operation.ejectScanTree(); @@ -390,7 +403,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); // Transfers and deletes use flat scans, an incomplete listing must fail them. - ScanOperation operation{*sftp, {.remotePath = remote("/home/test")}}; + ScanOperation operation{sftp, {.remotePath = remote("/home/test")}}; std::expected result{}; for (int iteration = 0; iteration < 100 && result.has_value() && result.value() != ScanOperation::WorkStatus::Complete; @@ -406,7 +419,7 @@ namespace Test ASSERT_TRUE(serverStartResult->command("rawName", {{"directory", "/home/test/Documents"}, {"name", ".."}})); auto [_, sftp] = createSftpSession(serverStartResult->port); - ScanOperation flat{*sftp, {.remotePath = remote("/home/test/Documents")}}; + ScanOperation flat{sftp, {.remotePath = remote("/home/test/Documents")}}; ASSERT_EQ(runScanToCompletion(flat), ScanOperation::WorkStatus::Complete); const auto rejected = flat.rejectedPaths(); ASSERT_EQ(rejected.size(), 1u); @@ -414,7 +427,7 @@ namespace Test for (auto const& entry : flat.ejectEntries()) EXPECT_NE(entry.name, "x/.."); - ScanOperation tree{*sftp, {.remotePath = remote("/home/test/Documents"), .buildTree = true}}; + ScanOperation tree{sftp, {.remotePath = remote("/home/test/Documents"), .buildTree = true}}; ASSERT_EQ(runScanToCompletion(tree), ScanOperation::WorkStatus::Complete); const auto root = tree.ejectScanTree(); const auto hostile = std::ranges::find_if(root.children, [](auto const& child) { diff --git a/backend/test/backend/test_sync_session.hpp b/backend/test/backend/test_sync_session.hpp index 2edea1ec..e6b33493 100644 --- a/backend/test/backend/test_sync_session.hpp +++ b/backend/test/backend/test_sync_session.hpp @@ -6,9 +6,12 @@ #include #include +#include #include #include +extern std::filesystem::path temporaryRoot; + namespace Test { namespace SyncSessionTestDetail @@ -186,4 +189,30 @@ namespace Test EXPECT_FALSE(session.resolve(Paths::RelativePath::make("dir/CON").value()).has_value()); EXPECT_EQ(session.resolve(Paths::RelativePath{})->remote.value(), "/srv/sync"); } + + TEST(SyncSessionCaseTests, CaseVariantsBlockOnlyTowardsASideThatFoldsCase) + { + using namespace SyncSessionTestDetail; + boost::asio::io_context context{}; + const auto blockedWith = [&](bool localFoldsCase, bool remoteFoldsCase, Direction direction) + { + SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::fromUtf8("/local", Paths::LocalPlatform::Posix).value(), + .remoteRoot = remote("/remote"), + .localFoldsCase = localFoldsCase, + .remoteFoldsCase = remoteFoldsCase, + }, + context.get_executor() + }; + session.setLocalTreeInStrand(directory("", {file("Readme", 1, 200)})); + session.setRemoteTreeInStrand(directory("", {file("readme", 1, 100)})); + return session.recomputeDiffInStrand({.direction = direction}, {}).blockedCount; + }; + EXPECT_EQ(blockedWith(false, false, Direction::Both), 0u); + EXPECT_EQ(blockedWith(true, false, Direction::Upload), 0u); + EXPECT_EQ(blockedWith(true, false, Direction::Download), 2u); + EXPECT_EQ(blockedWith(false, true, Direction::Upload), 2u); + EXPECT_EQ(blockedWith(false, true, Direction::Download), 0u); + } } diff --git a/backend/test/backend/test_transfer_failures.hpp b/backend/test/backend/test_transfer_failures.hpp index 4be7d0d7..09375095 100644 --- a/backend/test/backend/test_transfer_failures.hpp +++ b/backend/test/backend/test_transfer_failures.hpp @@ -2,7 +2,9 @@ #include #include +#include #include +#include #include "real_server_tests.hpp" #include @@ -82,6 +84,109 @@ namespace Test Utility::TemporaryDirectory localDirectory_{temporaryRoot, true}; }; + TEST_F(TransferFailureTests, RenameThatTimedOutRunsAfterTheOperationIsGone) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + std::promise release{}; + auto blocked = sftp->performPromise( + [released = release.get_future().share()]() + { + released.wait(); + } + ); + { + RenameOperation operation{ + sftp, + { + .sourcePath = remote("/home/test/file1.txt"), + .destinationPath = remote("/home/test/moved.txt"), + .futureTimeout = std::chrono::seconds{1}, + } + }; + const auto result = operation.work(); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::FutureTimeout); + } + release.set_value(); + blocked.wait(); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/moved.txt"), "Fake file content"); + } + +#ifndef _WIN32 + TEST_F(TransferFailureTests, DownloadGetsItsPermissionsAndTimeBeforeItIsMovedIntoPlace) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/private.txt"}, {"content", "private"}, {"mode", 0640}} + )); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + struct Case + { + bool inheritPermissions; + std::optional filePermissions; + std::filesystem::perms expected; + }; + using std::filesystem::perms; + const std::vector cases{ + {false, std::nullopt, Utility::newFilePermissions()}, + {true, std::nullopt, (perms::owner_read | perms::owner_write | perms::group_read) & ~Utility::processUmask()}, + {false, perms::owner_read | perms::owner_write, perms::owner_read | perms::owner_write}, + }; + for (std::size_t index = 0; index < cases.size(); ++index) + { + auto const& testCase = cases[index]; + const auto localPath = localDirectory_.path() / fmt::format("private_{}.txt", index); + SharedData::DirectoryEntry entry{}; + entry.name = "private.txt"; + entry.type = SharedData::FileType::Regular; + entry.size = 7u; + entry.mtime = 1700000000u; + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/private.txt"), + .localPath = toLocal(localPath), + .inheritPermissions = testCase.inheritPermissions, + .filePermissions = testCase.filePermissions, + .entry = entry, + .markAsDownloaded = false, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), "private"); + EXPECT_EQ(std::filesystem::status(localPath).permissions(), testCase.expected) << "case " << index; + const auto modified = std::chrono::file_clock::to_sys(std::filesystem::last_write_time(localPath)); + EXPECT_EQ(std::chrono::duration_cast(modified.time_since_epoch()).count(), 1700000000); + } + } + + TEST_F(TransferFailureTests, DownloadTempFileIsOnlyReadableByItsOwner) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "large.txt"; + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + } + }; + ASSERT_TRUE(operation.prepare().has_value()); + EXPECT_EQ( + std::filesystem::status(localPath.generic_string() + ".filepart").permissions(), + std::filesystem::perms::owner_read | std::filesystem::perms::owner_write + ); + } +#endif + TEST_F(TransferFailureTests, AsyncDownloadReadFailureKeepsExistingFile) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -92,7 +197,7 @@ namespace Test writeLocalFile(localPath, "original content"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -117,7 +222,7 @@ namespace Test writeLocalFile(localPath, "original content"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -142,7 +247,7 @@ namespace Test writeLocalFile(localPath, "original content"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -171,7 +276,7 @@ namespace Test entry.size = 70000u; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -187,6 +292,91 @@ namespace Test } } + TEST_F(TransferFailureTests, ReservedDownloadOfFileShorterThanListedKeepsExistingFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + for (const bool bigFileOptimized : {false, true}) + { + const auto localPath = localDirectory_.path() / fmt::format("reserved_{}.txt", bigFileOptimized); + writeLocalFile(localPath, "original content"); + + SharedData::DirectoryEntry entry{}; + entry.name = "large.txt"; + entry.type = SharedData::FileType::Regular; + entry.size = 70000u; + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .reserveSpace = true, + .bigFileOptimized = bigFileOptimized, + .entry = entry, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()) << "bigFileOptimized: " << bigFileOptimized; + if (!bigFileOptimized) + EXPECT_EQ(result.error().type, Operation::ErrorType::SizeMismatch) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), "original content") << "bigFileOptimized: " << bigFileOptimized; + } + } + + TEST_F(TransferFailureTests, ReservedDownloadResumesFromPartialFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + for (const bool bigFileOptimized : {false, true}) + { + const auto localPath = localDirectory_.path() / fmt::format("resumed_{}.txt", bigFileOptimized); + writeLocalFile(localPath.generic_string() + ".filepart", alphabet(65536).substr(0, 20000)); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + .reserveSpace = true, + .tryContinue = true, + .bigFileOptimized = bigFileOptimized, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), alphabet(65536)) << "bigFileOptimized: " << bigFileOptimized; + } + } + + TEST_F(TransferFailureTests, ReservedCompletePartialFileIsFinalized) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "file1.txt"; + writeLocalFile(localPath.generic_string() + ".filepart", "Fake file content"); + + DownloadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .reserveSpace = true, + .tryContinue = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), "Fake file content"); + } + TEST_F(TransferFailureTests, AsyncDownloadResumesFromPartialFile) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -196,7 +386,7 @@ namespace Test writeLocalFile(localPath.generic_string() + ".filepart", alphabet(65536).substr(0, 20000)); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/large.txt"), .localPath = toLocal(localPath), @@ -219,7 +409,7 @@ namespace Test writeLocalFile(localPath.generic_string() + ".filepart", "Fake file content"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -239,7 +429,7 @@ namespace Test const auto localPath = localDirectory_.path() / "followed.txt"; DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/link_to_file1.txt"), .localPath = toLocal(localPath), @@ -263,7 +453,7 @@ namespace Test writeLocalFile(localPath, alphabet(65536)); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -287,7 +477,7 @@ namespace Test writeLocalFile(localPath, "new content"); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -313,7 +503,7 @@ namespace Test writeLocalFile(localPath, content); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/resumed.txt"), .localPath = toLocal(localPath), @@ -339,7 +529,7 @@ namespace Test std::filesystem::create_symlink(outside, localPath.generic_string() + ".filepart"); DownloadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -365,7 +555,7 @@ namespace Test writeLocalFile(localPath, "new content"); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -394,7 +584,7 @@ namespace Test writeLocalFile(localPath, "new content"); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(localPath), @@ -405,6 +595,7 @@ namespace Test const auto result = runToEnd(operation); ASSERT_FALSE(result.has_value()); EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "Fake file content"); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt.filepart"), ""); } } @@ -419,7 +610,7 @@ namespace Test writeLocalFile(localPath, alphabet(4096)); ArchiveUploadOperation operation{ - *sftp, + sftp, { .localPaths = {toLocal(localPath)}, .remoteArchivePath = remote("/home/test/backup.tar"), @@ -445,7 +636,7 @@ namespace Test writeLocalFile(localPath, alphabet(4096)); ArchiveUploadOperation refused{ - *sftp, + sftp, { .localPaths = {toLocal(localPath)}, .remoteArchivePath = remote("/home/test/backup.tar"), @@ -458,7 +649,7 @@ namespace Test EXPECT_EQ(readRemoteFile(*sftp, "/home/test/backup.tar"), "old archive"); ArchiveUploadOperation replacing{ - *sftp, + sftp, { .localPaths = {toLocal(localPath)}, .remoteArchivePath = remote("/home/test/backup.tar"), diff --git a/backend/test/backend/test_upload_operation.hpp b/backend/test/backend/test_upload_operation.hpp index ace9d501..25512cbd 100644 --- a/backend/test/backend/test_upload_operation.hpp +++ b/backend/test/backend/test_upload_operation.hpp @@ -33,14 +33,14 @@ namespace Test { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - UploadOperation operation{*sftp, {}}; + UploadOperation operation{sftp, {}}; } TEST_F(UploadOperationTests, UploadOperationTypeIsUpload) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - UploadOperation operation{*sftp, {}}; + UploadOperation operation{sftp, {}}; EXPECT_EQ(operation.type(), SharedData::OperationType::Upload); } @@ -50,7 +50,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -79,7 +79,7 @@ namespace Test for (auto const& remotePath : {Paths::RemotePath{}, Paths::RemotePath::root()}) { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remotePath, .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -109,7 +109,7 @@ namespace Test ASSERT_TRUE(createResult.has_value()); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -139,7 +139,7 @@ namespace Test std::ofstream{temporaryRoot / "windows.txt"} << "text"; UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/windows.txt"), .localPath = toLocal(temporaryRoot / "windows.txt"), @@ -172,7 +172,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -230,7 +230,7 @@ namespace Test // First upload { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -255,7 +255,7 @@ namespace Test // Second upload without overwrite { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -277,7 +277,7 @@ namespace Test // First upload { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -302,7 +302,7 @@ namespace Test // Second upload with overwrite { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -354,7 +354,7 @@ namespace Test // Second upload continuing the partial file { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -431,7 +431,7 @@ namespace Test // Second upload continuing the partial file { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -488,7 +488,7 @@ namespace Test emptyFile.close(); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/emptyfile.txt"), .localPath = toLocal(temporaryRoot / "emptyfile.txt"), @@ -522,7 +522,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -557,7 +557,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -586,7 +586,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "does_not_exist.txt"), @@ -607,7 +607,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/nonexistent_dir/file.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -625,7 +625,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "does_not_exist.txt"), @@ -634,7 +634,7 @@ namespace Test auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); - EXPECT_EQ(workResult.error().type, UploadOperation::ErrorType::NotAFile); + EXPECT_EQ(workResult.error().type, UploadOperation::ErrorType::FileNotFound); } TEST_F(UploadOperationTests, UploadLocalDirectoryFails) @@ -643,7 +643,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(programDirectory), @@ -668,7 +668,7 @@ namespace Test } UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/largefile.txt"), .localPath = toLocal(temporaryRoot / "largefile.txt"), @@ -751,7 +751,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/uploaded_sym_to_file"), .localPath = toLocal(linkPath), @@ -788,7 +788,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/uploaded_sym_to_dir"), .localPath = toLocal(linkPath), @@ -820,7 +820,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/uploaded_dangling"), .localPath = toLocal(linkPath), @@ -851,7 +851,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/uploaded_sym_skip"), .localPath = toLocal(linkPath), @@ -880,7 +880,7 @@ namespace Test // First put a regular file in place. { UploadOperation first{ - *sftp, + sftp, { .remotePath = remote("/home/test/sym_overwrite_target"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -896,7 +896,7 @@ namespace Test temporaryRoot / "testfile.txt" ); UploadOperation symOp{ - *sftp, + sftp, { .remotePath = remote("/home/test/sym_overwrite_target"), .localPath = toLocal(linkPath), @@ -928,7 +928,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/sym_prepare_check"), .localPath = toLocal(linkPath), @@ -961,7 +961,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/newly_created/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -990,7 +990,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/a/b/c/d/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -1023,7 +1023,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -1043,7 +1043,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/still_missing_dir/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), @@ -1072,7 +1072,7 @@ namespace Test auto [_, sftp] = createSftpSession(serverStartResult->port); UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/missing/for/symlink/uploaded_link"), .localPath = toLocal(linkPath), @@ -1119,7 +1119,7 @@ namespace Test // Second upload attempting to continue the partial file { UploadOperation operation{ - *sftp, + sftp, { .remotePath = remote("/home/test/testfile.txt"), .localPath = toLocal(temporaryRoot / "testfile.txt"), diff --git a/frontend/include/frontend/file_explorer/local_side_model.hpp b/frontend/include/frontend/file_explorer/local_side_model.hpp index b3abac5f..0381147c 100644 --- a/frontend/include/frontend/file_explorer/local_side_model.hpp +++ b/frontend/include/frontend/file_explorer/local_side_model.hpp @@ -153,6 +153,16 @@ class LocalSideModel void enqueueSingleUpload(SharedData::BulkUploadEntry const& entry, bool allowOverwrite, bool insertRefresh); + /** + * @brief Deletes @p targets through the backend at once, each item by itself, and refreshes the listing. + */ + void deleteFilesDirectly(std::vector const& targets); + + /** + * @brief Shows why deleting local items failed. + */ + void showDeleteFailed(std::string const& reason); + private: /** * @brief Ask the backend once which external-open operations the platform can perform diff --git a/frontend/include/frontend/file_explorer/side_model.hpp b/frontend/include/frontend/file_explorer/side_model.hpp index 1801f57e..11be4529 100644 --- a/frontend/include/frontend/file_explorer/side_model.hpp +++ b/frontend/include/frontend/file_explorer/side_model.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -171,4 +172,6 @@ class SideModel : public NuiFileExplorer::ISideModel std::function refreshCallback_{nullptr}; bool reapplySelectionOnce_{false}; std::string dropMetadata_{}; + /** @brief Guards callbacks that can arrive after the side model is gone, like RPC replies and dialog closes. */ + Utility::LifetimeToken lifetime_{}; }; \ No newline at end of file diff --git a/frontend/include/frontend/session_components/local_operation_router.hpp b/frontend/include/frontend/session_components/local_operation_router.hpp new file mode 100644 index 00000000..31c8f2de --- /dev/null +++ b/frontend/include/frontend/session_components/local_operation_router.hpp @@ -0,0 +1,90 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +class OperationQueue; + +/** + * @brief Shows the operations of the backend's LocalOperationQueue, which belong to no connection, in the operation + * queues of the session tabs. + * + * An operation shows in the queue that started it. When that queue goes away, with its tab or on a reconnect, the + * card moves to the queue attached last, and without any queue to the next one that attaches. A delete that ends + * while no queue is attached is reported as a notification. + */ +class LocalOperationRouter +{ + public: + static LocalOperationRouter& instance(); + + LocalOperationRouter(LocalOperationRouter const&) = delete; + LocalOperationRouter& operator=(LocalOperationRouter const&) = delete; + LocalOperationRouter(LocalOperationRouter&&) = delete; + LocalOperationRouter& operator=(LocalOperationRouter&&) = delete; + ~LocalOperationRouter() = default; + + /** + * @brief @p queue can show cards from now on and takes the cards that have no queue. + */ + void attach(OperationQueue& queue); + + /** + * @brief @p queue goes away, its cards move to another queue. + */ + void detach(OperationQueue& queue); + + /** + * @brief The events of @p operationId go to @p queue. Call before the operation is queued. + */ + void host(Ids::OperationId const& operationId, OperationQueue& queue); + + /** + * @brief Drops @p operationId, which was not queued or whose card is gone. + */ + void forget(Ids::OperationId const& operationId); + + /** + * @brief Cancels every operation shown in @p queue and drops it, for a queue that clears its cards. + */ + void cancelHostedBy(OperationQueue const& queue); + + private: + LocalOperationRouter(); + + struct Routed + { + OperationQueue* host{nullptr}; + std::optional added{}; + std::optional scanProgress{}; + std::optional deleteProgress{}; + }; + + void onOperationAdded(SharedData::OperationAdded const& added); + void onScanProgress(SharedData::ScanProgress const& progress); + void onDeleteProgress(SharedData::BulkDeleteProgress const& progress); + void onOperationCompleted(Nui::val val); + + /** + * @brief Shows the card of @p routed in @p queue as far as its operation got. + */ + static void show(Routed& routed, OperationQueue& queue); + + /** + * @brief The queue attached last, nullptr without any. + */ + OperationQueue* newestQueue() const; + + std::vector queues_{}; + std::unordered_map routed_{}; + std::vector registrations_{}; +}; diff --git a/frontend/include/frontend/session_components/operation_queue.hpp b/frontend/include/frontend/session_components/operation_queue.hpp index 9b4d3d4c..32e16d56 100644 --- a/frontend/include/frontend/session_components/operation_queue.hpp +++ b/frontend/include/frontend/session_components/operation_queue.hpp @@ -39,6 +39,7 @@ class LocalSideModel; class RemoteSideModel; +class LocalOperationRouter; class OperationQueue { @@ -60,7 +61,7 @@ class OperationQueue RemoteSideModel* remoteModel ); - ROAR_PIMPL_SPECIAL_FUNCTIONS(OperationQueue); + ROAR_PIMPL_SPECIAL_FUNCTIONS_NO_MOVE(OperationQueue); void activate(std::shared_ptr fileEngine, Ids::SessionId sessionId); @@ -187,15 +188,38 @@ class OperationQueue std::function onEnqueued = {} ); - void addCompletionCallback(Ids::OperationId const& opId, std::function callback); - /** @brief Register a callback that receives the source paths of the entries a bulk operation - * could not transfer. Runs on completion, before the completion callback, because the - * final BulkProgress can arrive after the completion and is dropped then. + /** + * @brief Queues the delete of local items with their content in the backend's local operation queue, as one + * operation whose card shows here and refreshes the local listing when it ends. Needs no connection; with + * one the delete waits while this queue is paused. @p onAck gets the error when the backend refused it, + * nothing when it was queued. */ - void addFailedEntriesCallback( - Ids::OperationId const& opId, - std::function const& failedPaths)> callback + void enqueueLocalDelete( + std::vector const& targets, + std::function const& error)> onAck ); + + /** + * @brief The queue belongs to a connected session and can take operations. + */ + bool isActive() const; + + void addCompletionCallback(Ids::OperationId const& opId, std::function callback); + + /** + * @brief Receives the entries a bulk operation could not transfer or delete with why: by source path and, for a + * prescanned bulk transfer, by entry index. + */ + using FailedEntriesCallback = std::function> const& failedEntries, + std::vector const& indexedFailures + )>; + + /** @brief Register a callback that receives the entries a bulk operation could not transfer or + * delete. Runs on completion, before the completion callback, because the final BulkProgress + * can arrive after the completion and is dropped then. + */ + void addFailedEntriesCallback(Ids::OperationId const& opId, FailedEntriesCallback callback); /** @brief Register a callback that receives progress as a 0.0–1.0 fraction for an upload/download. * Automatically removed when the operation completes. */ @@ -313,6 +337,8 @@ class OperationQueue Nui::ElementRenderer operator()(); private: + friend class LocalOperationRouter; + template void cancelOperation(OperationCard const& operation); void cancelAll(); @@ -330,6 +356,7 @@ class OperationQueue void onDeleteProgress(SharedData::BulkDeleteProgress const& progress); void onScanProgress(SharedData::ScanProgress const& progress); void onOperationCompleted(Nui::val val); + void onOperationCompleted(SharedData::OperationCompleted completed); void onIsPaused(SharedData::ErrorOrSuccess const& result); void onOperationsReordered(SharedData::OperationsReordered const& evt); void requestMoveOperation(Ids::OperationId const& opId, std::size_t newIndex); diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp index 04e12887..16d386a6 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp @@ -13,7 +13,9 @@ #include #include -struct DisplayedBulkOperation : public OperationCard +struct DisplayedBulkOperation + : public OperationCard + , public ScanningCard { public: DisplayedBulkOperation( @@ -35,9 +37,9 @@ struct DisplayedBulkOperation : public OperationCard std::move(doDeletionCountdown), std::move(onCompleteAction) } + , ScanningCard{scanning} , localPath_{std::move(localPath)} , remotePath_{std::move(remotePath)} - , scanPhase_{scanning} , totalProgressBar_({ .height = std::string{progressHeight}, .min = 0, @@ -52,12 +54,6 @@ struct DisplayedBulkOperation : public OperationCard return true; } - void setScanProgress(std::uint64_t scannedBytes, std::uint64_t scannedEntries) - { - scanPhase_.setProgress(scannedBytes, scannedEntries); - Nui::globalEventContext.executeActiveEventsImmediately(); - } - void setProgress(SharedData::BulkProgress const& progress) { scanPhase_.end(); @@ -342,7 +338,6 @@ struct DisplayedBulkOperation : public OperationCard private: Paths::LocalPath localPath_; Paths::RemotePath remotePath_; - ScanPhaseStatus scanPhase_; Nui::Observed fileCurrentIndex{0ull}; Nui::Observed fileCount{0ull}; Nui::Observed> bytesPerSecond{0}; diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp index 1f4e5f70..b2d67175 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp @@ -6,7 +6,9 @@ #include #include -struct DisplayedDeleteOperation : public OperationCard +struct DisplayedDeleteOperation + : public OperationCard + , public ScanningCard { public: DisplayedDeleteOperation( @@ -28,10 +30,10 @@ struct DisplayedDeleteOperation : public OperationCard std::move(doDeletionCountdown), std::move(onCompleteAction) } + , ScanningCard{scanning} , displayPath_{std::move(displayPath)} , target_{std::move(target)} , recursive_{scanning} - , scanPhase_{scanning} , progressBar_({ .height = std::string{progressHeight}, .min = 0, @@ -53,12 +55,6 @@ struct DisplayedDeleteOperation : public OperationCard progressBar_.setZeroAsComplete(); } - void setScanProgress(std::uint64_t scannedBytes, std::uint64_t scannedEntries) - { - scanPhase_.setProgress(scannedBytes, scannedEntries); - Nui::globalEventContext.executeActiveEventsImmediately(); - } - void setProgress(SharedData::BulkDeleteProgress const& progress) { scanPhase_.end(); @@ -78,7 +74,7 @@ struct DisplayedDeleteOperation : public OperationCard std::optional resumableDescriptor() const override { - // A local delete runs in this process and needs no connection to be redone. + // A local delete runs in the local operation queue, which a reconnect does not touch. if (isCompletedState() || type_ == SharedData::OperationType::LocalDelete) return std::nullopt; // A bulk delete is re-issued by the backend from its own backup. @@ -123,9 +119,9 @@ struct DisplayedDeleteOperation : public OperationCard span{ class_ = "opq-status-text" }( - observe(filesDeleted_, totalFiles_, scanPhase_.scanning, scanPhase_.totalScanned), + observe(filesDeleted_, totalFiles_, state_, scanPhase_.scanning, scanPhase_.totalScanned), [this](){ - if (scanPhase_.scanning.value()) + if (scanPhase_.scanning.value() && !isCompletedState()) return scanPhase_.text(); return fmt::format("{}/{}", filesDeleted_.value(), totalFiles_.value()); } @@ -145,6 +141,5 @@ struct DisplayedDeleteOperation : public OperationCard Paths::RemotePath target_; /** @brief The delete scanned a directory, so it removes content and resumes doing the same. */ bool recursive_{false}; - ScanPhaseStatus scanPhase_; Components::ProgressBar progressBar_; }; \ No newline at end of file diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp index 43ca67b9..4fc8f4c6 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp @@ -7,7 +7,9 @@ #include -class DisplayedTransferOperation : public OperationCard +class DisplayedTransferOperation + : public OperationCard + , public ScanningCard { public: DisplayedTransferOperation( @@ -30,6 +32,7 @@ class DisplayedTransferOperation : public OperationCard totalBytes{0u}; Nui::Observed totalScanned{0u}; }; + +/** + * @brief The part of a card whose operation scans before its own work: the scan status and its progress setter. + */ +class ScanningCard +{ + public: + explicit ScanningCard(bool scanning) + : scanPhase_{scanning} + {} + + void setScanProgress(std::uint64_t scannedBytes, std::uint64_t scannedEntries) + { + scanPhase_.setProgress(scannedBytes, scannedEntries); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + + protected: + ScanPhaseStatus scanPhase_; +}; diff --git a/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp b/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp index fdc62019..2b56b097 100644 --- a/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp +++ b/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp @@ -30,9 +30,10 @@ class OperationQueue; * - @ref cancelDiff flips the backend cancel flag. * - @ref close releases backend memory. Idempotent. * - * The provider owns the lifecycle — callers should construct it once per sync flow - * and destroy it when the dialog closes. The destructor calls @ref close, replies that - * arrive afterwards are dropped. + * The provider owns the lifecycle: one per sync flow, held by the progress dialog until the + * first diff landed and then by the sync dialog until it closes or opens with another one. + * The destructor calls @ref close, replies that arrive afterwards are dropped. A call on a + * closed provider rejects (or reports a cancelled diff) right away. */ class BackendSyncProvider { diff --git a/frontend/include/frontend/sync_dialog/sync_dialog.hpp b/frontend/include/frontend/sync_dialog/sync_dialog.hpp index 61109863..21f5788a 100644 --- a/frontend/include/frontend/sync_dialog/sync_dialog.hpp +++ b/frontend/include/frontend/sync_dialog/sync_dialog.hpp @@ -9,8 +9,6 @@ #include #include -#include -#include #include #include @@ -41,28 +39,22 @@ class SyncDialog * @brief Opens the dialog against a backend-resident sync session whose first * diff has already completed. * - * @param provider Non-owning handle to the session; must outlive the dialog. + * @param provider The sync session, kept until the dialog closes or opens with another one. Callbacks of + * executions under a previous provider are dropped then. * @param snapshot The result of the initial recompute: summary and section roots. - * @param localPath Local scan root (for path column formatting only — data is + * @param localPath Local scan root (for path column formatting only, data is * served lazily from @p provider). * @param remotePath Remote scan root. * @param scanned Settings the scans of @p provider ran with. - * @param lastSyncTime When this pair was last synchronized, every recompute compares against it. */ void open( std::shared_ptr provider, SharedData::Sync::DiffSnapshot snapshot, std::filesystem::path localPath, std::filesystem::path remotePath, - ScanSettings scanned, - std::optional lastSyncTime + ScanSettings scanned ); - /** - * @brief Called when the user synchronized the whole selection, so the session can record the sync. - */ - void setOnSynchronized(std::function callback); - /** * @brief Scan + diff settings snapshot handed to the Recompare callback. */ diff --git a/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp b/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp index 7e9aeebb..8ec3cdf7 100644 --- a/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp +++ b/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp @@ -32,10 +32,11 @@ class SyncProgressDialog * The dialog keeps showing the Comparing spinner during the diff walk; * light diffs finish within a frame so the phase effectively flashes. * - * @param provider Non-owning handle to the sync session. Lifetime must - * outlive this call (typically owned by @c Session). + * @param provider The sync session, held until the first diff landed, a scan failed or the + * dialog was cancelled. * @param initialOptions The diff options to apply for the first recompute. - * @param onDone Called exactly once on success. Not invoked on cancel. + * @param onDone Called exactly once on success with the provider, which this dialog lets go + * of then. Not invoked on cancel or failure. */ void open( std::shared_ptr provider, @@ -45,7 +46,7 @@ class SyncProgressDialog bool recursive, bool ignoreHidden, SharedData::Sync::DiffOptions initialOptions, - std::function onDone + std::function, SharedData::Sync::DiffSnapshot)> onDone ); /** diff --git a/frontend/source/frontend/CMakeLists.txt b/frontend/source/frontend/CMakeLists.txt index 02c61618..fa02c681 100644 --- a/frontend/source/frontend/CMakeLists.txt +++ b/frontend/source/frontend/CMakeLists.txt @@ -141,6 +141,7 @@ target_sources( session_components/file_tracking.cpp session_components/session_options.cpp session_components/operation_queue.cpp + session_components/local_operation_router.cpp session_components/terminal_panel.cpp session_components/file_explorer_panel.cpp session_components/connection_loss_overlay.cpp diff --git a/frontend/source/frontend/file_explorer/local_side_model.cpp b/frontend/source/frontend/file_explorer/local_side_model.cpp index f8189763..0e885107 100644 --- a/frontend/source/frontend/file_explorer/local_side_model.cpp +++ b/frontend/source/frontend/file_explorer/local_side_model.cpp @@ -23,8 +23,11 @@ #include #include +#include + #include #include +#include #include #include #include @@ -490,25 +493,41 @@ void LocalSideModel::onDelete(std::vector const& items) return; std::vector listItems; - bool anyDirectory = false; - for (const auto& item : items) - { - listItems.push_back({item.name, item.fullPath.generic_string()}); - anyDirectory = anyDirectory || item.type == NuiFileExplorer::Item::Type::Directory; - } + listItems.reserve(items.size()); + std::ranges::transform( + items, + std::back_inserter(listItems), + [](NuiFileExplorer::Item const& item) + { + return ConfirmDialog::OpenOptions::ListElement{ + .text = item.name, .description = item.fullPath.generic_string() + }; + } + ); + const bool anyDirectory = std::ranges::any_of( + items, + [](NuiFileExplorer::Item const& item) + { + return item.type == NuiFileExplorer::Item::Type::Directory; + } + ); // Local directories go with their content in one step, unlike remote ones there is no second question. - auto text = language->get("localSideModel", "deleteSelectedConfirm"); - if (anyDirectory) - text += "\n" + language->get("localSideModel", "deleteDirectoriesWithContent"); + auto text = anyDirectory ? fmt::format( + "{}\n{}", + language->get("localSideModel", "deleteSelectedConfirm"), + language->get("localSideModel", "deleteDirectoriesWithContent") + ) + : language->get("localSideModel", "deleteSelectedConfirm"); - confirmDialog_->open( - {.styleVariant = ScriptNuiComponents::StyleVariant::Warning, - .headerText = language->get("localSideModel", "deleteLocalItemsQuestion"), - .text = std::move(text), - .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, - .focusButton = ConfirmDialog::Button::No, - .listItems = listItems, - .onClose = [this, items](std::optional button) + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Warning, + .headerText = language->get("localSideModel", "deleteLocalItemsQuestion"), + .text = std::move(text), + .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, + .focusButton = ConfirmDialog::Button::No, + .listItems = std::move(listItems), + .onClose = lifetime_.guarded( + [this, items, anyDirectory](std::optional button) { if (!button || *button != ConfirmDialog::Button::Yes) { @@ -516,59 +535,87 @@ void LocalSideModel::onDelete(std::vector const& items) return; } - Nui::val args = Nui::val::object(); - args.set("paths", Nui::val::array()); - args.set("recursive", true); - + std::vector targets; + targets.reserve(items.size()); for (const auto& item : items) { auto path = localPathOf(item); if (!path) return showPathRefused(language->get("localSideModel", "deleteFilesFailedTitle"), path.error()); - args["paths"].call("push", path->toUtf8()); + targets.push_back(*std::move(path)); } - Nui::RpcClient::callWithBackChannel( - "RpcFilesystem::removeSome", - [this](Nui::val val) - { - if (!val.hasOwnProperty("success")) + // Deleting a directory can take long, so it runs in the local operation queue with a card of its own. + if (!anyDirectory) + return deleteFilesDirectly(targets); + if (!operationQueue_) + { + Log::warn("Cannot queue the delete of local items: no operation queue is set"); + return showDeleteFailed(language->get("operationQueue", "unknownError")); + } + operationQueue_->enqueueLocalDelete( + targets, + lifetime_.guarded( + [this](std::optional const& error) { - Log::error("Invalid response from RpcFilesystem::remove"); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = language->get("localSideModel", "deleteFilesFailedTitle"), - .text = language->get("localSideModel", "invalidResponseFromBackend"), - .buttons = ConfirmDialog::Button::Ok, - }); - return; + if (!error) + return; + const auto reason = language->translate(*error); + Log::error("Failed to queue the delete of local items: {}", reason); + showDeleteFailed(reason); } + ) + ); + } + ), + }); +} - const auto success = val["success"].as(); - if (!success) - { - const auto error = language->translate(val["error"].as()); - Log::error("Failed to delete files: {}", error); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = language->get("localSideModel", "deleteFilesFailedTitle"), - .text = error, - .buttons = ConfirmDialog::Button::Ok, - }); - // The other items may be gone, the listing has to show it. - navigateTo(currentPath_.value()); - return; - } +void LocalSideModel::deleteFilesDirectly(std::vector const& targets) +{ + Nui::val args = Nui::val::object(); + args.set("paths", Nui::val::array()); + for (auto const& target : targets) + args["paths"].call("push", target.toUtf8()); - Log::info("Files deleted successfully"); - // Refresh list: - navigateTo(currentPath_.value()); - }, - args - ); - }} + Nui::RpcClient::callWithBackChannel( + "RpcFilesystem::removeSome", + lifetime_.guarded( + [this](Nui::val val) + { + const auto error = [&val]() -> std::optional + { + if (!val.hasOwnProperty("success")) + return language->get("localSideModel", "invalidResponseFromBackend"); + if (val["success"].as()) + return std::nullopt; + return language->translate(val["error"].as()); + }(); + if (error) + { + Log::error("Failed to delete local items: {}", *error); + showDeleteFailed(*error); + } + else + Log::info("Local items deleted"); + // The listing shows what is gone, also after a failure that kept some items. + navigateTo(currentPath_.value()); + } + ), + args ); } + +void LocalSideModel::showDeleteFailed(std::string const& reason) +{ + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = language->get("localSideModel", "deleteFilesFailedTitle"), + .text = reason, + .buttons = ConfirmDialog::Button::Ok, + }); +} + void LocalSideModel::onTransfer( std::vector const& items, std::optional const& subDir @@ -885,6 +932,8 @@ void LocalSideModel::onProperties(NuiFileExplorer::Item const& item) .mtimeNsec = pickU32("mtimeNsec"), .acl = pickString("acl"), }; + if (source.hasOwnProperty("nameIsValidUtf8")) + out.nameIsValidUtf8 = source["nameIsValidUtf8"].template as(); if (source.hasOwnProperty("linkTarget")) { const auto link = source["linkTarget"]; @@ -1046,6 +1095,8 @@ void LocalSideModel::navigateTo(std::filesystem::path const& path) .mtime = pickU64("mtime"), .mtimeNsec = pickU32("mtimeNsec"), }; + if (source.hasOwnProperty("nameIsValidUtf8")) + entry.nameIsValidUtf8 = source["nameIsValidUtf8"].template as(); if (source.hasOwnProperty("linkTarget")) { const auto link = source["linkTarget"]; @@ -1492,7 +1543,13 @@ std::expected LocalSideModel::localPathOf(Nu { if (item.isParentLink) return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::DotOrDotDot, .input = ".."}); - return Paths::LocalPathSegment::make(item.name) + return item.nameSegment() + .and_then( + [](Paths::PathSegment const& segment) + { + return Paths::LocalPathSegment::make(segment); + } + ) .and_then( [&item](Paths::LocalPathSegment const&) { diff --git a/frontend/source/frontend/file_explorer/remote_side_model.cpp b/frontend/source/frontend/file_explorer/remote_side_model.cpp index 91a53bd9..0cbaa2cc 100644 --- a/frontend/source/frontend/file_explorer/remote_side_model.cpp +++ b/frontend/source/frontend/file_explorer/remote_side_model.cpp @@ -286,7 +286,7 @@ std::expected RemoteSideModel::remotePathOf { if (item.isParentLink) return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::DotOrDotDot, .input = ".."}); - return Paths::PathSegment::make(item.name) + return item.nameSegment() .and_then( [&item](Paths::PathSegment const&) { @@ -514,10 +514,8 @@ void RemoteSideModel::enqueueDeletes( std::vector filesAndEmptyDirs ) { - // Combine into a single bulk-delete request. File / empty-dir entries - // become one bulk-delete card; non-empty directories become individual - // Scan+Delete pairs (their isDirectory=true triggers the recursive flow - // backend-side). One RPC for the whole user action. + // One bulk-delete request and card for the whole user action. Non-empty directories are scanned and deleted + // with their content by the backend (isDirectory), files and empty directories are deleted as they are. std::vector entries; entries.reserve(filesAndEmptyDirs.size() + nonEmpties.size()); for (auto& path : filesAndEmptyDirs) @@ -542,7 +540,8 @@ void RemoteSideModel::enqueueDeletes( .styleVariant = ScriptNuiComponents::StyleVariant::Danger, .headerText = language->get("remoteSideModel", "deleteFailed"), .text = fmt::format( - fmt::runtime(language->get("remoteSideModel", "failedToCreateDeleteOperation")), info + fmt::runtime(language->get("remoteSideModel", "failedToCreateDeleteOperation")), + language->translate(info) ), .buttons = ConfirmDialog::Button::Ok, }); diff --git a/frontend/source/frontend/main.cpp b/frontend/source/frontend/main.cpp index 375cda59..993a3c45 100644 --- a/frontend/source/frontend/main.cpp +++ b/frontend/source/frontend/main.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -38,6 +39,92 @@ namespace } } +void loadState() +{ + persistence->load( + [](std::optional const&, Persistence::StateHolder&, Persistence::LoadWarnings const&) + { + Log::info("State loaded, setting up frontend."); + frontendEvents = std::make_unique(); + frontendEvents->onLanguageChanged.value() = persistence->stateCache().localizationOptions.languageCode; + frontendEvents->selectedTheme.value() = persistence->stateCache().uiOptions.theme; + frontendEvents->darkLightMode.value() = persistence->stateCache().uiOptions.darkLightMode; + + themeController = std::make_unique(*frontendEvents); + + persistence->loadLanguageFiles( + [](std::optional lang) + { + Log::info("Language file loaded, continuing setup."); + try + { + language = std::make_unique( + frontendEvents.get(), + lang.value_or( + []() + { + auto res = nlohmann::json::object(); + res["en_US"] = nlohmann::json::object(); + return res; + }() + ) + ); + + Log::info("Language file loaded."); + Log::info("Creating main page."); + mainPage = + std::make_unique(persistence.get(), frontendEvents.get(), *themeController); + + Log::info("Creating DOM."); + dom = std::make_unique(); + + Log::info("Rendering main page into DOM."); + dom->setBody(Nui::Elements::body{}(mainPage->render())); + + Log::info("Calling setup completion function"); + mainPage->onSetupComplete(); + + Log::debug("Dumping nui_rpc.backend and nui_rpc.frontend for debugging:"); + auto nuiRpc = Nui::val::global("nui_rpc"); + if (!nuiRpc.isUndefined() && nuiRpc.hasOwnProperty("backend")) + { + Log::debug("nui_rpc.backend:"); + printKeys(nuiRpc["backend"]); + Log::debug("nui_rpc.frontend:"); + printKeys(nuiRpc["frontend"]); + } + } + catch (std::exception const& exc) + { + Log::error("Failed to continue setup after loading language file: {}", exc.what()); + } + } + ); + } + ); +} + +/** + * @brief Takes the platform of the backend for local paths, then runs @p then. Local paths follow the rules of the + * machine the backend runs on, not WebAssembly's, so nothing may read one before. + */ +void withLocalPlatform(void (*then)()) +{ + Nui::RpcClient::callWithBackChannel( + "Main::getVersion", + [then](Nui::val response) + { + if (response.hasOwnProperty("localPlatform")) + { + Paths::defaultLocalPlatform() = response["localPlatform"].as() == "windows" + ? Paths::LocalPlatform::Windows + : Paths::LocalPlatform::Posix; + } + then(); + } + ); +} + bool tryLoad(std::shared_ptr const& setupWait) { static int counter = 0; @@ -58,67 +145,7 @@ bool tryLoad(std::shared_ptr const& setupWait) Log::info("terminalUtility available."); persistence = std::make_unique(); - persistence->load( - [](std::optional const&, Persistence::StateHolder&, Persistence::LoadWarnings const&) - { - Log::info("State loaded, setting up frontend."); - frontendEvents = std::make_unique(); - frontendEvents->onLanguageChanged.value() = persistence->stateCache().localizationOptions.languageCode; - frontendEvents->selectedTheme.value() = persistence->stateCache().uiOptions.theme; - frontendEvents->darkLightMode.value() = persistence->stateCache().uiOptions.darkLightMode; - - themeController = std::make_unique(*frontendEvents); - - persistence->loadLanguageFiles( - [](std::optional lang) - { - Log::info("Language file loaded, continuing setup."); - try - { - language = std::make_unique( - frontendEvents.get(), - lang.value_or( - []() - { - auto res = nlohmann::json::object(); - res["en_US"] = nlohmann::json::object(); - return res; - }() - ) - ); - - Log::info("Language file loaded."); - Log::info("Creating main page."); - mainPage = - std::make_unique(persistence.get(), frontendEvents.get(), *themeController); - - Log::info("Creating DOM."); - dom = std::make_unique(); - - Log::info("Rendering main page into DOM."); - dom->setBody(Nui::Elements::body{}(mainPage->render())); - - Log::info("Calling setup completion function"); - mainPage->onSetupComplete(); - - Log::debug("Dumping nui_rpc.backend and nui_rpc.frontend for debugging:"); - auto nuiRpc = Nui::val::global("nui_rpc"); - if (!nuiRpc.isUndefined() && nuiRpc.hasOwnProperty("backend")) - { - Log::debug("nui_rpc.backend:"); - printKeys(nuiRpc["backend"]); - Log::debug("nui_rpc.frontend:"); - printKeys(nuiRpc["frontend"]); - } - } - catch (std::exception const& exc) - { - Log::error("Failed to continue setup after loading language file: {}", exc.what()); - } - } - ); - } - ); + withLocalPlatform(&loadState); } else Log::info("Waiting for terminalUtility to be available."); diff --git a/frontend/source/frontend/main_page.cpp b/frontend/source/frontend/main_page.cpp index 59c05814..6e607615 100644 --- a/frontend/source/frontend/main_page.cpp +++ b/frontend/source/frontend/main_page.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -133,13 +132,6 @@ void MainPage::onSetupComplete() "Main::getVersion", [this](Nui::val response) { - // Local paths are parsed by the rules of the machine the backend runs on, not WebAssembly's. - if (response.hasOwnProperty("localPlatform")) - { - Paths::defaultLocalPlatform() = response["localPlatform"].as() == "windows" - ? Paths::LocalPlatform::Windows - : Paths::LocalPlatform::Posix; - } impl_->events->appVersion = AppVersion{ .version = response["version"].as(), .gitTag = response["gitTag"].as(), diff --git a/frontend/source/frontend/session.cpp b/frontend/source/frontend/session.cpp index 4a3ea8fc..6a495b18 100644 --- a/frontend/source/frontend/session.cpp +++ b/frontend/source/frontend/session.cpp @@ -83,17 +83,9 @@ struct Session::Implementation std::unique_ptr layoutInitializer; SyncDialog syncDialog; SyncProgressDialog syncProgressDialog; - // Holds the active BackendSyncProvider between progress-dialog open and - // sync-dialog close. Destroyed (→ provider.close() → backend session - // released) when replaced or explicitly reset. - std::shared_ptr syncProvider_; // Last local+remote paths given to startSyncFlow; used by the dialog's // Recompare path to re-issue the flow without asking the user again. std::filesystem::path lastSyncLocalPath_{}; - /** @brief Names the sync pair of the running flow in State::syncBaselines, unset without a saved session. */ - std::optional syncBaselineKey_{}; - /** @brief When the running flow started its compare, recorded as the baseline once it is synchronized. */ - std::uint64_t syncFlowStart_{0}; std::filesystem::path lastSyncRemotePath_{}; std::unique_ptr connectionLossOverlay; std::unique_ptr snapshotManager; @@ -190,25 +182,7 @@ struct Session::Implementation { // Recompare: rerun the whole scan-and-diff flow with the dialog's current // settings, reusing the cached local+remote roots. The old provider is - // replaced by a fresh one; the backend session it held is closed on - // destruction. - // A synchronized selection makes the time its compare started the baseline of the next sync. - syncDialog.setOnSynchronized( - [this]() - { - if (!syncBaselineKey_ || !this->stateHolder) - return; - this->stateHolder->loadModifySave( - [key = *syncBaselineKey_, time = syncFlowStart_](Persistence::State& state) - { - if (!state.syncBaselines) - state.syncBaselines.emplace(); - (*state.syncBaselines)[key] = time; - } - ); - } - ); - + // replaced by a fresh one. syncDialog.setOnRecompareRequested( [this](SyncDialog::RecompareRequest req) { @@ -240,41 +214,20 @@ struct Session::Implementation lastSyncLocalPath_ = localPath; lastSyncRemotePath_ = remotePath; - // Only a saved session has a stable identity to remember the last sync of. - syncBaselineKey_.reset(); - std::optional lastSyncTime{}; - if (persistenceName && !persistenceName->empty()) - { - syncBaselineKey_ = - fmt::format("{}\n{}\n{}", *persistenceName, localPath.generic_string(), remotePath.generic_string()); - if (auto const& baselines = stateHolder->stateCache().syncBaselines; baselines) - { - if (const auto found = baselines->find(*syncBaselineKey_); found != baselines->end()) - lastSyncTime = found->second; - } - } - syncFlowStart_ = static_cast( - std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() - ); - initialDiffOptions.lastSyncTime = lastSyncTime; - - // Replace the old provider before opening the progress dialog so its - // backend session (if any) is closed before the new one opens. - syncProvider_ = std::make_shared(&operationQueue); - + // The progress dialog holds the provider while it scans and compares, then hands it to the sync dialog. syncProgressDialog.open( - syncProvider_, + std::make_shared(&operationQueue), localPath, remotePath, scan.respectIgnoreFiles, scan.recursive, scan.ignoreHidden, initialDiffOptions, - [this, provider = syncProvider_, localPath, remotePath, scan, lastSyncTime]( - SharedData::Sync::DiffSnapshot snapshot + [this, localPath, remotePath, scan]( + std::shared_ptr provider, SharedData::Sync::DiffSnapshot snapshot ) { - syncDialog.open(provider, std::move(snapshot), localPath, remotePath, scan, lastSyncTime); + syncDialog.open(std::move(provider), std::move(snapshot), localPath, remotePath, scan); Nui::globalEventContext.executeActiveEventsImmediately(); } ); @@ -572,6 +525,8 @@ std::optional Session::getLayout() const void Session::setupFileGrid() { impl_->fileExplorerPanel.setup(); + // Local deletes need no connection, their cards show in this queue from the start. + localSideModel().operationQueue(&impl_->operationQueue); // Sync dialog + progress dialog stay on Session; wire them through the // panel's synchronize callback. Every flow scans and diffs with the diff --git a/frontend/source/frontend/session_components/command_snippets_panel.cpp b/frontend/source/frontend/session_components/command_snippets_panel.cpp index 3ffa6737..43a7be5f 100644 --- a/frontend/source/frontend/session_components/command_snippets_panel.cpp +++ b/frontend/source/frontend/session_components/command_snippets_panel.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -45,6 +46,7 @@ #include #include #include +#include #include #include @@ -474,14 +476,11 @@ struct CommandSnippetsPanel::Implementation if (!unmatched) return {}; constexpr std::size_t excerptLength = 24u; - auto excerptEnd = std::min(command.size(), *unmatched + excerptLength); - // Never cut a UTF-8 sequence in half. - while (excerptEnd < command.size() && (static_cast(command[excerptEnd]) & 0xC0u) == 0x80u) - --excerptEnd; - const auto excerpt = command.substr(*unmatched, excerptEnd - *unmatched); + const auto rest = std::string_view{command}.substr(*unmatched); + const auto excerpt = Paths::truncatedUtf8(rest, excerptLength); return fmt::format( fmt::runtime(language->get("commandSnippetsPanel", "unmatchedBraces")), - excerptEnd < command.size() ? fmt::format("{}...", excerpt) : excerpt + excerpt.size() < rest.size() ? fmt::format("{}...", excerpt) : std::string{excerpt} ); } diff --git a/frontend/source/frontend/session_components/local_operation_router.cpp b/frontend/source/frontend/session_components/local_operation_router.cpp new file mode 100644 index 00000000..c25ed827 --- /dev/null +++ b/frontend/source/frontend/session_components/local_operation_router.cpp @@ -0,0 +1,231 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include + +namespace +{ + /** + * @brief Tells the user how a delete ended that no queue showed. + */ + void notifyUnshownCompletion(SharedData::OperationCompleted const& completed, std::string const& directory) + { + using enum SharedData::OperationCompletionReason; + switch (completed.reason) + { + case Completed: + { + if (completed.failedEntries.empty()) + return Notifications::success( + fmt::format(fmt::runtime(language->get("operationQueue", "localDeleteDone")), directory) + ); + return Notifications::warning( + fmt::format(fmt::runtime(language->get("operationQueue", "localDeletePartlyDone")), directory) + ); + } + case Failed: + { + return Notifications::error( + fmt::format( + fmt::runtime(language->get("operationQueue", "localDeleteFailed")), + directory, + completed.error ? completed.error->toString() : language->get("operationQueue", "unknownError") + ) + ); + } + case Canceled: + return; + } + } +} + +LocalOperationRouter& LocalOperationRouter::instance() +{ + static LocalOperationRouter router; + return router; +} + +LocalOperationRouter::LocalOperationRouter() +{ + registrations_.push_back( + Nui::RpcClient::autoRegisterFunction( + "LocalOperationQueue::onOperationAdded", + [this](SharedData::OperationAdded const& added) + { + onOperationAdded(added); + } + ) + ); + registrations_.push_back( + Nui::RpcClient::autoRegisterFunction( + "LocalOperationQueue::onScanProgress", + [this](SharedData::ScanProgress const& progress) + { + onScanProgress(progress); + } + ) + ); + registrations_.push_back( + Nui::RpcClient::autoRegisterFunction( + "LocalOperationQueue::onDeleteProgress", + [this](SharedData::BulkDeleteProgress const& progress) + { + onDeleteProgress(progress); + } + ) + ); + registrations_.push_back( + Nui::RpcClient::autoRegisterFunction( + "LocalOperationQueue::onOperationCompleted", + [this](Nui::val val) + { + onOperationCompleted(std::move(val)); + } + ) + ); +} + +void LocalOperationRouter::attach(OperationQueue& queue) +{ + queues_.push_back(&queue); + for (auto& [operationId, routed] : routed_) + { + if (routed.host) + continue; + routed.host = &queue; + show(routed, queue); + } +} + +void LocalOperationRouter::detach(OperationQueue& queue) +{ + std::erase(queues_, &queue); + auto* const successor = newestQueue(); + for (auto& [operationId, routed] : routed_) + { + if (routed.host != &queue) + continue; + routed.host = successor; + if (successor) + show(routed, *successor); + } +} + +void LocalOperationRouter::host(Ids::OperationId const& operationId, OperationQueue& queue) +{ + routed_[operationId].host = &queue; +} + +void LocalOperationRouter::forget(Ids::OperationId const& operationId) +{ + routed_.erase(operationId); +} + +void LocalOperationRouter::cancelHostedBy(OperationQueue const& queue) +{ + std::erase_if( + routed_, + [&queue](auto const& entry) + { + auto const& [operationId, routed] = entry; + if (routed.host != &queue) + return false; + Nui::RpcClient::callWithBackChannel( + "LocalOperationQueue::cancel", + [operationId](SharedData::ErrorOrSuccess<> const& result) + { + if (!result) + Log::warn( + "Failed to cancel the local operation '{}': {}", operationId.value(), result.error.value() + ); + }, + operationId + ); + return true; + } + ); +} + +void LocalOperationRouter::onOperationAdded(SharedData::OperationAdded const& added) +{ + auto [found, inserted] = routed_.try_emplace(added.operationId); + auto& routed = found->second; + if (inserted) + routed.host = newestQueue(); + routed.added = added; + if (routed.host) + routed.host->onOperationAdded(added); +} + +void LocalOperationRouter::onScanProgress(SharedData::ScanProgress const& progress) +{ + const auto found = routed_.find(progress.operationId); + if (found == routed_.end()) + return; + found->second.scanProgress = progress; + if (found->second.host) + found->second.host->onScanProgress(progress); +} + +void LocalOperationRouter::onDeleteProgress(SharedData::BulkDeleteProgress const& progress) +{ + const auto found = routed_.find(progress.operationId); + if (found == routed_.end()) + return; + found->second.deleteProgress = progress; + if (found->second.host) + found->second.host->onDeleteProgress(progress); +} + +void LocalOperationRouter::onOperationCompleted(Nui::val val) +{ + SharedData::OperationCompleted completed{}; + try + { + Nui::convertFromVal(val, completed); + } + catch (std::exception const& exception) + { + Log::error("Failed to read the completion of a local operation: {}", exception.what()); + return; + } + + auto routed = routed_.extract(completed.operationId); + if (routed.empty()) + { + Log::info("Local operation '{}' ended without a card.", completed.operationId.value()); + return; + } + if (auto* const host = routed.mapped().host; host) + return host->onOperationCompleted(std::move(completed)); + + auto const& added = routed.mapped().added; + notifyUnshownCompletion(completed, added && added->localPath ? added->localPath->toUtf8() : std::string{}); +} + +void LocalOperationRouter::show(Routed& routed, OperationQueue& queue) +{ + if (!routed.added) + return; + queue.onOperationAdded(*routed.added); + if (routed.scanProgress) + queue.onScanProgress(*routed.scanProgress); + if (routed.deleteProgress) + queue.onDeleteProgress(*routed.deleteProgress); +} + +OperationQueue* LocalOperationRouter::newestQueue() const +{ + return queues_.empty() ? nullptr : queues_.back(); +} diff --git a/frontend/source/frontend/session_components/operation_queue.cpp b/frontend/source/frontend/session_components/operation_queue.cpp index 8ac8f71f..ad77505b 100644 --- a/frontend/source/frontend/session_components/operation_queue.cpp +++ b/frontend/source/frontend/session_components/operation_queue.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +60,8 @@ struct OperationQueue::Implementation std::vector onUpdate; Nui::Observed paused{true}; + /** @brief The profile starts the queue of every connection paused, the backend creates it paused. */ + bool startsPaused{true}; std::shared_ptr> autoClean{std::make_shared>(false)}; ObservedRandomAccessMap priorityOperations; ObservedRandomAccessMap operations; @@ -86,8 +90,7 @@ struct OperationQueue::Implementation Nui::TimerHandle autoCleanTimer; std::unordered_map> completionCallbacks; - std::unordered_map const&)>> - failedEntriesCallbacks; + std::unordered_map failedEntriesCallbacks; std::unordered_map> transferProgressCallbacks; // Keyed by the aggregate bulk operation id (first pre-generated opId for // bulk up/download, or the aggregate id for bulk delete). Auto-erased on @@ -311,7 +314,8 @@ OperationQueue::OperationQueue( ), } { - impl_->paused = queueOptions.startInPausedState.value_or(true); + impl_->startsPaused = queueOptions.startInPausedState.value_or(true); + impl_->paused = impl_->startsPaused; *impl_->autoClean = queueOptions.autoRemoveCompletedOperations.value_or(false); impl_->liveQueuePageSize = std::clamp(queueOptions.liveQueuePageSize.value_or(200), 1, 1000); @@ -376,11 +380,17 @@ OperationQueue::OperationQueue( impl_->autoCleanTimer = std::move(handle); } ); + LocalOperationRouter::instance().attach(*this); +} + +OperationQueue::~OperationQueue() +{ + LocalOperationRouter::instance().detach(*this); } -ROAR_PIMPL_SPECIAL_FUNCTIONS_IMPL(OperationQueue); void OperationQueue::cancelAll() { + LocalOperationRouter::instance().cancelHostedBy(*this); impl_->operations.clear(); impl_->priorityOperations.clear(); impl_->history.clear(); @@ -402,10 +412,14 @@ void OperationQueue::cancelOperation(OperationCard const& operation) return; } - auto doCancel = [this, operationId = operation.operationId()]() + auto doCancel = [this, + operationId = operation.operationId(), + cancelName = operation.type() == SharedData::OperationType::LocalDelete + ? std::string{"LocalOperationQueue::cancel"} + : fmt::format("OperationQueue::{}::cancel", impl_->sessionId.value())]() { Nui::RpcClient::callWithBackChannel( - fmt::format("OperationQueue::{}::cancel", impl_->sessionId.value()), + cancelName, [this, operationId](SharedData::ErrorOrSuccess<> const& result) { if (!result) @@ -802,7 +816,8 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) cancelOperation(operation); }, impl_->autoClean, - std::function{} + localRefresh, + added.scanPhase.value_or(false) ); } else if (added.type == SharedData::OperationType::BulkDownload) @@ -956,7 +971,7 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) // Kick-to-top is meaningful only for the regular queue. Priority cards do // not get a handler — even though the button is rendered, CSS hides it // when the parent is the priority list (see .opq-priority-list rule). - if (added.mode != SharedData::OperationMode::PriorityQueued) + if (added.mode != SharedData::OperationMode::PriorityQueued && added.type != SharedData::OperationType::LocalDelete) { card->setKickToTopHandler( [this, opId = added.operationId]() @@ -1082,6 +1097,7 @@ void OperationQueue::onScanProgress(SharedData::ScanProgress const& progress) } case SharedData::OperationType::Delete: case SharedData::OperationType::BulkDelete: + case SharedData::OperationType::LocalDelete: { if (auto* card = operation->getCardSpecifically(); card) card->setScanProgress(progress.totalBytes, progress.totalScanned); @@ -1279,7 +1295,11 @@ void OperationQueue::onOperationCompleted(Nui::val val) Log::error("Failed to convert OperationCompleted: {}", e.what()); return; } + onOperationCompleted(std::move(completed)); +} +void OperationQueue::onOperationCompleted(SharedData::OperationCompleted completed) +{ Log::info("Received operation completed for operation id: {}", completed.operationId.value()); auto* operation = impl_->findOperation(completed.operationId); if (!operation) @@ -1293,14 +1313,8 @@ void OperationQueue::onOperationCompleted(Nui::val val) { auto callback = std::move(failedIt->second); impl_->failedEntriesCallbacks.erase(failedIt); - if (!completed.failedEntries.empty()) - { - std::vector failedPaths; - failedPaths.reserve(completed.failedEntries.size()); - for (auto const& entry : completed.failedEntries) - failedPaths.push_back(entry.first); - callback(failedPaths); - } + if (!completed.failedEntries.empty() || !completed.indexedFailures.empty()) + callback(completed.failedEntries, completed.indexedFailures); } bool partial = false; @@ -1363,15 +1377,48 @@ void OperationQueue::onOperationCompleted(Nui::val val) Nui::globalEventContext.executeActiveEventsImmediately(); } +void OperationQueue::enqueueLocalDelete( + std::vector const& targets, + std::function const& error)> onAck +) +{ + std::vector paths; + paths.reserve(targets.size()); + std::ranges::transform(targets, std::back_inserter(paths), &Paths::LocalPath::toUtf8); + Log::info("Frontend Operation Queue local delete: {} items", paths.size()); + + const auto operationId = Ids::generateOperationId(); + LocalOperationRouter::instance().host(operationId, *this); + // With a connection the delete is held while the queue of the connection is paused. + const auto rpcName = isActive() ? fmt::format("Session::{}::addLocalDelete", impl_->sessionId.value()) + : std::string{"LocalOperationQueue::addDelete"}; + Nui::RpcClient::callWithBackChannel( + rpcName, + [onAck = std::move(onAck), operationId](SharedData::ErrorOrSuccess<> const& result) + { + if (!result) + { + LocalOperationRouter::instance().forget(operationId); + return onAck(result.error); + } + onAck(std::nullopt); + }, + operationId.value(), + paths + ); +} + +bool OperationQueue::isActive() const +{ + return impl_->sessionId.isValid(); +} + void OperationQueue::addCompletionCallback(Ids::OperationId const& opId, std::function callback) { impl_->completionCallbacks[opId.value()] = std::move(callback); } -void OperationQueue::addFailedEntriesCallback( - Ids::OperationId const& opId, - std::function const&)> callback -) +void OperationQueue::addFailedEntriesCallback(Ids::OperationId const& opId, FailedEntriesCallback callback) { impl_->failedEntriesCallbacks[opId.value()] = std::move(callback); } @@ -1561,7 +1608,22 @@ void OperationQueue::executeSyncSelection( { if (!impl_->fileEngine) return onRejected("No file engine set"); - impl_->fileEngine->executeSyncSelection(std::move(request), std::move(onResolved), std::move(onRejected)); + const auto localDeleteId = request.localDeleteId; + LocalOperationRouter::instance().host(localDeleteId, *this); + impl_->fileEngine->executeSyncSelection( + std::move(request), + [onResolved = std::move(onResolved), localDeleteId](SharedData::Sync::SyncExecutionResult result) + { + if (result.localDeletes.empty()) + LocalOperationRouter::instance().forget(localDeleteId); + onResolved(std::move(result)); + }, + [onRejected = std::move(onRejected), localDeleteId](std::string const& reason) + { + LocalOperationRouter::instance().forget(localDeleteId); + onRejected(reason); + } + ); } void OperationQueue::cancelSyncDiff(Ids::SyncSessionId syncSessionId) @@ -1622,6 +1684,10 @@ void OperationQueue::requestMoveOperation(Ids::OperationId const& opId, std::siz // avoids a wasted round trip and a refused log on the backend side. return; } + // Local operations run in the backend's local operation queue, which keeps its own order. + if (const auto* operation = impl_->findOperation(opId); + !isActive() || (operation && operation->type() == SharedData::OperationType::LocalDelete)) + return; Nui::RpcClient::callWithBackChannel( fmt::format("OperationQueue::{}::moveOperation", impl_->sessionId.value()), [opId](SharedData::ErrorOrSuccess<> const& result) @@ -1641,6 +1707,8 @@ void OperationQueue::onIsPaused(SharedData::ErrorOrSuccess Log::info("Initial pause state of operation queue is: {}", result.paused ? "paused" : "running"); impl_->paused = result.paused; Nui::globalEventContext.executeActiveEventsImmediately(); + if (result.paused && !impl_->startsPaused) + unpause(); } else { @@ -1673,6 +1741,11 @@ void OperationQueue::togglePause() void OperationQueue::askBackendToCancelAll() { + if (!isActive()) + { + cancelAll(); + return; + } Nui::RpcClient::callWithBackChannel( fmt::format("OperationQueue::{}::cancelAll", impl_->sessionId.value()), [this](SharedData::ErrorOrSuccess<> const& result) diff --git a/frontend/source/frontend/settings/search/settings_search.cpp b/frontend/source/frontend/settings/search/settings_search.cpp index 34ba356d..53aad7aa 100644 --- a/frontend/source/frontend/settings/search/settings_search.cpp +++ b/frontend/source/frontend/settings/search/settings_search.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -137,7 +138,7 @@ namespace { auto text = value.is_string() ? value.get() : value.dump(); if (text.size() > maximumValueLength) - text = text.substr(0, maximumValueLength) + "..."; + return fmt::format("{}...", Paths::truncatedUtf8(text, maximumValueLength)); return text; } @@ -506,8 +507,8 @@ struct SettingsSearch::Implementation const auto key = event["key"].as(); const auto count = selectableCount(); const auto caret = inputElement.isUndefined() ? 0 : inputElement["selectionStart"].as(); - const auto valueLength = - inputElement.isUndefined() ? 0 : static_cast(inputElement["value"].as().size()); + // selectionStart counts UTF-16 code units, like the length of the JavaScript string. + const auto valueLength = inputElement.isUndefined() ? 0 : inputElement["value"]["length"].as(); const auto consume = [&event]() { diff --git a/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp index e8048136..c19e75cc 100644 --- a/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp +++ b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp @@ -88,7 +88,12 @@ void BackendSyncProvider::recompute( ) { if (!queue_ || closed_) + { + SharedData::Sync::DiffSnapshot nothing{}; + nothing.summary.cancelled = true; + onSnapshot(std::move(nothing)); return; + } queue_->recomputeSyncDiff( sessionId_, @@ -115,7 +120,7 @@ void BackendSyncProvider::loadChildren( ) { if (!queue_ || closed_) - return; + return onRejected("closed"); const auto expectedGeneration = generation_; queue_->loadSyncDiffChildren( @@ -135,7 +140,11 @@ void BackendSyncProvider::loadChildren( return; onResolved(std::move(nodes)); }, - std::move(onRejected) + [alive = std::weak_ptr{alive_}, onRejected = std::move(onRejected)](std::string const& message) + { + if (alive.lock()) + onRejected(message); + } ); } @@ -146,19 +155,24 @@ void BackendSyncProvider::executeSelection( ) { if (!queue_ || closed_) - return; + return onRejected("closed"); request.syncSessionId = sessionId_; request.generation = generation_; queue_->executeSyncSelection( std::move(request), - [alive = std::weak_ptr{alive_}, onResolved = std::move(onResolved)](SharedData::Sync::SyncExecutionResult result) + [alive = std::weak_ptr{alive_}, + onResolved = std::move(onResolved)](SharedData::Sync::SyncExecutionResult result) { // The operations are queued either way, the reply only maps them to rows of a dialog that may be gone. if (alive.lock()) onResolved(std::move(result)); }, - std::move(onRejected) + [alive = std::weak_ptr{alive_}, onRejected = std::move(onRejected)](std::string const& message) + { + if (alive.lock()) + onRejected(message); + } ); } diff --git a/frontend/source/frontend/sync_dialog/sync_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_dialog.cpp index 5ea8c275..ad7e11b8 100644 --- a/frontend/source/frontend/sync_dialog/sync_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_dialog.cpp @@ -3,8 +3,10 @@ #include #include #include +#include #include +#include #include #include @@ -41,7 +43,6 @@ #include #include #include -#include #include #include @@ -51,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -265,6 +267,10 @@ struct SyncDialog::Implementation bool selectionTouched{false}; /// What the tree shows, to skip re-rendering a section a recompute left unchanged. std::vector applied{}; + /** @brief What executions reported per row, see setRowState. */ + std::unordered_map executionStates{}; + /** @brief Why a failed row failed, for the failures a row explains. */ + std::unordered_map failureReasons{}; }; std::array sections_{}; @@ -279,9 +285,14 @@ struct SyncDialog::Implementation ConfirmDialog* confirmDialog_; OperationQueue* operationQueue_; std::function onRecompareRequested_{}; - std::function onSynchronized_{}; - /** @brief When the shown pair was last synchronized, part of every diff. */ - std::optional lastSyncTime_{}; + /** + * @brief False while a recompute runs or after one was cancelled: the backend's diff is not the one the rows + * show, so Synchronize is off. + */ + Nui::Observed diffCurrent_{true}; + /** @brief Operations of executions whose callbacks this dialog registered with the queue. */ + std::vector registeredOperations_{}; + Utility::LifetimeToken lifetime_{}; explicit Implementation(ConfirmDialog* confirmDialog, OperationQueue* operationQueue) : confirmDialog_{confirmDialog} @@ -291,6 +302,15 @@ struct SyncDialog::Implementation initTrees(); } + ~Implementation() + { + dropRegistrations(); + } + Implementation(Implementation const&) = delete; + Implementation& operator=(Implementation const&) = delete; + Implementation(Implementation&&) = delete; + Implementation& operator=(Implementation&&) = delete; + SectionState& stateFor(DiffSection section) { return sections_[static_cast(section)]; @@ -562,7 +582,10 @@ struct SyncDialog::Implementation } } - /** @brief Remembers rows the tree is about to show and gives file rows their progress observer. */ + /** + * @brief Remembers rows the tree is about to show and gives file rows their progress observer, which starts + * with what an execution already reported for the row. + */ void recordRows(DiffSection section, std::vector const& rows) { auto& state = stateFor(section); @@ -570,32 +593,64 @@ struct SyncDialog::Implementation for (auto const& row : rows) { state.rows.insert_or_assign(row.relKey, row); - if (!row.isDirectory) - progress.try_emplace(row.relKey, std::make_shared>(0.0)); + if (row.isDirectory) + continue; + const auto reported = state.executionStates.find(row.relKey); + progress.try_emplace( + row.relKey, + std::make_shared>( + reported == state.executionStates.end() ? 0.0 : reported->second + ) + ); } } - ProgressObserver progressFor(DiffSection section, std::string const& relKey) + /** + * @brief Shows the progress of a row: a fraction while it runs, above 1 when done, below 0 when it failed. Only + * loaded rows have an observer, the others pick the state up when they load. + */ + void setRowState(DiffSection section, std::string const& relKey, double value) { - auto& entry = progressMapFor(section)[relKey]; - if (!entry) - entry = std::make_shared>(0.0); - return entry; + auto& state = stateFor(section); + state.executionStates.insert_or_assign(relKey, value); + if (value >= 0.0) + state.failureReasons.erase(relKey); + auto const& progress = progressMapFor(section); + if (const auto observer = progress.find(relKey); observer != progress.end()) + *observer->second = value; } - /** @brief Observer of @p relKey plus those of the loaded file rows below it, so a directory - * entry paints the rows it covers. + /** + * @brief Shows a row as failed with the reason of @p type, for the failures a row explains: a destination or + * delete target that changed after the compare. A sync entry may only replace what the compare saw, so a + * destination that appeared since fails as existing. */ - std::vector progressCovering(DiffSection section, std::string const& relKey) + void setRowFailed(DiffSection section, std::string const& relKey, SharedData::OperationErrorType type) { - std::vector observers{progressFor(section, relKey)}; - const auto prefix = relKey + "/"; - for (auto const& [key, observer] : progressMapFor(section)) - { - if (key.starts_with(prefix)) - observers.push_back(observer); - } - return observers; + using enum SharedData::OperationErrorType; + if (type == ChangedSinceCompare || type == FileExists) + stateFor(section).failureReasons.insert_or_assign( + relKey, std::string{language->get("syncDialog", "changedSinceCompare")} + ); + setRowState(section, relKey, -1.0); + // A row shown as failed already only gets its reason. + auto const& progress = progressMapFor(section); + if (const auto observer = progress.find(relKey); observer != progress.end()) + observer->second->modify(); + } + + std::string failureReasonOf(DiffSection section, std::string const& relKey) const + { + auto const& reasons = stateFor(section).failureReasons; + const auto found = reasons.find(relKey); + return found == reasons.end() ? std::string{} : found->second; + } + + double rowState(DiffSection section, std::string const& relKey) const + { + auto const& states = stateFor(section).executionStates; + const auto found = states.find(relKey); + return found == states.end() ? 0.0 : found->second; } ScriptNuiComponents::Tree::ChildrenLoader makeChildrenLoader(DiffSection section) @@ -716,6 +771,20 @@ struct SyncDialog::Implementation return !state.rows.contains(entry.first); } ); + std::erase_if( + state.executionStates, + [&state](auto const& entry) + { + return !state.rows.contains(entry.first); + } + ); + std::erase_if( + state.failureReasons, + [&state](auto const& entry) + { + return !state.rows.contains(entry.first); + } + ); auto const& selection = selectionFor(section); auto& set = selection->value(); @@ -778,7 +847,6 @@ struct SyncDialog::Implementation .actionDelete = actionDelete_.value(), .recursive = recursive_.value(), .ignoreHidden = ignoreHidden_.value(), - .lastSyncTime = lastSyncTime_, }; } @@ -814,6 +882,7 @@ struct SyncDialog::Implementation return; } recomputeInFlight_ = true; + diffCurrent_ = false; provider_->recompute( buildDiffOptions(), SharedData::Sync::LoadedParents{ @@ -821,19 +890,25 @@ struct SyncDialog::Implementation .downloads = loadedParentsOf(DiffSection::Download), .deletes = loadedParentsOf(DiffSection::Delete), }, - [this](SharedData::Sync::DiffSnapshot snapshot) - { - recomputeInFlight_ = false; - if (recomputeQueued_) + lifetime_.guarded( + [this](SharedData::Sync::DiffSnapshot snapshot) { - recomputeQueued_ = false; - recomputeDiff(); - return; + recomputeInFlight_ = false; + if (recomputeQueued_) + { + recomputeQueued_ = false; + recomputeDiff(); + return; + } + if (snapshot.summary.cancelled) + { + Nui::globalEventContext.executeActiveEventsImmediately(); + return; + } + diffCurrent_ = true; + applySnapshot(std::move(snapshot), /*collapseZero=*/true); } - if (snapshot.summary.cancelled) - return; - applySnapshot(std::move(snapshot), /*collapseZero=*/true); - } + ) ); } @@ -961,9 +1036,12 @@ struct SyncDialog::Implementation enqueueSelection(section, {relKey}, SharedData::OperationMode::PriorityQueued); } + /** + * @brief Queues the selection of every section. + */ void enqueueOperations() { - if (!provider_) + if (!provider_ || !diffCurrent_.value()) return; for (const auto section : {DiffSection::Upload, DiffSection::Download, DiffSection::Delete}) { @@ -979,55 +1057,86 @@ struct SyncDialog::Implementation */ struct SyncPart { - /** @brief One observer per entry index; a directory delete lists its content as entries of their own. */ - std::vector observers{}; + DiffSection section{DiffSection::Upload}; + /** @brief The row of every entry index; a directory delete lists its content as entries of their own. */ + std::vector rows{}; + /** @brief Deletes report failures by path, transfers by entry index. */ std::unordered_map indexBySource{}; + std::vector failed{}; std::set seenInFlight{}; bool mapped{false}; std::optional completion{}; - std::vector failedSources{}; + std::vector> failedSources{}; + std::vector failedIndices{}; + }; - void markFailedSources() - { - for (auto const& source : failedSources) - { - if (const auto iter = indexBySource.find(source); iter != indexBySource.end()) - *observers[iter->second] = -1.0; - } - failedSources.clear(); - } + /** + * @brief Shows the entry @p index as failed. Without @p type the reason is not known yet, a later report may + * add it. + */ + void markFailedEntry(SyncPart& part, std::size_t index, std::optional type = {}) + { + if (index >= part.rows.size() || (part.failed[index] && !type)) + return; + part.failed[index] = true; + if (type) + setRowFailed(part.section, part.rows[index], *type); + else + setRowState(part.section, part.rows[index], -1.0); + } - void complete() + void markReportedFailures(SyncPart& part) + { + for (auto const& [source, type] : std::exchange(part.failedSources, {})) { - for (auto const& observer : observers) - { - // Rows already marked failed stay failed on an otherwise successful operation. - if (!(*completion && observer->value() < 0.0)) - *observer = *completion ? 1.1 : -1.0; - } + if (const auto iter = part.indexBySource.find(source); iter != part.indexBySource.end()) + markFailedEntry(part, iter->second, type); } + for (auto const& failure : std::exchange(part.failedIndices, {})) + markFailedEntry(part, failure.index, failure.type); + } - /** @brief The backend answered with the rows of this part. */ - void map() + void completePart(SyncPart& part) + { + // Rows already marked failed stay failed on an otherwise successful operation. + for (std::size_t index = 0; index < part.rows.size(); ++index) { - mapped = true; - markFailedSources(); - if (completion) - complete(); + if (!part.failed[index]) + setRowState(part.section, part.rows[index], *part.completion ? 1.1 : -1.0); } - }; + } + /** @brief The backend answered with the rows of @p part. */ + void mapPart(SyncPart& part, std::vector rows, std::vector const& sources) + { + part.rows = std::move(rows); + part.failed.assign(part.rows.size(), false); + for (std::size_t index = 0; index < sources.size() && index < part.rows.size(); ++index) + part.indexBySource.emplace(sources[index], index); + part.mapped = true; + markReportedFailures(part); + if (part.completion) + completePart(part); + } + + /** + * @brief Registers the completion of @p operationId with the queue, kept for as long as this dialog shows the + * execution, see @ref dropRegistrations. + */ void watchCompletion(Ids::OperationId const& operationId, std::shared_ptr part) { + registeredOperations_.push_back(operationId); operationQueue_->addCompletionCallback( operationId, - [part](bool success) - { - part->completion = success; - if (part->mapped) - part->complete(); - Nui::globalEventContext.executeActiveEventsImmediately(); - } + lifetime_.guarded( + [this, part](bool success) + { + part->completion = success; + if (part->mapped) + completePart(*part); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ) ); } @@ -1036,64 +1145,92 @@ struct SyncDialog::Implementation { operationQueue_->addBulkProgressCallback( operationId, - [part](SharedData::BulkProgress const& progress) - { - if (!part->mapped) - return; - auto& observers = part->observers; - for (const auto entryIndex : progress.failedEntryIndices) - { - if (entryIndex < observers.size() && observers[entryIndex]->value() >= 0.0) - *observers[entryIndex] = -1.0; - } - std::set stillInFlight; - for (auto const& file : progress.inFlight) + lifetime_.guarded( + [this, part](SharedData::BulkProgress const& progress) { - if (file.entryIndex >= observers.size()) - continue; - stillInFlight.insert(file.entryIndex); - part->seenInFlight.insert(file.entryIndex); - if (file.totalBytes > 0) - *observers[file.entryIndex] = - static_cast(file.bytes) / static_cast(file.totalBytes); - } - std::erase_if( - part->seenInFlight, - [&](std::uint64_t entryIndex) + if (!part->mapped) + return; + for (const auto entryIndex : progress.failedEntryIndices) + markFailedEntry(*part, entryIndex); + std::set stillInFlight; + for (auto const& file : progress.inFlight) { - if (stillInFlight.contains(entryIndex)) - return false; - auto& observer = observers[entryIndex]; - if (observer->value() >= 0.0 && observer->value() < 1.0) - *observer = 1.1; - return true; + if (file.entryIndex >= part->rows.size() || part->failed[file.entryIndex]) + continue; + stillInFlight.insert(file.entryIndex); + part->seenInFlight.insert(file.entryIndex); + if (file.totalBytes > 0) + setRowState( + part->section, + part->rows[file.entryIndex], + static_cast(file.bytes) / static_cast(file.totalBytes) + ); } - ); - Nui::globalEventContext.executeActiveEventsImmediately(); - } + std::erase_if( + part->seenInFlight, + [&](std::uint64_t entryIndex) + { + if (stillInFlight.contains(entryIndex)) + return false; + auto const& row = part->rows[entryIndex]; + if (!part->failed[entryIndex] && rowState(part->section, row) < 1.0) + setRowState(part->section, row, 1.1); + return true; + } + ); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ) ); watchFailedEntries(operationId, std::move(part)); } - /** @brief Failed entries arrive by their source path and mark the rows they belong to; then completion. */ + /** @brief Failed entries arrive on completion, by source path and for bulk transfers by index; then completion. */ void watchFailedEntries(Ids::OperationId const& operationId, std::shared_ptr part) { operationQueue_->addFailedEntriesCallback( operationId, - [part](std::vector const& failedPaths) - { - for (auto const& path : failedPaths) - part->failedSources.push_back(path); - if (part->mapped) - part->markFailedSources(); - } + lifetime_.guarded( + [this, part]( + std::vector> const& failedEntries, + std::vector const& indexedFailures + ) + { + std::ranges::transform( + failedEntries, + std::back_inserter(part->failedSources), + [](auto const& failure) + { + return std::pair{failure.first, failure.second.type}; + } + ); + part->failedIndices.insert( + part->failedIndices.end(), indexedFailures.begin(), indexedFailures.end() + ); + if (part->mapped) + markReportedFailures(*part); + } + ) ); watchCompletion(operationId, std::move(part)); } + /** + * @brief Removes the callbacks of every execution this dialog registered with the queue. Their operations keep + * running, only the rows they would paint are gone. + */ + void dropRegistrations() + { + if (!operationQueue_) + return; + for (auto const& operationId : std::exchange(registeredOperations_, {})) + operationQueue_->removeCallbacks(operationId); + } + /** * @brief Has the backend plan and queue the sparse selection @p relKeys. The operation ids are chosen here and - * listened to before the request goes out, the answer maps them to rows. + * listened to before the request goes out, the answer maps them to rows. A refused request queues + * nothing, its rows show as failed and the reason is shown. */ void enqueueSelection(DiffSection section, std::vector relKeys, SharedData::OperationMode mode) { @@ -1109,10 +1246,14 @@ struct SyncDialog::Implementation .remoteDeleteId = Ids::generateOperationId(), .localDeleteId = Ids::generateOperationId(), }; - auto uploads = std::make_shared(); - auto downloads = std::make_shared(); - auto remoteDeletes = std::make_shared(); - auto localDeletes = std::make_shared(); + const auto makePart = [section]() + { + return std::make_shared(SyncPart{.section = section}); + }; + auto uploads = makePart(); + auto downloads = makePart(); + auto remoteDeletes = makePart(); + auto localDeletes = makePart(); watchBulkPart(request.uploadId, uploads); watchBulkPart(request.downloadId, downloads); watchFailedEntries(request.remoteDeleteId, remoteDeletes); @@ -1120,62 +1261,53 @@ struct SyncDialog::Implementation provider_->executeSelection( request, - [this, section, request, uploads, downloads, remoteDeletes, localDeletes]( - SharedData::Sync::SyncExecutionResult result - ) - { - const auto mapBulk = [this, section]( - SyncPart& part, - std::vector const& rows, - std::vector const& sources - ) - { - for (std::size_t index = 0; index < rows.size(); ++index) - { - part.observers.push_back(progressFor(section, rows[index])); - if (index < sources.size()) - part.indexBySource.emplace(sources[index], index); - } - }; - const auto setCovering = [this, section](std::vector const& rows, double value) + lifetime_.guarded( + [this, section, request, uploads, downloads, remoteDeletes, localDeletes]( + SharedData::Sync::SyncExecutionResult result + ) { - for (auto const& row : rows) + mapPart(*uploads, std::move(result.uploads), {}); + mapPart(*downloads, std::move(result.downloads), {}); + mapPart(*remoteDeletes, std::move(result.remoteDeletes), result.remoteDeleteSources); + mapPart(*localDeletes, std::move(result.localDeletes), result.localDeleteSources); + + // A part without rows was never queued and will not complete. + for ( + auto const& [operationId, part] : std::array{ + std::pair{request.uploadId, uploads}, + std::pair{request.downloadId, downloads}, + std::pair{request.remoteDeleteId, remoteDeletes}, + std::pair{request.localDeleteId, localDeletes}, + } + ) { - for (auto const& observer : progressCovering(section, row)) - *observer = value; + if (part->rows.empty()) + operationQueue_->removeCallbacks(operationId); } - }; - - mapBulk(*uploads, result.uploads, result.uploadSources); - mapBulk(*downloads, result.downloads, result.downloadSources); - mapBulk(*remoteDeletes, result.remoteDeletes, result.remoteDeleteSources); - mapBulk(*localDeletes, result.localDeletes, result.localDeleteSources); - - // A part without rows was never queued and will not complete. - const std::array parts{ - std::pair{request.uploadId, uploads}, - std::pair{request.downloadId, downloads}, - std::pair{request.remoteDeleteId, remoteDeletes}, - std::pair{request.localDeleteId, localDeletes}, - }; - for (auto const& [operationId, part] : parts) + for (auto const& row : result.failed) + setRowState(section, row, -1.0); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ), + lifetime_.guarded( + [this, section, request](std::string const& message) { - if (part->observers.empty()) + Log::warn("Sync: the backend did not run the selection: {}", message); + for ( + auto const& operationId : + {request.uploadId, request.downloadId, request.remoteDeleteId, request.localDeleteId} + ) operationQueue_->removeCallbacks(operationId); - else - part->map(); + for (auto const& row : request.relKeys) + setRowState(section, row, -1.0); + Notifications::error( + fmt::format( + fmt::runtime(language->get("syncDialog", "executionRejected")), language->translate(message) + ) + ); + Nui::globalEventContext.executeActiveEventsImmediately(); } - setCovering(result.createdDirectories, 1.1); - setCovering(result.failed, -1.0); - Nui::globalEventContext.executeActiveEventsImmediately(); - }, - [this, request](std::string const& message) - { - Log::error("Sync: the backend did not run the selection: {}", message); - for (auto const& operationId : - {request.uploadId, request.downloadId, request.remoteDeleteId, request.localDeleteId}) - operationQueue_->removeCallbacks(operationId); - } + ) ); } }; @@ -1277,6 +1409,12 @@ ScriptNuiComponents::Tree::RowContentRenderer SyncDialog::Implementation::makeTr auto progress = progressIter->second; return div{ class_ = rowClass, + "title"_attr = Nui::observe(*progress).generate( + [this, section, relKey]() + { + return failureReasonOf(section, relKey); + } + ), style = Nui::observe(*progress).generate( [progress]() -> std::string { @@ -1318,11 +1456,6 @@ SyncDialog::~SyncDialog() SyncDialog::SyncDialog(SyncDialog&&) = default; SyncDialog& SyncDialog::operator=(SyncDialog&&) = default; -void SyncDialog::setOnSynchronized(std::function callback) -{ - impl_->onSynchronized_ = std::move(callback); -} - void SyncDialog::setOnRecompareRequested(std::function callback) { impl_->onRecompareRequested_ = std::move(callback); @@ -1338,12 +1471,13 @@ void SyncDialog::open( SharedData::Sync::DiffSnapshot snapshot, std::filesystem::path localPath, std::filesystem::path remotePath, - ScanSettings scanned, - std::optional lastSyncTime + ScanSettings scanned ) { + if (impl_->provider_ != provider) + impl_->dropRegistrations(); impl_->provider_ = std::move(provider); - impl_->lastSyncTime_ = lastSyncTime; + impl_->diffCurrent_ = true; impl_->localPath_ = std::move(localPath); impl_->remotePath_ = std::move(remotePath); impl_->scanned_ = scanned; @@ -1469,6 +1603,9 @@ Nui::ElementRenderer SyncDialog::operator()() onClick = [this]() { impl_->open_ = false; impl_->minimized_ = false; + // Closing ends the flow: the backend session and the rows' callbacks go. + impl_->dropRegistrations(); + impl_->provider_.reset(); if (impl_->operationQueue_) impl_->operationQueue_->hideMinimizedSync(); Nui::globalEventContext.executeActiveEventsImmediately(); @@ -1791,7 +1928,17 @@ Nui::ElementRenderer SyncDialog::operator()() .text = language->getObserved("syncDialog", "synchronize"), .icon = Ui5Icons::synchronize(), .attributes = { + disabled = observe(impl_->diffCurrent_).generate([this]() -> std::optional { + return impl_->diffCurrent_.value() ? std::nullopt : std::optional{true}; + }), + Nui::Attributes::title = observe(impl_->diffCurrent_).generate([this]() { + return impl_->diffCurrent_.value() + ? std::string{} + : language->get("syncDialog", "synchronizeUnavailable"); + }), onClick = [this](Nui::val) { + if (!impl_->diffCurrent_.value()) + return; const bool deletes = impl_->selectedTotalsOf(DiffSection::Delete).first > 0; impl_->confirmDialog_->open({ .styleVariant = deletes ? Snc::StyleVariant::Danger : Snc::StyleVariant::Warning, @@ -1800,11 +1947,8 @@ Nui::ElementRenderer SyncDialog::operator()() .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, .focusButton = deletes ? std::optional{ConfirmDialog::Button::No} : std::nullopt, .onClose = [this](std::optional btn) { - if (btn != ConfirmDialog::Button::Yes) - return; - impl_->enqueueOperations(); - if (impl_->onSynchronized_) - impl_->onSynchronized_(); + if (btn == ConfirmDialog::Button::Yes) + impl_->enqueueOperations(); } }); } diff --git a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp index 1a729a61..e1d4c08a 100644 --- a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp @@ -61,7 +61,7 @@ struct SyncProgressDialog::Implementation std::shared_ptr provider_{}; SharedData::Sync::DiffOptions initialOptions_{}; - std::function onDone_{}; + std::function, SharedData::Sync::DiffSnapshot)> onDone_{}; // Cancel token; replaced on each open() to invalidate stale callback captures. // The provider also has its own cancel flag that kills the backend walk — this @@ -86,7 +86,7 @@ void SyncProgressDialog::open( bool recursive, bool ignoreHidden, SharedData::Sync::DiffOptions initialOptions, - std::function onDone + std::function, SharedData::Sync::DiffSnapshot)> onDone ) { // Invalidate any stale callbacks from a previous open() by flipping the old @@ -150,8 +150,9 @@ void SyncProgressDialog::open( impl_->phase_ = SharedData::SyncPhase::Done; impl_->open_ = false; Nui::globalEventContext.executeActiveEventsImmediately(); - if (impl_->onDone_) - impl_->onDone_(std::move(snapshot)); + auto provider = std::exchange(impl_->provider_, nullptr); + if (auto onDone = std::exchange(impl_->onDone_, nullptr); onDone) + onDone(std::move(provider), std::move(snapshot)); } ); }, @@ -160,8 +161,10 @@ void SyncProgressDialog::open( { if (*token) return; - impl_->failureReason_ = reason; + impl_->failureReason_ = language->translate(reason); impl_->phase_ = SharedData::SyncPhase::Failed; + impl_->onDone_ = nullptr; + impl_->provider_.reset(); Nui::globalEventContext.executeActiveEventsImmediately(); }, // onDiffProgress @@ -181,8 +184,9 @@ void SyncProgressDialog::cancel() impl_->cancelToken_ = std::make_shared(false); impl_->open_ = false; impl_->phase_ = SharedData::SyncPhase::Idle; - if (impl_->provider_) - impl_->provider_->cancelDiff(); + impl_->onDone_ = nullptr; + if (auto provider = std::exchange(impl_->provider_, nullptr); provider) + provider->cancelDiff(); Nui::globalEventContext.executeActiveEventsImmediately(); } diff --git a/frontend/source/frontend/terminal/file_engine.cpp b/frontend/source/frontend/terminal/file_engine.cpp index 85c59623..a4776d30 100644 --- a/frontend/source/frontend/terminal/file_engine.cpp +++ b/frontend/source/frontend/terminal/file_engine.cpp @@ -688,8 +688,7 @@ void FileEngine::addBulkDownload( { Log::info("Requesting bulk download for {} entries", entries.size()); - // operationIds carries one id per entry plus one dedicated aggregate-bulk-card id - // at the back — see OperationQueue::addBulkDownloadOperation for rationale. + // One operation id per entry and, at the back, the id of the card of the files. if (entries.size() + 1 != operationIds.size()) { Log::error("addBulkDownload: entries and operationIds size mismatch"); @@ -704,7 +703,7 @@ void FileEngine::addBulkDownload( allowOverwrite, insertRefresh, mode, - onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) + onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) mutable { if (!channelId) { @@ -719,7 +718,7 @@ void FileEngine::addBulkDownload( operationIdStrings.push_back(id.value()); SharedData::BulkDownloadRequest request{ - .entries = entries, + .entries = std::move(entries), .allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, .mode = mode, @@ -757,8 +756,7 @@ void FileEngine::addBulkUpload( { Log::info("Requesting bulk upload for {} entries", entries.size()); - // operationIds carries one id per entry plus one dedicated aggregate-bulk-card id - // at the back — see OperationQueue::addBulkUploadOperation for rationale. + // One operation id per entry and, at the back, the id of the card of the files. if (entries.size() + 1 != operationIds.size()) { Log::error("addBulkUpload: entries and operationIds size mismatch"); @@ -773,7 +771,7 @@ void FileEngine::addBulkUpload( allowOverwrite, insertRefresh, mode, - onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) + onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) mutable { if (!channelId) { @@ -788,7 +786,7 @@ void FileEngine::addBulkUpload( operationIdStrings.push_back(id.value()); SharedData::BulkUploadRequest request{ - .entries = entries, + .entries = std::move(entries), .allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, .mode = mode, @@ -831,7 +829,7 @@ void FileEngine::addBulkDelete( bulkOperationId, insertRefresh, mode, - onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) + onBulkCreated = std::move(onBulkCreated)](auto const& channelId, std::string const& info) mutable { if (!channelId) { @@ -841,7 +839,7 @@ void FileEngine::addBulkDelete( } SharedData::BulkDeleteRequest request{ - .entries = entries, + .entries = std::move(entries), .insertRefresh = insertRefresh, .mode = mode, }; diff --git a/frontend/source/frontend/terminal/ssh_engine.cpp b/frontend/source/frontend/terminal/ssh_engine.cpp index 89ff3b61..ca4e360d 100644 --- a/frontend/source/frontend/terminal/ssh_engine.cpp +++ b/frontend/source/frontend/terminal/ssh_engine.cpp @@ -99,6 +99,8 @@ void SshTerminalEngine::disconnect(std::function onDisconnect, bool byLo Log::info("Disconnecting session: {}", impl_->sshSessionId.value()); if (!byLossOfConnection) { + Nui::val parameters = Nui::val::object(); + parameters.set("sessionId", impl_->sshSessionId.value()); Nui::RpcClient::callWithBackChannel( "SessionManager::disconnect", [onDisconnect = std::move(onDisconnect)](Nui::val) @@ -107,7 +109,7 @@ void SshTerminalEngine::disconnect(std::function onDisconnect, bool byLo Log::info("Frontend SshEngine: Disconnected"); onDisconnect(); }, - impl_->sshSessionId.value() + parameters ); } else diff --git a/frontend/source/frontend/terminal/terminal_channel.cpp b/frontend/source/frontend/terminal/terminal_channel.cpp index 9732d67e..4ae59924 100644 --- a/frontend/source/frontend/terminal/terminal_channel.cpp +++ b/frontend/source/frontend/terminal/terminal_channel.cpp @@ -235,7 +235,7 @@ namespace { debugPrint += "\\n"; } - else if (std::isprint(c)) + else if (std::isprint(static_cast(c))) { debugPrint += c; } diff --git a/log/source/log/logger_backend.cpp b/log/source/log/logger_backend.cpp index 556d67e7..9849cdd0 100644 --- a/log/source/log/logger_backend.cpp +++ b/log/source/log/logger_backend.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include @@ -29,16 +31,17 @@ namespace Log // ---- File sink (optional) ---- if (!disableFileLog) { - fs::path log_dir = - fmt::format(fmt::runtime(sinkDir.generic_string()), fmt::arg("appName", Constants::appName)); + const auto logDirectory = Paths::nativeFromUtf8( + fmt::format(fmt::runtime(Paths::nativeToUtf8(sinkDir)), fmt::arg("appName", Constants::appName)) + ); std::error_code ec; - fs::create_directories(log_dir, ec); // best-effort + fs::create_directories(logDirectory, ec); // best-effort - fs::path log_file = log_dir / (std::string{Constants::appName} + ".log"); + const auto logFile = logDirectory / Paths::nativeFromUtf8(fmt::format("{}.log", Constants::appName)); - // 5 MB per file, keep 3 rotated files + // 5 MB per file, keep 3 rotated files. The native name is wide on Windows (SPDLOG_WCHAR_FILENAMES). auto file_sink = - std::make_shared(log_file.string(), 5 * 1024 * 1024, 3, true); + std::make_shared(logFile.native(), 5 * 1024 * 1024, 3, true); file_sink->set_level(spdlogLevel); dist_sink->add_sink(file_sink); @@ -135,7 +138,7 @@ namespace Log { try { - hub->callRemote("log", static_cast(level), msg); + hub->callRemote("log", static_cast(level), Paths::sanitizedUtf8(msg)); } catch (const std::exception& e) { diff --git a/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp b/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp index 887e2439..1fc63722 100644 --- a/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp +++ b/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -71,7 +72,7 @@ namespace NuiFileExplorer if (home.empty()) return result; - result.push_back({{"kind", "home"}, {"name", "Home"}, {"path", home.generic_string()}}); + result.push_back({{"kind", "home"}, {"name", "Home"}, {"path", Paths::nativeToUtf8(home)}}); const auto userDirectories = readUserDirectories(home); for (auto const& definition : Utility::defaultPlaces) @@ -83,7 +84,7 @@ namespace NuiFileExplorer result.push_back( {{"kind", std::string{definition.kind}}, {"name", std::string{definition.name}}, - {"path", path.generic_string()}} + {"path", Paths::nativeToUtf8(path)}} ); } return result; diff --git a/nui-file-explorer/source/nui-file-explorer/support/default_places_windows.cpp b/nui-file-explorer/source/nui-file-explorer/support/default_places_windows.cpp index 70fca4f1..e3bc2e99 100644 --- a/nui-file-explorer/source/nui-file-explorer/support/default_places_windows.cpp +++ b/nui-file-explorer/source/nui-file-explorer/support/default_places_windows.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -9,6 +10,7 @@ #include #include +#include #include namespace NuiFileExplorer @@ -18,13 +20,11 @@ namespace NuiFileExplorer std::string resolveKnownFolder(KNOWNFOLDERID const& folderId) { PWSTR path = nullptr; - if (SUCCEEDED(SHGetKnownFolderPath(folderId, KF_FLAG_DEFAULT, nullptr, &path))) - { - std::wstring wide{path}; - CoTaskMemFree(path); - return std::filesystem::path{wide}.generic_string(); - } - return {}; + const auto result = SHGetKnownFolderPath(folderId, KF_FLAG_DEFAULT, nullptr, &path); + const std::unique_ptr owned{path, &CoTaskMemFree}; + if (FAILED(result) || !owned) + return {}; + return Paths::nativeToUtf8(std::filesystem::path{owned.get()}); } struct FolderDef diff --git a/nui-file-explorer/source/nui-file-explorer/support/windows_drives.cpp b/nui-file-explorer/source/nui-file-explorer/support/windows_drives.cpp index 59b8f169..7c26bf48 100644 --- a/nui-file-explorer/source/nui-file-explorer/support/windows_drives.cpp +++ b/nui-file-explorer/source/nui-file-explorer/support/windows_drives.cpp @@ -1,5 +1,8 @@ #include +#include +#include +#include #include #ifndef WIN32_LEAN_AND_MEAN @@ -23,12 +26,12 @@ namespace NuiFileExplorer if (!wide.empty()) { // "My Label (C:)" - auto letter = std::filesystem::path{rootPath}.root_name().string(); - return std::filesystem::path{wide}.string() + " (" + letter + ")"; + const auto letter = Paths::nativeToUtf8(std::filesystem::path{rootPath}.root_name()); + return fmt::format("{} ({})", Paths::wideToUtf8(wide), letter); } } // Fallback: just the drive letter - return std::filesystem::path{rootPath}.root_name().string(); + return Paths::nativeToUtf8(std::filesystem::path{rootPath}.root_name()); } } @@ -61,7 +64,7 @@ namespace NuiFileExplorer continue; auto displayName = driveDisplayName(root); - auto genericPath = std::filesystem::path{root}.generic_string(); + auto genericPath = Paths::nativeToUtf8(std::filesystem::path{root}); result.push_back({ {"name", displayName}, diff --git a/paths/include/paths/common_parent.hpp b/paths/include/paths/common_parent.hpp new file mode 100644 index 00000000..9ed07cc9 --- /dev/null +++ b/paths/include/paths/common_parent.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +namespace Paths +{ + /** + * @brief The closest directory that holds every one of @p paths below it. A path without a parent (a root) + * stands for itself, nothing for no paths. Works for LocalPath and RemotePath. + */ + template + std::optional commonParentOf(std::span paths) + { + if (paths.empty()) + return std::nullopt; + auto candidate = paths.front().parent(); + if (!candidate) + return paths.front(); + const auto containsAll = [&paths](PathT const& directory) + { + return std::ranges::all_of( + paths, + [&directory](PathT const& path) + { + return path != directory && path.isWithin(directory); + } + ); + }; + while (candidate && !containsAll(*candidate)) + candidate = candidate->parent(); + return candidate; + } +} diff --git a/paths/include/paths/local_path.hpp b/paths/include/paths/local_path.hpp index 9533d46b..29344aa9 100644 --- a/paths/include/paths/local_path.hpp +++ b/paths/include/paths/local_path.hpp @@ -1,9 +1,10 @@ #pragma once #include +#include #include -#include +#include #include #include #include @@ -19,13 +20,104 @@ namespace Paths { + namespace Detail + { + /** + * @brief True when @p text starts with @p prefix, ASCII letters compared without case. + */ + inline bool startsWithIgnoringCase(std::string_view text, std::string_view prefix) + { + const auto folded = [](char character) + { + return character >= 'A' && character <= 'Z' ? static_cast(character - 'A' + 'a') : character; + }; + return text.size() >= prefix.size() && + std::ranges::equal(text.substr(0, prefix.size()), prefix, {}, folded, folded); + } + + /** + * @brief Removes a \\?\ prefix from @p generic, a '/' separated Windows path: "//?/C:/x" becomes "C:/x" + * and "//?/UNC/server/share" becomes "//server/share". DevicePath for the device namespace "//./", + * any other "//?/" path and a server named ".", "?" or "..". + */ + inline std::expected removeWin32Prefix(std::string& generic) + { + const auto isLetter = [](char character) + { + return (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'); + }; + if (generic.starts_with("//?/")) + { + const auto rest = std::string_view{generic}.substr(4); + if (startsWithIgnoringCase(rest, "unc/")) + generic.erase(2, 6); + else if (rest.size() >= 2 && isLetter(rest[0]) && rest[1] == ':') + generic.erase(0, 4); + else + return std::unexpected(PathRejection::DevicePath); + } + if (!generic.starts_with("//")) + return {}; + const auto server = std::string_view{generic}.substr(2, generic.find('/', 2) - 2); + if (server == "." || server == "?" || server == "..") + return std::unexpected(PathRejection::DevicePath); + return {}; + } + + /** + * @brief Length of the root prefix of @p generic ("/", "C:/", "C:" or "//server/share"), 0 when it is not + * absolute on @p platform. Expects '/' separators and no \\?\ prefix. + */ + inline std::size_t rootLengthOf(std::string_view generic, LocalPlatform platform) + { + if (platform == LocalPlatform::Posix) + return generic.starts_with('/') ? 1u : 0u; + + const auto isLetter = [](char character) + { + return (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'); + }; + if (generic.size() >= 3 && isLetter(generic[0]) && generic[1] == ':' && generic[2] == '/') + return 3u; + if (generic.size() == 2 && isLetter(generic[0]) && generic[1] == ':') + return 2u; + if (generic.starts_with("//")) + { + const auto serverEnd = generic.find('/', 2); + if (serverEnd == std::string_view::npos || serverEnd == 2) + return 0u; + const auto shareEnd = std::min(generic.find('/', serverEnd + 1), generic.size()); + if (shareEnd == serverEnd + 1) + return 0u; + return shareEnd; + } + return 0u; + } + + /** + * @brief True when @p path is @p base or lies below it, both generic path text with @p base ending in '/' + * only when it is a root. @p equal compares two characters. + */ + template + bool isWithinText(std::string_view path, std::string_view base, EqualT equal) + { + if (path.size() < base.size() || !std::ranges::equal(path.substr(0, base.size()), base, equal)) + return false; + return path.size() == base.size() || base.back() == '/' || path[base.size()] == '/'; + } + } + /** * @brief An absolute, normalized path on the local machine, kept as generic UTF-8 ('/' separated). * * The frontend runs in WebAssembly where std::filesystem would misread "C:/x", so the type does its own parsing * for both platforms. The backend converts to a native std::filesystem::path only through @ref native. * + * On Windows the drive letter is upper case, a \\?\ prefix is dropped (\\?\UNC\server\share becomes + * //server/share) and device paths like \\.\PhysicalDrive0 are refused. + * * A default constructed LocalPath is unset (@ref empty), every operation that touches the disk must refuse it. + * Joining onto an unset path gives an unset path. */ class LocalPath { @@ -41,95 +133,100 @@ namespace Paths }; if (utf8.empty()) return refuse(PathRejection::Empty); + if (utf8.find('\0') != std::string_view::npos) + return refuse(PathRejection::ContainsNul); if (!isValidUtf8(utf8)) return refuse(PathRejection::InvalidUtf8); std::string generic{utf8}; if (platform == LocalPlatform::Windows) + { std::ranges::replace(generic, '\\', '/'); + if (const auto removed = Detail::removeWin32Prefix(generic); !removed) + return refuse(removed.error()); + } - auto rootLength = rootLengthOf(generic, platform); + const auto rootLength = Detail::rootLengthOf(generic, platform); if (rootLength == 0) return refuse(PathRejection::NotAbsolute); - LocalPath result; - result.platform_ = platform; - result.value_ = generic.substr(0, rootLength); - if (result.value_.back() != '/') - result.value_.push_back('/'); - result.rootLength_ = result.value_.size(); - - std::vector segments; - std::string_view rest{generic}; - rest.remove_prefix(rootLength); - std::size_t start = 0; - while (start <= rest.size()) + std::vector names; + for (auto const name : Detail::splitAtSlashes(std::string_view{generic}.substr(rootLength))) { - const auto end = std::min(rest.find('/', start), rest.size()); - const auto name = rest.substr(start, end - start); - start = end + 1; if (name.empty() || name == ".") continue; if (name == "..") { - if (segments.empty()) + if (names.empty()) return refuse(PathRejection::EscapesRoot); - segments.pop_back(); + names.pop_back(); continue; } - segments.push_back(name); - } - for (auto const name : segments) - { - auto segment = LocalPathSegment::make(name, platform); - if (!segment) - return refuse(segment.error().reason); - result = result / *segment; + if (const auto rejection = LocalPathSegment::rejectionOf(name, platform); rejection) + return refuse(*rejection); + names.push_back(name); } + + LocalPath result; + result.platform_ = platform; + result.value_.reserve(generic.size() + 1); + result.value_.append(generic, 0, rootLength); + if (result.value_.back() != '/') + result.value_.push_back('/'); + if (platform == LocalPlatform::Windows && result.value_[0] >= 'a' && result.value_[0] <= 'z') + result.value_[0] = static_cast(result.value_[0] - 'a' + 'A'); + result.rootLength_ = result.value_.size(); + for (auto const name : names) + Detail::appendSegment(result.value_, name); return result; } #ifndef __EMSCRIPTEN__ /** - * @brief Takes a native path, which has to be absolute. + * @brief Takes a native path, which has to be absolute. Refused with InvalidUtf8 when its text is not + * UTF-8 (POSIX) or has an unpaired surrogate (Windows). Never throws. */ static std::expected make(std::filesystem::path const& path) { - const auto utf8 = path.generic_u8string(); - return fromUtf8(std::string_view{reinterpret_cast(utf8.data()), utf8.size()}); + return checkedNativeToUtf8(path).and_then( + [](std::string const& utf8) + { + return fromUtf8(utf8); + } + ); } std::filesystem::path native() const { - return std::filesystem::path{ - std::u8string{reinterpret_cast(value_.data()), value_.size()} - }.make_preferred(); + return nativeFromUtf8(value_).make_preferred(); } #endif LocalPath operator/(LocalPathSegment const& segment) const { - assert(!empty()); + if (empty()) + return {}; LocalPath result{*this}; - if (result.value_.back() != '/') - result.value_.push_back('/'); - result.value_ += segment.value(); + Detail::appendSegment(result.value_, segment.value()); return result; } /** - * @brief Joins @p relative, every segment has to be a valid name on this path's platform. + * @brief Joins @p relative, every segment has to be a valid name on this path's platform. Empty when this + * path is unset. */ std::expected join(RelativePath const& relative) const { - LocalPath result{*this}; - for (auto const& segment : relative.segments()) + if (empty()) + return std::unexpected(PathError{.reason = PathRejection::Empty, .input = relative.value()}); + for (auto const name : relative.segmentViews()) { - auto local = LocalPathSegment::make(segment, platform_); - if (!local) - return std::unexpected(local.error()); - result = result / *local; + if (const auto rejection = LocalPathSegment::rejectionOf(name, platform_); rejection) + return std::unexpected(PathError{.reason = *rejection, .input = std::string{name}}); } + LocalPath result{*this}; + if (!relative.empty()) + Detail::appendSegment(result.value_, relative.value()); return result; } @@ -157,21 +254,39 @@ namespace Paths { if (empty() || isRoot()) return std::nullopt; - return *PathSegment::make(std::string_view{value_}.substr(value_.rfind('/') + 1)); + return PathSegment{std::string{Detail::lastSegment(value_)}}; } /** - * @brief True when this is @p base or lies below it. + * @brief True when this is @p base or lies below it, compared exactly. */ bool isWithin(LocalPath const& base) const { if (empty() || base.empty() || platform_ != base.platform_) return false; - if (value_ == base.value_) - return true; - if (!value_.starts_with(base.value_)) + return Detail::isWithinText(value_, base.value_, std::ranges::equal_to{}); + } + + /** + * @brief @ref isWithin with ASCII letters compared without case. For checks that must not miss a path the + * filesystem treats as the same, like the delete guards on Windows and macOS, which fold case. + */ + bool isWithinIgnoringCase(LocalPath const& base) const + { + if (empty() || base.empty() || platform_ != base.platform_) return false; - return base.value_.back() == '/' || value_[base.value_.size()] == '/'; + const auto folded = [](char character) + { + return character >= 'A' && character <= 'Z' ? static_cast(character - 'A' + 'a') : character; + }; + return Detail::isWithinText( + value_, + base.value_, + [&folded](char left, char right) + { + return folded(left) == folded(right); + } + ); } /** @@ -205,36 +320,6 @@ namespace Paths auto operator<=>(LocalPath const&) const = default; private: - /** - * @brief Length of the root prefix ("/", "C:/" or "//server/share"), 0 when not absolute. - */ - static std::size_t rootLengthOf(std::string_view generic, LocalPlatform platform) - { - if (platform == LocalPlatform::Posix) - return generic.starts_with('/') ? 1u : 0u; - - const auto isLetter = [](char character) - { - return (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'); - }; - if (generic.size() >= 3 && isLetter(generic[0]) && generic[1] == ':' && generic[2] == '/') - return 3u; - if (generic.size() == 2 && isLetter(generic[0]) && generic[1] == ':') - return 2u; - if (generic.starts_with("//")) - { - // UNC: //server/share is the root. - const auto serverEnd = generic.find('/', 2); - if (serverEnd == std::string_view::npos || serverEnd == 2) - return 0u; - const auto shareEnd = std::min(generic.find('/', serverEnd + 1), generic.size()); - if (shareEnd == serverEnd + 1) - return 0u; - return shareEnd; - } - return 0u; - } - std::string value_{}; std::size_t rootLength_{0}; LocalPlatform platform_{LocalPlatform::Posix}; diff --git a/paths/include/paths/local_path_segment.hpp b/paths/include/paths/local_path_segment.hpp index 5f345e28..1f71e691 100644 --- a/paths/include/paths/local_path_segment.hpp +++ b/paths/include/paths/local_path_segment.hpp @@ -1,10 +1,11 @@ #pragma once #include +#include #include #include -#include +#include #include #include #include @@ -34,7 +35,8 @@ namespace Paths } /** - * @brief The local platform paths default to. The frontend runs in WebAssembly and sets the backend's here. + * @brief The local platform paths default to. The frontend runs in WebAssembly and sets the backend's here + * before it reads any local path. */ inline LocalPlatform& defaultLocalPlatform() { @@ -46,8 +48,9 @@ namespace Paths * @brief A @ref PathSegment that can be created on the local filesystem of @p platform. * * Windows additionally refuses '\\', ':', '<', '>', '"', '|', '?', '*', control characters, device names - * like CON or LPT1 (also with an extension) and a trailing dot or space. A remote name has to pass here - * before it becomes part of a local path. + * like CON, CONIN$, COM0 or LPT¹ (also with an extension or spaces before it), a trailing dot or space and + * names longer than 255 UTF-16 code units. A remote name has to pass here before it becomes part of a local + * path. */ class LocalPathSegment { @@ -57,7 +60,7 @@ namespace Paths { if (platform == LocalPlatform::Windows) { - if (auto reason = windowsRejection(segment.view()); reason) + if (const auto reason = windowsRejection(segment.view()); reason) return std::unexpected(PathError{.reason = *reason, .input = segment.value()}); } return LocalPathSegment{segment}; @@ -74,6 +77,19 @@ namespace Paths ); } + /** + * @brief Why @p utf8 cannot be a local name on @p platform, nothing when it can. + */ + static std::optional + rejectionOf(std::string_view utf8, LocalPlatform platform = defaultLocalPlatform()) + { + if (const auto rejection = PathSegment::rejectionOf(utf8); rejection) + return rejection; + if (platform == LocalPlatform::Windows) + return windowsRejection(utf8); + return std::nullopt; + } + PathSegment const& segment() const noexcept { return segment_; @@ -91,36 +107,58 @@ namespace Paths : segment_{std::move(segment)} {} + /** + * @brief The Windows rules for a valid @ref PathSegment. + */ static std::optional windowsRejection(std::string_view name) { - for (const auto character : name) + constexpr std::size_t maximumNameLength = 255u; + const auto reserved = [](char character) { - const auto byte = static_cast(character); - if (byte < 0x20u || std::string_view{"\\:<>\"|?*"}.find(character) != std::string_view::npos) - return PathRejection::ReservedCharacter; - } + return static_cast(character) < 0x20u || + std::string_view{"\\:<>\"|?*"}.find(character) != std::string_view::npos; + }; + if (std::ranges::any_of(name, reserved)) + return PathRejection::ReservedCharacter; if (name.back() == '.' || name.back() == ' ') return PathRejection::TrailingDotOrSpace; + if (isDeviceName(name)) + return PathRejection::ReservedDeviceName; + if (utf16Length(name) > maximumNameLength) + return PathRejection::NameTooLong; + return std::nullopt; + } + + /** + * @brief True for a Windows device name. It counts with any extension and with spaces before it: + * "nul.txt" and "CON .log" are devices too. + */ + static bool isDeviceName(std::string_view name) + { + auto stem = name.substr(0, name.find('.')); + while (!stem.empty() && stem.back() == ' ') + stem.remove_suffix(1); - // Device names count with any extension: "nul.txt" is the NUL device too. - const auto stem = name.substr(0, name.find('.')); std::string upper{stem}; - std::transform( - upper.begin(), - upper.end(), + std::ranges::transform( + upper, upper.begin(), - [](unsigned char character) + [](char character) { - return static_cast(std::toupper(character)); + return character >= 'a' && character <= 'z' ? static_cast(character - 'a' + 'A') + : character; } ); - constexpr std::array devices{"CON", "PRN", "AUX", "NUL"}; + constexpr std::array devices{"CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$"}; if (std::ranges::find(devices, upper) != devices.end()) - return PathRejection::ReservedDeviceName; - if (upper.size() == 4 && (upper.starts_with("COM") || upper.starts_with("LPT")) && upper[3] >= '1' && - upper[3] <= '9') - return PathRejection::ReservedDeviceName; - return std::nullopt; + return true; + if (!upper.starts_with("COM") && !upper.starts_with("LPT")) + return false; + + const auto number = std::string_view{upper}.substr(3); + constexpr std::array superscripts{"\xC2\xB9", "\xC2\xB2", "\xC2\xB3"}; + return (number.size() == 1u && number[0] >= '0' && number[0] <= '9') || + std::ranges::find(superscripts, number) != superscripts.end(); } PathSegment segment_; diff --git a/paths/include/paths/native.hpp b/paths/include/paths/native.hpp index 7440378c..04105db6 100644 --- a/paths/include/paths/native.hpp +++ b/paths/include/paths/native.hpp @@ -1,29 +1,81 @@ #pragma once -#ifndef __EMSCRIPTEN__ -# include -# include -# include +#include +#include + +#include +#include +#include +#include +#include namespace Paths { /** - * @brief The text of a native path as generic UTF-8. For names and link literals, which are no LocalPath; an - * absolute path goes through LocalPath::make. + * @brief Native text converted to UTF-8, and whether that was lossless. + */ + struct NativeText + { + /** + * @brief The text as generic ('/' separated) UTF-8, always valid: U+FFFD stands for every byte that is not + * UTF-8 (POSIX) and every unpaired surrogate (Windows). + */ + std::string utf8{}; + /** + * @brief False when the native text had to be replaced in places, then @ref utf8 only shows it. + */ + bool lossless{true}; + }; + + /** + * @brief The text of a native path as generic UTF-8 and whether it converted losslessly. Never throws. + */ + inline NativeText convertNativeToUtf8(std::filesystem::path const& path) + { +#ifdef _WIN32 + const auto wide = path.generic_wstring(); + return NativeText{ + .utf8 = utf16ToUtf8(std::wstring_view{wide}), + .lossless = isValidUtf16(std::wstring_view{wide}), + }; +#else + auto const& bytes = path.native(); + return NativeText{.utf8 = sanitizedUtf8(bytes), .lossless = isValidUtf8(bytes)}; +#endif + } + + /** + * @brief The text of a native path as generic UTF-8, for display and logs: what is not UTF-8 becomes U+FFFD. + * A name or path that is acted on goes through @ref checkedNativeToUtf8 or LocalPath::make. */ inline std::string nativeToUtf8(std::filesystem::path const& path) { - const auto text = path.generic_u8string(); - return std::string{reinterpret_cast(text.data()), text.size()}; + return convertNativeToUtf8(path).utf8; } /** - * @brief A native path from UTF-8 text, the reverse of @ref nativeToUtf8. An absolute path goes through - * LocalPath::fromUtf8, which validates it. + * @brief The text of a native path as generic UTF-8, refused with InvalidUtf8 when it does not convert + * losslessly (the error input shows it with U+FFFD). + */ + inline std::expected checkedNativeToUtf8(std::filesystem::path const& path) + { + auto converted = convertNativeToUtf8(path); + if (!converted.lossless) + return std::unexpected(PathError{.reason = PathRejection::InvalidUtf8, .input = std::move(converted.utf8)}); + return std::move(converted.utf8); + } + + /** + * @brief A native path from UTF-8 text, the reverse of @ref nativeToUtf8. Never throws: on Windows every byte + * that is not UTF-8 becomes U+FFFD, so text from a server is checked with isValidUtf8 first. An absolute + * path goes through LocalPath::fromUtf8, which validates it. */ inline std::filesystem::path nativeFromUtf8(std::string_view utf8) { - return std::filesystem::path{std::u8string_view{reinterpret_cast(utf8.data()), utf8.size()}}; +#ifdef _WIN32 + return std::filesystem::path{utf8ToWide(utf8)}; +#else + return std::filesystem::path{std::string{utf8}}; +#endif } } -#endif diff --git a/paths/include/paths/path_error.hpp b/paths/include/paths/path_error.hpp index 2f7bd12b..5c0b7b76 100644 --- a/paths/include/paths/path_error.hpp +++ b/paths/include/paths/path_error.hpp @@ -1,6 +1,11 @@ #pragma once +#include + +#include + #include +#include #include #include @@ -17,14 +22,34 @@ namespace Paths ContainsNul, InvalidUtf8, NotAbsolute, - /// A character the local platform cannot store in a name, like '\\' or ':' on Windows. + /** + * @brief A character the local platform cannot store in a name, like '\\' or ':' on Windows. + */ ReservedCharacter, - /// A Windows device name like CON or LPT1, with or without an extension. + /** + * @brief A Windows device name like CON or LPT1, with or without an extension. + */ ReservedDeviceName, - /// Windows drops a trailing dot or space, the name would refer to another file. + /** + * @brief Windows drops a trailing dot or space, the name would refer to another file. + */ TrailingDotOrSpace, - /// The path leaves the directory it is meant to stay in. + /** + * @brief The path leaves the directory it is meant to stay in. + */ EscapesRoot, + /** + * @brief A Windows device or namespace path like \\.\PhysicalDrive0 or \\?\Volume{...}, no file path. + */ + DevicePath, + /** + * @brief A name longer than the 255 UTF-16 code units Windows allows. + */ + NameTooLong, + /** + * @brief An entry of a walked tree whose parent is not one of its entries. + */ + UnknownParent, }; inline std::string_view toString(PathRejection rejection) @@ -51,6 +76,12 @@ namespace Paths return "ends with a dot or space"; case PathRejection::EscapesRoot: return "leaves its root directory"; + case PathRejection::DevicePath: + return "is a device path"; + case PathRejection::NameTooLong: + return "has a name that is too long"; + case PathRejection::UnknownParent: + return "lies below an entry that was not listed"; } return "is invalid"; } @@ -63,24 +94,29 @@ namespace Paths PathRejection reason{PathRejection::Empty}; std::string input{}; + /** + * @brief The input in quotes followed by the reason. Control bytes and bytes that are not part of a valid + * UTF-8 sequence are written as \\xNN, so the text is always valid UTF-8. + */ std::string toString() const { std::string printable; printable.reserve(input.size()); - for (const auto character : input) + std::size_t index = 0; + while (index < input.size()) { - const auto byte = static_cast(character); - if (byte < 0x20u || byte == 0x7Fu) + const auto byte = static_cast(input[index]); + const auto length = Detail::utf8SequenceLength(input, index); + if (length == 0u || byte < 0x20u || byte == 0x7Fu) { - constexpr std::string_view digits = "0123456789ABCDEF"; - printable += "\\x"; - printable.push_back(digits[byte >> 4u]); - printable.push_back(digits[byte & 0x0Fu]); + fmt::format_to(std::back_inserter(printable), "\\x{:02X}", byte); + ++index; + continue; } - else - printable.push_back(character); + printable.append(input, index, length); + index += length; } - return "'" + printable + "' " + std::string{Paths::toString(reason)}; + return fmt::format("'{}' {}", printable, Paths::toString(reason)); } }; } diff --git a/paths/include/paths/path_segment.hpp b/paths/include/paths/path_segment.hpp index 92944454..aeb91a0d 100644 --- a/paths/include/paths/path_segment.hpp +++ b/paths/include/paths/path_segment.hpp @@ -1,66 +1,58 @@ #pragma once #include +#include #include #include #include +#include +#include #include #include #include namespace Paths { - /** - * @brief True when @p text is well-formed UTF-8 (no overlong forms, no surrogates). - */ - inline bool isValidUtf8(std::string_view text) + class RelativePath; + class RemotePath; + class LocalPath; + + namespace Detail { - std::size_t index = 0; - while (index < text.size()) + /** + * @brief The parts of @p text between '/' separators as string views, empty parts included. Empty text has + * no parts. + */ + inline auto splitAtSlashes(std::string_view text) { - const auto lead = static_cast(text[index]); - std::size_t length = 0; - std::uint32_t codePoint = 0; - if (lead < 0x80u) - { - ++index; - continue; - } - if ((lead & 0xE0u) == 0xC0u) - { - length = 2; - codePoint = lead & 0x1Fu; - } - else if ((lead & 0xF0u) == 0xE0u) - { - length = 3; - codePoint = lead & 0x0Fu; - } - else if ((lead & 0xF8u) == 0xF0u) - { - length = 4; - codePoint = lead & 0x07u; - } - else - return false; + return text | std::views::split('/') | + std::views::transform( + [](auto const& part) + { + return std::string_view{part.begin(), part.end()}; + } + ); + } + + /** + * @brief Appends @p tail to the path text @p value, with a '/' between them unless @p value ends in one. + */ + inline void appendSegment(std::string& value, std::string_view tail) + { + if (!value.empty() && value.back() != '/') + value.push_back('/'); + value.append(tail); + } - if (index + length > text.size()) - return false; - for (std::size_t offset = 1; offset < length; ++offset) - { - const auto continuation = static_cast(text[index + offset]); - if ((continuation & 0xC0u) != 0x80u) - return false; - codePoint = (codePoint << 6u) | (continuation & 0x3Fu); - } - constexpr std::uint32_t minimumForLength[] = {0u, 0u, 0x80u, 0x800u, 0x10000u}; - if (codePoint < minimumForLength[length] || codePoint > 0x10FFFFu || - (codePoint >= 0xD800u && codePoint <= 0xDFFFu)) - return false; - index += length; + /** + * @brief The text after the last '/' of @p value, all of it when there is none. + */ + inline std::string_view lastSegment(std::string_view value) + { + const auto slash = value.rfind('/'); + return slash == std::string_view::npos ? value : value.substr(slash + 1); } - return true; } /** @@ -74,21 +66,27 @@ namespace Paths public: static std::expected make(std::string_view utf8) { - const auto refuse = [utf8](PathRejection reason) - { - return std::unexpected(PathError{.reason = reason, .input = std::string{utf8}}); - }; + if (const auto rejection = rejectionOf(utf8); rejection) + return std::unexpected(PathError{.reason = *rejection, .input = std::string{utf8}}); + return PathSegment{std::string{utf8}}; + } + + /** + * @brief Why @p utf8 is no valid segment, nothing when it is one. + */ + static std::optional rejectionOf(std::string_view utf8) + { if (utf8.empty()) - return refuse(PathRejection::Empty); + return PathRejection::Empty; if (utf8 == "." || utf8 == "..") - return refuse(PathRejection::DotOrDotDot); + return PathRejection::DotOrDotDot; if (utf8.find('/') != std::string_view::npos) - return refuse(PathRejection::ContainsSeparator); + return PathRejection::ContainsSeparator; if (utf8.find('\0') != std::string_view::npos) - return refuse(PathRejection::ContainsNul); + return PathRejection::ContainsNul; if (!isValidUtf8(utf8)) - return refuse(PathRejection::InvalidUtf8); - return PathSegment{std::string{utf8}}; + return PathRejection::InvalidUtf8; + return std::nullopt; } std::string const& value() const noexcept @@ -104,6 +102,13 @@ namespace Paths auto operator<=>(PathSegment const&) const = default; private: + friend class RelativePath; + friend class RemotePath; + friend class LocalPath; + + /** + * @brief Takes @p value as it is, for text a path type already validated. + */ explicit PathSegment(std::string value) : value_{std::move(value)} {} diff --git a/paths/include/paths/paths.hpp b/paths/include/paths/paths.hpp index 0895c872..dc22b21e 100644 --- a/paths/include/paths/paths.hpp +++ b/paths/include/paths/paths.hpp @@ -31,15 +31,6 @@ namespace Paths // Paths travel as plain strings, invalid ones are refused when they are read. - inline void to_json(nlohmann::json& json, PathSegment const& segment) - { - json = segment.value(); - } - inline void from_json(nlohmann::json const& json, PathSegment& segment) - { - segment = Detail::orThrow(PathSegment::make(json.get())); - } - inline void to_json(nlohmann::json& json, RelativePath const& path) { json = path.value(); @@ -102,3 +93,19 @@ namespace Paths } #endif } + +/** + * @brief PathSegment has no unset state, so it is read through a serializer instead of from_json. + */ +template <> +struct nlohmann::adl_serializer +{ + static Paths::PathSegment from_json(nlohmann::json const& json) + { + return Paths::Detail::orThrow(Paths::PathSegment::make(json.get())); + } + static void to_json(nlohmann::json& json, Paths::PathSegment const& segment) + { + json = segment.value(); + } +}; diff --git a/paths/include/paths/relative_path.hpp b/paths/include/paths/relative_path.hpp index 4e5fa390..1f502dc6 100644 --- a/paths/include/paths/relative_path.hpp +++ b/paths/include/paths/relative_path.hpp @@ -25,43 +25,30 @@ namespace Paths static std::expected make(std::string_view utf8) { - RelativePath result; if (utf8.empty()) - return result; + return RelativePath{}; if (utf8.front() == '/') return std::unexpected(PathError{.reason = PathRejection::EscapesRoot, .input = std::string{utf8}}); - - std::size_t start = 0; - while (start <= utf8.size()) + for (auto const part : Detail::splitAtSlashes(utf8)) { - const auto end = std::min(utf8.find('/', start), utf8.size()); - auto segment = PathSegment::make(utf8.substr(start, end - start)); - if (!segment) - return std::unexpected(PathError{.reason = segment.error().reason, .input = std::string{utf8}}); - result = result / *segment; - start = end + 1; + if (const auto rejection = PathSegment::rejectionOf(part); rejection) + return std::unexpected(PathError{.reason = *rejection, .input = std::string{utf8}}); } - return result; + return RelativePath{std::string{utf8}}; } RelativePath operator/(PathSegment const& segment) const { RelativePath result{*this}; - if (!result.value_.empty()) - result.value_.push_back('/'); - result.value_ += segment.value(); + Detail::appendSegment(result.value_, segment.value()); return result; } RelativePath operator/(RelativePath const& other) const { - if (other.empty()) - return *this; - if (empty()) - return other; RelativePath result{*this}; - result.value_.push_back('/'); - result.value_ += other.value_; + if (!other.empty()) + Detail::appendSegment(result.value_, other.value_); return result; } @@ -70,16 +57,19 @@ namespace Paths return value_.empty(); } + /** + * @brief The names as views into this path, without copying or validating them again. + */ + auto segmentViews() const + { + return Detail::splitAtSlashes(value_); + } + std::vector segments() const { std::vector result; - std::size_t start = 0; - while (!value_.empty() && start <= value_.size()) - { - const auto end = std::min(value_.find('/', start), value_.size()); - result.push_back(*PathSegment::make(std::string_view{value_}.substr(start, end - start))); - start = end + 1; - } + for (auto const part : segmentViews()) + result.push_back(PathSegment{std::string{part}}); return result; } @@ -88,18 +78,14 @@ namespace Paths if (empty()) return std::nullopt; const auto slash = value_.rfind('/'); - RelativePath result; - if (slash != std::string::npos) - result.value_ = value_.substr(0, slash); - return result; + return RelativePath{slash == std::string::npos ? std::string{} : value_.substr(0, slash)}; } std::optional leaf() const { if (empty()) return std::nullopt; - const auto slash = value_.rfind('/'); - return *PathSegment::make(slash == std::string::npos ? value_ : value_.substr(slash + 1)); + return PathSegment{std::string{Detail::lastSegment(value_)}}; } /** @@ -126,6 +112,10 @@ namespace Paths auto operator<=>(RelativePath const&) const = default; private: + explicit RelativePath(std::string value) + : value_{std::move(value)} + {} + std::string value_{}; }; } diff --git a/paths/include/paths/remote_path.hpp b/paths/include/paths/remote_path.hpp index 74926388..bf574e9f 100644 --- a/paths/include/paths/remote_path.hpp +++ b/paths/include/paths/remote_path.hpp @@ -2,7 +2,9 @@ #include -#include +#include + +#include #include #include #include @@ -19,6 +21,7 @@ namespace Paths * platform. ".." is refused instead of resolved: the server decides what it means. * * A default constructed RemotePath is unset (@ref empty), every operation that touches the server must refuse it. + * Joining onto an unset path gives an unset path. */ class RemotePath { @@ -44,42 +47,37 @@ namespace Paths if (utf8.front() != '/') return refuse(PathRejection::NotAbsolute); - auto result = root(); - std::size_t start = 1; - while (start <= utf8.size()) + std::string value; + value.reserve(utf8.size()); + for (auto const name : Detail::splitAtSlashes(utf8)) { - const auto end = std::min(utf8.find('/', start), utf8.size()); - const auto name = utf8.substr(start, end - start); - start = end + 1; if (name.empty() || name == ".") continue; - auto segment = PathSegment::make(name); - if (!segment) - return refuse(segment.error().reason); - result = result / *segment; + if (const auto rejection = PathSegment::rejectionOf(name); rejection) + return refuse(*rejection); + value.push_back('/'); + value.append(name); } - return result; + if (value.empty()) + return root(); + return RemotePath{std::move(value)}; } RemotePath operator/(PathSegment const& segment) const { - assert(!empty()); + if (empty()) + return {}; RemotePath result{*this}; - if (result.value_.back() != '/') - result.value_.push_back('/'); - result.value_ += segment.value(); + Detail::appendSegment(result.value_, segment.value()); return result; } RemotePath operator/(RelativePath const& relative) const { - if (relative.empty()) + if (empty() || relative.empty()) return *this; - assert(!empty()); RemotePath result{*this}; - if (result.value_.back() != '/') - result.value_.push_back('/'); - result.value_ += relative.value(); + Detail::appendSegment(result.value_, relative.value()); return result; } @@ -91,7 +89,7 @@ namespace Paths const auto name = filename(); if (!name) return std::unexpected(PathError{.reason = PathRejection::Empty, .input = value_}); - return PathSegment::make(name->value() + std::string{suffix}) + return PathSegment::make(fmt::format("{}{}", name->view(), suffix)) .transform( [this](PathSegment const& segment) { @@ -117,26 +115,26 @@ namespace Paths if (empty()) return refuse(PathRejection::NotAbsolute); - auto result = linkContent.front() == '/' ? root() : parent().value_or(root()); - std::size_t start = 0; - while (start <= linkContent.size()) + std::string value = linkContent.front() == '/' ? std::string{} : parent().value_or(root()).value_; + if (value == "/") + value.clear(); + for (auto const name : Detail::splitAtSlashes(linkContent)) { - const auto end = std::min(linkContent.find('/', start), linkContent.size()); - const auto name = linkContent.substr(start, end - start); - start = end + 1; if (name.empty() || name == ".") continue; if (name == "..") { - result = result.parent().value_or(root()); + value.erase(std::min(value.rfind('/'), value.size())); continue; } - auto segment = PathSegment::make(name); - if (!segment) - return refuse(segment.error().reason); - result = result / *segment; + if (const auto rejection = PathSegment::rejectionOf(name); rejection) + return refuse(*rejection); + value.push_back('/'); + value.append(name); } - return result; + if (value.empty()) + return root(); + return RemotePath{std::move(value)}; } bool empty() const noexcept @@ -161,7 +159,7 @@ namespace Paths { if (empty() || isRoot()) return std::nullopt; - return *PathSegment::make(std::string_view{value_}.substr(value_.rfind('/') + 1)); + return PathSegment{std::string{Detail::lastSegment(value_)}}; } /** diff --git a/paths/include/paths/utf8.hpp b/paths/include/paths/utf8.hpp new file mode 100644 index 00000000..b79fe6da --- /dev/null +++ b/paths/include/paths/utf8.hpp @@ -0,0 +1,327 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace Paths +{ + /** + * @brief U+FFFD, written in place of every byte or code unit that is not part of a valid sequence. + */ + constexpr char32_t replacementCharacter = 0xFFFDu; + + /** + * @brief A 16 bit code unit type: char16_t, and wchar_t on Windows. + */ + template + concept Utf16CodeUnit = (std::same_as || std::same_as) && sizeof(CharT) == 2; + + namespace Detail + { + /** + * @brief Length of the well-formed UTF-8 sequence that starts at @p index of @p text, 0 when the bytes there + * are none (stray continuation byte, overlong form, surrogate, above U+10FFFF or truncated). + */ + constexpr std::size_t utf8SequenceLength(std::string_view text, std::size_t index) noexcept + { + const auto lead = static_cast(text[index]); + if (lead < 0x80u) + return 1u; + + const auto [length, initial] = [lead]() -> std::pair + { + if ((lead & 0xE0u) == 0xC0u) + return {2u, lead & 0x1Fu}; + if ((lead & 0xF0u) == 0xE0u) + return {3u, lead & 0x0Fu}; + if ((lead & 0xF8u) == 0xF0u) + return {4u, lead & 0x07u}; + return {0u, 0u}; + }(); + if (length == 0u || index + length > text.size()) + return 0u; + + std::uint32_t codePoint = initial; + for (std::size_t offset = 1; offset < length; ++offset) + { + const auto continuation = static_cast(text[index + offset]); + if ((continuation & 0xC0u) != 0x80u) + return 0u; + codePoint = (codePoint << 6u) | (continuation & 0x3Fu); + } + constexpr std::array minimumForLength{0u, 0u, 0x80u, 0x800u, 0x10000u}; + if (codePoint < minimumForLength[length] || codePoint > 0x10FFFFu || + (codePoint >= 0xD800u && codePoint <= 0xDFFFu)) + return 0u; + return length; + } + + /** + * @brief The code point of the valid sequence of @p length bytes at @p index of @p text, see + * @ref utf8SequenceLength. + */ + constexpr char32_t utf8CodePointAt(std::string_view text, std::size_t index, std::size_t length) noexcept + { + constexpr std::array leadMask{0u, 0x7Fu, 0x1Fu, 0x0Fu, 0x07u}; + std::uint32_t codePoint = static_cast(text[index]) & leadMask[length]; + for (std::size_t offset = 1; offset < length; ++offset) + codePoint = (codePoint << 6u) | (static_cast(text[index + offset]) & 0x3Fu); + return static_cast(codePoint); + } + + /** + * @brief True when @p codePoint is a Unicode scalar value: at most U+10FFFF and no surrogate. + */ + constexpr bool isScalarValue(char32_t codePoint) noexcept + { + return codePoint <= 0x10FFFFu && (codePoint < 0xD800u || codePoint > 0xDFFFu); + } + + /** + * @brief Appends @p codePoint to @p text as UTF-8, U+FFFD when it is no scalar value. + */ + constexpr void appendUtf8(std::string& text, char32_t codePoint) + { + if (!isScalarValue(codePoint)) + codePoint = replacementCharacter; + const auto byte = [](std::uint32_t value) + { + return static_cast(static_cast(value)); + }; + if (codePoint < 0x80u) + { + text.push_back(byte(codePoint)); + } + else if (codePoint < 0x800u) + { + text.push_back(byte(0xC0u | (codePoint >> 6u))); + text.push_back(byte(0x80u | (codePoint & 0x3Fu))); + } + else if (codePoint < 0x10000u) + { + text.push_back(byte(0xE0u | (codePoint >> 12u))); + text.push_back(byte(0x80u | ((codePoint >> 6u) & 0x3Fu))); + text.push_back(byte(0x80u | (codePoint & 0x3Fu))); + } + else + { + text.push_back(byte(0xF0u | (codePoint >> 18u))); + text.push_back(byte(0x80u | ((codePoint >> 12u) & 0x3Fu))); + text.push_back(byte(0x80u | ((codePoint >> 6u) & 0x3Fu))); + text.push_back(byte(0x80u | (codePoint & 0x3Fu))); + } + } + + /** + * @brief Calls @p onCodePoint with every code point of @p text, U+FFFD for each byte that is not part of a + * valid sequence. + */ + template + constexpr void forEachUtf8CodePoint(std::string_view text, FunctionT&& onCodePoint) + { + std::size_t index = 0; + while (index < text.size()) + { + const auto length = utf8SequenceLength(text, index); + if (length == 0u) + { + onCodePoint(replacementCharacter); + ++index; + continue; + } + onCodePoint(utf8CodePointAt(text, index, length)); + index += length; + } + } + + /** + * @brief Calls @p onCodePoint with every code point of the UTF-16 @p text, U+FFFD for each unpaired + * surrogate. + */ + template + constexpr void forEachUtf16CodePoint(std::basic_string_view text, FunctionT&& onCodePoint) + { + std::size_t index = 0; + while (index < text.size()) + { + const auto unit = static_cast(static_cast(text[index])); + const bool isHigh = unit >= 0xD800u && unit <= 0xDBFFu; + const auto next = index + 1 < text.size() + ? static_cast(static_cast(text[index + 1])) + : char32_t{0}; + if (isHigh && next >= 0xDC00u && next <= 0xDFFFu) + { + onCodePoint(0x10000u + ((unit - 0xD800u) << 10u) + (next - 0xDC00u)); + index += 2; + continue; + } + onCodePoint(unit >= 0xD800u && unit <= 0xDFFFu ? replacementCharacter : unit); + ++index; + } + } + } + + /** + * @brief True when @p text is well-formed UTF-8 (no overlong forms, no surrogates, nothing above U+10FFFF). + */ + constexpr bool isValidUtf8(std::string_view text) noexcept + { + std::size_t index = 0; + while (index < text.size()) + { + const auto length = Detail::utf8SequenceLength(text, index); + if (length == 0u) + return false; + index += length; + } + return true; + } + + /** + * @brief @p text with every byte that is not part of a valid UTF-8 sequence replaced by U+FFFD, so the result + * is always valid UTF-8. Valid text comes back unchanged. + */ + constexpr std::string sanitizedUtf8(std::string_view text) + { + if (isValidUtf8(text)) + return std::string{text}; + std::string result; + result.reserve(text.size() + 8u); + Detail::forEachUtf8CodePoint( + text, + [&result](char32_t codePoint) + { + Detail::appendUtf8(result, codePoint); + } + ); + return result; + } + + /** + * @brief The longest start of @p text with at most @p maximumBytes bytes that does not end inside a valid + * UTF-8 sequence. A byte outside any valid sequence counts as a sequence of its own. + */ + constexpr std::string_view truncatedUtf8(std::string_view text, std::size_t maximumBytes) noexcept + { + if (text.size() <= maximumBytes) + return text; + std::size_t end = 0; + while (end < text.size()) + { + const auto length = Detail::utf8SequenceLength(text, end); + const auto step = length == 0u ? 1u : length; + if (end + step > maximumBytes) + break; + end += step; + } + return text.substr(0, end); + } + + /** + * @brief The number of UTF-16 code units of the valid UTF-8 @p text, the unit Windows limits names in. + */ + constexpr std::size_t utf16Length(std::string_view text) noexcept + { + std::size_t units = 0; + std::size_t index = 0; + while (index < text.size()) + { + const auto length = Detail::utf8SequenceLength(text, index); + units += length == 4u ? 2u : 1u; + index += length == 0u ? 1u : length; + } + return units; + } + + /** + * @brief True when the UTF-16 @p text has no unpaired surrogate. + */ + template + constexpr bool isValidUtf16(std::basic_string_view text) noexcept + { + bool valid = true; + std::size_t index = 0; + while (index < text.size() && valid) + { + const auto unit = static_cast(text[index]); + if (unit >= 0xD800u && unit <= 0xDBFFu && index + 1 < text.size()) + { + const auto next = static_cast(text[index + 1]); + if (next >= 0xDC00u && next <= 0xDFFFu) + { + index += 2; + continue; + } + } + valid = unit < 0xD800u || unit > 0xDFFFu; + ++index; + } + return valid; + } + + /** + * @brief The UTF-8 @p text as UTF-16, every byte that is not part of a valid sequence becomes U+FFFD. + */ + template + constexpr std::basic_string utf8ToUtf16(std::string_view text) + { + std::basic_string result; + result.reserve(text.size()); + Detail::forEachUtf8CodePoint( + text, + [&result](char32_t codePoint) + { + if (codePoint < 0x10000u) + { + result.push_back(static_cast(codePoint)); + return; + } + const auto offset = codePoint - 0x10000u; + result.push_back(static_cast(0xD800u + (offset >> 10u))); + result.push_back(static_cast(0xDC00u + (offset & 0x3FFu))); + } + ); + return result; + } + + /** + * @brief The UTF-16 @p text as UTF-8, every unpaired surrogate becomes U+FFFD. + */ + template + constexpr std::string utf16ToUtf8(std::basic_string_view text) + { + std::string result; + result.reserve(text.size()); + Detail::forEachUtf16CodePoint( + text, + [&result](char32_t codePoint) + { + Detail::appendUtf8(result, codePoint); + } + ); + return result; + } + +#ifdef _WIN32 + /** + * @brief UTF-8 to the UTF-16 text of the wide Windows API, see @ref utf8ToUtf16. + */ + inline std::wstring utf8ToWide(std::string_view text) + { + return utf8ToUtf16(text); + } + + /** + * @brief Wide Windows API text to UTF-8, see @ref utf16ToUtf8. + */ + inline std::string wideToUtf8(std::wstring_view text) + { + return utf16ToUtf8(text); + } +#endif +} diff --git a/paths/source/paths/CMakeLists.txt b/paths/source/paths/CMakeLists.txt index 4cc13287..47391ec2 100644 --- a/paths/source/paths/CMakeLists.txt +++ b/paths/source/paths/CMakeLists.txt @@ -1,4 +1,4 @@ add_library(paths INTERFACE) target_include_directories(paths INTERFACE "${CMAKE_CURRENT_LIST_DIR}/../../include") -target_link_libraries(paths INTERFACE nlohmann_json::nlohmann_json) +target_link_libraries(paths INTERFACE nlohmann_json::nlohmann_json fmt::fmt) diff --git a/paths/test/paths/main.cpp b/paths/test/paths/main.cpp index 2fbe76bb..11ab01e4 100644 --- a/paths/test/paths/main.cpp +++ b/paths/test/paths/main.cpp @@ -2,6 +2,7 @@ #include "test_remote_path.hpp" #include "test_local_path.hpp" #include "test_native.hpp" +#include "test_utf8.hpp" #include diff --git a/paths/test/paths/test_local_path.hpp b/paths/test/paths/test_local_path.hpp index 1c5fd92a..963c2b00 100644 --- a/paths/test/paths/test_local_path.hpp +++ b/paths/test/paths/test_local_path.hpp @@ -1,9 +1,13 @@ #pragma once +#include #include #include +#include +#include + namespace Paths::Test { TEST(LocalPathTests, ParsesPosixPaths) @@ -22,7 +26,8 @@ namespace Paths::Test { EXPECT_EQ(LocalPath::fromUtf8("C:\\Users\\tim\\", LocalPlatform::Windows)->toUtf8(), "C:/Users/tim"); EXPECT_EQ(LocalPath::fromUtf8("C:/", LocalPlatform::Windows)->toUtf8(), "C:/"); - EXPECT_EQ(LocalPath::fromUtf8("c:", LocalPlatform::Windows)->toUtf8(), "c:/"); + EXPECT_EQ(LocalPath::fromUtf8("c:", LocalPlatform::Windows)->toUtf8(), "C:/"); + EXPECT_EQ(LocalPath::fromUtf8("d:\\Data", LocalPlatform::Windows)->toUtf8(), "D:/Data"); EXPECT_EQ(LocalPath::fromUtf8("\\\\server\\share\\dir", LocalPlatform::Windows)->toUtf8(), "//server/share/dir"); EXPECT_EQ(LocalPath::fromUtf8("/x", LocalPlatform::Windows).error().reason, PathRejection::NotAbsolute); EXPECT_EQ(LocalPath::fromUtf8("C:x", LocalPlatform::Windows).error().reason, PathRejection::NotAbsolute); @@ -64,13 +69,122 @@ namespace Paths::Test EXPECT_EQ(base.relativeTo(root)->value(), "home/tim"); } + TEST(LocalPathTests, Win32PrefixesAreDroppedAndDevicePathsRefused) + { + const auto windows = [](std::string_view text) + { + return LocalPath::fromUtf8(text, LocalPlatform::Windows); + }; + EXPECT_EQ(windows("\\\\?\\c:\\Users\\tim")->toUtf8(), "C:/Users/tim"); + EXPECT_EQ(windows("\\\\?\\C:")->toUtf8(), "C:/"); + EXPECT_EQ(windows("\\\\?\\UNC\\server\\share\\dir")->toUtf8(), "//server/share/dir"); + EXPECT_EQ(windows("//?/unc/server/share")->toUtf8(), "//server/share/"); + EXPECT_EQ(*windows("\\\\?\\C:\\Users"), *windows("C:/Users")); + for (auto const* text : { + "\\\\.\\PhysicalDrive0", + "\\\\.\\C:\\x", + "\\\\?\\Volume{12345678-1234-1234-1234-123456789abc}\\x", + "\\\\?\\GLOBALROOT\\Device\\x", + "//?/x", + "//../share/x", + "//./share", + }) + { + const auto path = windows(text); + ASSERT_FALSE(path.has_value()) << text; + EXPECT_EQ(path.error().reason, PathRejection::DevicePath) << text; + } + EXPECT_EQ(windows("\\\\?\\UNC\\server").error().reason, PathRejection::NotAbsolute); + EXPECT_EQ(windows("\\\\?\\UNC\\..\\share").error().reason, PathRejection::DevicePath); + + // On POSIX these are ordinary names below the root. + EXPECT_EQ(LocalPath::fromUtf8("//?/x", LocalPlatform::Posix)->toUtf8(), "/?/x"); + } + + TEST(LocalPathTests, RootLengthCoversDrivesAndShares) + { + using Detail::rootLengthOf; + EXPECT_EQ(rootLengthOf("/a", LocalPlatform::Posix), 1u); + EXPECT_EQ(rootLengthOf("a", LocalPlatform::Posix), 0u); + EXPECT_EQ(rootLengthOf("C:/a", LocalPlatform::Windows), 3u); + EXPECT_EQ(rootLengthOf("C:", LocalPlatform::Windows), 2u); + EXPECT_EQ(rootLengthOf("C:a", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("1:/a", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("//server/share/a", LocalPlatform::Windows), 14u); + EXPECT_EQ(rootLengthOf("//server/share", LocalPlatform::Windows), 14u); + EXPECT_EQ(rootLengthOf("//server", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("//server/", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("///share", LocalPlatform::Windows), 0u); + EXPECT_EQ(rootLengthOf("/a", LocalPlatform::Windows), 0u); + } + + TEST(LocalPathTests, RefusesNulAndInvalidUtf8) + { + EXPECT_EQ(LocalPath::fromUtf8(std::string{"/a\0b", 4}, LocalPlatform::Posix).error().reason, PathRejection::ContainsNul); + EXPECT_EQ( + LocalPath::fromUtf8(std::string{"//ser\0ver/share", 16}, LocalPlatform::Windows).error().reason, + PathRejection::ContainsNul + ); + EXPECT_EQ(LocalPath::fromUtf8("/a/\xFF", LocalPlatform::Posix).error().reason, PathRejection::InvalidUtf8); + EXPECT_EQ(LocalPath::fromUtf8("", LocalPlatform::Posix).error().reason, PathRejection::Empty); + } + + TEST(LocalPathTests, WithinComparesThePlatformAndOptionallyIgnoresCase) + { + const auto windowsHome = LocalPath::fromUtf8("C:/Users/Me", LocalPlatform::Windows).value(); + const auto windowsUsers = LocalPath::fromUtf8("c:/USERS", LocalPlatform::Windows).value(); + EXPECT_FALSE(windowsHome.isWithin(windowsUsers)); + EXPECT_TRUE(windowsHome.isWithinIgnoringCase(windowsUsers)); + EXPECT_TRUE(windowsHome.isWithinIgnoringCase(windowsHome)); + EXPECT_TRUE(windowsHome.isWithinIgnoringCase(LocalPath::fromUtf8("C:/", LocalPlatform::Windows).value())); + EXPECT_FALSE(windowsHome.isWithinIgnoringCase(LocalPath::fromUtf8("C:/Use", LocalPlatform::Windows).value())); + EXPECT_FALSE(windowsHome.isWithinIgnoringCase(LocalPath::fromUtf8("D:/", LocalPlatform::Windows).value())); + + const auto posix = LocalPath::fromUtf8("/C:/Users/Me", LocalPlatform::Posix).value(); + EXPECT_FALSE(posix.isWithin(LocalPath::fromUtf8("/", LocalPlatform::Windows).value_or(LocalPath{}))); + EXPECT_FALSE(windowsHome.isWithin(LocalPath::fromUtf8("/", LocalPlatform::Posix).value())); + EXPECT_FALSE(windowsHome.isWithinIgnoringCase(LocalPath::fromUtf8("/", LocalPlatform::Posix).value())); + EXPECT_TRUE(posix.isWithin(LocalPath::fromUtf8("/", LocalPlatform::Posix).value())); + EXPECT_FALSE(LocalPath{}.isWithin(LocalPath{})); + + const auto share = LocalPath::fromUtf8("//server/share/a", LocalPlatform::Windows).value(); + EXPECT_TRUE(share.isWithin(LocalPath::fromUtf8("//server/share", LocalPlatform::Windows).value())); + EXPECT_FALSE(share.isWithin(LocalPath::fromUtf8("//server/sha", LocalPlatform::Windows).value())); + } + + TEST(LocalPathTests, JoinAndAppendOnAnUnsetPathStayUnset) + { + const LocalPath unset{}; + EXPECT_EQ(unset.join(RelativePath::make("a").value()).error().reason, PathRejection::Empty); + EXPECT_EQ(unset.join(RelativePath{}).error().reason, PathRejection::Empty); + EXPECT_TRUE((unset / LocalPathSegment::make("a", LocalPlatform::Posix).value()).empty()); + EXPECT_FALSE(unset.parent().has_value()); + EXPECT_FALSE(unset.filename().has_value()); + + const auto root = LocalPath::fromUtf8("/", LocalPlatform::Posix).value(); + EXPECT_EQ(root.join(RelativePath{})->toUtf8(), "/"); + EXPECT_EQ(root.join(RelativePath::make("a/b").value())->toUtf8(), "/a/b"); + const auto drive = LocalPath::fromUtf8("C:/", LocalPlatform::Windows).value(); + EXPECT_EQ(drive.join(RelativePath::make("a").value())->toUtf8(), "C:/a"); + EXPECT_EQ(drive.join(RelativePath::make("a/x.").value()).error().reason, PathRejection::TrailingDotOrSpace); + } + + TEST(LocalPathTests, RoundTripsThroughJson) + { + const nlohmann::json json = LocalPath::fromUtf8("/home/t\xC3\xA4st").value(); + EXPECT_EQ(json.get(), "/home/t\xC3\xA4st"); + EXPECT_EQ(json.get().toUtf8(), "/home/t\xC3\xA4st"); + EXPECT_THROW(nlohmann::json{"relative"}.at(0).get(), std::invalid_argument); + } + #ifndef __EMSCRIPTEN__ TEST(LocalPathTests, ConvertsToAndFromNativePaths) { - const auto native = std::filesystem::current_path() / "t\xC3\xA4st"; + const auto native = std::filesystem::current_path() / nativeFromUtf8("t\xC3\xA4st \xE6\x9D\xB1\xE4\xBA\xAC"); const auto path = LocalPath::make(native); ASSERT_TRUE(path.has_value()) << path.error().toString(); EXPECT_EQ(path->native(), native.lexically_normal()); + EXPECT_EQ(nativeToUtf8(path->native().filename()), "t\xC3\xA4st \xE6\x9D\xB1\xE4\xBA\xAC"); EXPECT_FALSE(LocalPath::make(std::filesystem::path{"relative"}).has_value()); } #endif diff --git a/paths/test/paths/test_native.hpp b/paths/test/paths/test_native.hpp index 3d239379..a6552220 100644 --- a/paths/test/paths/test_native.hpp +++ b/paths/test/paths/test_native.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -22,6 +23,52 @@ namespace Paths::Test EXPECT_EQ(native.filename().extension(), nativeFromUtf8(".txt")); } + TEST(NativePathTests, ValidTextConvertsLosslessly) + { + const auto converted = convertNativeToUtf8(nativeFromUtf8("/home/Jürgen")); + EXPECT_EQ(converted.utf8, "/home/Jürgen"); + EXPECT_TRUE(converted.lossless); + EXPECT_EQ(checkedNativeToUtf8(nativeFromUtf8("/home/Jürgen")), "/home/Jürgen"); + } + +#ifndef _WIN32 + TEST(NativePathTests, BytesThatAreNotUtf8AreReplacedAndReported) + { + const std::filesystem::path latin1{std::string{"/home/Fran\xE7ois"}}; + const auto converted = convertNativeToUtf8(latin1); + EXPECT_EQ(converted.utf8, "/home/Fran\xEF\xBF\xBDois"); + EXPECT_FALSE(converted.lossless); + EXPECT_EQ(nativeToUtf8(latin1), converted.utf8); + + const auto checked = checkedNativeToUtf8(latin1); + ASSERT_FALSE(checked.has_value()); + EXPECT_EQ(checked.error().reason, PathRejection::InvalidUtf8); + } + + TEST(NativePathTests, LocalPathRefusesANativePathThatIsNotUtf8) + { + const auto made = LocalPath::make(std::filesystem::path{std::string{"/home/Fran\xE7ois"}}); + ASSERT_FALSE(made.has_value()); + EXPECT_EQ(made.error().reason, PathRejection::InvalidUtf8); + } +#else + TEST(NativePathTests, UnpairedSurrogatesAreReplacedAndReported) + { + const std::wstring wide{L'C', L':', L'/', L'a', static_cast(0xD800u)}; + const auto converted = convertNativeToUtf8(std::filesystem::path{wide}); + EXPECT_EQ(converted.utf8, "C:/a\xEF\xBF\xBD"); + EXPECT_FALSE(converted.lossless); + const auto made = LocalPath::make(std::filesystem::path{wide}); + ASSERT_FALSE(made.has_value()); + EXPECT_EQ(made.error().reason, PathRejection::InvalidUtf8); + } + + TEST(NativePathTests, TextThatIsNotUtf8BecomesReplacementCharacters) + { + EXPECT_EQ(nativeFromUtf8("a\xFF").native(), std::wstring{L"a\uFFFD"}); + } +#endif + #ifdef _WIN32 TEST(NativePathTests, BackslashesBecomeForwardSlashes) { diff --git a/paths/test/paths/test_path_segment.hpp b/paths/test/paths/test_path_segment.hpp index c78cd828..00ab7ae4 100644 --- a/paths/test/paths/test_path_segment.hpp +++ b/paths/test/paths/test_path_segment.hpp @@ -5,6 +5,8 @@ #include #include +#include +#include namespace Paths::Test { @@ -56,21 +58,118 @@ namespace Paths::Test {"nul.txt", PathRejection::ReservedDeviceName}, {"Lpt9", PathRejection::ReservedDeviceName}, {"com1.tar.gz", PathRejection::ReservedDeviceName}, + {"COM0", PathRejection::ReservedDeviceName}, + {"lpt0.txt", PathRejection::ReservedDeviceName}, + {"COM\xC2\xB9", PathRejection::ReservedDeviceName}, + {"lpt\xC2\xB3.log", PathRejection::ReservedDeviceName}, + {"CONIN$", PathRejection::ReservedDeviceName}, + {"conout$.txt", PathRejection::ReservedDeviceName}, + {"CON .txt", PathRejection::ReservedDeviceName}, + {"nul .tar.gz", PathRejection::ReservedDeviceName}, }) { const auto segment = LocalPathSegment::make(name, LocalPlatform::Windows); ASSERT_FALSE(segment.has_value()) << name; EXPECT_EQ(segment.error().reason, reason) << name; } - for (auto const* name : {"console", "com10", "lpt", "a.b", "nul_"}) + for (auto const* name : {"console", "com10", "lpt", "a.b", "nul_", "COM\xC2\xB4", "conin", " CON"}) EXPECT_TRUE(LocalPathSegment::make(name, LocalPlatform::Windows).has_value()) << name; } + TEST(LocalPathSegmentTests, WindowsLimitsNamesTo255Utf16Units) + { + EXPECT_TRUE(LocalPathSegment::make(std::string(255, 'a'), LocalPlatform::Windows).has_value()); + EXPECT_EQ( + LocalPathSegment::make(std::string(256, 'a'), LocalPlatform::Windows).error().reason, + PathRejection::NameTooLong + ); + + // A code point above U+FFFF takes two UTF-16 units, so 128 of them are 256 units. + const auto rockets = [](std::size_t count) + { + std::string name; + for (std::size_t index = 0; index < count; ++index) + name += "\xF0\x9F\x9A\x80"; + return name; + }; + EXPECT_TRUE(LocalPathSegment::make(rockets(127), LocalPlatform::Windows).has_value()); + EXPECT_EQ( + LocalPathSegment::make(rockets(128), LocalPlatform::Windows).error().reason, PathRejection::NameTooLong + ); + EXPECT_TRUE(LocalPathSegment::make(rockets(128), LocalPlatform::Posix).has_value()); + } + + TEST(LocalPathSegmentTests, RejectionOfAppliesTheSegmentRulesFirst) + { + EXPECT_EQ(LocalPathSegment::rejectionOf("..", LocalPlatform::Windows), PathRejection::DotOrDotDot); + EXPECT_EQ(LocalPathSegment::rejectionOf("a/b", LocalPlatform::Posix), PathRejection::ContainsSeparator); + EXPECT_EQ(LocalPathSegment::rejectionOf("a:b", LocalPlatform::Windows), PathRejection::ReservedCharacter); + EXPECT_FALSE(LocalPathSegment::rejectionOf("a:b", LocalPlatform::Posix).has_value()); + } + + TEST(Utf8Tests, AcceptsOnlyWellFormedSequences) + { + for (auto const* text : {"", "ascii", "\xC3\xA4", "\xE2\x82\xAC", "\xF0\x9F\x9A\x80", "\xF4\x8F\xBF\xBF"}) + EXPECT_TRUE(isValidUtf8(text)) << text; + for (auto const* text : { + "\x80", + "\xBF", + "\xC3", + "\xE2\x82", + "\xF0\x9F\x9A", + "\xF4\x90\x80\x80", + "\xF5\x80\x80\x80", + "\xF8\x88\x80\x80\x80", + "\xFC\x84\x80\x80\x80\x80", + "\xFF", + "\xE0\x80\xAF", + "\xF0\x80\x80\xAF", + "\xED\xBF\xBF", + "a\xC3(b", + }) + EXPECT_FALSE(isValidUtf8(text)) << text; + } + + TEST(Utf8Tests, CountsUtf16Units) + { + EXPECT_EQ(utf16Length(""), 0u); + EXPECT_EQ(utf16Length("abc"), 3u); + EXPECT_EQ(utf16Length("\xC3\xA4\xE2\x82\xAC"), 2u); + EXPECT_EQ(utf16Length("\xF0\x9F\x9A\x80"), 2u); + } + + TEST(PathErrorTests, EscapesControlBytesAndInvalidUtf8) + { + const auto printable = [](std::string input) + { + return PathError{.reason = PathRejection::InvalidUtf8, .input = std::move(input)}.toString(); + }; + EXPECT_EQ(printable("plain"), "'plain' is not valid UTF-8"); + EXPECT_EQ(printable("tab\there\x7F"), "'tab\\x09here\\x7F' is not valid UTF-8"); + EXPECT_EQ(printable(std::string{"a\0b", 3}), "'a\\x00b' is not valid UTF-8"); + EXPECT_EQ(printable("Gr\xC3\xBC\xC3\x9F\xF0\x9F\x9A\x80"), "'Gr\xC3\xBC\xC3\x9F\xF0\x9F\x9A\x80' is not valid UTF-8"); + EXPECT_EQ(printable("a\xC3(\xFF\xE2\x82"), "'a\\xC3(\\xFF\\xE2\\x82' is not valid UTF-8"); + EXPECT_EQ(printable("\xED\xA0\x80"), "'\\xED\\xA0\\x80' is not valid UTF-8"); + EXPECT_TRUE(isValidUtf8(printable("\xC0\xAF\xF8\x88\x80\x80\x80"))); + } + TEST(RelativePathTests, ParsesAndJoinsSegments) { const auto path = RelativePath::make("a/b/c.txt"); ASSERT_TRUE(path.has_value()); - EXPECT_EQ(path->segments().size(), 3u); + ASSERT_EQ(path->segments().size(), 3u); + EXPECT_EQ(path->segments()[1].value(), "b"); + auto views = path->segmentViews(); + EXPECT_EQ( + std::vector(views.begin(), views.end()), (std::vector{"a", "b", "c.txt"}) + ); + EXPECT_TRUE(RelativePath{}.segments().empty()); + EXPECT_EQ((RelativePath{} / PathSegment::make("x").value()).value(), "x"); + EXPECT_EQ((*path / RelativePath{}).value(), "a/b/c.txt"); + EXPECT_EQ((RelativePath{} / *path).value(), "a/b/c.txt"); + EXPECT_EQ((*RelativePath::make("a") / *RelativePath::make("b/c")).value(), "a/b/c"); + EXPECT_FALSE(RelativePath{}.parent().has_value()); + EXPECT_EQ(RelativePath::make("a")->parent()->value(), ""); EXPECT_EQ(path->leaf()->value(), "c.txt"); EXPECT_EQ(path->parent()->value(), "a/b"); EXPECT_EQ(RelativePath{}.value(), ""); @@ -80,7 +179,21 @@ namespace Paths::Test TEST(RelativePathTests, RefusesAnythingThatCouldLeaveItsRoot) { - for (auto const* text : {"/a", "..", "a/../b", "a//b", "a/", "./a", "x/.."}) + for (auto const* text : {"/a", "..", "a/../b", "a//b", "a/", "./a", "x/..", "a/\xFF"}) EXPECT_FALSE(RelativePath::make(text).has_value()) << text; + EXPECT_EQ(RelativePath::make("/a").error().reason, PathRejection::EscapesRoot); + EXPECT_EQ(RelativePath::make(std::string{"a/b\0c", 5}).error().reason, PathRejection::ContainsNul); + } + + TEST(PathSplitTests, SplitsAtEverySlash) + { + const auto parts = [](std::string_view text) + { + auto split = Detail::splitAtSlashes(text); + return std::vector(split.begin(), split.end()); + }; + EXPECT_TRUE(parts("").empty()); + EXPECT_EQ(parts("a"), (std::vector{"a"})); + EXPECT_EQ(parts("/a//b/"), (std::vector{"", "a", "", "b", ""})); } } diff --git a/paths/test/paths/test_remote_path.hpp b/paths/test/paths/test_remote_path.hpp index 300632dc..7f92e6d4 100644 --- a/paths/test/paths/test_remote_path.hpp +++ b/paths/test/paths/test_remote_path.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -21,6 +22,37 @@ namespace Paths::Test EXPECT_EQ(RemotePath::make("/a/../b").error().reason, PathRejection::DotOrDotDot); } + TEST(RemotePathTests, RefusesNulAndInvalidUtf8) + { + EXPECT_EQ(RemotePath::make(std::string{"/a\0b", 4}).error().reason, PathRejection::ContainsNul); + EXPECT_EQ(RemotePath::make("/home/\xC3").error().reason, PathRejection::InvalidUtf8); + EXPECT_EQ(RemotePath::make("/\xED\xA0\x80").error().reason, PathRejection::InvalidUtf8); + EXPECT_EQ(RemotePath::make("/home/t\xC3\xA4st")->value(), "/home/t\xC3\xA4st"); + } + + TEST(RemotePathTests, JoiningOntoAnUnsetPathStaysUnset) + { + const RemotePath unset{}; + EXPECT_TRUE((unset / PathSegment::make("a").value()).empty()); + EXPECT_TRUE((unset / RelativePath::make("a/b").value()).empty()); + EXPECT_TRUE((unset / RelativePath{}).empty()); + EXPECT_EQ((RemotePath::root() / RelativePath{}).value(), "/"); + EXPECT_EQ((RemotePath::root() / PathSegment::make("a").value()).value(), "/a"); + } + + TEST(RemotePathTests, SegmentsAndRelativePathsRoundTripThroughJson) + { + const nlohmann::json segment = PathSegment::make("a\\b").value(); + EXPECT_EQ(segment.get().value(), "a\\b"); + EXPECT_THROW(nlohmann::json{"a/b"}.at(0).get(), std::invalid_argument); + + const nlohmann::json relative = RelativePath::make("a/b").value(); + EXPECT_EQ(relative.get(), "a/b"); + EXPECT_EQ(relative.get().value(), "a/b"); + EXPECT_EQ(nlohmann::json{""}.at(0).get().value(), ""); + EXPECT_THROW(nlohmann::json{"a/../b"}.at(0).get(), std::invalid_argument); + } + TEST(RemotePathTests, BackslashesAndColonsAreOrdinaryCharacters) { const auto base = RemotePath::make("/home/test").value(); @@ -96,4 +128,21 @@ namespace Paths::Test EXPECT_FALSE(unset.isWithin(RemotePath::root())); EXPECT_FALSE(RemotePath::root().isWithin(unset)); } + + TEST(RemotePathTests, CommonParentHoldsEveryPathBelowIt) + { + const auto path = [](std::string_view text) + { + return RemotePath::make(text).value(); + }; + const std::vector siblings{path("/r/src/a.cpp"), path("/r/README.md")}; + EXPECT_EQ(commonParentOf(std::span{siblings}), path("/r")); + const std::vector nested{path("/r/a/b/c"), path("/r/a/d")}; + EXPECT_EQ(commonParentOf(std::span{nested}), path("/r/a")); + const std::vector single{path("/r/x")}; + EXPECT_EQ(commonParentOf(std::span{single}), path("/r")); + const std::vector root{RemotePath::root()}; + EXPECT_EQ(commonParentOf(std::span{root}), RemotePath::root()); + EXPECT_FALSE(commonParentOf(std::span{})); + } } diff --git a/paths/test/paths/test_utf8.hpp b/paths/test/paths/test_utf8.hpp new file mode 100644 index 00000000..7b681e87 --- /dev/null +++ b/paths/test/paths/test_utf8.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace Paths::Test +{ + TEST(Utf8Tests, SanitizedKeepsValidText) + { + for (const std::string_view text : {"", "plain", "Grüße 東京 \xF0\x9F\x9A\x80"}) + EXPECT_EQ(sanitizedUtf8(text), text); + } + + TEST(Utf8Tests, SanitizedReplacesEveryInvalidByte) + { + EXPECT_EQ(sanitizedUtf8("a\xFF" + "b"), + "a\xEF\xBF\xBD" + "b"); + EXPECT_EQ(sanitizedUtf8("\xC0\xAF"), "\xEF\xBF\xBD\xEF\xBF\xBD"); + EXPECT_EQ(sanitizedUtf8("x\xC3"), "x\xEF\xBF\xBD"); + EXPECT_EQ(sanitizedUtf8("Fran\xE7ois"), "Fran\xEF\xBF\xBDois"); + EXPECT_TRUE(isValidUtf8(sanitizedUtf8("\xED\xA0\x80\xF4\x90\x80\x80\xF8\x88\x80\x80\x80"))); + } + + TEST(Utf8Tests, TruncatedNeverEndsInsideASequence) + { + EXPECT_EQ(truncatedUtf8("abc", 5u), "abc"); + EXPECT_EQ(truncatedUtf8("abc", 2u), "ab"); + EXPECT_EQ(truncatedUtf8("a\xC3\xA4", 2u), "a"); + EXPECT_EQ(truncatedUtf8("a\xC3\xA4", 3u), "a\xC3\xA4"); + EXPECT_EQ(truncatedUtf8("\xF0\x9F\x9A\x80x", 3u), ""); + EXPECT_EQ(truncatedUtf8("a\xFF\xFF", 2u), "a\xFF"); + EXPECT_EQ(truncatedUtf8("", 0u), ""); + } + + TEST(Utf8Tests, Utf16RoundTrip) + { + const std::string text = "Grüße 東京 \xF0\x9F\x9A\x80"; + const auto utf16 = utf8ToUtf16(text); + EXPECT_EQ(utf16, u"Grüße 東京 \U0001F680"); + EXPECT_TRUE(isValidUtf16(std::u16string_view{utf16})); + EXPECT_EQ(utf16ToUtf8(std::u16string_view{utf16}), text); + } + + TEST(Utf8Tests, InvalidUtf8BecomesReplacementInUtf16) + { + EXPECT_EQ(utf8ToUtf16("a\xFF"), u"a�"); + } + + TEST(Utf8Tests, UnpairedSurrogatesBecomeReplacement) + { + const std::u16string lone{u'a', char16_t{0xD800u}, u'b', char16_t{0xDC00u}}; + EXPECT_FALSE(isValidUtf16(std::u16string_view{lone})); + EXPECT_EQ(utf16ToUtf8(std::u16string_view{lone}), "a\xEF\xBF\xBD" + "b\xEF\xBF\xBD"); + const std::u16string trailingHigh{u'x', char16_t{0xDBFFu}}; + EXPECT_FALSE(isValidUtf16(std::u16string_view{trailingHigh})); + EXPECT_EQ(utf16ToUtf8(std::u16string_view{trailingHigh}), "x\xEF\xBF\xBD"); + } +} diff --git a/persistence/include/persistence/state/sftp_options.hpp b/persistence/include/persistence/state/sftp_options.hpp index 41a068fc..1c93f371 100644 --- a/persistence/include/persistence/state/sftp_options.hpp +++ b/persistence/include/persistence/state/sftp_options.hpp @@ -2,6 +2,8 @@ #include +#include + #include #include #include @@ -34,6 +36,16 @@ namespace Persistence return *configured; } + /** + * @brief The suffix a destination is moved aside to while a finished transfer replaces it on a server without + * posix-rename. + * @param tempFileSuffix The effective temp file suffix, see effectiveTempFileSuffix. + */ + inline std::string asideFileSuffix(std::string_view tempFileSuffix) + { + return fmt::format("{}.old", tempFileSuffix); + } + struct CommonTransferOptions : public DefaultMissingMember { std::optional tempFileSuffix{std::nullopt}; diff --git a/persistence/include/persistence/state/state.hpp b/persistence/include/persistence/state/state.hpp index 98db154e..364ccd30 100644 --- a/persistence/include/persistence/state/state.hpp +++ b/persistence/include/persistence/state/state.hpp @@ -14,7 +14,6 @@ #include #include -#include #include #include @@ -59,11 +58,6 @@ namespace Persistence LocalizationOptions localizationOptions{}; std::optional lastDirectConnect{}; std::optional lastDirectConnectInheritables{}; - /** - * @brief When each sync pair (session, local root and remote root) was last synchronized, in seconds since - * the epoch. Sync compares modification times against it to see which side changed since. - */ - std::optional> syncBaselines{}; State fullyResolve() const; @@ -95,7 +89,6 @@ namespace Persistence fileTrackingOptions, localFilesystemOptions, lastDirectConnect, - lastDirectConnectInheritables, - syncBaselines) + lastDirectConnectInheritables) ) } \ No newline at end of file diff --git a/persistence/source/persistence/state_holder_backend.cpp b/persistence/source/persistence/state_holder_backend.cpp index 5ad4f03e..d8acd670 100644 --- a/persistence/source/persistence/state_holder_backend.cpp +++ b/persistence/source/persistence/state_holder_backend.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -26,7 +28,7 @@ namespace Persistence void setupPersistence() { const auto path = Nui::resolvePath(Constants::persistencePath); - const auto parentPath = std::filesystem::path{path}.parent_path().string(); + const auto parentPath = std::filesystem::path{path}.parent_path(); if (!std::filesystem::exists(parentPath)) std::filesystem::create_directories(parentPath); @@ -39,7 +41,8 @@ namespace Persistence const auto now = std::chrono::system_clock::now(); const auto time = fmt::format("{:%Y-%m-%d_%H-%M-%S}", now); - return path.parent_path() / (path.filename().string() + ".backup_" + time); + return path.parent_path() / + Paths::nativeFromUtf8(fmt::format("{}.backup_{}", Paths::nativeToUtf8(path.filename()), time)); }(); { @@ -48,16 +51,16 @@ namespace Persistence writer << reader.rdbuf(); } - Log::info("Copied config file to backup: {}", backupFileName.string()); + Log::info("Copied config file to backup: {}", Paths::nativeToUtf8(backupFileName)); return backupFileName; } LoadWarning unreadableConfig(std::filesystem::path const& path, std::exception const& exception) { - Log::error("Failed to parse config file: {}", exception.what()); + Log::error("Failed to parse config file: {}", Utility::errorMessage(exception)); return LoadWarning{ .kind = LoadWarningKind::ConfigUnreadable, - .arguments = {makeBackup(path).string(), exception.what()}, + .arguments = {Paths::nativeToUtf8(makeBackup(path)), Utility::errorMessage(exception)}, }; } } @@ -134,8 +137,8 @@ namespace Persistence } catch (std::exception const& e) { - Log::error("Failed to load config file: {}", e.what()); - error = e.what(); + Log::error("Failed to load config file: {}", Utility::errorMessage(e)); + error = Utility::errorMessage(e); } onLoad(error, *this, cachedWarnings_); @@ -461,7 +464,7 @@ namespace Persistence } catch (std::exception const& e) { - Log::error("Failed to save config file: {}", e.what()); + Log::error("Failed to save config file: {}", Utility::errorMessage(e)); throw; } } @@ -527,7 +530,8 @@ namespace Persistence hub.callRemote( responseId, nlohmann::json{ - {"error", Utility::localizedMessage("backend.persistence.saveFailed", e.what())}, + {"error", + Utility::localizedMessage("backend.persistence.saveFailed", Utility::errorMessage(e))}, } ); } @@ -583,7 +587,14 @@ namespace Persistence try { - YAML::Node config = YAML::LoadFile(filePath.generic_string()); + std::ifstream file{filePath, std::ios::binary}; + if (!file) + { + Log::error("Cannot open the language file '{}'.", Paths::nativeToUtf8(filePath)); + onLoadComplete(std::nullopt); + return; + } + YAML::Node config = YAML::Load(file); auto translateNode = [&](this const auto& translateNode, YAML::Node const& node) -> nlohmann::json { if (node.IsScalar()) @@ -605,7 +616,7 @@ namespace Persistence } catch (std::exception const& e) { - Log::error("Failed to parse language file: {}", e.what()); + Log::error("Failed to parse language file: {}", Utility::errorMessage(e)); onLoadComplete(std::nullopt); return; } @@ -635,11 +646,11 @@ namespace Persistence ); if (!content) { - Log::error("Failed to load language file: {}", file.string()); + Log::error("Failed to load language file: {}", Paths::nativeToUtf8(file)); onLoadComplete(std::nullopt); return; } - assembled[file.stem().string()] = *content; + assembled[Paths::nativeToUtf8(file.stem())] = *content; } onLoadComplete(std::move(assembled)); } diff --git a/shared_data/benchmark/shared_data/sync_benchmark.cpp b/shared_data/benchmark/shared_data/sync_benchmark.cpp index c9fdfb3f..4c1536e6 100644 --- a/shared_data/benchmark/shared_data/sync_benchmark.cpp +++ b/shared_data/benchmark/shared_data/sync_benchmark.cpp @@ -177,7 +177,7 @@ int main(int argc, char** argv) { return listing.at(relative.empty() ? std::string{"/root"} : "/root/" + relative.value()); }; - TreeDirectoryWalker walker{std::move(scanner)}; + TreeDirectoryWalker walker{std::move(scanner)}; if (!walker.walkAll()) std::abort(); auto tree = std::move(walker).ejectTree(); diff --git a/shared_data/include/shared_data/directory_entry.hpp b/shared_data/include/shared_data/directory_entry.hpp index 8ecdb0f8..3da03398 100644 --- a/shared_data/include/shared_data/directory_entry.hpp +++ b/shared_data/include/shared_data/directory_entry.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -88,17 +90,30 @@ namespace SharedData using FileType = SharedData::FileType; /** - * @brief The name as listed, UTF-8. A server or a hostile local name can be anything, like "..", "x/.." or - * empty, so it only becomes part of a path through Paths::PathSegment::make. Empty for the root of a - * walk and for stat results. + * @brief The name as listed, always valid UTF-8. A server or a hostile local name can be anything, like "..", + * "x/.." or empty, so it only becomes part of a path through @ref nameSegment. A listed name that was + * not UTF-8 is kept for display with U+FFFD for every invalid byte and @ref nameIsValidUtf8 false. + * Empty for the root of a walk and for stat results. */ std::string name{}; + /** + * @brief False when the listed name was not UTF-8 and @ref name only shows it: such an entry can be listed + * but never named in a path, so no action can hit a different item that is called like the display + * text. + */ + bool nameIsValidUtf8{true}; std::uint32_t flags{0}; FileType type{FileType::Unknown}; std::uint64_t size{0}; std::uint32_t uid{0}; std::uint32_t gid{0}; + /** + * @brief The owner name, valid UTF-8 (U+FFFD for every invalid byte the server sent). + */ std::string owner{}; + /** + * @brief The group name, valid UTF-8 (U+FFFD for every invalid byte the server sent). + */ std::string group{}; std::filesystem::perms permissions{std::filesystem::perms::unknown}; std::uint64_t atime{0}; @@ -108,12 +123,26 @@ namespace SharedData std::uint64_t mtime{0}; std::uint32_t mtimeNsec{0}; std::string acl{}; - // For symlinks: raw value returned by readlink (i.e. the link literal, not the resolved target) as UTF-8. - // nullopt when not a symlink or when unavailable. This is what the sync diff compares by. + /** + * @brief For symlinks: the link literal readlink returned (not the resolved target), valid UTF-8 with U+FFFD + * for every invalid byte. nullopt when not a symlink or when unavailable. The sync diff compares by it, + * a transfer of the link reads the literal itself. + */ std::optional linkTarget{}; // For symlinks: the full entry of the symlink target (if known). nullptr means unknown. std::shared_ptr resolvedTarget{nullptr}; + /** + * @brief The name as a path segment. Refused like Paths::PathSegment::make refuses it, and with InvalidUtf8 + * when the listed name was not UTF-8. + */ + std::expected nameSegment() const + { + if (!nameIsValidUtf8) + return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::InvalidUtf8, .input = name}); + return Paths::PathSegment::make(name); + } + bool isDirectory() const { return type == FileType::Directory; @@ -224,29 +253,13 @@ namespace SharedData /** * @brief The path of @p entry below the root of its walk, following the parent indices. Empty for the root, - * which is the entry without a parent. Refused when a name on the way is no path segment. + * which is the entry without a parent. Refused when a name on the way is no path segment or a parent index + * lies outside @p entries. */ inline std::expected relativePath(std::vector const& entries, DirectoryEntry const& entry) { - if (!entry.parent) - return Paths::RelativePath{}; - const auto parentIndex = entry.parent.value(); - if (parentIndex >= entries.size()) - throw std::out_of_range("Parent index is out of range"); - return Paths::PathSegment::make(entry.name) - .and_then( - [&entries, parentIndex](Paths::PathSegment const& segment) - { - return relativePath(entries, entries[parentIndex]) - .transform( - [&segment](Paths::RelativePath const& parent) - { - return parent / segment; - } - ); - } - ); + return Utility::walkedRelativePath(entries, entry); } void to_json(nlohmann::json& j, DirectoryEntry const& entry); diff --git a/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp b/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp index f5ec99ca..8e3d0118 100644 --- a/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp +++ b/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -8,6 +9,7 @@ #include #include +#include #include namespace SharedData @@ -21,12 +23,17 @@ namespace SharedData bool isDirectory{false}; /** @brief Only shown on the card for files. */ std::uint64_t sizeBytes{0}; + /** + * @brief What a sync compare saw at the target, which is then only deleted while it still holds that. Set + * by the backend's sync only, it is not part of the request the frontend sends. + */ + std::optional compared{}; }; BOOST_DESCRIBE_STRUCT(BulkDeleteEntry, (), (target, isDirectory, sizeBytes)) /** - * @brief Single-RPC payload for deleting many remote items. Files become one aggregate delete, every directory - * becomes a delete of its own that scans first. + * @brief Single-RPC payload for deleting many remote items as one delete operation: files are deleted as listed, + * every directory is scanned first and deleted with its content. */ struct BulkDeleteRequest { diff --git a/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp b/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp index aee44469..e1c49df7 100644 --- a/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp +++ b/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -8,6 +9,7 @@ #include #include +#include #include namespace SharedData @@ -22,13 +24,28 @@ namespace SharedData /** @brief Trusted for files, the backend does not stat them. */ std::uint64_t sizeBytes{0}; bool isDirectory{false}; + /** + * @brief A directory that is created without its content, as an entry of the files' bulk instead of a bulk + * of its own that scans it. + */ + bool withoutContent{false}; /** * @brief The user agreed to replace this existing destination. Without it, or the request's allowOverwrite, * a destination that appeared after the existence check is not replaced. */ bool mayOverwrite{false}; + /** + * @brief What a sync compare saw at the destination: the entry fails with ChangedSinceCompare instead of + * replacing a destination that no longer holds it. Set by the backend's sync only, it is not part of + * the request the frontend sends. + */ + std::optional comparedDestination{}; }; - BOOST_DESCRIBE_STRUCT(BulkUploadEntry, (), (source, destination, sizeBytes, isDirectory, mayOverwrite)) + BOOST_DESCRIBE_STRUCT( + BulkUploadEntry, + (), + (source, destination, sizeBytes, isDirectory, withoutContent, mayOverwrite) + ) /** * @brief One remote item of a bulk download. A directory is scanned and downloaded with its content. @@ -40,16 +57,20 @@ namespace SharedData /** @brief Trusted for files, the backend does not stat them. */ std::uint64_t sizeBytes{0}; bool isDirectory{false}; + /** @brief See @ref BulkUploadEntry::withoutContent. */ + bool withoutContent{false}; /** @brief Modification time of the remote file, restored on the local copy so the next sync sees no change. */ std::uint64_t mtime{0}; std::uint32_t mtimeNsec{0}; /** @brief See @ref BulkUploadEntry::mayOverwrite. */ bool mayOverwrite{false}; + /** @brief See @ref BulkUploadEntry::comparedDestination. */ + std::optional comparedDestination{}; }; BOOST_DESCRIBE_STRUCT( BulkDownloadEntry, (), - (source, destination, sizeBytes, isDirectory, mtime, mtimeNsec, mayOverwrite) + (source, destination, sizeBytes, isDirectory, withoutContent, mtime, mtimeNsec, mayOverwrite) ) /** @@ -62,8 +83,14 @@ namespace SharedData bool allowOverwrite{false}; bool insertRefresh{false}; OperationMode mode{OperationMode::Queued}; + /** + * @brief The directories the files' bulk works below, the labels of its card. Unset, the closest directories + * holding every source and every destination. + */ + std::optional localRoot{}; + std::optional remoteRoot{}; }; - BOOST_DESCRIBE_STRUCT(BulkUploadRequest, (), (entries, allowOverwrite, insertRefresh, mode)) + BOOST_DESCRIBE_STRUCT(BulkUploadRequest, (), (entries, allowOverwrite, insertRefresh, mode, localRoot, remoteRoot)) /** * @brief Download analogue of @ref BulkUploadRequest. @@ -74,6 +101,17 @@ namespace SharedData bool allowOverwrite{false}; bool insertRefresh{false}; OperationMode mode{OperationMode::Queued}; + /** + * @brief The directories the files' bulk works below, the labels of its card. No link may lie between the + * local one and a destination. Unset, the closest directories holding every source and every + * destination. + */ + std::optional localRoot{}; + std::optional remoteRoot{}; }; - BOOST_DESCRIBE_STRUCT(BulkDownloadRequest, (), (entries, allowOverwrite, insertRefresh, mode)) + BOOST_DESCRIBE_STRUCT( + BulkDownloadRequest, + (), + (entries, allowOverwrite, insertRefresh, mode, localRoot, remoteRoot) + ) } diff --git a/shared_data/include/shared_data/file_operations/compared_state.hpp b/shared_data/include/shared_data/file_operations/compared_state.hpp new file mode 100644 index 00000000..1cc113c4 --- /dev/null +++ b/shared_data/include/shared_data/file_operations/compared_state.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include + +#include +#include +#include + +namespace SharedData +{ + /** + * @brief What a sync compare saw at a path that one of its entries replaces or deletes: nothing, or the type, + * size and modification time of what was there, the link literal for a symlink. + */ + struct ComparedState + { + bool exists{false}; + FileType type{FileType::Unknown}; + std::uint64_t size{0}; + std::uint64_t mtime{0}; + std::optional linkTarget{}; + + bool operator==(ComparedState const&) const = default; + }; + + /** + * @brief @p entry, as an lstat or a listing describes it, in the terms of a compare. + */ + ComparedState comparedStateOf(DirectoryEntry const& entry); + + /** + * @brief True when @p current is still what the compare saw in @p compared. A path seen absent must still be + * absent. One seen present may be gone now, nothing there can be lost. Otherwise the type must be the + * same, a symlink its literal, a regular file or a special one its size and modification time; a + * directory changes by its type only. + */ + bool unchangedSinceCompare(ComparedState const& compared, ComparedState const& current); +} diff --git a/shared_data/include/shared_data/file_operations/operation_completed.hpp b/shared_data/include/shared_data/file_operations/operation_completed.hpp index 5b27fdc3..0961c9d2 100644 --- a/shared_data/include/shared_data/file_operations/operation_completed.hpp +++ b/shared_data/include/shared_data/file_operations/operation_completed.hpp @@ -10,13 +10,25 @@ #include -#include #include +#include +#include +#include namespace SharedData { BOOST_DEFINE_ENUM_CLASS(OperationCompletionReason, Completed, Canceled, Failed) + /** + * @brief A skipped entry of a prescanned bulk up- or download, by its index, and why it was skipped. + */ + struct IndexedFailure + { + std::uint32_t index{0}; + OperationErrorType type{OperationErrorType::UnknownError}; + }; + BOOST_DESCRIBE_STRUCT(IndexedFailure, (), (index, type)) + struct OperationCompleted { OperationCompletionReason reason; @@ -27,10 +39,12 @@ namespace SharedData std::optional error{std::nullopt}; /** @brief The skipped entries of a bulk by their source path, a remote path for downloads. */ std::vector> failedEntries{}; + /** @brief The skipped entries of a prescanned bulk up- or download by their index. */ + std::vector indexedFailures{}; }; BOOST_DESCRIBE_STRUCT( OperationCompleted, (), - (reason, operationId, completionTime, localPath, remotePath, error, failedEntries) + (reason, operationId, completionTime, localPath, remotePath, error, failedEntries, indexedFailures) ) } \ No newline at end of file diff --git a/shared_data/include/shared_data/file_operations/operation_error_type.hpp b/shared_data/include/shared_data/file_operations/operation_error_type.hpp index 20ce72dd..8349d97c 100644 --- a/shared_data/include/shared_data/file_operations/operation_error_type.hpp +++ b/shared_data/include/shared_data/file_operations/operation_error_type.hpp @@ -47,6 +47,8 @@ namespace SharedData ScanIncomplete, DirectoryUnreadable, InvalidName, - LinkInLocalPath + LinkInLocalPath, + ChannelClosed, + ChangedSinceCompare ); } \ No newline at end of file diff --git a/shared_data/include/shared_data/sync/case_fold.hpp b/shared_data/include/shared_data/sync/case_fold.hpp new file mode 100644 index 00000000..a1cd54a2 --- /dev/null +++ b/shared_data/include/shared_data/sync/case_fold.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace SharedData::Sync +{ + /** + * @brief The key under which names meet on a filesystem that folds case or normalization, like NTFS, APFS and + * HFS+: every letter is lowercased (ASCII, Latin-1, Latin Extended-A, Greek, Cyrillic, fullwidth Latin) + * and every precomposed letter of those scripts is decomposed into its base and combining marks, so an + * NFC and an NFD spelling of a name get the same key. Text that is not UTF-8 is its own key. + */ + std::string caseFoldedName(std::string_view name); + + /** + * @brief False when @p name is its own case folded key, which holds for most names and costs no allocation. + */ + bool foldingChanges(std::string_view name); +} diff --git a/shared_data/include/shared_data/sync/diff.hpp b/shared_data/include/shared_data/sync/diff.hpp index 1d9add0d..9e50265c 100644 --- a/shared_data/include/shared_data/sync/diff.hpp +++ b/shared_data/include/shared_data/sync/diff.hpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -33,17 +32,28 @@ namespace SharedData::Sync bool recursive{true}; /// When true, entries with any path segment starting with '.' are dropped. bool ignoreHidden{false}; - /// The local filesystem folds case (Windows, macOS), set by the backend. Remote names that differ only in - /// case would land on one local file then. + /** + * @brief The local filesystem folds case (Windows, macOS), set by the backend. Names that differ only in case + * or normalization would land on one local file then. + */ bool localCaseInsensitive{false}; - /// When this pair was last synchronized (seconds since the epoch); a side whose modification time is later - /// changed since. Unset for the first sync, newest wins then. - std::optional lastSyncTime{}; + /** + * @brief The server looks like it folds case (Windows or macOS paths), set by the backend. Names that differ + * only in case or normalization would land on one remote file then. + */ + bool remoteCaseInsensitive{false}; }; BOOST_DESCRIBE_STRUCT( DiffOptions, (), - (direction, actionUpload, actionDownload, actionDelete, recursive, ignoreHidden, localCaseInsensitive, lastSyncTime) + (direction, + actionUpload, + actionDownload, + actionDelete, + recursive, + ignoreHidden, + localCaseInsensitive, + remoteCaseInsensitive) ) /** @@ -102,6 +112,12 @@ namespace SharedData::Sync /** @brief True for names the "ignore hidden" option drops. */ bool isHiddenName(std::string_view name); + /** + * @brief Whether a child inside a directory that is transferred or deleted as a whole is part of it: listed and + * not dropped as hidden. + */ + bool isPlannable(ScanNode const& child, DiffOptions const& options); + /** * @brief Parallel merge walk of two sorted scan trees. * @@ -114,14 +130,11 @@ namespace SharedData::Sync * whose @ref ScanNode::childrenKnown is false, * - `options.ignoreHidden` filters per-name at every level. * + * A file that differs on both sides goes from the newer side to the other one in Both mode. + * * @see DiffSink::onProgress heartbeats fire every 512 compares. */ - void diffScanTrees( - ScanNode const& local, - ScanNode const& remote, - DiffOptions const& options, - DiffSink& sink - ); + void diffScanTrees(ScanNode const& local, ScanNode const& remote, DiffOptions const& options, DiffSink& sink); inline void to_json(nlohmann::json& j, DiffOptions const& s) { diff --git a/shared_data/include/shared_data/sync/diff_summary.hpp b/shared_data/include/shared_data/sync/diff_summary.hpp index d8d29dac..bee72953 100644 --- a/shared_data/include/shared_data/sync/diff_summary.hpp +++ b/shared_data/include/shared_data/sync/diff_summary.hpp @@ -13,16 +13,7 @@ namespace SharedData::Sync /** * @brief Why the diff left an entry alone even though it exists on one or both sides. */ - BOOST_DEFINE_ENUM_CLASS( - BlockReason, - Unreadable, - Excluded, - InvalidName, - TypeConflict, - CaseConflict, - ChangedOnBothSides, - ChangedOnDestination - ) + BOOST_DEFINE_ENUM_CLASS(BlockReason, Unreadable, Excluded, InvalidName, TypeConflict, CaseConflict) /** * @brief An entry sync does not act on, with the reason. diff --git a/shared_data/include/shared_data/sync/diff_tree_store.hpp b/shared_data/include/shared_data/sync/diff_tree_store.hpp index de398bfb..42872693 100644 --- a/shared_data/include/shared_data/sync/diff_tree_store.hpp +++ b/shared_data/include/shared_data/sync/diff_tree_store.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -33,6 +34,12 @@ namespace SharedData::Sync */ ScanNode const* findScanNode(ScanNode const& root, std::string_view relKey); + /** + * @brief What the scan @p root saw at @p relKey: absent when no node is there, else its type, size and + * modification time and the link literal of a symlink. + */ + ComparedState comparedStateAt(ScanNode const& root, std::string_view relKey); + /** @brief True when @p action reads from the local side (upload, local delete). */ bool actionSourceIsLocal(Action action); diff --git a/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp b/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp index 8c14244c..ec94584c 100644 --- a/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp +++ b/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp @@ -1,29 +1,19 @@ #pragma once -#include #include -#include - #include #include namespace SharedData::Sync { /** - * @brief One row of the backend-computed enqueue plan. - * - * The backend collapses the frontend's selected relKey set into a minimal list - * that: + * @brief One entry of the plan @ref DiffTreeStore::buildPlan makes from a sparse selection. * - * - emits a one-sided directory as a single bulk-dir entry when every descendant - * is selected (saves N transfers for N descendants), - * - emits an individual leaf otherwise, - * - skips intermediate directory rows with partial selection — the enqueue - * side creates missing directories as needed via the bulk upload/download - * flow. - * - * Entries carry no paths: the backend resolves @ref relKey below its own sync roots. + * A transfer lists the files and symlinks of a selected directory, plus every directory that has no + * transferred content and so has to be created by itself; the files create their parents. A delete lists a + * directory before its listed content. Entries name rows by relKey, the backend resolves them below its own + * sync roots. */ struct EnqueuePlanEntry { @@ -34,23 +24,4 @@ namespace SharedData::Sync std::uint32_t mtimeNsec{0}; bool isDirectory{false}; }; - BOOST_DESCRIBE_STRUCT( - EnqueuePlanEntry, - (), - (relKey, action, sizeBytes, mtime, mtimeNsec, isDirectory) - ) - - inline void to_json(nlohmann::json& j, EnqueuePlanEntry const& e) - { - SharedData::to_json(j, e); - } - inline void from_json(nlohmann::json const& j, EnqueuePlanEntry& e) - { - SharedData::from_json(j, e); - } - -#ifdef NUI_FRONTEND - // No to_val/from_val forwarder for EnqueuePlanEntry — described-struct path - // in nui's convertToVal handles it natively. -#endif } diff --git a/shared_data/include/shared_data/sync/sync_execution.hpp b/shared_data/include/shared_data/sync/sync_execution.hpp index fe000bf2..f2978d45 100644 --- a/shared_data/include/shared_data/sync/sync_execution.hpp +++ b/shared_data/include/shared_data/sync/sync_execution.hpp @@ -13,8 +13,8 @@ namespace SharedData::Sync { /** - * @brief Asks the backend to plan and queue the selected rows of one section. Paths never travel: the backend - * resolves every row below its own roots. + * @brief Asks the backend to plan and queue the selected rows of one section. The request names rows only, the + * backend resolves every row below its own roots. * * The operation ids are chosen by the frontend, so it can listen to them before the reply arrives. An id is * unused when its part of the plan is empty. @@ -43,19 +43,15 @@ namespace SharedData::Sync */ struct SyncExecutionResult { - /** @brief Rows of the bulk upload, in the order of its entry indices. */ + /** @brief Rows of the bulk upload, in the order of its entry indices, by which it reports failures. */ std::vector uploads{}; - /** @brief The source each upload row reports when it fails, in the same order. */ - std::vector uploadSources{}; std::vector downloads{}; - std::vector downloadSources{}; /** @brief Every listed item of a directory delete is a row of its own, its content after it. */ std::vector remoteDeletes{}; + /** @brief The path each delete row reports when it fails, in the same order. */ std::vector remoteDeleteSources{}; std::vector localDeletes{}; std::vector localDeleteSources{}; - /** @brief Directories without content, created right away. */ - std::vector createdDirectories{}; /** @brief Rows that were refused or failed right away, nothing was queued for them. */ std::vector failed{}; }; @@ -63,14 +59,11 @@ namespace SharedData::Sync SyncExecutionResult, (), (uploads, - uploadSources, downloads, - downloadSources, remoteDeletes, remoteDeleteSources, localDeletes, localDeleteSources, - createdDirectories, failed) ) diff --git a/shared_data/include/shared_data/sync/tree_directory_walker.hpp b/shared_data/include/shared_data/sync/tree_directory_walker.hpp index 5537bf68..74a749c9 100644 --- a/shared_data/include/shared_data/sync/tree_directory_walker.hpp +++ b/shared_data/include/shared_data/sync/tree_directory_walker.hpp @@ -34,7 +34,7 @@ namespace SharedData::Sync * No second pass is needed. Each directory's children are assigned once, sorted, before any of * them is queued, so the queued pointers stay valid. */ - template + template requires std::is_invocable_r_v< std::expected, WalkErrorType>, ScannerT, @@ -145,9 +145,9 @@ namespace SharedData::Sync if (raw.type == SharedData::FileType::Regular) totalBytes_ += raw.size; - // The listed name as it is, "x/.." stays an invalid name. + // The listed name as it is, "x/.." or a name that was not UTF-8 stays an invalid name. + const auto availability = nameAvailability(raw); std::string nameUtf8 = std::move(raw.name); - const auto availability = nameAvailability(nameUtf8, raw.type); children.push_back( ScanNode{ .name = std::move(nameUtf8), @@ -244,9 +244,11 @@ namespace SharedData::Sync type == SharedData::FileType::Symlink; } - Availability nameAvailability(std::string const& name, SharedData::FileType type) const + Availability nameAvailability(SharedData::DirectoryEntry const& entry) const { - const auto segment = Paths::PathSegment::make(name); + auto const& name = entry.name; + const auto type = entry.type; + const auto segment = entry.nameSegment(); if (!segment) return Availability::InvalidName; if (requireLocalNames_ && !Paths::LocalPathSegment::make(*segment).has_value()) diff --git a/shared_data/source/shared_data/CMakeLists.txt b/shared_data/source/shared_data/CMakeLists.txt index 5511a387..5fbaf3b3 100644 --- a/shared_data/source/shared_data/CMakeLists.txt +++ b/shared_data/source/shared_data/CMakeLists.txt @@ -2,7 +2,9 @@ add_library( shared-data STATIC directory_entry.cpp + file_operations/compared_state.cpp ignore_rules.cpp + sync/case_fold.cpp sync/diff.cpp sync/diff_tree_store.cpp sync/enqueue_minimizer.cpp diff --git a/shared_data/source/shared_data/directory_entry.cpp b/shared_data/source/shared_data/directory_entry.cpp index 61265b8b..81172095 100644 --- a/shared_data/source/shared_data/directory_entry.cpp +++ b/shared_data/source/shared_data/directory_entry.cpp @@ -12,6 +12,8 @@ namespace SharedData {"mtime", entry.mtime}, {"mtimeNsec", entry.mtimeNsec}, }; + if (!entry.nameIsValidUtf8) + j["nameIsValidUtf8"] = false; if (entry.flags != 0) j["flags"] = entry.flags; if (entry.uid != 0) @@ -46,6 +48,7 @@ namespace SharedData j.at("name").get_to(entry.name); j.at("type").get_to(entry.type); + getOptional("nameIsValidUtf8", entry.nameIsValidUtf8); getOptional("flags", entry.flags); getOptional("size", entry.size); getOptional("uid", entry.uid); diff --git a/shared_data/source/shared_data/file_operations/compared_state.cpp b/shared_data/source/shared_data/file_operations/compared_state.cpp new file mode 100644 index 00000000..6ccce2c5 --- /dev/null +++ b/shared_data/source/shared_data/file_operations/compared_state.cpp @@ -0,0 +1,34 @@ +#include + +namespace SharedData +{ + ComparedState comparedStateOf(DirectoryEntry const& entry) + { + return ComparedState{ + .exists = true, + .type = entry.type, + .size = entry.size, + .mtime = entry.mtime, + .linkTarget = entry.type == FileType::Symlink ? entry.linkTarget : std::nullopt, + }; + } + + bool unchangedSinceCompare(ComparedState const& compared, ComparedState const& current) + { + if (!compared.exists) + return !current.exists; + if (!current.exists) + return true; + if (compared.type != current.type) + return false; + switch (compared.type) + { + case FileType::Directory: + return true; + case FileType::Symlink: + return compared.linkTarget == current.linkTarget; + default: + return compared.size == current.size && compared.mtime == current.mtime; + } + } +} diff --git a/shared_data/source/shared_data/ignore_rules.cpp b/shared_data/source/shared_data/ignore_rules.cpp index 7dcffaba..e072466e 100644 --- a/shared_data/source/shared_data/ignore_rules.cpp +++ b/shared_data/source/shared_data/ignore_rules.cpp @@ -1,5 +1,6 @@ #include +#include #include namespace SharedData @@ -135,9 +136,8 @@ namespace SharedData continue; const auto relToOriginStr = ruleSet.originDir.empty() - ? relPath.value() - : relPath.value().substr(ruleSet.originDir.value().size() + 1u); - const auto relToOrigin = *Paths::RelativePath::make(relToOriginStr); + ? std::string_view{relPath.value()} + : std::string_view{relPath.value()}.substr(ruleSet.originDir.value().size() + 1u); for (auto const& pattern : ruleSet.patterns) { @@ -151,14 +151,13 @@ namespace SharedData } else { - for (auto const& component : relToOrigin.segments()) - { - if (globMatch(pattern.pattern, component.view())) + matched = std::ranges::any_of( + Paths::Detail::splitAtSlashes(relToOriginStr), + [&pattern](std::string_view component) { - matched = true; - break; + return globMatch(pattern.pattern, component); } - } + ); if (!matched) matched = globMatch(pattern.pattern, relToOriginStr); } diff --git a/shared_data/source/shared_data/sync/case_fold.cpp b/shared_data/source/shared_data/sync/case_fold.cpp new file mode 100644 index 00000000..3919d9e5 --- /dev/null +++ b/shared_data/source/shared_data/sync/case_fold.cpp @@ -0,0 +1,319 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace SharedData::Sync +{ + namespace + { + struct Decomposition + { + char32_t composed; + char32_t base; + char32_t mark; + }; + + /** + * @brief Canonical decompositions into two code points of Latin-1, Latin Extended-A, Greek and Cyrillic, + * sorted by the composed code point. A base can decompose again. + */ + constexpr std::array decompositions = std::to_array( + {{0xC0u, 0x41u, 0x300u}, + {0xC1u, 0x41u, 0x301u}, + {0xC2u, 0x41u, 0x302u}, + {0xC3u, 0x41u, 0x303u}, + {0xC4u, 0x41u, 0x308u}, + {0xC5u, 0x41u, 0x30Au}, + {0xC7u, 0x43u, 0x327u}, + {0xC8u, 0x45u, 0x300u}, + {0xC9u, 0x45u, 0x301u}, + {0xCAu, 0x45u, 0x302u}, + {0xCBu, 0x45u, 0x308u}, + {0xCCu, 0x49u, 0x300u}, + {0xCDu, 0x49u, 0x301u}, + {0xCEu, 0x49u, 0x302u}, + {0xCFu, 0x49u, 0x308u}, + {0xD1u, 0x4Eu, 0x303u}, + {0xD2u, 0x4Fu, 0x300u}, + {0xD3u, 0x4Fu, 0x301u}, + {0xD4u, 0x4Fu, 0x302u}, + {0xD5u, 0x4Fu, 0x303u}, + {0xD6u, 0x4Fu, 0x308u}, + {0xD9u, 0x55u, 0x300u}, + {0xDAu, 0x55u, 0x301u}, + {0xDBu, 0x55u, 0x302u}, + {0xDCu, 0x55u, 0x308u}, + {0xDDu, 0x59u, 0x301u}, + {0xE0u, 0x61u, 0x300u}, + {0xE1u, 0x61u, 0x301u}, + {0xE2u, 0x61u, 0x302u}, + {0xE3u, 0x61u, 0x303u}, + {0xE4u, 0x61u, 0x308u}, + {0xE5u, 0x61u, 0x30Au}, + {0xE7u, 0x63u, 0x327u}, + {0xE8u, 0x65u, 0x300u}, + {0xE9u, 0x65u, 0x301u}, + {0xEAu, 0x65u, 0x302u}, + {0xEBu, 0x65u, 0x308u}, + {0xECu, 0x69u, 0x300u}, + {0xEDu, 0x69u, 0x301u}, + {0xEEu, 0x69u, 0x302u}, + {0xEFu, 0x69u, 0x308u}, + {0xF1u, 0x6Eu, 0x303u}, + {0xF2u, 0x6Fu, 0x300u}, + {0xF3u, 0x6Fu, 0x301u}, + {0xF4u, 0x6Fu, 0x302u}, + {0xF5u, 0x6Fu, 0x303u}, + {0xF6u, 0x6Fu, 0x308u}, + {0xF9u, 0x75u, 0x300u}, + {0xFAu, 0x75u, 0x301u}, + {0xFBu, 0x75u, 0x302u}, + {0xFCu, 0x75u, 0x308u}, + {0xFDu, 0x79u, 0x301u}, + {0xFFu, 0x79u, 0x308u}, + {0x100u, 0x41u, 0x304u}, + {0x101u, 0x61u, 0x304u}, + {0x102u, 0x41u, 0x306u}, + {0x103u, 0x61u, 0x306u}, + {0x104u, 0x41u, 0x328u}, + {0x105u, 0x61u, 0x328u}, + {0x106u, 0x43u, 0x301u}, + {0x107u, 0x63u, 0x301u}, + {0x108u, 0x43u, 0x302u}, + {0x109u, 0x63u, 0x302u}, + {0x10Au, 0x43u, 0x307u}, + {0x10Bu, 0x63u, 0x307u}, + {0x10Cu, 0x43u, 0x30Cu}, + {0x10Du, 0x63u, 0x30Cu}, + {0x10Eu, 0x44u, 0x30Cu}, + {0x10Fu, 0x64u, 0x30Cu}, + {0x112u, 0x45u, 0x304u}, + {0x113u, 0x65u, 0x304u}, + {0x114u, 0x45u, 0x306u}, + {0x115u, 0x65u, 0x306u}, + {0x116u, 0x45u, 0x307u}, + {0x117u, 0x65u, 0x307u}, + {0x118u, 0x45u, 0x328u}, + {0x119u, 0x65u, 0x328u}, + {0x11Au, 0x45u, 0x30Cu}, + {0x11Bu, 0x65u, 0x30Cu}, + {0x11Cu, 0x47u, 0x302u}, + {0x11Du, 0x67u, 0x302u}, + {0x11Eu, 0x47u, 0x306u}, + {0x11Fu, 0x67u, 0x306u}, + {0x120u, 0x47u, 0x307u}, + {0x121u, 0x67u, 0x307u}, + {0x122u, 0x47u, 0x327u}, + {0x123u, 0x67u, 0x327u}, + {0x124u, 0x48u, 0x302u}, + {0x125u, 0x68u, 0x302u}, + {0x128u, 0x49u, 0x303u}, + {0x129u, 0x69u, 0x303u}, + {0x12Au, 0x49u, 0x304u}, + {0x12Bu, 0x69u, 0x304u}, + {0x12Cu, 0x49u, 0x306u}, + {0x12Du, 0x69u, 0x306u}, + {0x12Eu, 0x49u, 0x328u}, + {0x12Fu, 0x69u, 0x328u}, + {0x130u, 0x49u, 0x307u}, + {0x134u, 0x4Au, 0x302u}, + {0x135u, 0x6Au, 0x302u}, + {0x136u, 0x4Bu, 0x327u}, + {0x137u, 0x6Bu, 0x327u}, + {0x139u, 0x4Cu, 0x301u}, + {0x13Au, 0x6Cu, 0x301u}, + {0x13Bu, 0x4Cu, 0x327u}, + {0x13Cu, 0x6Cu, 0x327u}, + {0x13Du, 0x4Cu, 0x30Cu}, + {0x13Eu, 0x6Cu, 0x30Cu}, + {0x143u, 0x4Eu, 0x301u}, + {0x144u, 0x6Eu, 0x301u}, + {0x145u, 0x4Eu, 0x327u}, + {0x146u, 0x6Eu, 0x327u}, + {0x147u, 0x4Eu, 0x30Cu}, + {0x148u, 0x6Eu, 0x30Cu}, + {0x14Cu, 0x4Fu, 0x304u}, + {0x14Du, 0x6Fu, 0x304u}, + {0x14Eu, 0x4Fu, 0x306u}, + {0x14Fu, 0x6Fu, 0x306u}, + {0x150u, 0x4Fu, 0x30Bu}, + {0x151u, 0x6Fu, 0x30Bu}, + {0x154u, 0x52u, 0x301u}, + {0x155u, 0x72u, 0x301u}, + {0x156u, 0x52u, 0x327u}, + {0x157u, 0x72u, 0x327u}, + {0x158u, 0x52u, 0x30Cu}, + {0x159u, 0x72u, 0x30Cu}, + {0x15Au, 0x53u, 0x301u}, + {0x15Bu, 0x73u, 0x301u}, + {0x15Cu, 0x53u, 0x302u}, + {0x15Du, 0x73u, 0x302u}, + {0x15Eu, 0x53u, 0x327u}, + {0x15Fu, 0x73u, 0x327u}, + {0x160u, 0x53u, 0x30Cu}, + {0x161u, 0x73u, 0x30Cu}, + {0x162u, 0x54u, 0x327u}, + {0x163u, 0x74u, 0x327u}, + {0x164u, 0x54u, 0x30Cu}, + {0x165u, 0x74u, 0x30Cu}, + {0x168u, 0x55u, 0x303u}, + {0x169u, 0x75u, 0x303u}, + {0x16Au, 0x55u, 0x304u}, + {0x16Bu, 0x75u, 0x304u}, + {0x16Cu, 0x55u, 0x306u}, + {0x16Du, 0x75u, 0x306u}, + {0x16Eu, 0x55u, 0x30Au}, + {0x16Fu, 0x75u, 0x30Au}, + {0x170u, 0x55u, 0x30Bu}, + {0x171u, 0x75u, 0x30Bu}, + {0x172u, 0x55u, 0x328u}, + {0x173u, 0x75u, 0x328u}, + {0x174u, 0x57u, 0x302u}, + {0x175u, 0x77u, 0x302u}, + {0x176u, 0x59u, 0x302u}, + {0x177u, 0x79u, 0x302u}, + {0x178u, 0x59u, 0x308u}, + {0x179u, 0x5Au, 0x301u}, + {0x17Au, 0x7Au, 0x301u}, + {0x17Bu, 0x5Au, 0x307u}, + {0x17Cu, 0x7Au, 0x307u}, + {0x17Du, 0x5Au, 0x30Cu}, + {0x17Eu, 0x7Au, 0x30Cu}, + {0x385u, 0xA8u, 0x301u}, + {0x386u, 0x391u, 0x301u}, + {0x388u, 0x395u, 0x301u}, + {0x389u, 0x397u, 0x301u}, + {0x38Au, 0x399u, 0x301u}, + {0x38Cu, 0x39Fu, 0x301u}, + {0x38Eu, 0x3A5u, 0x301u}, + {0x38Fu, 0x3A9u, 0x301u}, + {0x390u, 0x3CAu, 0x301u}, + {0x3AAu, 0x399u, 0x308u}, + {0x3ABu, 0x3A5u, 0x308u}, + {0x3ACu, 0x3B1u, 0x301u}, + {0x3ADu, 0x3B5u, 0x301u}, + {0x3AEu, 0x3B7u, 0x301u}, + {0x3AFu, 0x3B9u, 0x301u}, + {0x3B0u, 0x3CBu, 0x301u}, + {0x3CAu, 0x3B9u, 0x308u}, + {0x3CBu, 0x3C5u, 0x308u}, + {0x3CCu, 0x3BFu, 0x301u}, + {0x3CDu, 0x3C5u, 0x301u}, + {0x3CEu, 0x3C9u, 0x301u}, + {0x3D3u, 0x3D2u, 0x301u}, + {0x3D4u, 0x3D2u, 0x308u}, + {0x400u, 0x415u, 0x300u}, + {0x401u, 0x415u, 0x308u}, + {0x403u, 0x413u, 0x301u}, + {0x407u, 0x406u, 0x308u}, + {0x40Cu, 0x41Au, 0x301u}, + {0x40Du, 0x418u, 0x300u}, + {0x40Eu, 0x423u, 0x306u}, + {0x419u, 0x418u, 0x306u}, + {0x439u, 0x438u, 0x306u}, + {0x450u, 0x435u, 0x300u}, + {0x451u, 0x435u, 0x308u}, + {0x453u, 0x433u, 0x301u}, + {0x457u, 0x456u, 0x308u}, + {0x45Cu, 0x43Au, 0x301u}, + {0x45Du, 0x438u, 0x300u}, + {0x45Eu, 0x443u, 0x306u}, + {0x476u, 0x474u, 0x30Fu}, + {0x477u, 0x475u, 0x30Fu}, + {0x4C1u, 0x416u, 0x306u}, + {0x4C2u, 0x436u, 0x306u}, + {0x4D0u, 0x410u, 0x306u}, + {0x4D1u, 0x430u, 0x306u}, + {0x4D2u, 0x410u, 0x308u}, + {0x4D3u, 0x430u, 0x308u}, + {0x4D6u, 0x415u, 0x306u}, + {0x4D7u, 0x435u, 0x306u}, + {0x4DAu, 0x4D8u, 0x308u}, + {0x4DBu, 0x4D9u, 0x308u}, + {0x4DCu, 0x416u, 0x308u}, + {0x4DDu, 0x436u, 0x308u}, + {0x4DEu, 0x417u, 0x308u}, + {0x4DFu, 0x437u, 0x308u}, + {0x4E2u, 0x418u, 0x304u}, + {0x4E3u, 0x438u, 0x304u}, + {0x4E4u, 0x418u, 0x308u}, + {0x4E5u, 0x438u, 0x308u}, + {0x4E6u, 0x41Eu, 0x308u}, + {0x4E7u, 0x43Eu, 0x308u}, + {0x4EAu, 0x4E8u, 0x308u}, + {0x4EBu, 0x4E9u, 0x308u}, + {0x4ECu, 0x42Du, 0x308u}, + {0x4EDu, 0x44Du, 0x308u}, + {0x4EEu, 0x423u, 0x304u}, + {0x4EFu, 0x443u, 0x304u}, + {0x4F0u, 0x423u, 0x308u}, + {0x4F1u, 0x443u, 0x308u}, + {0x4F2u, 0x423u, 0x30Bu}, + {0x4F3u, 0x443u, 0x30Bu}, + {0x4F4u, 0x427u, 0x308u}, + {0x4F5u, 0x447u, 0x308u}, + {0x4F8u, 0x42Bu, 0x308u}, + {0x4F9u, 0x44Bu, 0x308u}} + ); + + char32_t foldedLetter(char32_t codePoint) + { + // Simple case folding differs from the lowercase mapping in these. + switch (codePoint) + { + case 0xB5u: + return 0x3BCu; + case 0x17Fu: + return U's'; + case 0x3C2u: + return 0x3C3u; + default: + return Utility::FuzzySearch::toLowerCase(codePoint); + } + } + + void appendDecomposed(char32_t codePoint, std::u32string& out) + { + const auto found = std::ranges::lower_bound(decompositions, codePoint, {}, &Decomposition::composed); + if (found == decompositions.end() || found->composed != codePoint) + { + out.push_back(foldedLetter(codePoint)); + return; + } + appendDecomposed(found->base, out); + out.push_back(found->mark); + } + } + + bool foldingChanges(std::string_view name) + { + return std::ranges::any_of( + name, + [](char character) + { + const auto byte = static_cast(character); + return byte >= 0x80u || (byte >= 'A' && byte <= 'Z'); + } + ); + } + + std::string caseFoldedName(std::string_view name) + { + if (!foldingChanges(name)) + return std::string{name}; + if (!Paths::isValidUtf8(name)) + return std::string{name}; + std::u32string folded; + folded.reserve(name.size()); + for (const auto codePoint : Utility::Utf8::decode(name)) + appendDecomposed(codePoint, folded); + return Utility::Utf8::encode(folded); + } +} diff --git a/shared_data/source/shared_data/sync/diff.cpp b/shared_data/source/shared_data/sync/diff.cpp index fe5a7907..dbad1f31 100644 --- a/shared_data/source/shared_data/sync/diff.cpp +++ b/shared_data/source/shared_data/sync/diff.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -17,6 +18,16 @@ namespace SharedData::Sync { constexpr std::uint64_t progressCheckpointInterval = 512; + /** + * @brief State of one merge walk. + */ + struct Walk + { + DiffOptions const& options; + DiffSink& sink; + std::uint64_t compared{0}; + }; + std::string joinRel(std::string const& parentRelKey, std::string const& name) { if (parentRelKey.empty()) @@ -138,16 +149,11 @@ namespace SharedData::Sync } } - /** - * @brief Forward declaration — recursion point for directory pairs. - */ void mergeChildren( std::vector const& localChildren, std::vector const& remoteChildren, std::string const& parentRelKey, - DiffOptions const& options, - DiffSink& sink, - std::uint64_t& comparedCounter + Walk& walk ); std::optional blockReasonOf(ScanNode const& node) @@ -166,36 +172,16 @@ namespace SharedData::Sync return BlockReason::Excluded; } - /** - * @brief The latest modification time in @p node and everything listed below it. - */ - std::uint64_t newestModification(ScanNode const& node) - { - auto newest = node.mtime; - for (auto const& child : node.children) - newest = std::max(newest, newestModification(child)); - return newest; - } - - /** - * @brief A delete must not remove what appeared or changed on that side since the last sync, nobody has - * seen it on the other side. - */ - bool changedSinceLastSync(ScanNode const& node, DiffOptions const& options) - { - return options.lastSyncTime && newestModification(node) > *options.lastSyncTime; - } - void handleBothPresent( ScanNode const& localChild, ScanNode const& remoteChild, std::string const& relKey, std::string const& parentRelKey, - DiffOptions const& options, - DiffSink& sink, - std::uint64_t& comparedCounter + Walk& walk ) { + auto const& options = walk.options; + auto& sink = walk.sink; // Unknown or untransferable on one side: neither side may be changed for this name. if (auto reason = blockReasonOf(localChild).or_else([&]() { return blockReasonOf(remoteChild); }); reason) { @@ -220,32 +206,13 @@ namespace SharedData::Sync if (sameDir) { if (options.recursive) - mergeChildren(localChild.children, remoteChild.children, relKey, options, sink, comparedCounter); + mergeChildren(localChild.children, remoteChild.children, relKey, walk); return; } if (!entriesDiffer(localChild, remoteChild)) return; - // With a known last sync, the side changed since then is the one that holds new data. - if (options.lastSyncTime) - { - const bool localChanged = localChild.mtime > *options.lastSyncTime; - const bool remoteChanged = remoteChild.mtime > *options.lastSyncTime; - const bool destinationChanged = (options.direction == Direction::Upload && remoteChanged) || - (options.direction == Direction::Download && localChanged); - if (options.direction == Direction::Both && localChanged && remoteChanged) - { - sink.onBlocked(relKey, BlockReason::ChangedOnBothSides); - return; - } - if (destinationChanged) - { - sink.onBlocked(relKey, BlockReason::ChangedOnDestination); - return; - } - } - const auto action = chooseTwoSidedAction(localChild, remoteChild, options); if (!action) return; @@ -256,22 +223,11 @@ namespace SharedData::Sync sink.emitDownload(std::move(row), parentRelKey); } - std::string foldedCase(std::string_view name) - { - std::string folded{name}; - for (auto& character : folded) - { - if (character >= 'A' && character <= 'Z') - character = static_cast(character - 'A' + 'a'); - } - return folded; - } - /** - * @brief Names of one directory that would meet as one file on a case folding filesystem: a local-only and a - * remote-only name that differ only in case, and remote names that differ only in case when the local - * filesystem folds case. Transferring either could overwrite the other and a delete could hit both. - * ASCII letters only. + * @brief Names of one directory that would meet as one file on a destination that folds case: variants that + * differ only in case or normalization across the sides, and variants on the side transferred from. + * Transferring one could overwrite the other and a delete could hit both. Only the directions whose + * destination folds count, a case sensitive destination stores every variant. See @ref caseFoldedName. */ std::unordered_set caseConflictsOf( std::vector const& localChildren, @@ -279,18 +235,24 @@ namespace SharedData::Sync DiffOptions const& options ) { - const auto hasUpper = [](std::vector const& children) + const bool localDestinationFolds = options.direction != Direction::Upload && options.localCaseInsensitive; + const bool remoteDestinationFolds = + options.direction != Direction::Download && options.remoteCaseInsensitive; + if (!localDestinationFolds && !remoteDestinationFolds) + return {}; + + const auto anyFolds = [](std::vector const& children) { return std::ranges::any_of( children, [](ScanNode const& node) { - return std::ranges::any_of(node.name, [](char character) { return character >= 'A' && character <= 'Z'; }); + return foldingChanges(node.name); } ); }; - // Without an upper case letter folding changes nothing, which is the common case. - if (!hasUpper(localChildren) && !hasUpper(remoteChildren)) + // Lower case ASCII names fold to themselves, which is the common case. + if (!anyFolds(localChildren) && !anyFolds(remoteChildren)) return {}; struct Variants @@ -300,9 +262,9 @@ namespace SharedData::Sync }; std::unordered_map byFolded; for (auto const& node : localChildren) - byFolded[foldedCase(node.name)].local.push_back(node.name); + byFolded[caseFoldedName(node.name)].local.push_back(node.name); for (auto const& node : remoteChildren) - byFolded[foldedCase(node.name)].remote.push_back(node.name); + byFolded[caseFoldedName(node.name)].remote.push_back(node.name); std::unordered_set conflicts; for (auto const& [folded, variants] : byFolded) @@ -316,8 +278,9 @@ namespace SharedData::Sync ); } ); - const bool remoteVariants = options.localCaseInsensitive && variants.remote.size() > 1; - if (!crossSide && !remoteVariants) + const bool remoteVariants = localDestinationFolds && variants.remote.size() > 1; + const bool localVariants = remoteDestinationFolds && variants.local.size() > 1; + if (!crossSide && !remoteVariants && !localVariants) continue; for (auto const name : variants.local) conflicts.emplace(name); @@ -331,11 +294,11 @@ namespace SharedData::Sync std::vector const& localChildren, std::vector const& remoteChildren, std::string const& parentRelKey, - DiffOptions const& options, - DiffSink& sink, - std::uint64_t& comparedCounter + Walk& walk ) { + auto const& options = walk.options; + auto& sink = walk.sink; std::size_t li = 0; std::size_t ri = 0; const auto caseConflicts = caseConflictsOf(localChildren, remoteChildren, options); @@ -347,10 +310,11 @@ namespace SharedData::Sync return true; }; - const auto bumpAndCheckpoint = [&]() { - ++comparedCounter; - if (comparedCounter % progressCheckpointInterval == 0) - sink.onProgress(comparedCounter); + const auto bumpAndCheckpoint = [&]() + { + ++walk.compared; + if (walk.compared % progressCheckpointInterval == 0) + sink.onProgress(walk.compared); }; while (li < localChildren.size() || ri < remoteChildren.size()) @@ -373,7 +337,7 @@ namespace SharedData::Sync if ((!options.ignoreHidden || !isHiddenName(lc.name)) && !blockedForCase(lc.name)) { const auto relKey = joinRel(parentRelKey, lc.name); - handleBothPresent(lc, rc, relKey, parentRelKey, options, sink, comparedCounter); + handleBothPresent(lc, rc, relKey, parentRelKey, walk); } ++li; ++ri; @@ -389,9 +353,6 @@ namespace SharedData::Sync else if (auto action = chooseLocalOnlyAction(lc, options); action) { const auto relKey = joinRel(parentRelKey, lc.name); - if (*action == Action::DeleteLocal && changedSinceLastSync(lc, options)) - sink.onBlocked(relKey, BlockReason::ChangedOnDestination); - else emitOneSided( makeOneSidedRow(lc, relKey, *action, true), *action, parentRelKey, options, sink ); @@ -410,9 +371,6 @@ namespace SharedData::Sync else if (auto action = chooseRemoteOnlyAction(rc, options); action) { const auto relKey = joinRel(parentRelKey, rc.name); - if (*action == Action::DeleteRemote && changedSinceLastSync(rc, options)) - sink.onBlocked(relKey, BlockReason::ChangedOnDestination); - else emitOneSided( makeOneSidedRow(rc, relKey, *action, false), *action, parentRelKey, options, sink ); @@ -455,6 +413,11 @@ namespace SharedData::Sync return !name.empty() && name.front() == '.'; } + bool isPlannable(ScanNode const& child, DiffOptions const& options) + { + return child.availability == Availability::Listed && (!options.ignoreHidden || !isHiddenName(child.name)); + } + /** * @brief Link literals compare with repeated separators collapsed, "a//b" points where "a/b" does. */ @@ -492,17 +455,12 @@ namespace SharedData::Sync return false; } - void diffScanTrees( - ScanNode const& local, - ScanNode const& remote, - DiffOptions const& options, - DiffSink& sink - ) + void diffScanTrees(ScanNode const& local, ScanNode const& remote, DiffOptions const& options, DiffSink& sink) { - std::uint64_t compared = 0; - mergeChildren(local.children, remote.children, std::string{}, options, sink, compared); + Walk walk{.options = options, .sink = sink}; + mergeChildren(local.children, remote.children, std::string{}, walk); // Final heartbeat so observers see the completed count even if it didn't // land on a checkpoint boundary. - sink.onProgress(compared); + sink.onProgress(walk.compared); } } diff --git a/shared_data/source/shared_data/sync/diff_tree_store.cpp b/shared_data/source/shared_data/sync/diff_tree_store.cpp index 1962649d..b123bee9 100644 --- a/shared_data/source/shared_data/sync/diff_tree_store.cpp +++ b/shared_data/source/shared_data/sync/diff_tree_store.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -75,6 +76,20 @@ namespace SharedData::Sync return current; } + ComparedState comparedStateAt(ScanNode const& root, std::string_view relKey) + { + auto const* node = findScanNode(root, relKey); + if (!node) + return ComparedState{}; + return ComparedState{ + .exists = true, + .type = node->type, + .size = node->size, + .mtime = node->mtime, + .linkTarget = node->type == FileType::Symlink ? node->linkTarget : std::nullopt, + }; + } + bool actionSourceIsLocal(Action action) { return action == Action::Upload || action == Action::DeleteLocal; @@ -192,9 +207,7 @@ namespace SharedData::Sync keys.reserve(directory->children.size()); for (auto const& child : directory->children) { - if (options_.ignoreHidden && isHiddenName(child.name)) - continue; - if (child.availability != Availability::Listed) + if (!isPlannable(child, options_)) continue; auto childKey = joinRelKey(parentRelKey, child.name); nodes_.insert_or_assign(childKey, makeOneSidedRow(child, childKey, parent.action, sourceIsLocal)); @@ -224,33 +237,23 @@ namespace SharedData::Sync }; }; - // Downloaded onto a case folding filesystem, names that differ only in case would meet as one file. + // Transferred onto a case folding filesystem, names that differ only in case would meet as one file. std::unordered_map foldedCounts; - const bool foldsCase = action == Action::Download && options_.localCaseInsensitive; - const auto folded = [](std::string name) - { - for (auto& character : name) - { - if (character >= 'A' && character <= 'Z') - character = static_cast(character - 'A' + 'a'); - } - return name; - }; + const bool foldsCase = (action == Action::Download && options_.localCaseInsensitive) || + (action == Action::Upload && options_.remoteCaseInsensitive); if (foldsCase) { for (auto const& child : directory.children) - ++foldedCounts[folded(child.name)]; + ++foldedCounts[caseFoldedName(child.name)]; } bool anyChild = false; for (auto const& child : directory.children) { - if (options_.ignoreHidden && isHiddenName(child.name)) - continue; // Unknown or untransferable entries are never planned, not even inside a transferred directory. - if (child.availability != Availability::Listed) + if (!isPlannable(child, options_)) continue; - if (foldsCase && foldedCounts[folded(child.name)] > 1) + if (foldsCase && foldedCounts[caseFoldedName(child.name)] > 1) continue; anyChild = true; auto childKey = joinRelKey(relKey, child.name); @@ -282,9 +285,7 @@ namespace SharedData::Sync ); for (auto const& child : directory.children) { - if (options_.ignoreHidden && isHiddenName(child.name)) - continue; - if (child.availability != Availability::Listed) + if (!isPlannable(child, options_)) continue; auto childKey = joinRelKey(relKey, child.name); if (child.type == FileType::Directory) diff --git a/shared_data/test/shared_data/main.cpp b/shared_data/test/shared_data/main.cpp index e83708e2..7c47bdb9 100644 --- a/shared_data/test/shared_data/main.cpp +++ b/shared_data/test/shared_data/main.cpp @@ -1,5 +1,6 @@ #include "test_bulk_delete_request.hpp" #include "test_bulk_transfer_request.hpp" +#include "test_compared_state.hpp" #include "test_operation_added.hpp" #include "test_diff.hpp" #include "test_diff_tree_store.hpp" diff --git a/shared_data/test/shared_data/test_bulk_transfer_request.hpp b/shared_data/test/shared_data/test_bulk_transfer_request.hpp index 77deef0a..b39f0882 100644 --- a/shared_data/test/shared_data/test_bulk_transfer_request.hpp +++ b/shared_data/test/shared_data/test_bulk_transfer_request.hpp @@ -36,18 +36,27 @@ namespace SharedData::Test { {.source = local("a.txt"), .destination = remote("/srv/a\\b"), .sizeBytes = 3}, {.source = local("dir"), .destination = remote("/srv/dir"), .isDirectory = true}, + {.source = local("empty"), + .destination = remote("/srv/empty"), + .isDirectory = true, + .withoutContent = true}, }, .allowOverwrite = true, .insertRefresh = true, .mode = OperationMode::PriorityQueued, + .remoteRoot = remote("/srv"), }; const nlohmann::json json = request; const auto read = json.get(); - ASSERT_EQ(read.entries.size(), 2u); + ASSERT_EQ(read.entries.size(), 3u); EXPECT_EQ(read.entries[0].source, local("a.txt")); EXPECT_EQ(read.entries[0].destination.value(), "/srv/a\\b"); EXPECT_EQ(read.entries[0].sizeBytes, 3u); EXPECT_TRUE(read.entries[1].isDirectory); + EXPECT_FALSE(read.entries[1].withoutContent); + EXPECT_TRUE(read.entries[2].withoutContent); + EXPECT_EQ(read.remoteRoot, remote("/srv")); + EXPECT_FALSE(read.localRoot); EXPECT_TRUE(read.allowOverwrite); EXPECT_TRUE(read.insertRefresh); EXPECT_EQ(read.mode, OperationMode::PriorityQueued); @@ -59,11 +68,12 @@ namespace SharedData::Test .entries = { {.source = remote("/srv/a.txt"), - .destination = local("a.txt"), - .sizeBytes = 7, - .mtime = 1700000000u, - .mtimeNsec = 5u}, + .destination = local("a.txt"), + .sizeBytes = 7, + .mtime = 1700000000u, + .mtimeNsec = 5u}, }, + .localRoot = local(""), }; const nlohmann::json json = request; const auto read = json.get(); @@ -73,6 +83,8 @@ namespace SharedData::Test EXPECT_EQ(read.entries[0].sizeBytes, 7u); EXPECT_EQ(read.entries[0].mtime, 1700000000u); EXPECT_EQ(read.entries[0].mtimeNsec, 5u); + EXPECT_EQ(read.localRoot, local("")); + EXPECT_FALSE(read.remoteRoot); } TEST(BulkTransferRequestTests, InvalidPathsAreRefusedOnRead) diff --git a/shared_data/test/shared_data/test_compared_state.hpp b/shared_data/test/shared_data/test_compared_state.hpp new file mode 100644 index 00000000..82dded67 --- /dev/null +++ b/shared_data/test/shared_data/test_compared_state.hpp @@ -0,0 +1,82 @@ +#pragma once + +#include + +#include + +namespace SharedData::Test +{ + namespace ComparedStateTestDetail + { + inline ComparedState file(std::uint64_t size, std::uint64_t mtime) + { + return ComparedState{.exists = true, .type = FileType::Regular, .size = size, .mtime = mtime}; + } + + inline ComparedState link(std::string target) + { + return ComparedState{.exists = true, .type = FileType::Symlink, .mtime = 5, .linkTarget = std::move(target)}; + } + } + + TEST(ComparedStateTests, AnUnchangedFilePasses) + { + using namespace ComparedStateTestDetail; + EXPECT_TRUE(unchangedSinceCompare(file(3, 100), file(3, 100))); + } + + TEST(ComparedStateTests, AFileWithAnotherSizeOrTimeChanged) + { + using namespace ComparedStateTestDetail; + EXPECT_FALSE(unchangedSinceCompare(file(3, 100), file(4, 100))); + EXPECT_FALSE(unchangedSinceCompare(file(3, 100), file(3, 101))); + } + + TEST(ComparedStateTests, SomethingThatAppearedWhereNothingWasChanged) + { + using namespace ComparedStateTestDetail; + EXPECT_TRUE(unchangedSinceCompare(ComparedState{}, ComparedState{})); + EXPECT_FALSE(unchangedSinceCompare(ComparedState{}, file(3, 100))); + EXPECT_FALSE(unchangedSinceCompare(ComparedState{}, ComparedState{.exists = true, .type = FileType::Directory})); + } + + TEST(ComparedStateTests, SomethingThatIsGoneLosesNothing) + { + using namespace ComparedStateTestDetail; + EXPECT_TRUE(unchangedSinceCompare(file(3, 100), ComparedState{})); + } + + TEST(ComparedStateTests, AnotherTypeChanged) + { + using namespace ComparedStateTestDetail; + EXPECT_FALSE(unchangedSinceCompare(file(0, 5), link("target"))); + EXPECT_FALSE(unchangedSinceCompare(ComparedState{.exists = true, .type = FileType::Directory}, file(0, 0))); + } + + TEST(ComparedStateTests, ASymlinkChangesByItsLiteralOnly) + { + using namespace ComparedStateTestDetail; + auto later = link("target"); + later.mtime = 99; + EXPECT_TRUE(unchangedSinceCompare(link("target"), later)); + EXPECT_FALSE(unchangedSinceCompare(link("target"), link("other"))); + } + + TEST(ComparedStateTests, ADirectoryChangesByItsTypeOnly) + { + const ComparedState compared{.exists = true, .type = FileType::Directory, .size = 4096, .mtime = 1}; + const ComparedState current{.exists = true, .type = FileType::Directory, .size = 8192, .mtime = 2}; + EXPECT_TRUE(unchangedSinceCompare(compared, current)); + } + + TEST(ComparedStateTests, AnEntryDescribesItsState) + { + const DirectoryEntry entry{.type = FileType::Symlink, .size = 7, .mtime = 9, .linkTarget = "a/b"}; + EXPECT_EQ( + comparedStateOf(entry), + (ComparedState{.exists = true, .type = FileType::Symlink, .size = 7, .mtime = 9, .linkTarget = "a/b"}) + ); + const DirectoryEntry regular{.type = FileType::Regular, .size = 7, .mtime = 9, .linkTarget = "ignored"}; + EXPECT_FALSE(comparedStateOf(regular).linkTarget); + } +} diff --git a/shared_data/test/shared_data/test_diff.hpp b/shared_data/test/shared_data/test_diff.hpp index d24557cc..d19bfaa7 100644 --- a/shared_data/test/shared_data/test_diff.hpp +++ b/shared_data/test/shared_data/test_diff.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -796,84 +797,63 @@ namespace SharedData::Sync::Test EXPECT_EQ(row.descendantByteTotal, 0u); } - TEST(SyncDiffTests, WithoutALastSyncTheNewestSideWins) + TEST(SyncDiffTests, NamesDifferingOnlyInCaseAcrossSidesAreBlockedForACaseFoldingServer) { + // On a case insensitive server "Foo" replaces "foo", the delete of "foo" would then remove the upload. CollectingSink sink; diffScanTrees( - makeDir("", {makeFile("a.txt", 1, 300)}), - makeDir("", {makeFile("a.txt", 2, 200)}), - {.direction = Direction::Both}, + makeDir("", {makeFile("Foo", 3, 200), makeFile("other", 1, 100)}), + makeDir("", {makeFile("foo", 3, 100)}), + {.direction = Direction::Upload, .actionDelete = true, .remoteCaseInsensitive = true}, sink ); - EXPECT_EQ(sink.uploads.size(), 1u); - EXPECT_TRUE(sink.blocked.empty()); + ASSERT_EQ(sink.uploads.size(), 1u); + EXPECT_EQ(sink.uploads.front().node.relKey, "other"); + EXPECT_TRUE(sink.deletes.empty()); + ASSERT_EQ(sink.blocked.size(), 2u); + for (auto const& blocked : sink.blocked) + EXPECT_EQ(blocked.reason, BlockReason::CaseConflict) << blocked.relKey; } - TEST(SyncDiffTests, BothSidesChangedSinceTheLastSyncIsBlocked) + TEST(SyncCaseFoldTests, LettersOfEveryCoveredScriptFoldToOneKey) { - CollectingSink sink; - diffScanTrees( - makeDir("", {makeFile("both.txt", 1, 300), makeFile("local.txt", 1, 300)}), - makeDir("", {makeFile("both.txt", 2, 250), makeFile("local.txt", 2, 50)}), - {.direction = Direction::Both, .lastSyncTime = 100}, - sink - ); - ASSERT_EQ(sink.blocked.size(), 1u); - EXPECT_EQ(sink.blocked.front().relKey, "both.txt"); - EXPECT_EQ(sink.blocked.front().reason, BlockReason::ChangedOnBothSides); - ASSERT_EQ(sink.uploads.size(), 1u) << "only the local side changed, it wins"; - EXPECT_EQ(sink.uploads.front().node.relKey, "local.txt"); + EXPECT_EQ(caseFoldedName("README.md"), "readme.md"); + EXPECT_EQ(caseFoldedName("\xC3\x84PFEL"), caseFoldedName("\xC3\xA4pfel")) << "Ä and ä"; + EXPECT_EQ(caseFoldedName("ŁÓDŹ"), caseFoldedName("łódź")) << "Latin Extended-A"; + EXPECT_EQ( + caseFoldedName("\xCE\xA3\xCE\x9F\xCE\xA6\xCE\x9F\xCE\xA3"), + caseFoldedName("\xCF\x83\xCE\xBF\xCF\x86\xCE\xBF\xCF\x82") + ) << "ΣΟΦΟΣ and σοφος with a final sigma"; + EXPECT_EQ(caseFoldedName("\xD0\x81\xD0\x96"), caseFoldedName("\xD1\x91\xD0\xB6")) << "ЁЖ and ёж"; + EXPECT_NE(caseFoldedName("a"), caseFoldedName("b")); + EXPECT_EQ(caseFoldedName("caf\xFF"), "caf\xFF") << "text that is not UTF-8 is its own key"; } - TEST(SyncDiffTests, UploadNeverReplacesWhatChangedOnTheServerSinceTheLastSync) + TEST(SyncCaseFoldTests, ComposedAndDecomposedSpellingsFoldToOneKey) { - CollectingSink sink; - diffScanTrees( - makeDir("", {makeFile("a.txt", 1, 50), makeFile("b.txt", 1, 300)}), - makeDir("", {makeFile("a.txt", 2, 200), makeFile("b.txt", 2, 50)}), - {.direction = Direction::Upload, .lastSyncTime = 100}, - sink - ); - ASSERT_EQ(sink.blocked.size(), 1u); - EXPECT_EQ(sink.blocked.front().relKey, "a.txt"); - EXPECT_EQ(sink.blocked.front().reason, BlockReason::ChangedOnDestination); - ASSERT_EQ(sink.uploads.size(), 1u); - EXPECT_EQ(sink.uploads.front().node.relKey, "b.txt"); + EXPECT_EQ(caseFoldedName("caf\xC3\xA9"), caseFoldedName("cafe\xCC\x81")) << "é as NFC and NFD"; + EXPECT_EQ(caseFoldedName("CAF\xC3\x89"), caseFoldedName("cafe\xCC\x81")) << "É folds to e and the accent"; + EXPECT_EQ(caseFoldedName("\xCE\x90"), caseFoldedName("\xCE\xB9\xCC\x88\xCC\x81")) << "ΐ decomposes twice"; + EXPECT_FALSE(foldingChanges("plain-name_1.txt")); + EXPECT_TRUE(foldingChanges("Name")); + EXPECT_TRUE(foldingChanges("caf\xC3\xA9")); } - TEST(SyncDiffTests, DeleteNeverRemovesWhatAppearedSinceTheLastSync) + TEST(SyncDiffTests, NonAsciiNamesDifferingOnlyInCaseOrNormalizationAreBlocked) { CollectingSink sink; diffScanTrees( - makeDir("", {}), makeDir( - "", - {makeFile("old.txt", 1, 50), - makeFile("new.txt", 1, 200), - makeDir("dir", {makeFile("old", 1, 50), makeFile("fresh", 1, 200)})} + "", {makeFile("\xC3\x84pfel.txt", 1, 100), makeFile("cafe\xCC\x81", 1, 100), makeFile("other", 1, 100)} ), - {.direction = Direction::Upload, .actionDelete = true, .lastSyncTime = 100}, - sink - ); - ASSERT_EQ(sink.deletes.size(), 1u); - EXPECT_EQ(sink.deletes.front().node.relKey, "old.txt"); - EXPECT_EQ(sink.blocked.size(), 2u) << "new.txt and the directory holding a new file"; - } - - TEST(SyncDiffTests, NamesDifferingOnlyInCaseAcrossSidesAreBlocked) - { - // On a case insensitive server "Foo" replaces "foo", the delete of "foo" would then remove the upload. - CollectingSink sink; - diffScanTrees( - makeDir("", {makeFile("Foo", 3, 200), makeFile("other", 1, 100)}), - makeDir("", {makeFile("foo", 3, 100)}), - {.direction = Direction::Upload, .actionDelete = true}, + makeDir("", {makeFile("\xC3\xA4pfel.txt", 2, 100), makeFile("caf\xC3\xA9", 2, 100)}), + {.direction = Direction::Upload, .actionDelete = true, .remoteCaseInsensitive = true}, sink ); ASSERT_EQ(sink.uploads.size(), 1u); EXPECT_EQ(sink.uploads.front().node.relKey, "other"); EXPECT_TRUE(sink.deletes.empty()); - ASSERT_EQ(sink.blocked.size(), 2u); + EXPECT_EQ(sink.blocked.size(), 4u); for (auto const& blocked : sink.blocked) EXPECT_EQ(blocked.reason, BlockReason::CaseConflict) << blocked.relKey; } @@ -896,6 +876,63 @@ namespace SharedData::Sync::Test } } + TEST(SyncDiffTests, LocalCaseVariantsAreBlockedOnlyForACaseFoldingServer) + { + const auto local = makeDir("", {makeFile("Readme", 1, 100), makeFile("readme", 1, 100)}); + const auto remote = makeDir("", {}); + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Upload, .localCaseInsensitive = true}, sink); + EXPECT_EQ(sink.uploads.size(), 2u) << "only the destination folding case counts"; + EXPECT_TRUE(sink.blocked.empty()); + } + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Upload, .remoteCaseInsensitive = true}, sink); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_EQ(sink.blocked.size(), 2u); + } + } + + TEST(SyncDiffTests, CaseSensitiveSidesNeverBlockCaseOrNormalizationDifferences) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("Foo", 3, 200), makeFile("cafe\xCC\x81", 1, 100), makeFile("Readme", 1, 100)}), + makeDir("", {makeFile("foo", 3, 100), makeFile("caf\xC3\xA9", 2, 100), makeFile("README", 1, 100)}), + {.direction = Direction::Both}, + sink + ); + EXPECT_TRUE(sink.blocked.empty()); + EXPECT_EQ(sink.uploads.size(), 3u); + EXPECT_EQ(sink.downloads.size(), 3u); + } + + TEST(SyncDiffTests, CaseVariantsAcrossSidesCountOnlyInTheDirectionsWhoseDestinationFolds) + { + const auto local = makeDir("", {makeFile("Foo", 3, 200)}); + const auto remote = makeDir("", {makeFile("foo", 3, 100)}); + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Download, .remoteCaseInsensitive = true}, sink); + EXPECT_TRUE(sink.blocked.empty()) << "a download onto a case sensitive filesystem keeps both"; + EXPECT_EQ(sink.downloads.size(), 1u); + } + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Download, .localCaseInsensitive = true}, sink); + EXPECT_EQ(sink.blocked.size(), 2u); + EXPECT_TRUE(sink.downloads.empty()); + } + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Both, .localCaseInsensitive = true}, sink); + EXPECT_EQ(sink.blocked.size(), 2u); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + } + } + TEST(SyncDiffTests, UnreadableLocalDirectoryNeverDeletesItsRemoteContent) { CollectingSink sink; diff --git a/shared_data/test/shared_data/test_diff_tree_store.hpp b/shared_data/test/shared_data/test_diff_tree_store.hpp index df25dc31..9d5264df 100644 --- a/shared_data/test/shared_data/test_diff_tree_store.hpp +++ b/shared_data/test/shared_data/test_diff_tree_store.hpp @@ -342,6 +342,38 @@ namespace SharedData::Sync::Test EXPECT_EQ(relKeys(stores.downloads.buildPlan({"new"}, stores.trees())), (std::vector{"new/b"})); } + TEST(DiffTreeStoreTests, UploadPlanSkipsCaseVariantsOnlyForACaseFoldingServer) + { + const auto local = + makeDir("", {makeDir("new", {makeFile("A.txt", 1, 100), makeFile("a.txt", 1, 100), makeFile("b", 1, 100)})}); + const DiffStores folding{local, makeDir("", {}), {.direction = Direction::Upload, .remoteCaseInsensitive = true}}; + EXPECT_EQ(relKeys(folding.uploads.buildPlan({"new"}, folding.trees())), (std::vector{"new/b"})); + + const DiffStores sensitive{local, makeDir("", {}), {.direction = Direction::Upload, .localCaseInsensitive = true}}; + EXPECT_EQ( + relKeys(sensitive.uploads.buildPlan({"new"}, sensitive.trees())), + (std::vector{"new/A.txt", "new/a.txt", "new/b"}) + ); + } + + TEST(DiffTreeStoreTests, ComparedStateAtNamesWhatTheScanSaw) + { + const auto tree = makeDir( + "", {makeDir("dir", {makeFile("a.txt", 3, 100), makeSymlink("link", "../x", 7)}), makeFile("b", 1, 2)} + ); + EXPECT_EQ( + comparedStateAt(tree, "dir/a.txt"), + (ComparedState{.exists = true, .type = FileType::Regular, .size = 3, .mtime = 100}) + ); + EXPECT_EQ( + comparedStateAt(tree, "dir/link"), + (ComparedState{.exists = true, .type = FileType::Symlink, .mtime = 7, .linkTarget = "../x"}) + ); + EXPECT_EQ(comparedStateAt(tree, "dir").type, FileType::Directory); + EXPECT_FALSE(comparedStateAt(tree, "dir/missing").exists); + EXPECT_FALSE(comparedStateAt(tree, "b/below").exists); + } + TEST(DiffTreeStoreTests, PlanForAPartlySelectedExpandedDirectory) { DiffStores stores{ diff --git a/shared_data/test/shared_data/test_tree_directory_walker.hpp b/shared_data/test/shared_data/test_tree_directory_walker.hpp index e2de0a34..f14cef19 100644 --- a/shared_data/test/shared_data/test_tree_directory_walker.hpp +++ b/shared_data/test/shared_data/test_tree_directory_walker.hpp @@ -53,12 +53,10 @@ namespace SharedData::Sync::Test }; } - template inline std::expected walkFake(FakeFilesystem& filesystem, bool recursive = true) { auto scanner = makeScanner(filesystem); - TreeDirectoryWalker walker{std::move(scanner), recursive - }; + TreeDirectoryWalker walker{std::move(scanner), recursive}; auto result = walker.walkAll(); if (!result) return std::unexpected(result.error()); @@ -117,7 +115,7 @@ namespace SharedData::Sync::Test filesystem.addFile("/root", ".hidden", 1); filesystem.addFile("/root", "file", 1); - const auto tree = walkFake(filesystem); + const auto tree = walkFake(filesystem); ASSERT_TRUE(tree.has_value()); EXPECT_EQ(childNames(*tree), (std::vector{".hidden", "file"})); } @@ -156,6 +154,23 @@ namespace SharedData::Sync::Test EXPECT_EQ(tree->children[0].children[0].children[0].children[0].name, "deep.txt"); } + TEST(TreeDirectoryWalkerTests, NamesThatWereNotUtf8AreInvalidAndNeverEntered) + { + FakeFilesystem filesystem; + filesystem.directories["/root"].push_back( + DirectoryEntry{.name = "Fran\xEF\xBF\xBDois", .nameIsValidUtf8 = false, .type = FileType::Directory} + ); + filesystem.directories["/root"].push_back( + DirectoryEntry{.name = "x\xEF\xBF\xBD", .nameIsValidUtf8 = false, .type = FileType::Regular, .size = 3} + ); + const auto tree = walkFake(filesystem); + ASSERT_TRUE(tree.has_value()) << tree.error(); + ASSERT_EQ(tree->children.size(), 2u); + for (auto const& child : tree->children) + EXPECT_EQ(child.availability, Availability::InvalidName) << child.name; + EXPECT_EQ(filesystem.listed, (std::vector{"/root"})); + } + TEST(TreeDirectoryWalkerTests, ScannerErrorsAbortTheWalk) { FakeFilesystem filesystem; @@ -174,7 +189,7 @@ namespace SharedData::Sync::Test filesystem.addFile("/root/dir", "b", 50); auto scanner = makeScanner(filesystem); - TreeDirectoryWalker walker{std::move(scanner)}; + TreeDirectoryWalker walker{std::move(scanner)}; ASSERT_TRUE(walker.walkAll().has_value()); EXPECT_TRUE(walker.completed()); EXPECT_EQ(walker.totalEntries(), 4u) << "root, a, dir, b"; diff --git a/ssh/include/ssh/async/transfer_context.hpp b/ssh/include/ssh/async/transfer_context.hpp index 5d7580d8..774304df 100644 --- a/ssh/include/ssh/async/transfer_context.hpp +++ b/ssh/include/ssh/async/transfer_context.hpp @@ -68,7 +68,6 @@ namespace SecureShell private: std::atomic outcome_{Outcome::Running}; - std::atomic_bool ended_{false}; mutable std::mutex errorMutex_{}; std::optional error_{}; std::atomic_bool paused_{false}; diff --git a/ssh/include/ssh/key_file.hpp b/ssh/include/ssh/key_file.hpp new file mode 100644 index 00000000..c968441d --- /dev/null +++ b/ssh/include/ssh/key_file.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace SecureShell +{ + namespace Detail + { + /** + * @brief Overwrites @p target with zero bytes through a volatile pointer and clears it, so sensitive data + * does not linger in process memory. + */ + void secureWipe(std::string& target) noexcept; + + /** + * @brief The key type name and base64 blob of an OpenSSH public key line "type base64 [comment]", nothing + * for any other text. + */ + std::optional> splitPublicKeyLine(std::string_view text); + } + + /** + * @brief Imports the private key file @p keyFile, read by its native path so any name works on Windows. Returns + * what ssh_pki_import_privkey_file would: SSH_EOF when the file cannot be read, SSH_ERROR when it holds no + * valid key or the passphrase is wrong. + */ + int importPrivateKeyFile( + std::filesystem::path const& keyFile, + char const* passphrase, + ssh_auth_callback askPass, + void* userData, + ssh_key* key + ); + + /** + * @brief Imports the public key file @p keyFile, read by its native path. An OpenSSH public key line is imported + * from memory, any other format (like PEM) through libssh by the UTF-8 path. SSH_EOF when the file cannot + * be read. + */ + int importPublicKeyFile(std::filesystem::path const& keyFile, ssh_key* key); +} diff --git a/ssh/include/ssh/sftp_session.hpp b/ssh/include/ssh/sftp_session.hpp index 49928968..54ce88c5 100644 --- a/ssh/include/ssh/sftp_session.hpp +++ b/ssh/include/ssh/sftp_session.hpp @@ -14,9 +14,12 @@ #include #include +#include #include #include #include +#include +#include #include #include #include @@ -69,18 +72,22 @@ namespace SecureShell * @brief Lists the contents of a directory. * * @param path + * @param resolveLinks Reads the literal and the target of every symlink, two more round trips each. Without + * it a symlink is listed as the link itself only. * @return std::future, Error>> */ std::future, Error>> - listDirectory(Paths::RemotePath const& path); + listDirectory(Paths::RemotePath const& path, bool resolveLinks = true); /** * @brief In-strand variant of listDirectory. Must be called from within the processing thread. * * @param path + * @param resolveLinks See @ref listDirectory. * @return std::expected, Error> */ - std::expected, Error> listDirectoryInStrand(Paths::RemotePath const& path); + std::expected, Error> + listDirectoryInStrand(Paths::RemotePath const& path, bool resolveLinks = true); /** * @brief Create a directory. @@ -271,16 +278,26 @@ namespace SecureShell renameInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination); /** - * @brief Renames without ever replacing @p destination. Fails with SSH_FX_FILE_ALREADY_EXISTS when it - * exists. Regular files move atomically through hardlink@openssh.com where the server has it. + * @brief Renames without replacing @p destination. Fails with SSH_FX_FILE_ALREADY_EXISTS when it exists. + * + * A regular file moves atomically through hardlink@openssh.com where the server has it and the link + * succeeds. Otherwise a server without posix-rename@openssh.com refuses an existing destination itself, + * and on one with it the destination is checked before the rename, which leaves a window. + * + * @param sourceType The type of @p source when the caller knows it, saves an lstat. */ - std::expected - renameNoReplaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination); + std::expected renameNoReplaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + std::optional sourceType = std::nullopt + ); /** - * @brief Renames over an existing @p destination without a moment where it is missing: atomically with - * posix-rename@openssh.com, otherwise by moving the destination aside to @p asideSuffix and - * back if the rename fails. + * @brief Renames over an existing @p destination. + * + * Atomic with posix-rename@openssh.com. Otherwise the destination is moved aside to @p asideSuffix first + * and moved back when the rename fails; if moving it back fails too, the error names where it is kept. + * A leftover aside is removed when the destination exists, and put back when it does not. */ std::expected renameReplaceInStrand( Paths::RemotePath const& source, @@ -288,6 +305,18 @@ namespace SecureShell std::string_view asideSuffix ); + /** + * @brief Moves a finished transfer from @p source to @p destination with @ref renameReplaceInStrand when + * @p replace is set, otherwise with @ref renameNoReplaceInStrand. + */ + std::expected moveIntoPlaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + bool replace, + std::string_view asideSuffix, + std::optional sourceType = std::nullopt + ); + /** * @brief Whether the server announced the SFTP extension @p name with @p version. */ @@ -326,6 +355,19 @@ namespace SecureShell */ std::expected homeDirectoryInStrand(); + /** + * @brief The directory the server starts the user in, resolved by the server (realpath of "." on OpenSSH), + * without any fallback: an error when the server does not answer with an absolute path. Must be + * called from within the processing thread. + */ + std::expected canonicalHomeDirectoryInStrand(); + + /** + * @brief The server's canonical form of the existing @p path, with every symlink resolved (realpath on + * OpenSSH). Must be called from within the processing thread. + */ + std::expected canonicalPathInStrand(Paths::RemotePath const& path); + /** * @brief Reads the XDG user directories from /.config/user-dirs.dirs. A missing file yields no * directories instead of an error. @@ -392,6 +434,12 @@ namespace SecureShell */ std::expected readLinkDeepInStrand(Paths::RemotePath const& path, int maxDepth = 10); + /** + * @brief The literal of the symlink @p path, valid UTF-8 like a listed link target (U+FFFD for every invalid + * byte). Must be called from within the processing thread. + */ + std::expected readLinkInStrand(Paths::RemotePath const& path); + /** * @brief Creates a symlink at @p linkPath with the content @p target, which may be relative. */ @@ -405,9 +453,38 @@ namespace SecureShell private: /** - * @brief Refuses an unset path before it reaches the server. + * @brief Asserts the call runs in the processing thread and refuses an unset path before it reaches the + * server, where an empty path would mean the start directory. + */ + template + std::expected requireSetInStrand(PathTs const&... paths) const + { + assert(strand_->withinProcessingThread()); + if ((paths.empty() || ...)) + return std::unexpected(unsetPathError()); + return {}; + } + + static SftpError unsetPathError(); + + /** + * @brief The absolute path the server canonicalizes @p path to. + */ + std::expected canonicalizeInStrand(char const* path); + + static SftpError alreadyExists(Paths::RemotePath const& destination); + + /** + * @brief Moves the regular file @p source to @p destination by a hard link and removing @p source, or by a + * rename when the server refuses the link and @p destination does not exist. + */ + std::expected + hardlinkIntoPlaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination); + + /** + * @brief Renames @p destination to @p aside, removing a file left at @p aside first when the rename fails. */ - static std::expected requireSet(Paths::RemotePath const& path); + std::expected moveAsideInStrand(Paths::RemotePath const& destination, Paths::RemotePath const& aside); /** * @brief Reads a file into a string, stopping after maximumSize bytes. diff --git a/ssh/source/ssh/CMakeLists.txt b/ssh/source/ssh/CMakeLists.txt index 77bd4247..e0333a24 100644 --- a/ssh/source/ssh/CMakeLists.txt +++ b/ssh/source/ssh/CMakeLists.txt @@ -9,6 +9,7 @@ add_library( sftp_session.cpp file_stream.cpp file_information.cpp + key_file.cpp ) target_include_directories( diff --git a/ssh/source/ssh/async/transfer_context.cpp b/ssh/source/ssh/async/transfer_context.cpp index dc30247b..f3d759a5 100644 --- a/ssh/source/ssh/async/transfer_context.cpp +++ b/ssh/source/ssh/async/transfer_context.cpp @@ -6,7 +6,6 @@ namespace SecureShell { auto expected = Outcome::Running; outcome_.compare_exchange_strong(expected, outcome); - ended_.store(true); } void AsyncTransferContext::cancel() @@ -32,7 +31,7 @@ namespace SecureShell bool AsyncTransferContext::hasEnded() const { - return ended_.load(); + return outcome_.load() != Outcome::Running; } AsyncTransferContext::Outcome AsyncTransferContext::outcome() const diff --git a/ssh/source/ssh/file_information.cpp b/ssh/source/ssh/file_information.cpp index ea1016d8..3ebcc35f 100644 --- a/ssh/source/ssh/file_information.cpp +++ b/ssh/source/ssh/file_information.cpp @@ -1,19 +1,29 @@ #include +#include + +#include +#include + namespace SecureShell { FileInformation fromSftpAttributes(sftp_attributes attributes) { + const auto serverText = [](char const* text) + { + return text ? Paths::sanitizedUtf8(text) : std::string{}; + }; return FileInformation{ SharedData::DirectoryEntry{ - .name = attributes->name ? std::string{attributes->name} : std::string{}, + .name = serverText(attributes->name), + .nameIsValidUtf8 = attributes->name == nullptr || Paths::isValidUtf8(attributes->name), .flags = attributes->flags, .type = static_cast(attributes->type), .size = attributes->size, .uid = attributes->uid, .gid = attributes->gid, - .owner = attributes->owner ? std::string{attributes->owner} : std::string{}, - .group = attributes->group ? std::string{attributes->group} : std::string{}, + .owner = serverText(attributes->owner), + .group = serverText(attributes->group), .permissions = static_cast(attributes->permissions), .atime = attributes->atime, .atimeNsec = attributes->atime_nseconds, @@ -22,7 +32,7 @@ namespace SecureShell .mtime = attributes->mtime, .mtimeNsec = attributes->mtime_nseconds, .acl = attributes->acl - ? std::string{ssh_string_get_char(attributes->acl), ssh_string_len(attributes->acl)} + ? Paths::sanitizedUtf8({ssh_string_get_char(attributes->acl), ssh_string_len(attributes->acl)}) : std::string{}, }, }; diff --git a/ssh/source/ssh/key_file.cpp b/ssh/source/ssh/key_file.cpp new file mode 100644 index 00000000..582cdb51 --- /dev/null +++ b/ssh/source/ssh/key_file.cpp @@ -0,0 +1,93 @@ +#include +#include + +#include + +#include +#include +#include + +namespace SecureShell +{ + namespace + { + std::optional readKeyFile(std::filesystem::path const& keyFile) + { + std::ifstream stream{keyFile, std::ios::binary}; + if (!stream) + return std::nullopt; + std::string content{std::istreambuf_iterator{stream}, std::istreambuf_iterator{}}; + if (stream.bad()) + { + Detail::secureWipe(content); + return std::nullopt; + } + return content; + } + + bool isSpace(char character) + { + return std::isspace(static_cast(character)) != 0; + } + + std::string_view nextToken(std::string_view& text) + { + std::size_t start = 0; + while (start < text.size() && isSpace(text[start])) + ++start; + std::size_t end = start; + while (end < text.size() && !isSpace(text[end])) + ++end; + const auto token = text.substr(start, end - start); + text.remove_prefix(end); + return token; + } + } + + namespace Detail + { + void secureWipe(std::string& target) noexcept + { + volatile char* volatileData = target.data(); + for (std::size_t index = 0; index < target.size(); ++index) + volatileData[index] = '\0'; + target.clear(); + } + + std::optional> splitPublicKeyLine(std::string_view text) + { + const auto type = nextToken(text); + const auto blob = nextToken(text); + if (type.empty() || blob.empty() || type.starts_with("-----") || + ssh_key_type_from_name(std::string{type}.c_str()) == SSH_KEYTYPE_UNKNOWN) + return std::nullopt; + return std::pair{std::string{type}, std::string{blob}}; + } + } + + int importPrivateKeyFile( + std::filesystem::path const& keyFile, + char const* passphrase, + ssh_auth_callback askPass, + void* userData, + ssh_key* key + ) + { + auto content = readKeyFile(keyFile); + if (!content) + return SSH_EOF; + const auto result = ssh_pki_import_privkey_base64(content->c_str(), passphrase, askPass, userData, key); + Detail::secureWipe(*content); + return result; + } + + int importPublicKeyFile(std::filesystem::path const& keyFile, ssh_key* key) + { + const auto content = readKeyFile(keyFile); + if (!content) + return SSH_EOF; + if (const auto line = Detail::splitPublicKeyLine(*content); line) + return ssh_pki_import_pubkey_base64(line->second.c_str(), ssh_key_type_from_name(line->first.c_str()), key); + return ssh_pki_import_pubkey_file(Paths::nativeToUtf8(keyFile).c_str(), key); + } +} diff --git a/ssh/source/ssh/session.cpp b/ssh/source/ssh/session.cpp index 779f0af8..152c7183 100644 --- a/ssh/source/ssh/session.cpp +++ b/ssh/source/ssh/session.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -11,23 +12,9 @@ namespace SecureShell { namespace { - /** - * @brief Overwrite the contents of a string buffer in place so sensitive - * data does not linger in process memory after use. - * @param target String whose storage will be overwritten with zero bytes - * through a volatile pointer to prevent optimizer elision. - */ - void secureWipe(std::string& target) - { - volatile char* volatilePtr = target.data(); - for (std::size_t index = 0; index < target.size(); ++index) - volatilePtr[index] = '\0'; - target.clear(); - } - /** * @brief Describes why reading a key file failed, libssh only returns SSH_EOF or SSH_ERROR for it. - * @param importResult Return value of ssh_pki_import_pubkey_file or ssh_pki_import_privkey_file. + * @param importResult Return value of importPublicKeyFile or importPrivateKeyFile. * @param keyKind "public" or "private". * @param keyFile The key file that was read. */ @@ -352,7 +339,7 @@ namespace SecureShell { lastAttemptedAppliedOption = "SSH Directory"; return static_cast(*session).setOption( - SSH_OPTIONS_SSH_DIR, sshOptions.sshDirectory.value().generic_string().c_str() + SSH_OPTIONS_SSH_DIR, Paths::nativeToUtf8(sshOptions.sshDirectory.value()).c_str() ); } return 0; @@ -378,7 +365,7 @@ namespace SecureShell { lastAttemptedAppliedOption = "Known Hosts File"; return static_cast(*session).setOption( - SSH_OPTIONS_KNOWNHOSTS, sshOptions.knownHostsFile.value().generic_string().c_str() + SSH_OPTIONS_KNOWNHOSTS, Paths::nativeToUtf8(sshOptions.knownHostsFile.value()).c_str() ); } return 0; @@ -634,9 +621,7 @@ namespace SecureShell { if (maybeSshKeyPublic) { - return ssh_pki_import_pubkey_file( - Paths::nativeToUtf8(*maybeSshKeyPublic).c_str(), &publicKey.underlyingReferenced() - ); + return importPublicKeyFile(*maybeSshKeyPublic, &publicKey.underlyingReferenced()); } return SSH_OK; }, @@ -654,12 +639,8 @@ namespace SecureShell }, [&sshKeyPrivate, &privateKey, askPass, askPassUserDataKeyPhrase]() { - return ssh_pki_import_privkey_file( - Paths::nativeToUtf8(sshKeyPrivate).c_str(), - nullptr, - askPass, - askPassUserDataKeyPhrase, - &privateKey.underlyingReferenced() + return importPrivateKeyFile( + sshKeyPrivate, nullptr, askPass, askPassUserDataKeyPhrase, &privateKey.underlyingReferenced() ); }, [&privateKey, &session]() @@ -699,7 +680,7 @@ namespace SecureShell if (pwFromCache.has_value()) { buf = pwFromCache.value(); - secureWipe(*pwFromCache); + Detail::secureWipe(*pwFromCache); pwFromCache.reset(); const auto result = static_cast(*session).userauthPassword(buf.data()); if (result == SSH_AUTH_SUCCESS) @@ -721,7 +702,7 @@ namespace SecureShell return (int)SSH_AUTH_DENIED; } ); - secureWipe(buf); + Detail::secureWipe(buf); } if (keyFailure && result.result == SSH_EOF) diff --git a/ssh/source/ssh/sftp_session.cpp b/ssh/source/ssh/sftp_session.cpp index c5968330..0ae8d187 100644 --- a/ssh/source/ssh/sftp_session.cpp +++ b/ssh/source/ssh/sftp_session.cpp @@ -1,8 +1,6 @@ #include -#include #include #include -#include #include #include @@ -72,15 +70,13 @@ namespace SecureShell } std::expected, SftpSession::Error> - SftpSession::listDirectoryInStrand(Paths::RemotePath const& path) + SftpSession::listDirectoryInStrand(Paths::RemotePath const& path, bool resolveLinks) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); int closeResult = 0; std::vector entries{}; - std::vector names{}; { std::unique_ptr> dir{ @@ -106,7 +102,6 @@ namespace SecureShell for (; entry != nullptr; entry.reset(sftp_readdir(session_, dir.get()))) { entries.push_back(fromSftpAttributes(entry.get())); - names.emplace_back(entry->name != nullptr ? entry->name : ""); } } @@ -120,13 +115,12 @@ namespace SecureShell return std::unexpected(lastError()); } - for (std::size_t index = 0; index < entries.size(); ++index) + for (auto& entry : entries) { - auto& entry = entries[index]; - if (entry.type == SharedData::FileType::Symlink) + if (resolveLinks && entry.type == SharedData::FileType::Symlink) { // A name that is no valid segment, like "x/..", must not become a path to look up. - const auto name = Paths::PathSegment::make(names[index]); + const auto name = entry.nameSegment(); if (!name) continue; const auto fullPath = path / *name; @@ -137,7 +131,7 @@ namespace SecureShell if (linkLiteral) { const auto* raw = static_cast(linkLiteral.get()); - entry.linkTarget = std::string{raw ? raw : ""}; + entry.linkTarget = Paths::sanitizedUtf8(raw ? raw : ""); } std::unique_ptr targetAttrs{ @@ -153,12 +147,12 @@ namespace SecureShell } std::future, SftpSession::Error>> - SftpSession::listDirectory(Paths::RemotePath const& path) + SftpSession::listDirectory(Paths::RemotePath const& path, bool resolveLinks) { return performPromise( - [this, path]() + [this, path, resolveLinks]() { - return listDirectoryInStrand(path); + return listDirectoryInStrand(path, resolveLinks); } ); } @@ -166,8 +160,7 @@ namespace SecureShell std::expected SftpSession::createDirectoryInStrand(Paths::RemotePath const& path, std::filesystem::perms permissions) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_mkdir( session_, path.c_str(), static_cast(permissions & std::filesystem::perms::mask) @@ -193,8 +186,7 @@ namespace SecureShell std::filesystem::perms permissions ) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); std::unique_ptr attributes{ sftp_lstat(session_, path.c_str()), sftp_attributes_free @@ -237,8 +229,7 @@ namespace SecureShell std::expected SftpSession::createFileInStrand(Paths::RemotePath const& path, std::filesystem::perms permissions) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); std::unique_ptr> file{ // O_EXCL: an existing file is an error, not reported as created. @@ -276,8 +267,7 @@ namespace SecureShell std::expected SftpSession::removeFileInStrand(Paths::RemotePath const& path) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_unlink(session_, path.c_str()); if (result != SSH_OK) @@ -297,8 +287,7 @@ namespace SecureShell std::expected SftpSession::removeDirectoryInStrand(Paths::RemotePath const& path) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_rmdir(session_, path.c_str()); if (result != SSH_OK) @@ -326,7 +315,7 @@ namespace SecureShell std::vector nonEmpties; for (const auto& path : directories) { - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); std::unique_ptr attributes{ sftp_lstat(session_, path.c_str()), sftp_attributes_free @@ -397,7 +386,7 @@ namespace SecureShell { for (const auto& path : paths) { - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); std::unique_ptr attributes{ sftp_lstat(session_, path.c_str()), sftp_attributes_free @@ -426,8 +415,7 @@ namespace SecureShell std::expected SftpSession::statInStrand(Paths::RemotePath const& path) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); const auto callStart = std::chrono::steady_clock::now(); std::unique_ptr attributes{ @@ -452,8 +440,7 @@ namespace SecureShell std::expected SftpSession::lstatInStrand(Paths::RemotePath const& path) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); const auto callStart = std::chrono::steady_clock::now(); std::unique_ptr attributes{ @@ -480,8 +467,7 @@ namespace SecureShell std::expected SftpSession::statInStrand(Paths::RemotePath const& path, sftp_attributes attributes) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_setstat(session_, path.c_str(), attributes); if (result != SSH_OK) @@ -503,10 +489,7 @@ namespace SecureShell std::expected SftpSession::renameInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(source); !valid) - return std::unexpected(valid.error()); - if (auto valid = requireSet(destination); !valid) + if (auto valid = requireSetInStrand(source, destination); !valid) return std::unexpected(valid.error()); auto result = sftp_rename(session_, source.c_str(), destination.c_str()); if (result != SSH_OK) @@ -522,43 +505,73 @@ namespace SecureShell return sftp_extension_supported(session_, name, version) == 1; } - std::expected - SftpSession::renameNoReplaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination) + std::expected SftpSession::renameNoReplaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + std::optional sourceType + ) { assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(source); !valid) - return std::unexpected(valid.error()); - if (auto valid = requireSet(destination); !valid) - return std::unexpected(valid.error()); - const auto alreadyExists = [&destination]() + const auto type = sourceType.has_value() + ? std::expected{*sourceType} + : lstatInStrand(source).transform( + [](FileInformation const& information) + { + return information.type; + } + ); + if (!type.has_value()) + return std::unexpected(type.error()); + + if (*type == SharedData::FileType::Regular && supportsExtensionInStrand("hardlink@openssh.com")) + return hardlinkIntoPlaceInStrand(source, destination); + + // libssh sends posix-rename@openssh.com when the server offers it, which replaces, while a plain SFTP v3 + // rename refuses an existing destination. + if (!supportsExtensionInStrand("posix-rename@openssh.com")) { - return SftpError{ - .message = fmt::format("'{}' already exists", destination.value()), - .sftpError = SSH_FX_FILE_ALREADY_EXISTS, - }; - }; + const auto renamed = renameInStrand(source, destination); + if (!renamed.has_value() && lstatInStrand(destination).has_value()) + return std::unexpected(alreadyExists(destination)); + return renamed; + } - const auto sourceInfo = lstatInStrand(source); - if (!sourceInfo.has_value()) - return std::unexpected(sourceInfo.error()); + // Leaves a window between the check and the rename. + if (lstatInStrand(destination).has_value()) + return std::unexpected(alreadyExists(destination)); + return renameInStrand(source, destination); + } - // A hard link fails when the destination exists, so the move cannot replace anything. - if (sourceInfo->isRegularFile() && supportsExtensionInStrand("hardlink@openssh.com")) + std::expected + SftpSession::hardlinkIntoPlaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination) + { + if (auto valid = requireSetInStrand(source, destination); !valid) + return valid; + + if (sftp_hardlink(session_, source.c_str(), destination.c_str()) == SSH_OK) { - if (sftp_hardlink(session_, source.c_str(), destination.c_str()) != SSH_OK) + if (auto removed = removeFileInStrand(source); !removed.has_value()) { - if (lstatInStrand(destination).has_value()) - return std::unexpected(alreadyExists()); - return std::unexpected(lastError()); + Log::warn( + "SftpSession: Moved '{}' to '{}' but could not remove '{}': {}", + source.value(), + destination.value(), + source.value(), + removed.error().toString() + ); } - return removeFileInStrand(source); + return {}; } - // libssh sends posix-rename@openssh.com when the server offers it, which replaces. Plain SFTP v3 rename - // refuses an existing destination. So only a server with posix-rename but without hardlink leaves the - // window between this check and the rename; OpenSSH offers both. - if (lstatInStrand(destination).has_value()) - return std::unexpected(alreadyExists()); + const auto linkError = lastError(); + const auto existing = lstatInStrand(destination); + if (existing.has_value()) + return std::unexpected(alreadyExists(destination)); + if (existing.error().sftpError != SSH_FX_NO_SUCH_FILE) + return std::unexpected(linkError); + + // Filesystems without hard links, and Linux protected_hardlinks, refuse the link. + Log::debug("SftpSession: Cannot link '{}': {}, renaming instead.", source.value(), linkError.toString()); return renameInStrand(source, destination); } @@ -569,42 +582,57 @@ namespace SecureShell ) { assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(source); !valid) - return std::unexpected(valid.error()); - if (auto valid = requireSet(destination); !valid) - return std::unexpected(valid.error()); if (supportsExtensionInStrand("posix-rename@openssh.com")) return renameInStrand(source, destination); - // Plain SFTP rename refuses an existing destination: move it aside instead of deleting it first. - const auto destinationInfo = lstatInStrand(destination); - if (!destinationInfo.has_value()) - return renameInStrand(source, destination); - if (destinationInfo->isDirectory()) + // Plain SFTP rename refuses an existing destination, so it is moved aside instead of deleted first. + const auto aside = destination.withFilenameSuffix(asideSuffix); + if (!aside) { return std::unexpected( SftpError{ - .message = fmt::format("'{}' is a directory", destination.value()), - .sftpError = SSH_FX_FAILURE, + .message = fmt::format("Invalid suffix '{}' for '{}'", asideSuffix, destination.value()), + .wrapperError = WrapperErrors::InvalidPath, } ); } - const auto aside = destination.withFilenameSuffix(asideSuffix); - if (!aside) + const auto destinationInfo = lstatInStrand(destination); + if (!destinationInfo.has_value()) + { + if (destinationInfo.error().sftpError != SSH_FX_NO_SUCH_FILE || !lstatInStrand(*aside).has_value()) + return renameInStrand(source, destination); + // A destination that only exists aside, from a rollback that failed, is put back first. + if (auto restored = renameInStrand(*aside, destination); !restored.has_value()) + return restored; + } + else if (destinationInfo->isDirectory()) { return std::unexpected( SftpError{ - .message = fmt::format("Invalid suffix '{}' for '{}'", asideSuffix, destination.value()), - .wrapperError = WrapperErrors::InvalidPath, + .message = fmt::format("'{}' is a directory", destination.value()), + .sftpError = SSH_FX_FAILURE, } ); } - if (auto movedAside = renameInStrand(destination, *aside); !movedAside.has_value()) + + if (auto movedAside = moveAsideInStrand(destination, *aside); !movedAside.has_value()) return movedAside; if (auto renamed = renameInStrand(source, destination); !renamed.has_value()) { - std::ignore = renameInStrand(*aside, destination); + if (auto restored = renameInStrand(*aside, destination); !restored.has_value()) + { + Log::error( + "SftpSession: Could not restore '{}' from '{}': {}", + destination.value(), + aside->value(), + restored.error().toString() + ); + auto error = renamed.error(); + error.message = + fmt::format("{}. The previous '{}' is kept as '{}'", error.message, destination.value(), aside->value()); + return std::unexpected(std::move(error)); + } return renamed; } if (auto removed = removeFileInStrand(*aside); !removed.has_value()) @@ -612,6 +640,40 @@ namespace SecureShell return {}; } + std::expected + SftpSession::moveAsideInStrand(Paths::RemotePath const& destination, Paths::RemotePath const& aside) + { + const auto movedAside = renameInStrand(destination, aside); + if (movedAside.has_value() || !lstatInStrand(aside).has_value()) + return movedAside; + + // A leftover of an earlier replace holds the name. + if (auto removed = removeFileInStrand(aside); !removed.has_value()) + return removed; + return renameInStrand(destination, aside); + } + + std::expected SftpSession::moveIntoPlaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + bool replace, + std::string_view asideSuffix, + std::optional sourceType + ) + { + if (replace) + return renameReplaceInStrand(source, destination, asideSuffix); + return renameNoReplaceInStrand(source, destination, sourceType); + } + + SftpError SftpSession::alreadyExists(Paths::RemotePath const& destination) + { + return SftpError{ + .message = fmt::format("'{}' already exists", destination.value()), + .sftpError = SSH_FX_FILE_ALREADY_EXISTS, + }; + } + std::future> SftpSession::rename(Paths::RemotePath const& source, Paths::RemotePath const& destination) { @@ -626,8 +688,7 @@ namespace SecureShell std::expected SftpSession::chownInStrand(Paths::RemotePath const& path, uid_t owner, gid_t group) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_chown(session_, path.c_str(), owner, group); if (result != SSH_OK) @@ -649,8 +710,7 @@ namespace SecureShell std::expected SftpSession::chmodInStrand(Paths::RemotePath const& path, std::filesystem::perms perms) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto result = sftp_chmod(session_, path.c_str(), static_cast(perms)); if (result != SSH_OK) @@ -669,17 +729,45 @@ namespace SecureShell ); } - std::expected SftpSession::homeDirectoryInStrand() + std::expected SftpSession::canonicalizeInStrand(char const* path) { assert(strand_->withinProcessingThread()); const std::unique_ptr canonical{ - sftp_canonicalize_path(session_, "."), ssh_string_free_char + sftp_canonicalize_path(session_, path), ssh_string_free_char }; - if (canonical != nullptr && canonical.get()[0] == '/') - { - if (auto home = Paths::RemotePath::make(canonical.get()); home) - return *std::move(home); - } + if (canonical == nullptr) + return std::unexpected(lastError()); + return Paths::RemotePath::make(canonical.get()) + .transform_error( + [](Paths::PathError const& error) + { + return Error{ + .message = fmt::format("The server canonicalized to {}", error.toString()), + .sftpError = SSH_FX_FAILURE, + .wrapperError = WrapperErrors::InvalidPath, + }; + } + ); + } + + std::expected SftpSession::canonicalHomeDirectoryInStrand() + { + return canonicalizeInStrand("."); + } + + std::expected + SftpSession::canonicalPathInStrand(Paths::RemotePath const& path) + { + if (auto valid = requireSetInStrand(path); !valid) + return std::unexpected(valid.error()); + return canonicalizeInStrand(path.c_str()); + } + + std::expected SftpSession::homeDirectoryInStrand() + { + assert(strand_->withinProcessingThread()); + if (auto home = canonicalHomeDirectoryInStrand(); home) + return home; char* rawUser = nullptr; const auto optionResult = ssh_options_get(session_->session, SSH_OPTIONS_USER, &rawUser); @@ -709,8 +797,7 @@ namespace SecureShell std::expected SftpSession::readSmallFileInStrand(Paths::RemotePath const& path, std::size_t maximumSize) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); const std::unique_ptr file{ sftp_open(session_, path.c_str(), O_RDONLY, 0), sftp_close @@ -737,7 +824,7 @@ namespace SecureShell SftpSession::userDirectoriesInStrand(Paths::RemotePath const& home) { constexpr std::size_t maximumFileSize = 64u * 1024u; - if (auto valid = requireSet(home); !valid) + if (auto valid = requireSetInStrand(home); !valid) return std::unexpected(valid.error()); return readSmallFileInStrand(home / *Paths::RelativePath::make(".config/user-dirs.dirs"), maximumFileSize) .transform( @@ -767,17 +854,13 @@ namespace SecureShell ); } - std::expected SftpSession::requireSet(Paths::RemotePath const& path) + SftpError SftpSession::unsetPathError() { - if (!path.empty()) - return {}; - return std::unexpected( - Error{ - .message = "The remote path is not set", - .sftpError = SSH_FX_FAILURE, - .wrapperError = WrapperErrors::InvalidPath, - } - ); + return Error{ + .message = "The remote path is not set", + .sftpError = SSH_FX_FAILURE, + .wrapperError = WrapperErrors::InvalidPath, + }; } SftpError SftpSession::lastError() const @@ -815,8 +898,7 @@ namespace SecureShell std::filesystem::perms permissions ) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); const auto callStart = std::chrono::steady_clock::now(); std::unique_ptr> file{ @@ -863,8 +945,7 @@ namespace SecureShell std::expected SftpSession::readLinkDeepInStrand(Paths::RemotePath const& path, int maxDepth) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(path); !valid) + if (auto valid = requireSetInStrand(path); !valid) return std::unexpected(valid.error()); auto currentPath = path; std::optional linkContent{}; @@ -941,11 +1022,22 @@ namespace SecureShell ); } + std::expected SftpSession::readLinkInStrand(Paths::RemotePath const& path) + { + if (auto valid = requireSetInStrand(path); !valid) + return std::unexpected(valid.error()); + const std::unique_ptr literal{ + sftp_readlink(session_, path.c_str()), ssh_string_free_char + }; + if (!literal) + return std::unexpected(lastError()); + return Paths::sanitizedUtf8(literal.get()); + } + std::expected SftpSession::createSymLinkInStrand(std::string const& target, Paths::RemotePath const& linkPath) { - assert(strand_->withinProcessingThread()); - if (auto valid = requireSet(linkPath); !valid) + if (auto valid = requireSetInStrand(linkPath); !valid) return std::unexpected(valid.error()); const auto result = sftp_symlink(session_, target.c_str(), linkPath.c_str()); if (result != SSH_OK) diff --git a/ssh/test/ssh/main.cpp b/ssh/test/ssh/main.cpp index 8930f777..089072f2 100644 --- a/ssh/test/ssh/main.cpp +++ b/ssh/test/ssh/main.cpp @@ -5,6 +5,7 @@ #include "test_ssh_session.hpp" #include "test_sftp.hpp" #include "test_sftp_in_strand.hpp" +#include "test_key_file.hpp" #include diff --git a/ssh/test/ssh/test_key_file.hpp b/ssh/test/ssh/test_key_file.hpp new file mode 100644 index 00000000..8b45bbd0 --- /dev/null +++ b/ssh/test/ssh/test_key_file.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include "common_fixture.hpp" +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace SecureShell::Test +{ + TEST(KeyFileTests, SplitsAnOpenSshPublicKeyLine) + { + const auto line = Detail::splitPublicKeyLine("ssh-ed25519 AAAAC3Nza comment with spaces\n"); + ASSERT_TRUE(line.has_value()); + EXPECT_EQ(line->first, "ssh-ed25519"); + EXPECT_EQ(line->second, "AAAAC3Nza"); + EXPECT_FALSE(Detail::splitPublicKeyLine("-----BEGIN PUBLIC KEY-----\nMIIC\n").has_value()); + EXPECT_FALSE(Detail::splitPublicKeyLine("unknown-type AAAA").has_value()); + EXPECT_FALSE(Detail::splitPublicKeyLine("ssh-rsa").has_value()); + EXPECT_FALSE(Detail::splitPublicKeyLine("").has_value()); + } + + TEST(KeyFileTests, ImportsKeysFromFilesWithNonAsciiNames) + { + const auto directory = temporaryRoot / Paths::nativeFromUtf8("Schlüssel 東京"); + std::filesystem::create_directories(directory); + const auto privateFile = directory / Paths::nativeFromUtf8("ключ"); + { + std::ofstream stream{privateFile, std::ios::binary}; + stream.write(privateKey, static_cast(std::strlen(privateKey))); + } + + SshKey imported{}; + ASSERT_EQ( + importPrivateKeyFile(privateFile, nullptr, nullptr, nullptr, &imported.underlyingReferenced()), SSH_OK + ); + ASSERT_TRUE(imported); + + char* blob = nullptr; + ASSERT_EQ(ssh_pki_export_pubkey_base64(imported.underlying(), &blob), SSH_OK); + const std::unique_ptr ownedBlob{blob, &ssh_string_free_char}; + const auto publicFile = directory / Paths::nativeFromUtf8("ключ.pub"); + { + std::ofstream stream{publicFile, std::ios::binary}; + stream << fmt::format( + "{} {} test@host\n", ssh_key_type_to_char(ssh_key_type(imported.underlying())), ownedBlob.get() + ); + } + SshKey importedPublic{}; + ASSERT_EQ(importPublicKeyFile(publicFile, &importedPublic.underlyingReferenced()), SSH_OK); + EXPECT_EQ(ssh_key_cmp(importedPublic.underlying(), imported.underlying(), SSH_KEY_CMP_PUBLIC), 0); + } + + TEST(KeyFileTests, AMissingKeyFileIsReportedLikeLibssh) + { + SshKey key{}; + const auto missing = temporaryRoot / Paths::nativeFromUtf8("fehlt-ü"); + EXPECT_EQ(importPrivateKeyFile(missing, nullptr, nullptr, nullptr, &key.underlyingReferenced()), SSH_EOF); + EXPECT_EQ(importPublicKeyFile(missing, &key.underlyingReferenced()), SSH_EOF); + } +} diff --git a/ssh/test/ssh/test_sftp.hpp b/ssh/test/ssh/test_sftp.hpp index 845b1e8f..d1747c5e 100644 --- a/ssh/test/ssh/test_sftp.hpp +++ b/ssh/test/ssh/test_sftp.hpp @@ -908,6 +908,42 @@ namespace SecureShell::Test EXPECT_NE(iter, listResult->end()); } + TEST_F(SftpTests, ListedNamesThatAreNotUtf8AreShownButNeverNamed) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("rawName", {{"directory", "/home/test"}, {"nameHex", "61ff62"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto listFut = sftp->listDirectory(remote("/home/test")); + ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); + const auto listResult = listFut.get(); + ASSERT_TRUE(listResult.has_value()); + const auto iter = std::find_if(listResult->begin(), listResult->end(), [](auto const& entry) { + return entry.name == "a\xEF\xBF\xBD" + "b"; + }); + ASSERT_NE(iter, listResult->end()); + EXPECT_FALSE(iter->nameIsValidUtf8); + const auto segment = iter->nameSegment(); + ASSERT_FALSE(segment.has_value()); + EXPECT_EQ(segment.error().reason, Paths::PathRejection::InvalidUtf8); + + const nlohmann::json listing = *listResult; + const auto dumped = listing.dump(); + const auto parsed = nlohmann::json::parse(dumped).get>(); + const auto parsedIter = std::find_if(parsed.begin(), parsed.end(), [](auto const& entry) { + return !entry.nameIsValidUtf8; + }); + ASSERT_NE(parsedIter, parsed.end()); + EXPECT_EQ(parsedIter->name, iter->name); + EXPECT_EQ( + std::count_if(parsed.begin(), parsed.end(), [](auto const& entry) { + return entry.nameIsValidUtf8; + }), + static_cast(parsed.size() - 1u) + ); + } + TEST_F(SftpTests, MockServerPosixRenameExtensionOverwrites) { CREATE_SERVER_AND_JOINER(Sftp); @@ -1042,28 +1078,164 @@ namespace SecureShell::Test } } + namespace + { + std::expected renameReplace(SftpSession& sftp) + { + auto future = sftp.performPromise( + [&sftp]() + { + return sftp.renameReplaceInStrand( + remote("/home/test/source.txt"), remote("/home/test/target.txt"), ".aside" + ); + } + ); + if (future.wait_for(3s) != std::future_status::ready) + return std::unexpected(SftpError{.message = "timeout"}); + return future.get(); + } + + std::expected renameNoReplace( + SftpSession& sftp, + std::string_view destination, + std::optional sourceType = std::nullopt + ) + { + auto future = sftp.performPromise( + [&sftp, destination, sourceType]() + { + return sftp.renameNoReplaceInStrand(remote("/home/test/source.txt"), remote(destination), sourceType); + } + ); + if (future.wait_for(3s) != std::future_status::ready) + return std::unexpected(SftpError{.message = "timeout"}); + return future.get(); + } + } + TEST_F(SftpTests, FailedRenameReplaceRestoresTheDestination) { CREATE_SERVER_AND_JOINER(Sftp); ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); - // Moving the destination aside succeeds, the rename into place fails. + // Moving the destination aside succeeds, the rename into place fails, moving it back succeeds. + ASSERT_TRUE(serverStartResult->command("failRenames", {{"after", 1}, {"count", 1}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + EXPECT_FALSE(renameReplace(*sftp).has_value()); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "target"); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/target.txt.aside")); + EXPECT_EQ(readRemote(*sftp, "/home/test/source.txt"), "source"); + } + + TEST_F(SftpTests, FailedRollbackNamesWhereTheDestinationIsKept) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); ASSERT_TRUE(serverStartResult->command("failRenames", {{"after", 1}})); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto future = sftp->performPromise( - [&sftp]() - { - return sftp->renameReplaceInStrand(remote("/home/test/source.txt"), remote("/home/test/target.txt"), ".aside"); - } - ); - ASSERT_EQ(future.wait_for(3s), std::future_status::ready); - EXPECT_FALSE(future.get().has_value()); + const auto result = renameReplace(*sftp); + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().message.find("/home/test/target.txt.aside"), std::string::npos) << result.error().message; ASSERT_TRUE(serverStartResult->command("failRenames", {{"enabled", false}})); - // The restore is a rename as well and ran while renames failed, so the original sits aside: never lost. - const bool restored = readRemote(*sftp, "/home/test/target.txt") == "target"; - const bool keptAside = readRemote(*sftp, "/home/test/target.txt.aside") == "target"; - EXPECT_TRUE(restored || keptAside); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt.aside"), "target"); EXPECT_EQ(readRemote(*sftp, "/home/test/source.txt"), "source"); } + + TEST_F(SftpTests, RenameReplaceRemovesALeftoverAside) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt.aside"}, {"content", "old"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + ASSERT_TRUE(renameReplace(*sftp).has_value()); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "source"); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/target.txt.aside")); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/source.txt")); + } + + TEST_F(SftpTests, RenameReplacePutsBackADestinationThatOnlyExistsAside) + { + for (const bool failInPlace : {false, true}) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE( + serverStartResult->command("createFile", {{"path", "/home/test/target.txt.aside"}, {"content", "target"}}) + ); + // Putting it back and moving it aside again succeed, the rename into place fails. + if (failInPlace) + ASSERT_TRUE(serverStartResult->command("failRenames", {{"after", 2}, {"count", 1}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto result = renameReplace(*sftp); + EXPECT_EQ(result.has_value(), !failInPlace); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), failInPlace ? "target" : "source"); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/target.txt.aside")) << "failInPlace: " << failInPlace; + } + } + + TEST_F(SftpTests, RenameNoReplaceRenamesWhenTheServerRefusesHardlinks) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "hardlink@openssh.com"}})); + ASSERT_TRUE(serverStartResult->command("failHardlinks", nlohmann::json::object())); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto refused = renameNoReplace(*sftp, "/home/test/target.txt", SharedData::FileType::Regular); + ASSERT_FALSE(refused.has_value()); + EXPECT_EQ(refused.error().sftpError, SSH_FX_FILE_ALREADY_EXISTS); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "target"); + + ASSERT_TRUE(renameNoReplace(*sftp, "/home/test/moved.txt", SharedData::FileType::Regular).has_value()); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/source.txt")); + EXPECT_EQ(readRemote(*sftp, "/home/test/moved.txt"), "source"); + } + + TEST_F(SftpTests, RenameNoReplaceSucceedsWhenTheLinkedSourceCannotBeRemoved) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "hardlink@openssh.com"}})); + ASSERT_TRUE(serverStartResult->command("failRemoves", nlohmann::json::object())); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + ASSERT_TRUE(renameNoReplace(*sftp, "/home/test/moved.txt").has_value()); + EXPECT_EQ(readRemote(*sftp, "/home/test/moved.txt"), "source"); + } + + TEST_F(SftpTests, MoveIntoPlaceReplacesOnlyWhenAllowed) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto move = [&sftp](bool replace) + { + auto future = sftp->performPromise( + [&sftp, replace]() + { + return sftp->moveIntoPlaceInStrand( + remote("/home/test/source.txt"), remote("/home/test/target.txt"), replace, ".aside" + ); + } + ); + return future.get(); + }; + const auto refused = move(false); + ASSERT_FALSE(refused.has_value()); + EXPECT_EQ(refused.error().sftpError, SSH_FX_FILE_ALREADY_EXISTS); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "target"); + + ASSERT_TRUE(move(true).has_value()); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "source"); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/source.txt")); + } } diff --git a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs index c84a80c9..fff6dbef 100644 --- a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs +++ b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs @@ -169,6 +169,10 @@ const faults = { failRenames: false, // Renames that still succeed before failRenames applies, null for none. renamesBeforeFailure: null, + // Renames that fail before renames succeed again, null for all. + renamesFailing: null, + failHardlinks: false, + failRemoves: false, deniedDirectories: new Set(), deniedStats: new Set(), bytesRead: 0, @@ -185,6 +189,11 @@ const renameFails = () => { --faults.renamesBeforeFailure; return false; } + if (faults.renamesFailing !== null) { + if (faults.renamesFailing === 0) + return false; + --faults.renamesFailing; + } return true; }; @@ -235,6 +244,31 @@ const advertiseExtensions = (sftpStream) => { }; }; +// Resolves every symlink on the way like realpath, undefined when a component does not exist. +const canonicalize = (path, depth = 0) => { + if (depth > 40) + return undefined; + const parts = path.split('/').filter(part => part.length > 0 && part !== '.'); + let current = '/'; + for (let index = 0; index < parts.length; ++index) { + if (parts[index] === '..') { + current = pathUtil.posix.dirname(current); + continue; + } + const candidate = pathUtil.posix.join(current, parts[index]); + const entry = fakeFilesystem.find(candidate); + if (entry === undefined) + return undefined; + if (entry.type === 'symlink') { + const target = entry.target.startsWith('/') ? entry.target : pathUtil.posix.join(current, entry.target); + const rest = parts.slice(index + 1).join('/'); + return canonicalize(rest.length > 0 ? `${target}/${rest}` : target, depth + 1); + } + current = candidate; + } + return current; +}; + const resolveFileName = (path) => { if (path === '.') { return defaultPath; @@ -472,19 +506,18 @@ const server = new Server({ return sftpStream.name(reqid, [{ filename, longname: filename, attrs: {} }]); } - path = resolveFileName(path); - - const result = fakeFilesystem.find(path); + path = canonicalize(resolveFileName(path)); + const result = path === undefined ? undefined : fakeFilesystem.find(path); if (result === undefined) { logMessage(`Path not found: ${path}`); return sftpStream.status(reqid, STATUS_CODE.NO_SUCH_FILE); } - logMessage('REALPATH', result.path(), result); + logMessage('REALPATH', path, result); const name = [{ - filename: result.path(), + filename: path, longname: makeLongName(result), attrs: result.stat }]; @@ -567,7 +600,8 @@ const server = new Server({ return { filename: child.name, longname: makeLongName(child), - attrs: child.stat + attrs: child.stat, + rawName: child.rawName }; }); } @@ -577,7 +611,7 @@ const server = new Server({ logMessage(`Sending ${entries.length} entries`); logMessage(entries); logMessage(`Remaining entries: ${handle.remainingReadDirEntries.length}`); - sftpStream.name(reqid, entries); + sendNames(sftpStream, reqid, entries); return; } @@ -598,6 +632,10 @@ const server = new Server({ logMessage('Cannot remove directories'); return sftpStream.status(reqid, STATUS_CODE.FAILURE); } + if (faults.failRemoves) { + logMessage('Fault: failing REMOVE'); + return sftpStream.status(reqid, STATUS_CODE.PERMISSION_DENIED); + } result.parent.removeChild(result.name); @@ -663,6 +701,10 @@ const server = new Server({ } if (name === 'hardlink@openssh.com') { + if (faults.failHardlinks) { + logMessage('Fault: failing hardlink'); + return sftpStream.status(reqid, STATUS_CODE.PERMISSION_DENIED); + } if (newResult !== undefined || oldResult.type !== 'file') return sftpStream.status(reqid, STATUS_CODE.FAILURE); const parent = fakeFilesystem.find(pathUtil.dirname(newPath)); @@ -919,6 +961,14 @@ cli.on('dropAfterBytes', (command) => { cli.on('failRenames', (command) => { faults.failRenames = command.enabled !== false; faults.renamesBeforeFailure = command.after === undefined ? null : Number(command.after); + faults.renamesFailing = command.count === undefined ? null : Number(command.count); +}); +// Hard links answer permission denied, like a filesystem without them or Linux protected_hardlinks. +cli.on('failHardlinks', (command) => { + faults.failHardlinks = command.enabled !== false; +}); +cli.on('failRemoves', (command) => { + faults.failRemoves = command.enabled !== false; }); cli.on('denyDirectory', (command) => { faults.deniedDirectories.add(command.path); @@ -931,14 +981,45 @@ cli.on('advertise', (command) => { advertisedExtensions[command.name] = command.data ?? '1'; }); // Inserts an entry under a raw name that may be invalid as a path segment, like '..' or 'a/b'. +/** + * ssh2 writes names as UTF-8 strings, so a name whose bytes are no UTF-8 is listed under its ASCII placeholder of the + * same length and its bytes are put into the NAME packet in place of the placeholder. + */ +function sendNames(sftpStream, reqid, entries) { + const rawEntries = entries.filter((entry) => entry.rawName !== undefined); + if (rawEntries.length === 0) + return sftpStream.name(reqid, entries); + const protocol = sftpStream._protocol; + const chunks = []; + sftpStream._protocol = { channelData: (_id, data) => chunks.push(Buffer.from(data)) }; + try { + sftpStream.name(reqid, entries); + } finally { + sftpStream._protocol = protocol; + } + const packet = Buffer.concat(chunks); + for (const entry of rawEntries) { + const at = packet.indexOf(Buffer.from(entry.filename, 'latin1')); + if (at >= 0) + entry.rawName.copy(packet, at); + } + protocol.channelData(sftpStream.outgoing.id, packet); +} + cli.on('rawName', (command) => { const parent = fakeFilesystem.find(command.directory); if (parent === undefined || parent.type !== 'directory') return; - if (command.type === 'directory') - parent.insert(directory({ name: command.name }, [])); - else - parent.insert(file(command.name, command.content ?? '')); + // nameHex gives bytes that need not be UTF-8, they are listed through a placeholder of the same length. + const rawName = command.nameHex === undefined ? undefined : Buffer.from(command.nameHex, 'hex'); + const name = rawName === undefined + ? command.name + : Array.from({ length: rawName.length }, () => String.fromCharCode(65 + Math.floor(Math.random() * 26))).join(''); + const node = command.type === 'directory' + ? directory({ name }, []) + : file(name, command.content ?? ''); + node.rawName = rawName; + parent.insert(node); }); cli.on('createFile', (command) => { const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); @@ -949,13 +1030,14 @@ cli.on('createFile', (command) => { const content = command.size !== undefined ? Buffer.alloc(Number(command.size), command.fill ?? 'x') : (command.content ?? ''); - parent.insert(file(name, content)); + parent.insert(file(name, content, command.mode === undefined ? undefined : { mode: Number(command.mode) })); }); cli.on('createDirectory', (command) => { const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); if (parent === undefined || parent.type !== 'directory') return; - parent.insert(directory({ name: pathUtil.posix.basename(command.path) }, [])); + const stat = command.mode === undefined ? undefined : { mode: Number(command.mode) }; + parent.insert(directory({ name: pathUtil.posix.basename(command.path), stat }, [])); }); cli.on('createSymlink', (command) => { const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); diff --git a/ssh/test/ssh/test_ssh2_servers/source/cli.mjs b/ssh/test/ssh/test_ssh2_servers/source/cli.mjs index 2f176b4e..3aa70557 100644 --- a/ssh/test/ssh/test_ssh2_servers/source/cli.mjs +++ b/ssh/test/ssh/test_ssh2_servers/source/cli.mjs @@ -18,7 +18,7 @@ class CommandLineInterface { const command = JSON.parse(input); if (Object.hasOwnProperty.call(command, 'command')) { - // Exit runs once: a second run wrote to the closed log and the crash waited for stdin on Windows. + // Exit runs its handler once and closes stdin. if (command.command === 'exit') { if (this.eventMap.has('exit')) { this.eventMap.get('exit')(); diff --git a/static/assets/languages/de_DE.yaml b/static/assets/languages/de_DE.yaml index 82c752cf..eaf0e33c 100644 --- a/static/assets/languages/de_DE.yaml +++ b/static/assets/languages/de_DE.yaml @@ -153,6 +153,9 @@ operationQueue: deletingItem: "Lösche: '{}'" deletingSelectedIn: "Lösche ausgewählte Elemente in {}" deletingLocallyIn: "Lösche lokale Elemente in {}" + localDeleteDone: "Die lokalen Elemente in {} wurden gelöscht." + localDeletePartlyDone: "Nicht alle lokalen Elemente in {} konnten gelöscht werden." + localDeleteFailed: "Die lokalen Elemente in {} konnten nicht gelöscht werden: {}" entriesFailed: "{} Einträge fehlgeschlagen:\n" errorButton: "Fehler!" livePage: "Live" @@ -197,16 +200,17 @@ syncDialog: deleteSideLocal: "lokal" confirmDeleteRowHeader: "Jetzt löschen" confirmDeleteRow: "'{}' {} jetzt löschen?" - blockedNotice: "{} Einträge bleiben unverändert, weil sie nicht lesbar oder nicht übertragbar sind, ein Verzeichnis auf einen symbolischen Link trifft sich ihre Namen nur in der Groß- und Kleinschreibung unterscheiden oder sie sich seit der letzten Synchronisation so geändert haben, dass Daten verloren gingen. Zum Anzeigen klicken." + blockedNotice: "{} Einträge bleiben unverändert, weil sie nicht lesbar oder nicht übertragbar sind, ein Verzeichnis auf einen symbolischen Link trifft oder sich ihre Namen nur in der Groß- und Kleinschreibung unterscheiden. Zum Anzeigen klicken." blockedTitle: "Unveränderte Einträge" + synchronizeUnavailable: "Der Vergleich läuft noch oder wurde abgebrochen. Vergleichen Sie erneut, um zu synchronisieren." + executionRejected: "Diese Einträge wurden nicht synchronisiert: {}" + changedSinceCompare: "Nach dem Vergleich geändert und deshalb unverändert gelassen. Erneut vergleichen, um es zu synchronisieren." blockReason: Unreadable: "nicht lesbar" Excluded: "nicht übertragbar" InvalidName: "ungültiger Name" TypeConflict: "Verzeichnis gegen symbolischen Link" CaseConflict: "Namen unterscheiden sich nur in der Groß- und Kleinschreibung" - ChangedOnBothSides: "seit der letzten Synchronisation auf beiden Seiten geändert" - ChangedOnDestination: "seit der letzten Synchronisation auf der Seite geändert, die ersetzt oder gelöscht würde" tree: loading: "Lädt…" loadFailed: "Fehlgeschlagen: {}" @@ -1151,6 +1155,12 @@ terminalPanel: saveDialogTitle: "Terminalinhalt speichern" saveToFile: "Terminal in Datei speichern" backend: + deleteGuard: + noPath: "Es wurde nichts gelöscht. Ein Pfad fehlt." + root: "Es wurde nichts gelöscht. '{}' ist das Wurzelverzeichnis." + containsHome: "Es wurde nichts gelöscht. '{}' ist oder enthält das Heimatverzeichnis '{}'." + windowsName: "Es wurde nichts gelöscht. Der Server verwendet Windows-Pfade, unter denen '{}' ein anderes Element bezeichnen könnte." + notVerified: "Es wurde nichts gelöscht, weil der Server nicht rechtzeitig beantwortet hat, ob '{}' das Heimatverzeichnis enthält." rename: targetExists: "Ein Eintrag mit diesem Namen existiert bereits." invalidName: "Der neue Name muss ein einfacher Name im selben Verzeichnis sein." @@ -1196,3 +1206,8 @@ backend: connectFailedUnspecified: "Verbindung zum SSH-Server aus unbekanntem Grund fehlgeschlagen." persistence: saveFailed: "Der Zustand konnte nicht gespeichert werden: {}" + sync: + staleComparison: "der Vergleich hat sich seit der Anzeige geändert, bitte erneut vergleichen" + scanCancelled: "Das Auflisten wurde abgebrochen." + localScanFailed: "Das lokale Verzeichnis konnte nicht aufgelistet werden: {}" + remoteScanFailed: "Das Verzeichnis auf dem Server konnte nicht aufgelistet werden: {}" diff --git a/static/assets/languages/en_US.yaml b/static/assets/languages/en_US.yaml index bfa5ed6e..8af1bdb4 100644 --- a/static/assets/languages/en_US.yaml +++ b/static/assets/languages/en_US.yaml @@ -93,6 +93,9 @@ operationQueue: deletingItem: "Deleting: '{}'" deletingSelectedIn: "Deleting selected items in {}" deletingLocallyIn: "Deleting local items in {}" + localDeleteDone: "The local items in {} were deleted." + localDeletePartlyDone: "Not every local item in {} could be deleted." + localDeleteFailed: "The local items in {} could not be deleted: {}" entriesFailed: "{} entries failed:\n" errorButton: "Error!" livePage: "Live" @@ -137,16 +140,17 @@ syncDialog: deleteSideLocal: "locally" confirmDeleteRowHeader: "Delete Now" confirmDeleteRow: "Delete '{}' {} now?" - blockedNotice: "{} entries are left alone because they cannot be read, cannot be transferred, a directory faces a symlink, their names differ only in case, or they changed since the last sync where that would lose data. Click to see them." + blockedNotice: "{} entries are left alone because they cannot be read, cannot be transferred, a directory faces a symlink, or their names differ only in case. Click to see them." blockedTitle: "Entries left alone" + synchronizeUnavailable: "The comparison is running or was cancelled. Compare again to synchronize." + executionRejected: "These entries were not synchronized: {}" + changedSinceCompare: "Changed after the compare, so it was left as it is. Compare again to synchronize it." blockReason: Unreadable: "unreadable" Excluded: "not transferable" InvalidName: "invalid name" TypeConflict: "directory against symlink" CaseConflict: "names differ only in case" - ChangedOnBothSides: "changed on both sides since the last sync" - ChangedOnDestination: "changed on the side it would replace or delete since the last sync" tree: loading: "Loading…" loadFailed: "Failed: {}" @@ -1153,6 +1157,12 @@ terminalPanel: saveDialogTitle: "Save terminal contents" saveToFile: "Save terminal to file" backend: + deleteGuard: + noPath: "Nothing was deleted. A path is missing." + root: "Nothing was deleted. '{}' is the root directory." + containsHome: "Nothing was deleted. '{}' is or contains the home directory '{}'." + windowsName: "Nothing was deleted. The server uses Windows paths, where '{}' could name a different item." + notVerified: "Nothing was deleted, because the server did not answer in time whether '{}' contains the home directory." rename: targetExists: "An item with this name already exists." invalidName: "The new name must be a plain name in the same directory." @@ -1198,3 +1208,8 @@ backend: connectFailedUnspecified: "Failed to connect to ssh server for unspecified reason." persistence: saveFailed: "Failed to save state to disk: {}" + sync: + staleComparison: "the comparison changed since they were shown, compare again" + scanCancelled: "The listing was cancelled." + localScanFailed: "The local directory could not be listed: {}" + remoteScanFailed: "The directory on the server could not be listed: {}" diff --git a/tar-archive/include/tar_archive/header.hpp b/tar-archive/include/tar_archive/header.hpp index 7bb9eec1..8232a8da 100644 --- a/tar-archive/include/tar_archive/header.hpp +++ b/tar-archive/include/tar_archive/header.hpp @@ -93,8 +93,9 @@ namespace TarArchive }; /** - * @brief Build the on-disk tar records for a DirectoryEntry, emitting PAX extended - * headers when the entry's path, link target, or size exceed plain USTAR limits. + * @brief Build the on-disk tar records for a member, emitting PAX extended headers when its path, link + * target, or size exceed plain USTAR limits. Unknown permissions are written as 0755 for a directory, + * 0777 for a symlink and 0644 otherwise. */ std::expected buildRecords(MemberHeader const& meta); diff --git a/tar-archive/source/tar_archive/archive.cpp b/tar-archive/source/tar_archive/archive.cpp index b39bb431..4104d0cb 100644 --- a/tar-archive/source/tar_archive/archive.cpp +++ b/tar-archive/source/tar_archive/archive.cpp @@ -2,6 +2,9 @@ #include #include "compression_internal.hpp" +#include + +#include #include @@ -18,7 +21,7 @@ namespace TarArchive { if (!std::filesystem::exists(path_)) return std::unexpected(makeError( - TarErrorCode::FileNotFound, "archive does not exist: " + path_.string() + TarErrorCode::FileNotFound, fmt::format("archive does not exist: {}", Paths::nativeToUtf8(path_)) )); const Compression byExtension = compressionFromExtension(path_); auto byMagic = detectCompressionByMagic(path_); diff --git a/tar-archive/source/tar_archive/compression/factory.cpp b/tar-archive/source/tar_archive/compression/factory.cpp index b70dbc5d..9c9884e2 100644 --- a/tar-archive/source/tar_archive/compression/factory.cpp +++ b/tar-archive/source/tar_archive/compression/factory.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -27,7 +28,7 @@ namespace TarArchive std::string lowercaseExtension(std::filesystem::path const& path) { - return toLower(path.extension().string()); + return toLower(Paths::nativeToUtf8(path.extension())); } struct MagicSignature diff --git a/tar-archive/source/tar_archive/compression/raw_stream.cpp b/tar-archive/source/tar_archive/compression/raw_stream.cpp index a89f8d9e..f4625245 100644 --- a/tar-archive/source/tar_archive/compression/raw_stream.cpp +++ b/tar-archive/source/tar_archive/compression/raw_stream.cpp @@ -2,6 +2,9 @@ #include #include +#include + +#include #include #include @@ -15,9 +18,7 @@ namespace TarArchive::Detail std::ofstream output{path, std::ios::binary | std::ios::trunc}; if (!output.is_open()) return std::unexpected(makeError( - TarErrorCode::IoError, - "could not open '" + path.string() + "' for writing", - errno + TarErrorCode::IoError, fmt::format("could not open '{}' for writing", Paths::nativeToUtf8(path)), errno )); return output; } @@ -26,14 +27,12 @@ namespace TarArchive::Detail { if (!std::filesystem::exists(path)) return std::unexpected(makeError( - TarErrorCode::FileNotFound, "archive does not exist: " + path.string() + TarErrorCode::FileNotFound, fmt::format("archive does not exist: {}", Paths::nativeToUtf8(path)) )); std::ifstream input{path, std::ios::binary}; if (!input.is_open()) return std::unexpected(makeError( - TarErrorCode::IoError, - "could not open '" + path.string() + "' for reading", - errno + TarErrorCode::IoError, fmt::format("could not open '{}' for reading", Paths::nativeToUtf8(path)), errno )); return input; } diff --git a/tar-archive/source/tar_archive/header.cpp b/tar-archive/source/tar_archive/header.cpp index 7189f707..c6ed9212 100644 --- a/tar-archive/source/tar_archive/header.cpp +++ b/tar-archive/source/tar_archive/header.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -227,6 +228,17 @@ namespace TarArchive return std::nullopt; } + bool hasNonAsciiByte(std::string_view text) + { + return std::ranges::any_of( + text, + [](char character) + { + return static_cast(character) >= 0x80u; + } + ); + } + std::string normaliseArchivePath(std::string value) { while (!value.empty() && value.front() == '/') @@ -266,6 +278,21 @@ namespace TarArchive return payload; } + /** + * @brief The mode field: the permission bits, or what a newly created member of this type commonly gets + * when they are unknown. + */ + std::uint32_t modeOf(std::filesystem::perms permissions, TypeFlag typeflag) + { + if (permissions != std::filesystem::perms::unknown) + return static_cast(permissions) & 07777u; + if (typeflag == TypeFlag::Directory) + return 0755u; + if (typeflag == TypeFlag::SymbolicLink) + return 0777u; + return 0644u; + } + void populateUstarHeader( RawRecord& record, std::string const& name, @@ -279,7 +306,7 @@ namespace TarArchive record.fill(std::byte{0u}); writeString(record, fieldName, name); writeString(record, fieldPrefix, prefix); - writeNumeric(record, fieldMode, static_cast(meta.permissions) & 07777u); + writeNumeric(record, fieldMode, modeOf(meta.permissions, typeflag)); writeNumeric(record, fieldUid, meta.uid); writeNumeric(record, fieldGid, meta.gid); writeNumeric(record, fieldSize, payloadSize); @@ -632,20 +659,21 @@ namespace TarArchive std::string truncatedPath = normalisedPath; std::string truncatedLink = linkTargetText; + // A name with non-ASCII bytes also gets a PAX record, which readers take as UTF-8, the ustar fields have no + // encoding of their own. auto split = trySplitUstarPath(normalisedPath, isDirectory); - const bool pathNeedsPax = !split.has_value(); - if (pathNeedsPax) - { + if (!split.has_value() || hasNonAsciiByte(normalisedPath)) paxEntries.emplace_back("path", normalisedPath); - truncatedPath = normalisedPath.substr(0u, fieldName.length); + if (!split.has_value()) + { + truncatedPath = std::string{Paths::truncatedUtf8(normalisedPath, fieldName.length)}; split = std::pair{std::string{}, truncatedPath}; } - const bool linkNeedsPax = linkTargetText.size() > fieldLinkName.length; - if (linkNeedsPax) + if (linkTargetText.size() > fieldLinkName.length || hasNonAsciiByte(linkTargetText)) { paxEntries.emplace_back("linkpath", linkTargetText); - truncatedLink = linkTargetText.substr(0u, fieldLinkName.length); + truncatedLink = std::string{Paths::truncatedUtf8(linkTargetText, fieldLinkName.length)}; } if (sizeExceedsOctal) @@ -668,9 +696,8 @@ namespace TarArchive const std::string paxName = "PaxHeaders/" + (truncatedPath.empty() ? std::string{"entry"} : truncatedPath); const auto paxSplit = trySplitUstarPath(paxName, false); - const std::string paxNameField = paxSplit - ? paxSplit->second - : paxName.substr(0u, fieldName.length); + const std::string paxNameField = + paxSplit ? paxSplit->second : std::string{Paths::truncatedUtf8(paxName, fieldName.length)}; const std::string paxPrefixField = paxSplit ? paxSplit->first : std::string{}; RawRecord paxRecord; diff --git a/tar-archive/test/tar_archive/test_header.hpp b/tar-archive/test/tar_archive/test_header.hpp index 292e59d1..c3460b12 100644 --- a/tar-archive/test/tar_archive/test_header.hpp +++ b/tar-archive/test/tar_archive/test_header.hpp @@ -2,12 +2,14 @@ #include #include +#include #include #include #include #include +#include #include namespace TarArchive::Test @@ -63,6 +65,26 @@ namespace TarArchive::Test EXPECT_EQ(parsed->uName, "alice"); } + TEST(TarHeader, UnknownPermissionsGetTheModeOfANewMember) + { + const auto modeOf = [](SharedData::FileType type) + { + TarArchive::MemberHeader entry{}; + entry.path = "member"; + entry.type = type; + if (type == SharedData::FileType::Symlink) + entry.linkTarget = "target"; + const auto built = buildRecords(entry); + EXPECT_TRUE(built.has_value()); + const auto parsed = parseRecord(built->records.back()); + EXPECT_TRUE(parsed.has_value()); + return static_cast(parsed->permissions); + }; + EXPECT_EQ(modeOf(SharedData::FileType::Regular), 0644u); + EXPECT_EQ(modeOf(SharedData::FileType::Directory), 0755u); + EXPECT_EQ(modeOf(SharedData::FileType::Symlink), 0777u); + } + TEST(TarHeader, BuildRecordsForDirectoryHasTrailingSlashAndNoSize) { TarArchive::MemberHeader entry{}; @@ -149,6 +171,65 @@ namespace TarArchive::Test EXPECT_EQ(fileTypeFromTypeFlag(TypeFlag::FifoSpecial), FileType::Fifo); } + TEST(TarHeader, NonAsciiNamesGetPaxRecords) + { + const std::string name = "Grüße/東京.txt"; + const auto built = buildRecords(makeRegularEntry(name, 4u)); + ASSERT_TRUE(built.has_value()) << built.error().toString(); + ASSERT_EQ(built->records.size(), 3u); + + const auto paxHeader = parseRecord(built->records[0]); + ASSERT_TRUE(paxHeader.has_value()); + EXPECT_EQ(paxHeader->typeflag, TypeFlag::ExtendedHeader); + const auto overrides = parsePaxPayload( + std::span{built->records[1]}.first(static_cast(paxHeader->size)) + ); + ASSERT_TRUE(overrides.has_value()) << overrides.error().toString(); + EXPECT_EQ(overrides->path, name); + + const auto main = parseRecord(built->records[2]); + ASSERT_TRUE(main.has_value()); + EXPECT_EQ(main->fullName(), name); + } + + TEST(TarHeader, NonAsciiLinkTargetsGetPaxRecords) + { + TarArchive::MemberHeader entry{}; + entry.path = "link"; + entry.type = SharedData::FileType::Symlink; + entry.linkTarget = "Ärger.txt"; + entry.permissions = static_cast(0777u); + + const auto built = buildRecords(entry); + ASSERT_TRUE(built.has_value()); + ASSERT_EQ(built->records.size(), 3u); + const auto paxHeader = parseRecord(built->records[0]); + ASSERT_TRUE(paxHeader.has_value()); + const auto overrides = parsePaxPayload( + std::span{built->records[1]}.first(static_cast(paxHeader->size)) + ); + ASSERT_TRUE(overrides.has_value()); + EXPECT_EQ(overrides->linkPath, "Ärger.txt"); + EXPECT_FALSE(overrides->path.has_value()); + } + + TEST(TarHeader, TruncatedUstarNamesEndAtACodePointBoundary) + { + std::string name; + for (int index = 0; index < 120; ++index) + name += "ü"; + const auto built = buildRecords(makeRegularEntry(name, 1u)); + ASSERT_TRUE(built.has_value()); + const auto main = parseRecord(built->records.back()); + ASSERT_TRUE(main.has_value()); + EXPECT_EQ(main->fullName().size(), 100u); + EXPECT_TRUE(Paths::isValidUtf8(main->fullName())); + + const auto paxHeader = parseRecord(built->records.front()); + ASSERT_TRUE(paxHeader.has_value()); + EXPECT_TRUE(Paths::isValidUtf8(paxHeader->fullName())); + } + TEST(TarHeader, LargeSizeTriggersPaxExtendedHeader) { TarArchive::MemberHeader entry{}; diff --git a/tar-archive/test/tar_archive/test_pax.hpp b/tar-archive/test/tar_archive/test_pax.hpp index dceb3caf..9515fba0 100644 --- a/tar-archive/test/tar_archive/test_pax.hpp +++ b/tar-archive/test/tar_archive/test_pax.hpp @@ -63,6 +63,34 @@ namespace TarArchive::Test EXPECT_EQ(entry.header().size, contents.size()); } + TEST(TarPax, NonAsciiPathRoundTrips) + { + const std::string name = "Grüße/東京 \xF0\x9F\x9A\x80.txt"; + TarArchive::MemberHeader meta{}; + meta.path = name; + meta.type = SharedData::FileType::Regular; + meta.size = 0u; + meta.permissions = static_cast(0644u); + + const auto path = paxDirectory() / "nonascii.tar"; + Archive archive{path}; + { + auto writer = archive.openWriter(); + ASSERT_TRUE(writer.has_value()); + auto entry = writer->beginEntry(meta); + ASSERT_TRUE(entry.has_value()); + ASSERT_TRUE(std::move(*entry).close().has_value()); + ASSERT_TRUE(writer->finalize().has_value()); + } + + auto reader = archive.openReader(); + ASSERT_TRUE(reader.has_value()); + auto next = reader->nextEntry(); + ASSERT_TRUE(next.has_value()); + ASSERT_TRUE(next->has_value()); + EXPECT_EQ((**next).header().path, name); + } + TEST(TarPax, LongSymlinkTargetRoundTripsViaExtendedHeader) { const std::string linkText = std::string(250u, 'x'); diff --git a/utility/include/utility/algorithm/case_convert.hpp b/utility/include/utility/algorithm/case_convert.hpp index 72373fb1..ca77bce5 100644 --- a/utility/include/utility/algorithm/case_convert.hpp +++ b/utility/include/utility/algorithm/case_convert.hpp @@ -9,19 +9,42 @@ namespace Utility::Algorithm { /** - * @brief Converts the passed string to upper case by out paramter. + * @brief @p character in lower case when it is an ASCII letter, unchanged otherwise. Bytes of a multibyte UTF-8 + * sequence are never touched and the locale plays no part. + */ + constexpr char asciiToLower(char character) noexcept + { + return character >= 'A' && character <= 'Z' ? static_cast(character - 'A' + 'a') : character; + } + + /** + * @brief @p character in upper case when it is an ASCII letter, unchanged otherwise, see @ref asciiToLower. + */ + constexpr char asciiToUpper(char character) noexcept + { + return character >= 'a' && character <= 'z' ? static_cast(character - 'a' + 'A') : character; + } + + /** + * @brief True for the ASCII upper case letters. + */ + constexpr bool isAsciiUpper(char character) noexcept + { + return character >= 'A' && character <= 'Z'; + } + + /** + * @brief Converts the ASCII letters of the passed string to upper case by out paramter. * * @param input The string to convert. */ inline void toUpperCaseInplace(std::string& input) { - std::transform(input.begin(), input.end(), input.begin(), [](char c) { - return std::toupper(c); - }); + std::ranges::transform(input, input.begin(), asciiToUpper); } /** - * @brief Converts the passed string to upper case and returns it. + * @brief Converts the ASCII letters of the passed string to upper case and returns it. * * @param input The string to convert. * @return std::string The string in upper case. @@ -29,26 +52,22 @@ namespace Utility::Algorithm inline std::string toUpperCase(std::string const& input) { std::string result(input.size(), '\0'); - std::transform(input.begin(), input.end(), result.begin(), [](char c) { - return std::toupper(c); - }); + std::ranges::transform(input, result.begin(), asciiToUpper); return result; } /** - * @brief Converts the passed string to lower case by out paramter. + * @brief Converts the ASCII letters of the passed string to lower case by out paramter. * * @param input The string to convert. */ inline void toLowerCaseInplace(std::string& input) { - std::transform(input.begin(), input.end(), input.begin(), [](char c) { - return std::tolower(c); - }); + std::ranges::transform(input, input.begin(), asciiToLower); } /** - * @brief Converts the passed string to lower case and returns it. + * @brief Converts the ASCII letters of the passed string to lower case and returns it. * * @param input The string to convert. * @return std::string The string in lower case. @@ -56,14 +75,12 @@ namespace Utility::Algorithm inline std::string toLowerCase(std::string const& input) { std::string result(input.size(), '\0'); - std::transform(input.begin(), input.end(), result.begin(), [](char c) { - return std::tolower(c); - }); + std::ranges::transform(input, result.begin(), asciiToLower); return result; } /** - * @brief Case-insensitive lexicographic comparison of two strings. + * @brief Lexicographic comparison of two strings, ASCII letters compared without case. * * @param lhs The left-hand string. * @param rhs The right-hand string. @@ -75,8 +92,8 @@ namespace Utility::Algorithm const auto commonLength = std::min(lhs.size(), rhs.size()); for (std::size_t index = 0; index < commonLength; ++index) { - const auto leftChar = std::tolower(static_cast(lhs[index])); - const auto rightChar = std::tolower(static_cast(rhs[index])); + const auto leftChar = static_cast(asciiToLower(lhs[index])); + const auto rightChar = static_cast(asciiToLower(rhs[index])); if (leftChar != rightChar) return leftChar < rightChar ? -1 : 1; } diff --git a/utility/include/utility/convert_naming_convention.hpp b/utility/include/utility/convert_naming_convention.hpp index 63f6d17a..d8db38ec 100644 --- a/utility/include/utility/convert_naming_convention.hpp +++ b/utility/include/utility/convert_naming_convention.hpp @@ -1,7 +1,9 @@ #pragma once -#include +#include + #include +#include namespace Utility { @@ -19,7 +21,7 @@ namespace Utility { if (!normalizedSegments[i].empty()) { - result += std::toupper(normalizedSegments[i][0]); + result += Algorithm::asciiToUpper(normalizedSegments[i][0]); result += normalizedSegments[i].substr(1); } } @@ -36,7 +38,7 @@ namespace Utility { if (!segment.empty()) { - result += std::toupper(segment[0]); + result += Algorithm::asciiToUpper(segment[0]); result += segment.substr(1); } } @@ -67,7 +69,7 @@ namespace Utility { result += '_'; for (auto const& ch : normalizedSegments[i]) - result += std::toupper(ch); + result += Algorithm::asciiToUpper(ch); } return result; } @@ -110,12 +112,12 @@ namespace Utility std::string currentSegment{}; for (auto const& ch : input) { - if (std::isupper(ch) && !currentSegment.empty()) + if (Algorithm::isAsciiUpper(ch) && !currentSegment.empty()) { result.normalizedSegments.push_back(currentSegment); currentSegment.clear(); } - currentSegment += std::tolower(ch); + currentSegment += Algorithm::asciiToLower(ch); } if (!currentSegment.empty()) result.normalizedSegments.push_back(currentSegment); @@ -131,12 +133,12 @@ namespace Utility std::string currentSegment{}; for (auto const& ch : input) { - if (std::isupper(ch) && !currentSegment.empty()) + if (Algorithm::isAsciiUpper(ch) && !currentSegment.empty()) { result.normalizedSegments.push_back(currentSegment); currentSegment.clear(); } - currentSegment += std::tolower(ch); + currentSegment += Algorithm::asciiToLower(ch); } if (!currentSegment.empty()) result.normalizedSegments.push_back(currentSegment); @@ -159,7 +161,7 @@ namespace Utility } else { - currentSegment += std::tolower(ch); + currentSegment += Algorithm::asciiToLower(ch); } } if (!currentSegment.empty()) @@ -183,7 +185,7 @@ namespace Utility } else { - currentSegment += std::tolower(ch); + currentSegment += Algorithm::asciiToLower(ch); } } if (!currentSegment.empty()) diff --git a/utility/include/utility/directory_traversal.hpp b/utility/include/utility/directory_traversal.hpp index c71faabb..171dcdd3 100644 --- a/utility/include/utility/directory_traversal.hpp +++ b/utility/include/utility/directory_traversal.hpp @@ -1,8 +1,12 @@ #pragma once +#include #include #include +#include +#include + #include #include #include @@ -10,6 +14,8 @@ #include #include #include +#include +#include namespace Utility { @@ -35,12 +41,61 @@ namespace Utility std::uint64_t totalBytes_{0}; }; + /** + * @brief The path of @p entry below the root of its walk, following the parent indices through @p entries. + * Empty for the root, which is the entry without a parent. Refused when a name on the way is no path + * segment or a parent index lies outside @p entries. + */ + template + std::expected + walkedRelativePath(std::vector const& entries, EntryT const& entry) + { + std::vector ancestry; + for (auto const* current = &entry; current->parent; current = &entries[*current->parent]) + { + if (*current->parent >= entries.size()) + return std::unexpected( + Paths::PathError{.reason = Paths::PathRejection::UnknownParent, .input = current->name} + ); + ancestry.push_back(current); + } + Paths::RelativePath relative{}; + for (auto const* ancestor : ancestry | std::views::reverse) + { + auto segment = ancestor->nameSegment(); + if (!segment) + return std::unexpected(std::move(segment).error()); + relative = relative / *segment; + } + return relative; + } + + /** + * @brief @p entry of a walk for display: @p root followed by the names as listed on the way down to it, so it can + * name an entry whose name is no path segment. Starts below a parent index outside @p entries. + */ + template + std::string describeWalkedEntry(std::string_view root, std::vector const& entries, EntryT const& entry) + { + std::vector names; + for (auto const* current = &entry; current->parent; current = &entries[*current->parent]) + { + names.push_back(current->name); + if (*current->parent >= entries.size()) + break; + } + const auto below = fmt::format("{}", fmt::join(names | std::views::reverse, "/")); + if (root.empty() || below.empty()) + return root.empty() ? below : std::string{root}; + return root.ends_with('/') ? fmt::format("{}{}", root, below) : fmt::format("{}/{}", root, below); + } + /** * @brief Walks a directory tree breadth first. The walker never sees the root path: the scanner lists the * directory at a path relative to the root, and entries are named relative to their parent. The root - * is entry 0 and has an empty name. + * is entry 0 and has an empty name. Every "." and ".." a scanner lists is dropped. */ - template + template requires std:: is_invocable_r_v, WalkErrorType>, ScannerT, Paths::RelativePath const&> class DeepDirectoryWalker : public BaseDirectoryWalker @@ -53,6 +108,7 @@ namespace Utility , entries_{EntryT{ .type = EntryT::FileType::Directory, }} + , directoryPaths_(1) {} std::vector ejectEntries() && @@ -75,6 +131,7 @@ namespace Utility entries_ = {EntryT{ .type = EntryT::FileType::Directory, }}; + directoryPaths_.assign(1, Paths::RelativePath{}); rejectedEntries_.clear(); currentIndex_ = 0; totalBytes_ = 0; @@ -89,25 +146,17 @@ namespace Utility std::expected walk() { if (completed()) - return false; + return true; if (entries_[currentIndex_].isDirectory()) { - auto result = scanner_(relativePath(entries_[currentIndex_])); + auto result = scanner_(directoryPaths_[currentIndex_]); if (!result) return std::unexpected(std::move(result).error()); moveEntries(std::move(result).value(), currentIndex_); ++currentIndex_; } - else - { - // Log::warn( - // "DeepDirectoryWalker: Encountered non-directory entry '{}' during walk at a point where a " - // "directory was expected, skipping.", - // relativePath(entries_[currentIndex_]).value() - // ); - } // Move over all non dirs for (; currentIndex_ < entries_.size(); ++currentIndex_) @@ -123,17 +172,25 @@ namespace Utility } /** - * @brief The path of a walked entry below the root, empty for the root. + * @brief The path of a walked entry below the root, empty for the root. Refused for an entry whose parent is + * not among the walked entries. */ - Paths::RelativePath relativePath(EntryT const& entry) const + std::expected relativePath(EntryT const& entry) const { if (!entry.parent) - return {}; + return Paths::RelativePath{}; const auto parentIndex = entry.parent.value(); if (parentIndex >= entries_.size()) - throw std::out_of_range("Parent index is out of range"); + return std::unexpected( + Paths::PathError{.reason = Paths::PathRejection::UnknownParent, .input = entry.name} + ); // Only valid segments are adopted, see moveEntries. - return relativePath(entries_[parentIndex]) / *Paths::PathSegment::make(nameOf(entry)); + return entry.nameSegment().transform( + [this, parentIndex](Paths::PathSegment const& segment) + { + return directoryPaths_[parentIndex] / segment; + } + ); } /** @@ -141,22 +198,19 @@ namespace Utility */ std::string describeRejected(EntryT const& entry) const { - if (!entry.parent || entry.parent.value() >= entries_.size()) - return nameOf(entry); - const auto parent = relativePath(entries_[entry.parent.value()]); - return parent.empty() ? nameOf(entry) : parent.value() + "/" + nameOf(entry); + return describeWalkedEntry({}, entries_, entry); } std::expected walkAll() { - decltype(walk()) res; - do + while (true) { - res = walk(); - if (!res) - return std::unexpected(std::move(res).error()); - } while (!res.value()); - return {}; + auto result = walk(); + if (!result) + return std::unexpected(std::move(result).error()); + if (*result) + return {}; + } } bool completed() const override @@ -165,8 +219,9 @@ namespace Utility } /** - * @brief Entries whose name is not a valid path segment, like ".." or "a/b" from a server. They are never - * walked or returned by @ref entries, their parent index is set so their full path can be shown. + * @brief Entries whose name is not a valid path segment, like ".." or "a/b" from a server or a name that was + * not UTF-8. They are never walked or returned by @ref entries, their parent index is set so their + * full path can be shown. */ std::vector const& rejectedEntries() const { @@ -183,26 +238,27 @@ namespace Utility for (auto& entry : newEntries) { entry.parent = parent; - const auto name = nameOf(entry); + auto const& name = entry.name; if (name == "." || name == "..") continue; - if (!Paths::PathSegment::make(name).has_value()) + auto segment = entry.nameSegment(); + if (!segment.has_value()) { rejectedEntries_.push_back(std::move(entry)); continue; } + directoryPaths_.push_back( + entry.isDirectory() ? directoryPaths_[parent] / *segment : Paths::RelativePath{} + ); entries_.push_back(std::move(entry)); } } - static std::string const& nameOf(EntryT const& entry) - { - return entry.name; - } - private: ScannerT scanner_; std::vector entries_{}; + /** @brief Index aligned with entries_: the path of each directory below the root, empty for the others. */ + std::vector directoryPaths_{}; std::vector rejectedEntries_{}; }; -} \ No newline at end of file +} diff --git a/utility/include/utility/error_message.hpp b/utility/include/utility/error_message.hpp new file mode 100644 index 00000000..34bf11c6 --- /dev/null +++ b/utility/include/utility/error_message.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include + +namespace Utility +{ + /** + * @brief The text of @p error as valid UTF-8. On Windows a system error is read with FormatMessageW, which the + * narrow message() would give in the ANSI code page; any other text has every byte that is not UTF-8 + * replaced by U+FFFD. + */ + std::string errorMessage(std::error_code const& error); + + /** + * @brief The text of a filesystem error as valid UTF-8: the message of its code and the paths it names. + */ + std::string errorMessage(std::filesystem::filesystem_error const& error); + + /** + * @brief The text of any exception as valid UTF-8: a filesystem error as above, any other what() with every byte + * that is not UTF-8 replaced by U+FFFD. + */ + std::string errorMessage(std::exception const& error); +} diff --git a/utility/include/utility/json_utf8.hpp b/utility/include/utility/json_utf8.hpp new file mode 100644 index 00000000..a5c6698b --- /dev/null +++ b/utility/include/utility/json_utf8.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include + +#include + +#include + +namespace Utility +{ + /** + * @brief @p json with every string and object key that is not valid UTF-8 written with U+FFFD for each invalid + * byte, so dump() never refuses it. Valid text stays as it is. + */ + inline nlohmann::json withValidUtf8(nlohmann::json json) + { + if (json.is_string()) + { + auto const& text = json.get_ref(); + if (!Paths::isValidUtf8(text)) + json = Paths::sanitizedUtf8(text); + return json; + } + if (json.is_array()) + { + for (auto& element : json) + element = withValidUtf8(std::move(element)); + return json; + } + if (json.is_object()) + { + nlohmann::json result = nlohmann::json::object(); + for (auto& [key, value] : json.items()) + result[Paths::sanitizedUtf8(key)] = withValidUtf8(std::move(value)); + return result; + } + return json; + } +} diff --git a/utility/include/utility/lifetime_token.hpp b/utility/include/utility/lifetime_token.hpp new file mode 100644 index 00000000..0bdd64b4 --- /dev/null +++ b/utility/include/utility/lifetime_token.hpp @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include + +namespace Utility +{ + /** + * @brief Tells a callback whether the object that made it still exists. The object holds the token as a member + * and wraps callbacks that capture it with @ref guarded, which then do nothing once it is gone. + * + * A token belongs to the address of its owner: a copied or moved owner gets a new token, and assigning one owner + * to another keeps each one's own token, so callbacks of the old address never run on the new one. + */ + class LifetimeToken + { + public: + LifetimeToken() = default; + ~LifetimeToken() = default; + LifetimeToken(LifetimeToken const&) + : LifetimeToken{} + {} + LifetimeToken(LifetimeToken&&) noexcept + : LifetimeToken{} + {} + LifetimeToken& operator=(LifetimeToken const&) noexcept + { + return *this; + } + LifetimeToken& operator=(LifetimeToken&&) noexcept + { + return *this; + } + + /** + * @brief @p callable wrapped so it runs only while this token exists. The wrapper takes the same parameters, + * so code that inspects them, like an RPC back channel, sees the original signature. @p callable has + * one call operator, which returns nothing: a call after the owner is gone has nothing to return. + */ + template + auto guarded(CallableT&& callable) const + { + return guardedFor(std::forward(callable), &std::remove_cvref_t::operator()); + } + + /** + * @brief Checks whether the owner still exists, for callbacks that need the check without @ref guarded. + */ + std::weak_ptr watch() const + { + return alive_; + } + + private: + template + auto guardedFor(CallableT&& callable, void (ClassT::*)(ArgumentTs...) const) const + { + return guardedWith(std::forward(callable)); + } + + template + auto guardedFor(CallableT&& callable, void (ClassT::*)(ArgumentTs...)) const + { + return guardedWith(std::forward(callable)); + } + + template + auto guardedWith(CallableT&& callable) const + { + return [alive = std::weak_ptr{alive_}, + callable = std::forward(callable)](ArgumentTs... arguments) mutable + { + if (const auto locked = alive.lock(); !locked) + return; + callable(std::forward(arguments)...); + }; + } + + private: + std::shared_ptr alive_{std::make_shared(true)}; + }; +} diff --git a/utility/include/utility/link_safe_remove.hpp b/utility/include/utility/link_safe_remove.hpp index 711d770f..0e398e05 100644 --- a/utility/include/utility/link_safe_remove.hpp +++ b/utility/include/utility/link_safe_remove.hpp @@ -2,29 +2,116 @@ #include #include +#include #include +#include #include +#include namespace Utility { + namespace Detail + { + /** + * @brief True for a Windows reparse tag that makes its item a link: a name surrogate such as a symlink + * (IO_REPARSE_TAG_SYMLINK), a junction (IO_REPARSE_TAG_MOUNT_POINT) or a WSL symlink + * (IO_REPARSE_TAG_LX_SYMLINK). Cloud file placeholders, deduplicated files and AF_UNIX sockets are + * reparse points too, but no links. + */ + bool isLinkReparseTag(std::uint32_t tag) noexcept; + } + + /** + * @brief What an item is, inspected without following it. + */ + enum class NoFollowKind + { + Missing, + /** @brief A symlink or, on Windows, a junction or another name surrogate reparse point. */ + Link, + Directory, + /** @brief A regular file or any other item that is no directory and no link. */ + Other + }; + + /** + * @brief Inspects @p path without following it. A missing path is no error. + */ + NoFollowKind noFollowKindOf(std::filesystem::path const& path, std::error_code& error); + /** - * @brief True for a symlink and, on Windows, for every reparse point such as a junction. std::filesystem may - * report a junction as a plain directory, and walking into it leaves the tree it belongs to. + * @brief True for a symlink and, on Windows, for a junction or another reparse point whose tag is a name + * surrogate (see Detail::isLinkReparseTag). std::filesystem may report a junction as a plain directory, + * and walking into it leaves the tree it belongs to. */ bool isLinkOrJunction(std::filesystem::path const& path, std::error_code& error); + /** + * @brief Directories already found to be real directories, so @ref firstLinkBelow inspects each of them once. + * One set belongs to one operation, which writes below a single root. Thread safe. + */ + class VerifiedDirectories + { + public: + bool contains(std::filesystem::path const& directory) const; + void insert(std::filesystem::path const& directory); + + private: + mutable std::mutex mutex_{}; + std::set directories_{}; + }; + /** * @brief Finds the first existing component of @p path below @p root (root excluded, @p path included) that is - * a symlink or junction. Writing through it would leave @p root. + * a symlink or junction. Writing through it would leave @p root. Components after the first missing one + * are not inspected, they cannot exist. + * @param verified Directories inspected before, skipped here; the real directories found are added to it. * @return The offending component, or nothing when there is none. @p error is set when @p path is not lexically * below @p root or a component cannot be inspected. */ - std::optional - firstLinkBelow(std::filesystem::path const& root, std::filesystem::path const& path, std::error_code& error); + std::optional firstLinkBelow( + std::filesystem::path const& root, + std::filesystem::path const& path, + std::error_code& error, + VerifiedDirectories* verified = nullptr + ); + + /** + * @brief Removes @p path itself without following it: a file, a link (never its target) or an empty directory. + * On Windows a read-only item has its read-only attribute cleared and is removed again. + * @return True when @p path was removed, false when it did not exist or @p error is set. + */ + bool removeNoFollow(std::filesystem::path const& path, std::error_code& error); + + /** + * @brief An item that could not be removed, and why. + */ + struct RemoveFailure + { + std::filesystem::path path; + std::error_code error; + }; + + /** + * @brief The outcome of @ref removeAllNoFollow. + */ + struct RemoveAllResult + { + /** @brief Every entry removed, the given path included. */ + std::uintmax_t removed{0}; + /** @brief The items that stopped the removal, by their own path. Their parent directories stay as well. */ + std::vector failures{}; + }; /** * @brief Removes @p path and, for a real directory, everything below it. A symlink or junction is removed - * itself and never followed, so nothing outside @p path is touched. + * itself and never followed, so nothing outside @p path is touched. An item that cannot be removed does + * not stop the others, only the directories containing it stay. + */ + RemoveAllResult removeAllNoFollow(std::filesystem::path const& path); + + /** + * @brief @ref removeAllNoFollow that sets @p error to the error of the first item that could not be removed. * @return The number of removed entries, 0 when @p path does not exist. */ std::uintmax_t removeAllNoFollow(std::filesystem::path const& path, std::error_code& error); diff --git a/utility/include/utility/no_follow_tree_walk.hpp b/utility/include/utility/no_follow_tree_walk.hpp new file mode 100644 index 00000000..156e5608 --- /dev/null +++ b/utility/include/utility/no_follow_tree_walk.hpp @@ -0,0 +1,105 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace Utility +{ + /** + * @brief Walks a path and, for a real directory, the tree below it in bounded steps without following links or + * junctions. It either counts the entries or removes them, the content of a directory before the + * directory itself. A link is counted or removed itself. + * + * An item that cannot be removed does not stop the walk, only the directories containing it stay. + */ + class NoFollowTreeWalk + { + public: + enum class Mode + { + Count, + Remove + }; + + NoFollowTreeWalk(std::filesystem::path root, Mode mode); + ~NoFollowTreeWalk() = default; + NoFollowTreeWalk(NoFollowTreeWalk const&) = delete; + NoFollowTreeWalk& operator=(NoFollowTreeWalk const&) = delete; + NoFollowTreeWalk(NoFollowTreeWalk&&) = default; + NoFollowTreeWalk& operator=(NoFollowTreeWalk&&) = default; + + /** + * @brief Works until the walk is done or @p deadline has passed, handling at least one entry per call. + * @return True when the walk is done. + */ + bool step(std::chrono::steady_clock::time_point deadline); + + bool done() const noexcept + { + return done_; + } + + /** + * @brief The entries counted or removed so far, the root included. + */ + std::uintmax_t entries() const noexcept + { + return entries_; + } + + /** + * @brief Count mode: the size of the regular files counted so far. + */ + std::uintmax_t bytes() const noexcept + { + return bytes_; + } + + /** + * @brief Remove mode: the items that could not be removed or listed, by their own path. + */ + std::vector const& failures() const noexcept + { + return failures_; + } + + /** + * @brief The entry handled last, empty before the first step. + */ + std::filesystem::path const& lastPath() const noexcept + { + return lastPath_; + } + + private: + struct Frame + { + std::filesystem::path directory; + std::filesystem::directory_iterator iterator; + /** @brief Something below could not be removed, so the directory stays. */ + bool keepsContent{false}; + }; + + void start(); + void handleEntry(std::filesystem::directory_entry const& entry); + void enter(std::filesystem::path const& directory); + void finishDirectory(); + void finishItem(std::filesystem::path const& path, NoFollowKind kind, bool directoryEntry, std::uintmax_t size); + void fail(std::filesystem::path const& path, std::error_code const& error); + + private: + std::filesystem::path root_; + Mode mode_; + bool started_{false}; + bool done_{false}; + std::vector stack_{}; + std::uintmax_t entries_{0}; + std::uintmax_t bytes_{0}; + std::vector failures_{}; + std::filesystem::path lastPath_{}; + }; +} diff --git a/utility/include/utility/permissions.hpp b/utility/include/utility/permissions.hpp index 56c661fd..6f83dac5 100644 --- a/utility/include/utility/permissions.hpp +++ b/utility/include/utility/permissions.hpp @@ -1,7 +1,9 @@ #pragma once +#include #include #include +#include namespace Utility { @@ -15,7 +17,8 @@ namespace Utility std::optional sanitizePermissions(std::filesystem::perms permissions); /** - * @brief The file mode creation mask of this process, empty where there is none. + * @brief The file mode creation mask of this process, empty where there is none. Read once; outside Linux + * reading it sets it for a moment, so the first call has to happen before other threads create files. */ std::filesystem::perms processUmask(); @@ -24,6 +27,18 @@ namespace Utility */ std::optional inheritablePermissions(std::filesystem::perms permissions); + /** + * @brief The permissions the system gives a new file: read and write for everyone, minus the process umask. + */ + std::filesystem::perms newFilePermissions(); + + /** + * @brief Creates @p path when it is missing and lets only its owner read and write it, so nothing written to it + * before its final permissions are set is readable by others. Never follows a link at @p path. Does + * nothing on Windows, where a new file takes the access rules of its directory. + */ + std::expected restrictToOwner(std::filesystem::path const& path); + /** * @brief The permissions of a local file as far as they mean anything. Unknown on Windows, where they are made up * from the read-only attribute and mark every file executable. diff --git a/utility/include/utility/process_temporary_directory.hpp b/utility/include/utility/process_temporary_directory.hpp index 2b8622bf..130195e0 100644 --- a/utility/include/utility/process_temporary_directory.hpp +++ b/utility/include/utility/process_temporary_directory.hpp @@ -19,7 +19,12 @@ namespace Utility public: explicit ProcessTemporaryDirectory(std::filesystem::path const& base) { - if (char const* inherited = std::getenv(environmentName); inherited != nullptr && *inherited != '\0') +#ifdef _WIN32 + auto const* inherited = ::_wgetenv(environmentName); +#else + auto const* inherited = std::getenv(environmentName); +#endif + if (inherited != nullptr && *inherited != 0) { path_ = std::filesystem::path{inherited}; std::filesystem::create_directories(path_); @@ -28,7 +33,7 @@ namespace Utility owned_.emplace(base, true); path_ = owned_->path(); #ifdef _WIN32 - ::_putenv_s(environmentName, path_.string().c_str()); + ::_wputenv_s(environmentName, path_.c_str()); #else ::setenv(environmentName, path_.c_str(), 1); #endif @@ -40,7 +45,11 @@ namespace Utility } private: +#ifdef _WIN32 + static constexpr wchar_t const* environmentName = L"NUI_SFTP_TEST_PROCESS_TEMPORARY"; +#else static constexpr char const* environmentName = "NUI_SFTP_TEST_PROCESS_TEMPORARY"; +#endif std::optional owned_{}; std::filesystem::path path_{}; diff --git a/utility/include/utility/rename.hpp b/utility/include/utility/rename.hpp index d144470d..da278baf 100644 --- a/utility/include/utility/rename.hpp +++ b/utility/include/utility/rename.hpp @@ -10,8 +10,40 @@ namespace Utility * @brief Renames @p from to @p to and fails with std::errc::file_exists instead of replacing an existing @p to. * * Atomic where the platform offers it: renameat2 RENAME_NOREPLACE on Linux, renamex_np RENAME_EXCL on macOS, - * MoveFileExW without MOVEFILE_REPLACE_EXISTING on Windows. A Linux filesystem without RENAME_NOREPLACE falls - * back to link and unlink for files. + * MoveFileExW without MOVEFILE_REPLACE_EXISTING on Windows. Where the filesystem refuses those, a regular file + * moves by link and unlink on Linux, and anything else by @ref Detail::renameUnlessExists, which leaves a window + * between the check and the rename. */ std::expected renameNoReplace(std::filesystem::path const& from, std::filesystem::path const& to); + + /** + * @brief Moves a finished file from @p from to @p to. + * + * @param replace Whether an existing @p to is replaced. Without it, an existing @p to fails with + * std::errc::file_exists, see @ref renameNoReplace. + */ + std::expected + moveIntoPlace(std::filesystem::path const& from, std::filesystem::path const& to, bool replace); + + namespace Detail + { + /** + * @brief Whether @p error of an atomic no-replace rename or a hard link means the filesystem does not offer + * it, rather than that the move itself failed. + */ + bool isNoReplaceUnsupported(std::error_code error); + + /** + * @brief Moves the regular file @p from to @p to by a hard link and removing @p from. The link fails on an + * existing @p to. When @p from cannot be removed, the link is removed again. + */ + std::expected + renameByLink(std::filesystem::path const& from, std::filesystem::path const& to); + + /** + * @brief Renames @p from to @p to unless something, a dangling link included, exists at @p to. + */ + std::expected + renameUnlessExists(std::filesystem::path const& from, std::filesystem::path const& to); + } } diff --git a/utility/include/utility/temporary_directory.hpp b/utility/include/utility/temporary_directory.hpp index 7bde3acc..cdd74c0d 100644 --- a/utility/include/utility/temporary_directory.hpp +++ b/utility/include/utility/temporary_directory.hpp @@ -1,14 +1,15 @@ #pragma once #include - -#include +#include #include #include -#include #include +#include +#include +#include #include namespace Utility @@ -29,7 +30,9 @@ namespace Utility } else if (!std::filesystem::is_directory(path_)) { - throw std::runtime_error(std::string{"Given path exists and is not a directory: "} + path_.string()); + throw std::runtime_error( + fmt::format("Given path exists and is not a directory: {}", Paths::nativeToUtf8(path_)) + ); } if (valid && randomSubFolder) { @@ -40,7 +43,9 @@ namespace Utility } if (!valid) { - throw std::runtime_error(std::string{"Could not setup temporary directory: "} + path_.string()); + throw std::runtime_error( + fmt::format("Could not setup temporary directory: {}", Paths::nativeToUtf8(path_)) + ); } } ~TemporaryDirectory() diff --git a/utility/include/utility/utf8.hpp b/utility/include/utility/utf8.hpp index 698c35dd..43279ddc 100644 --- a/utility/include/utility/utf8.hpp +++ b/utility/include/utility/utf8.hpp @@ -6,8 +6,8 @@ namespace Utility::Utf8 { /** - * @brief Decodes UTF-8 into code points. Malformed or truncated sequences, overlong forms and surrogates each - * become one U+FFFD, so arbitrary bytes never fail to decode. + * @brief Decodes UTF-8 into code points. Every byte that is not part of a valid sequence (malformed, truncated, + * overlong, surrogate) becomes one U+FFFD, so arbitrary bytes never fail to decode. * * @param utf8 The text to decode. */ diff --git a/utility/source/utility/CMakeLists.txt b/utility/source/utility/CMakeLists.txt index a669f51a..8544627e 100644 --- a/utility/source/utility/CMakeLists.txt +++ b/utility/source/utility/CMakeLists.txt @@ -21,9 +21,11 @@ else() utility PRIVATE node/node.cpp + error_message.cpp permissions.cpp rename.cpp link_safe_remove.cpp + no_follow_tree_walk.cpp download_mark.cpp resources.cpp ) diff --git a/utility/source/utility/error_message.cpp b/utility/source/utility/error_message.cpp new file mode 100644 index 00000000..0627765e --- /dev/null +++ b/utility/source/utility/error_message.cpp @@ -0,0 +1,78 @@ +#include + +#include +#include + +#include + +#include +#include + +#ifdef _WIN32 +# include +#endif + +namespace Utility +{ + namespace + { +#ifdef _WIN32 + struct LocalFreeDeleter + { + void operator()(wchar_t* buffer) const noexcept + { + LocalFree(buffer); + } + }; + + std::string systemMessage(int code) + { + wchar_t* buffer = nullptr; + const auto length = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + nullptr, + static_cast(code), + 0, + reinterpret_cast(&buffer), + 0, + nullptr + ); + const std::unique_ptr owned{buffer}; + if (length == 0 || !owned) + return fmt::format("System error {}", code); + std::wstring_view text{owned.get(), length}; + while (!text.empty() && (text.back() == L'\r' || text.back() == L'\n' || text.back() == L' ')) + text.remove_suffix(1); + return Paths::wideToUtf8(text); + } +#endif + } + + std::string errorMessage(std::error_code const& error) + { +#ifdef _WIN32 + if (error.category() == std::system_category()) + return systemMessage(error.value()); +#endif + return Paths::sanitizedUtf8(error.message()); + } + + std::string errorMessage(std::filesystem::filesystem_error const& error) + { + const auto message = errorMessage(error.code()); + if (error.path1().empty()) + return message; + if (error.path2().empty()) + return fmt::format("{}: '{}'", message, Paths::nativeToUtf8(error.path1())); + return fmt::format( + "{}: '{}', '{}'", message, Paths::nativeToUtf8(error.path1()), Paths::nativeToUtf8(error.path2()) + ); + } + + std::string errorMessage(std::exception const& error) + { + if (const auto* filesystemError = dynamic_cast(&error)) + return errorMessage(*filesystemError); + return Paths::sanitizedUtf8(error.what()); + } +} diff --git a/utility/source/utility/link_safe_remove.cpp b/utility/source/utility/link_safe_remove.cpp index b2ed6b2c..a1048b54 100644 --- a/utility/source/utility/link_safe_remove.cpp +++ b/utility/source/utility/link_safe_remove.cpp @@ -1,4 +1,9 @@ #include +#include +#include "no_follow_inspection.hpp" + +#include +#include #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN @@ -7,29 +12,165 @@ namespace Utility { - bool isLinkOrJunction(std::filesystem::path const& path, std::error_code& error) + namespace Detail { - error.clear(); + bool isLinkReparseTag(std::uint32_t tag) noexcept + { + // IsReparseTagNameSurrogate: the item stands for another named item, like a symlink or junction does. + constexpr std::uint32_t nameSurrogateBit = 0x20000000u; + return (tag & nameSurrogateBit) != 0; + } + #ifdef _WIN32 - const auto attributes = ::GetFileAttributesW(path.c_str()); - if (attributes == INVALID_FILE_ATTRIBUTES) + namespace + { + struct FindCloser + { + void operator()(HANDLE handle) const noexcept + { + if (handle != INVALID_HANDLE_VALUE) + ::FindClose(handle); + } + }; + using FindHandle = std::unique_ptr; + + bool isMissingError(DWORD lastError) + { + return lastError == ERROR_FILE_NOT_FOUND || lastError == ERROR_PATH_NOT_FOUND; + } + } + + NoFollowInspection inspectNoFollow(std::filesystem::path const& path, std::error_code& error) + { + error.clear(); + const auto attributes = ::GetFileAttributesW(path.c_str()); + if (attributes == INVALID_FILE_ATTRIBUTES) + { + const auto lastError = ::GetLastError(); + if (!isMissingError(lastError)) + error = std::error_code{static_cast(lastError), std::system_category()}; + return {}; + } + const bool directoryEntry = (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + if ((attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) + { + WIN32_FIND_DATAW data{}; + const FindHandle find{::FindFirstFileW(path.c_str(), &data)}; + if (find.get() == INVALID_HANDLE_VALUE) + { + const auto lastError = ::GetLastError(); + if (!isMissingError(lastError)) + error = std::error_code{static_cast(lastError), std::system_category()}; + return {}; + } + if (isLinkReparseTag(data.dwReserved0)) + return {.kind = NoFollowKind::Link, .directoryEntry = directoryEntry}; + } + return { + .kind = directoryEntry ? NoFollowKind::Directory : NoFollowKind::Other, + .directoryEntry = directoryEntry, + }; + } + + bool removeInspected(std::filesystem::path const& path, NoFollowInspection const& inspection, std::error_code& error) { - const auto lastError = ::GetLastError(); - if (lastError != ERROR_FILE_NOT_FOUND && lastError != ERROR_PATH_NOT_FOUND) - error = std::error_code{static_cast(lastError), std::system_category()}; + error.clear(); + if (inspection.kind == NoFollowKind::Missing) + return false; + // A junction or directory link is removed like an empty directory, its target stays. + const auto removeOnce = [&path, &inspection]() + { + return inspection.directoryEntry ? ::RemoveDirectoryW(path.c_str()) != 0 : ::DeleteFileW(path.c_str()) != 0; + }; + if (removeOnce()) + return true; + auto lastError = ::GetLastError(); + if (isMissingError(lastError)) + return false; + if (lastError == ERROR_ACCESS_DENIED && inspection.kind != NoFollowKind::Link) + { + const auto attributes = ::GetFileAttributesW(path.c_str()); + if (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_READONLY) != 0) + { + const auto writable = attributes & ~static_cast(FILE_ATTRIBUTE_READONLY); + if (::SetFileAttributesW(path.c_str(), writable == 0 ? FILE_ATTRIBUTE_NORMAL : writable) != 0) + { + if (removeOnce()) + return true; + lastError = ::GetLastError(); + ::SetFileAttributesW(path.c_str(), attributes); + } + } + } + error = std::error_code{static_cast(lastError), std::system_category()}; return false; } - return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0; #else - const auto status = std::filesystem::symlink_status(path, error); - if (status.type() == std::filesystem::file_type::not_found) + NoFollowInspection inspectionOf(std::filesystem::file_status const& status) + { + switch (status.type()) + { + case std::filesystem::file_type::none: + case std::filesystem::file_type::not_found: + return {}; + case std::filesystem::file_type::symlink: + return {.kind = NoFollowKind::Link}; + case std::filesystem::file_type::directory: + return {.kind = NoFollowKind::Directory, .directoryEntry = true}; + default: + return {.kind = NoFollowKind::Other}; + } + } + + NoFollowInspection inspectNoFollow(std::filesystem::path const& path, std::error_code& error) + { + const auto status = std::filesystem::symlink_status(path, error); + if (status.type() == std::filesystem::file_type::not_found) + error.clear(); + if (error) + return {}; + return inspectionOf(status); + } + + bool removeInspected(std::filesystem::path const& path, NoFollowInspection const& inspection, std::error_code& error) + { error.clear(); - return !error && std::filesystem::is_symlink(status); + if (inspection.kind == NoFollowKind::Missing) + return false; + // remove(3) unlinks a link itself and removes a directory only when it is empty. + return std::filesystem::remove(path, error); + } #endif } - std::optional - firstLinkBelow(std::filesystem::path const& root, std::filesystem::path const& path, std::error_code& error) + NoFollowKind noFollowKindOf(std::filesystem::path const& path, std::error_code& error) + { + return Detail::inspectNoFollow(path, error).kind; + } + + bool isLinkOrJunction(std::filesystem::path const& path, std::error_code& error) + { + return noFollowKindOf(path, error) == NoFollowKind::Link; + } + + bool VerifiedDirectories::contains(std::filesystem::path const& directory) const + { + const std::scoped_lock lock{mutex_}; + return directories_.contains(directory); + } + + void VerifiedDirectories::insert(std::filesystem::path const& directory) + { + const std::scoped_lock lock{mutex_}; + directories_.insert(directory); + } + + std::optional firstLinkBelow( + std::filesystem::path const& root, + std::filesystem::path const& path, + std::error_code& error, + VerifiedDirectories* verified + ) { error.clear(); const auto relative = path.lexically_relative(root); @@ -44,70 +185,49 @@ namespace Utility if (component == ".") continue; current /= component; - const bool isLink = isLinkOrJunction(current, error); + if (verified && verified->contains(current)) + continue; + const auto kind = noFollowKindOf(current, error); if (error) return std::nullopt; - if (isLink) - return current; + switch (kind) + { + case NoFollowKind::Link: + return current; + case NoFollowKind::Missing: + return std::nullopt; + case NoFollowKind::Directory: + { + if (verified) + verified->insert(current); + break; + } + case NoFollowKind::Other: + break; + } } return std::nullopt; } - namespace + bool removeNoFollow(std::filesystem::path const& path, std::error_code& error) { - bool removeLink(std::filesystem::path const& path, std::error_code& error) - { -#ifdef _WIN32 - // A junction or directory symlink is removed like an empty directory, its target stays. - const auto attributes = ::GetFileAttributesW(path.c_str()); - const bool removed = (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0 ? ::RemoveDirectoryW(path.c_str()) != 0 - : ::DeleteFileW(path.c_str()) != 0; - if (!removed) - error = std::error_code{static_cast(::GetLastError()), std::system_category()}; - return removed; -#else - return std::filesystem::remove(path, error); -#endif - } + const auto inspection = Detail::inspectNoFollow(path, error); + if (error) + return false; + return Detail::removeInspected(path, inspection, error); } - std::uintmax_t removeAllNoFollow(std::filesystem::path const& path, std::error_code& error) + RemoveAllResult removeAllNoFollow(std::filesystem::path const& path) { - error.clear(); - const bool isLink = isLinkOrJunction(path, error); - if (error) - return 0; - if (isLink) - return removeLink(path, error) ? 1u : 0u; - - const auto status = std::filesystem::symlink_status(path, error); - if (status.type() == std::filesystem::file_type::not_found) - { - error.clear(); - return 0; - } - if (error) - return 0; + NoFollowTreeWalk walk{path, NoFollowTreeWalk::Mode::Remove}; + walk.step(std::chrono::steady_clock::time_point::max()); + return {.removed = walk.entries(), .failures = walk.failures()}; + } - std::uintmax_t removed = 0; - if (std::filesystem::is_directory(status)) - { - std::filesystem::directory_iterator iter{path, error}; - if (error) - return removed; - for (; iter != std::filesystem::directory_iterator{}; iter.increment(error)) - { - if (error) - return removed; - removed += removeAllNoFollow(iter->path(), error); - if (error) - return removed; - } - if (error) - return removed; - } - if (std::filesystem::remove(path, error)) - ++removed; - return removed; + std::uintmax_t removeAllNoFollow(std::filesystem::path const& path, std::error_code& error) + { + auto result = removeAllNoFollow(path); + error = result.failures.empty() ? std::error_code{} : result.failures.front().error; + return result.removed; } } diff --git a/utility/source/utility/no_follow_inspection.hpp b/utility/source/utility/no_follow_inspection.hpp new file mode 100644 index 00000000..336c142d --- /dev/null +++ b/utility/source/utility/no_follow_inspection.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include +#include + +namespace Utility::Detail +{ + /** + * @brief What an item is, inspected without following it, with what removing it needs to know. + */ + struct NoFollowInspection + { + NoFollowKind kind{NoFollowKind::Missing}; + /** @brief The item has the directory attribute, which on Windows includes directory links and junctions. */ + bool directoryEntry{false}; + }; + + /** + * @brief Inspects @p path without following it, a missing path is no error. + */ + NoFollowInspection inspectNoFollow(std::filesystem::path const& path, std::error_code& error); + +#ifndef _WIN32 + /** + * @brief The inspection a symlink_status gives, which on POSIX is all there is to know. + */ + NoFollowInspection inspectionOf(std::filesystem::file_status const& status); +#endif + + /** + * @brief Removes the item at @p path that @p inspection describes, never following it. + * @return True when it was removed, false when it was gone or @p error is set. + */ + bool removeInspected(std::filesystem::path const& path, NoFollowInspection const& inspection, std::error_code& error); +} diff --git a/utility/source/utility/no_follow_tree_walk.cpp b/utility/source/utility/no_follow_tree_walk.cpp new file mode 100644 index 00000000..5e807576 --- /dev/null +++ b/utility/source/utility/no_follow_tree_walk.cpp @@ -0,0 +1,172 @@ +#include +#include "no_follow_inspection.hpp" + +#include + +namespace Utility +{ + namespace + { + Detail::NoFollowInspection inspectEntry(std::filesystem::directory_entry const& entry, std::error_code& error) + { +#ifdef _WIN32 + // The cached type does not tell a junction from a directory, the reparse tag does. + return Detail::inspectNoFollow(entry.path(), error); +#else + const auto status = entry.symlink_status(error); + if (status.type() == std::filesystem::file_type::not_found) + error.clear(); + if (error) + return {}; + return Detail::inspectionOf(status); +#endif + } + } + + NoFollowTreeWalk::NoFollowTreeWalk(std::filesystem::path root, Mode mode) + : root_{std::move(root)} + , mode_{mode} + {} + + bool NoFollowTreeWalk::step(std::chrono::steady_clock::time_point deadline) + { + if (done_) + return true; + if (!started_) + { + started_ = true; + start(); + } + do + { + if (stack_.empty()) + { + done_ = true; + return true; + } + auto& frame = stack_.back(); + if (frame.iterator == std::filesystem::directory_iterator{}) + { + finishDirectory(); + continue; + } + const auto entry = *frame.iterator; + std::error_code error{}; + frame.iterator.increment(error); + if (error) + { + fail(frame.directory, error); + frame.iterator = std::filesystem::directory_iterator{}; + } + handleEntry(entry); + } while (std::chrono::steady_clock::now() < deadline); + return false; + } + + void NoFollowTreeWalk::start() + { + std::error_code error{}; + const auto inspection = Detail::inspectNoFollow(root_, error); + if (error) + return fail(root_, error); + switch (inspection.kind) + { + case NoFollowKind::Missing: + return; + case NoFollowKind::Directory: + return enter(root_); + case NoFollowKind::Link: + case NoFollowKind::Other: + { + const auto size = mode_ == Mode::Count && inspection.kind == NoFollowKind::Other + ? std::filesystem::file_size(root_, error) + : std::uintmax_t{0}; + return finishItem(root_, inspection.kind, inspection.directoryEntry, error ? 0 : size); + } + } + } + + void NoFollowTreeWalk::handleEntry(std::filesystem::directory_entry const& entry) + { + std::error_code error{}; + const auto inspection = inspectEntry(entry, error); + if (error) + return fail(entry.path(), error); + switch (inspection.kind) + { + case NoFollowKind::Missing: + return; + case NoFollowKind::Directory: + return enter(entry.path()); + case NoFollowKind::Link: + case NoFollowKind::Other: + { + const bool countsBytes = mode_ == Mode::Count && inspection.kind == NoFollowKind::Other && + entry.is_regular_file(error); + const auto size = countsBytes ? entry.file_size(error) : std::uintmax_t{0}; + return finishItem(entry.path(), inspection.kind, inspection.directoryEntry, error ? 0 : size); + } + } + } + + void NoFollowTreeWalk::enter(std::filesystem::path const& directory) + { + lastPath_ = directory; + std::error_code error{}; + std::filesystem::directory_iterator iterator{directory, error}; + if (error) + { + if (mode_ == Mode::Count) + { + ++entries_; + return; + } + return fail(directory, error); + } + stack_.push_back(Frame{.directory = directory, .iterator = std::move(iterator)}); + } + + void NoFollowTreeWalk::finishDirectory() + { + auto frame = std::move(stack_.back()); + stack_.pop_back(); + if (frame.keepsContent) + { + if (!stack_.empty()) + stack_.back().keepsContent = true; + return; + } + finishItem(frame.directory, NoFollowKind::Directory, true, 0); + } + + void NoFollowTreeWalk::finishItem( + std::filesystem::path const& path, + NoFollowKind kind, + bool directoryEntry, + std::uintmax_t size + ) + { + lastPath_ = path; + if (mode_ == Mode::Count) + { + ++entries_; + bytes_ += size; + return; + } + std::error_code error{}; + if (Detail::removeInspected(path, {.kind = kind, .directoryEntry = directoryEntry}, error)) + ++entries_; + else if (error) + fail(path, error); + } + + void NoFollowTreeWalk::fail(std::filesystem::path const& path, std::error_code const& error) + { + lastPath_ = path; + if (mode_ == Mode::Count) + return; + failures_.push_back({.path = path, .error = error}); + if (!stack_.empty()) + stack_.back().keepsContent = true; + } +} diff --git a/utility/source/utility/node/node.cpp b/utility/source/utility/node/node.cpp index 56cd6560..0aa69c04 100644 --- a/utility/source/utility/node/node.cpp +++ b/utility/source/utility/node/node.cpp @@ -1,4 +1,6 @@ #include +#include +#include #pragma clang diagnostic push #pragma clang diagnostic ignored "-Weverything" @@ -45,7 +47,7 @@ namespace SecureShell::Test boost::asio::write(stdinPipe, boost::asio::buffer(commandWithNewline), ec); if (ec) { - std::cerr << "Failed to write command: " << ec.message() << std::endl; + std::cerr << "Failed to write command: " << Utility::errorMessage(ec) << std::endl; return; } // Node on Windows cannot finish exiting while a read on its stdin pipe is pending. @@ -71,7 +73,7 @@ namespace SecureShell::Test boost::asio::write(stdinPipe, boost::asio::buffer(commandWithNewline), ec); if (ec) { - std::cerr << "Failed to write command: " << ec.message() << std::endl; + std::cerr << "Failed to write command: " << Utility::errorMessage(ec) << std::endl; return false; } @@ -82,7 +84,7 @@ namespace SecureShell::Test const auto readAmount = stdoutPipe.read_some(boost::asio::buffer(buffer), ec); if (ec || readAmount == 0) { - std::cerr << "Failed to read command acknowledgement: " << ec.message() << std::endl; + std::cerr << "Failed to read command acknowledgement: " << Utility::errorMessage(ec) << std::endl; return false; } received.append(buffer.data(), readAmount); @@ -103,7 +105,7 @@ namespace SecureShell::Test const int exitCode = process(*this)->wait(errc); if (errc && errc != boost::system::errc::no_child_process) { - std::cerr << "node process wait failed: " << errc.message() << std::endl; + std::cerr << "node process wait failed: " << Utility::errorMessage(errc) << std::endl; } return exitCode; } @@ -119,7 +121,7 @@ namespace SecureShell::Test } catch (std::exception const& e) { - std::cerr << "Failed to terminate process: " << e.what() << std::endl; + std::cerr << "Failed to terminate process: " << Utility::errorMessage(e) << std::endl; } } @@ -134,7 +136,7 @@ namespace SecureShell::Test packageJsonFile << packageJson.dump(4); } - const auto command = "\"cd "s + directory.generic_string() + " && npm install\""; + const auto command = fmt::format("\"cd {} && npm install\"", Paths::nativeToUtf8(directory)); #ifdef _WIN32 const auto shell = MSYS2_BASH; const auto npmShellArgs = std::vector{ @@ -156,7 +158,7 @@ namespace SecureShell::Test shell, npmShellArgs, bp2::process_environment{bp2::environment::current()}, - bp2::process_start_dir{directory.generic_string()}}; + bp2::process_start_dir{directory.native()}}; child.wait(); } @@ -201,7 +203,7 @@ namespace SecureShell::Test node, std::vector{"main.mjs", "--log-file=./log.txt"}, bp2::process_environment{bp2::environment::current()}, - bp2::process_start_dir{isolateDirectory.path().generic_string()}, + bp2::process_start_dir{isolateDirectory.path().native()}, bp2::process_stdio{ .in = result->stdinPipe, .out = result->stdoutPipe, @@ -223,7 +225,7 @@ namespace SecureShell::Test } catch (std::exception const& e) { - std::cerr << "Failed to terminate process: " << e.what() << std::endl; + std::cerr << "Failed to terminate process: " << Utility::errorMessage(e) << std::endl; } } }); @@ -236,7 +238,7 @@ namespace SecureShell::Test int readAmount = result->stdoutPipe.read_some(boost::asio::buffer(buffer), ec); if (ec) { - std::cerr << "Failed to read port: " << ec.message() << std::endl; + std::cerr << "Failed to read port: " << Utility::errorMessage(ec) << std::endl; return result; } if (readAmount == 0) @@ -249,7 +251,7 @@ namespace SecureShell::Test } catch (std::exception const& e) { - std::cerr << "Failed to parse port object: " << e.what() << std::endl; + std::cerr << "Failed to parse port object: " << Utility::errorMessage(e) << std::endl; return result; } diff --git a/utility/source/utility/permissions.cpp b/utility/source/utility/permissions.cpp index 956c6d12..dce93265 100644 --- a/utility/source/utility/permissions.cpp +++ b/utility/source/utility/permissions.cpp @@ -4,7 +4,10 @@ #include #ifndef _WIN32 +# include +# include # include +# include #endif namespace Utility @@ -31,6 +34,33 @@ namespace Utility return static_cast(mask); #endif } + +#ifndef _WIN32 + class UniqueDescriptor + { + public: + explicit UniqueDescriptor(int descriptor) + : descriptor_{descriptor} + {} + ~UniqueDescriptor() + { + if (descriptor_ >= 0) + ::close(descriptor_); + } + UniqueDescriptor(UniqueDescriptor const&) = delete; + UniqueDescriptor& operator=(UniqueDescriptor const&) = delete; + UniqueDescriptor(UniqueDescriptor&&) = delete; + UniqueDescriptor& operator=(UniqueDescriptor&&) = delete; + + int get() const + { + return descriptor_; + } + + private: + int descriptor_; + }; +#endif } std::optional sanitizePermissions(std::filesystem::perms permissions) @@ -54,6 +84,31 @@ namespace Utility return *sanitized & ~processUmask(); } + std::filesystem::perms newFilePermissions() + { + constexpr auto readWrite = std::filesystem::perms::owner_read | std::filesystem::perms::owner_write | + std::filesystem::perms::group_read | std::filesystem::perms::group_write | + std::filesystem::perms::others_read | std::filesystem::perms::others_write; + return readWrite & ~processUmask(); + } + + std::expected restrictToOwner(std::filesystem::path const& path) + { +#ifdef _WIN32 + static_cast(path); + return {}; +#else + const UniqueDescriptor descriptor{ + ::open(path.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC | O_NOFOLLOW, S_IRUSR | S_IWUSR) + }; + if (descriptor.get() < 0) + return std::unexpected(std::error_code{errno, std::generic_category()}); + if (::fchmod(descriptor.get(), S_IRUSR | S_IWUSR) != 0) + return std::unexpected(std::error_code{errno, std::generic_category()}); + return {}; +#endif + } + std::filesystem::perms localPermissions(std::filesystem::file_status const& status) { #ifdef _WIN32 diff --git a/utility/source/utility/rename.cpp b/utility/source/utility/rename.cpp index 184d5c59..c8b3b22b 100644 --- a/utility/source/utility/rename.cpp +++ b/utility/source/utility/rename.cpp @@ -13,9 +13,54 @@ namespace Utility { namespace { - std::unexpected lastErrno() +#ifndef _WIN32 + std::error_code lastErrno() { - return std::unexpected(std::error_code{errno, std::generic_category()}); + return std::error_code{errno, std::generic_category()}; + } +#endif + } + + namespace Detail + { + bool isNoReplaceUnsupported(std::error_code error) + { + return error == std::errc::invalid_argument || error == std::errc::function_not_supported || + error == std::errc::operation_not_supported || error == std::errc::not_supported || + error == std::errc::operation_not_permitted; + } + + std::expected + renameByLink(std::filesystem::path const& from, std::filesystem::path const& to) + { + std::error_code error{}; + std::filesystem::create_hard_link(from, to, error); + if (error) + return std::unexpected(error); + std::filesystem::remove(from, error); + if (error) + { + std::error_code undoError{}; + std::filesystem::remove(to, undoError); + return std::unexpected(error); + } + return {}; + } + + std::expected + renameUnlessExists(std::filesystem::path const& from, std::filesystem::path const& to) + { + std::error_code error{}; + const auto status = std::filesystem::symlink_status(to, error); + if (status.type() == std::filesystem::file_type::none) + return std::unexpected(error); + if (std::filesystem::exists(status)) + return std::unexpected(std::make_error_code(std::errc::file_exists)); + error.clear(); + std::filesystem::rename(from, to, error); + if (error) + return std::unexpected(error); + return {}; } } @@ -31,30 +76,37 @@ namespace Utility } return {}; #elif defined(__APPLE__) - if (::renamex_np(from.c_str(), to.c_str(), RENAME_EXCL) != 0) - return lastErrno(); - return {}; + if (::renamex_np(from.c_str(), to.c_str(), RENAME_EXCL) == 0) + return {}; + if (const auto error = lastErrno(); !Detail::isNoReplaceUnsupported(error)) + return std::unexpected(error); + return Detail::renameUnlessExists(from, to); #else if (::renameat2(AT_FDCWD, from.c_str(), AT_FDCWD, to.c_str(), RENAME_NOREPLACE) == 0) return {}; - if (errno != EINVAL && errno != ENOSYS) - return lastErrno(); + if (const auto error = lastErrno(); !Detail::isNoReplaceUnsupported(error)) + return std::unexpected(error); - // The filesystem cannot rename without replacing. A hard link fails on an existing target as well. std::error_code statusError{}; if (std::filesystem::is_regular_file(std::filesystem::symlink_status(from, statusError))) { - if (::link(from.c_str(), to.c_str()) != 0) - return lastErrno(); - if (::unlink(from.c_str()) != 0) - return lastErrno(); - return {}; + const auto linked = Detail::renameByLink(from, to); + if (linked || !Detail::isNoReplaceUnsupported(linked.error())) + return linked; } - if (std::filesystem::exists(std::filesystem::symlink_status(to, statusError))) - return std::unexpected(std::make_error_code(std::errc::file_exists)); - if (::rename(from.c_str(), to.c_str()) != 0) - return lastErrno(); - return {}; + return Detail::renameUnlessExists(from, to); #endif } + + std::expected + moveIntoPlace(std::filesystem::path const& from, std::filesystem::path const& to, bool replace) + { + if (!replace) + return renameNoReplace(from, to); + std::error_code error{}; + std::filesystem::rename(from, to, error); + if (error) + return std::unexpected(error); + return {}; + } } diff --git a/utility/source/utility/resources.cpp b/utility/source/utility/resources.cpp index f50960ae..40733422 100644 --- a/utility/source/utility/resources.cpp +++ b/utility/source/utility/resources.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #ifndef NDEBUG # include #endif @@ -59,10 +60,11 @@ namespace const auto& seg = segments[index]; if (!hasGlobChars(seg)) { - const auto next = currentDir / seg; + const auto nativeSegment = Paths::nativeFromUtf8(seg); + const auto next = currentDir / nativeSegment; std::error_code ec; if (std::filesystem::exists(next, ec)) - matchPatternInDirImpl(next, segments, index + 1, relative / seg, results); + matchPatternInDirImpl(next, segments, index + 1, relative / nativeSegment, results); } else { @@ -71,9 +73,11 @@ namespace return; for (const auto& entry : std::filesystem::directory_iterator(currentDir, ec)) { - const auto name = entry.path().filename().string(); + const auto name = Paths::nativeToUtf8(entry.path().filename()); if (segmentMatchesPattern(name, seg)) - matchPatternInDirImpl(entry.path(), segments, index + 1, relative / name, results); + matchPatternInDirImpl( + entry.path(), segments, index + 1, relative / entry.path().filename(), results + ); } } } @@ -83,8 +87,8 @@ namespace std::vector searchPaths; #ifndef NDEBUG - searchPaths.push_back(std::filesystem::path{SOURCE_DIR}); - searchPaths.push_back(std::filesystem::path{SOURCE_DIR} / "static"); + searchPaths.push_back(Paths::nativeFromUtf8(SOURCE_DIR)); + searchPaths.push_back(Paths::nativeFromUtf8(SOURCE_DIR) / "static"); #endif searchPaths.push_back(relativeRoot); searchPaths.push_back( @@ -161,7 +165,7 @@ bool pointsToWithinDir(std::filesystem::path const& relativeRoot, std::filesyste { // make absolute and check if it starts with relativeRoot const auto absPath = std::filesystem::absolute(path); - return absPath.string().starts_with(relativeRoot.string()); + return Paths::nativeToUtf8(absPath).starts_with(Paths::nativeToUtf8(relativeRoot)); } std::vector getThemeDirs(std::filesystem::path const& relativeRoot) @@ -175,12 +179,12 @@ matchPatternInDir(std::filesystem::path const& baseDir, std::filesystem::path co std::vector segments; for (auto const& part : relativePattern) { - const auto s = part.string(); + const auto s = Paths::nativeToUtf8(part); if (!s.empty() && s != "." && s != "/" && s != "\\") segments.push_back(s); } std::vector results; - matchPatternInDirImpl(baseDir.generic_string(), segments, 0, {}, results); + matchPatternInDirImpl(baseDir, segments, 0, {}, results); return results; } @@ -195,7 +199,7 @@ std::vector findFilesInSearchPaths( { for (auto const& relPath : matchPatternInDir(searchPath, relativePattern)) { - if (seen.insert(relPath.generic_string()).second) + if (seen.insert(Paths::nativeToUtf8(relPath)).second) result.push_back(searchPath / relPath); } } @@ -231,7 +235,7 @@ mapUrlToFile(std::filesystem::path const& resourceDir, std::string const& urlPat const auto path = [&]() -> std::optional { // make path relative to / to avoid directory traversal - const auto relative = std::filesystem::relative(urlPathString, "/"); + const auto relative = std::filesystem::relative(Paths::nativeFromUtf8(urlPathString), "/"); if (endsWith(".css") && relative.parent_path().filename() == "themes") return firstOf(findFilesInSearchPaths(getThemeDirs(resourceDir), relative.filename())); diff --git a/utility/source/utility/shell_integration.cpp b/utility/source/utility/shell_integration.cpp index 3e6907c0..09561ae0 100644 --- a/utility/source/utility/shell_integration.cpp +++ b/utility/source/utility/shell_integration.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -99,7 +100,7 @@ namespace ShellIntegration ShellKind detectShellKind(std::filesystem::path const& command) { - auto name = command.filename().string(); + auto name = Paths::nativeToUtf8(command.filename()); std::ranges::transform(name, name.begin(), [](char character) { return static_cast(std::tolower(static_cast(character))); }); diff --git a/utility/source/utility/typed_line_buffer.cpp b/utility/source/utility/typed_line_buffer.cpp index c9426f75..165c0ada 100644 --- a/utility/source/utility/typed_line_buffer.cpp +++ b/utility/source/utility/typed_line_buffer.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include @@ -56,11 +58,8 @@ namespace Utility case '\x7f': case '\b': { - // A multibyte character goes as a whole: its continuation bytes, then its lead. - while (!line_.empty() && (static_cast(line_.back()) & 0xC0) == 0x80) - line_.pop_back(); if (!line_.empty()) - line_.pop_back(); + line_.resize(Paths::truncatedUtf8(line_, line_.size() - 1u).size()); continue; } case '\x03': // Ctrl-C diff --git a/utility/source/utility/utf8.cpp b/utility/source/utility/utf8.cpp index 099bc9d0..70be7489 100644 --- a/utility/source/utility/utf8.cpp +++ b/utility/source/utility/utf8.cpp @@ -1,84 +1,20 @@ #include -#include +#include namespace Utility::Utf8 { - namespace - { - constexpr char32_t replacementCharacter = 0xFFFDu; - - bool isContinuationByte(unsigned char byte) - { - return (byte & 0xC0u) == 0x80u; - } - - bool isScalarValue(char32_t codePoint) - { - return codePoint <= 0x10FFFFu && (codePoint < 0xD800u || codePoint > 0xDFFFu); - } - } - std::u32string decode(std::string_view utf8) { std::u32string result; result.reserve(utf8.size()); - - std::size_t position = 0; - while (position < utf8.size()) - { - const auto leadByte = static_cast(utf8[position]); - if (leadByte < 0x80u) - { - result.push_back(leadByte); - ++position; - continue; - } - - std::size_t length = 0; - char32_t codePoint = 0; - char32_t minimum = 0; - if ((leadByte & 0xE0u) == 0xC0u) - { - length = 2; - codePoint = leadByte & 0x1Fu; - minimum = 0x80u; - } - else if ((leadByte & 0xF0u) == 0xE0u) - { - length = 3; - codePoint = leadByte & 0x0Fu; - minimum = 0x800u; - } - else if ((leadByte & 0xF8u) == 0xF0u) - { - length = 4; - codePoint = leadByte & 0x07u; - minimum = 0x10000u; - } - else + Paths::Detail::forEachUtf8CodePoint( + utf8, + [&result](char32_t codePoint) { - result.push_back(replacementCharacter); - ++position; - continue; - } - - std::size_t consumed = 1; - while ( - consumed < length && position + consumed < utf8.size() && - isContinuationByte(static_cast(utf8[position + consumed])) - ) - { - codePoint = (codePoint << 6u) | (static_cast(utf8[position + consumed]) & 0x3Fu); - ++consumed; - } - - if (consumed != length || codePoint < minimum || !isScalarValue(codePoint)) - result.push_back(replacementCharacter); - else result.push_back(codePoint); - position += consumed; - } + } + ); return result; } @@ -86,35 +22,8 @@ namespace Utility::Utf8 { std::string result; result.reserve(codePoints.size()); - - for (auto codePoint : codePoints) - { - if (!isScalarValue(codePoint)) - codePoint = replacementCharacter; - - if (codePoint < 0x80u) - { - result.push_back(static_cast(codePoint)); - } - else if (codePoint < 0x800u) - { - result.push_back(static_cast(0xC0u | (codePoint >> 6u))); - result.push_back(static_cast(0x80u | (codePoint & 0x3Fu))); - } - else if (codePoint < 0x10000u) - { - result.push_back(static_cast(0xE0u | (codePoint >> 12u))); - result.push_back(static_cast(0x80u | ((codePoint >> 6u) & 0x3Fu))); - result.push_back(static_cast(0x80u | (codePoint & 0x3Fu))); - } - else - { - result.push_back(static_cast(0xF0u | (codePoint >> 18u))); - result.push_back(static_cast(0x80u | ((codePoint >> 12u) & 0x3Fu))); - result.push_back(static_cast(0x80u | ((codePoint >> 6u) & 0x3Fu))); - result.push_back(static_cast(0x80u | (codePoint & 0x3Fu))); - } - } + for (const auto codePoint : codePoints) + Paths::Detail::appendUtf8(result, codePoint); return result; } } diff --git a/utility/test/utility/main.cpp b/utility/test/utility/main.cpp index 445e8490..d3195ad3 100644 --- a/utility/test/utility/main.cpp +++ b/utility/test/utility/main.cpp @@ -1,9 +1,13 @@ #include +#include "test_case_convert.hpp" #include "test_command_template.hpp" #include "test_directory_traversal.hpp" #include "test_echo_suppressor.hpp" +#include "test_error_message.hpp" #include "test_fuzzy_search.hpp" +#include "test_json_utf8.hpp" #include "test_keyed_diff.hpp" +#include "test_lifetime_token.hpp" #include "test_localized_message.hpp" #include "test_move_to_front.hpp" #include "test_permissions.hpp" diff --git a/utility/test/utility/test_case_convert.hpp b/utility/test/utility/test_case_convert.hpp new file mode 100644 index 00000000..e17c7765 --- /dev/null +++ b/utility/test/utility/test_case_convert.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include + +#include + +namespace Utility::Test +{ + TEST(CaseConvertTests, OnlyAsciiLettersChangeCase) + { + EXPECT_EQ(Algorithm::toLowerCase("ÄBC Größe"), "Äbc größe"); + EXPECT_EQ(Algorithm::toUpperCase("äbc größe"), "äBC GRößE"); + std::string text = "\xC3\x84X\xFF"; + Algorithm::toLowerCaseInplace(text); + EXPECT_EQ(text, "\xC3\x84x\xFF"); + } + + TEST(CaseConvertTests, CompareIgnoresAsciiCaseOnly) + { + EXPECT_EQ(Algorithm::caseInsensitiveCompare("ABC", "abc"), 0); + EXPECT_NE(Algorithm::caseInsensitiveCompare("Ä", "ä"), 0); + EXPECT_LT(Algorithm::caseInsensitiveCompare("a", "\xC3\xA4"), 0); + } + + TEST(CaseConvertTests, NamingConventionsKeepUtf8Intact) + { + const SegmentedString segments{.normalizedSegments = {"größe", "übung"}}; + EXPECT_EQ(segments.camelCase(), "größeübung"); + EXPECT_EQ(segments.screamingSnakeCase(), "größe_üBUNG"); + } +} diff --git a/utility/test/utility/test_directory_traversal.hpp b/utility/test/utility/test_directory_traversal.hpp index 76e666d2..13fb24b4 100644 --- a/utility/test/utility/test_directory_traversal.hpp +++ b/utility/test/utility/test_directory_traversal.hpp @@ -20,7 +20,6 @@ namespace Utility::Test class DirectoryTraversalTests : public ::testing::Test { public: - template auto withWalkerDo(auto&& fn) { auto scan = [this](Paths::RelativePath const& relative) @@ -30,8 +29,7 @@ namespace Utility::Test else return std::unexpected("No Scanner Installed In Test"s); }; - using WalkerType = - DeepDirectoryWalker; + using WalkerType = DeepDirectoryWalker; if (!walker_) walker_ = std::make_unique(std::move(scan)); return fn(static_cast(*walker_)); @@ -347,9 +345,15 @@ namespace Utility::Test const auto walkResult = withWalkerDo([this, &paths](auto& walker) { auto res = walker.walkAll(); auto const& entries = walker.entries(); - std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [this, &walker](auto const& entry) { - return absolutePath(walker.relativePath(entry)); - }); + std::transform( + entries.begin(), + entries.end(), + std::back_inserter(paths), + [this, &walker](auto const& entry) + { + return absolutePath(*walker.relativePath(entry)); + } + ); return res; }); @@ -431,14 +435,23 @@ namespace Utility::Test }; std::vector paths{}; - const auto walkResult = withWalkerDo([this, &paths](auto& walker) { - auto res = walker.walk(); - auto const& entries = walker.entries(); - std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [this, &walker](auto const& entry) { - return absolutePath(walker.relativePath(entry)); - }); - return res; - }); + const auto walkResult = withWalkerDo( + [this, &paths](auto& walker) + { + auto res = walker.walk(); + auto const& entries = walker.entries(); + std::transform( + entries.begin(), + entries.end(), + std::back_inserter(paths), + [this, &walker](auto const& entry) + { + return absolutePath(*walker.relativePath(entry)); + } + ); + return res; + } + ); ASSERT_TRUE(walkResult.has_value()) << "Error during walk: " << (walkResult.has_value() ? "" : walkResult.error()); @@ -488,15 +501,24 @@ namespace Utility::Test auto doTheWalk = [this, &paths]() { paths.clear(); - const auto walkResult = withWalkerDo([this, &paths](auto& walker) { - walker.reset(); - auto res = walker.walk(); - auto const& entries = walker.entries(); - std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [this, &walker](auto const& entry) { - return absolutePath(walker.relativePath(entry)); - }); - return res; - }); + const auto walkResult = withWalkerDo( + [this, &paths](auto& walker) + { + walker.reset(); + auto res = walker.walk(); + auto const& entries = walker.entries(); + std::transform( + entries.begin(), + entries.end(), + std::back_inserter(paths), + [this, &walker](auto const& entry) + { + return absolutePath(*walker.relativePath(entry)); + } + ); + return res; + } + ); ASSERT_TRUE(walkResult.has_value()) << "Error during walk: " << (walkResult.has_value() ? "" : walkResult.error()); }; @@ -522,4 +544,73 @@ namespace Utility::Test doTheWalk(); EXPECT_THAT(paths, ::testing::UnorderedElementsAreArray(expectedPaths)); } -} \ No newline at end of file + + namespace + { + std::vector smallTree() + { + using SharedData::DirectoryEntry; + return { + DirectoryEntry{.type = DirectoryEntry::FileType::Directory}, + DirectoryEntry{.name = "sub", .type = DirectoryEntry::FileType::Directory, .parent = 0}, + DirectoryEntry{.name = "file.txt", .type = DirectoryEntry::FileType::Regular, .parent = 1}, + DirectoryEntry{.name = "a/b", .type = DirectoryEntry::FileType::Regular, .parent = 1}, + DirectoryEntry{.name = "lost.txt", .type = DirectoryEntry::FileType::Regular, .parent = 9}, + }; + } + } + + TEST(WalkedEntryTests, RelativePathsFollowTheParents) + { + const auto entries = smallTree(); + const auto root = walkedRelativePath(entries, entries[0]); + ASSERT_TRUE(root.has_value()); + EXPECT_TRUE(root->empty()); + const auto file = walkedRelativePath(entries, entries[2]); + ASSERT_TRUE(file.has_value()) << file.error().toString(); + EXPECT_EQ(file->value(), "sub/file.txt"); + } + + TEST(WalkedEntryTests, RelativePathsRefuseInvalidNamesAndUnknownParentsWithoutThrowing) + { + const auto entries = smallTree(); + const auto invalid = walkedRelativePath(entries, entries[3]); + ASSERT_FALSE(invalid.has_value()); + EXPECT_EQ(invalid.error().reason, Paths::PathRejection::ContainsSeparator); + const auto lost = walkedRelativePath(entries, entries[4]); + ASSERT_FALSE(lost.has_value()); + EXPECT_EQ(lost.error().reason, Paths::PathRejection::UnknownParent); + const auto shared = SharedData::relativePath(entries, entries[4]); + ASSERT_FALSE(shared.has_value()); + EXPECT_EQ(shared.error().reason, Paths::PathRejection::UnknownParent); + } + + TEST(WalkedEntryTests, DescriptionsUseTheNamesAsListed) + { + const auto entries = smallTree(); + EXPECT_EQ(describeWalkedEntry("/home/test", entries, entries[3]), "/home/test/sub/a/b"); + EXPECT_EQ(describeWalkedEntry("/", entries, entries[2]), "/sub/file.txt"); + EXPECT_EQ(describeWalkedEntry("", entries, entries[2]), "sub/file.txt"); + EXPECT_EQ(describeWalkedEntry("/home/test", entries, entries[0]), "/home/test"); + EXPECT_EQ(describeWalkedEntry("/home/test", entries, entries[4]), "/home/test/lost.txt"); + } + + TEST_F(DirectoryTraversalTests, ACompletedWalkReportsItsEnd) + { + createSomeFilesIn(isolateDirectory_.path(), 2); + makeDefaultScan(); + withWalkerDo( + [](auto& walker) + { + ASSERT_TRUE(walker.walkAll().has_value()); + ASSERT_TRUE(walker.completed()); + const auto again = walker.walk(); + ASSERT_TRUE(again.has_value()); + EXPECT_TRUE(*again) << "a walk that is done says so"; + EXPECT_EQ(walker.entries().size(), 3u); + for (auto const& entry : walker.entries()) + EXPECT_TRUE(walker.relativePath(entry).has_value()) << entry.name; + } + ); + } +} diff --git a/utility/test/utility/test_error_message.hpp b/utility/test/utility/test_error_message.hpp new file mode 100644 index 00000000..5e78e642 --- /dev/null +++ b/utility/test/utility/test_error_message.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace Utility::Test +{ + TEST(ErrorMessageTests, GenericErrorsKeepTheirMessage) + { + const auto error = std::make_error_code(std::errc::no_such_file_or_directory); + EXPECT_EQ(errorMessage(error), Paths::sanitizedUtf8(error.message())); + EXPECT_TRUE(Paths::isValidUtf8(errorMessage(error))); + } + + TEST(ErrorMessageTests, SystemErrorsAreUtf8) + { + const std::error_code error{5, std::system_category()}; + const auto message = errorMessage(error); + EXPECT_FALSE(message.empty()); + EXPECT_TRUE(Paths::isValidUtf8(message)); + } + + TEST(ErrorMessageTests, FilesystemErrorsNameTheirPathsAsUtf8) + { + const auto code = std::make_error_code(std::errc::permission_denied); + const std::filesystem::filesystem_error error{ + "remove", Paths::nativeFromUtf8("/x/Grüße"), Paths::nativeFromUtf8("/y/東京"), code + }; + EXPECT_EQ(errorMessage(error), fmt::format("{}: '/x/Grüße', '/y/東京'", errorMessage(code))); + + const std::filesystem::filesystem_error single{"remove", Paths::nativeFromUtf8("/x/a"), code}; + EXPECT_EQ(errorMessage(single), fmt::format("{}: '/x/a'", errorMessage(code))); + + const std::filesystem::filesystem_error none{"remove", code}; + EXPECT_EQ(errorMessage(none), errorMessage(code)); + } + + TEST(ErrorMessageTests, ExceptionsAreUtf8) + { + const std::runtime_error plain{"bad \xFF"}; + EXPECT_EQ(errorMessage(plain), "bad \xEF\xBF\xBD"); + + const auto code = std::make_error_code(std::errc::permission_denied); + const std::filesystem::filesystem_error filesystemError{"remove", Paths::nativeFromUtf8("/x/ü"), code}; + std::exception const& asException = filesystemError; + EXPECT_EQ(errorMessage(asException), errorMessage(filesystemError)); + } + +#ifndef _WIN32 + TEST(ErrorMessageTests, FilesystemErrorsReplaceBytesThatAreNotUtf8) + { + const std::filesystem::filesystem_error error{ + "remove", std::filesystem::path{std::string{"/x/Fran\xE7ois"}}, std::make_error_code(std::errc::io_error) + }; + const auto message = errorMessage(error); + EXPECT_TRUE(Paths::isValidUtf8(message)); + EXPECT_NE(message.find("/x/Fran\xEF\xBF\xBDois"), std::string::npos); + } +#endif +} diff --git a/utility/test/utility/test_json_utf8.hpp b/utility/test/utility/test_json_utf8.hpp new file mode 100644 index 00000000..c09c0eeb --- /dev/null +++ b/utility/test/utility/test_json_utf8.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace Utility::Test +{ + TEST(JsonUtf8Tests, StringsAndKeysThatAreNotUtf8BecomeDumpable) + { + const nlohmann::json json{ + {"name", "a\xFF"}, + {"list", {"ok", "b\xC3"}}, + {"nested", {{"c\xFE", 1}}}, + {"number", 42}, + }; + EXPECT_THROW(std::ignore = json.dump(), nlohmann::json::type_error); + + const auto valid = withValidUtf8(json); + EXPECT_NO_THROW(std::ignore = valid.dump()); + EXPECT_EQ(valid["name"], "a\xEF\xBF\xBD"); + EXPECT_EQ(valid["list"][0], "ok"); + EXPECT_EQ(valid["list"][1], "b\xEF\xBF\xBD"); + EXPECT_EQ(valid["nested"]["c\xEF\xBF\xBD"], 1); + EXPECT_EQ(valid["number"], 42); + } + + TEST(JsonUtf8Tests, ValidJsonStaysTheSame) + { + const nlohmann::json json{{"name", "Grüße"}, {"values", {1, 2, 3}}, {"flag", true}}; + EXPECT_EQ(withValidUtf8(json), json); + } +} diff --git a/utility/test/utility/test_lifetime_token.hpp b/utility/test/utility/test_lifetime_token.hpp new file mode 100644 index 00000000..c5135439 --- /dev/null +++ b/utility/test/utility/test_lifetime_token.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace Utility::Test +{ + TEST(LifetimeTokenTests, AGuardedCallbackRunsOnlyWhileItsTokenExists) + { + int calls = 0; + std::function callback; + { + const LifetimeToken token; + callback = token.guarded( + [&calls](int amount) + { + calls += amount; + } + ); + callback(2); + EXPECT_EQ(calls, 2); + using Guarded = decltype(token.guarded([](int) {})); + static_assert(std::is_invocable_v); + static_assert(!std::is_invocable_v, "the wrapper keeps the parameter types"); + } + callback(5); + EXPECT_EQ(calls, 2) << "the owner is gone"; + } + + TEST(LifetimeTokenTests, ACopiedOrMovedOwnerHasATokenOfItsOwn) + { + int calls = 0; + auto original = std::make_optional(); + auto callback = original->guarded( + [&calls]() + { + ++calls; + } + ); + const LifetimeToken copy{*original}; + const LifetimeToken moved{std::move(*original)}; + LifetimeToken assigned; + assigned = copy; + callback(); + EXPECT_EQ(calls, 1) << "the original still exists after being moved from"; + + original.reset(); + auto stillCalls = callback; + stillCalls(); + EXPECT_EQ(calls, 1) << "neither the copy, the moved-to nor the assigned token keeps it alive"; + EXPECT_TRUE(copy.watch().lock()); + EXPECT_TRUE(moved.watch().lock()); + } + + TEST(LifetimeTokenTests, GuardedCallablesKeepTheirArguments) + { + const LifetimeToken token; + std::string received; + auto callback = token.guarded( + [&received](std::string text, std::unique_ptr number) + { + received = text + std::to_string(*number); + } + ); + callback(std::string{"value "}, std::make_unique(7)); + EXPECT_EQ(received, "value 7"); + } +} diff --git a/utility/test/utility/test_link_safe_remove.hpp b/utility/test/utility/test_link_safe_remove.hpp index 1dfd4a34..6e1da5eb 100644 --- a/utility/test/utility/test_link_safe_remove.hpp +++ b/utility/test/utility/test_link_safe_remove.hpp @@ -1,15 +1,22 @@ #pragma once #include +#include #include #include +#include #include #include #include +#include #include +#ifndef _WIN32 +# include +#endif + extern std::filesystem::path programDirectory; extern std::filesystem::path temporaryRoot; @@ -33,9 +40,47 @@ namespace Utility::Test #endif } + /** + * @brief A tree with content, a nested directory link out of it and a file link: tree/a.txt (1 byte), + * tree/sub/b.txt (2 bytes), tree/sub/deeper/c.txt (3 bytes), tree/sub/deeper/out -> outside and + * tree/fileLink -> outside/keep.txt. 8 entries counting tree itself. + */ + std::filesystem::path makeTreeWithLinks() + { + const auto outside = root_.path() / "outside"; + std::filesystem::create_directories(outside); + std::ofstream{outside / "keep.txt"} << "keep"; + const auto tree = root_.path() / "tree"; + std::filesystem::create_directories(tree / "sub" / "deeper"); + std::ofstream{tree / "a.txt"} << "a"; + std::ofstream{tree / "sub" / "b.txt"} << "bb"; + std::ofstream{tree / "sub" / "deeper" / "c.txt"} << "ccc"; + EXPECT_TRUE(makeDirectoryLink(tree / "sub" / "deeper" / "out", outside)); + std::error_code error; + std::filesystem::create_symlink(outside / "keep.txt", tree / "fileLink", error); + EXPECT_FALSE(error) << error.message(); + return tree; + } + TemporaryDirectory root_{temporaryRoot, true}; }; + TEST(LinkReparseTagTests, OnlyNameSurrogatesAreLinks) + { + // IO_REPARSE_TAG_SYMLINK, IO_REPARSE_TAG_MOUNT_POINT (junction), IO_REPARSE_TAG_LX_SYMLINK + EXPECT_TRUE(Detail::isLinkReparseTag(0xA000000Cu)); + EXPECT_TRUE(Detail::isLinkReparseTag(0xA0000003u)); + EXPECT_TRUE(Detail::isLinkReparseTag(0xA000001Du)); + // IO_REPARSE_TAG_CLOUD_6 (OneDrive placeholder), IO_REPARSE_TAG_DEDUP, IO_REPARSE_TAG_AF_UNIX, + // IO_REPARSE_TAG_APPEXECLINK, IO_REPARSE_TAG_WOF + EXPECT_FALSE(Detail::isLinkReparseTag(0x9000601Au)); + EXPECT_FALSE(Detail::isLinkReparseTag(0x80000013u)); + EXPECT_FALSE(Detail::isLinkReparseTag(0x80000023u)); + EXPECT_FALSE(Detail::isLinkReparseTag(0x8000001Bu)); + EXPECT_FALSE(Detail::isLinkReparseTag(0x80000017u)); + EXPECT_FALSE(Detail::isLinkReparseTag(0u)); + } + TEST_F(LinkSafeRemoveTests, DirectoryLinkIsALinkAndADirectoryIsNot) { const auto target = root_.path() / "target"; @@ -129,4 +174,184 @@ namespace Utility::Test EXPECT_EQ(removeAllNoFollow(root_.path() / "missing", error), 0u); EXPECT_FALSE(error); } + + TEST_F(LinkSafeRemoveTests, KindsAreInspectedWithoutFollowing) + { + const auto target = root_.path() / "target"; + std::filesystem::create_directories(target); + std::ofstream{root_.path() / "file.txt"} << "file"; + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + + std::error_code error; + EXPECT_EQ(noFollowKindOf(target, error), NoFollowKind::Directory); + EXPECT_EQ(noFollowKindOf(root_.path() / "file.txt", error), NoFollowKind::Other); + EXPECT_EQ(noFollowKindOf(root_.path() / "link", error), NoFollowKind::Link); + EXPECT_EQ(noFollowKindOf(root_.path() / "missing", error), NoFollowKind::Missing); + EXPECT_FALSE(error) << error.message(); + } + + TEST_F(LinkSafeRemoveTests, RemoveNoFollowRemovesOneItemOnly) + { + const auto target = root_.path() / "target"; + std::filesystem::create_directories(target / "full"); + std::ofstream{target / "full" / "keep.txt"} << "keep"; + std::filesystem::create_directories(target / "empty"); + std::ofstream{target / "file.txt"} << "file"; + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + + std::error_code error; + EXPECT_TRUE(removeNoFollow(root_.path() / "link", error)); + EXPECT_FALSE(error) << error.message(); + EXPECT_TRUE(std::filesystem::exists(target / "full" / "keep.txt")) << "the link target must survive"; + + EXPECT_TRUE(removeNoFollow(target / "file.txt", error)); + EXPECT_TRUE(removeNoFollow(target / "empty", error)); + EXPECT_FALSE(removeNoFollow(target / "full", error)); + EXPECT_TRUE(error) << "a directory with content is not removed"; + EXPECT_TRUE(std::filesystem::exists(target / "full" / "keep.txt")); + + EXPECT_FALSE(removeNoFollow(root_.path() / "missing", error)); + EXPECT_FALSE(error) << "a missing item is no error"; + } + +#ifdef _WIN32 + TEST_F(LinkSafeRemoveTests, ReadOnlyItemsAreRemoved) + { + const auto tree = root_.path() / "readOnly"; + std::filesystem::create_directories(tree); + std::ofstream{tree / "locked.txt"} << "locked"; + std::filesystem::permissions(tree / "locked.txt", std::filesystem::perms::owner_write, std::filesystem::perm_options::remove); + + const auto result = removeAllNoFollow(tree); + EXPECT_TRUE(result.failures.empty()); + EXPECT_EQ(result.removed, 2u); + EXPECT_FALSE(std::filesystem::exists(tree)); + } +#endif + +#ifndef _WIN32 + TEST_F(LinkSafeRemoveTests, AnUndeletableEntryKeepsOnlyItsDirectories) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root removes entries of read-only directories"; + + const auto tree = root_.path() / "stuck"; + std::filesystem::create_directories(tree / "b-locked"); + std::filesystem::create_directories(tree / "c-free"); + std::ofstream{tree / "a.txt"} << "a"; + std::ofstream{tree / "b-locked" / "inner.txt"} << "inner"; + std::ofstream{tree / "c-free" / "free.txt"} << "free"; + std::ofstream{tree / "d.txt"} << "d"; + std::filesystem::permissions(tree / "b-locked", std::filesystem::perms::owner_write, std::filesystem::perm_options::remove); + + const auto result = removeAllNoFollow(tree); + std::filesystem::permissions(tree / "b-locked", std::filesystem::perms::owner_write, std::filesystem::perm_options::add); + + ASSERT_EQ(result.failures.size(), 1u); + EXPECT_EQ(result.failures.front().path, tree / "b-locked" / "inner.txt") << "the real path that failed"; + EXPECT_TRUE(result.failures.front().error); + EXPECT_EQ(result.removed, 4u) << "a.txt, c-free, free.txt and d.txt"; + EXPECT_TRUE(std::filesystem::exists(tree / "b-locked" / "inner.txt")); + EXPECT_FALSE(std::filesystem::exists(tree / "a.txt")); + EXPECT_FALSE(std::filesystem::exists(tree / "c-free")); + EXPECT_FALSE(std::filesystem::exists(tree / "d.txt")); + + std::error_code error; + removeAllNoFollow(tree, error); + EXPECT_FALSE(error) << "the rest goes once it can"; + } + + TEST_F(LinkSafeRemoveTests, AnUnreadableDirectoryIsAFailureOfItsOwn) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root reads every directory"; + + const auto tree = root_.path() / "unreadable"; + std::filesystem::create_directories(tree / "closed"); + std::ofstream{tree / "closed" / "inner.txt"} << "inner"; + std::ofstream{tree / "open.txt"} << "open"; + std::filesystem::permissions(tree / "closed", std::filesystem::perms::owner_read, std::filesystem::perm_options::remove); + + const auto result = removeAllNoFollow(tree); + std::filesystem::permissions(tree / "closed", std::filesystem::perms::owner_read, std::filesystem::perm_options::add); + + ASSERT_EQ(result.failures.size(), 1u); + EXPECT_EQ(result.failures.front().path, tree / "closed"); + EXPECT_FALSE(std::filesystem::exists(tree / "open.txt")); + EXPECT_TRUE(std::filesystem::exists(tree / "closed" / "inner.txt")); + } +#endif + + TEST_F(LinkSafeRemoveTests, CountingWalksTheTreeWithoutFollowingLinks) + { + const auto tree = makeTreeWithLinks(); + + NoFollowTreeWalk walk{tree, NoFollowTreeWalk::Mode::Count}; + EXPECT_TRUE(walk.step(std::chrono::steady_clock::time_point::max())); + EXPECT_TRUE(walk.done()); + EXPECT_EQ(walk.entries(), 8u) << "tree, a.txt, sub, b.txt, deeper, c.txt, out and fileLink"; + EXPECT_EQ(walk.bytes(), 6u) << "links count no bytes and are not followed"; + EXPECT_TRUE(walk.failures().empty()); + EXPECT_TRUE(std::filesystem::exists(tree / "sub" / "deeper" / "c.txt")) << "counting removes nothing"; + } + + TEST_F(LinkSafeRemoveTests, RemovingInSmallStepsDeletesTheTreeAndKeepsLinkTargets) + { + const auto tree = makeTreeWithLinks(); + + NoFollowTreeWalk walk{tree, NoFollowTreeWalk::Mode::Remove}; + int steps = 0; + std::set seen; + while (!walk.step(std::chrono::steady_clock::time_point::min())) + { + ++steps; + seen.insert(walk.lastPath()); + ASSERT_LT(steps, 100); + } + EXPECT_GE(steps, 7) << "an expired deadline still handles one entry per step"; + EXPECT_TRUE(seen.contains(tree / "sub" / "deeper" / "out")); + EXPECT_EQ(walk.entries(), 8u); + EXPECT_TRUE(walk.failures().empty()); + EXPECT_FALSE(std::filesystem::exists(tree)); + EXPECT_TRUE(std::filesystem::exists(root_.path() / "outside" / "keep.txt")); + } + + TEST_F(LinkSafeRemoveTests, WalkingALinkRootTouchesOnlyTheLink) + { + const auto target = root_.path() / "target"; + std::filesystem::create_directories(target); + std::ofstream{target / "keep.txt"} << "keep"; + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + + NoFollowTreeWalk count{root_.path() / "link", NoFollowTreeWalk::Mode::Count}; + count.step(std::chrono::steady_clock::time_point::max()); + EXPECT_EQ(count.entries(), 1u); + + NoFollowTreeWalk walk{root_.path() / "missing", NoFollowTreeWalk::Mode::Remove}; + EXPECT_TRUE(walk.step(std::chrono::steady_clock::time_point::max())); + EXPECT_EQ(walk.entries(), 0u); + EXPECT_TRUE(walk.failures().empty()); + } + + TEST_F(LinkSafeRemoveTests, VerifiedDirectoriesAreInspectedOnce) + { + const auto outside = root_.path() / "outside"; + std::filesystem::create_directories(outside); + const auto tree = root_.path() / "tree"; + std::filesystem::create_directories(tree / "real" / "deeper"); + + VerifiedDirectories verified; + std::error_code error; + EXPECT_FALSE(firstLinkBelow(tree, tree / "real" / "deeper" / "missing" / "file.txt", error, &verified)); + EXPECT_FALSE(error) << error.message(); + EXPECT_TRUE(verified.contains(tree / "real")); + EXPECT_TRUE(verified.contains(tree / "real" / "deeper")); + EXPECT_FALSE(verified.contains(tree / "real" / "deeper" / "missing")) << "a missing component is not verified"; + + ASSERT_TRUE(makeDirectoryLink(tree / "real" / "deeper" / "missing", outside)); + EXPECT_EQ( + firstLinkBelow(tree, tree / "real" / "deeper" / "missing" / "file.txt", error, &verified), + tree / "real" / "deeper" / "missing" + ) << "a component created later is inspected"; + } } diff --git a/utility/test/utility/test_permissions.hpp b/utility/test/utility/test_permissions.hpp index d0fb0cb7..b570dc2d 100644 --- a/utility/test/utility/test_permissions.hpp +++ b/utility/test/utility/test_permissions.hpp @@ -1,10 +1,16 @@ #pragma once #include +#include #include #include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; namespace Utility::Test { @@ -30,4 +36,43 @@ namespace Utility::Test EXPECT_EQ(*inherited, perms::all & ~processUmask()); EXPECT_EQ(*inherited & processUmask(), perms::none); } + +#ifndef _WIN32 + TEST(PermissionsTests, NewFilePermissionsAreThoseOfACreatedFile) + { + const TemporaryDirectory directory{temporaryRoot, true}; + std::ofstream{directory.path() / "created"} << "x"; + EXPECT_EQ(std::filesystem::status(directory.path() / "created").permissions(), newFilePermissions()); + } + + TEST(PermissionsTests, RestrictToOwnerCreatesAPrivateFile) + { + const TemporaryDirectory directory{temporaryRoot, true}; + const auto path = directory.path() / "private"; + ASSERT_TRUE(restrictToOwner(path).has_value()); + EXPECT_EQ(std::filesystem::status(path).permissions(), perms::owner_read | perms::owner_write); + } + + TEST(PermissionsTests, RestrictToOwnerKeepsTheContentOfAnExistingFile) + { + const TemporaryDirectory directory{temporaryRoot, true}; + const auto path = directory.path() / "existing"; + std::ofstream{path} << "content"; + std::filesystem::permissions(path, perms::owner_all | perms::group_read | perms::others_read); + ASSERT_TRUE(restrictToOwner(path).has_value()); + EXPECT_EQ(std::filesystem::status(path).permissions(), perms::owner_read | perms::owner_write); + std::ifstream file{path}; + EXPECT_EQ(std::string(std::istreambuf_iterator{file}, std::istreambuf_iterator{}), "content"); + } + + TEST(PermissionsTests, RestrictToOwnerRefusesALink) + { + const TemporaryDirectory directory{temporaryRoot, true}; + std::ofstream{directory.path() / "target"} << "x"; + const auto before = std::filesystem::status(directory.path() / "target").permissions(); + std::filesystem::create_symlink(directory.path() / "target", directory.path() / "link"); + EXPECT_FALSE(restrictToOwner(directory.path() / "link").has_value()); + EXPECT_EQ(std::filesystem::status(directory.path() / "target").permissions(), before); + } +#endif } diff --git a/utility/test/utility/test_rename.hpp b/utility/test/utility/test_rename.hpp index f96170f8..ebeae1a7 100644 --- a/utility/test/utility/test_rename.hpp +++ b/utility/test/utility/test_rename.hpp @@ -69,6 +69,84 @@ namespace Utility::Test EXPECT_EQ(read(directory_.path() / "target" / "inside"), "x"); } + TEST_F(RenameNoReplaceTests, RenameUnlessExistsRenamesToAFreeName) + { + write(directory_.path() / "a", "a"); + ASSERT_TRUE(Detail::renameUnlessExists(directory_.path() / "a", directory_.path() / "b").has_value()); + EXPECT_FALSE(std::filesystem::exists(directory_.path() / "a")); + EXPECT_EQ(read(directory_.path() / "b"), "a"); + } + + TEST_F(RenameNoReplaceTests, RenameUnlessExistsRefusesAnExistingFile) + { + write(directory_.path() / "a", "a"); + write(directory_.path() / "b", "b"); + const auto result = Detail::renameUnlessExists(directory_.path() / "a", directory_.path() / "b"); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), std::errc::file_exists); + EXPECT_EQ(read(directory_.path() / "a"), "a"); + EXPECT_EQ(read(directory_.path() / "b"), "b"); + } + +#ifndef _WIN32 + TEST_F(RenameNoReplaceTests, RenameUnlessExistsRefusesADanglingLink) + { + write(directory_.path() / "a", "a"); + std::filesystem::create_symlink(directory_.path() / "missing", directory_.path() / "b"); + const auto result = Detail::renameUnlessExists(directory_.path() / "a", directory_.path() / "b"); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), std::errc::file_exists); + EXPECT_TRUE(std::filesystem::is_symlink(std::filesystem::symlink_status(directory_.path() / "b"))); + EXPECT_EQ(read(directory_.path() / "a"), "a"); + } + + TEST_F(RenameNoReplaceTests, RenameByLinkMovesAFile) + { + write(directory_.path() / "a", "a"); + ASSERT_TRUE(Detail::renameByLink(directory_.path() / "a", directory_.path() / "b").has_value()); + EXPECT_FALSE(std::filesystem::exists(directory_.path() / "a")); + EXPECT_EQ(read(directory_.path() / "b"), "a"); + } + + TEST_F(RenameNoReplaceTests, RenameByLinkRefusesAnExistingFile) + { + write(directory_.path() / "a", "a"); + write(directory_.path() / "b", "b"); + const auto result = Detail::renameByLink(directory_.path() / "a", directory_.path() / "b"); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), std::errc::file_exists); + EXPECT_FALSE(Detail::isNoReplaceUnsupported(result.error())); + EXPECT_EQ(read(directory_.path() / "a"), "a"); + EXPECT_EQ(read(directory_.path() / "b"), "b"); + } +#endif + + TEST(RenameNoReplaceFallbackTests, OnlyMissingSupportFallsBack) + { + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::invalid_argument))); + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::function_not_supported))); + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::operation_not_supported))); + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::not_supported))); + EXPECT_TRUE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::operation_not_permitted))); + EXPECT_FALSE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::file_exists))); + EXPECT_FALSE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::no_such_file_or_directory))); + EXPECT_FALSE(Detail::isNoReplaceUnsupported(std::make_error_code(std::errc::permission_denied))); + } + + TEST_F(RenameNoReplaceTests, MoveIntoPlaceReplacesOnlyWhenAllowed) + { + write(directory_.path() / "a", "a"); + write(directory_.path() / "b", "b"); + const auto refused = moveIntoPlace(directory_.path() / "a", directory_.path() / "b", false); + ASSERT_FALSE(refused.has_value()); + EXPECT_EQ(refused.error(), std::errc::file_exists); + EXPECT_EQ(read(directory_.path() / "b"), "b"); + + ASSERT_TRUE(moveIntoPlace(directory_.path() / "a", directory_.path() / "b", true).has_value()); + EXPECT_FALSE(std::filesystem::exists(directory_.path() / "a")); + EXPECT_EQ(read(directory_.path() / "b"), "a"); + } + TEST(RenameTargetTests, OnlyANewNameInTheSameDirectoryIsARenameInPlace) { const std::filesystem::path old{"/data/old.txt"}; diff --git a/utility/test/utility/test_utf8.hpp b/utility/test/utility/test_utf8.hpp index ad17e593..b059201d 100644 --- a/utility/test/utility/test_utf8.hpp +++ b/utility/test/utility/test_utf8.hpp @@ -37,8 +37,8 @@ namespace Utility::Tests ), replacement + U"a" ); - EXPECT_EQ(Utf8::decode("\xC0\xAF"), replacement); - EXPECT_EQ(Utf8::decode("\xED\xA0\x80"), replacement); + EXPECT_EQ(Utf8::decode("\xC0\xAF"), replacement + replacement); + EXPECT_EQ(Utf8::decode("\xED\xA0\x80"), replacement + replacement + replacement); } TEST_F(Utf8Tests, EncodeRoundTripsDecode)