diff --git a/CMakeLists.txt b/CMakeLists.txt index 2361e373..cfa040cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,9 @@ add_executable(${PROJECT_NAME}) target_link_libraries(${PROJECT_NAME} PUBLIC core-target) if (WIN32) + # A GUI program: a console program lives in a console window and exits with it, closing that window or a crash + # of Windows Terminal ended the app silently. --console attaches to the console it was started from. + set_target_properties(${PROJECT_NAME} PROPERTIES WIN32_EXECUTABLE TRUE) include("${CMAKE_CURRENT_LIST_DIR}/_cmake/windows/icon.cmake") file(TO_CMAKE_PATH "${NUI_SFTP_ICO}" NUI_SFTP_ICO_FORWARD) configure_file( @@ -139,6 +142,7 @@ add_subdirectory("${CMAKE_SOURCE_DIR}/dependencies/gimo" EXCLUDE_FROM_ALL) add_subdirectory("${CMAKE_SOURCE_DIR}/dependencies/traits" EXCLUDE_FROM_ALL) add_subdirectory("${CMAKE_SOURCE_DIR}/constants/source/constants") add_subdirectory("${CMAKE_SOURCE_DIR}/ids/source/ids") +add_subdirectory("${CMAKE_SOURCE_DIR}/paths/source/paths") add_subdirectory("${CMAKE_SOURCE_DIR}/shared_data/source/shared_data") add_subdirectory("${CMAKE_SOURCE_DIR}/utility/source/utility") add_subdirectory("${CMAKE_SOURCE_DIR}/log/source/log") @@ -190,6 +194,7 @@ else() add_subdirectory(ssh/test/ssh) add_subdirectory(backend/test/backend) add_subdirectory(utility/test/utility) + add_subdirectory(paths/test/paths) add_subdirectory(shared_data/test/shared_data) add_subdirectory(shared_data/benchmark/shared_data) add_subdirectory(tar-archive/test/tar_archive) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef23f7e2..e5cbd57e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -226,8 +226,9 @@ cmake --build build/clang_debug --target \ nui-sftp-shared-data-test nui-sftp-ssh-test \ nui-sftp-tar-archive-test nui-sftp-command-store-test -# Run the whole suite. -ctest --test-dir build/clang_debug --output-on-failure +# Run the whole suite. Tests run in parallel; the timeout makes a hang fail +# after two minutes instead of blocking the run. +ctest --test-dir build/clang_debug --output-on-failure --timeout 120 -j$(nproc) # Or run an individual binary. ./build/clang_debug/test/nui-sftp-utility-test @@ -328,7 +329,7 @@ update both workflows if it changes the build surface. 1. Fork the repository and create a feature branch from `main`. 2. Keep commits focused; prefer small, reviewable changes over large rewrites. 3. Run a debug build with tests enabled locally and make sure `ctest - --output-on-failure` passes from `build/clang_debug`. + --output-on-failure --timeout 120 -j$(nproc)` passes from `build/clang_debug`. 4. Format any C++ changes with clang-format. 5. Open the PR against `main`. Describe the user-visible behaviour, the motivation, and anything reviewers should pay particular attention to. @@ -377,7 +378,7 @@ needed. "type": "shell", "dependsOn": "build_debug", "command": "ctest", - "args": ["--test-dir", "build/clang_debug", "--output-on-failure"], + "args": ["--test-dir", "build/clang_debug", "--output-on-failure", "--timeout", "120", "-j", "8"], "group": "test" } ] diff --git a/backend/include/backend/dropped_entry.hpp b/backend/include/backend/dropped_entry.hpp index 765060b6..20dc1653 100644 --- a/backend/include/backend/dropped_entry.hpp +++ b/backend/include/backend/dropped_entry.hpp @@ -1,11 +1,12 @@ #pragma once -#include +#include #include +#include /** * @brief Describes a file dropped onto the window from the desktop, with just enough detail for the transfer. - * @param path Local path of the dropped file or directory. + * @param path Local path of the dropped file or directory, nothing when it is not an absolute path. */ -SharedData::DirectoryEntry leanDirectoryEntryFromPath(std::filesystem::path const& path); +std::optional droppedItemFromPath(std::filesystem::path const& path); diff --git a/backend/include/backend/existence_check.hpp b/backend/include/backend/existence_check.hpp new file mode 100644 index 00000000..d985d39a --- /dev/null +++ b/backend/include/backend/existence_check.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include +#include +#include +#include + +/** + * @brief Whether each of @p paths exists on the local disk. Only "not found" is absent, a dangling symlink exists. A + * path that cannot be checked fails the whole check: a transfer overwrites what it takes for absent. + */ +std::expected, std::string> localPathsExist(std::vector const& paths); + +/** + * @brief Whether each of @p paths exists on the server, with the same rules as localPathsExist. Few paths are + * checked with one lstat each, more with one listing per parent directory. + */ +std::expected, std::string> remotePathsExist( + SecureShell::SftpSession& session, + std::vector const& paths, + std::chrono::milliseconds timeout +); diff --git a/backend/include/backend/local_delete_guard.hpp b/backend/include/backend/local_delete_guard.hpp new file mode 100644 index 00000000..55120717 --- /dev/null +++ b/backend/include/backend/local_delete_guard.hpp @@ -0,0 +1,56 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include + +/** + * @brief Refuses a root directory and @p home or a directory containing it. + */ +inline std::expected +refuseProtectedLocalDelete(Paths::LocalPath const& target, std::optional const& home) +{ + if (target.empty()) + return std::unexpected(std::string{"no path"}); + if (target.isRoot()) + return std::unexpected(fmt::format("'{}' is a root directory", target.toUtf8())); + if (home && home->isWithin(target)) + return std::unexpected(fmt::format("'{}' contains the home directory", target.toUtf8())); + return {}; +} + +/** + * @brief A local path that may be deleted, or the reason to refuse it: not absolute, containing '..', a root + * directory, or @p home or a directory containing it. + */ +inline std::expected +localDeleteTarget(std::string const& utf8, std::optional const& home) +{ + // LocalPath resolves ".." lexically, which goes elsewhere when a directory on the way is a symlink. + for (auto const segment : std::views::split(std::string_view{utf8}, '/')) + { + for (auto const part : std::views::split(segment, '\\')) + { + if (std::string_view{part.begin(), part.end()} == "..") + return std::unexpected(fmt::format("'{}' contains '..'", utf8)); + } + } + + auto target = Paths::LocalPath::fromUtf8(utf8); + if (!target) + return std::unexpected(fmt::format("'{}' {}", utf8, Paths::toString(target.error().reason))); + if (auto allowed = refuseProtectedLocalDelete(*target, home); !allowed) + return std::unexpected(std::move(allowed).error()); + return *std::move(target); +} + +/** + * @brief The home directory of the user running the backend, empty when it cannot be determined. + */ +std::optional localHomeDirectory(); diff --git a/backend/include/backend/opener.hpp b/backend/include/backend/opener.hpp index 9585bd92..9b33641d 100644 --- a/backend/include/backend/opener.hpp +++ b/backend/include/backend/opener.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include class Opener @@ -19,6 +20,13 @@ class Opener std::expected openFile(std::filesystem::path const& path, bool openWith); + /** + * @brief Why opening @p path could run code: a program, a script, a shortcut or an application bundle. Nothing + * for an ordinary document or a folder. + * @return A localized message key describing the risk. + */ + std::optional riskOf(std::filesystem::path const& path) const; + /** * @brief Open the native file manager showing @p path. * @details For a directory, opens the file manager at that directory. For a file, opens the diff --git a/backend/include/backend/rpc_filesystem.hpp b/backend/include/backend/rpc_filesystem.hpp index 967208e1..2fed420a 100644 --- a/backend/include/backend/rpc_filesystem.hpp +++ b/backend/include/backend/rpc_filesystem.hpp @@ -4,6 +4,8 @@ #include #include +#include + class RpcFilesystem : public RpcHelper::StrandRpc { public: @@ -12,7 +14,8 @@ class RpcFilesystem : public RpcHelper::StrandRpc Nui::Window& wnd, Nui::RpcHub& hub, Persistence::LocalFilesystemOptions options, - Opener& opener + Opener& opener, + std::function riskyFilePolicy = []() { return Persistence::RiskyFilePolicy::Warn; } ); private: @@ -34,4 +37,6 @@ class RpcFilesystem : public RpcHelper::StrandRpc private: Persistence::LocalFilesystemOptions options_; Opener* opener_; + /** @brief Read on every open, so a changed setting applies at once. */ + std::function riskyFilePolicy_; }; \ No newline at end of file diff --git a/backend/include/backend/session.hpp b/backend/include/backend/session.hpp index 5137f527..35d5571b 100644 --- a/backend/include/backend/session.hpp +++ b/backend/include/backend/session.hpp @@ -209,7 +209,7 @@ class Session void registerRpcSftpOpenSyncSession(); void registerRpcSftpRecomputeSyncDiff(); void registerRpcSftpLoadSyncDiffChildren(); - void registerRpcSftpBuildSyncEnqueuePlan(); + void registerRpcSftpExecuteSyncSelection(); void registerRpcSftpCancelSyncDiff(); void registerRpcSftpCloseSyncSession(); void registerOperationQueuePauseUnpause(); diff --git a/backend/include/backend/sftp/all_operations.hpp b/backend/include/backend/sftp/all_operations.hpp index 5c1693ae..0ae456e6 100644 --- a/backend/include/backend/sftp/all_operations.hpp +++ b/backend/include/backend/sftp/all_operations.hpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,10 @@ auto Operation::visit(FunctionT&& func) const { return func(static_cast(*this)); } + case LocalDelete: + { + return func(static_cast(*this)); + } case ArchiveDownload: { return func(static_cast(*this)); diff --git a/backend/include/backend/sftp/archive_download_operation.hpp b/backend/include/backend/sftp/archive_download_operation.hpp index 07c4c71a..a7305ed4 100644 --- a/backend/include/backend/sftp/archive_download_operation.hpp +++ b/backend/include/backend/sftp/archive_download_operation.hpp @@ -1,6 +1,9 @@ #pragma once #include +#include +#include +#include #include #include #include @@ -10,6 +13,8 @@ #include #include #include +#include +#include #include #include @@ -62,13 +67,12 @@ class ArchiveDownloadOperation : public Operation * `foo/sub/`, `foo/sub/baz.txt`. * - Other types (symlinks, devices, ...) are silently skipped. * - * `fullPath` must carry the full remote path; `path` is used only if - * `fullPath` is empty. + * A root that is the remote root directory has no name and is refused. */ - std::vector entries{}; + std::vector roots{}; /** @brief Destination archive path on the local filesystem. */ - std::filesystem::path localArchivePath{}; + Paths::LocalPath localArchivePath{}; /** @brief Compression codec. Use TarArchive::Compression::Auto to pick from the extension. */ TarArchive::Compression compression{TarArchive::Compression::Auto}; @@ -81,6 +85,8 @@ class ArchiveDownloadOperation : public Operation /** @brief Refuse to overwrite a pre-existing final archive. */ bool mayOverwrite{false}; + /** @brief Mark the archive as downloaded from the internet, see Utility::markAsDownloaded. */ + bool markAsDownloaded{true}; /** @brief Create any missing parents of @ref localArchivePath. */ bool createMissingDirectories{false}; @@ -98,9 +104,25 @@ class ArchiveDownloadOperation : public Operation std::expected work() override; std::expected workInStrand() override; + // The scan phase blocks on SFTP futures, so it runs off the strand like a standalone scan. + bool usesStrand() const noexcept override + { + return scanPhase_ == nullptr; + } + bool isBarrier() const noexcept override { - return false; + return scanPhase_ != nullptr; + } + + /** + * @brief Scans the directory root of @p scan before packing; roots are scanned in the order added. + */ + void addScanRoot(std::unique_ptr scan); + + bool scanning() const noexcept + { + return scanPhase_ != nullptr; } int parallelWorkDoable(int) const noexcept override @@ -129,16 +151,15 @@ class ArchiveDownloadOperation : public Operation * rather than silently buried inside prepare. * * Safe to call multiple times — one call per directory root in - * @ref Options::entries. Matching is by @p remoteRootPath which - * must equal the root entry's full remote path - * (fullPath-or-fallback-to-path in the root @ref DirectoryEntry). + * @ref Options::roots. Matching is by @p remoteRootPath, the + * path of the root. * * Threading: invoked on the queue's asio strand between SFTP-strand * work batches, so no extra synchronisation is required against * prepareInStrand (which runs on the SFTP strand). */ void setScanResultForRoot( - std::filesystem::path const& remoteRootPath, + Paths::RemotePath const& remoteRootPath, std::vector entries, std::uint64_t totalBytes ); @@ -152,8 +173,8 @@ class ArchiveDownloadOperation : public Operation */ struct ResolvedEntry { - SharedData::DirectoryEntry tarMeta{}; - std::filesystem::path remoteFullPath{}; + TarArchive::MemberHeader header{}; + Paths::RemotePath source{}; }; std::expected prepareInStrand(); @@ -162,14 +183,14 @@ class ArchiveDownloadOperation : public Operation * @brief Expand a preceding ScanOperation's flat entry vector into the * op's ResolvedEntry list, prefixing each entry's tar path with * @p archivePrefix and composing remote paths under - * @p remoteRootFullPath. Index 0 (the synthetic scan root) is + * @p remoteRoot. Index 0 (the synthetic scan root) is * skipped because the root directory entry is emitted separately * by flattenRootsInStrand. */ void appendPrescannedInStrand( std::vector const& entries, - std::filesystem::path const& remoteRootFullPath, - std::filesystem::path const& archivePrefix + Paths::RemotePath const& remoteRoot, + Paths::RelativePath const& archivePrefix ); std::expected openNextEntryInStrand(); std::expected readChunkInStrand(); @@ -191,6 +212,8 @@ class ArchiveDownloadOperation : public Operation std::uint64_t totalBytesTransferred_{0u}; std::uint64_t totalPayloadBytes_{0u}; SecureShell::BufferLease buffer_; + /** @brief options_.localArchivePath for the filesystem calls, empty when none was given. */ + std::filesystem::path localArchiveNativePath_{}; std::filesystem::path tempPath_{}; /** * @brief Pre-scanned entry lists delivered by preceding ScanOperation(s), @@ -198,7 +221,7 @@ class ArchiveDownloadOperation : public Operation * looks each directory root up here before falling back to a live * recurseIntoDirectoryInStrand listing. */ - std::map, std::uint64_t>> + std::map, std::uint64_t>> prescannedRoots_{}; /** * @brief Reused purely for its bytes-transferred + B/s meter machinery; @@ -208,4 +231,5 @@ class ArchiveDownloadOperation : public Operation */ std::shared_ptr throughputMeter_{ std::make_shared()}; + std::unique_ptr> scanPhase_{}; }; diff --git a/backend/include/backend/sftp/archive_upload_operation.hpp b/backend/include/backend/sftp/archive_upload_operation.hpp index 3a76387e..c0eddf13 100644 --- a/backend/include/backend/sftp/archive_upload_operation.hpp +++ b/backend/include/backend/sftp/archive_upload_operation.hpp @@ -1,15 +1,20 @@ #pragma once #include +#include +#include #include #include #include #include +#include +#include #include #include #include #include #include +#include #include #include @@ -50,10 +55,10 @@ class ArchiveUploadOperation : public Operation progressCallback = [](auto, auto, auto, auto) {}; /** @brief Local filesystem paths to pack. Each is lstat'd in-strand on start. */ - std::vector localPaths{}; + std::vector localPaths{}; /** @brief Destination archive path on the remote side. */ - std::filesystem::path remoteArchivePath{}; + Paths::RemotePath remoteArchivePath{}; /** @brief Compression codec; Auto means derive from remoteArchivePath extension. */ TarArchive::Compression compression{TarArchive::Compression::Auto}; @@ -64,6 +69,9 @@ class ArchiveUploadOperation : public Operation /** @brief Refuse to overwrite an existing remote file. */ bool mayOverwrite{false}; + /** @brief The archive is written next to its destination under this suffix and renamed into place. */ + std::string tempFileSuffix{".filepart"}; + std::chrono::seconds futureTimeout{5}; }; @@ -77,10 +85,27 @@ class ArchiveUploadOperation : public Operation std::expected work() override; std::expected workInStrand() override; + // The scan phase blocks on local filesystem calls, so it runs off the strand like a standalone scan. + bool usesStrand() const noexcept override + { + return scanPhase_ == nullptr; + } + bool isBarrier() const noexcept override { - return false; + return scanPhase_ != nullptr; } + + /** + * @brief Scans the directory root of @p scan before packing; roots are scanned in the order added. + */ + void addScanRoot(std::unique_ptr scan); + + bool scanning() const noexcept + { + return scanPhase_ != nullptr; + } + int parallelWorkDoable(int) const noexcept override { return 1; @@ -111,7 +136,7 @@ class ArchiveUploadOperation : public Operation * passed in the options — prepareInStrand looks up the same key. */ void setScanResultForRoot( - std::filesystem::path const& localRootPath, + Paths::LocalPath const& localRootPath, std::vector entries, std::uint64_t totalBytes ); @@ -119,13 +144,13 @@ class ArchiveUploadOperation : public Operation private: /** * @brief Archive-relative metadata paired with the full local source path. - * Directory entries have an empty localFullPath since they carry no + * Directory entries have an unset source since they carry no * payload. */ struct ResolvedEntry { - SharedData::DirectoryEntry tarMeta{}; - std::filesystem::path localFullPath{}; + TarArchive::MemberHeader header{}; + Paths::LocalPath source{}; }; std::expected prepareInStrand(); @@ -139,8 +164,8 @@ class ArchiveUploadOperation : public Operation */ void appendPrescannedFromScan( std::vector const& entries, - std::filesystem::path const& localRootFullPath, - std::filesystem::path const& archivePrefix + Paths::LocalPath const& localRoot, + Paths::RelativePath const& archivePrefix ); std::expected openNextEntryInStrand(); std::expected readChunkInStrand(); @@ -169,7 +194,7 @@ class ArchiveUploadOperation : public Operation * path. flattenRoots looks each directory root up here and errors * out if none is present — there is no self-contained recurse. */ - std::map, std::uint64_t>> + std::map, std::uint64_t>> prescannedRoots_{}; /** * @brief Reused purely for its bytes-transferred + B/s meter machinery; @@ -179,4 +204,13 @@ class ArchiveUploadOperation : public Operation */ std::shared_ptr throughputMeter_{ std::make_shared()}; + /** @brief A scanned directory or name that cannot go into the archive, set once the scan delivered it. */ + std::optional incompleteScan_{}; + std::unique_ptr> scanPhase_{}; + /** @brief options_.remoteArchivePath, unset when none was given. */ + Paths::RemotePath remoteArchivePath_{}; + /** @brief Where the archive is written before it replaces the destination. */ + Paths::RemotePath tempPath_{}; + /** @brief The temp file exists on the server and must go if the upload does not complete. */ + bool tempCreated_{false}; }; diff --git a/backend/include/backend/sftp/bulk_download_operation.hpp b/backend/include/backend/sftp/bulk_download_operation.hpp index ffadf62a..d782adc5 100644 --- a/backend/include/backend/sftp/bulk_download_operation.hpp +++ b/backend/include/backend/sftp/bulk_download_operation.hpp @@ -13,8 +13,10 @@ class BulkDownloadOperation : public BulkTransferOperation struct BulkDownloadOperationOptions { BulkProgressCallback overallProgressCallback = [](SharedData::BulkProgress const&) {}; - std::filesystem::path remotePath{}; - std::filesystem::path localPath{}; + /** @brief The scanned remote directory, only a label for a flat list. */ + Paths::RemotePath remotePath{}; + /** @brief Where the scanned directory is created, only a label for a flat list. */ + Paths::LocalPath localPath{}; DownloadOperation::DownloadOperationOptions individualOptions = {}; bool asArchive{false}; std::string archiveFormat{"tar"}; @@ -43,21 +45,30 @@ class BulkDownloadOperation : public BulkTransferOperation */ struct PrescannedFile { - std::filesystem::path remoteSrc; - std::filesystem::path localDst; + Paths::RemotePath source; + Paths::LocalPath destination; std::uint64_t sizeBytes; std::uint64_t mtime{0}; std::uint32_t mtimeNsec{0}; + bool mayOverwrite{false}; }; void setPrescannedFileList(std::vector files); protected: std::expected prepareRootInStrand(SharedData::DirectoryEntry const& root) override; - std::expected createDirectoryInStrand(std::size_t entryIndex) override; - std::unique_ptr - makeChild(std::size_t entryIndex, std::function onProgress) override; - std::filesystem::path displayPath(std::size_t entryIndex) const override; + std::expected createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) override; + std::unique_ptr makeChild( + std::size_t entryIndex, + EntryPaths const& paths, + std::function onProgress + ) override; bool isSkippableError(Error const& error) const override; + bool isSkippableDirectoryError(Error const& error) const override; + + bool sourceIsRemote() const noexcept override + { + return true; + } std::string_view logName() const noexcept override { @@ -65,7 +76,8 @@ class BulkDownloadOperation : public BulkTransferOperation } private: - std::filesystem::path localPathOf(std::size_t entryIndex) const; + std::expected + createLocalDirectory(Paths::LocalPath const& path, SharedData::DirectoryEntry const& entry); std::expected applyPermsToDirectory(std::filesystem::path const& path, SharedData::DirectoryEntry const& entryToInheritFrom); diff --git a/backend/include/backend/sftp/bulk_resume_registry.hpp b/backend/include/backend/sftp/bulk_resume_registry.hpp index c966616d..25a5b3af 100644 --- a/backend/include/backend/sftp/bulk_resume_registry.hpp +++ b/backend/include/backend/sftp/bulk_resume_registry.hpp @@ -1,35 +1,28 @@ #pragma once #include -#include +#include +#include #include #include #include #include #include +#include /** * @brief Carries the state needed to re-enqueue a bulk operation against a * replacement Session after a seamless reconnect. * - * The kind tracks which addBulk* path to take; the request mirrors - * the original BulkAddRequest minus already-completed entries (and - * the head entry that was mid-transfer when the connection died, on - * the assumption that the per-file resume path is not byte-accurate - * across a reconnect). + * One entry per frontend card: the request's type picks the addBulk* + * path. A transfer entry holds either the files of one bulk card or a + * single directory, so adopting it re-issues exactly that card under + * its own id. */ struct BulkResumeEntry { - enum class Kind - { - BulkDownload, - BulkUpload, - BulkDelete, - }; - - Kind kind; - SharedData::BulkAddRequest request; + std::variant request; std::chrono::steady_clock::time_point storedAt{std::chrono::steady_clock::now()}; }; diff --git a/backend/include/backend/sftp/bulk_transfer_operation.hpp b/backend/include/backend/sftp/bulk_transfer_operation.hpp index 59fab7ec..a5a44c4e 100644 --- a/backend/include/backend/sftp/bulk_transfer_operation.hpp +++ b/backend/include/backend/sftp/bulk_transfer_operation.hpp @@ -1,9 +1,11 @@ #pragma once #include +#include #include #include #include +#include #include #include @@ -35,8 +37,10 @@ class BulkTransferOperation : public Operation struct CommonOptions { BulkProgressCallback overallProgressCallback = [](SharedData::BulkProgress const&) {}; - std::filesystem::path remotePath{}; - std::filesystem::path localPath{}; + /** @brief Root of a scanned tree on the server, only a label for a flat list. */ + Paths::RemotePath remotePath{}; + /** @brief Root of a scanned tree on this machine, only a label for a flat list. */ + Paths::LocalPath localPath{}; bool failFast{false}; // Upper bound of files transferred at once. The queue may grant fewer slots per step. int concurrency{1}; @@ -45,6 +49,17 @@ class BulkTransferOperation : public Operation std::chrono::milliseconds progressEmitInterval{100}; }; + /** + * @brief Both ends of one entry. + */ + struct EntryPaths + { + Paths::RemotePath remote{}; + Paths::LocalPath local{}; + /** @brief This entry's destination may be replaced even when the bulk's options do not allow it. */ + bool mayOverwrite{false}; + }; + ~BulkTransferOperation() override; BulkTransferOperation(BulkTransferOperation const&) = delete; BulkTransferOperation(BulkTransferOperation&&) = delete; @@ -61,14 +76,15 @@ class BulkTransferOperation : public Operation void pause(bool doPause) override; SecureShell::ProcessingStrand* strand() const override; + // The scan phase blocks on SFTP futures, so it runs off the strand like a standalone scan. bool usesStrand() const noexcept override { - return true; + return scanPhase_ == nullptr; } bool isBarrier() const noexcept override { - return false; + return scanPhase_ != nullptr; } /** @@ -83,7 +99,47 @@ class BulkTransferOperation : public Operation */ void setScanResult(std::vector&& entries, std::uint64_t totalBytes); - std::vector> getFailed() const; + /** + * @brief Runs @p scan before transferring; the scanned tree becomes this bulk's entries. + * + * @tparam ScanT ScanOperation for downloads, LocalScanOperation for uploads. + */ + template + void scanFirst(std::unique_ptr scan) + { + auto phase = std::make_unique>( + [this](ScanT& finished) + { + const auto scannedBytes = finished.totalBytes(); + addRejectedEntries(finished.rejectedPaths()); + if constexpr (requires { finished.unreadableDirectories(); }) + addUnreadableDirectories(finished.unreadableDirectories()); + setScanResult(finished.ejectEntries(), scannedBytes); + } + ); + phase->addRoot(std::move(scan)); + scanPhase_ = std::move(phase); + } + + bool scanning() const noexcept + { + return scanPhase_ != nullptr; + } + + /** + * @brief Records entries the scan refused because of their name as failed, they are never transferred. + */ + void addRejectedEntries(std::vector const& paths); + + /** + * @brief Records directories the scan could not list as failed, their content is not transferred. + */ + void addUnreadableDirectories(std::vector const& paths); + + /** + * @brief The entries that were skipped, by their source path. + */ + std::vector> getFailed() const; std::size_t inFlightCount() const noexcept { @@ -104,24 +160,39 @@ class BulkTransferOperation : public Operation BulkTransferOperation(SecureShell::SftpSession& sftp, CommonOptions common); /** - * @brief Flat-list mode: every entry is a file with an absolute source in entry.path and the - * matching absolute destination in @p destinations (index aligned). + * @brief Flat-list mode: every entry is a file, @p paths holds both of its ends (index aligned). */ void setPrescannedEntries( std::vector entries, - std::vector destinations, + std::vector paths, std::uint64_t totalBytes ); bool prescanned() const noexcept { - return !prescannedDestinations_.empty(); + return entrySource_ == EntrySource::Flat; } - std::filesystem::path const& prescannedDestination(std::size_t entryIndex) const - { - return prescannedDestinations_[entryIndex]; - } + /** + * @brief Both ends of the entry at @p entryIndex. In tree mode the entry's names are joined below the roots, + * so an entry whose name the local side cannot hold fails with InvalidName. + */ + std::expected entryPaths(std::size_t entryIndex) const; + + /** + * @brief Path shown to the user for the entry, the remote side for both directions. + */ + std::string displayPath(std::size_t entryIndex) const; + + /** + * @brief The source path of the entry, the key a failure is reported under. + */ + std::string failureSource(std::size_t entryIndex) const; + + /** + * @brief The entry below the source root by its names as listed, for entries without valid paths. + */ + std::string describeEntry(std::size_t entryIndex) const; std::vector const& entries() const noexcept { @@ -146,31 +217,43 @@ class BulkTransferOperation : public Operation /** * @brief Creates the directory described by the entry at @p entryIndex. */ - virtual std::expected createDirectoryInStrand(std::size_t entryIndex) = 0; + virtual std::expected createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) = 0; /** * @brief Builds the child transfer for the file at @p entryIndex. * * @param onProgress Must be wired to the child's progress callback as (current, max). */ - virtual std::unique_ptr - makeChild(std::size_t entryIndex, std::function onProgress) = 0; + virtual std::unique_ptr makeChild( + std::size_t entryIndex, + EntryPaths const& paths, + std::function onProgress + ) = 0; /** - * @brief Path shown to the user for the entry (the remote side for both directions). + * @brief Whether a failed child may be recorded and skipped instead of failing the bulk. */ - virtual std::filesystem::path displayPath(std::size_t entryIndex) const = 0; + virtual bool isSkippableError(Error const& error) const = 0; /** - * @brief Whether a failed child may be recorded and skipped instead of failing the bulk. + * @brief Whether a directory that could not be created may be recorded and skipped instead of failing the bulk. + * Its descendants then fail on their own. */ - virtual bool isSkippableError(Error const& error) const = 0; + virtual bool isSkippableDirectoryError(Error const&) const + { + return false; + } /** * @brief Log prefix, e.g. "BulkDownloadOperation". */ virtual std::string_view logName() const noexcept = 0; + /** + * @brief The remote side is the source, as for a download. + */ + virtual bool sourceIsRemote() const noexcept = 0; + private: struct Slot { @@ -202,11 +285,24 @@ class BulkTransferOperation : public Operation std::int64_t updateBulkBytesPerSecond(std::uint64_t bytesNow); private: + /** + * @brief Where the entries came from. A scanned tree always holds its root, so an empty one + * means the scan never delivered. + */ + enum class EntrySource + { + None, + Tree, + Flat, + }; + SecureShell::SftpSession* sftp_; CommonOptions common_; + std::unique_ptr scanPhase_{}; + EntrySource entrySource_{EntrySource::None}; std::vector entries_{}; // Index aligned with entries_ in prescanned mode, empty in tree mode. - std::vector prescannedDestinations_{}; + std::vector prescannedPaths_{}; std::vector slots_{}; // Entries whose child could not get a transfer buffer; retried once a slot frees up. std::deque deferred_{}; @@ -217,7 +313,7 @@ class BulkTransferOperation : public Operation std::uint64_t completedBytes_{0}; std::uint64_t totalBytes_{0}; int budget_{1}; - std::vector> failedEntries_{}; + std::vector> failedEntries_{}; std::vector failedEntryIndices_{}; // Rolling bulk-level throughput. Per-file rates reset between files, which made the number // flicker; sampling the cumulative byte count keeps one coherent rate per bulk. diff --git a/backend/include/backend/sftp/bulk_upload_operation.hpp b/backend/include/backend/sftp/bulk_upload_operation.hpp index 3d52c246..4eb22e1e 100644 --- a/backend/include/backend/sftp/bulk_upload_operation.hpp +++ b/backend/include/backend/sftp/bulk_upload_operation.hpp @@ -13,8 +13,10 @@ class BulkUploadOperation : public BulkTransferOperation struct BulkUploadOperationOptions { BulkProgressCallback overallProgressCallback = [](SharedData::BulkProgress const&) {}; - std::filesystem::path remotePath{}; - std::filesystem::path localPath{}; + /** @brief Where the scanned directory is created, only a label for a flat list. */ + Paths::RemotePath remotePath{}; + /** @brief The scanned local directory, only a label for a flat list. */ + Paths::LocalPath localPath{}; UploadOperation::UploadOperationOptions individualOptions = {}; bool failFast{false}; // Files transferred at once, bounded by the queue's slot budget. @@ -34,31 +36,37 @@ class BulkUploadOperation : public BulkTransferOperation /** @brief See BulkDownloadOperation::PrescannedFile. */ struct PrescannedFile { - std::filesystem::path localSrc; - std::filesystem::path remoteDst; + Paths::LocalPath source; + Paths::RemotePath destination; std::uint64_t sizeBytes; + bool mayOverwrite{false}; }; void setPrescannedFileList(std::vector files); protected: std::expected prepareRootInStrand(SharedData::DirectoryEntry const& root) override; - std::expected createDirectoryInStrand(std::size_t entryIndex) override; - std::unique_ptr - makeChild(std::size_t entryIndex, std::function onProgress) override; - std::filesystem::path displayPath(std::size_t entryIndex) const override; + std::expected createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) override; + std::unique_ptr makeChild( + std::size_t entryIndex, + EntryPaths const& paths, + std::function onProgress + ) override; bool isSkippableError(Error const& error) const override; + bool sourceIsRemote() const noexcept override + { + return false; + } + std::string_view logName() const noexcept override { return "BulkUploadOperation"; } private: - std::filesystem::path localPathOf(std::size_t entryIndex) const; - std::filesystem::path remotePathOf(std::size_t entryIndex) const; std::filesystem::perms determinePerms(SharedData::DirectoryEntry const& entry) const; std::expected - createRemoteDirectoryInStrand(std::filesystem::path const& path, SharedData::DirectoryEntry const& entry); + createRemoteDirectoryInStrand(Paths::RemotePath const& path, SharedData::DirectoryEntry const& entry); private: UploadOperation::UploadOperationOptions individualOptions_; diff --git a/backend/include/backend/sftp/delete_operation.hpp b/backend/include/backend/sftp/delete_operation.hpp index 2486c119..8c907408 100644 --- a/backend/include/backend/sftp/delete_operation.hpp +++ b/backend/include/backend/sftp/delete_operation.hpp @@ -1,8 +1,11 @@ #pragma once #include +#include +#include #include #include +#include #include #include @@ -15,7 +18,8 @@ class DeleteOperation : public Operation { std::function filesRemovedProgress = [](std::string const&, auto, auto) {}; - std::filesystem::path remotePath{}; + /** @brief What is deleted, or for a recursive delete the root every scanned entry must lie within. */ + Paths::RemotePath remotePath{}; std::chrono::seconds futureTimeout{5}; bool recursive{false}; }; @@ -38,11 +42,39 @@ class DeleteOperation : public Operation std::expected work() override; + /** + * @brief The scanned content of remotePath, see @ref appendScanResult. + */ void setScanResult(std::vector&& entries, std::uint64_t /* totalBytes */); + /** + * @brief Adds the scanned content of @p root, which has to lie below it, otherwise nothing is deleted. + */ + void appendScanResult(Paths::RemotePath const& root, std::vector const& entries); + + /** + * @brief Deletes exactly @p targets, last first, without a scan. For the files of a bulk delete. + */ + void setTargets(std::vector targets); + + /** + * @brief Scans with @p scan before deleting; every scanned entry is deleted, deepest first. + */ + void scanFirst(std::unique_ptr scan); + + /** + * @brief Like @ref scanFirst for one more directory, so one operation deletes several directories. + */ + void addScanRoot(std::unique_ptr scan); + + bool scanning() const noexcept + { + return scanPhase_ != nullptr; + } + bool isBarrier() const noexcept override { - return false; + return scanPhase_ != nullptr; } /** @@ -61,23 +93,37 @@ class DeleteOperation : public Operation return SharedData::OperationType::Delete; } - std::filesystem::path remotePath() const + Paths::RemotePath const& remotePath() const { return remotePath_; } std::expected cancel(bool adoptCancelState) override; + /** + * @brief Items that could not be deleted, with why. The rest was still deleted. + */ + std::vector> const& getFailed() const noexcept + { + return failedEntries_; + } + private: - std::expected removeOnce(std::filesystem::path const& path); + std::expected removeOnce(Paths::RemotePath const& path); private: SecureShell::SftpSession* sftp_; - std::filesystem::path remotePath_; + Paths::RemotePath remotePath_; std::function filesRemovedProgress_; std::chrono::seconds futureTimeout_; std::size_t deletedThusFar_{0}; std::size_t totalToDelete_{0}; - std::vector entries_{}; + /** @brief Deleted from the back, so a scanned directory goes after its content. */ + std::vector targets_{}; + /** @brief A scanned entry was not a valid path within remotePath_, nothing may be deleted. */ + std::optional invalidScanEntry_{}; + std::vector> failedEntries_{}; + std::unique_ptr> scanPhase_{}; + bool scanDelivered_{false}; bool recursive_{false}; }; \ No newline at end of file diff --git a/backend/include/backend/sftp/download_operation.hpp b/backend/include/backend/sftp/download_operation.hpp index f727bfab..1f0296da 100644 --- a/backend/include/backend/sftp/download_operation.hpp +++ b/backend/include/backend/sftp/download_operation.hpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include @@ -23,8 +25,8 @@ class DownloadOperation : public Operation std::make_signed_t bytesPerSecond )> progressCallback = [](auto, auto, auto, auto) {}; - std::filesystem::path remotePath{}; - std::filesystem::path localPath{}; + Paths::RemotePath remotePath{}; + Paths::LocalPath localPath{}; std::string tempFileSuffix{".filepart"}; bool mayOverwrite{false}; bool reserveSpace{false}; @@ -42,6 +44,13 @@ class DownloadOperation : public Operation * may not yet exist locally. */ bool createMissingDirectories{false}; + /** @brief Mark the file as downloaded from the internet, see Utility::markAsDownloaded. */ + bool markAsDownloaded{true}; + /** + * @brief When set, a local path running through a symlink or junction below this directory is refused, so + * a link planted inside a downloaded tree cannot redirect the download out of it. + */ + Paths::LocalPath linkFreeRoot{}; }; SecureShell::ProcessingStrand* strand() const override @@ -82,12 +91,12 @@ class DownloadOperation : public Operation return SharedData::OperationType::Download; } - std::filesystem::path remotePath() const + Paths::RemotePath const& remotePath() const { return options_.remotePath; } - std::filesystem::path localPath() const + Paths::LocalPath const& localPath() const { return options_.localPath; } @@ -122,8 +131,14 @@ class DownloadOperation : public Operation std::expected openOrAdoptFileInStrand(SecureShell::IFileStream& stream); + /** + * @brief Refuses @p path when it runs through a symlink or junction, below the link free root when one is set, + * otherwise only at @p path itself. + */ + std::expected refuseLinkedPath(std::filesystem::path const& path) const; + std::expected - readSymlinkInStrand(std::filesystem::path const& remoteFullPath); + readSymlinkInStrand(Paths::RemotePath const& remoteFullPath); void cleanup(); @@ -131,7 +146,17 @@ class DownloadOperation : public Operation SecureShell::SftpSession* sftp_; std::weak_ptr fileStream_; DownloadOperationOptions options_; + /** @brief options_.remotePath, unset when none was given. */ + Paths::RemotePath remotePath_{}; + /** @brief options_.localPath for the filesystem calls, empty when none was given. */ + std::filesystem::path localNativePath_{}; std::ofstream localFile_; SecureShell::BufferLease buffer_; std::shared_ptr asyncTransferContext_; + /** @brief Bytes already present in an adopted partial file. */ + std::uint64_t resumeOffset_{0u}; + /** @brief Bytes in the temp file so far, counted instead of tellp, which append mode misreports on Windows. */ + std::uint64_t bytesInFile_{0u}; + /** @brief An adopted partial file already holds the whole content. */ + bool transferComplete_{false}; }; \ No newline at end of file diff --git a/backend/include/backend/sftp/local_delete_operation.hpp b/backend/include/backend/sftp/local_delete_operation.hpp new file mode 100644 index 00000000..a60c00d9 --- /dev/null +++ b/backend/include/backend/sftp/local_delete_operation.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +/** + * @brief Deletes local items through the queue, so they honour pause and show progress like remote deletes. + * Exactly the given items are removed: a directory only once empty, a link as the link itself. + */ +class LocalDeleteOperation : public Operation +{ + public: + struct LocalDeleteOperationOptions + { + std::function itemsRemovedProgress = + [](std::string const&, auto, auto) {}; + /** @brief Deleted from the back, so a directory goes before its content in this list. */ + std::vector targets{}; + }; + + explicit LocalDeleteOperation(LocalDeleteOperationOptions options); + ~LocalDeleteOperation() override = default; + LocalDeleteOperation(LocalDeleteOperation const&) = delete; + LocalDeleteOperation(LocalDeleteOperation&&) = delete; + LocalDeleteOperation& operator=(LocalDeleteOperation const&) = delete; + LocalDeleteOperation& operator=(LocalDeleteOperation&&) = delete; + + SecureShell::ProcessingStrand* strand() const override + { + return nullptr; + } + + bool usesStrand() const noexcept override + { + return false; + } + + SharedData::OperationType type() const override + { + return SharedData::OperationType::LocalDelete; + } + + bool isBarrier() const noexcept override + { + return false; + } + + int parallelWorkDoable(int) const noexcept override + { + return 1; + } + + std::expected work() override; + std::expected cancel(bool adoptCancelState) override; + + /** + * @brief Items that could not be deleted, with why. The rest was still deleted. + */ + std::vector> const& getFailed() const noexcept + { + return failedEntries_; + } + + private: + LocalDeleteOperationOptions options_; + std::vector> failedEntries_{}; + std::size_t deletedThusFar_{0}; +}; diff --git a/backend/include/backend/sftp/local_scan_operation.hpp b/backend/include/backend/sftp/local_scan_operation.hpp index 6ae09011..cf4d8da8 100644 --- a/backend/include/backend/sftp/local_scan_operation.hpp +++ b/backend/include/backend/sftp/local_scan_operation.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -22,7 +23,7 @@ class LocalScanOperation : public Operation { std::function progressCallback = [](auto, auto, auto) {}; - std::filesystem::path localPath{}; + Paths::LocalPath localPath{}; bool respectIgnoreFiles{false}; /// When false, only the root directory is listed; subdirectories are not descended into. bool recursive{true}; @@ -31,6 +32,8 @@ class LocalScanOperation : public Operation /// When true, the operation builds a sorted @ref SharedData::Sync::ScanNode tree /// instead of a flat entry vector. bool buildTree{false}; + /** @brief Tree mode only: names ending in one of these, the temp files of transfers, stay excluded. */ + std::vector excludedSuffixes{}; }; LocalScanOperation(ScanOperationOptions options); @@ -67,7 +70,7 @@ class LocalScanOperation : public Operation return 1; } - std::filesystem::path localPath() const + Paths::LocalPath const& localPath() const { return localPath_; } @@ -76,13 +79,13 @@ class LocalScanOperation : public Operation auto withWalkerDo(auto&& fn) { - auto scan = [this](std::filesystem::path const& path) + auto scan = [this](Paths::RelativePath const& relative) { - return scanner(path); + return scanner(relative); }; using WalkerType = Utility::DeepDirectoryWalker; if (!walker_) - walker_ = std::make_unique(localPath_, std::move(scan)); + walker_ = std::make_unique(std::move(scan)); return fn(static_cast(*walker_)); } @@ -92,13 +95,22 @@ class LocalScanOperation : public Operation */ auto withTreeWalkerDo(auto&& fn) { - auto scan = [this](std::filesystem::path const& path) + auto scan = [this](Paths::RelativePath const& relative) { - return scanner(path); + return scanner(relative); }; using WalkerType = SharedData::Sync::TreeDirectoryWalker; if (!walker_) - walker_ = std::make_unique(localPath_, std::move(scan), recursive_); + walker_ = std::make_unique( + std::move(scan), + recursive_, + [](Error const& error) + { + return error.type == ErrorType::DirectoryUnreadable; + }, + /*requireLocalNames=*/false, + excludedSuffixes_ + ); return fn(static_cast(*walker_)); } @@ -116,6 +128,35 @@ class LocalScanOperation : public Operation ); } + /** + * @brief Full paths of listed entries whose name is not a valid path segment, for display. Flat scans only, + * the tree of a sync scan keeps them as invalid name nodes. + */ + std::vector rejectedPaths() + { + return withWalkerDo( + [this](auto& walker) + { + std::vector paths; + auto root = localPath_.toUtf8(); + if (!root.ends_with('/')) + root.push_back('/'); + for (auto const& entry : walker.rejectedEntries()) + paths.push_back(root + walker.describeRejected(entry)); + return paths; + } + ); + } + + /** + * @brief Full paths of directories below the root that could not be listed. Flat scans only, they are left out + * of the entries; the tree of a sync scan keeps them as unreadable nodes. + */ + std::vector const& unreadableDirectories() const noexcept + { + return unreadableDirectories_; + } + /** * @brief Eject the scanned node tree. Only valid when @ref buildsTree() is true. */ @@ -136,16 +177,32 @@ class LocalScanOperation : public Operation std::uint64_t totalBytes() const; - std::expected, Error> scanner(std::filesystem::path const& path); + /** + * @brief Lists the directory at @p relative below the scanned root. + */ + std::expected, Error> scanner(Paths::RelativePath const& relative); + + private: + /** + * @brief A directory below the root that vanished or denies listing is unreadable, a sync tree keeps + * it as such. Any other listing error fails the scan. + */ + std::expected, Error> listingFailed( + std::filesystem::path const& path, + Paths::RelativePath const& relative, + std::error_code const& errorCode + ); private: - std::filesystem::path localPath_; + std::vector unreadableDirectories_{}; + Paths::LocalPath localPath_; std::function progressCallback_; bool respectIgnoreFiles_; bool recursive_; bool ignoreHidden_; bool buildTree_; + std::vector excludedSuffixes_; bool rootScanned_{false}; SharedData::IgnoreMatcher ignoreMatcher_; std::unique_ptr walker_; diff --git a/backend/include/backend/sftp/operation.hpp b/backend/include/backend/sftp/operation.hpp index 45b436dd..683d2f28 100644 --- a/backend/include/backend/sftp/operation.hpp +++ b/backend/include/backend/sftp/operation.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -164,6 +165,16 @@ class Operation state_ = newState; } + /** + * @brief The error of an async transfer that ended without completing. + */ + static Error asyncTransferError(SecureShell::AsyncTransferContext const& context) + { + if (auto error = context.error(); error.has_value()) + return Error{.type = ErrorType::SftpError, .sftpError = std::move(error)}; + return Error{.type = ErrorType::SftpError, .extraInfo = "The transfer ended before it completed"}; + } + protected: OperationState state_{OperationState::NotStarted}; std::optional error_{std::nullopt}; diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index 69e45c22..6af445b5 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -10,12 +10,16 @@ #include #include #include -#include +#include +#include +#include +#include #include #include #include #include #include +#include #include #include @@ -82,8 +86,8 @@ class OperationQueue std::expected addDownloadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::filesystem::path const& localPath, - std::filesystem::path const& remotePath, + Paths::LocalPath const& localPath, + Paths::RemotePath const& remotePath, bool allowOverwrite, bool isBigFile, bool insertRefresh, @@ -94,8 +98,8 @@ class OperationQueue std::expected addUploadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::filesystem::path const& localPath, - std::filesystem::path const& remotePath, + Paths::LocalPath const& localPath, + Paths::RemotePath const& remotePath, bool allowOverwrite, bool isBigFile, bool insertRefresh, @@ -121,8 +125,8 @@ class OperationQueue std::expected addArchiveDownloadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::vector entries, - std::filesystem::path const& localArchivePath, + std::vector roots, + Paths::LocalPath const& localArchivePath, TarArchive::Compression compression, int compressionLevel, bool mayOverwrite, @@ -138,18 +142,22 @@ class OperationQueue std::expected addArchiveUploadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::vector localPaths, - std::filesystem::path const& remoteArchivePath, + std::vector localPaths, + Paths::RemotePath const& remoteArchivePath, TarArchive::Compression compression, int compressionLevel, bool mayOverwrite, SharedData::OperationMode mode = SharedData::OperationMode::Queued ); + /** + * @brief Queues the delete of @p remotePath, refused for the root and for the home directory or any directory + * that contains it. + */ std::expected addDeleteOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::filesystem::path const& remotePath, + Paths::RemotePath const& remotePath, bool recursive, bool insertRefresh, SharedData::OperationMode mode = SharedData::OperationMode::Queued @@ -172,7 +180,7 @@ class OperationQueue */ std::size_t addBulkDownloadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkAddRequest const& request, + SharedData::BulkDownloadRequest const& request, std::function operationIdFor, Ids::OperationId bulkCardId ); @@ -187,7 +195,7 @@ class OperationQueue * @p operationIdFor. */ std::size_t addBulkUploadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkAddRequest const& request, + SharedData::BulkUploadRequest const& request, std::function operationIdFor, Ids::OperationId bulkCardId ); @@ -200,23 +208,50 @@ class OperationQueue * recursively removed. * * @param sftp SFTP session to use. - * @param request Bulk request — `dst` is unused for delete, only - * `src` (target path) and `isDirectory` matter. + * @param request The remote items; directories are scanned and deleted with their content. * @param bulkOperationId Single OperationId for the aggregate file-bulk * card. Per-directory ids are generated locally. * @return Number of entries successfully enqueued. */ std::size_t addBulkDeleteOperation( SecureShell::SftpSession& sftp, - SharedData::BulkAddRequest const& request, + SharedData::BulkDeleteRequest const& request, Ids::OperationId bulkOperationId ); + /** + * @brief Queues a resolved sync plan under the operation ids of @p request: files of each direction as one + * bulk transfer, remote and local deletes as one operation each. Directories without content are created + * right away. Every path comes from @p plan, which resolved them below the sync roots. + */ + SharedData::Sync::SyncExecutionResult executeSyncPlan( + SecureShell::SftpSession& sftp, + SyncSession const& session, + SyncSession::ResolvedPlan plan, + SharedData::Sync::SyncExecutionRequest const& request + ); + + /** + * @brief Queues the delete of local items as one operation with one card. Refused as a whole when one of them + * is a root directory or contains the home directory. + */ + std::expected addLocalDeleteOperation( + Ids::OperationId operationId, + std::vector targets, + SharedData::OperationMode mode = SharedData::OperationMode::Queued + ); + + /** + * @brief Refuses an unset delete target, the root, and the home directory or any directory that contains it. + */ + std::expected + refuseProtectedDeleteTarget(SecureShell::SftpSession& sftp, Paths::RemotePath const& target); + std::expected addRenameOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::filesystem::path const& sourcePath, - std::filesystem::path const& destinationPath, + Paths::RemotePath const& sourcePath, + Paths::RemotePath const& destinationPath, SharedData::OperationMode mode = SharedData::OperationMode::Queued ); @@ -239,8 +274,8 @@ class OperationQueue Ids::SyncSessionId syncSessionId, Ids::OperationId remoteScanId, Ids::OperationId localScanId, - std::filesystem::path const& remotePath, - std::filesystem::path const& localPath, + Paths::RemotePath const& remotePath, + Paths::LocalPath const& localPath, bool respectIgnoreFiles, bool recursive, bool ignoreHidden @@ -265,6 +300,11 @@ class OperationQueue bool paused() const; void paused(bool pause); + /** + * @brief Observes every completion right before it is sent to the frontend. Used by tests. + */ + void setCompletionObserver(std::function observer); + bool hasPriorityWork() const; /** @@ -303,12 +343,23 @@ class OperationQueue private: bool workQueue(std::deque>>& queue); + /** + * @brief Tells the sync session waiting for the scan @p id, if any, that it failed or was cancelled. + */ + void failSyncScan(Ids::OperationId const& id, std::string const& reason); + + /** + * @brief Hands the tree of a completed standalone scan to the sync session registered for its id. + */ + void deliverSyncScan(Ids::OperationId const& id, Operation& operation); + private: Persistence::SftpOptions sftpOpts_{}; Ids::SessionId sessionId_{}; std::deque>> priorityOperations_{}; std::deque>> operations_{}; std::atomic_bool paused_{true}; + std::function completionObserver_{}; // Transfer slots handed out per queue tick. int parallelism_{1}; // Upper bound on the slots a single bulk transfer may take, clamped to parallelism_. @@ -316,7 +367,8 @@ class OperationQueue // Keyed by operationId.value(). Called when a sync-only scan completes and // hands off its ScanNode tree. The scan is known to be build-tree mode so // @ref ScanOperation::ejectScanTree() has the payload. - std::map> syncScanCallbacks_{}; + std::map)>> + syncScanCallbacks_{}; // SyncSessions active on this channel. Keyed by SyncSessionId.value() because // Ids::IdHash isn't specialized on the strongly-typed id; the existing @@ -327,4 +379,7 @@ class OperationQueue // op and erased once the op completes (success or failure). Keyed by // the bulk's OperationId. Read by dumpBulkResumes() during teardown. std::unordered_map bulkResumeStash_{}; + + /** @brief The server's home directory, asked once for the delete guard. */ + std::optional remoteHome_{}; }; diff --git a/backend/include/backend/sftp/rename_operation.hpp b/backend/include/backend/sftp/rename_operation.hpp index 8fa0bb71..d573c0fe 100644 --- a/backend/include/backend/sftp/rename_operation.hpp +++ b/backend/include/backend/sftp/rename_operation.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -11,8 +12,8 @@ class RenameOperation : public Operation public: struct RenameOperationOptions { - std::filesystem::path sourcePath{}; - std::filesystem::path destinationPath{}; + Paths::RemotePath sourcePath{}; + Paths::RemotePath destinationPath{}; std::chrono::seconds futureTimeout{5}; }; @@ -51,19 +52,19 @@ class RenameOperation : public Operation return SharedData::OperationType::Rename; } - std::filesystem::path const& sourcePath() const + Paths::RemotePath const& sourcePath() const { return sourcePath_; } - std::filesystem::path const& destinationPath() const + Paths::RemotePath const& destinationPath() const { return destinationPath_; } private: SecureShell::SftpSession* sftp_; - std::filesystem::path sourcePath_; - std::filesystem::path destinationPath_; + Paths::RemotePath sourcePath_; + Paths::RemotePath destinationPath_; std::chrono::seconds futureTimeout_; }; diff --git a/backend/include/backend/sftp/resume_check.hpp b/backend/include/backend/sftp/resume_check.hpp new file mode 100644 index 00000000..b81b53ff --- /dev/null +++ b/backend/include/backend/sftp/resume_check.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace ResumeCheck +{ + /** @brief How many bytes before the resume point have to agree. */ + constexpr std::uint64_t comparedBytes = 64u * 1024u; + + /** + * @brief Whether the bytes right before @p size agree in the local file and the remote stream, so a leftover + * partial file belongs to this source and can be continued. A partial file of another program or of an + * older version of the source fails this. The position of @p remote is unspecified afterwards. + * + * @return False when they differ or either side is shorter than @p size. + */ + std::expected + tailsMatchInStrand(SecureShell::IFileStream& remote, std::filesystem::path const& local, std::uint64_t size); +} diff --git a/backend/include/backend/sftp/scan_operation.hpp b/backend/include/backend/sftp/scan_operation.hpp index 5b50e696..2d85228a 100644 --- a/backend/include/backend/sftp/scan_operation.hpp +++ b/backend/include/backend/sftp/scan_operation.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -20,7 +21,7 @@ class ScanOperation : public Operation { std::function progressCallback = [](auto, auto, auto) {}; - std::filesystem::path remotePath{}; + Paths::RemotePath remotePath{}; std::chrono::seconds futureTimeout{5}; bool respectIgnoreFiles{false}; /// When false, only the root directory is listed; subdirectories are not descended into. @@ -31,6 +32,8 @@ class ScanOperation : public Operation /// instead of a flat entry vector. Only one shape is active at a time; use /// @ref ejectScanTree() in that case and @ref ejectEntries() otherwise. bool buildTree{false}; + /** @brief Tree mode only: names ending in one of these, the temp files of transfers, stay excluded. */ + std::vector excludedSuffixes{}; }; SecureShell::ProcessingStrand* strand() const override; @@ -67,7 +70,7 @@ class ScanOperation : public Operation return 1; } - std::filesystem::path remotePath() const + Paths::RemotePath const& remotePath() const { return remotePath_; } @@ -76,13 +79,13 @@ class ScanOperation : public Operation auto withWalkerDo(auto&& fn) { - auto scan = [this](std::filesystem::path const& path) + auto scan = [this](Paths::RelativePath const& relative) { - return scanner(path); + return scanner(relative); }; using WalkerType = Utility::DeepDirectoryWalker; if (!walker_) - walker_ = std::make_unique(remotePath_, std::move(scan)); + walker_ = std::make_unique(std::move(scan)); return fn(static_cast(*walker_)); } @@ -92,13 +95,22 @@ class ScanOperation : public Operation */ auto withTreeWalkerDo(auto&& fn) { - auto scan = [this](std::filesystem::path const& path) + auto scan = [this](Paths::RelativePath const& relative) { - return scanner(path); + return scanner(relative); }; using WalkerType = SharedData::Sync::TreeDirectoryWalker; if (!walker_) - walker_ = std::make_unique(remotePath_, std::move(scan), recursive_); + walker_ = std::make_unique( + std::move(scan), + recursive_, + [](Error const& error) + { + return error.type == ErrorType::DirectoryUnreadable; + }, + /*requireLocalNames=*/true, + excludedSuffixes_ + ); return fn(static_cast(*walker_)); } @@ -116,6 +128,24 @@ class ScanOperation : public Operation ); } + /** + * @brief Full paths of listed entries whose name is not a valid path segment, for display. Flat scans only, + * the tree of a sync scan keeps them as invalid name nodes. + */ + std::vector rejectedPaths() + { + return withWalkerDo( + [this](auto& walker) + { + std::vector paths; + const auto root = remotePath_.isRoot() ? std::string{} : remotePath_.value(); + for (auto const& entry : walker.rejectedEntries()) + paths.push_back(root + "/" + walker.describeRejected(entry)); + return paths; + } + ); + } + /** * @brief Eject the scanned node tree. Only valid when @ref buildsTree() is true. */ @@ -136,11 +166,14 @@ class ScanOperation : public Operation std::uint64_t totalBytes() const; - std::expected, Error> scanner(std::filesystem::path const& path); + /** + * @brief Lists the directory at @p relative below the scanned root. + */ + std::expected, Error> scanner(Paths::RelativePath const& relative); private: SecureShell::SftpSession* sftp_; - std::filesystem::path remotePath_; + Paths::RemotePath remotePath_; std::function progressCallback_; std::chrono::seconds futureTimeout_; @@ -148,6 +181,7 @@ class ScanOperation : public Operation bool recursive_; bool ignoreHidden_; bool buildTree_; + std::vector excludedSuffixes_; bool rootScanned_{false}; SharedData::IgnoreMatcher ignoreMatcher_; std::unique_ptr walker_; diff --git a/backend/include/backend/sftp/scan_phase.hpp b/backend/include/backend/sftp/scan_phase.hpp new file mode 100644 index 00000000..ad5a3a82 --- /dev/null +++ b/backend/include/backend/sftp/scan_phase.hpp @@ -0,0 +1,93 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +/** + * @brief A scan that belongs to the operation consuming its result. The consumer steps it before + * its own work, so cancelling, reordering or failing the consumer acts on its scan too. + */ +class ScanPhaseDriver +{ + public: + virtual ~ScanPhaseDriver() = default; + + /** + * @brief Steps the current scan once. Complete when every root was scanned and delivered. + */ + virtual std::expected step() = 0; + + virtual void cancel(bool adoptCancelState) = 0; +}; + +/** + * @brief Scans one or more roots in order and hands each finished scan to the consumer. + * + * @tparam ScanT ScanOperation or LocalScanOperation. + */ +template +class ScanPhase : public ScanPhaseDriver +{ + public: + /** + * @brief Receives a finished scan; the consumer ejects its entries. + */ + using DeliverFunction = std::function; + + explicit ScanPhase(DeliverFunction deliver) + : deliver_{std::move(deliver)} + {} + + void addRoot(std::unique_ptr scan) + { + scans_.push_back(std::move(scan)); + } + + std::expected step() override + { + if (scans_.empty()) + return Operation::WorkStatus::Complete; + + auto& current = *scans_.front(); + auto result = current.work(); + if (!result.has_value()) + return std::unexpected(std::move(result).error()); + if (result.value() != Operation::WorkStatus::Complete) + return Operation::WorkStatus::MoreWork; + + deliver_(current); + scans_.pop_front(); + return scans_.empty() ? Operation::WorkStatus::Complete : Operation::WorkStatus::MoreWork; + } + + void cancel(bool adoptCancelState) override + { + for (auto& scan : scans_) + std::ignore = scan->cancel(adoptCancelState); + } + + private: + DeliverFunction deliver_; + std::deque> scans_{}; +}; + +/** + * @brief Steps @p phase and drops it once every root is delivered. + * + * @return MoreWork while scanning or right after the last delivery, the scan's error otherwise. + */ +template +std::expected stepScanPhase(std::unique_ptr& phase) +{ + auto result = phase->step(); + if (!result.has_value()) + return result; + if (result.value() == Operation::WorkStatus::Complete) + phase.reset(); + return Operation::WorkStatus::MoreWork; +} diff --git a/backend/include/backend/sftp/upload_operation.hpp b/backend/include/backend/sftp/upload_operation.hpp index 2f5a7a3e..618e5251 100644 --- a/backend/include/backend/sftp/upload_operation.hpp +++ b/backend/include/backend/sftp/upload_operation.hpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include @@ -23,8 +25,8 @@ class UploadOperation : public Operation std::make_signed_t bytesPerSecond )> progressCallback = [](auto, auto, auto, auto) {}; - std::filesystem::path remotePath{}; - std::filesystem::path localPath{}; + Paths::RemotePath remotePath{}; + Paths::LocalPath localPath{}; std::string tempFileSuffix{".filepart"}; bool mayOverwrite{false}; bool tryContinue{false}; @@ -75,12 +77,12 @@ class UploadOperation : public Operation return SharedData::OperationType::Upload; } - std::filesystem::path remotePath() const + Paths::RemotePath const& remotePath() const { return options_.remotePath; } - std::filesystem::path localPath() const + Paths::LocalPath const& localPath() const { return options_.localPath; } @@ -115,6 +117,11 @@ class UploadOperation : public Operation /// Remote-heavy portion of finalize() — runs under a single strand umbrella. std::expected finalizeInStrand(); + /** + * @brief Whether the existing remote temp file of @p tempSize bytes ends like the same range of the source. + */ + bool tempBelongsToSourceInStrand(std::uint64_t tempSize); + /** @brief Ensures every ancestor directory of @p dir exists on the remote, creating * any missing ones. Idempotent: existing directories are left alone. * @@ -122,7 +129,7 @@ class UploadOperation : public Operation * the parent, so deeply-nested missing paths are created in one pass. * Must be called from within the processing thread. */ - std::expected ensureRemoteDirectoryExistsInStrand(std::filesystem::path const& dir); + std::expected ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& dir); std::filesystem::perms determinePerms(std::filesystem::perms localPerms) const; @@ -132,6 +139,12 @@ class UploadOperation : public Operation SecureShell::SftpSession* sftp_; std::weak_ptr fileStream_; UploadOperationOptions options_; + /** @brief options_.remotePath, unset when none was given. */ + Paths::RemotePath remotePath_{}; + /** @brief options_.localPath for the filesystem calls, empty when none was given. */ + std::filesystem::path localNativePath_{}; + /** @brief Where the upload is written before it replaces remotePath_. */ + Paths::RemotePath tempPath_{}; std::ifstream localFile_; bool isSymlink_{false}; std::size_t leftToUpload_{0}; diff --git a/backend/include/backend/sync/sync_session.hpp b/backend/include/backend/sync/sync_session.hpp index 88f883af..8f4a4f51 100644 --- a/backend/include/backend/sync/sync_session.hpp +++ b/backend/include/backend/sync/sync_session.hpp @@ -1,6 +1,9 @@ #pragma once #include +#include +#include +#include #include #include #include @@ -50,8 +53,8 @@ class SyncSession : public std::enable_shared_from_this struct Options { Ids::SyncSessionId sessionId{}; - std::filesystem::path localRoot{}; - std::filesystem::path remoteRoot{}; + Paths::LocalPath localRoot{}; + Paths::RemotePath remoteRoot{}; /// Number of scan entries (sum of both sides) above which /// @ref SharedData::Sync::DiffSummary::heavyCompare is set to true. std::uint64_t heavyCompareThreshold{5000}; @@ -73,6 +76,31 @@ class SyncSession : public std::enable_shared_from_this return generation_; } + Paths::LocalPath const& localRoot() const noexcept + { + return options_.localRoot; + } + + Paths::RemotePath const& remoteRoot() const noexcept + { + return options_.remoteRoot; + } + + /** + * @brief Where a row of the diff lives on both sides. + */ + struct ResolvedPaths + { + Paths::LocalPath local; + Paths::RemotePath remote; + }; + + /** + * @brief The local and remote path of @p relative below the session roots. Fails for a name the local + * platform cannot store, nothing can leave the roots since @p relative holds no '..'. + */ + std::expected resolve(Paths::RelativePath const& relative) const; + boost::asio::strand& strand() { return strand_; @@ -142,6 +170,28 @@ class SyncSession : public std::enable_shared_from_this std::unordered_set const& selectedRelKeys ) const; + struct ResolvedPlanEntry + { + SharedData::Sync::EnqueuePlanEntry entry; + ResolvedPaths paths; + }; + + struct ResolvedPlan + { + std::vector entries{}; + /** @brief Keys of entries that have no path on one side, like a name Windows cannot store. */ + std::vector refused{}; + }; + + /** + * @brief @ref buildEnqueuePlanInStrand with every entry resolved below the session roots. + * Precondition: running on @ref strand(). + */ + ResolvedPlan resolvePlanInStrand( + SharedData::Sync::DiffSection section, + std::unordered_set const& selectedRelKeys + ) const; + private: SharedData::Sync::DiffTreeStore& storeFor(SharedData::Sync::DiffSection section); SharedData::Sync::DiffTreeStore const& storeFor(SharedData::Sync::DiffSection section) const; diff --git a/backend/include/backend/windows/main_windows.hpp b/backend/include/backend/windows/main_windows.hpp index 95bb41ee..fca772ad 100644 --- a/backend/include/backend/windows/main_windows.hpp +++ b/backend/include/backend/windows/main_windows.hpp @@ -8,4 +8,10 @@ struct Main::PlatformSpecifics PlatformSpecifics(Nui::Window& window, Nui::RpcHub& hub); EnableWindowsDragDrop enableDragDrop; -}; \ No newline at end of file +}; + +/** + * @brief Attaches to the console of the process that started this one, if there is one, and points the standard + * streams at it. The program is a GUI program, without this its output goes nowhere. + */ +void attachParentConsole(); \ No newline at end of file diff --git a/backend/source/backend/CMakeLists.txt b/backend/source/backend/CMakeLists.txt index 7e5abd0c..c2071970 100644 --- a/backend/source/backend/CMakeLists.txt +++ b/backend/source/backend/CMakeLists.txt @@ -4,6 +4,8 @@ add_library( session_manager.cpp session.cpp rpc_filesystem.cpp + local_delete_guard.cpp + existence_check.cpp rpc_system.cpp theme_finder.cpp dropped_entry.cpp @@ -22,6 +24,7 @@ add_library( sftp/operation_queue.cpp sftp/download_operation.cpp sftp/upload_operation.cpp + sftp/resume_check.cpp sftp/scan_operation.cpp sftp/local_scan_operation.cpp sftp/bulk_transfer_operation.cpp @@ -29,6 +32,7 @@ add_library( sftp/bulk_upload_operation.cpp sftp/bulk_resume_registry.cpp sftp/delete_operation.cpp + sftp/local_delete_operation.cpp sftp/rename_operation.cpp sftp/archive_download_operation.cpp sftp/archive_upload_operation.cpp diff --git a/backend/source/backend/dropped_entry.cpp b/backend/source/backend/dropped_entry.cpp index 5d75e1b0..1e76b3e7 100644 --- a/backend/source/backend/dropped_entry.cpp +++ b/backend/source/backend/dropped_entry.cpp @@ -1,11 +1,15 @@ #include -SharedData::DirectoryEntry leanDirectoryEntryFromPath(std::filesystem::path const& path) +std::optional droppedItemFromPath(std::filesystem::path const& path) { using FileType = SharedData::DirectoryEntry::FileType; - return SharedData::DirectoryEntry{ - .path = path, + auto localPath = Paths::LocalPath::make(path); + if (!localPath) + return std::nullopt; + const auto name = localPath->filename(); + auto entry = SharedData::DirectoryEntry{ + .name = name ? name->value() : std::string{}, .type = [&path]() { std::error_code statusError; @@ -28,4 +32,5 @@ SharedData::DirectoryEntry leanDirectoryEntryFromPath(std::filesystem::path cons return FileType::Unknown; }() }; + return SharedData::DroppedItem{.path = *std::move(localPath), .entry = std::move(entry)}; } diff --git a/backend/source/backend/existence_check.cpp b/backend/source/backend/existence_check.cpp new file mode 100644 index 00000000..85b293ea --- /dev/null +++ b/backend/source/backend/existence_check.cpp @@ -0,0 +1,131 @@ +#include +#include + +#include +#include + +#include +#include +#include +#include + +std::expected, std::string> localPathsExist(std::vector const& paths) +{ + std::vector results; + results.reserve(paths.size()); + for (auto const& pathUtf8 : paths) + { + const auto path = Paths::LocalPath::fromUtf8(pathUtf8); + if (!path) + return std::unexpected(path.error().toString()); + std::error_code ec; + const auto status = std::filesystem::symlink_status(path->native(), ec); + if (status.type() == std::filesystem::file_type::not_found) + { + results.push_back(false); + continue; + } + if (ec) + return std::unexpected(fmt::format("'{}': {}", pathUtf8, ec.message())); + results.push_back(true); + } + return results; +} + +namespace +{ + // Below this many paths a stat per path is cheaper than reading a potentially huge directory in full. + constexpr std::size_t listingThreshold = 10; + + std::expected, std::string> remotePathsExistByStat( + SecureShell::SftpSession& session, + std::vector const& paths, + std::chrono::milliseconds timeout + ) + { + std::vector results; + results.reserve(paths.size()); + for (auto const& path : paths) + { + auto future = session.lstat(path); + if (future.wait_for(timeout) != std::future_status::ready) + return std::unexpected(fmt::format("'{}': stat timed out", path.value())); + const auto result = future.get(); + if (!result && result.error().sftpError != SSH_FX_NO_SUCH_FILE) + return std::unexpected(fmt::format("'{}': {}", path.value(), result.error().toString())); + results.push_back(result.has_value()); + } + return results; + } + + /** + * @brief One listing per parent directory instead of a stat per path. A parent that does not exist yet holds + * nothing. + */ + std::expected, std::string> remotePathsExistByListing( + SecureShell::SftpSession& session, + std::vector const& paths, + std::chrono::milliseconds timeout + ) + { + const auto listParent = + [&session, timeout](Paths::RemotePath const& parent) -> std::expected, std::string> + { + auto future = session.listDirectory(parent); + if (future.wait_for(timeout) != std::future_status::ready) + return std::unexpected(fmt::format("'{}': listing timed out", parent.value())); + const auto result = future.get(); + if (!result) + { + if (result.error().sftpError == SSH_FX_NO_SUCH_FILE) + return std::unordered_set{}; + return std::unexpected(fmt::format("'{}': {}", parent.value(), result.error().toString())); + } + std::unordered_set names; + names.reserve(result->size()); + for (auto const& entry : *result) + names.insert(entry.name); + return names; + }; + + std::unordered_map> listingByParent; + std::vector results; + results.reserve(paths.size()); + for (auto const& path : paths) + { + const auto parent = path.parent(); + if (!parent) + return std::unexpected(fmt::format("'{}' has no parent directory", path.value())); + auto iter = listingByParent.find(parent->value()); + if (iter == listingByParent.end()) + { + auto listing = listParent(*parent); + if (!listing) + return std::unexpected(listing.error()); + iter = listingByParent.emplace(parent->value(), *std::move(listing)).first; + } + results.push_back(iter->second.contains(std::string{path.filename()->view()})); + } + return results; + } +} + +std::expected, std::string> remotePathsExist( + SecureShell::SftpSession& session, + std::vector const& paths, + std::chrono::milliseconds timeout +) +{ + std::vector remotePaths; + remotePaths.reserve(paths.size()); + for (auto const& path : paths) + { + auto remotePath = Paths::RemotePath::make(path); + if (!remotePath) + return std::unexpected(remotePath.error().toString()); + remotePaths.push_back(*std::move(remotePath)); + } + if (remotePaths.size() < listingThreshold) + return remotePathsExistByStat(session, remotePaths, timeout); + return remotePathsExistByListing(session, remotePaths, timeout); +} diff --git a/backend/source/backend/file_tracking/change_filter.cpp b/backend/source/backend/file_tracking/change_filter.cpp index 96bbd898..298c3aff 100644 --- a/backend/source/backend/file_tracking/change_filter.cpp +++ b/backend/source/backend/file_tracking/change_filter.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -35,7 +36,7 @@ namespace FileTracking bool isTransientFile(std::string const& filename, std::string const& downloadTempSuffix) { - const auto name = std::filesystem::path{filename}.filename().string(); + const auto name = Paths::nativeToUtf8(Paths::nativeFromUtf8(filename).filename()); if (name.starts_with(".goutputstream")) return true; if (!downloadTempSuffix.empty() && name.size() > downloadTempSuffix.size() && diff --git a/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp b/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp index 68cc1e8c..1191f3d5 100644 --- a/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp +++ b/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp @@ -1,18 +1,35 @@ #include +#include +#include +#include -#include #include #include #include #include +#include #include namespace FileTracking { namespace { + /** + * @brief Parse a metadata file, the stream is closed again on return. + * + * @param metaPath Path to the metadata.json file. + * @return The parsed json. + */ + nlohmann::json readMetadata(std::filesystem::path const& metaPath) + { + std::ifstream metaIn{metaPath}; + nlohmann::json meta; + metaIn >> meta; + return meta; + } + /** * @brief Scan @p tempRootDir for dead instance directories and handle them. * @@ -136,7 +153,7 @@ namespace FileTracking Log::info("forceCleanOrphanedInstanceDirs: removing orphaned instance '{}'", dirPath.filename().string()); std::error_code rmErr; - fs::remove_all(dirPath, rmErr); + Utility::removeAllNoFollow(dirPath, rmErr); if (rmErr) Log::warn("forceCleanOrphanedInstanceDirs: failed to remove '{}': {}", dirPath.filename().string(), rmErr.message()); else @@ -222,45 +239,15 @@ namespace FileTracking continue; } + // The metadata stream must be closed before remove_all, Windows refuses to delete open files. + std::optional deadAt; try { - std::ifstream metaIn{metaPath}; - nlohmann::json meta; - metaIn >> meta; - + auto const meta = readMetadata(metaPath); if (meta.contains("deadAt") && !meta["deadAt"].is_null()) - { - auto deadAtTs = meta["deadAt"].get(); - auto deadAt = std::chrono::system_clock::from_time_t( - static_cast(deadAtTs) + deadAt = std::chrono::system_clock::from_time_t( + static_cast(meta["deadAt"].get()) ); - auto elapsed = - std::chrono::duration_cast(now - deadAt); - - if (elapsed >= retentionHours) - { - Log::info( - "Removing dead instance '{}' (dead for {} hours)", - dirName, - elapsed.count() - ); - std::error_code rmErr; - fs::remove_all(dirPath, rmErr); - if (rmErr) - { - Log::warn( - "Failed to remove dead instance '{}': {}", - dirName, - rmErr.message() - ); - } - } - // else: dead but within retention window, leave it - } - else - { - writeDeadAt(); - } } catch (std::exception const& exc) { @@ -269,8 +256,23 @@ namespace FileTracking dirName, exc.what() ); + } + + if (!deadAt) + { writeDeadAt(); + continue; } + + auto const elapsed = std::chrono::duration_cast(now - *deadAt); + if (elapsed < retentionHours) + continue; + + Log::info("Removing dead instance '{}' (dead for {} hours)", dirName, elapsed.count()); + std::error_code rmErr; + Utility::removeAllNoFollow(dirPath, rmErr); + if (rmErr) + Log::warn("Failed to remove dead instance '{}': {}", dirName, rmErr.message()); } } } // namespace @@ -307,7 +309,7 @@ namespace FileTracking reply({ {"success", true}, {"instanceId", inst->instanceId()}, - {"instanceDir", inst->instanceDir().string()}, + {"instanceDir", Paths::nativeToUtf8(inst->instanceDir())}, }); } ); @@ -344,7 +346,13 @@ namespace FileTracking reply.error("Instance not found: " + instanceId); return; } - auto watch = inst->addWatch(Utility::pathFromUtf8(path), recursive, std::move(downloadTempSuffix)); + const auto watchedPath = Paths::LocalPath::fromUtf8(path); + if (!watchedPath) + { + reply.error(fmt::format("Cannot watch '{}': {}", path, watchedPath.error().toString())); + return; + } + auto watch = inst->addWatch(watchedPath->native(), recursive, std::move(downloadTempSuffix)); if (!watch) { reply.error("Failed to add watch on path: " + path); @@ -364,7 +372,7 @@ namespace FileTracking { arr.push_back({ {"instanceId", instId}, - {"instanceDir", inst->instanceDir().string()}, + {"instanceDir", Paths::nativeToUtf8(inst->instanceDir())}, }); } reply({{"success", true}, {"instances", std::move(arr)}}); @@ -384,11 +392,53 @@ namespace FileTracking reply({ {"success", true}, {"instanceId", instanceId}, - {"instanceDir", inst->instanceDir().string()}, + {"instanceDir", Paths::nativeToUtf8(inst->instanceDir())}, }); } ); + // Size and modification time of every file in an instance, taken right after the download: downloads keep + // the server's modification time, so this is what the server had. File tracking compares against it + // before replacing the server's copy. + registerOnStrand( + "FileTracking::fileStates", + [this](RpcHelper::RpcOnce&& reply, std::string instanceId) + { + auto* inst = findInstance(instanceId); + if (!inst) + { + reply.error("Instance not found: " + instanceId); + return; + } + nlohmann::json files = nlohmann::json::array(); + std::error_code walkError{}; + for (std::filesystem::recursive_directory_iterator iter{inst->instanceDir(), walkError}, end; + !walkError && iter != end; + iter.increment(walkError)) + { + std::error_code entryError{}; + if (!iter->is_regular_file(entryError) || iter->is_symlink(entryError)) + continue; + const auto size = iter->file_size(entryError); + const auto modified = iter->last_write_time(entryError); + if (entryError) + continue; + const auto seconds = std::chrono::duration_cast( + std::chrono::file_clock::to_sys(modified).time_since_epoch() + ); + // Canonical like the watched directory, so the paths match the change events. + files.push_back({ + {"path", Paths::nativeToUtf8(std::filesystem::weakly_canonical(iter->path(), entryError))}, + {"size", size}, + {"mtime", seconds.count()}, + }); + } + if (walkError) + Log::warn("FileTracking::fileStates: walking '{}' failed: {}", instanceId, walkError.message()); + reply({{"success", true}, {"files", std::move(files)}}); + } + ); + registerOnStrand( "FileTracking::manualCleanup", [this](RpcHelper::RpcOnce&& reply) diff --git a/backend/source/backend/file_tracking/temp_dir_instancing.cpp b/backend/source/backend/file_tracking/temp_dir_instancing.cpp index b6e95410..4133f8c9 100644 --- a/backend/source/backend/file_tracking/temp_dir_instancing.cpp +++ b/backend/source/backend/file_tracking/temp_dir_instancing.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -11,6 +13,7 @@ #include #include +#include #include #include @@ -87,7 +90,10 @@ namespace FileTracking return; std::error_code statusError; - const bool isDirectory = std::filesystem::is_directory(std::filesystem::path(dir) / filename, statusError); + // efsw reports UTF-8 on every platform. + const bool isDirectory = std::filesystem::is_directory( + Paths::nativeFromUtf8(dir) / Paths::nativeFromUtf8(filename), statusError + ); const auto change = filterFileChange( {toFileAction(action), filename, std::move(oldFilename)}, isDirectory, downloadTempSuffix_ ); @@ -96,9 +102,9 @@ namespace FileTracking nlohmann::json payload = { {"action", Utility::enumToString(change->action)}, - {"directory", std::filesystem::path(dir).generic_string()}, - {"filename", std::filesystem::path(change->filename).generic_string()}, - {"oldFilename", std::filesystem::path(change->oldFilename).generic_string()}, + {"directory", Paths::nativeToUtf8(Paths::nativeFromUtf8(dir))}, + {"filename", Paths::nativeToUtf8(Paths::nativeFromUtf8(change->filename))}, + {"oldFilename", Paths::nativeToUtf8(Paths::nativeFromUtf8(change->oldFilename))}, }; boost::asio::post( @@ -127,7 +133,7 @@ namespace FileTracking Config config; std::string instanceId; std::filesystem::path instanceDir; - InstanceLock lock; + std::optional lock; boost::asio::strand strand; Nui::RpcHub* hub; std::shared_ptr watcher; @@ -149,7 +155,7 @@ namespace FileTracking : config{std::move(cfg)} , instanceId{std::move(instId)} , instanceDir{prepareInstanceDir(config.tempRootDir / instanceId)} - , lock{instanceDir / (instanceId + ".lock")} + , lock{std::in_place, instanceDir / (instanceId + ".lock")} , strand{std::move(strd)} , hub{hubPtr} , watcher{std::make_shared()} @@ -208,15 +214,19 @@ namespace FileTracking absPath = fs::weakly_canonical(absPath); auto rel = fs::relative(absPath, impl_->instanceDir); - if (rel.string().starts_with("..")) + if (rel.empty() || *rel.begin() == "..") { - Log::warn("addWatch: '{}' is outside instance dir '{}'", absPath.string(), impl_->instanceDir.string()); + Log::warn( + "addWatch: '{}' is outside instance dir '{}'", + Paths::nativeToUtf8(absPath), + Paths::nativeToUtf8(impl_->instanceDir) + ); return std::nullopt; } if (!fs::exists(absPath)) { - Log::warn("addWatch: path '{}' does not exist", absPath.string()); + Log::warn("addWatch: path '{}' does not exist", Paths::nativeToUtf8(absPath)); return std::nullopt; } @@ -230,10 +240,11 @@ namespace FileTracking impl_->instanceId, impl_->strand, impl_->hub, std::move(downloadTempSuffix) ); - auto watchId = impl_->watcher->addWatch(absPath.string(), impl_->listener.get(), recursive); + // efsw takes UTF-8, a narrow string is the ANSI code page on Windows. + auto watchId = impl_->watcher->addWatch(Paths::nativeToUtf8(absPath), impl_->listener.get(), recursive); if (watchId < 0) { - Log::warn("addWatch: efsw rejected watch for '{}'", absPath.string()); + Log::warn("addWatch: efsw rejected watch for '{}'", Paths::nativeToUtf8(absPath)); return std::nullopt; } @@ -248,9 +259,10 @@ namespace FileTracking impl_->watcher.reset(); impl_->listener.reset(); + impl_->lock.reset(); std::error_code err; - std::filesystem::remove_all(impl_->instanceDir, err); + Utility::removeAllNoFollow(impl_->instanceDir, err); if (err) Log::warn("cleanupNow: failed to remove '{}': {}", impl_->instanceDir.string(), err.message()); diff --git a/backend/source/backend/linux/opener_linux.cpp b/backend/source/backend/linux/opener_linux.cpp index b84df4cb..19e87821 100644 --- a/backend/source/backend/linux/opener_linux.cpp +++ b/backend/source/backend/linux/opener_linux.cpp @@ -32,6 +32,10 @@ #include #include +#include +#include +#include +#include #include namespace @@ -404,6 +408,24 @@ Opener::Opener(Opener&&) noexcept = default; Opener& Opener::operator=(Opener&&) noexcept = default; +std::optional Opener::riskOf(std::filesystem::path const& path) const +{ + std::error_code statusError{}; + const auto status = std::filesystem::status(path, statusError); + if (statusError || !std::filesystem::is_regular_file(status)) + return std::nullopt; + using std::filesystem::perms; + if ((status.permissions() & (perms::owner_exec | perms::group_exec | perms::others_exec)) != perms::none) + return std::string{"backend.opener.risk.program"}; + // Types a desktop starts as programs instead of showing them. + static constexpr std::string_view runnable[] = {".desktop", ".appimage", ".run", ".jar"}; + auto extension = path.extension().string(); + std::ranges::transform(extension, extension.begin(), [](unsigned char character) { return std::tolower(character); }); + if (std::ranges::find(runnable, std::string_view{extension}) != std::end(runnable)) + return std::string{"backend.opener.risk.dangerousType"}; + return std::nullopt; +} + std::expected Opener::openFile(std::filesystem::path const& path, bool openWith) { Log::info("Opener: openFile path='{}' openWith={} parentWindow='{}'", path.string(), openWith, impl_->parentWindow); diff --git a/backend/source/backend/local_delete_guard.cpp b/backend/source/backend/local_delete_guard.cpp new file mode 100644 index 00000000..cd786edc --- /dev/null +++ b/backend/source/backend/local_delete_guard.cpp @@ -0,0 +1,11 @@ +#include + +#include + +std::optional localHomeDirectory() +{ + auto home = Paths::LocalPath::make(Nui::resolvePath("%userprofile%")); + if (!home) + return std::nullopt; + return *std::move(home); +} diff --git a/backend/source/backend/mac/opener_mac.mm b/backend/source/backend/mac/opener_mac.mm index a1f96940..215d7e96 100644 --- a/backend/source/backend/mac/opener_mac.mm +++ b/backend/source/backend/mac/opener_mac.mm @@ -8,6 +8,10 @@ #include #include +#include +#include +#include +#include #include #include @@ -20,8 +24,7 @@ } /** - * @brief Mirrors the Windows refusal of binary executables: LaunchServices would run a - * Mach-O or script through Terminal and launch application bundles directly. + * @brief LaunchServices would run a Mach-O or script through Terminal and launch application bundles directly. */ bool isExecutable(std::filesystem::path const& path, bool isDirectory) { @@ -69,6 +72,23 @@ void chooseApplicationAndOpen(NSURL* url) } } +std::optional Opener::riskOf(std::filesystem::path const& path) const +{ + std::error_code statusError{}; + const bool isDirectory = std::filesystem::is_directory(path, statusError); + if (isExecutable(path, isDirectory)) + return std::string{isDirectory ? "backend.opener.risk.application" : "backend.opener.risk.program"}; + // Types LaunchServices runs or installs instead of showing. + static constexpr std::string_view runnable[] = { + ".command", ".tool", ".terminal", ".pkg", ".mpkg", ".workflow", ".scpt", ".applescript", ".jar", ".webloc", + }; + auto extension = path.extension().string(); + std::ranges::transform(extension, extension.begin(), [](unsigned char character) { return std::tolower(character); }); + if (std::ranges::find(runnable, std::string_view{extension}) != std::end(runnable)) + return std::string{"backend.opener.risk.dangerousType"}; + return std::nullopt; +} + struct Opener::Implementation {}; @@ -90,9 +110,8 @@ void chooseApplicationAndOpen(NSURL* url) if (!path.is_absolute() || !std::filesystem::exists(path, existsError)) return std::unexpected{std::string{"backend.opener.mac.fileNotFound"}}; + // Programs, scripts and bundles are judged by riskOf and the user's policy before this is called. const bool isDirectory = std::filesystem::is_directory(path, existsError); - if (isExecutable(path, isDirectory)) - return std::unexpected{std::string{"backend.opener.mac.executable"}}; @autoreleasepool { diff --git a/backend/source/backend/mac/web_view_hooks.mm b/backend/source/backend/mac/web_view_hooks.mm index ee2a0865..9b374946 100644 --- a/backend/source/backend/mac/web_view_hooks.mm +++ b/backend/source/backend/mac/web_view_hooks.mm @@ -107,9 +107,12 @@ - (void)willOpenMenu:(NSMenu*)menu withEvent:(NSEvent*)event { const auto& request = arguments.is_array() && !arguments.empty() ? arguments[0] : arguments; - std::vector entries; + std::vector entries; for (auto const& path : takeDroppedFiles()) - entries.push_back(leanDirectoryEntryFromPath(path)); + { + if (auto item = droppedItemFromPath(path); item) + entries.push_back(*std::move(item)); + } if (entries.empty()) { Log::warn("NativeFileDrop::claim: no dropped files are pending."); diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index 3707f3ed..1e05f773 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -13,6 +13,7 @@ #endif #include +#include #include #include #include @@ -194,7 +195,7 @@ Main::LoggerSetup::LoggerSetup(Persistence::StateHolder& stateHolder) Main::Main(ProgramOptions options, ForkPool* forkPool) : shuttingDown_{false} - , programDir_{boost::dll::program_location().parent_path().string()} + , programDir_{boost::dll::program_location().parent_path().native()} , stateHolder_{programDir_} , loggerSetup_{stateHolder_} , window_{ @@ -312,7 +313,17 @@ void Main::registerRpc() } rpcFilesystem_ = std::make_unique( - window_.getExecutor(), window_, hub_, holder.stateCache().localFilesystemOptions, *opener_ + window_.getExecutor(), + window_, + hub_, + holder.stateCache().localFilesystemOptions, + *opener_, + [this]() + { + return stateHolder_.stateCache().localFilesystemOptions.openRiskyFiles.value_or( + Persistence::RiskyFilePolicy::Warn + ); + } ); hub_.markRpcAsInitialized(); @@ -389,6 +400,8 @@ void Main::registerVersionGetter() {"version", VERSION}, {"gitTag", GIT_TAG}, {"gitBranch", GIT_BRANCH}, + {"localPlatform", + Paths::compiledLocalPlatform() == Paths::LocalPlatform::Windows ? "windows" : "posix"}, } ); } @@ -430,6 +443,18 @@ void Main::startChildSignalTimer() int main(int const argc, char const* const* argv) { +#ifdef _WIN32 + // A GUI program has no console; help and --console write to the one it was started from. + for (int index = 1; index < argc; ++index) + { + const std::string_view argument{argv[index]}; + if (argument == "--help" || argument == "--console") + { + attachParentConsole(); + break; + } + } +#endif auto options = parseProgramOptions(argc, argv); if (!options) return 0; @@ -493,4 +518,4 @@ int main(int const argc, char const* const* argv) } ssh_finalize(); -} \ No newline at end of file +} diff --git a/backend/source/backend/process/process.cpp b/backend/source/backend/process/process.cpp index 77a9fa21..041612d8 100644 --- a/backend/source/backend/process/process.cpp +++ b/backend/source/backend/process/process.cpp @@ -309,6 +309,11 @@ void Process::spawn( auto executable = bp2::environment::find_executable(processName, env); if (executable.empty()) executable = processName; +#ifdef _WIN32 + // cmd.exe reads its own path as given: in "C:/Windows/System32/cmd.exe" it takes "/c" as a switch and runs + // "md.exe" instead of starting a shell. + executable.make_preferred(); +#endif { std::scoped_lock lock{impl_->awaitExit}; @@ -317,6 +322,18 @@ void Process::spawn( if (!launcher) { +#ifdef _WIN32 + // The app is a GUI program without a console, a console child would open a window of its own. + bp2::windows::default_launcher windowlessLauncher{}; + windowlessLauncher.creation_flags |= CREATE_NO_WINDOW; + impl_->child = std::make_unique(windowlessLauncher( + impl_->executor, + executable, + arguments, + bp2::process_environment{env}, + bp2::process_stdio(impl_->stdinPipe, impl_->stdoutPipe, impl_->stderrPipe) + )); +#else impl_->child = std::make_unique( impl_->executor, executable, @@ -324,6 +341,7 @@ void Process::spawn( bp2::process_environment{env}, bp2::process_stdio(impl_->stdinPipe, impl_->stdoutPipe, impl_->stderrPipe) ); +#endif } else { diff --git a/backend/source/backend/program_options.cpp b/backend/source/backend/program_options.cpp index c8f14f44..77b858eb 100644 --- a/backend/source/backend/program_options.cpp +++ b/backend/source/backend/program_options.cpp @@ -11,7 +11,7 @@ std::optional parseProgramOptions(int argc, char const* const* a boost::program_options::options_description desc("Allowed options"); desc.add_options()("help", "produce help message")( "enable-dev-tools", "Enable developer tools (DevTools in Browser)" - ); + )("console", "Windows: write output to the console the program was started from"); boost::program_options::variables_map vm; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), vm); diff --git a/backend/source/backend/rpc_filesystem.cpp b/backend/source/backend/rpc_filesystem.cpp index 054feafa..bdf0f912 100644 --- a/backend/source/backend/rpc_filesystem.cpp +++ b/backend/source/backend/rpc_filesystem.cpp @@ -1,11 +1,18 @@ #include +#include #include #include -#include +#include #include +#include +#include +#include +#include #include +#include + #include #ifndef _WIN32 @@ -21,6 +28,31 @@ namespace { + /** + * @brief The native form of the absolute local path @p utf8, or why it is refused. + */ + std::expected localPathParameter(std::string const& utf8) + { + return Paths::LocalPath::fromUtf8(utf8) + .transform( + [](Paths::LocalPath const& path) + { + return path.native(); + } + ) + .transform_error( + [](Paths::PathError const& error) + { + return error.toString(); + } + ); + } + + std::expected deletableLocalPath(std::string const& utf8) + { + return localDeleteTarget(utf8, localHomeDirectory()); + } + #ifndef _WIN32 /** * @brief Per-call uid/gid → name cache, so a directory full of files owned by the same user @@ -145,11 +177,13 @@ RpcFilesystem::RpcFilesystem( Nui::Window& wnd, Nui::RpcHub& hub, Persistence::LocalFilesystemOptions options, - Opener& opener + Opener& opener, + std::function riskyFilePolicy ) : RpcHelper::StrandRpc{executor, wnd, hub} , options_{std::move(options)} , opener_{&opener} + , riskyFilePolicy_{std::move(riskyFilePolicy)} { registerRemove(); registerRemoveMultiple(); @@ -189,7 +223,13 @@ void RpcFilesystem::registerRemove() const bool recursive = parameters.value("recursive", false); const auto pathUtf8 = parameters["path"].get(); - const auto path = Utility::pathFromUtf8(pathUtf8); + const auto target = deletableLocalPath(pathUtf8); + if (!target) + { + Log::error("RpcFilesystem::remove refused: {}", target.error()); + return reply.error(Utility::localizedMessage("backend.rpcFilesystem.removeRefused", pathUtf8)); + } + const auto path = target->native(); if (!recursive) { @@ -205,7 +245,7 @@ void RpcFilesystem::registerRemove() } std::error_code ec; - const auto filesRemoved = std::filesystem::remove_all(path, ec); + const auto filesRemoved = Utility::removeAllNoFollow(path, ec); if (ec) { Log::error("Failed to recursively remove path: {}: {}", pathUtf8, ec.message()); @@ -243,14 +283,32 @@ void RpcFilesystem::registerRemoveMultiple() const bool recursive = parameters.value("recursive", false); const auto pathsUtf8 = parameters["paths"].get>(); - size_t removedCount = 0; + + // Nothing is removed when one of the paths is refused. + std::vector targets; + targets.reserve(pathsUtf8.size()); for (const auto& pathUtf8 : pathsUtf8) { - const auto path = Utility::pathFromUtf8(pathUtf8); + auto target = deletableLocalPath(pathUtf8); + if (!target) + { + Log::error("RpcFilesystem::removeSome refused: {}", target.error()); + return reply.error(Utility::localizedMessage("backend.rpcFilesystem.removeRefused", pathUtf8)); + } + targets.push_back(*std::move(target)); + } + + // One item that cannot be removed does not keep the others, every failure is reported. + size_t removedCount = 0; + std::vector failures; + for (const auto& target : targets) + { + const auto path = target.native(); + const auto& pathUtf8 = target.toUtf8(); std::error_code ec; if (recursive) { - removedCount += std::filesystem::remove_all(path, ec); + removedCount += Utility::removeAllNoFollow(path, ec); } else { @@ -261,9 +319,19 @@ void RpcFilesystem::registerRemoveMultiple() if (ec) { Log::error("Failed to remove path '{}': {}", pathUtf8, ec.message()); - return reply.error(Utility::localizedMessage("backend.rpcFilesystem.removeFailed", pathUtf8, ec.message())); + failures.push_back(fmt::format("{}: {}", pathUtf8, ec.message())); } } + if (!failures.empty()) + { + return reply.error( + Utility::localizedMessage( + "backend.rpcFilesystem.removeSomeFailed", + failures.size(), + fmt::format("{}", fmt::join(failures, "\n")) + ) + ); + } Log::info("Successfully removed all specified paths."); return reply({{"success", true}, {"removedCount", removedCount}}); } @@ -293,11 +361,29 @@ void RpcFilesystem::registerRename() const auto oldPathUtf8 = parameters["oldPath"].get(); const auto newPathUtf8 = parameters["newPath"].get(); - const auto oldPath = Utility::pathFromUtf8(oldPathUtf8); - const auto newPath = Utility::pathFromUtf8(newPathUtf8); + const auto oldPathParameter = localPathParameter(oldPathUtf8); + const auto newPathParameter = localPathParameter(newPathUtf8); + if (!oldPathParameter || !newPathParameter) + return reply.error(oldPathParameter ? newPathParameter.error() : oldPathParameter.error()); + auto const& oldPath = *oldPathParameter; + auto const& newPath = *newPathParameter; + const bool overwrite = parameters.contains("overwrite") && parameters["overwrite"].is_boolean() && + parameters["overwrite"].get(); + + if (!Utility::isRenameInPlace(oldPath, newPath)) + { + Log::error("RpcFilesystem::rename refused to move '{}' to '{}'.", oldPathUtf8, newPathUtf8); + return reply.error("backend.rename.invalidName"); + } + // Never replaces an existing item unless the user confirmed it. std::error_code ec; - std::filesystem::rename(oldPath, newPath, ec); + if (overwrite) + std::filesystem::rename(oldPath, newPath, ec); + else if (auto renamed = Utility::renameNoReplace(oldPath, newPath); !renamed.has_value()) + ec = renamed.error(); + if (ec == std::errc::file_exists) + return reply.error("backend.rename.targetExists"); if (ec) { Log::error("Failed to rename from '{}' to '{}': {}", oldPathUtf8, newPathUtf8, ec.message()); @@ -324,22 +410,21 @@ void RpcFilesystem::registerListFiles() return reply.error("Missing required parameter 'path'."); } - auto directoryPath = Utility::pathFromUtf8(parameters["path"].get()); - const auto fileNameOnly = parameters.value("fileNameOnly", false); + // A drive "C:" is read as its root "C:/". + const auto directoryParameter = localPathParameter(parameters["path"].get()); + if (!directoryParameter) + return reply.error(directoryParameter.error()); + auto const& directoryPath = *directoryParameter; nlohmann::json fileList = nlohmann::json::array(); -#ifdef _WIN32 - if (directoryPath == directoryPath.root_path()) - directoryPath += "/"; -#endif - Log::info("Listing files in directory: {}", Utility::pathToUtf8Generic(directoryPath)); + Log::info("Listing files in directory: {}", Paths::nativeToUtf8(directoryPath)); std::error_code ec; auto iter = std::filesystem::directory_iterator(directoryPath, ec); if (ec) { Log::error( - "Failed to list files in directory '{}': {}", Utility::pathToUtf8Generic(directoryPath), ec.message() + "Failed to list files in directory '{}': {}", Paths::nativeToUtf8(directoryPath), ec.message() ); return reply.error(ec.message()); } @@ -352,7 +437,7 @@ void RpcFilesystem::registerListFiles() const auto entryPath = entry.path(); SharedData::DirectoryEntry dirEntry{}; - dirEntry.path = fileNameOnly ? entryPath.filename() : entryPath; + dirEntry.name = Paths::nativeToUtf8(entryPath.filename()); dirEntry.type = SharedData::fileTypeFromStdFilesystemType(entry.symlink_status().type()); dirEntry.permissions = entry.symlink_status().permissions(); @@ -365,18 +450,11 @@ void RpcFilesystem::registerListFiles() if (dirEntry.type == SharedData::FileType::Symlink) { std::error_code linkEc; - auto linkTarget = std::filesystem::read_symlink(entryPath, linkEc); + const auto linkTarget = std::filesystem::read_symlink(entryPath, linkEc); if (!linkEc) - dirEntry.linkTarget = std::move(linkTarget); + dirEntry.linkTarget = Paths::nativeToUtf8(linkTarget); auto resolved = std::make_shared(); - std::error_code pathEc; - auto canonical = std::filesystem::canonical(entryPath, pathEc); - if (!pathEc) - resolved->path = std::move(canonical); - else if (dirEntry.linkTarget.has_value()) - resolved->path = *dirEntry.linkTarget; - resolved->type = SharedData::fileTypeFromStdFilesystemType(entry.status().type()); resolved->permissions = entry.status().permissions(); #ifndef _WIN32 @@ -390,7 +468,7 @@ void RpcFilesystem::registerListFiles() fileList.push_back(nlohmann::json(dirEntry)); } - Log::info("Successfully listed files in directory '{}'", Utility::pathToUtf8Generic(directoryPath)); + Log::info("Successfully listed files in directory '{}'", Paths::nativeToUtf8(directoryPath)); return reply({{"success", true}, {"files", fileList}}); } ); @@ -418,7 +496,10 @@ void RpcFilesystem::registerCreateFile() } const auto filePathUtf8 = parameters["filePath"].get(); - const auto filePath = Utility::pathFromUtf8(filePathUtf8); + const auto filePathParameter = localPathParameter(filePathUtf8); + if (!filePathParameter) + return reply.error(filePathParameter.error()); + auto const& filePath = *filePathParameter; std::error_code ec; std::ofstream fileStream(filePath, std::ios_base::noreplace | std::ios_base::binary); @@ -458,7 +539,10 @@ void RpcFilesystem::registerCreateDirectory() } const auto directoryPathUtf8 = parameters["path"].get(); - const auto directoryPath = Utility::pathFromUtf8(directoryPathUtf8); + const auto directoryParameter = localPathParameter(directoryPathUtf8); + if (!directoryParameter) + return reply.error(directoryParameter.error()); + auto const& directoryPath = *directoryParameter; std::error_code ec; std::filesystem::create_directory(directoryPath, ec); @@ -492,16 +576,20 @@ void RpcFilesystem::registerProperties() return reply.error("Missing required parameter 'path'."); } - const auto path = Utility::pathFromUtf8(parameters["path"].get()); + const auto pathParameter = localPathParameter(parameters["path"].get()); + if (!pathParameter) + return reply.error(pathParameter.error()); + auto const& path = *pathParameter; SharedData::DirectoryEntry entry{}; - entry.path = path; + entry.name = Paths::nativeToUtf8(path.filename()); + std::string resolvedPath{}; std::error_code symEc; const auto symStatus = std::filesystem::symlink_status(path, symEc); if (symEc) { - Log::error("Failed to get properties for path '{}': {}", Utility::pathToUtf8Generic(path), symEc.message()); + Log::error("Failed to get properties for path '{}': {}", Paths::nativeToUtf8(path), symEc.message()); return reply.error(symEc.message()); } entry.type = SharedData::fileTypeFromStdFilesystemType(symStatus.type()); @@ -519,17 +607,20 @@ void RpcFilesystem::registerProperties() if (entry.type == SharedData::FileType::Symlink) { std::error_code linkEc; - auto linkTarget = std::filesystem::read_symlink(path, linkEc); + const auto linkTarget = std::filesystem::read_symlink(path, linkEc); if (!linkEc) - entry.linkTarget = std::move(linkTarget); + entry.linkTarget = Paths::nativeToUtf8(linkTarget); auto resolved = std::make_shared(); std::error_code pathEc; - auto canonical = std::filesystem::canonical(path, pathEc); + const auto canonical = std::filesystem::canonical(path, pathEc); if (!pathEc) - resolved->path = std::move(canonical); + { + resolvedPath = Paths::nativeToUtf8(canonical); + resolved->name = Paths::nativeToUtf8(canonical.filename()); + } else if (entry.linkTarget.has_value()) - resolved->path = *entry.linkTarget; + resolvedPath = *entry.linkTarget; std::error_code statusEc; const auto resolvedStatus = std::filesystem::status(path, statusEc); @@ -550,8 +641,14 @@ void RpcFilesystem::registerProperties() entry.resolvedTarget = std::move(resolved); } - Log::info("Successfully retrieved properties for path '{}'", Utility::pathToUtf8Generic(path)); - return reply({{"success", true}, {"entry", nlohmann::json(entry)}}); + Log::info("Successfully retrieved properties for path '{}'", Paths::nativeToUtf8(path)); + // The paths go along for display, the entries carry names only. + return reply( + {{"success", true}, + {"entry", nlohmann::json(entry)}, + {"path", Paths::nativeToUtf8(path)}, + {"resolvedPath", resolvedPath}} + ); } ); } @@ -567,14 +664,14 @@ void RpcFilesystem::registerGetHome() if (options.homeOverride.has_value()) { std::filesystem::path homePath = *options.homeOverride; - const auto homePathUtf8 = Utility::pathToUtf8Generic(homePath); + const auto homePathUtf8 = Paths::nativeToUtf8(homePath); Log::info("Returning overridden home directory path: {}", homePathUtf8); return reply({{"success", true}, {"path", homePathUtf8}}); } auto home = Nui::resolvePath("%userprofile%"); - const auto homeUtf8 = Utility::pathToUtf8Generic(home); + const auto homeUtf8 = Paths::nativeToUtf8(home); Log::info("Returning initial directory path: {}", homeUtf8); return reply({{"success", true}, {"path", homeUtf8}}); } @@ -594,9 +691,11 @@ void RpcFilesystem::registerDoesExist() } const auto pathUtf8 = parameters["path"].get(); - const auto path = Utility::pathFromUtf8(pathUtf8); + const auto path = localPathParameter(pathUtf8); + if (!path) + return reply.error(path.error()); std::error_code ec; - const auto status = std::filesystem::exists(path, ec); + const auto status = std::filesystem::exists(*path, ec); if (ec) { Log::error("Failed to get properties for path '{}': {}", pathUtf8, ec.message()); @@ -619,30 +718,58 @@ void RpcFilesystem::registerDoesExistBatch() .perform( [](RpcHelper::RpcOnce&& reply, std::vector const& paths) { - std::vector results; - results.reserve(paths.size()); - for (auto const& pathUtf8 : paths) + auto results = localPathsExist(paths); + if (!results) { - std::error_code ec; - const auto status = std::filesystem::exists(Utility::pathFromUtf8(pathUtf8), ec); - if (ec) - { - Log::warn( - "RpcFilesystem::existsBatch: stat failed for '{}': {} (treating as not-exists)", - pathUtf8, - ec.message() - ); - results.push_back(false); - continue; - } - results.push_back(status); + Log::error("RpcFilesystem::existsBatch: {}", results.error()); + return reply.error(results.error()); } Log::info("RpcFilesystem::existsBatch: probed {} paths", paths.size()); - return reply({{"success", true}, {"exists", results}}); + return reply({{"success", true}, {"exists", *results}}); } ); } +namespace +{ + /** + * @brief Writes @p content to the UTF-8 path @p filePath and replies. Without @p replace an existing file is + * left alone and the reply's error is backend.rpcFilesystem.fileExists. + */ + void writeFileAndReply( + RpcHelper::RpcOnce&& reply, + std::string const& filePath, + std::string const& content, + bool replace + ) + { + // The path is UTF-8, the narrow ofstream constructor reads the ANSI code page on Windows. + const auto path = localPathParameter(filePath); + if (!path) + return reply.error(path.error()); + + const auto mode = replace ? std::ios_base::binary : std::ios_base::binary | std::ios_base::noreplace; + std::ofstream fileStream(*path, mode); + if (!fileStream) + { + if (std::error_code statusError{}; !replace && std::filesystem::exists(std::filesystem::symlink_status(*path, statusError))) + return reply.error(Utility::localizedMessage("backend.rpcFilesystem.fileExists", filePath)); + Log::error("Failed to open file '{}' for writing.", filePath); + return reply.error(fmt::format("Cannot open '{}' for writing", filePath)); + } + + fileStream.write(content.data(), static_cast(content.size())); + if (!fileStream) + { + Log::error("Failed to write to file '{}'.", filePath); + return reply.error(fmt::format("Cannot write '{}'", filePath)); + } + + Log::info("Successfully wrote to file '{}'", filePath); + return reply({{"success", true}}); + } +} + void RpcFilesystem::registerWriteFile() { on("RpcFilesystem::writeFile") @@ -650,24 +777,15 @@ void RpcFilesystem::registerWriteFile() [](RpcHelper::RpcOnce&& reply, std::string const& filePath, std::string const& content) { Log::info("RpcFilesystem::writeFile called for file: {}", filePath); - - std::error_code ec; - std::ofstream fileStream(filePath, std::ios_base::binary); - if (!fileStream) - { - Log::error("Failed to open file '{}' for writing: {}", filePath, ec.message()); - return reply.error(ec.message()); - } - - fileStream.write(content.data(), static_cast(content.size())); - if (!fileStream) - { - Log::error("Failed to write to file '{}': {}", filePath, ec.message()); - return reply.error(ec.message()); - } - - Log::info("Successfully wrote to file '{}'", filePath); - return reply({{"success", true}}); + writeFileAndReply(std::move(reply), filePath, content, /*replace=*/true); + } + ); + on("RpcFilesystem::writeNewFile") + .perform( + [](RpcHelper::RpcOnce&& reply, std::string const& filePath, std::string const& content) + { + Log::info("RpcFilesystem::writeNewFile called for file: {}", filePath); + writeFileAndReply(std::move(reply), filePath, content, /*replace=*/false); } ); } @@ -676,10 +794,27 @@ void RpcFilesystem::registerOpen() { on("RpcFilesystem::open") .perform( - [this](RpcHelper::RpcOnce&& reply, std::string const& filePath, bool openWith) + [this](RpcHelper::RpcOnce&& reply, std::string const& filePath, bool openWith, bool confirmed) { Log::info("RpcFilesystem::open called for file: {}", filePath); - auto result = opener_->openFile(Utility::pathFromUtf8(filePath), openWith); + const auto path = localPathParameter(filePath); + if (!path) + return reply.error(path.error()); + + // Files that can run code follow the user's policy; the frontend asks and calls again confirmed. + if (const auto risk = opener_->riskOf(*path); risk) + { + const auto policy = riskyFilePolicy_ ? riskyFilePolicy_() : Persistence::RiskyFilePolicy::Warn; + if (policy == Persistence::RiskyFilePolicy::Never) + { + Log::warn("Refusing to open '{}': {}, opening such files is turned off.", filePath, *risk); + return reply.error(Utility::localizedMessage("backend.opener.risk.refused", filePath)); + } + if (policy == Persistence::RiskyFilePolicy::Warn && !confirmed) + return reply({{"success", false}, {"needsConfirmation", true}, {"risk", *risk}}); + } + + auto result = opener_->openFile(*path, openWith); if (!result) { Log::error("Failed to open file '{}': {}", filePath, result.error()); @@ -715,7 +850,10 @@ void RpcFilesystem::registerOpenInFileManager() [this](RpcHelper::RpcOnce&& reply, std::string const& filePath) { Log::info("RpcFilesystem::openInFileManager called for path: {}", filePath); - auto result = opener_->openInFileManager(Utility::pathFromUtf8(filePath)); + const auto path = localPathParameter(filePath); + if (!path) + return reply.error(path.error()); + auto result = opener_->openInFileManager(*path); if (!result) { Log::error("Failed to open file manager for '{}': {}", filePath, result.error()); diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index 6320931e..38e4752e 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -1,11 +1,15 @@ #include +#include +#include -#include +#include #include #include #include -#include +#include +#include +#include #include @@ -15,6 +19,66 @@ using namespace std::chrono_literals; +namespace +{ + /** + * @brief A remote path from the frontend, or the message to reply with when it is not a valid absolute path. + */ + std::expected parseRemotePath(std::string const& path) + { + return Paths::RemotePath::make(path).transform_error( + [](Paths::PathError const& error) + { + return fmt::format("Invalid remote path: {}", error.toString()); + } + ); + } + + /** + * @brief A local path from the frontend, or the message to reply with when it is not a valid absolute path. + */ + std::expected parseLocalPath(std::string const& path) + { + return Paths::LocalPath::fromUtf8(path).transform_error( + [](Paths::PathError const& error) + { + return fmt::format("Invalid local path: {}", error.toString()); + } + ); + } + + /** + * @brief A JSON request from the frontend, or the message to reply with when it does not parse. Paths are + * validated while parsing. + */ + template + std::expected parseRequest(nlohmann::json const& json, std::string_view what) + { + try + { + return json.get(); + } + catch (std::exception const& exception) + { + return std::unexpected(fmt::format("Invalid {}: {}", what, exception.what())); + } + } + + std::expected, std::string> parseRemotePaths(std::vector const& paths) + { + std::vector result; + result.reserve(paths.size()); + for (auto const& path : paths) + { + auto parsed = parseRemotePath(path); + if (!parsed) + return std::unexpected(std::move(parsed).error()); + result.push_back(*std::move(parsed)); + } + return result; + } +} + Session::Session( Ids::SessionId id, std::unique_ptr session, @@ -76,7 +140,7 @@ void Session::start() self->registerRpcSftpOpenSyncSession(); self->registerRpcSftpRecomputeSyncDiff(); self->registerRpcSftpLoadSyncDiffChildren(); - self->registerRpcSftpBuildSyncEnqueuePlan(); + self->registerRpcSftpExecuteSyncSelection(); self->registerRpcSftpCancelSyncDiff(); self->registerRpcSftpCloseSyncSession(); self->registerOperationQueuePauseUnpause(); @@ -573,7 +637,10 @@ void Session::registerRpcSftpListDirectory() Ids::makeChannelId(channelIdString), [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - auto fut = channel->listDirectory(path); + const auto remotePath = parseRemotePath(path); + if (!remotePath) + return reply.error(remotePath.error()); + auto fut = channel->listDirectory(*remotePath); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error("Timed out"); @@ -619,7 +686,7 @@ void Session::registerRpcSftpHomeDirectory() if (!result.has_value()) return reply.error("Failed to get home directory: " + result.error().toString()); - const auto home = Utility::pathToUtf8Generic(*result); + const auto& home = result->value(); Log::info("Remote home directory is '{}'", home); reply({{"path", home}}); }, @@ -645,7 +712,10 @@ void Session::registerRpcSftpUserDirectories() Ids::makeChannelId(channelIdString), [home](RpcHelper::RpcOnce&& reply, auto&& channel) { - auto fut = channel->userDirectories(Utility::pathFromUtf8(home)); + const auto homePath = parseRemotePath(home); + if (!homePath) + return reply.error(homePath.error()); + auto fut = channel->userDirectories(*homePath); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error("Failed to read user directories: timeout"); @@ -657,7 +727,7 @@ void Session::registerRpcSftpUserDirectories() for (auto const& directory : *result) { directories.push_back( - {{"name", directory.name}, {"path", Utility::pathToUtf8Generic(directory.path)}} + {{"name", directory.name}, {"path", directory.path}} ); } Log::info("Read {} user directories in '{}'", directories.size(), home); @@ -676,7 +746,7 @@ void Session::registerRpcQueuedRemoves() [weak = weak_from_this()]( RpcHelper::RpcOnce&& reply, std::string const& channelIdString, - std::vector paths, + std::vector const& pathStrings, bool recursive, bool insertRefresh, int mode @@ -686,9 +756,13 @@ void Session::registerRpcQueuedRemoves() if (!self) return reply.error("Session no longer exists"); + auto parsed = parseRemotePaths(pathStrings); + if (!parsed) + return reply.error(parsed.error()); + self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), paths = std::move(paths), recursive, insertRefresh, mode]( + [weak = self->weak_from_this(), paths = *std::move(parsed), recursive, insertRefresh, mode]( RpcHelper::RpcOnce&& reply, auto&& channel ) mutable { @@ -696,6 +770,14 @@ void Session::registerRpcQueuedRemoves() if (!self) return reply.error("Session no longer exists"); + // Nothing is queued when one of the targets is protected. + for (auto const& path : paths) + { + if (auto allowed = self->operationQueue_->refuseProtectedDeleteTarget(*channel, path); + !allowed) + return reply.error(fmt::format("Refusing to delete '{}'", path.value())); + } + std::vector operationIds; for (auto const& path : paths) { @@ -708,6 +790,11 @@ void Session::registerRpcQueuedRemoves() insertRefresh, static_cast(mode) ); + if (!result) + { + Log::error("Could not queue the delete of '{}': {}", path.value(), result.error().toString()); + continue; + } operationIds.push_back(newOperationId.value()); } @@ -735,7 +822,10 @@ void Session::registerRpcSftpCreateDirectory() Ids::makeChannelId(channelIdString), [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - auto fut = channel->createDirectory(path); + const auto remotePath = parseRemotePath(path); + if (!remotePath) + return reply.error(remotePath.error()); + auto fut = channel->createDirectory(*remotePath); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error(fmt::format("Failed to create directory '{}': timeout", path)); @@ -770,7 +860,10 @@ void Session::registerRpcSftpCreateFile() Ids::makeChannelId(channelIdString), [path](RpcHelper::RpcOnce&& reply, auto&& channel) { - auto fut = channel->createFile(path); + const auto remotePath = parseRemotePath(path); + if (!remotePath) + return reply.error(remotePath.error()); + auto fut = channel->createFile(*remotePath); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error("Failed to create file: timeout"); @@ -808,12 +901,19 @@ void Session::registerRpcSftpAddDownloadOperation() if (!self) return reply({{"error", "Session no longer exists"}}); + auto remote = parseRemotePath(remotePath); + if (!remote) + return reply.error(remote.error()); + auto local = parseLocalPath(localPath); + if (!local) + return reply.error(local.error()); + self->withSftpChannelDo( Ids::makeChannelId(channelIdString), [weak = self->weak_from_this(), newOperationIdString, - localPath, - remotePath, + local = *std::move(local), + remote = *std::move(remote), allowOverwrite, bigFile, insertRefresh, @@ -827,8 +927,8 @@ void Session::registerRpcSftpAddDownloadOperation() const auto result = self->operationQueue_->addDownloadOperation( *channel, Ids::makeOperationId(newOperationIdString), - localPath, - remotePath, + local, + remote, allowOverwrite, bigFile, insertRefresh, @@ -840,8 +940,8 @@ void Session::registerRpcSftpAddDownloadOperation() { Log::error( "Failed to add download operation for file '{}' to '{}': {}", - remotePath, - localPath, + remote.value(), + local.toUtf8(), result.error().toString() ); return reply.error("Failed to add download operation: " + result.error().toString()); @@ -850,8 +950,8 @@ void Session::registerRpcSftpAddDownloadOperation() Log::info( "Added download operation with id '{}' for file '{}' to '{}'", newOperationIdString, - remotePath, - localPath + remote.value(), + local.toUtf8() ); self->resetQueueThrottle(); @@ -874,7 +974,7 @@ void Session::registerRpcSftpAddArchiveDownloadOperation() RpcHelper::RpcOnce&& reply, std::string const& channelIdString, std::string const& newOperationIdString, - std::string const& entriesJson, + std::string const& rootsJson, std::string const& localArchivePath, int compressionCodec, int compressionLevel, @@ -886,23 +986,27 @@ void Session::registerRpcSftpAddArchiveDownloadOperation() if (!self) return reply({{"error", "Session no longer exists"}}); - std::vector entries; + // An invalid remote path fails the parse. + std::vector roots; try { - entries = nlohmann::json::parse(entriesJson).get>(); + roots = nlohmann::json::parse(rootsJson).get>(); } catch (std::exception const& exc) { - Log::error("addArchiveDownload: failed to parse entries JSON: {}", exc.what()); - return reply({{"error", std::string{"Invalid entries payload: "} + exc.what()}}); + Log::error("addArchiveDownload: failed to parse roots JSON: {}", exc.what()); + return reply({{"error", std::string{"Invalid roots payload: "} + exc.what()}}); } + auto archivePath = parseLocalPath(localArchivePath); + if (!archivePath) + return reply.error(archivePath.error()); self->withSftpChannelDo( Ids::makeChannelId(channelIdString), [weak = self->weak_from_this(), newOperationIdString, - entries = std::move(entries), - localArchivePath, + roots = std::move(roots), + localArchivePath = *std::move(archivePath), compressionCodec, compressionLevel, mayOverwrite, @@ -917,8 +1021,8 @@ void Session::registerRpcSftpAddArchiveDownloadOperation() const auto result = self->operationQueue_->addArchiveDownloadOperation( *channel, Ids::makeOperationId(newOperationIdString), - std::move(entries), - std::filesystem::path{localArchivePath}, + std::move(roots), + localArchivePath, codec, compressionLevel, mayOverwrite, @@ -929,7 +1033,7 @@ void Session::registerRpcSftpAddArchiveDownloadOperation() { Log::error( "Failed to add archive-download operation to '{}': {}", - localArchivePath, + localArchivePath.toUtf8(), result.error().toString() ); return reply.error( @@ -938,10 +1042,10 @@ void Session::registerRpcSftpAddArchiveDownloadOperation() } Log::info( - "Added archive-download operation '{}' → '{}' ({} entries)", + "Added archive-download operation '{}' → '{}' ({} roots)", newOperationIdString, - localArchivePath, - entries.size() + localArchivePath.toUtf8(), + roots.size() ); self->resetQueueThrottle(); @@ -973,17 +1077,25 @@ void Session::registerRpcSftpAddArchiveUploadOperation() if (!self) return reply({{"error", "Session no longer exists"}}); - std::vector localPaths; + std::vector localPaths; localPaths.reserve(localPathStrings.size()); for (auto const& pathString : localPathStrings) - localPaths.emplace_back(pathString); + { + auto localPath = parseLocalPath(pathString); + if (!localPath) + return reply.error(localPath.error()); + localPaths.push_back(*std::move(localPath)); + } + auto archivePath = parseRemotePath(remoteArchivePath); + if (!archivePath) + return reply.error(archivePath.error()); self->withSftpChannelDo( Ids::makeChannelId(channelIdString), [weak = self->weak_from_this(), newOperationIdString, localPaths = std::move(localPaths), - remoteArchivePath, + remoteArchivePath = *std::move(archivePath), compressionCodec, compressionLevel, mayOverwrite, @@ -998,7 +1110,7 @@ void Session::registerRpcSftpAddArchiveUploadOperation() *channel, Ids::makeOperationId(newOperationIdString), std::move(localPaths), - std::filesystem::path{remoteArchivePath}, + remoteArchivePath, codec, compressionLevel, mayOverwrite, @@ -1008,13 +1120,15 @@ void Session::registerRpcSftpAddArchiveUploadOperation() { Log::error( "Failed to add archive-upload operation to '{}': {}", - remoteArchivePath, + remoteArchivePath.value(), result.error().toString() ); return reply.error("Failed to add archive-upload operation: " + result.error().toString()); } Log::info( - "Added archive-upload operation '{}' → '{}'", newOperationIdString, remoteArchivePath + "Added archive-upload operation '{}' → '{}'", + newOperationIdString, + remoteArchivePath.value() ); self->resetQueueThrottle(); reply({{"success", true}}); @@ -1036,13 +1150,19 @@ void Session::registerRpcSftpAddBulkDownloadOperation() RpcHelper::RpcOnce&& reply, std::string const& channelIdString, std::vector const& operationIdStrings, - SharedData::BulkAddRequest const& request + nlohmann::json const& requestJson ) { auto self = weak.lock(); if (!self) return reply({{"error", "Session no longer exists"}}); + // Parsed here, so an invalid path is answered with an error instead of no reply at all. + auto parsed = parseRequest(requestJson, "bulk download"); + if (!parsed) + return reply.error(parsed.error()); + auto request = *std::move(parsed); + // Expect N per-entry ids + 1 dedicated aggregate-bulk-card id at the end. if (operationIdStrings.size() != request.entries.size() + 1) return reply.error("addBulkDownload: operationIds and entries length mismatch"); @@ -1088,13 +1208,19 @@ void Session::registerRpcSftpAddBulkUploadOperation() RpcHelper::RpcOnce&& reply, std::string const& channelIdString, std::vector const& operationIdStrings, - SharedData::BulkAddRequest const& request + nlohmann::json const& requestJson ) { auto self = weak.lock(); if (!self) return reply({{"error", "Session no longer exists"}}); + // Parsed here, so an invalid path is answered with an error instead of no reply at all. + auto parsed = parseRequest(requestJson, "bulk upload"); + if (!parsed) + return reply.error(parsed.error()); + auto request = *std::move(parsed); + if (operationIdStrings.size() != request.entries.size() + 1) return reply.error("addBulkUpload: operationIds and entries length mismatch"); @@ -1150,12 +1276,19 @@ void Session::registerRpcSftpAddUploadOperation() if (!self) return reply({{"error", "Session no longer exists"}}); + auto local = parseLocalPath(localPath); + if (!local) + return reply.error(local.error()); + auto remote = parseRemotePath(remotePath); + if (!remote) + return reply.error(remote.error()); + self->withSftpChannelDo( Ids::makeChannelId(channelIdString), [weak = self->weak_from_this(), newOperationIdString, - localPath, - remotePath, + local = *std::move(local), + remote = *std::move(remote), allowOverwrite, bigFile, insertRefresh, @@ -1169,8 +1302,8 @@ void Session::registerRpcSftpAddUploadOperation() const auto result = self->operationQueue_->addUploadOperation( *channel, Ids::makeOperationId(newOperationIdString), - localPath, - remotePath, + local, + remote, allowOverwrite, bigFile, insertRefresh, @@ -1182,8 +1315,8 @@ void Session::registerRpcSftpAddUploadOperation() { Log::error( "Failed to add upload operation for file '{}' to '{}': {}", - localPath, - remotePath, + local.toUtf8(), + remote.value(), result.error().toString() ); return reply.error("Failed to add upload operation: " + result.error().toString()); @@ -1192,8 +1325,8 @@ void Session::registerRpcSftpAddUploadOperation() Log::info( "Added upload operation with id '{}' for file '{}' to '{}'", newOperationIdString, - localPath, - remotePath + local.toUtf8(), + remote.value() ); self->resetQueueThrottle(); @@ -1222,11 +1355,20 @@ void Session::registerRpcSftpAddRenameOperation() if (!self) return reply({{"error", "Session no longer exists"}}); + auto source = parseRemotePath(sourcePath); + if (!source) + return reply.error(source.error()); + auto destination = parseRemotePath(destinationPath); + if (!destination) + return reply.error(destination.error()); + self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), newOperationIdString, sourcePath, destinationPath, mode]( - RpcHelper::RpcOnce&& reply, auto&& channel - ) + [weak = self->weak_from_this(), + newOperationIdString, + source = *std::move(source), + destination = *std::move(destination), + mode](RpcHelper::RpcOnce&& reply, auto&& channel) { auto self = weak.lock(); if (!self) @@ -1235,8 +1377,8 @@ void Session::registerRpcSftpAddRenameOperation() const auto result = self->operationQueue_->addRenameOperation( *channel, Ids::makeOperationId(newOperationIdString), - sourcePath, - destinationPath, + source, + destination, static_cast(mode) ); @@ -1244,8 +1386,8 @@ void Session::registerRpcSftpAddRenameOperation() { Log::error( "Failed to add rename operation for '{}' to '{}': {}", - sourcePath, - destinationPath, + source.value(), + destination.value(), result.error().toString() ); return reply.error("Failed to add rename operation: " + result.error().toString()); @@ -1254,8 +1396,8 @@ void Session::registerRpcSftpAddRenameOperation() Log::info( "Added rename operation with id '{}' for '{}' to '{}'", newOperationIdString, - sourcePath, - destinationPath + source.value(), + destination.value() ); self->resetQueueThrottle(); @@ -1313,14 +1455,22 @@ void Session::registerRpcSftpOpenSyncSession() if (!self) return reply({{"error", "Session no longer exists"}}); + // The roots every synchronized path is resolved against, refused here when they are not absolute. + auto remoteRoot = parseRemotePath(remotePath); + if (!remoteRoot) + return reply.error(remoteRoot.error()); + auto localRoot = Paths::LocalPath::fromUtf8(localPath); + if (!localRoot) + return reply.error(fmt::format("Invalid local path: {}", localRoot.error().toString())); + self->withSftpChannelDo( Ids::makeChannelId(channelIdString), [weak = self->weak_from_this(), syncSessionIdString, remoteScanIdString, localScanIdString, - remotePath, - localPath, + remotePath = *std::move(remoteRoot), + localPath = *std::move(localRoot), respectIgnoreFiles, recursive, ignoreHidden](RpcHelper::RpcOnce&& reply, auto&& channel) @@ -1456,49 +1606,61 @@ void Session::registerRpcSftpLoadSyncDiffChildren() ); } -void Session::registerRpcSftpBuildSyncEnqueuePlan() +void Session::registerRpcSftpExecuteSyncSelection() { - on(fmt::format("Session::{}::sftp::buildSyncEnqueuePlan", id_.value())) + // The backend plans the selection and resolves every row below its own sync roots, so no absolute path + // round-trips through the frontend. + on(fmt::format("Session::{}::sftp::executeSyncSelection", id_.value())) .perform( [weak = weak_from_this()]( - RpcHelper::RpcOnce&& reply, - std::string const& syncSessionIdString, - SharedData::Sync::DiffSection section, - std::vector const& selectedRelKeys, - std::string const& expectedGenerationString + RpcHelper::RpcOnce&& reply, std::string const& channelIdString, nlohmann::json const& requestJson ) { auto self = weak.lock(); if (!self || !self->operationQueue_) - return reply({{"error", "Session no longer exists"}}); + return reply.error("Session no longer exists"); - const auto syncSessionId = Ids::makeSyncSessionId(syncSessionIdString); - auto session = self->operationQueue_->syncSession(syncSessionId); - if (!session) - return reply({{"error", "Sync session not found"}}); + SharedData::Sync::SyncExecutionRequest request{}; + try + { + request = requestJson.get(); + } + catch (std::exception const& exception) + { + return reply.error(fmt::format("Invalid sync request: {}", exception.what())); + } - const std::uint64_t expectedGeneration = std::stoull(expectedGenerationString); - auto selected = std::unordered_set{selectedRelKeys.begin(), selectedRelKeys.end()}; + auto session = self->operationQueue_->syncSession(request.syncSessionId); + if (!session) + return reply.error("Sync session not found"); auto replyShared = std::make_shared(std::move(reply)); boost::asio::post( session->strand(), - [session, section, selected = std::move(selected), expectedGeneration, replyShared]() + [self, session, request = std::move(request), channelIdString, replyShared]() mutable { - if (session->generation() != expectedGeneration) - { - replyShared->error("stale generation"); - return; - } - const auto plan = session->buildEnqueuePlanInStrand(section, selected); - nlohmann::json payload = nlohmann::json::array(); - for (auto const& entry : plan) - { - nlohmann::json j; - SharedData::to_json(j, entry); - payload.push_back(std::move(j)); - } - (*replyShared)(std::move(payload)); + if (session->generation() != request.generation) + return replyShared->error("stale generation"); + + auto plan = session->resolvePlanInStrand( + request.section, std::unordered_set{request.relKeys.begin(), request.relKeys.end()} + ); + self->withSftpChannelDo( + Ids::makeChannelId(channelIdString), + [self, session, plan = std::move(plan), request = std::move(request)]( + RpcHelper::RpcOnce&& reply, auto&& channel + ) mutable + { + auto result = self->operationQueue_->executeSyncPlan( + *channel, *session, std::move(plan), request + ); + self->resetQueueThrottle(); + nlohmann::json json; + SharedData::to_json(json, result); + reply(std::move(json)); + }, + std::move(*replyShared) + ); } ); } @@ -1546,7 +1708,7 @@ void Session::registerRpcSftpPreDeleteChecks() [weak = weak_from_this()]( RpcHelper::RpcOnce&& reply, std::string const& channelIdString, - std::vector const& directories + std::vector const& directories ) { auto self = weak.lock(); @@ -1561,7 +1723,10 @@ void Session::registerRpcSftpPreDeleteChecks() if (!self) return reply.error("Session no longer exists"); - auto fut = channel->filterOutEmptyDirectories(directories); + auto remotePaths = parseRemotePaths(directories); + if (!remotePaths) + return reply.error(remotePaths.error()); + auto fut = channel->filterOutEmptyDirectories(*std::move(remotePaths)); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error("Failed to perform pre-delete checks: timeout"); @@ -1595,19 +1760,16 @@ void Session::registerRpcSftpDeleteFiles() if (!self) return reply.error("Session no longer exists"); - std::vector transformedPaths; - transformedPaths.resize(paths.size()); - std::transform( - paths.begin(), - paths.end(), - transformedPaths.begin(), - [](std::string const& p) - { - return Utility::pathFromUtf8(p); - } - ); - - auto fut = channel->removeAll(transformedPaths); + auto remotePaths = parseRemotePaths(paths); + if (!remotePaths) + return reply.error(remotePaths.error()); + for (auto const& path : *remotePaths) + { + if (auto allowed = self->operationQueue_->refuseProtectedDeleteTarget(*channel, path); + !allowed) + return reply.error(fmt::format("Refusing to delete '{}'", path.value())); + } + auto fut = channel->removeAll(*std::move(remotePaths)); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error("Failed to delete files: timeout"); @@ -1630,7 +1792,8 @@ void Session::registerRpcSftpRename() RpcHelper::RpcOnce&& reply, std::string const& channelIdString, std::string const& oldName, - std::string const& newName + std::string const& newName, + bool overwrite ) { auto self = weak.lock(); @@ -1639,19 +1802,37 @@ void Session::registerRpcSftpRename() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), - [weak = self->weak_from_this(), oldName, newName](RpcHelper::RpcOnce&& reply, auto&& channel) + [weak = self->weak_from_this(), oldName, newName, overwrite]( + RpcHelper::RpcOnce&& reply, auto&& channel + ) { auto self = weak.lock(); if (!self) return reply.error("Session no longer exists"); - auto fut = channel->rename(Utility::pathFromUtf8(oldName), Utility::pathFromUtf8(newName)); + const auto oldPath = Paths::RemotePath::make(oldName); + const auto newPath = Paths::RemotePath::make(newName); + if (!oldPath || !newPath || !oldPath->parent() || oldPath->parent() != newPath->parent()) + return reply.error("backend.rename.invalidName"); + + // Never replaces an existing item unless the user confirmed it. + auto fut = channel->performPromise( + [channel, oldPath = *oldPath, newPath = *newPath, overwrite]() + { + return overwrite ? channel->renameReplaceInStrand(oldPath, newPath, ".nui-sftp-old") + : channel->renameNoReplaceInStrand(oldPath, newPath); + } + ); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error("Failed to rename file: timeout"); const auto result = fut.get(); if (!result.has_value()) + { + if (result.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) + return reply.error("backend.rename.targetExists"); return reply.error("Failed to rename file: " + result.error().toString()); + } reply({{"success", true}}); }, std::move(reply) @@ -1671,13 +1852,24 @@ void Session::registerRpcSftpAddBulkDeleteOperation() RpcHelper::RpcOnce&& reply, std::string const& channelIdString, std::string const& bulkOperationIdString, - SharedData::BulkAddRequest const& request + nlohmann::json const& requestJson ) { auto self = weak.lock(); if (!self) return reply({{"error", "Session no longer exists"}}); + // Parsed here, so an invalid target is answered with an error instead of no reply at all. + SharedData::BulkDeleteRequest request{}; + try + { + request = requestJson.get(); + } + catch (std::exception const& exception) + { + return reply.error(fmt::format("Invalid bulk delete: {}", exception.what())); + } + self->withSftpChannelDo( Ids::makeChannelId(channelIdString), [weak = self->weak_from_this(), @@ -1688,6 +1880,14 @@ void Session::registerRpcSftpAddBulkDeleteOperation() if (!self) return reply({{"error", "Session no longer exists"}}); + // Nothing is queued when one of the targets is protected. + for (auto const& entry : request.entries) + { + if (auto allowed = self->operationQueue_->refuseProtectedDeleteTarget(*channel, entry.target); + !allowed) + return reply.error(fmt::format("Refusing to delete '{}'", entry.target.value())); + } + const auto enqueued = self->operationQueue_->addBulkDeleteOperation( *channel, request, Ids::makeOperationId(bulkOperationIdString) ); @@ -1706,9 +1906,9 @@ void Session::registerRpcSftpExistsBatch() { // Batched existence probe over SFTP. Lets the frontend ask whether N // remote paths exist in a single round-trip — eliminates the per-file - // SFTP stat latency that made multi-file uploads feel slow. Stat - // failures degrade to "not exists" per-entry rather than failing the - // whole batch (mirrors RpcFilesystem::existsBatch on the local side). + // SFTP stat latency that made multi-file uploads feel slow. Only "no such + // file" is absent, any other failure fails the whole batch: the frontend + // overwrites what it takes for absent. on(fmt::format("Session::{}::sftp::existsBatch", id_.value())) .perform( [weak = weak_from_this()]( @@ -1723,87 +1923,14 @@ void Session::registerRpcSftpExistsBatch() Ids::makeChannelId(channelIdString), [paths](RpcHelper::RpcOnce&& reply, auto&& channel) { - // Small drops: a handful of stat round-trips is cheaper than reading - // a potentially huge target directory in full. Above the threshold the - // single readdir below wins (one round-trip instead of N). - constexpr std::size_t listingThreshold = 10; - if (paths.size() < listingThreshold) + auto results = remotePathsExist(*channel, paths, futureTimeout); + if (!results) { - std::vector results; - results.reserve(paths.size()); - for (auto const& path : paths) - { - auto fut = channel->stat(Utility::pathFromUtf8(path)); - if (fut.wait_for(futureTimeout) != std::future_status::ready) - { - Log::warn( - "sftp::existsBatch: stat timeout for '{}' (treating as not-exists)", path - ); - results.push_back(false); - continue; - } - const auto result = fut.get(); - results.push_back(result.has_value()); - } - Log::info("sftp::existsBatch: probed {} paths via stat", paths.size()); - reply({{"success", true}, {"exists", results}}); - return; - } - - // Larger drops: group destinations by parent directory and list each - // parent once (readdir) rather than issuing one stat round-trip per - // path. A bulk drop targets a single directory, so this collapses N - // sequential SFTP round-trips into a single listing — a shallow - // existence diff, like sync does. A listing failure (e.g. the parent - // doesn't exist yet) degrades to "nothing exists", matching the - // per-stat fallback. - std::unordered_map>> listingByParent; - - auto listParent = [&channel]( - std::filesystem::path const& parent - ) -> std::optional> - { - auto fut = channel->listDirectory(parent); - if (fut.wait_for(futureTimeout) != std::future_status::ready) - { - Log::warn( - "sftp::existsBatch: listing timed out for '{}' (treating contents as not-exists)", - parent.generic_string() - ); - return std::nullopt; - } - const auto result = fut.get(); - if (!result.has_value()) - return std::nullopt; - std::unordered_set names; - names.reserve(result->size()); - for (auto const& entry : *result) - names.insert(entry.path.filename().generic_string()); - return names; - }; - - std::vector results; - results.reserve(paths.size()); - for (auto const& path : paths) - { - const auto fsPath = Utility::pathFromUtf8(path); - const auto parentKey = fsPath.parent_path().generic_string(); - auto it = listingByParent.find(parentKey); - if (it == listingByParent.end()) - it = listingByParent.emplace(parentKey, listParent(fsPath.parent_path())).first; - - auto const& listing = it->second; - results.push_back( - listing.has_value() && - listing->find(fsPath.filename().generic_string()) != listing->end() - ); + Log::error("sftp::existsBatch: {}", results.error()); + return reply.error(results.error()); } - Log::info( - "sftp::existsBatch: probed {} paths across {} dir listing(s)", - paths.size(), - listingByParent.size() - ); - reply({{"success", true}, {"exists", results}}); + Log::info("sftp::existsBatch: probed {} paths", paths.size()); + reply({{"success", true}, {"exists", *results}}); }, std::move(reply) ); @@ -1831,7 +1958,10 @@ void Session::registerRpcSftpStat() if (!self) return reply.error("Session no longer exists"); - auto fut = channel->stat(Utility::pathFromUtf8(path)); + const auto remotePath = parseRemotePath(path); + if (!remotePath) + return reply.error(remotePath.error()); + auto fut = channel->stat(*remotePath); if (fut.wait_for(futureTimeout) != std::future_status::ready) return reply.error("Failed to stat file: timeout"); diff --git a/backend/source/backend/sftp/archive_download_operation.cpp b/backend/source/backend/sftp/archive_download_operation.cpp index 6fd128c0..53986882 100644 --- a/backend/source/backend/sftp/archive_download_operation.cpp +++ b/backend/source/backend/sftp/archive_download_operation.cpp @@ -1,9 +1,12 @@ #include +#include +#include #include #include #include +#include #include #include @@ -104,7 +107,7 @@ namespace * @brief True for entries whose payload needs to stream over SFTP (regular * files only; directories and other typeflags carry no payload). */ - bool isRegularFile(SharedData::DirectoryEntry const& entry) noexcept + bool isRegularFile(TarArchive::MemberHeader const& entry) noexcept { return entry.type == SharedData::FileType::Regular; } @@ -112,7 +115,7 @@ namespace bool ArchiveDownloadOperation::isPayloadEntry(ResolvedEntry const& entry) const noexcept { - return isRegularFile(entry.tarMeta); + return isRegularFile(entry.header); } ArchiveDownloadOperation::ArchiveDownloadOperation(SecureShell::SftpSession& sftp, Options options) @@ -122,8 +125,12 @@ ArchiveDownloadOperation::ArchiveDownloadOperation(SecureShell::SftpSession& sft { options_.tempFileSuffix = Persistence::effectiveTempFileSuffix(options_.tempFileSuffix); - tempPath_ = options_.localArchivePath; - tempPath_ += options_.tempFileSuffix; + if (!options_.localArchivePath.empty()) + { + localArchiveNativePath_ = options_.localArchivePath.native(); + tempPath_ = localArchiveNativePath_; + tempPath_ += options_.tempFileSuffix; + } } ArchiveDownloadOperation::~ArchiveDownloadOperation() @@ -138,6 +145,14 @@ ArchiveDownloadOperation::~ArchiveDownloadOperation() std::expected ArchiveDownloadOperation::work() { + if (scanPhase_) + { + auto result = stepScanPhase(scanPhase_); + if (!result.has_value()) + return enterErrorState(std::move(result).error()); + return result; + } + auto future = sftp_->performPromise([this]() { return workInStrand(); }); if (future.wait_for(options_.futureTimeout) != std::future_status::ready) { @@ -152,6 +167,14 @@ ArchiveDownloadOperation::workInStrand() { using enum OperationState; + if (scanPhase_) + { + Log::error("ArchiveDownloadOperation: The scan phase cannot run on the processing thread."); + return enterErrorState( + {.type = ErrorType::ImplementationError, .extraInfo = "Scan phase stepped on the processing thread."} + ); + } + switch (state_) { case NotStarted: @@ -163,11 +186,11 @@ ArchiveDownloadOperation::workInStrand() { if (options_.localArchivePath.empty()) return enterErrorState({.type = ErrorType::InvalidPath}); - if (options_.entries.empty()) + if (options_.roots.empty()) return enterErrorState( {.type = ErrorType::ImplementationError, .extraInfo = "archive has no roots"} ); - if (std::filesystem::exists(options_.localArchivePath) && !options_.mayOverwrite) + if (std::filesystem::exists(localArchiveNativePath_) && !options_.mayOverwrite) return enterErrorState({.type = ErrorType::FileExists}); const auto prepareResult = prepareInStrand(); @@ -269,9 +292,15 @@ std::expected ArchiveDownloadOperation::prepareInStrand( .extraInfo = "no archivable entries under the selected roots", }); - auto sinkOrError = openArchiveSink( - tempPath_, options_.localArchivePath, options_.compression, options_.compressionOptions - ); + // Opening for writing follows a symlink planted at the temp path, out of the target directory. + if (std::error_code linkError{}; std::filesystem::is_symlink(std::filesystem::symlink_status(tempPath_, linkError))) + { + Log::error("ArchiveDownloadOperation: temp path '{}' is a symlink.", tempPath_.generic_string()); + return std::unexpected(Error{.type = ErrorType::OpenFailure, .extraInfo = "The temporary file path is a symlink"}); + } + + auto sinkOrError = + openArchiveSink(tempPath_, localArchiveNativePath_, options_.compression, options_.compressionOptions); if (!sinkOrError.has_value()) { Log::error( @@ -299,58 +328,57 @@ std::expected ArchiveDownloadOperation::prepareInStrand( std::expected ArchiveDownloadOperation::flattenRootsInStrand() { - for (auto const& root : options_.entries) + for (auto const& root : options_.roots) { - const auto rootFullPath = root.fullPath.empty() ? root.path : root.fullPath; - const auto basename = - rootFullPath.has_filename() ? rootFullPath.filename() : rootFullPath; + const auto name = root.path.filename(); + if (!name) + { + Log::error("ArchiveDownloadOperation: refusing root '{}', it has no name", root.path.value()); + return std::unexpected(Error{ + .type = ErrorType::InvalidPath, + .extraInfo = fmt::format("'{}' cannot be archived under its own name", root.path.value()), + }); + } + const auto archivePrefix = Paths::RelativePath{} / *name; - if (root.type == SharedData::FileType::Regular) + if (root.entry.type == SharedData::FileType::Regular) { - ResolvedEntry resolved{}; - resolved.tarMeta = root; - resolved.tarMeta.path = basename; - resolved.tarMeta.fullPath = basename; - resolved.remoteFullPath = rootFullPath; - totalPayloadBytes_ += resolved.tarMeta.size; + ResolvedEntry resolved{ + .header = TarArchive::memberHeaderOf(archivePrefix.value(), root.entry), + .source = root.path, + }; + totalPayloadBytes_ += resolved.header.size; resolvedEntries_.push_back(std::move(resolved)); } - else if (root.type == SharedData::FileType::Directory) + else if (root.entry.type == SharedData::FileType::Directory) { - ResolvedEntry dirEntry{}; - dirEntry.tarMeta = root; - dirEntry.tarMeta.path = basename; - dirEntry.tarMeta.fullPath = basename; - dirEntry.tarMeta.size = 0u; + ResolvedEntry dirEntry{.header = TarArchive::memberHeaderOf(archivePrefix.value(), root.entry)}; + dirEntry.header.size = 0u; resolvedEntries_.push_back(std::move(dirEntry)); // Directory roots are expanded exclusively from a preceding // ScanOperation's results — there is no self-contained recurse // fallback, so a visible scan card is always the source of truth // for this op's entry list. - const auto it = prescannedRoots_.find(rootFullPath); + const auto it = prescannedRoots_.find(root.path); if (it == prescannedRoots_.end()) { Log::error( - "ArchiveDownloadOperation: missing pre-scanned entries for directory root '{}'", - rootFullPath.generic_string() + "ArchiveDownloadOperation: missing pre-scanned entries for directory root '{}'", root.path.value() ); return std::unexpected(Error{ .type = ErrorType::ImplementationError, - .extraInfo = fmt::format( - "no pre-scanned entries for directory root '{}'", - rootFullPath.generic_string() - ), + .extraInfo = fmt::format("no pre-scanned entries for directory root '{}'", root.path.value()), }); } - appendPrescannedInStrand(it->second.first, rootFullPath, basename); + appendPrescannedInStrand(it->second.first, root.path, archivePrefix); } else { Log::warn( "ArchiveDownloadOperation: skipping non-regular, non-directory root '{}' (type={})", - rootFullPath.generic_string(), - static_cast(root.type) + root.path.value(), + static_cast(root.entry.type) ); } } @@ -359,8 +387,8 @@ std::expected ArchiveDownloadOperation::flattenRootsInSt void ArchiveDownloadOperation::appendPrescannedInStrand( std::vector const& entries, - std::filesystem::path const& remoteRootFullPath, - std::filesystem::path const& archivePrefix + Paths::RemotePath const& remoteRoot, + Paths::RelativePath const& archivePrefix ) { // Index 0 is the synthetic scan root (the directory itself); the caller @@ -369,34 +397,35 @@ void ArchiveDownloadOperation::appendPrescannedInStrand( for (std::size_t idx = 1u; idx < entries.size(); ++idx) { auto const& child = entries[idx]; - const auto relPath = SharedData::fullPathRelative(entries, child); - const auto childArchivePath = archivePrefix / relPath; - const auto childFullRemote = remoteRootFullPath / relPath; + const auto relative = SharedData::relativePath(entries, child); + if (!relative) + { + Log::warn("ArchiveDownloadOperation: skipping pre-scanned child: {}", relative.error().toString()); + continue; + } + const auto memberPath = archivePrefix / *relative; + const auto source = remoteRoot / *relative; if (child.type == SharedData::FileType::Regular) { - ResolvedEntry resolved{}; - resolved.tarMeta = child; - resolved.tarMeta.path = childArchivePath; - resolved.tarMeta.fullPath = childArchivePath; - resolved.remoteFullPath = childFullRemote; - totalPayloadBytes_ += resolved.tarMeta.size; + ResolvedEntry resolved{ + .header = TarArchive::memberHeaderOf(memberPath.value(), child), + .source = source, + }; + totalPayloadBytes_ += resolved.header.size; resolvedEntries_.push_back(std::move(resolved)); } else if (child.type == SharedData::FileType::Directory) { - ResolvedEntry dirEntry{}; - dirEntry.tarMeta = child; - dirEntry.tarMeta.path = childArchivePath; - dirEntry.tarMeta.fullPath = childArchivePath; - dirEntry.tarMeta.size = 0u; + ResolvedEntry dirEntry{.header = TarArchive::memberHeaderOf(memberPath.value(), child)}; + dirEntry.header.size = 0u; resolvedEntries_.push_back(std::move(dirEntry)); } else { Log::warn( "ArchiveDownloadOperation: skipping non-regular pre-scanned child '{}' (type={})", - childFullRemote.generic_string(), + source.value(), static_cast(child.type) ); } @@ -417,12 +446,12 @@ std::expected ArchiveDownloadOperation::openNextEntryInS { // Directory (or other payload-less) entry — just emit the tar header // and move on without opening a remote stream. - auto entryOrError = writer_->beginEntry(resolved.tarMeta); + auto entryOrError = writer_->beginEntry(resolved.header); if (!entryOrError.has_value()) { Log::error( "ArchiveDownloadOperation: cannot begin tar entry '{}': {}", - resolved.tarMeta.path.generic_string(), + resolved.header.path, entryOrError.error().toString() ); return std::unexpected(archiveErrorToOperationError(entryOrError.error())); @@ -435,7 +464,7 @@ std::expected ArchiveDownloadOperation::openNextEntryInS } const auto streamOpenResult = sftp_->openFileInStrand( - resolved.remoteFullPath, + resolved.source, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown ); @@ -443,25 +472,25 @@ std::expected ArchiveDownloadOperation::openNextEntryInS { Log::error( "ArchiveDownloadOperation: cannot open remote '{}': {}", - resolved.remoteFullPath.generic_string(), + resolved.source.value(), streamOpenResult.error().message ); return std::unexpected(Error{ .type = ErrorType::SftpError, .sftpError = streamOpenResult.error(), .extraInfo = fmt::format( - "Opening remote file: {}", resolved.remoteFullPath.generic_string() + "Opening remote file: {}", resolved.source.value() ), }); } currentStream_ = streamOpenResult.value(); - auto entryWriterOrError = writer_->beginEntry(resolved.tarMeta); + auto entryWriterOrError = writer_->beginEntry(resolved.header); if (!entryWriterOrError.has_value()) { Log::error( "ArchiveDownloadOperation: cannot begin tar entry '{}': {}", - resolved.tarMeta.path.generic_string(), + resolved.header.path, entryWriterOrError.error().toString() ); return std::unexpected(archiveErrorToOperationError(entryWriterOrError.error())); @@ -486,7 +515,7 @@ std::expected ArchiveDownloadOperation::readChunkInStran }); auto const& resolved = resolvedEntries_[currentEntryIndex_]; - const std::uint64_t remaining = resolved.tarMeta.size - currentEntryBytesRead_; + const std::uint64_t remaining = resolved.header.size - currentEntryBytesRead_; if (remaining == 0u) { const auto closeResult = finaliseEntryInStrand(); @@ -503,29 +532,31 @@ std::expected ArchiveDownloadOperation::readChunkInStran { Log::error( "ArchiveDownloadOperation: read failure on '{}': {}", - resolved.remoteFullPath.generic_string(), + resolved.source.value(), readResult.error().message ); return std::unexpected(Error{ .type = ErrorType::SftpError, .sftpError = readResult.error(), .extraInfo = fmt::format( - "Reading remote file: {}", resolved.remoteFullPath.generic_string() + "Reading remote file: {}", resolved.source.value() ), }); } const auto bytesRead = readResult.value(); if (bytesRead == 0) { - Log::warn( + Log::error( "ArchiveDownloadOperation: premature EOF on '{}' (expected {} more bytes)", - resolved.remoteFullPath.generic_string(), + resolved.source.value(), remaining ); - const auto closeResult = finaliseEntryInStrand(); - if (!closeResult.has_value()) - return std::unexpected(closeResult.error()); - return true; + return std::unexpected(Error{ + .type = ErrorType::SizeMismatch, + .extraInfo = fmt::format( + "'{}' ended {} bytes early", resolved.source.value(), remaining + ), + }); } const auto writeResult = currentEntry_->write(std::span{ @@ -536,7 +567,7 @@ std::expected ArchiveDownloadOperation::readChunkInStran { Log::error( "ArchiveDownloadOperation: tar write failure on '{}': {}", - resolved.tarMeta.path.generic_string(), + resolved.header.path, writeResult.error().toString() ); return std::unexpected(archiveErrorToOperationError(writeResult.error())); @@ -552,7 +583,7 @@ std::expected ArchiveDownloadOperation::readChunkInStran 0u, totalPayloadBytes_, totalBytesTransferred_, throughputMeter_->bytesPerSecond() ); - if (currentEntryBytesRead_ >= resolved.tarMeta.size) + if (currentEntryBytesRead_ >= resolved.header.size) { const auto closeResult = finaliseEntryInStrand(); if (!closeResult.has_value()) @@ -576,16 +607,23 @@ std::expected ArchiveDownloadOperation::finaliseEntryInS currentStream_.reset(); auto const& resolved = resolvedEntries_[currentEntryIndex_]; - if (currentEntryBytesRead_ != resolved.tarMeta.size) + if (currentEntryBytesRead_ != resolved.header.size) { - Log::warn( - "ArchiveDownloadOperation: entry '{}' truncated ({} of {} bytes) — archive padded with zeros", - resolved.tarMeta.path.generic_string(), + Log::error( + "ArchiveDownloadOperation: entry '{}' is incomplete ({} of {} bytes)", + resolved.header.path, currentEntryBytesRead_, - resolved.tarMeta.size + resolved.header.size ); - ++currentEntryIndex_; - return {}; + return std::unexpected(Error{ + .type = ErrorType::SizeMismatch, + .extraInfo = fmt::format( + "'{}' holds {} of {} bytes", + resolved.header.path, + currentEntryBytesRead_, + resolved.header.size + ), + }); } const auto closeResult = std::move(handle).close(); @@ -593,7 +631,7 @@ std::expected ArchiveDownloadOperation::finaliseEntryInS { Log::error( "ArchiveDownloadOperation: tar entry close failed for '{}': {}", - resolved.tarMeta.path.generic_string(), + resolved.header.path, closeResult.error().toString() ); return std::unexpected(archiveErrorToOperationError(closeResult.error())); @@ -621,11 +659,20 @@ std::expected ArchiveDownloadOperation::finaliseArchiveI return std::unexpected(archiveErrorToOperationError(finaliseResult.error())); } - if (std::filesystem::exists(options_.localArchivePath) && !options_.mayOverwrite) - return std::unexpected(Error{.type = ErrorType::FileExists}); + if (options_.markAsDownloaded) + { + if (const auto marked = Utility::markAsDownloaded(tempPath_); marked) + Log::warn("ArchiveDownloadOperation: Cannot mark the archive as downloaded: {}", marked.message()); + } + // Without permission to overwrite, the rename itself refuses an existing archive: no check-then-rename race. std::error_code renameErr{}; - std::filesystem::rename(tempPath_, options_.localArchivePath, renameErr); + if (options_.mayOverwrite) + std::filesystem::rename(tempPath_, localArchiveNativePath_, renameErr); + else if (auto renamed = Utility::renameNoReplace(tempPath_, localArchiveNativePath_); !renamed.has_value()) + renameErr = renamed.error(); + if (renameErr == std::errc::file_exists) + return std::unexpected(Error{.type = ErrorType::FileExists}); if (renameErr) { Log::error("ArchiveDownloadOperation: rename failed: {}", renameErr.message()); @@ -636,11 +683,13 @@ std::expected ArchiveDownloadOperation::finaliseArchiveI std::expected ArchiveDownloadOperation::cancel(bool adoptCancelState) { + if (scanPhase_) + scanPhase_->cancel(adoptCancelState); if (adoptCancelState) { Log::info( "ArchiveDownloadOperation: cancelled ({} roots, {} bytes planned).", - options_.entries.size(), + options_.roots.size(), totalPayloadBytes_ ); state_ = OperationState::Canceled; @@ -650,7 +699,7 @@ std::expected ArchiveDownloadOperation::cancel(bool adop } void ArchiveDownloadOperation::setScanResultForRoot( - std::filesystem::path const& remoteRootPath, + Paths::RemotePath const& remoteRootPath, std::vector entries, std::uint64_t totalBytes ) @@ -674,3 +723,18 @@ void ArchiveDownloadOperation::cleanup() std::filesystem::remove(tempPath_, removeErr); } } + +void ArchiveDownloadOperation::addScanRoot(std::unique_ptr scan) +{ + if (!scanPhase_) + { + scanPhase_ = std::make_unique>( + [this](ScanOperation& finished) + { + const auto scannedBytes = finished.totalBytes(); + setScanResultForRoot(finished.remotePath(), finished.ejectEntries(), scannedBytes); + } + ); + } + scanPhase_->addRoot(std::move(scan)); +} diff --git a/backend/source/backend/sftp/archive_upload_operation.cpp b/backend/source/backend/sftp/archive_upload_operation.cpp index c4ed4205..c406cfce 100644 --- a/backend/source/backend/sftp/archive_upload_operation.cpp +++ b/backend/source/backend/sftp/archive_upload_operation.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -69,15 +70,9 @@ namespace } /** - * @brief Build a DirectoryEntry from a local filesystem path. The @p archivePath - * is what goes into the tar header (relative to the archive root); the - * payload is read from @p localPath. + * @brief The metadata of @p localPath for its tar header. */ - std::optional - localPathToEntry( - std::filesystem::path const& localPath, - std::filesystem::path const& archivePath - ) + std::optional localPathToEntry(std::filesystem::path const& localPath) { std::error_code statErr{}; const auto status = std::filesystem::symlink_status(localPath, statErr); @@ -92,8 +87,6 @@ namespace } SharedData::DirectoryEntry entry{}; - entry.path = archivePath; - entry.fullPath = archivePath; entry.permissions = status.permissions(); entry.type = SharedData::fileTypeFromStdFilesystemType(status.type()); @@ -122,14 +115,17 @@ namespace bool ArchiveUploadOperation::isPayloadEntry(ResolvedEntry const& entry) const noexcept { - return entry.tarMeta.type == SharedData::FileType::Regular; + return entry.header.type == SharedData::FileType::Regular; } ArchiveUploadOperation::ArchiveUploadOperation(SecureShell::SftpSession& sftp, Options options) : Operation{} , sftp_{&sftp} , options_{std::move(options)} -{} +{ + remoteArchivePath_ = options_.remoteArchivePath; + tempPath_ = remoteArchivePath_.withFilenameSuffix(options_.tempFileSuffix).value_or(Paths::RemotePath{}); +} ArchiveUploadOperation::~ArchiveUploadOperation() { @@ -140,6 +136,14 @@ ArchiveUploadOperation::~ArchiveUploadOperation() std::expected ArchiveUploadOperation::work() { + if (scanPhase_) + { + auto result = stepScanPhase(scanPhase_); + if (!result.has_value()) + return enterErrorState(std::move(result).error()); + return result; + } + auto future = sftp_->performPromise([this]() { return workInStrand(); }); if (future.wait_for(options_.futureTimeout) != std::future_status::ready) { @@ -153,6 +157,19 @@ std::expected ArchiveUploadOperation::w { using enum OperationState; + if (scanPhase_) + { + Log::error("ArchiveUploadOperation: The scan phase cannot run on the processing thread."); + return enterErrorState( + {.type = ErrorType::ImplementationError, .extraInfo = "Scan phase stepped on the processing thread."} + ); + } + if (incompleteScan_ && state_ == OperationState::NotStarted) + { + Log::error("ArchiveUploadOperation: The scan is incomplete: {}", incompleteScan_->toString()); + return enterErrorState(*incompleteScan_); + } + switch (state_) { case NotStarted: @@ -247,27 +264,36 @@ std::expected ArchiveUploadOperation::flattenRoots() for (auto const& localRoot : options_.localPaths) { - const auto basename = - localRoot.has_filename() ? localRoot.filename() : localRoot; - auto rootEntry = localPathToEntry(localRoot, basename); + const auto name = localRoot.filename(); + if (!name) + { + Log::error("ArchiveUploadOperation: refusing root '{}', it has no name", localRoot.toUtf8()); + return std::unexpected(Error{ + .type = ErrorType::InvalidPath, + .extraInfo = fmt::format("'{}' cannot be archived under its own name", localRoot.toUtf8()), + }); + } + const auto archivePrefix = Paths::RelativePath{} / *name; + auto rootEntry = localPathToEntry(localRoot.native()); if (!rootEntry) continue; if (rootEntry->type == SharedData::FileType::Regular) { totalPayloadBytes_ += rootEntry->size; - ResolvedEntry resolved{}; - resolved.tarMeta = std::move(*rootEntry); - resolved.localFullPath = localRoot; - resolvedEntries_.push_back(std::move(resolved)); + resolvedEntries_.push_back( + ResolvedEntry{ + .header = TarArchive::memberHeaderOf(archivePrefix.value(), *rootEntry), + .source = localRoot, + } + ); } else if (rootEntry->type == SharedData::FileType::Directory) { rootEntry->size = 0u; - ResolvedEntry dirEntry{}; - dirEntry.tarMeta = std::move(*rootEntry); - dirEntry.localFullPath.clear(); - resolvedEntries_.push_back(std::move(dirEntry)); + resolvedEntries_.push_back( + ResolvedEntry{.header = TarArchive::memberHeaderOf(archivePrefix.value(), *rootEntry)} + ); // Directory roots are expanded exclusively from a preceding // LocalScanOperation's results — the archive op no longer @@ -276,24 +302,20 @@ std::expected ArchiveUploadOperation::flattenRoots() if (it == prescannedRoots_.end()) { Log::error( - "ArchiveUploadOperation: missing pre-scanned entries for directory root '{}'", - localRoot.generic_string() + "ArchiveUploadOperation: missing pre-scanned entries for directory root '{}'", localRoot.toUtf8() ); return std::unexpected(Error{ .type = ErrorType::ImplementationError, - .extraInfo = fmt::format( - "no pre-scanned entries for directory root '{}'", - localRoot.generic_string() - ), + .extraInfo = fmt::format("no pre-scanned entries for directory root '{}'", localRoot.toUtf8()), }); } - appendPrescannedFromScan(it->second.first, localRoot, basename); + appendPrescannedFromScan(it->second.first, localRoot, archivePrefix); } else { Log::warn( "ArchiveUploadOperation: skipping non-regular, non-directory root '{}' (type={})", - localRoot.generic_string(), + localRoot.toUtf8(), static_cast(rootEntry->type) ); } @@ -303,8 +325,8 @@ std::expected ArchiveUploadOperation::flattenRoots() void ArchiveUploadOperation::appendPrescannedFromScan( std::vector const& entries, - std::filesystem::path const& localRootFullPath, - std::filesystem::path const& archivePrefix + Paths::LocalPath const& localRoot, + Paths::RelativePath const& archivePrefix ) { // Index 0 is the synthetic scan root (the directory itself); the caller @@ -314,35 +336,41 @@ void ArchiveUploadOperation::appendPrescannedFromScan( for (std::size_t idx = 1u; idx < entries.size(); ++idx) { auto const& child = entries[idx]; - const auto relPath = SharedData::fullPathRelative(entries, child); - const auto childArchivePath = archivePrefix / relPath; - const auto childLocalFull = localRootFullPath / relPath; + auto source = SharedData::relativePath(entries, child) + .and_then( + [&localRoot](Paths::RelativePath const& relative) + { + return localRoot.join(relative); + } + ); + if (!source) + { + Log::warn("ArchiveUploadOperation: skipping pre-scanned child: {}", source.error().toString()); + continue; + } + const auto memberPath = archivePrefix / *SharedData::relativePath(entries, child); if (child.type == SharedData::FileType::Regular) { - ResolvedEntry resolved{}; - resolved.tarMeta = child; - resolved.tarMeta.path = childArchivePath; - resolved.tarMeta.fullPath = childArchivePath; - resolved.localFullPath = childLocalFull; - totalPayloadBytes_ += resolved.tarMeta.size; - resolvedEntries_.push_back(std::move(resolved)); + totalPayloadBytes_ += child.size; + resolvedEntries_.push_back( + ResolvedEntry{ + .header = TarArchive::memberHeaderOf(memberPath.value(), child), + .source = *std::move(source), + } + ); } else if (child.type == SharedData::FileType::Directory) { - ResolvedEntry dirEntry{}; - dirEntry.tarMeta = child; - dirEntry.tarMeta.path = childArchivePath; - dirEntry.tarMeta.fullPath = childArchivePath; - dirEntry.tarMeta.size = 0u; - dirEntry.localFullPath.clear(); + ResolvedEntry dirEntry{.header = TarArchive::memberHeaderOf(memberPath.value(), child)}; + dirEntry.header.size = 0u; resolvedEntries_.push_back(std::move(dirEntry)); } else { Log::warn( "ArchiveUploadOperation: skipping non-regular pre-scanned child '{}' (type={})", - childLocalFull.generic_string(), + source->toUtf8(), static_cast(child.type) ); } @@ -351,6 +379,16 @@ void ArchiveUploadOperation::appendPrescannedFromScan( std::expected ArchiveUploadOperation::prepareInStrand() { + if (remoteArchivePath_.empty() || remoteArchivePath_.isRoot() || tempPath_.empty()) + { + return std::unexpected( + Error{ + .type = ErrorType::InvalidPath, + .extraInfo = options_.remoteArchivePath.value(), + } + ); + } + const auto flattenResult = flattenRoots(); if (!flattenResult.has_value()) return std::unexpected(flattenResult.error()); @@ -361,38 +399,39 @@ std::expected ArchiveUploadOperation::prepareInStrand() .extraInfo = "no local files could be stat'd", }); - using OpenType = SecureShell::SftpSession::OpenType; - OpenType openType = OpenType::Write | OpenType::Create | OpenType::Truncate; - if (!options_.mayOverwrite) - openType = OpenType::Write | OpenType::Create | OpenType::Exclusive; + // Fail early instead of packing everything first, the final rename refuses to replace it anyway. + if (!options_.mayOverwrite && sftp_->lstatInStrand(remoteArchivePath_).has_value()) + return std::unexpected(Error{.type = ErrorType::FileExists}); + // Written under the temp name, so a failed or cancelled upload never leaves a truncated destination. + using OpenType = SecureShell::SftpSession::OpenType; const auto openResult = sftp_->openFileInStrand( - options_.remoteArchivePath, openType, std::filesystem::perms{0644u} + tempPath_, OpenType::Write | OpenType::Create | OpenType::Truncate, std::filesystem::perms{0644u} ); if (!openResult.has_value()) { Log::error( "ArchiveUploadOperation: cannot open remote '{}': {}", - options_.remoteArchivePath.generic_string(), + options_.remoteArchivePath.value(), openResult.error().message ); - return std::unexpected(Error{ - .type = ErrorType::SftpError, - .sftpError = openResult.error(), - .extraInfo = fmt::format( - "Opening remote archive file: {}", options_.remoteArchivePath.generic_string() - ), - }); + return std::unexpected( + Error{ + .type = ErrorType::SftpError, + .sftpError = openResult.error(), + .extraInfo = fmt::format("Opening remote archive file: {}", options_.remoteArchivePath.value()), + } + ); } remoteStream_ = openResult.value(); + tempCreated_ = true; // Wrap the SFTP terminal sink in the requested codec. Auto means derive // from the user-facing remote path so the bytes on the wire match the // declared file extension. - const auto resolvedCodec = - options_.compression == TarArchive::Compression::Auto - ? TarArchive::compressionFromExtension(options_.remoteArchivePath) - : options_.compression; + const auto resolvedCodec = options_.compression == TarArchive::Compression::Auto + ? TarArchive::compressionFromExtension(Paths::nativeFromUtf8(options_.remoteArchivePath.value())) + : options_.compression; auto wrappedSink = TarArchive::wrapSinkWithCodec( std::make_unique(remoteStream_), @@ -432,12 +471,12 @@ std::expected ArchiveUploadOperation::openNextEntryInStr if (!isPayloadEntry(resolved)) { // Directory entry — emit the tar header only and advance. - auto entryOrError = writer_->beginEntry(resolved.tarMeta); + auto entryOrError = writer_->beginEntry(resolved.header); if (!entryOrError.has_value()) { Log::error( "ArchiveUploadOperation: cannot begin tar entry '{}': {}", - resolved.tarMeta.path.generic_string(), + resolved.header.path, entryOrError.error().toString() ); return std::unexpected(archiveErrorToOperationError(entryOrError.error())); @@ -449,25 +488,25 @@ std::expected ArchiveUploadOperation::openNextEntryInStr return {}; } - currentLocalFile_.open(resolved.localFullPath, std::ios::binary); + currentLocalFile_.open(resolved.source.native(), std::ios::binary); if (!currentLocalFile_.is_open()) { Log::error( "ArchiveUploadOperation: cannot open local '{}'", - resolved.localFullPath.generic_string() + resolved.source.toUtf8() ); return std::unexpected(Error{ .type = ErrorType::OpenFailure, - .extraInfo = resolved.localFullPath.generic_string(), + .extraInfo = resolved.source.toUtf8(), }); } - auto entryWriterOrError = writer_->beginEntry(resolved.tarMeta); + auto entryWriterOrError = writer_->beginEntry(resolved.header); if (!entryWriterOrError.has_value()) { Log::error( "ArchiveUploadOperation: cannot begin tar entry '{}': {}", - resolved.tarMeta.path.generic_string(), + resolved.header.path, entryWriterOrError.error().toString() ); return std::unexpected(archiveErrorToOperationError(entryWriterOrError.error())); @@ -480,7 +519,7 @@ std::expected ArchiveUploadOperation::openNextEntryInStr std::expected ArchiveUploadOperation::readChunkInStrand() { auto const& resolved = resolvedEntries_[currentEntryIndex_]; - const std::uint64_t remaining = resolved.tarMeta.size - currentEntryBytesRead_; + const std::uint64_t remaining = resolved.header.size - currentEntryBytesRead_; if (remaining == 0u) { const auto closeResult = finaliseEntryInStrand(); @@ -496,15 +535,17 @@ std::expected ArchiveUploadOperation::readChunkInStrand( const auto bytesRead = static_cast(currentLocalFile_.gcount()); if (bytesRead == 0u) { - Log::warn( + Log::error( "ArchiveUploadOperation: local '{}' truncated mid-read (expected {} more bytes)", - resolved.localFullPath.generic_string(), + resolved.source.toUtf8(), remaining ); - const auto closeResult = finaliseEntryInStrand(); - if (!closeResult.has_value()) - return std::unexpected(closeResult.error()); - return true; + return std::unexpected(Error{ + .type = ErrorType::SizeMismatch, + .extraInfo = fmt::format( + "'{}' ended {} bytes early", resolved.source.toUtf8(), remaining + ), + }); } const auto writeResult = currentEntry_->write(std::span{ @@ -514,7 +555,7 @@ std::expected ArchiveUploadOperation::readChunkInStrand( { Log::error( "ArchiveUploadOperation: tar write failure on '{}': {}", - resolved.tarMeta.path.generic_string(), + resolved.header.path, writeResult.error().toString() ); return std::unexpected(archiveErrorToOperationError(writeResult.error())); @@ -530,7 +571,7 @@ std::expected ArchiveUploadOperation::readChunkInStrand( 0u, totalPayloadBytes_, totalBytesTransferred_, throughputMeter_->bytesPerSecond() ); - if (currentEntryBytesRead_ >= resolved.tarMeta.size) + if (currentEntryBytesRead_ >= resolved.header.size) { const auto closeResult = finaliseEntryInStrand(); if (!closeResult.has_value()) @@ -549,16 +590,23 @@ std::expected ArchiveUploadOperation::finaliseEntryInStr currentLocalFile_.close(); auto const& resolved = resolvedEntries_[currentEntryIndex_]; - if (currentEntryBytesRead_ != resolved.tarMeta.size) + if (currentEntryBytesRead_ != resolved.header.size) { - Log::warn( - "ArchiveUploadOperation: entry '{}' truncated ({} of {} bytes) — archive padded with zeros", - resolved.tarMeta.path.generic_string(), + Log::error( + "ArchiveUploadOperation: entry '{}' is incomplete ({} of {} bytes)", + resolved.header.path, currentEntryBytesRead_, - resolved.tarMeta.size + resolved.header.size ); - ++currentEntryIndex_; - return {}; + return std::unexpected(Error{ + .type = ErrorType::SizeMismatch, + .extraInfo = fmt::format( + "'{}' holds {} of {} bytes", + resolved.header.path, + currentEntryBytesRead_, + resolved.header.size + ), + }); } const auto closeResult = std::move(handle).close(); @@ -566,7 +614,7 @@ std::expected ArchiveUploadOperation::finaliseEntryInStr { Log::error( "ArchiveUploadOperation: tar entry close failed for '{}': {}", - resolved.tarMeta.path.generic_string(), + resolved.header.path, closeResult.error().toString() ); return std::unexpected(archiveErrorToOperationError(closeResult.error())); @@ -597,11 +645,29 @@ std::expected ArchiveUploadOperation::finaliseArchiveInS if (auto stream = remoteStream_.lock(); stream) stream->close(false); remoteStream_.reset(); + + const auto renameResult = options_.mayOverwrite + ? sftp_->renameReplaceInStrand(tempPath_, remoteArchivePath_, options_.tempFileSuffix + ".old") + : sftp_->renameNoReplaceInStrand(tempPath_, remoteArchivePath_); + if (!renameResult.has_value()) + { + Log::error( + "ArchiveUploadOperation: cannot move the archive to '{}': {}", + options_.remoteArchivePath.value(), + renameResult.error().message + ); + if (renameResult.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) + return std::unexpected(Error{.type = ErrorType::FileExists}); + return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = renameResult.error()}); + } + tempCreated_ = false; return {}; } std::expected ArchiveUploadOperation::cancel(bool adoptCancelState) { + if (scanPhase_) + scanPhase_->cancel(adoptCancelState); if (adoptCancelState) { Log::info( @@ -616,7 +682,7 @@ std::expected ArchiveUploadOperation::cancel(bool adoptC } void ArchiveUploadOperation::setScanResultForRoot( - std::filesystem::path const& localRootPath, + Paths::LocalPath const& localRootPath, std::vector entries, std::uint64_t totalBytes ) @@ -634,4 +700,43 @@ void ArchiveUploadOperation::cleanup() if (auto stream = remoteStream_.lock(); stream) stream->close(false); remoteStream_.reset(); + + if (tempCreated_) + { + tempCreated_ = false; + if (auto* processingStrand = sftp_->strand(); processingStrand && processingStrand->withinProcessingThread()) + std::ignore = sftp_->removeFileInStrand(tempPath_); + else + std::ignore = sftp_->removeFile(tempPath_).wait_for(options_.futureTimeout); + } +} + +void ArchiveUploadOperation::addScanRoot(std::unique_ptr scan) +{ + if (!scanPhase_) + { + scanPhase_ = std::make_unique>( + [this](LocalScanOperation& finished) + { + const auto scannedBytes = finished.totalBytes(); + // An archive cannot be partly successful, missing content fails it. + if (const auto& unreadable = finished.unreadableDirectories(); !unreadable.empty() && !incompleteScan_) + { + incompleteScan_ = Error{ + .type = ErrorType::DirectoryUnreadable, + .extraInfo = fmt::format("Cannot list '{}'", unreadable.front()), + }; + } + if (const auto rejected = finished.rejectedPaths(); !rejected.empty() && !incompleteScan_) + { + incompleteScan_ = Error{ + .type = ErrorType::InvalidName, + .extraInfo = fmt::format("'{}' is not a valid name", rejected.front()), + }; + } + setScanResultForRoot(finished.localPath(), finished.ejectEntries(), scannedBytes); + } + ); + } + scanPhase_->addRoot(std::move(scan)); } diff --git a/backend/source/backend/sftp/bulk_download_operation.cpp b/backend/source/backend/sftp/bulk_download_operation.cpp index 652bea6e..d3d54d70 100644 --- a/backend/source/backend/sftp/bulk_download_operation.cpp +++ b/backend/source/backend/sftp/bulk_download_operation.cpp @@ -1,6 +1,9 @@ #include +#include #include #include +#include +#include namespace { @@ -30,41 +33,43 @@ SharedData::OperationType BulkDownloadOperation::type() const return SharedData::OperationType::BulkDownload; } -std::filesystem::path BulkDownloadOperation::localPathOf(std::size_t entryIndex) const -{ - if (prescanned()) - return prescannedDestination(entryIndex); - return (common().localPath / SharedData::fullPathRelative(entries(), entries()[entryIndex])).lexically_normal(); -} - -std::filesystem::path BulkDownloadOperation::displayPath(std::size_t entryIndex) const +std::expected +BulkDownloadOperation::prepareRootInStrand(SharedData::DirectoryEntry const& root) { - return SharedData::fullPath(entries(), entries()[entryIndex]); + // Base directory of the download, everything after this will be relative to this: + return createLocalDirectory(common().localPath, root); } std::expected -BulkDownloadOperation::prepareRootInStrand(SharedData::DirectoryEntry const& root) +BulkDownloadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) { - // Base directory of the download, everything after this will be relative to this: - const auto path = common().localPath; - std::error_code ec; - std::filesystem::create_directories(path, ec); - if (ec) + // create_directories follows a link planted inside the tree and would create and fill directories outside it. + std::error_code error{}; + const auto link = Utility::firstLinkBelow(common().localPath.native(), paths.local.native(), error); + if (error) { - Log::error("BulkDownloadOperation: Failed to create local directory: {}: {}", path.string(), ec.message()); - return std::unexpected( - Error{ - .type = ErrorType::CannotCreateDirectory, - .extraInfo = fmt::format("Creating local directory: {}: {}", path.string(), ec.message()) - } - ); + return std::unexpected(Error{ + .type = ErrorType::FileStatFailed, + .extraInfo = fmt::format("Inspecting local directory: {}: {}", paths.local.toUtf8(), error.message()) + }); } - return applyPermsToDirectory(path, root); + if (link) + { + Log::error("BulkDownloadOperation: '{}' runs through the link '{}'.", paths.local.toUtf8(), Paths::nativeToUtf8(*link)); + return std::unexpected(Error{ + .type = ErrorType::LinkInLocalPath, + .extraInfo = fmt::format("The local path runs through a link: {}", Paths::nativeToUtf8(*link)) + }); + } + return createLocalDirectory(paths.local, entries()[entryIndex]); } -std::expected BulkDownloadOperation::createDirectoryInStrand(std::size_t entryIndex) +std::expected +BulkDownloadOperation::createLocalDirectory(Paths::LocalPath const& localPath, SharedData::DirectoryEntry const& entry) { - const auto path = localPathOf(entryIndex); + if (localPath.empty()) + return std::unexpected(Error{.type = ErrorType::ImplementationError, .extraInfo = "No local directory"}); + const auto path = localPath.native(); std::error_code ec; std::filesystem::create_directories(path, ec); if (ec) @@ -77,23 +82,28 @@ std::expected BulkDownloadOperation::createD } ); } - return applyPermsToDirectory(path, entries()[entryIndex]); + return applyPermsToDirectory(path, entry); } -std::unique_ptr -BulkDownloadOperation::makeChild(std::size_t entryIndex, std::function onProgress) +std::unique_ptr BulkDownloadOperation::makeChild( + std::size_t entryIndex, + EntryPaths const& paths, + std::function onProgress +) { auto const& entry = entries()[entryIndex]; auto downloadOptions = individualOptions_; - downloadOptions.remotePath = displayPath(entryIndex); - downloadOptions.localPath = localPathOf(entryIndex); + downloadOptions.remotePath = paths.remote; + downloadOptions.localPath = paths.local; + downloadOptions.mayOverwrite = downloadOptions.mayOverwrite || paths.mayOverwrite; downloadOptions.bigFileOptimized = entry.size >= Constants::bigFileCutOff; downloadOptions.progressCallback = [onProgress = std::move(onProgress)](auto, auto max, auto current, auto) { onProgress(current, max); }; downloadOptions.entry = entry; + downloadOptions.linkFreeRoot = common().localPath; return std::make_unique(sftp(), std::move(downloadOptions)); } @@ -102,23 +112,31 @@ bool BulkDownloadOperation::isSkippableError(Error const& error) const { // Missing or unreadable single files, and local files that cannot be written, do not compromise // the rest of the bulk. - if (error.type == ErrorType::OpenFailure || error.type == ErrorType::CannotCreateDirectory) + // A destination that appeared after the existence check and may not be replaced fails only its own entry. + if (error.type == ErrorType::OpenFailure || error.type == ErrorType::CannotCreateDirectory || + error.type == ErrorType::LinkInLocalPath || error.type == ErrorType::FileExists) return true; return error.sftpError && (error.sftpError->sftpError == SSH_FX_NO_SUCH_FILE || error.sftpError->sftpError == SSH_FX_PERMISSION_DENIED || error.sftpError->sftpError == SSH_FX_FAILURE); } +bool BulkDownloadOperation::isSkippableDirectoryError(Error const& error) const +{ + return error.type == ErrorType::LinkInLocalPath; +} + std::expected BulkDownloadOperation::applyPermsToDirectory( std::filesystem::path const& path, SharedData::DirectoryEntry const& entryToInheritFrom ) { + // Unknown permissions (the scan root carries none) leave the directory as created. std::optional toApplyPerms{std::nullopt}; if (individualOptions_.inheritPermissions) - toApplyPerms = entryToInheritFrom.permissions; + toApplyPerms = Utility::inheritablePermissions(entryToInheritFrom.permissions); else if (individualOptions_.directoryPermissions) - toApplyPerms = individualOptions_.directoryPermissions; + toApplyPerms = Utility::sanitizePermissions(*individualOptions_.directoryPermissions); if (!toApplyPerms) return {}; @@ -143,24 +161,27 @@ std::expected BulkDownloadOperation::applyPe void BulkDownloadOperation::setPrescannedFileList(std::vector files) { std::vector entries; - std::vector destinations; + std::vector paths; entries.reserve(files.size()); - destinations.reserve(files.size()); + paths.reserve(files.size()); std::uint64_t totalBytes = 0; for (auto& file : files) { SharedData::DirectoryEntry entry{}; - // entry.path carries the absolute remote source so SharedData::fullPath returns it - // verbatim (flat entry, no parent). The local destination lives in the aligned vector. - entry.path = file.remoteSrc; - entry.fullPath = file.remoteSrc; + // Flat entries carry their name only, the paths come from the prescanned paths. + if (const auto name = file.source.filename(); name) + entry.name = name->value(); entry.type = SharedData::FileType::Regular; entry.size = file.sizeBytes; entry.mtime = file.mtime; entry.mtimeNsec = file.mtimeNsec; totalBytes += file.sizeBytes; entries.push_back(std::move(entry)); - destinations.push_back(std::move(file.localDst)); + paths.push_back( + EntryPaths{ + .remote = std::move(file.source), .local = std::move(file.destination), .mayOverwrite = file.mayOverwrite + } + ); } - setPrescannedEntries(std::move(entries), std::move(destinations), totalBytes); + setPrescannedEntries(std::move(entries), std::move(paths), totalBytes); } diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 0237ddb1..19fc4572 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -19,6 +20,14 @@ BulkTransferOperation::~BulkTransferOperation() std::expected BulkTransferOperation::work() { + if (scanPhase_) + { + auto result = stepScanPhase(scanPhase_); + if (!result.has_value()) + return enterErrorState(std::move(result).error()); + return result; + } + auto fut = sftp_->performPromise( [this]() { @@ -37,6 +46,14 @@ std::expected B { using enum OperationState; + if (scanPhase_) + { + Log::error("{}: The scan phase cannot run on the processing thread.", logName()); + return enterErrorState( + Error{.type = ErrorType::ImplementationError, .extraInfo = "Scan phase stepped on the processing thread."} + ); + } + switch (state()) { case (NotStarted): @@ -81,8 +98,20 @@ std::expected B std::expected BulkTransferOperation::start() { + if (entrySource_ == EntrySource::None) + { + Log::error("{}: Started without entries.", logName()); + return enterErrorState( + Error{.type = ErrorType::ImplementationError, .extraInfo = "Neither a scan result nor a file list was set."} + ); + } if (entries_.empty()) { + if (entrySource_ == EntrySource::Tree) + { + Log::error("{}: The scan delivered no entries.", logName()); + return enterErrorState(Error{.type = ErrorType::ScanIncomplete}); + } Log::info("{}: No entries to transfer.", logName()); enterState(OperationState::Completed); return WorkStatus::Complete; @@ -97,13 +126,20 @@ std::expected B else { auto const& root = entries_[0]; - if (!root.isDirectory()) + if (!root.isDirectory() || root.parent) { - Log::error("{}: First entry is not a directory: {}.", logName(), root.path.string()); + Log::error("{}: The first entry is not the root directory of the scan.", logName()); return enterErrorState( Error{.type = ErrorType::ImplementationError, .extraInfo = "First entry must be a directory."} ); } + if (common_.remotePath.empty() || common_.localPath.empty()) + { + Log::error("{}: A scanned tree needs a remote and a local root.", logName()); + return enterErrorState( + Error{.type = ErrorType::ImplementationError, .extraInfo = "A tree needs both roots."} + ); + } const auto result = prepareRootInStrand(root); if (!result.has_value()) return enterErrorState(result.error()); @@ -185,13 +221,29 @@ std::expected BulkTransferOperation::fillSlo } auto const& entry = entries_[index]; + const auto paths = entryPaths(index); + if (!paths) + { + Log::warn("{}: Skipping '{}': {}", logName(), displayPath(index), paths.error().toString()); + failedEntries_.emplace_back(failureSource(index), paths.error()); + failedEntryIndices_.push_back(static_cast(index)); + completedBytes_ += entry.isRegularFile() ? entry.size : 0u; + ++completedEntries_; + continue; + } if (entry.isDirectory()) { // Created inline while the cursor passes it, so files listed after it only start once // it exists. Scan order lists parents before children. - const auto result = createDirectoryInStrand(index); + const auto result = createDirectoryInStrand(index, *paths); if (!result.has_value()) - return std::unexpected(result.error()); + { + if (!isSkippableDirectoryError(result.error())) + return std::unexpected(result.error()); + Log::warn("{}: Skipping directory '{}': {}", logName(), displayPath(index), result.error().toString()); + failedEntries_.emplace_back(failureSource(index), result.error()); + failedEntryIndices_.push_back(static_cast(index)); + } ++completedEntries_; ++directoriesCreated; continue; @@ -200,6 +252,7 @@ std::expected BulkTransferOperation::fillSlo { auto child = makeChild( index, + *paths, [this, index](std::uint64_t current, std::uint64_t max) { if (auto* slot = slotFor(index); slot) @@ -212,7 +265,7 @@ std::expected BulkTransferOperation::fillSlo slots_.push_back(Slot{.entryIndex = index, .child = std::move(child), .totalBytes = entry.size}); continue; } - Log::warn("{}: Skipping unsupported file type for entry: {}.", logName(), displayPath(index).string()); + Log::warn("{}: Skipping unsupported file type for entry: {}.", logName(), displayPath(index)); ++completedEntries_; } return {}; @@ -247,13 +300,11 @@ std::expected BulkTransferOperation::stepSlo continue; } - Log::error( - "{}: Transfer failed for: {}: {}", logName(), displayPath(slot.entryIndex).string(), error.toString() - ); + Log::error("{}: Transfer failed for: {}: {}", logName(), displayPath(slot.entryIndex), error.toString()); if (isSkippableError(error)) { // Not critical for the bulk as a whole: record the entry for the user and move on. - failedEntries_.emplace_back(SharedData::fullPath(entries_, entries_[slot.entryIndex]), error); + failedEntries_.emplace_back(failureSource(slot.entryIndex), error); failedEntryIndices_.push_back(static_cast(slot.entryIndex)); finishSlot(slot); if (common_.failFast) @@ -336,7 +387,7 @@ void BulkTransferOperation::emitProgress(bool force) continue; update.inFlight.push_back( SharedData::BulkFileProgress{ - .file = displayPath(slot.entryIndex).string(), + .file = displayPath(slot.entryIndex), .entryIndex = slot.entryIndex, .bytes = slot.bytes, .totalBytes = slot.totalBytes, @@ -348,7 +399,7 @@ void BulkTransferOperation::emitProgress(bool force) { update.inFlight.push_back( SharedData::BulkFileProgress{ - .file = displayPath(entryIndex).string(), + .file = displayPath(entryIndex), .entryIndex = entryIndex, .bytes = 0, .totalBytes = entries_[entryIndex].size, @@ -410,25 +461,123 @@ void BulkTransferOperation::setScanResult(std::vector entries, - std::vector destinations, + std::vector paths, std::uint64_t totalBytes ) { entries_ = std::move(entries); - prescannedDestinations_ = std::move(destinations); + prescannedPaths_ = std::move(paths); totalBytes_ = totalBytes; + entrySource_ = EntrySource::Flat; +} + +void BulkTransferOperation::addRejectedEntries(std::vector const& paths) +{ + for (auto const& path : paths) + { + Log::warn("{}: Skipping '{}', its name is not a valid path segment.", logName(), path); + failedEntries_.emplace_back( + path, Error{.type = ErrorType::InvalidName, .extraInfo = "The name is not a valid path segment"} + ); + } +} + +void BulkTransferOperation::addUnreadableDirectories(std::vector const& paths) +{ + for (auto const& path : paths) + { + Log::warn("{}: Skipping the content of '{}', it cannot be listed.", logName(), path); + failedEntries_.emplace_back( + path, Error{.type = ErrorType::DirectoryUnreadable, .extraInfo = "The directory cannot be listed"} + ); + } } -std::vector> BulkTransferOperation::getFailed() const +std::vector> BulkTransferOperation::getFailed() const { return failedEntries_; } +std::expected +BulkTransferOperation::entryPaths(std::size_t entryIndex) const +{ + if (prescanned()) + return prescannedPaths_[entryIndex]; + if (common_.remotePath.empty() || common_.localPath.empty()) + return std::unexpected(Error{.type = ErrorType::ImplementationError, .extraInfo = "A tree needs both roots"}); + + const auto invalidName = [](Paths::PathError const& error) + { + return Error{.type = ErrorType::InvalidName, .extraInfo = error.toString()}; + }; + + // The names from the entry up to, but not including, the root. + std::vector names; + for (auto const* current = &entries_[entryIndex]; current->parent; current = &entries_[*current->parent]) + { + if (*current->parent >= entries_.size()) + return std::unexpected(Error{.type = ErrorType::ImplementationError, .extraInfo = "Parent out of range"}); + names.push_back(¤t->name); + } + + Paths::RelativePath relative{}; + for (auto name = names.rbegin(); name != names.rend(); ++name) + { + auto segment = Paths::PathSegment::make(**name); + if (!segment) + return std::unexpected(invalidName(segment.error())); + relative = relative / *segment; + } + + return common_.localPath.join(relative) + .transform( + [this, &relative](Paths::LocalPath local) + { + return EntryPaths{.remote = common_.remotePath / relative, .local = std::move(local)}; + } + ) + .transform_error(invalidName); +} + +std::string BulkTransferOperation::displayPath(std::size_t entryIndex) const +{ + if (auto paths = entryPaths(entryIndex); paths) + return paths->remote.value(); + return describeEntry(entryIndex); +} + +std::string BulkTransferOperation::failureSource(std::size_t entryIndex) const +{ + if (auto paths = entryPaths(entryIndex); paths) + return sourceIsRemote() ? paths->remote.value() : paths->local.toUtf8(); + return describeEntry(entryIndex); +} + +std::string BulkTransferOperation::describeEntry(std::size_t entryIndex) const +{ + // The names as listed, one of them is no path segment or the entry would have paths. + std::vector names; + for (auto const* current = &entries_[entryIndex]; current->parent && *current->parent < entries_.size(); + current = &entries_[*current->parent]) + { + names.push_back(current->name); + } + auto root = sourceIsRemote() ? common_.remotePath.value() : common_.localPath.toUtf8(); + for (auto name = names.rbegin(); name != names.rend(); ++name) + { + if (!root.ends_with('/')) + root.push_back('/'); + root += *name; + } + return root; +} + void BulkTransferOperation::cancelChildrenInStrand(bool adoptCancelState) { for (auto& slot : slots_) @@ -442,6 +591,9 @@ void BulkTransferOperation::cancelChildrenInStrand(bool adoptCancelState) std::expected BulkTransferOperation::cancel(bool adoptCancelState) { + if (scanPhase_) + scanPhase_->cancel(adoptCancelState); + // Each child's cancel and destructor would block on the processing thread separately; // hop once and do all of them there. auto* processingStrand = sftp_->strand(); diff --git a/backend/source/backend/sftp/bulk_upload_operation.cpp b/backend/source/backend/sftp/bulk_upload_operation.cpp index 5cc8993e..c09678ae 100644 --- a/backend/source/backend/sftp/bulk_upload_operation.cpp +++ b/backend/source/backend/sftp/bulk_upload_operation.cpp @@ -1,6 +1,8 @@ #include +#include #include #include +#include namespace { @@ -30,26 +32,6 @@ SharedData::OperationType BulkUploadOperation::type() const return SharedData::OperationType::BulkUpload; } -std::filesystem::path BulkUploadOperation::localPathOf(std::size_t entryIndex) const -{ - // Prescanned entries carry the absolute local source in entry.path. - if (prescanned()) - return entries()[entryIndex].path; - return (common().localPath / SharedData::fullPathRelative(entries(), entries()[entryIndex])).lexically_normal(); -} - -std::filesystem::path BulkUploadOperation::remotePathOf(std::size_t entryIndex) const -{ - if (prescanned()) - return prescannedDestination(entryIndex); - return (common().remotePath / SharedData::fullPathRelative(entries(), entries()[entryIndex])).lexically_normal(); -} - -std::filesystem::path BulkUploadOperation::displayPath(std::size_t entryIndex) const -{ - return remotePathOf(entryIndex); -} - std::expected BulkUploadOperation::prepareRootInStrand(SharedData::DirectoryEntry const& root) { @@ -57,13 +39,14 @@ BulkUploadOperation::prepareRootInStrand(SharedData::DirectoryEntry const& root) return createRemoteDirectoryInStrand(common().remotePath, root); } -std::expected BulkUploadOperation::createDirectoryInStrand(std::size_t entryIndex) +std::expected +BulkUploadOperation::createDirectoryInStrand(std::size_t entryIndex, EntryPaths const& paths) { - return createRemoteDirectoryInStrand(remotePathOf(entryIndex), entries()[entryIndex]); + return createRemoteDirectoryInStrand(paths.remote, entries()[entryIndex]); } std::expected BulkUploadOperation::createRemoteDirectoryInStrand( - std::filesystem::path const& path, + Paths::RemotePath const& path, SharedData::DirectoryEntry const& entry ) { @@ -75,21 +58,25 @@ std::expected BulkUploadOperation::createRemot Error{ .type = ErrorType::SftpError, .sftpError = result.error(), - .extraInfo = fmt::format("Creating remote directory: {}", path.string()) + .extraInfo = fmt::format("Creating remote directory: {}", path.value()) } ); } return {}; } -std::unique_ptr -BulkUploadOperation::makeChild(std::size_t entryIndex, std::function onProgress) +std::unique_ptr BulkUploadOperation::makeChild( + std::size_t entryIndex, + EntryPaths const& paths, + std::function onProgress +) { auto const& entry = entries()[entryIndex]; auto uploadOptions = individualOptions_; - uploadOptions.remotePath = remotePathOf(entryIndex); - uploadOptions.localPath = localPathOf(entryIndex); + uploadOptions.remotePath = paths.remote; + uploadOptions.localPath = paths.local; + uploadOptions.mayOverwrite = uploadOptions.mayOverwrite || paths.mayOverwrite; uploadOptions.bigFileOptimized = entry.size >= Constants::bigFileCutOff; uploadOptions.progressCallback = [onProgress = std::move(onProgress)](auto, auto max, auto current, auto) { @@ -110,12 +97,15 @@ bool BulkUploadOperation::isSkippableError(Error const& error) const std::filesystem::perms BulkUploadOperation::determinePerms(SharedData::DirectoryEntry const& entry) const { + // Unknown permissions (the scan root carries none) fall back to the defaults below. if (individualOptions_.inheritPermissions) { - auto inherited = entry.permissions; - if (entry.isRegularFile()) - inherited |= std::filesystem::perms::owner_write; - return inherited; + if (auto inherited = Utility::sanitizePermissions(entry.permissions); inherited) + { + if (entry.isRegularFile()) + *inherited |= std::filesystem::perms::owner_write; + return *inherited; + } } if (entry.isDirectory()) @@ -129,8 +119,9 @@ std::filesystem::perms BulkUploadOperation::determinePerms(SharedData::Directory if (individualOptions_.filePermissions) return individualOptions_.filePermissions.value() | std::filesystem::perms::owner_write; // Inherit execute permissions anyway: - const auto ownerExecute = entry.permissions & std::filesystem::perms::owner_exec; - const auto groupExecute = entry.permissions & std::filesystem::perms::group_exec; + const auto known = Utility::sanitizePermissions(entry.permissions).value_or(std::filesystem::perms::none); + const auto ownerExecute = known & std::filesystem::perms::owner_exec; + const auto groupExecute = known & std::filesystem::perms::group_exec; return std::filesystem::perms::owner_read | std::filesystem::perms::owner_write | ownerExecute | std::filesystem::perms::group_read | std::filesystem::perms::group_write | groupExecute; } @@ -138,20 +129,25 @@ std::filesystem::perms BulkUploadOperation::determinePerms(SharedData::Directory void BulkUploadOperation::setPrescannedFileList(std::vector files) { std::vector entries; - std::vector destinations; + std::vector paths; entries.reserve(files.size()); - destinations.reserve(files.size()); + paths.reserve(files.size()); std::uint64_t totalBytes = 0; for (auto& file : files) { SharedData::DirectoryEntry entry{}; - entry.path = file.localSrc; - entry.fullPath = file.localSrc; + // Flat entries carry their name only, the paths come from the prescanned paths. + if (const auto name = file.source.filename(); name) + entry.name = name->value(); entry.type = SharedData::FileType::Regular; entry.size = file.sizeBytes; totalBytes += file.sizeBytes; entries.push_back(std::move(entry)); - destinations.push_back(std::move(file.remoteDst)); + paths.push_back( + EntryPaths{ + .remote = std::move(file.destination), .local = std::move(file.source), .mayOverwrite = file.mayOverwrite + } + ); } - setPrescannedEntries(std::move(entries), std::move(destinations), totalBytes); + setPrescannedEntries(std::move(entries), std::move(paths), totalBytes); } diff --git a/backend/source/backend/sftp/delete_operation.cpp b/backend/source/backend/sftp/delete_operation.cpp index 789077d7..55af74a3 100644 --- a/backend/source/backend/sftp/delete_operation.cpp +++ b/backend/source/backend/sftp/delete_operation.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -22,6 +23,14 @@ std::expected DeleteOperati { using enum OperationState; + if (scanPhase_) + { + auto result = stepScanPhase(scanPhase_); + if (!result.has_value()) + return enterErrorState(std::move(result).error()); + return result; + } + switch (state_) { case (NotStarted): @@ -33,9 +42,30 @@ std::expected DeleteOperati { if (!recursive_) { + if (remotePath_.empty() || remotePath_.isRoot()) + { + Log::error("DeleteOperation: Refusing to delete '{}'.", remotePath_.value()); + return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = remotePath_.value()}); + } + targets_ = {remotePath_}; totalToDelete_ = 1; filesRemovedProgress_ = {}; } + else if (!scanDelivered_) + { + // Without the scanned tree a recursive delete would silently delete nothing. + Log::error("DeleteOperation: Recursive delete of '{}' has no scan result.", remotePath_.value()); + return enterErrorState({.type = ErrorType::ScanIncomplete}); + } + else if (invalidScanEntry_) + { + Log::error( + "DeleteOperation: Scan of '{}' returned '{}', which is not a path below it.", + remotePath_.value(), + *invalidScanEntry_ + ); + return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = *invalidScanEntry_}); + } state_ = Prepared; [[fallthrough]]; @@ -47,27 +77,19 @@ std::expected DeleteOperati } case (Running): { - if (deletedThusFar_ == totalToDelete_) + if (deletedThusFar_ < totalToDelete_) { - state_ = Completed; - return WorkStatus::Complete; + auto removeResult = removeOnce(targets_[totalToDelete_ - deletedThusFar_ - 1]); + if (!removeResult.has_value()) + return std::unexpected(removeResult.error()); + if (deletedThusFar_ < totalToDelete_) + return WorkStatus::MoreWork; } - const auto currentPath = [this]() - { - if (!recursive_) - return remotePath_; - // go from back: - return fullPath(entries_, entries_[totalToDelete_ - deletedThusFar_ - 1]); - }(); - auto removeResult = removeOnce(currentPath); - if (!removeResult.has_value()) - return std::unexpected(removeResult.error()); - if (deletedThusFar_ == totalToDelete_) - { - state_ = Completed; - return WorkStatus::Complete; - } - return WorkStatus::MoreWork; + // Nothing deleted at all is a failure, some items left behind a partial success. + if (totalToDelete_ > 0 && failedEntries_.size() == totalToDelete_) + return enterErrorState(failedEntries_.back().second); + state_ = Completed; + return WorkStatus::Complete; } case (Finalizing): { @@ -103,35 +125,108 @@ std::expected DeleteOperati return enterErrorState({.type = ErrorType::UnknownWorkState}); } -std::expected DeleteOperation::removeOnce(std::filesystem::path const& path) +std::expected DeleteOperation::removeOnce(Paths::RemotePath const& path) { auto fut = sftp_->removeAll({path}); if (fut.wait_for(futureTimeout_) != std::future_status::ready) return enterErrorState({.type = ErrorType::FutureTimeout}); - const auto result = fut.get(); - if (!result.has_value()) - return enterErrorState( - {.type = ErrorType::DeleteFailed, .sftpError = result.error(), .extraInfo = path.generic_string()} + // One item that cannot be deleted does not keep the others, its directories then stay as well. An item that is + // already gone, after a resume for one, counts as deleted. + if (const auto result = fut.get(); !result.has_value() && result.error().sftpError != SSH_FX_NO_SUCH_FILE) + { + Log::error("DeleteOperation: Failed to delete '{}': {}", path.value(), result.error().toString()); + failedEntries_.emplace_back( + path.value(), Error{.type = ErrorType::DeleteFailed, .sftpError = result.error(), .extraInfo = path.value()} ); + } ++deletedThusFar_; if (filesRemovedProgress_) - filesRemovedProgress_(path.generic_string(), deletedThusFar_, totalToDelete_); + filesRemovedProgress_(path.value(), deletedThusFar_, totalToDelete_); return {}; } void DeleteOperation::setScanResult(std::vector&& entries, std::uint64_t /* totalBytes */) { - entries_ = std::move(entries); - totalToDelete_ = entries_.size(); + appendScanResult(remotePath_, entries); +} + +void DeleteOperation::appendScanResult(Paths::RemotePath const& root, std::vector const& entries) +{ + scanDelivered_ = true; + for (std::size_t index = 0; index < entries.size(); ++index) + { + auto const& entry = entries[index]; + // The scan root is the first entry and the only one without a parent, everything else lies below it. + if (index > 0 && !entry.parent) + { + invalidScanEntry_ = fmt::format("{} has a second root", root.value()); + break; + } + const auto relative = SharedData::relativePath(entries, entry); + if (!relative || root.empty()) + { + invalidScanEntry_ = relative ? std::string{"no scan root"} : relative.error().toString(); + break; + } + auto target = root / *relative; + if (target.isRoot()) + { + invalidScanEntry_ = target.value(); + break; + } + targets_.push_back(std::move(target)); + } + if (invalidScanEntry_) + targets_.clear(); + totalToDelete_ = targets_.size(); +} + +void DeleteOperation::setTargets(std::vector targets) +{ + scanDelivered_ = true; + for (auto& target : targets) + { + if (target.empty() || target.isRoot()) + { + invalidScanEntry_ = target.value(); + break; + } + targets_.push_back(std::move(target)); + } + if (invalidScanEntry_) + targets_.clear(); + totalToDelete_ = targets_.size(); +} + +void DeleteOperation::scanFirst(std::unique_ptr scan) +{ + addScanRoot(std::move(scan)); +} + +void DeleteOperation::addScanRoot(std::unique_ptr scan) +{ + if (!scanPhase_) + { + scanPhase_ = std::make_unique>( + [this](ScanOperation& finished) + { + appendScanResult(finished.remotePath(), finished.ejectEntries()); + } + ); + } + scanPhase_->addRoot(std::move(scan)); } std::expected DeleteOperation::cancel(bool adoptCancelState) { + if (scanPhase_) + scanPhase_->cancel(adoptCancelState); + if (adoptCancelState) { - Log::info("DeleteOperation: Delete of '{}' canceled.", remotePath_.generic_string()); + Log::info("DeleteOperation: Delete of '{}' canceled.", remotePath_.value()); state_ = OperationState::Canceled; } return {}; diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index 43e9985d..0302e2fd 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -1,7 +1,14 @@ #include +#include +#include #include #include +#include +#include +#include +#include + #include using namespace std::chrono_literals; @@ -13,6 +20,9 @@ DownloadOperation::DownloadOperation(SecureShell::SftpSession& sftp, DownloadOpe , localFile_{} { options_.tempFileSuffix = Persistence::effectiveTempFileSuffix(options_.tempFileSuffix); + remotePath_ = options_.remotePath; + if (!options_.localPath.empty()) + localNativePath_ = options_.localPath.native(); } DownloadOperation::~DownloadOperation() @@ -66,11 +76,11 @@ std::expected DownloadO Log::error("DownloadOperation: Invalid local path."); return enterErrorState({.type = ErrorType::InvalidPath}); } - if (std::filesystem::exists(options_.localPath) && !options_.mayOverwrite) + if (std::filesystem::exists(localNativePath_) && !options_.mayOverwrite) { Log::error( "DownloadOperation: File '{}' already exists and may not be overwritten.", - options_.localPath.generic_string() + options_.localPath.toUtf8() ); return enterErrorState({.type = ErrorType::FileExists}); } @@ -99,6 +109,11 @@ std::expected DownloadO options_.progressCallback(1, 1, 1, 0); return WorkStatus::Complete; } + if (transferComplete_) + { + state_ = Finalizing; + return WorkStatus::MoreWork; + } if (options_.bigFileOptimized) { auto stream = fileStream_.lock(); @@ -108,7 +123,7 @@ std::expected DownloadO return enterErrorState({.type = ErrorType::FileStreamExpired}); } const auto contextResult = stream->readAsyncInStrand( - options_.entry->size, + static_cast(options_.entry->size - resumeOffset_), buffer_.data(), buffer_.size(), [this](SecureShell::IFileStream::SignedSizeType bytesRead) @@ -153,6 +168,8 @@ std::expected DownloadO { if (asyncTransferContext_->hasEnded()) { + if (asyncTransferContext_->outcome() != SecureShell::AsyncTransferContext::Outcome::Completed) + return enterErrorState(asyncTransferError(*asyncTransferContext_)); options_.progressCallback( 0ull, options_.entry->size, options_.entry->size, asyncTransferContext_->bytesPerSecond() ); @@ -164,7 +181,7 @@ std::expected DownloadO options_.progressCallback( 0ull, options_.entry->size, - asyncTransferContext_->bytesTransferred(), + resumeOffset_ + static_cast(asyncTransferContext_->bytesTransferred()), asyncTransferContext_->bytesPerSecond() ); return WorkStatus::MoreWork; @@ -231,39 +248,46 @@ std::expected DownloadOperation::handleSymlinkIn { if (options_.symlinkHandling == Persistence::SymlinkHandling::SkipSymlink) { - Log::info("DownloadOperation: Skipping symlink as per options: {}.", options_.remotePath.string()); + Log::info("DownloadOperation: Skipping symlink as per options: {}.", options_.remotePath.value()); return {}; } - const auto readResult = readSymlinkInStrand(options_.remotePath); + const auto readResult = readSymlinkInStrand(remotePath_); if (!readResult.has_value()) return std::unexpected(readResult.error()); auto const& linkEntry = readResult.value().targetInfo; + // An existing link at the leaf makes creating the new one fail, only the directories above it matter. + if (!options_.linkFreeRoot.empty()) + { + if (const auto refused = refuseLinkedPath(localNativePath_.parent_path()); !refused) + return refused; + } + std::error_code ec{}; if (options_.createMissingDirectories) { - std::filesystem::create_directories(options_.localPath.parent_path(), ec); + std::filesystem::create_directories(localNativePath_.parent_path(), ec); if (ec) { Log::error( "DownloadOperation: Failed to create parent dirs for symlink '{}': {}.", - options_.localPath.generic_string(), + options_.localPath.toUtf8(), ec.message() ); return std::unexpected(Error{.type = ErrorType::CannotCreateDirectory}); } } + // The copy keeps the link's own content, so a relative link still points into the downloaded tree. + const auto linkContent = Paths::nativeFromUtf8(readResult.value().linkContent); if (linkEntry && linkEntry->isDirectory()) - std::filesystem::create_directory_symlink(readResult.value().linkTarget, options_.localPath, ec); + std::filesystem::create_directory_symlink(linkContent, localNativePath_, ec); else - std::filesystem::create_symlink(readResult.value().linkTarget, options_.localPath, ec); + std::filesystem::create_symlink(linkContent, localNativePath_, ec); if (ec) { Log::error( - "DownloadOperation: Failed to create symlink at '{}': {}.", - options_.localPath.generic_string(), - ec.message() + "DownloadOperation: Failed to create symlink at '{}': {}.", options_.localPath.toUtf8(), ec.message() ); return std::unexpected(Error{.type = ErrorType::CannotCreateSymlink}); } @@ -328,15 +352,14 @@ std::expected DownloadOperation::readOnceInStran return false; } - std::uint64_t tellp = 0; std::uint64_t fileSize = 0; bool good = true; { localFile_.write(buffer_.data(), static_cast(readAmount)); - tellp = static_cast(localFile_.tellp()); + bytesInFile_ += static_cast(readAmount); fileSize = options_.entry->size; good = localFile_.good(); - options_.progressCallback(0ull, fileSize, tellp, 0); + options_.progressCallback(0ull, fileSize, bytesInFile_, 0); } if (!good) { @@ -346,7 +369,7 @@ std::expected DownloadOperation::readOnceInStran }); return false; } - return good && tellp < fileSize; + return good && bytesInFile_ < fileSize; } bool DownloadOperation::commitBufferToFile(SecureShell::IFileStream::SignedSizeType bytesRead) @@ -363,12 +386,43 @@ bool DownloadOperation::commitBufferToFile(SecureShell::IFileStream::SignedSizeT return true; } +std::expected +DownloadOperation::refuseLinkedPath(std::filesystem::path const& path) const +{ + std::error_code error{}; + std::optional link{}; + if (!options_.linkFreeRoot.empty()) + link = Utility::firstLinkBelow(options_.linkFreeRoot.native(), path, error); + else if (Utility::isLinkOrJunction(path, error)) + link = path; + if (error) + { + Log::error("DownloadOperation: Cannot inspect '{}': {}.", Paths::nativeToUtf8(path), error.message()); + return std::unexpected(Error{.type = ErrorType::FileStatFailed, .extraInfo = Paths::nativeToUtf8(path)}); + } + if (link) + { + Log::error("DownloadOperation: '{}' runs through the link '{}'.", Paths::nativeToUtf8(path), Paths::nativeToUtf8(*link)); + return std::unexpected( + Error{ + .type = ErrorType::LinkInLocalPath, + .extraInfo = fmt::format("The local path runs through a link: {}", Paths::nativeToUtf8(*link)) + } + ); + } + return {}; +} + std::expected DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) { - auto tempPath = options_.localPath; + auto tempPath = localNativePath_; tempPath += options_.tempFileSuffix; + // Opening for writing follows a link planted at the temp path or above it, out of the target directory. + if (const auto refused = refuseLinkedPath(tempPath); !refused) + return refused; + if (options_.createMissingDirectories) { std::error_code mkErr{}; @@ -386,35 +440,56 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) if (options_.tryContinue && std::filesystem::exists(tempPath)) { - localFile_.open(tempPath, std::ios::binary | std::ios::app); - if (!localFile_.is_open()) + // The size on disk, tellp right after opening in append mode reports 0 on Windows. + std::error_code sizeError{}; + const auto existingSize = std::filesystem::file_size(tempPath, sizeError); + if (sizeError) { - Log::error("DownloadOperation: Failed to open file for appending: {}", tempPath.generic_string()); - return std::unexpected(Error{.type = ErrorType::OpenFailure}); + Log::error("DownloadOperation: Cannot read the size of '{}': {}", tempPath.generic_string(), sizeError.message()); + return std::unexpected(Error{.type = ErrorType::FileStatFailed}); + } + + // A leftover of another program or of an older version of the source must not be continued. + bool belongsToSource = existingSize > 0 && existingSize <= options_.entry->size; + if (belongsToSource) + { + const auto matches = ResumeCheck::tailsMatchInStrand(stream, tempPath, existingSize); + if (!matches) + return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = matches.error()}); + belongsToSource = *matches; } - if (static_cast(localFile_.tellp()) == options_.entry->size) + if (belongsToSource && existingSize == options_.entry->size) { Log::info( "DownloadOperation: File '{}' already complete, will be renamed in finalize() step.", tempPath.generic_string() ); - localFile_.close(); + transferComplete_ = true; return {}; } - else if (static_cast(localFile_.tellp()) > options_.entry->size) + else if (!belongsToSource) { Log::info( - "DownloadOperation: File '{}' is larger than expected, discarding and starting over.", + "DownloadOperation: File '{}' is not a part of this source, discarding and starting over.", tempPath.generic_string() ); - localFile_.close(); + if (auto rewound = stream.seekInStrand(0); !rewound) + return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = rewound.error()}); localFile_.open(tempPath, std::ios::binary | std::ios::trunc); } else { Log::info("DownloadOperation: File '{}' is incomplete, continuing download.", tempPath.generic_string()); - auto seekResult = stream.seekInStrand(localFile_.tellp()); + localFile_.open(tempPath, std::ios::binary | std::ios::app); + if (!localFile_.is_open()) + { + Log::error("DownloadOperation: Failed to open file for appending: {}", tempPath.generic_string()); + return std::unexpected(Error{.type = ErrorType::OpenFailure}); + } + resumeOffset_ = existingSize; + bytesInFile_ = existingSize; + auto seekResult = stream.seekInStrand(resumeOffset_); if (!seekResult.has_value()) return std::unexpected(Error{.type = ErrorType::FileStatFailed}); } @@ -434,7 +509,7 @@ DownloadOperation::openOrAdoptFileInStrand(SecureShell::IFileStream& stream) } std::expected -DownloadOperation::readSymlinkInStrand(std::filesystem::path const& remoteFullPath) +DownloadOperation::readSymlinkInStrand(Paths::RemotePath const& remoteFullPath) { auto deepLinkResult = sftp_->readLinkDeepInStrand(remoteFullPath); if (!deepLinkResult.has_value()) @@ -442,13 +517,13 @@ DownloadOperation::readSymlinkInStrand(std::filesystem::path const& remoteFullPa Log::error( "DownloadOperation: Failed to read symlink target: {} for entry: {}.", deepLinkResult.error().toString(), - remoteFullPath.generic_string() + remoteFullPath.value() ); return std::unexpected( Error{ .type = ErrorType::SftpError, .sftpError = deepLinkResult.error(), - .extraInfo = fmt::format("Reading symlink target for entry: {}", remoteFullPath.generic_string()) + .extraInfo = fmt::format("Reading symlink target for entry: {}", remoteFullPath.value()) } ); } @@ -464,13 +539,12 @@ std::expected DownloadOperation::prepare() } // Initial check. Check again later before rename. - if (std::filesystem::exists(options_.localPath)) + if (std::filesystem::exists(localNativePath_)) { if (!options_.mayOverwrite) { Log::error( - "DownloadOperation: File '{}' already exists and may not be overwritten.", - options_.localPath.generic_string() + "DownloadOperation: File '{}' already exists and may not be overwritten.", options_.localPath.toUtf8() ); return enterErrorState({.type = ErrorType::FileExists}); } @@ -495,9 +569,15 @@ std::expected DownloadOperation::prepare() std::expected DownloadOperation::prepareInStrand() { + if (remotePath_.empty() || remotePath_.isRoot()) + { + Log::error("DownloadOperation: Invalid remote path '{}'.", options_.remotePath.value()); + return std::unexpected(Error{.type = ErrorType::InvalidPath, .extraInfo = options_.remotePath.value()}); + } + if (!options_.entry) { - const auto fileInfo = sftp_->lstatInStrand(options_.remotePath); + const auto fileInfo = sftp_->lstatInStrand(remotePath_); if (!fileInfo.has_value()) { Log::error("DownloadOperation: Failed to stat file."); @@ -511,7 +591,7 @@ std::expected DownloadOperation::prepareInStrand return {}; const auto streamOpenResult = sftp_->openFileInStrand( - options_.remotePath, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown + remotePath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown ); if (!streamOpenResult.has_value()) { @@ -520,7 +600,7 @@ std::expected DownloadOperation::prepareInStrand Error{ .type = ErrorType::SftpError, .sftpError = streamOpenResult.error(), - .extraInfo = fmt::format("Opening remote file: {}", options_.remotePath.string()) + .extraInfo = fmt::format("Opening remote file: {}", options_.remotePath.value()) } ); } @@ -533,6 +613,24 @@ std::expected DownloadOperation::prepareInStrand return std::unexpected(Error{.type = ErrorType::FileStreamExpired}); } + if (options_.entry->isSymlink()) + { + // Following a link transfers its target, so the target's size and times apply. + const auto targetInfo = stream->statInStrand(); + if (!targetInfo.has_value()) + { + Log::error("DownloadOperation: Failed to stat the symlink target."); + return std::unexpected(Error{.type = ErrorType::FileStatFailed, .sftpError = targetInfo.error()}); + } + if (!targetInfo->isRegularFile()) + return std::unexpected(Error{.type = ErrorType::NotAFile}); + options_.entry->type = targetInfo->type; + options_.entry->size = targetInfo->size; + options_.entry->mtime = targetInfo->mtime; + options_.entry->mtimeNsec = targetInfo->mtimeNsec; + options_.entry->permissions = targetInfo->permissions; + } + auto openResult = openOrAdoptFileInStrand(*stream); if (!openResult.has_value()) { @@ -540,9 +638,6 @@ std::expected DownloadOperation::prepareInStrand return std::unexpected(openResult.error()); } - if (options_.entry->isSymlink()) - return {}; - buffer_ = sftp_->bufferProvider().leaseForTransfer(options_.entry->size, stream->readLengthLimit()); if (buffer_.empty()) { @@ -580,8 +675,8 @@ std::expected DownloadOperation::cancel(bool ado { Log::info( "DownloadOperation: Download of '{}' to '{}' canceled.", - options_.remotePath.generic_string(), - options_.localPath.generic_string() + options_.remotePath.value(), + options_.localPath.toUtf8() ); state_ = OperationState::Canceled; } @@ -602,7 +697,7 @@ void DownloadOperation::cleanup() localFile_.close(); - auto tempPath = options_.localPath; + auto tempPath = localNativePath_; tempPath += options_.tempFileSuffix; if (options_.doCleanup && std::filesystem::exists(tempPath)) std::filesystem::remove(tempPath); @@ -640,19 +735,54 @@ std::expected DownloadOperation::finalizeInStran localFile_.close(); - if (std::filesystem::exists(options_.localPath) && !options_.mayOverwrite) + auto tempPath = localNativePath_; + tempPath += options_.tempFileSuffix; + + // A short file must never replace the destination. + std::error_code sizeError{}; + const auto writtenSize = std::filesystem::file_size(tempPath, sizeError); + if (sizeError || writtenSize != options_.entry->size) { Log::error( - "DownloadOperation: File '{}' already exists and may not be overwritten.", - options_.localPath.generic_string() + "DownloadOperation: '{}' holds {} bytes instead of {}.", + tempPath.generic_string(), + sizeError ? 0u : writtenSize, + options_.entry->size ); - return std::unexpected(Error{.type = ErrorType::FileExists}); + return std::unexpected( + Error{ + .type = ErrorType::SizeMismatch, + .extraInfo = fmt::format( + "Received {} of {} bytes for {}", + sizeError ? 0u : writtenSize, + options_.entry->size, + options_.remotePath.value() + ), + } + ); + } + + // Marked before the rename, so the final file never exists without the mark. A filesystem without streams or + // extended attributes, FAT for one, cannot carry it. + if (options_.markAsDownloaded) + { + if (const auto marked = Utility::markAsDownloaded(tempPath); marked) + Log::warn("DownloadOperation: Cannot mark '{}' as downloaded: {}", Paths::nativeToUtf8(tempPath), marked.message()); } + // Without permission to overwrite, the rename itself refuses an existing file: no check-then-rename race. std::error_code ec{}; - auto tempPath = options_.localPath; - tempPath += options_.tempFileSuffix; - std::filesystem::rename(tempPath, options_.localPath, ec); + if (options_.mayOverwrite) + std::filesystem::rename(tempPath, localNativePath_, ec); + else if (auto renamed = Utility::renameNoReplace(tempPath, localNativePath_); !renamed.has_value()) + ec = renamed.error(); + if (ec == std::errc::file_exists) + { + Log::error( + "DownloadOperation: File '{}' already exists and may not be overwritten.", options_.localPath.toUtf8() + ); + return std::unexpected(Error{.type = ErrorType::FileExists}); + } if (ec) { Log::error("DownloadOperation: Failed to rename file: {}", ec.message()); @@ -675,8 +805,10 @@ std::expected DownloadOperation::finalizeInStran return std::unexpected(Error{.type = ErrorType::FileStatFailed, .sftpError = fileInfo.error()}); } + // Special bits never travel and the umask applies, like for a file created locally. std::error_code permissionsError{}; - std::filesystem::permissions(options_.localPath, fileInfo->permissions, permissionsError); + if (const auto inherited = Utility::inheritablePermissions(fileInfo->permissions); inherited) + std::filesystem::permissions(localNativePath_, *inherited, permissionsError); if (permissionsError) { Log::error("DownloadOperation: Failed to set permissions: {}", permissionsError.message()); @@ -686,7 +818,8 @@ std::expected DownloadOperation::finalizeInStran else if (options_.filePermissions) { std::error_code permissionsError{}; - std::filesystem::permissions(options_.localPath, *options_.filePermissions, permissionsError); + if (const auto configured = Utility::sanitizePermissions(*options_.filePermissions); configured) + std::filesystem::permissions(localNativePath_, *configured, permissionsError); if (permissionsError) { Log::error("DownloadOperation: Failed to set permissions: {}", permissionsError.message()); @@ -706,13 +839,11 @@ std::expected DownloadOperation::finalizeInStran )}; const auto fileTime = std::chrono::file_clock::from_sys(sysTime); std::error_code mtimeError{}; - std::filesystem::last_write_time(options_.localPath, fileTime, mtimeError); + std::filesystem::last_write_time(localNativePath_, fileTime, mtimeError); if (mtimeError) { Log::warn( - "DownloadOperation: Failed to set mtime on '{}': {}.", - options_.localPath.generic_string(), - mtimeError.message() + "DownloadOperation: Failed to set mtime on '{}': {}.", options_.localPath.toUtf8(), mtimeError.message() ); } } diff --git a/backend/source/backend/sftp/local_delete_operation.cpp b/backend/source/backend/sftp/local_delete_operation.cpp new file mode 100644 index 00000000..7362f48a --- /dev/null +++ b/backend/source/backend/sftp/local_delete_operation.cpp @@ -0,0 +1,88 @@ +#include +#include + +#include +#include + +LocalDeleteOperation::LocalDeleteOperation(LocalDeleteOperationOptions options) + : Operation{} + , options_{std::move(options)} +{} + +std::expected LocalDeleteOperation::work() +{ + using enum OperationState; + + switch (state_) + { + case NotStarted: + case Preparing: + case Prepared: + { + for (auto const& target : options_.targets) + { + if (target.empty() || target.isRoot()) + { + Log::error("LocalDeleteOperation: Refusing to delete '{}'.", target.toUtf8()); + return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = target.toUtf8()}); + } + } + state_ = Running; + [[fallthrough]]; + } + case Running: + { + const auto total = options_.targets.size(); + if (deletedThusFar_ < total) + { + auto const& target = options_.targets[total - deletedThusFar_ - 1]; + std::error_code error{}; + if (Utility::isLinkOrJunction(target.native(), error) && !error) + Utility::removeAllNoFollow(target.native(), error); + else if (!error) + std::filesystem::remove(target.native(), error); + // One item that cannot be deleted does not keep the others, its directories then stay as well. + if (error) + { + Log::error("LocalDeleteOperation: Failed to delete '{}': {}", target.toUtf8(), error.message()); + failedEntries_.emplace_back( + target.toUtf8(), + Error{ + .type = ErrorType::DeleteFailed, + .extraInfo = fmt::format("{}: {}", target.toUtf8(), error.message()) + } + ); + } + ++deletedThusFar_; + options_.itemsRemovedProgress(target.toUtf8(), deletedThusFar_, total); + if (deletedThusFar_ < total) + return WorkStatus::MoreWork; + } + // Nothing deleted at all is a failure, some items left behind a partial success. + if (total > 0 && failedEntries_.size() == total) + return enterErrorState(failedEntries_.back().second); + state_ = Completed; + return WorkStatus::Complete; + } + case Finalizing: + { + state_ = Completed; + return WorkStatus::Complete; + } + case Completed: + case PartialSuccess: + return std::unexpected(Error{.type = ErrorType::CannotWorkCompletedOperation}); + case Failed: + return std::unexpected(Error{.type = ErrorType::CannotWorkFailedOperation}); + case Canceled: + return std::unexpected(Error{.type = ErrorType::CannotWorkCanceledOperation}); + } + return enterErrorState({.type = ErrorType::UnknownWorkState}); +} + +std::expected LocalDeleteOperation::cancel(bool adoptCancelState) +{ + if (adoptCancelState) + state_ = OperationState::Canceled; + return {}; +} diff --git a/backend/source/backend/sftp/local_scan_operation.cpp b/backend/source/backend/sftp/local_scan_operation.cpp index dfc1a58a..8d74bdf8 100644 --- a/backend/source/backend/sftp/local_scan_operation.cpp +++ b/backend/source/backend/sftp/local_scan_operation.cpp @@ -1,5 +1,8 @@ #include +#include +#include #include +#include #include @@ -32,12 +35,13 @@ LocalScanOperation::LocalScanOperation(ScanOperationOptions options) , recursive_{options.recursive} , ignoreHidden_{options.ignoreHidden} , buildTree_{options.buildTree} + , excludedSuffixes_{std::move(options.excludedSuffixes)} {} LocalScanOperation::~LocalScanOperation() = default; std::expected, LocalScanOperation::Error> -LocalScanOperation::scanner(std::filesystem::path const& path) +LocalScanOperation::scanner(Paths::RelativePath const& relative) { // Non-recursive mode: only list the root directory. The walker still invokes the // scanner for each child directory it discovers, but we short-circuit those calls so @@ -46,28 +50,22 @@ LocalScanOperation::scanner(std::filesystem::path const& path) return std::vector{}; rootScanned_ = true; - std::error_code iterErrc{}; - auto iter = std::filesystem::directory_iterator( - path, std::filesystem::directory_options::skip_permission_denied, iterErrc - ); - if (iterErrc) + const auto joined = localPath_.join(relative); + if (!joined) { - Log::error( - "LocalScanOperation: Failed to open directory '{}': {}", path.generic_string(), iterErrc.message() - ); return enterErrorState>( - {.type = ErrorType::FilesystemError, .extraInfo = iterErrc.message()} + {.type = ErrorType::InvalidName, .extraInfo = joined.error().toString()} ); } - const auto end = std::filesystem::directory_iterator(); + const auto path = joined->native(); - const auto relDir = [&]() -> std::filesystem::path { - std::error_code errc{}; - auto rel = std::filesystem::relative(path, localPath_, errc); - if (errc || rel.empty() || rel == std::filesystem::path{"."}) - return {}; - return rel; - }(); + // No skip_permission_denied: an unreadable directory would list as empty and sync would delete the + // remote copy of its content. + std::error_code iterErrc{}; + auto iter = std::filesystem::directory_iterator(path, iterErrc); + if (iterErrc) + return listingFailed(path, relative, iterErrc); + const auto end = std::filesystem::directory_iterator(); if (respectIgnoreFiles_) { @@ -78,26 +76,36 @@ LocalScanOperation::scanner(std::filesystem::path const& path) if (!std::filesystem::is_regular_file(ignorePath, errc)) continue; if (auto content = readLocalFile(ignorePath)) - ignoreMatcher_.addFile(relDir, *content); + ignoreMatcher_.addFile(relative, *content); } } std::vector entries; - for (; iter != end; ++iter) + for (; iter != end; iter.increment(iterErrc)) { + if (iterErrc) + return listingFailed(path, relative, iterErrc); try { auto const& entry = *iter; const auto status = entry.symlink_status(); SharedData::DirectoryEntry::FileType type; +#ifdef _WIN32 + // A junction is a reparse point that std::filesystem reports as a directory. + std::error_code linkError{}; + const bool isSymlink = std::filesystem::is_symlink(status) || Utility::isLinkOrJunction(entry.path(), linkError); +#else const bool isSymlink = std::filesystem::is_symlink(status); +#endif if (isSymlink) { #ifdef _WIN32 - // Windows symlinks require elevated privileges to create and are rarely used by - // ordinary users. Skipping them here keeps sync predictable on Windows. - continue; + // Windows symlinks and junctions are not transferred. A sync tree keeps them as excluded + // placeholders, so the name still counts as present and the remote side is left alone. + if (!buildTree_) + continue; + type = SharedData::DirectoryEntry::FileType::Unknown; #else type = SharedData::DirectoryEntry::FileType::Symlink; #endif @@ -108,8 +116,13 @@ LocalScanOperation::scanner(std::filesystem::path const& path) type = SharedData::DirectoryEntry::FileType::Regular; else { - // Dont add anything thats not uploadable: - continue; + // Sockets, devices and FIFOs are not uploadable, a sync tree keeps them as excluded placeholders. + if (!buildTree_) + continue; + type = SharedData::fileTypeFromStdFilesystemType(status.type()); + if (type == SharedData::DirectoryEntry::FileType::Regular || + type == SharedData::DirectoryEntry::FileType::Directory) + type = SharedData::DirectoryEntry::FileType::Unknown; } std::uint64_t mtimeSecs = 0; @@ -135,34 +148,33 @@ LocalScanOperation::scanner(std::filesystem::path const& path) } } - std::optional linkTarget{}; + std::optional linkTarget{}; if (isSymlink) { std::error_code errc{}; - auto target = std::filesystem::read_symlink(entry.path(), errc); + const auto target = std::filesystem::read_symlink(entry.path(), errc); if (!errc) - linkTarget = std::move(target); + linkTarget = Paths::nativeToUtf8(target); } - const auto filename = entry.path().filename(); - if (ignoreHidden_) - { - const auto name = filename.string(); - if (!name.empty() && name.front() == '.') - continue; - } + const auto name = Paths::nativeToUtf8(entry.path().filename()); + if (ignoreHidden_ && !name.empty() && name.front() == '.') + continue; if (respectIgnoreFiles_ && !ignoreMatcher_.empty()) { - const auto childRel = relDir.empty() ? filename : (relDir / filename); - if (ignoreMatcher_.isIgnored(childRel, type == SharedData::DirectoryEntry::FileType::Directory)) + // A name that is no segment cannot match a rule, the walker rejects it. + const auto segment = Paths::PathSegment::make(name); + if (segment && + ignoreMatcher_.isIgnored(relative / *segment, type == SharedData::DirectoryEntry::FileType::Directory)) continue; } entries.push_back( SharedData::DirectoryEntry{ - .path = filename, + .name = name, .type = type, .size = entry.is_regular_file() ? entry.file_size() : 0, + .permissions = Utility::localPermissions(status), .mtime = mtimeSecs, .linkTarget = std::move(linkTarget), } @@ -179,6 +191,38 @@ LocalScanOperation::scanner(std::filesystem::path const& path) return {std::move(entries)}; } +std::expected, LocalScanOperation::Error> +LocalScanOperation::listingFailed( + std::filesystem::path const& path, + Paths::RelativePath const& relative, + std::error_code const& errorCode +) +{ + const bool belowRoot = !relative.empty(); + const bool deniedOrVanished = + errorCode == std::errc::permission_denied || errorCode == std::errc::no_such_file_or_directory; + if (belowRoot && deniedOrVanished) + { + Log::warn("LocalScanOperation: Cannot list '{}': {}", path.generic_string(), errorCode.message()); + if (!buildTree_) + { + // Left out of the entries, the consumer reports it instead of transferring an empty directory. + unreadableDirectories_.push_back(Paths::nativeToUtf8(path)); + return std::vector{}; + } + return std::unexpected( + Error{ + .type = ErrorType::DirectoryUnreadable, + .extraInfo = fmt::format("{}: {}", path.generic_string(), errorCode.message()), + } + ); + } + Log::error("LocalScanOperation: Failed to list '{}': {}", path.generic_string(), errorCode.message()); + return enterErrorState>( + {.type = ErrorType::FilesystemError, .extraInfo = errorCode.message()} + ); +} + std::uint64_t LocalScanOperation::totalBytes() const { return walker_->totalBytes(); @@ -192,8 +236,15 @@ std::expected LocalSc { case (NotStarted): { + if (localPath_.empty()) + { + Log::error("LocalScanOperation: No root to scan."); + return enterErrorState( + {.type = ErrorType::ImplementationError, .extraInfo = "The scan has no root."} + ); + } state_ = Running; - Log::info("LocalScanOperation: Starting scan of '{}'.", localPath_.generic_string()); + Log::info("LocalScanOperation: Starting scan of '{}'.", localPath_.toUtf8()); progressCallback_(0, 0, 0); return WorkStatus::MoreWork; } @@ -204,7 +255,7 @@ std::expected LocalSc { if (walker.completed()) { - Log::info("LocalScanOperation: Scan of '{}' completed.", localPath_.generic_string()); + Log::info("LocalScanOperation: Scan of '{}' completed.", localPath_.toUtf8()); state_ = Completed; return WorkStatus::Complete; } @@ -258,7 +309,7 @@ std::expected LocalScanOperation::cancel(bool a { if (adoptCancelState) { - Log::info("LocalScanOperation: Scan of '{}' canceled.", localPath_.generic_string()); + Log::info("LocalScanOperation: Scan of '{}' canceled.", localPath_.toUtf8()); enterState(OperationState::Canceled); } return {}; diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 2571ac31..baac44e8 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -23,6 +25,17 @@ using namespace std::chrono_literals; namespace { + /** + * @brief No path instead of an unset one, which the frontend would refuse to read. + */ + template + std::optional unlessEmpty(PathT path) + { + if (path.empty()) + return std::nullopt; + return path; + } + OperationQueue::OperationCompleted makeCompletedOperation( OperationQueue::CompletionReason reason, Ids::OperationId operationId, @@ -38,8 +51,8 @@ namespace .reason = reason, .operationId = operationId, .completionTime = std::chrono::system_clock::now(), - .localPath = op.localPath(), - .remotePath = op.remotePath(), + .localPath = unlessEmpty(op.localPath()), + .remotePath = unlessEmpty(op.remotePath()), .error = error, }; }, @@ -49,7 +62,7 @@ namespace .reason = reason, .operationId = operationId, .completionTime = std::chrono::system_clock::now(), - .remotePath = op.remotePath(), + .remotePath = unlessEmpty(op.remotePath()), .error = error, }; }, @@ -59,7 +72,7 @@ namespace .reason = reason, .operationId = operationId, .completionTime = std::chrono::system_clock::now(), - .localPath = op.localPath(), + .localPath = unlessEmpty(op.localPath()), .error = error, }; }, @@ -73,12 +86,15 @@ namespace .failedEntries = op.getFailed(), }; }, - [reason, operationId, error](UploadOperation const&) + [reason, operationId, error](UploadOperation const& op) { return OperationQueue::OperationCompleted{ .reason = reason, .operationId = operationId, .completionTime = std::chrono::system_clock::now(), + .localPath = unlessEmpty(op.localPath()), + .remotePath = unlessEmpty(op.remotePath()), + .error = error, }; }, [reason, operationId, error](BulkUploadOperation const& op) @@ -97,8 +113,19 @@ namespace .reason = reason, .operationId = operationId, .completionTime = std::chrono::system_clock::now(), - .remotePath = op.remotePath(), + .remotePath = unlessEmpty(op.remotePath()), .error = error, + .failedEntries = op.getFailed(), + }; + }, + [reason, operationId, error](LocalDeleteOperation const& op) + { + return OperationQueue::OperationCompleted{ + .reason = reason, + .operationId = operationId, + .completionTime = std::chrono::system_clock::now(), + .error = error, + .failedEntries = op.getFailed(), }; }, [reason, operationId, error](RenameOperation const& op) @@ -107,8 +134,7 @@ namespace .reason = reason, .operationId = operationId, .completionTime = std::chrono::system_clock::now(), - .localPath = op.sourcePath(), - .remotePath = op.destinationPath(), + .remotePath = unlessEmpty(op.destinationPath()), .error = error, }; }, @@ -158,6 +184,7 @@ namespace opts.customDirectoryPermissions ? opts.customDirectoryPermissions : def.directoryPermissions, .futureTimeout = futureTimeout, .symlinkHandling = opts.symlinkHandling.value_or(def.symlinkHandling), + .markAsDownloaded = opts.markAsDownloaded.value_or(def.markAsDownloaded), }; } @@ -270,6 +297,9 @@ void OperationQueue::cancelAll() self->operations_.clear(); self->priorityOperations_.clear(); + auto waiting = std::exchange(self->syncScanCallbacks_, {}); + for (auto& [id, callback] : waiting) + callback(std::unexpected(std::string{"The scan was cancelled."})); Log::info("All operations in the queue have been canceled."); } ); @@ -295,6 +325,7 @@ void OperationQueue::cancel(Ids::OperationId id) const auto erasedFromRegular = std::erase_if(self->operations_, cancelPredicate); if (erasedFromRegular == 0) std::erase_if(self->priorityOperations_, cancelPredicate); + self->failSyncScan(id, "The scan was cancelled."); } ); } @@ -404,11 +435,18 @@ void OperationQueue::completeOperation(SharedData::OperationCompleted&& operatio // a backup for reconnect adoption. self->bulkResumeStash_.erase(operationCompleted.operationId); + if (self->completionObserver_) + self->completionObserver_(operationCompleted); self->hub_->callRemote(self->rpcName("onOperationCompleted"), std::move(operationCompleted)); } ); } +void OperationQueue::setCompletionObserver(std::function observer) +{ + completionObserver_ = std::move(observer); +} + void OperationQueue::dumpBulkResumes(BulkResumeRegistry& registry) { if (bulkResumeStash_.empty()) @@ -426,34 +464,41 @@ void OperationQueue::adoptBulkResume( ) { Log::info( - "OperationQueue::adoptBulkResume: re-issuing bulk operation '{}' (kind={}, {} entries)", + "OperationQueue::adoptBulkResume: re-issuing bulk operation '{}' ({} entries)", operationId.value(), - static_cast(entry.kind), - entry.request.entries.size() + std::visit( + [](auto const& request) + { + return request.entries.size(); + }, + entry.request + ) ); - // For bulk download/upload the frontend reserves one extra id at the - // back of the per-entry id list for the aggregate card. On adopt we - // reuse the original aggregate id (so the surviving frontend card - // re-attaches) and synthesise fresh per-entry ids since their cards - // no longer exist in the new session. - auto operationIdFor = [](std::size_t) + // A transfer backup holds either the files of one bulk card or a single directory, both resume under the + // stashed id so the surviving card re-attaches. + auto operationIdFor = [&operationId](std::size_t) { - return Ids::generateOperationId(); + return operationId; }; - switch (entry.kind) - { - case BulkResumeEntry::Kind::BulkDownload: - addBulkDownloadOperation(sftp, entry.request, operationIdFor, operationId); - break; - case BulkResumeEntry::Kind::BulkUpload: - addBulkUploadOperation(sftp, entry.request, operationIdFor, operationId); - break; - case BulkResumeEntry::Kind::BulkDelete: - addBulkDeleteOperation(sftp, entry.request, operationId); - break; - } + std::visit( + Utility::overloaded{ + [&](SharedData::BulkDownloadRequest const& request) + { + addBulkDownloadOperation(sftp, request, operationIdFor, operationId); + }, + [&](SharedData::BulkUploadRequest const& request) + { + addBulkUploadOperation(sftp, request, operationIdFor, operationId); + }, + [&](SharedData::BulkDeleteRequest const& request) + { + addBulkDeleteOperation(sftp, request, operationId); + }, + }, + entry.request + ); } void OperationQueue::deepPause(bool pause) @@ -545,6 +590,7 @@ bool OperationQueue::workQueue(std::dequetype() == SharedData::OperationType::Scan) - { - // Walk past any additional queued scans to find the archive - // op that consumes multi-root scan batches — each per-root - // scan delivers its slice into the same ArchiveDownload. - Operation* nextNonScan = nullptr; - for (std::size_t lookahead = item.queueIndex + 1; lookahead < queue.size(); ++lookahead) - { - auto* candidate = queue[lookahead].second.get(); - if (candidate->type() != SharedData::OperationType::Scan) - { - nextNonScan = candidate; - break; - } - } - if (next && next->type() == SharedData::OperationType::BulkDownload) - { - auto* scan = static_cast(operation.get()); - static_cast(next)->setScanResult(scan->ejectEntries(), scan->totalBytes()); - } - else if (next && next->type() == SharedData::OperationType::Delete) - { - auto* scan = static_cast(operation.get()); - static_cast(next)->setScanResult(scan->ejectEntries(), scan->totalBytes()); - } - else if (nextNonScan && nextNonScan->type() == SharedData::OperationType::ArchiveDownload) - { - auto* scan = static_cast(operation.get()); - static_cast(nextNonScan) - ->setScanResultForRoot(scan->remotePath(), scan->ejectEntries(), scan->totalBytes()); - } - else - { - const auto cbIt = syncScanCallbacks_.find(id.value()); - if (cbIt != syncScanCallbacks_.end()) - { - auto* scan = static_cast(operation.get()); - auto tree = scan->ejectScanTree(); - auto callback = std::move(cbIt->second); - syncScanCallbacks_.erase(cbIt); - callback(std::move(tree)); - } - else - Log::error("Scan operation completed but no following BulkOperation to set results to."); - } - } - else if (operation->type() == SharedData::OperationType::LocalScan) - { - // Mirror the remote-Scan branch: walk past other LocalScans so - // multi-root ArchiveUpload sits any number of slots ahead. - Operation* nextNonScan = nullptr; - for (std::size_t lookahead = item.queueIndex + 1; lookahead < queue.size(); ++lookahead) - { - auto* candidate = queue[lookahead].second.get(); - if (candidate->type() != SharedData::OperationType::LocalScan) - { - nextNonScan = candidate; - break; - } - } - if (next && next->type() == SharedData::OperationType::BulkUpload) - { - auto* scan = static_cast(operation.get()); - static_cast(next)->setScanResult(scan->ejectEntries(), scan->totalBytes()); - } - else if (nextNonScan && nextNonScan->type() == SharedData::OperationType::ArchiveUpload) - { - auto* scan = static_cast(operation.get()); - static_cast(nextNonScan) - ->setScanResultForRoot(scan->localPath(), scan->ejectEntries(), scan->totalBytes()); - } - else - { - const auto cbIt = syncScanCallbacks_.find(id.value()); - if (cbIt != syncScanCallbacks_.end()) - { - auto* scan = static_cast(operation.get()); - auto tree = scan->ejectScanTree(); - auto callback = std::move(cbIt->second); - syncScanCallbacks_.erase(cbIt); - callback(std::move(tree)); - } - else - Log::error("LocalScan operation completed but no following BulkOperation to set results to."); - } - } - + deliverSyncScan(id, *operation); completeOperation(makeCompletedOperation(OperationQueue::CompletionReason::Completed, id, *operation)); queue.erase(queue.begin() + static_cast(item.queueIndex)); return true; @@ -656,6 +615,32 @@ bool OperationQueue::workQueue(std::deque(operation).ejectScanTree() + : static_cast(operation).ejectScanTree(); + callback.mapped()(std::move(tree)); +} + bool OperationQueue::work() { // Assumed in strand @@ -699,8 +684,8 @@ void OperationQueue::paused(bool pause) std::expected OperationQueue::addDownloadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::filesystem::path const& localPath, - std::filesystem::path const& remotePath, + Paths::LocalPath const& localPath, + Paths::RemotePath const& remotePath, bool allowOverwrite, bool isBigFile, bool insertRefresh, @@ -780,6 +765,7 @@ std::expected OperationQueue::addDownloadOperation( .totalBytes = result->size, .localPath = localPath, .remotePath = remotePath, + .allowOverwrite = allowOverwrite, } ); @@ -787,19 +773,6 @@ std::expected OperationQueue::addDownloadOperation( } else if (result->isDirectory()) { - // operationId is assigned to BulkDownload so the frontend's completion callback - // (registered on operationId) fires only after all files are fully downloaded, - // not when the preceding scan finishes. - const auto scanId = Ids::generateOperationId(); - auto scan = std::make_unique( - sftp, - ScanOperation::ScanOperationOptions{ - .progressCallback = makeScanProgressCallback("onScanProgress", scanId), - .remotePath = remotePath, - .futureTimeout = std::chrono::seconds{5}, - } - ); - auto bulk = std::make_unique( sftp, BulkDownloadOperation::BulkDownloadOperationOptions{ @@ -812,19 +785,21 @@ std::expected OperationQueue::addDownloadOperation( } ); + // The scan is part of the bulk, so cancelling or moving the card acts on both. + bulk->scanFirst( + std::make_unique( + sftp, + ScanOperation::ScanOperationOptions{ + .progressCallback = makeScanProgressCallback("onScanProgress", operationId), + .remotePath = remotePath, + .futureTimeout = resolvedTimeout, + } + ) + ); + auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - targetQueue.emplace_back(scanId, std::move(scan)); targetQueue.emplace_back(operationId, std::move(bulk)); - hub_->callRemote( - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = scanId, - .type = SharedData::OperationType::Scan, - .mode = mode, - .remotePath = remotePath, - } - ); hub_->callRemote( rpcName("onOperationAdded"), SharedData::OperationAdded{ @@ -834,21 +809,22 @@ std::expected OperationQueue::addDownloadOperation( .insertRefresh = insertRefresh, .localPath = localPath, .remotePath = remotePath, + .scanPhase = true, } ); // Synthetic single-entry resume backup: re-issuing this through // adopt walks the same scan+bulk path, so the entry list itself // doesn't matter — we just need the directory paths preserved. - SharedData::BulkAddRequest synthetic{ - .entries = {SharedData::BulkAddEntry{.src = remotePath, .dst = localPath, .isDirectory = true}}, + SharedData::BulkDownloadRequest synthetic{ + .entries = {SharedData::BulkDownloadEntry{ + .source = remotePath, .destination = localPath, .isDirectory = true + }}, .allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, .mode = mode, }; - bulkResumeStash_.insert_or_assign( - operationId, BulkResumeEntry{.kind = BulkResumeEntry::Kind::BulkDownload, .request = std::move(synthetic)} - ); + bulkResumeStash_.insert_or_assign(operationId, BulkResumeEntry{.request = std::move(synthetic)}); return {}; } @@ -897,8 +873,8 @@ namespace std::expected OperationQueue::addArchiveDownloadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::vector entries, - std::filesystem::path const& localArchivePath, + std::vector roots, + Paths::LocalPath const& localArchivePath, TarArchive::Compression compression, int compressionLevel, bool mayOverwrite, @@ -907,9 +883,9 @@ std::expected OperationQueue::addArchiveDownloadOperatio { // Assumed in strand. - if (entries.empty()) + if (roots.empty()) { - Log::error("addArchiveDownloadOperation: no entries provided."); + Log::error("addArchiveDownloadOperation: no roots provided."); return std::unexpected( Operation::Error{ .type = Operation::ErrorType::InvalidPath, @@ -919,33 +895,44 @@ std::expected OperationQueue::addArchiveDownloadOperatio } std::uint64_t totalPayloadBytes = 0u; - for (auto const& entry : entries) + for (auto const& root : roots) { - if (entry.type == SharedData::FileType::Regular) - totalPayloadBytes += entry.size; + // Every root is packed under its own name, the remote root directory has none. + if (!root.path.filename()) + { + return std::unexpected( + Operation::Error{ + .type = Operation::ErrorType::InvalidPath, + .extraInfo = fmt::format("'{}' cannot be archived under its own name", root.path.value()), + } + ); + } + if (root.entry.type == SharedData::FileType::Regular) + totalPayloadBytes += root.entry.size; } - const auto rootCount = entries.size(); + const auto rootCount = roots.size(); // Collect each directory root's full remote path so we can enqueue a // preceding ScanOperation per root. The archive operation consumes those // scans' results via setScanResultForRoot and never recurses itself, so // scan work is always visible as a separate queue card. - std::vector directoryRootPaths; - for (auto const& entry : entries) + std::vector directoryRootPaths; + for (auto const& root : roots) { - if (entry.type == SharedData::FileType::Directory) - { - const auto rootFullPath = entry.fullPath.empty() ? entry.path : entry.fullPath; - directoryRootPaths.push_back(rootFullPath); - } + if (root.entry.type == SharedData::FileType::Directory) + directoryRootPaths.push_back(root.path); } ArchiveDownloadOperation::Options opts{}; - opts.entries = std::move(entries); + opts.roots = std::move(roots); opts.localArchivePath = localArchivePath; + opts.tempFileSuffix = Persistence::effectiveTempFileSuffix( + sftpOpts_.downloadOptions.value_or(Persistence::DownloadOptions{}).tempFileSuffix + ); opts.compression = compression; opts.compressionOptions = compressionOptionsFromLevel(compression, compressionLevel); opts.mayOverwrite = mayOverwrite; + opts.markAsDownloaded = sftpOpts_.downloadOptions.value_or(Persistence::DownloadOptions{}).markAsDownloaded.value_or(true); opts.createMissingDirectories = true; opts.futureTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); opts.progressCallback = [weak = weak_from_this(), operationId, name = rpcName("onArchiveDownloadProgress")]( @@ -977,34 +964,23 @@ std::expected OperationQueue::addArchiveDownloadOperatio auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - // Scans first so the dispatcher can hand each scan's results to the - // archive op sitting one-or-more slots ahead. They share a single queue - // position (adjacency scan → ... → scan → archive), which the scan- - // completion handler in workQueue walks past to reach the archive op. + // Directory roots are scanned by the archive operation itself before it packs. + auto archive = std::make_unique(sftp, std::move(opts)); for (auto const& rootPath : directoryRootPaths) { - const auto scanId = Ids::generateOperationId(); - auto scan = std::make_unique( - sftp, - ScanOperation::ScanOperationOptions{ - .progressCallback = makeScanProgressCallback("onScanProgress", scanId), - .remotePath = rootPath, - .futureTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout), - } - ); - targetQueue.emplace_back(scanId, std::move(scan)); - hub_->callRemote( - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = scanId, - .type = SharedData::OperationType::Scan, - .mode = mode, - .remotePath = rootPath, - } + archive->addScanRoot( + std::make_unique( + sftp, + ScanOperation::ScanOperationOptions{ + .progressCallback = makeScanProgressCallback("onScanProgress", operationId), + .remotePath = rootPath, + .futureTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout), + } + ) ); } - - targetQueue.emplace_back(operationId, std::make_unique(sftp, std::move(opts))); + const auto scanPhase = archive->scanning(); + targetQueue.emplace_back(operationId, std::move(archive)); Log::info( "OperationQueue::{}::onOperationAdded (ArchiveDownload, {} entries, {} bytes, {} pre-scans)", @@ -1022,6 +998,7 @@ std::expected OperationQueue::addArchiveDownloadOperatio .insertRefresh = true, .totalBytes = totalPayloadBytes, .localPath = localArchivePath, + .scanPhase = scanPhase, } ); @@ -1031,8 +1008,8 @@ std::expected OperationQueue::addArchiveDownloadOperatio std::expected OperationQueue::addArchiveUploadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::vector localPaths, - std::filesystem::path const& remoteArchivePath, + std::vector localPaths, + Paths::RemotePath const& remoteArchivePath, TarArchive::Compression compression, int compressionLevel, bool mayOverwrite, @@ -1058,7 +1035,7 @@ std::expected OperationQueue::addArchiveUploadOperation( for (auto const& localPath : localPaths) { std::error_code sizeErr{}; - const auto size = std::filesystem::file_size(localPath, sizeErr); + const auto size = std::filesystem::file_size(localPath.native(), sizeErr); if (!sizeErr) estimatedTotalBytes += size; } @@ -1066,11 +1043,11 @@ std::expected OperationQueue::addArchiveUploadOperation( // Local directory roots get a visible LocalScanOperation each; the // archive op itself never recurses and instead consumes each scan's // output via setScanResultForRoot — keyed by the root's local path. - std::vector directoryRootPaths; + std::vector directoryRootPaths; for (auto const& localPath : localPaths) { std::error_code isDirErr{}; - if (std::filesystem::is_directory(localPath, isDirErr)) + if (std::filesystem::is_directory(localPath.native(), isDirErr)) directoryRootPaths.push_back(localPath); } const auto rootCount = localPaths.size(); @@ -1078,6 +1055,9 @@ std::expected OperationQueue::addArchiveUploadOperation( ArchiveUploadOperation::Options opts{}; opts.localPaths = std::move(localPaths); opts.remoteArchivePath = remoteArchivePath; + opts.tempFileSuffix = Persistence::effectiveTempFileSuffix( + sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}).tempFileSuffix + ); opts.compression = compression; opts.compressionOptions = compressionOptionsFromLevel(compression, compressionLevel); opts.mayOverwrite = mayOverwrite; @@ -1111,29 +1091,17 @@ std::expected OperationQueue::addArchiveUploadOperation( auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - // Scans first: the scan-complete dispatcher walks past subsequent scans - // to reach this archive op and feeds each LocalScan's slice into its - // setScanResultForRoot. Queue layout: [LocalScan...] ... [ArchiveUpload]. + // Directory roots are scanned by the archive operation itself before it packs. + auto archive = std::make_unique(sftp, std::move(opts)); for (auto const& rootPath : directoryRootPaths) { - const auto scanId = Ids::generateOperationId(); - auto scan = std::make_unique(LocalScanOperation::ScanOperationOptions{ - .progressCallback = makeScanProgressCallback("onLocalScanProgress", scanId), + archive->addScanRoot(std::make_unique(LocalScanOperation::ScanOperationOptions{ + .progressCallback = makeScanProgressCallback("onLocalScanProgress", operationId), .localPath = rootPath, - }); - targetQueue.emplace_back(scanId, std::move(scan)); - hub_->callRemote( - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = scanId, - .type = SharedData::OperationType::LocalScan, - .mode = mode, - .localPath = rootPath, - } - ); + })); } - - targetQueue.emplace_back(operationId, std::make_unique(sftp, std::move(opts))); + const auto scanPhase = archive->scanning(); + targetQueue.emplace_back(operationId, std::move(archive)); Log::info( "OperationQueue::{}::onOperationAdded (ArchiveUpload, {} paths, ~{} bytes, {} pre-scans)", @@ -1151,6 +1119,7 @@ std::expected OperationQueue::addArchiveUploadOperation( .insertRefresh = true, .totalBytes = estimatedTotalBytes, .remotePath = remoteArchivePath, + .scanPhase = scanPhase, } ); return {}; @@ -1158,7 +1127,7 @@ std::expected OperationQueue::addArchiveUploadOperation( std::size_t OperationQueue::addBulkDownloadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkAddRequest const& request, + SharedData::BulkDownloadRequest const& request, std::function operationIdFor, Ids::OperationId bulkCardId ) @@ -1178,6 +1147,12 @@ std::size_t OperationQueue::addBulkDownloadOperation( std::vector files; files.reserve(request.entries.size()); std::size_t directoryCount = 0; + // Each directory resumes as its own unit under its own id, so the card's backup holds only the files. + SharedData::BulkDownloadRequest fileUnit{ + .allowOverwrite = request.allowOverwrite, + .insertRefresh = request.insertRefresh, + .mode = request.mode, + }; for (std::size_t idx = 0; idx < request.entries.size(); ++idx) { auto const& entry = request.entries[idx]; @@ -1186,9 +1161,9 @@ std::size_t OperationQueue::addBulkDownloadOperation( const auto result = addDownloadOperation( sftp, operationIdFor(idx), - entry.dst, - entry.src, - request.allowOverwrite, + entry.destination, + entry.source, + request.allowOverwrite || entry.mayOverwrite, /*isBigFile*/ false, // Propagate the request's flag so manual folder downloads refresh // the local panel. Sync passes false to avoid refresh spam; a @@ -1202,7 +1177,7 @@ std::size_t OperationQueue::addBulkDownloadOperation( { Log::error( "Bulk download: directory entry '{}' failed to queue: {}", - entry.src.generic_string(), + entry.source.value(), result.error().toString() ); continue; @@ -1210,13 +1185,15 @@ std::size_t OperationQueue::addBulkDownloadOperation( ++directoryCount; continue; } + fileUnit.entries.push_back(entry); files.push_back( BulkDownloadOperation::PrescannedFile{ - .remoteSrc = entry.src, - .localDst = entry.dst, + .source = entry.source, + .destination = entry.destination, .sizeBytes = entry.sizeBytes, .mtime = entry.mtime, .mtimeNsec = entry.mtimeNsec, + .mayOverwrite = entry.mayOverwrite, } ); } @@ -1229,8 +1206,9 @@ std::size_t OperationQueue::addBulkDownloadOperation( // file entry drives the UI's "localPath"/"remotePath" labels for the // card — arbitrary pick; per-file paths show up in progress events. const auto bulkOpId = bulkCardId; - const auto firstSrc = files.front().remoteSrc; - const auto firstDst = files.front().localDst; + // The card's labels, a file directly below the root labels with the root itself. + const auto remoteLabel = files.front().source.parent().value_or(Paths::RemotePath::root()); + const auto localLabel = files.front().destination.parent().value_or(files.front().destination); auto downloadOpts = resolveDownloadOptions(transferOptions, resolvedTimeout); // Sync plans list files of directories that do not exist locally yet. @@ -1240,8 +1218,8 @@ std::size_t OperationQueue::addBulkDownloadOperation( sftp, BulkDownloadOperation::BulkDownloadOperationOptions{ .overallProgressCallback = makeBulkProgressCallback("onBulkDownloadProgress", bulkOpId), - .remotePath = firstSrc.parent_path(), - .localPath = firstDst.parent_path(), + .remotePath = remoteLabel, + .localPath = localLabel, .individualOptions = std::move(downloadOpts), .failFast = transferOptions.failFast.value_or(false), .concurrency = bulkConcurrency_, @@ -1257,24 +1235,17 @@ std::size_t OperationQueue::addBulkDownloadOperation( .type = SharedData::OperationType::BulkDownload, .mode = request.mode, .insertRefresh = request.insertRefresh, - .localPath = firstDst.parent_path(), - .remotePath = firstSrc.parent_path(), + .localPath = localLabel, + .remotePath = remoteLabel, } ); } Log::info("Bulk download: queued one bulk card (for files) + {} directory entries", directoryCount); - // Stash the request keyed by the aggregate bulk card id so a teardown - // can hand it to the resume registry. Mixed file/directory entries - // are preserved as-is — adopt re-issues the request through this same - // function which performs the same file/directory partition. - if (!request.entries.empty()) - { - bulkResumeStash_.insert_or_assign( - bulkCardId, BulkResumeEntry{.kind = BulkResumeEntry::Kind::BulkDownload, .request = request} - ); - } + // Stashed under the card id so a teardown can hand it to the resume registry. + if (!fileUnit.entries.empty()) + bulkResumeStash_.insert_or_assign(bulkCardId, BulkResumeEntry{.request = std::move(fileUnit)}); return request.entries.size(); } @@ -1282,8 +1253,8 @@ std::size_t OperationQueue::addBulkDownloadOperation( std::expected OperationQueue::addUploadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::filesystem::path const& localPath, - std::filesystem::path const& remotePath, + Paths::LocalPath const& localPath, + Paths::RemotePath const& remotePath, bool allowOverwrite, bool isBigFile, bool insertRefresh, @@ -1293,14 +1264,16 @@ std::expected OperationQueue::addUploadOperation( { // Assumed in strand + const auto localNativePath = localPath.native(); + std::error_code ec; // lstat-style: do not follow links here so dangling symlinks can still be uploaded as // symlinks (UploadOperation::handleSymlink recreates the literal on the remote). - const auto symStat = std::filesystem::symlink_status(localPath, ec); + const auto symStat = std::filesystem::symlink_status(localNativePath, ec); if (ec) { - Log::error("Failed to stat local file '{}': {}", localPath.generic_string(), ec.message()); + Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), ec.message()); return std::unexpected(Operation::Error{.type = Operation::ErrorType::FileStatFailed}); } @@ -1308,10 +1281,10 @@ std::expected OperationQueue::addUploadOperation( std::filesystem::file_status stat = symStat; if (!isSymlink) { - stat = std::filesystem::status(localPath, ec); + stat = std::filesystem::status(localNativePath, ec); if (ec) { - Log::error("Failed to stat local file '{}': {}", localPath.generic_string(), ec.message()); + Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), ec.message()); return std::unexpected(Operation::Error{.type = Operation::ErrorType::FileStatFailed}); } } @@ -1370,6 +1343,7 @@ std::expected OperationQueue::addUploadOperation( .insertRefresh = insertRefresh, .localPath = localPath, .remotePath = remotePath, + .allowOverwrite = allowOverwrite, } ); @@ -1377,16 +1351,10 @@ std::expected OperationQueue::addUploadOperation( } else if (stat.type() == std::filesystem::file_type::directory) { - auto scan = std::make_unique(LocalScanOperation::ScanOperationOptions{ - .progressCallback = makeScanProgressCallback("onLocalScanProgress", operationId), - .localPath = localPath, - }); - - const auto bulkId = Ids::generateOperationId(); auto bulk = std::make_unique( sftp, BulkUploadOperation::BulkUploadOperationOptions{ - .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", bulkId), + .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", operationId), .remotePath = remotePath, .localPath = localPath, .individualOptions = resolveUploadOptions(transferOptions, resolvedTimeout), @@ -1394,45 +1362,40 @@ std::expected OperationQueue::addUploadOperation( .concurrency = bulkConcurrency_, } ); + // The scan is part of the bulk, so cancelling or moving the card acts on both. + bulk->scanFirst( + std::make_unique(LocalScanOperation::ScanOperationOptions{ + .progressCallback = makeScanProgressCallback("onLocalScanProgress", operationId), + .localPath = localPath, + }) + ); auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - targetQueue.emplace_back(operationId, std::move(scan)); - targetQueue.emplace_back(bulkId, std::move(bulk)); + targetQueue.emplace_back(operationId, std::move(bulk)); hub_->callRemote( rpcName("onOperationAdded"), SharedData::OperationAdded{ .operationId = operationId, - .type = SharedData::OperationType::LocalScan, - .mode = mode, - .localPath = localPath, - } - ); - hub_->callRemote( - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = bulkId, .type = SharedData::OperationType::BulkUpload, .mode = mode, .insertRefresh = insertRefresh, .localPath = localPath, .remotePath = remotePath, + .scanPhase = true, } ); - // Synthetic single-entry resume — keyed by the bulk card's id - // (bulkId here, since the BulkUpload onOperationAdded uses bulkId - // as the frontend's card id). Adopt walks the same scan+bulk - // path on replay. - SharedData::BulkAddRequest synthetic{ - .entries = {SharedData::BulkAddEntry{.src = localPath, .dst = remotePath, .isDirectory = true}}, + // Synthetic single-entry resume backup: adopting it walks the same scan and bulk path. + SharedData::BulkUploadRequest synthetic{ + .entries = {SharedData::BulkUploadEntry{ + .source = localPath, .destination = remotePath, .isDirectory = true + }}, .allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, .mode = mode, }; - bulkResumeStash_.insert_or_assign( - bulkId, BulkResumeEntry{.kind = BulkResumeEntry::Kind::BulkUpload, .request = std::move(synthetic)} - ); + bulkResumeStash_.insert_or_assign(operationId, BulkResumeEntry{.request = std::move(synthetic)}); return {}; } @@ -1445,7 +1408,7 @@ std::expected OperationQueue::addUploadOperation( std::size_t OperationQueue::addBulkUploadOperation( SecureShell::SftpSession& sftp, - SharedData::BulkAddRequest const& request, + SharedData::BulkUploadRequest const& request, std::function operationIdFor, Ids::OperationId bulkCardId ) @@ -1462,6 +1425,12 @@ std::size_t OperationQueue::addBulkUploadOperation( std::vector files; files.reserve(request.entries.size()); std::size_t directoryCount = 0; + // Each directory resumes as its own unit under its own id, so the card's backup holds only the files. + SharedData::BulkUploadRequest fileUnit{ + .allowOverwrite = request.allowOverwrite, + .insertRefresh = request.insertRefresh, + .mode = request.mode, + }; for (std::size_t idx = 0; idx < request.entries.size(); ++idx) { auto const& entry = request.entries[idx]; @@ -1470,9 +1439,9 @@ std::size_t OperationQueue::addBulkUploadOperation( const auto result = addUploadOperation( sftp, operationIdFor(idx), - entry.src, - entry.dst, - request.allowOverwrite, + entry.source, + entry.destination, + request.allowOverwrite || entry.mayOverwrite, /*isBigFile*/ false, // Propagate the request's flag so manual folder uploads refresh // the remote panel. Sync passes false to avoid refresh spam; a @@ -1486,7 +1455,7 @@ std::size_t OperationQueue::addBulkUploadOperation( { Log::error( "Bulk upload: directory entry '{}' failed to queue: {}", - entry.src.generic_string(), + entry.source.toUtf8(), result.error().toString() ); continue; @@ -1494,11 +1463,13 @@ std::size_t OperationQueue::addBulkUploadOperation( ++directoryCount; continue; } + fileUnit.entries.push_back(entry); files.push_back( BulkUploadOperation::PrescannedFile{ - .localSrc = entry.src, - .remoteDst = entry.dst, + .source = entry.source, + .destination = entry.destination, .sizeBytes = entry.sizeBytes, + .mayOverwrite = entry.mayOverwrite, } ); } @@ -1507,8 +1478,8 @@ std::size_t OperationQueue::addBulkUploadOperation( { // Separate reserved id — see addBulkDownloadOperation for rationale. const auto bulkOpId = bulkCardId; - const auto firstSrc = files.front().localSrc; - const auto firstDst = files.front().remoteDst; + const auto localLabel = files.front().source.parent().value_or(files.front().source); + const auto remoteLabel = files.front().destination.parent().value_or(Paths::RemotePath::root()); auto uploadOpts = resolveUploadOptions(transferOptions, resolvedTimeout); uploadOpts.createMissingDirectories = true; @@ -1517,8 +1488,8 @@ std::size_t OperationQueue::addBulkUploadOperation( sftp, BulkUploadOperation::BulkUploadOperationOptions{ .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", bulkOpId), - .remotePath = firstDst.parent_path(), - .localPath = firstSrc.parent_path(), + .remotePath = remoteLabel, + .localPath = localLabel, .individualOptions = std::move(uploadOpts), .failFast = transferOptions.failFast.value_or(false), .concurrency = bulkConcurrency_, @@ -1534,28 +1505,277 @@ std::size_t OperationQueue::addBulkUploadOperation( .type = SharedData::OperationType::BulkUpload, .mode = request.mode, .insertRefresh = request.insertRefresh, - .localPath = firstSrc.parent_path(), - .remotePath = firstDst.parent_path(), + .localPath = localLabel, + .remotePath = remoteLabel, } ); } Log::info("Bulk upload: queued one bulk card (for files) + {} directory entries", directoryCount); - if (!request.entries.empty()) + if (!fileUnit.entries.empty()) + bulkResumeStash_.insert_or_assign(bulkCardId, BulkResumeEntry{.request = std::move(fileUnit)}); + + return request.entries.size(); +} + +namespace +{ + /** + * @brief Creates @p directory and its missing parents below @p root, which has to exist. + */ + bool createRemoteDirectoryBelow( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& root, + Paths::RemotePath const& directory, + std::chrono::seconds timeout + ) + { + std::vector chain; + for (auto current = std::optional{directory}; current && *current != root && current->isWithin(root); + current = current->parent()) + chain.push_back(*current); + for (auto iter = chain.rbegin(); iter != chain.rend(); ++iter) + { + auto created = sftp.createDirectoryIfItDoesntExist(*iter); + if (created.wait_for(timeout) != std::future_status::ready || !created.get().has_value()) + { + Log::error("Sync: could not create remote directory '{}'.", iter->value()); + return false; + } + } + return true; + } +} + +SharedData::Sync::SyncExecutionResult OperationQueue::executeSyncPlan( + SecureShell::SftpSession& sftp, + SyncSession const& session, + SyncSession::ResolvedPlan plan, + SharedData::Sync::SyncExecutionRequest const& request +) +{ + // Assumed in strand + using SharedData::Sync::Action; + const auto timeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); + SharedData::Sync::SyncExecutionResult result{.failed = std::move(plan.refused)}; + + SharedData::BulkUploadRequest uploads{.allowOverwrite = true, .mode = request.mode}; + SharedData::BulkDownloadRequest downloads{.allowOverwrite = true, .mode = request.mode}; + SharedData::BulkDeleteRequest remoteDeletes{.mode = request.mode, .listedContentOnly = true}; + std::vector localDeletes; + const auto localHome = localHomeDirectory(); + + for (auto& [entry, paths] : plan.entries) { - bulkResumeStash_.insert_or_assign( - bulkCardId, BulkResumeEntry{.kind = BulkResumeEntry::Kind::BulkUpload, .request = request} + switch (entry.action) + { + case Action::Upload: + { + if (entry.isDirectory) + { + auto& target = createRemoteDirectoryBelow(sftp, session.remoteRoot(), paths.remote, timeout) + ? result.createdDirectories + : result.failed; + target.push_back(std::move(entry.relKey)); + break; + } + result.uploadSources.push_back(paths.local.toUtf8()); + uploads.entries.push_back({ + .source = std::move(paths.local), + .destination = std::move(paths.remote), + .sizeBytes = entry.sizeBytes, + }); + result.uploads.push_back(std::move(entry.relKey)); + break; + } + case Action::Download: + { + if (entry.isDirectory) + { + std::error_code error{}; + std::filesystem::create_directories(paths.local.native(), error); + (error ? result.failed : result.createdDirectories).push_back(std::move(entry.relKey)); + break; + } + result.downloadSources.push_back(paths.remote.value()); + downloads.entries.push_back({ + .source = std::move(paths.remote), + .destination = std::move(paths.local), + .sizeBytes = entry.sizeBytes, + .mtime = entry.mtime, + .mtimeNsec = entry.mtimeNsec, + }); + result.downloads.push_back(std::move(entry.relKey)); + break; + } + case Action::DeleteRemote: + { + if (!refuseProtectedDeleteTarget(sftp, paths.remote).has_value()) + { + result.failed.push_back(std::move(entry.relKey)); + break; + } + result.remoteDeleteSources.push_back(paths.remote.value()); + remoteDeletes.entries.push_back({.target = std::move(paths.remote), .isDirectory = entry.isDirectory}); + result.remoteDeletes.push_back(std::move(entry.relKey)); + break; + } + case Action::DeleteLocal: + { + if (auto allowed = refuseProtectedLocalDelete(paths.local, localHome); !allowed) + { + Log::error("Sync: refusing to delete locally: {}", allowed.error()); + result.failed.push_back(std::move(entry.relKey)); + break; + } + result.localDeleteSources.push_back(paths.local.toUtf8()); + localDeletes.push_back(std::move(paths.local)); + result.localDeletes.push_back(std::move(entry.relKey)); + break; + } + } + } + + if (!uploads.entries.empty()) + { + addBulkUploadOperation( + sftp, + uploads, + [](std::size_t) + { + return Ids::generateOperationId(); + }, + request.uploadId ); } + if (!downloads.entries.empty()) + { + addBulkDownloadOperation( + sftp, + downloads, + [](std::size_t) + { + return Ids::generateOperationId(); + }, + request.downloadId + ); + } + if (!remoteDeletes.entries.empty() && addBulkDeleteOperation(sftp, remoteDeletes, request.remoteDeleteId) == 0) + { + std::ranges::move(result.remoteDeletes, std::back_inserter(result.failed)); + result.remoteDeletes.clear(); + result.remoteDeleteSources.clear(); + } + if (!localDeletes.empty() && !addLocalDeleteOperation(request.localDeleteId, std::move(localDeletes), request.mode)) + { + std::ranges::move(result.localDeletes, std::back_inserter(result.failed)); + result.localDeletes.clear(); + result.localDeleteSources.clear(); + } - return request.entries.size(); + Log::info( + "Sync: queued {} uploads, {} downloads, {} remote and {} local deletes, created {} directories, {} failed", + result.uploads.size(), + result.downloads.size(), + result.remoteDeletes.size(), + result.localDeletes.size(), + result.createdDirectories.size(), + result.failed.size() + ); + return result; +} + +std::expected OperationQueue::addLocalDeleteOperation( + Ids::OperationId operationId, + std::vector targets, + SharedData::OperationMode mode +) +{ + // Assumed in strand + if (targets.empty()) + return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = "no targets"}); + const auto home = localHomeDirectory(); + for (auto const& target : targets) + { + if (auto allowed = refuseProtectedLocalDelete(target, home); !allowed) + { + Log::error("Refusing to delete locally: {}", allowed.error()); + return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = target.toUtf8()}); + } + } + + const auto commonParent = targets.front().parent().value_or(targets.front()); + auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; + targetQueue.emplace_back( + operationId, + std::make_unique(LocalDeleteOperation::LocalDeleteOperationOptions{ + .itemsRemovedProgress = + [weak = weak_from_this(), operationId, name = rpcName("onDeleteProgress")]( + std::string const& path, std::uint64_t itemsDeleted, std::uint64_t totalItems + ) + { + auto self = weak.lock(); + if (!self) + return; + self->hub_->callRemote( + name, + SharedData::BulkDeleteProgress{ + .operationId = operationId, + .currentFile = path, + .filesDeleted = itemsDeleted, + .totalFiles = totalItems, + } + ); + }, + .targets = std::move(targets), + }) + ); + + hub_->callRemote( + rpcName("onOperationAdded"), + SharedData::OperationAdded{ + .operationId = operationId, + .type = SharedData::OperationType::LocalDelete, + .mode = mode, + .localPath = commonParent, + } + ); + return {}; +} + +std::expected +OperationQueue::refuseProtectedDeleteTarget(SecureShell::SftpSession& sftp, Paths::RemotePath const& target) +{ + const auto refuse = [&target](std::string_view why) + { + Log::error("Refusing to delete '{}': {}", target.value(), why); + return std::unexpected(Operation::Error{.type = Operation::ErrorType::InvalidPath, .extraInfo = target.value()}); + }; + if (target.empty()) + return refuse("no path"); + if (target.isRoot()) + return refuse("it is the root directory"); + + if (!remoteHome_) + { + auto homeFuture = sftp.homeDirectory(); + if (homeFuture.wait_for(sftpOpts_.operationTimeout.value_or(defaultFutureTimeout)) != std::future_status::ready) + return refuse("the home directory could not be determined in time"); + auto home = homeFuture.get(); + if (!home) + return refuse("the home directory could not be determined"); + remoteHome_ = *std::move(home); + } + if (remoteHome_->isWithin(target)) + return refuse(fmt::format("it contains the home directory '{}'", remoteHome_->value())); + return {}; } std::expected OperationQueue::addDeleteOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::filesystem::path const& remotePath, + Paths::RemotePath const& remotePath, bool recursive, bool insertRefresh, SharedData::OperationMode mode @@ -1564,42 +1784,20 @@ std::expected OperationQueue::addDeleteOperation( // Assumed in strand auto& targetQueue = (mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; - auto statFut = sftp.stat(remotePath); + if (auto allowed = refuseProtectedDeleteTarget(sftp, remotePath); !allowed) + return allowed; + + // lstat: a symlink to a directory is removed as a link, recursing would delete the target's content. + auto statFut = sftp.lstat(remotePath); if (statFut.wait_for(sftpOpts_.operationTimeout.value_or(defaultFutureTimeout)) != std::future_status::ready) { - Log::error("addDeleteOperation: stat timed out for '{}'", remotePath.string()); + Log::error("addDeleteOperation: lstat timed out for '{}'", remotePath.value()); return std::unexpected(Operation::Error{.type = Operation::ErrorType::FutureTimeout}); } const auto statResult = statFut.get(); - const bool isDirectory = statResult.has_value() && statResult->isDirectoryLike(); + const bool isDirectory = statResult.has_value() && statResult->isDirectory(); const auto isBulk = recursive && isDirectory; - if (isBulk) - { - const auto scanId = Ids::generateOperationId(); - - auto scan = std::make_unique( - sftp, - ScanOperation::ScanOperationOptions{ - .progressCallback = makeScanProgressCallback("onScanProgress", scanId), - .remotePath = remotePath, - .futureTimeout = std::chrono::seconds{5}, - } - ); - - targetQueue.emplace_back(scanId, std::move(scan)); - - hub_->callRemote( - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = scanId, - .type = SharedData::OperationType::Scan, - .mode = mode, - .remotePath = remotePath, - } - ); - } - auto operation = std::make_unique( sftp, DeleteOperation::DeleteOperationOptions{ @@ -1623,9 +1821,23 @@ std::expected OperationQueue::addDeleteOperation( }, .remotePath = remotePath, .futureTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout), - .recursive = recursive && isDirectory, + .recursive = isBulk, } ); + if (isBulk) + { + // The scan is part of the delete, so cancelling or moving the card acts on both. + operation->scanFirst( + std::make_unique( + sftp, + ScanOperation::ScanOperationOptions{ + .progressCallback = makeScanProgressCallback("onScanProgress", operationId), + .remotePath = remotePath, + .futureTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout), + } + ) + ); + } targetQueue.emplace_back(operationId, std::move(operation)); @@ -1637,6 +1849,7 @@ std::expected OperationQueue::addDeleteOperation( .mode = mode, .insertRefresh = insertRefresh, .remotePath = remotePath, + .scanPhase = isBulk, } ); @@ -1645,134 +1858,113 @@ std::expected OperationQueue::addDeleteOperation( std::size_t OperationQueue::addBulkDeleteOperation( SecureShell::SftpSession& sftp, - SharedData::BulkAddRequest const& request, + SharedData::BulkDeleteRequest const& request, Ids::OperationId bulkOperationId ) { - // Assumed in strand. Strategy: build a single DeleteOperation with - // pre-filled entries for all *file* paths in the request — that gives - // one bulk-delete card on the frontend driven by the existing - // BulkDeleteProgress emit path. Each *directory* entry still needs the - // recursive scan-then-delete flow so descendants are removed; those - // become standard single-Delete cards via addDeleteOperation. - + // Assumed in strand. One DeleteOperation, so one card and one completion for the whole request: files are + // deleted as given, every real directory is scanned first and deleted with its content. auto& targetQueue = (request.mode == SharedData::OperationMode::PriorityQueued) ? priorityOperations_ : operations_; const auto resolvedTimeout = sftpOpts_.operationTimeout.value_or(defaultFutureTimeout); - std::vector fileEntries; - fileEntries.reserve(request.entries.size()); - std::size_t enqueuedDirs = 0; + auto operation = std::make_unique( + sftp, + DeleteOperation::DeleteOperationOptions{ + .filesRemovedProgress = + [weak = weak_from_this(), bulkOperationId, name = rpcName("onDeleteProgress")]( + auto const& path, std::uint64_t filesDeleted, std::uint64_t totalFiles + ) + { + auto self = weak.lock(); + if (!self) + return; + self->hub_->callRemote( + name, + SharedData::BulkDeleteProgress{ + .operationId = bulkOperationId, + .currentFile = path, + .filesDeleted = filesDeleted, + .totalFiles = totalFiles, + } + ); + }, + .futureTimeout = resolvedTimeout, + .recursive = true, + } + ); + std::vector fileTargets; + std::size_t scannedDirectories = 0; for (auto const& entry : request.entries) { - if (!entry.isDirectory) - { - SharedData::DirectoryEntry de{}; - de.path = entry.src; - de.fullPath = entry.src; - de.type = SharedData::FileType::Regular; - de.size = entry.sizeBytes; - fileEntries.push_back(std::move(de)); + if (!refuseProtectedDeleteTarget(sftp, entry.target).has_value()) continue; - } - // Directory entry → existing single-add path which sets up the - // Scan + Delete pair. These render as standard Scan / Delete - // cards (separate from the bulk-files card below). - const auto dirOpId = Ids::generateOperationId(); - const auto result = addDeleteOperation( - sftp, - dirOpId, - entry.src, - /*recursive*/ true, - /*insertRefresh*/ false, - request.mode - ); - if (!result.has_value()) + if (entry.isDirectory && !request.listedContentOnly) { - Log::error( - "Bulk delete: directory entry '{}' failed to queue: {}", - entry.src.generic_string(), - result.error().toString() - ); - continue; + // lstat: a symlink to a directory is removed as a link, recursing would delete the target's content. + auto statFuture = sftp.lstat(entry.target); + if (statFuture.wait_for(resolvedTimeout) != std::future_status::ready) + { + Log::error("Bulk delete: lstat timed out for '{}'", entry.target.value()); + continue; + } + const auto information = statFuture.get(); + if (information && information->isDirectory()) + { + operation->addScanRoot( + std::make_unique( + sftp, + ScanOperation::ScanOperationOptions{ + .progressCallback = makeScanProgressCallback("onScanProgress", bulkOperationId), + .remotePath = entry.target, + .futureTimeout = resolvedTimeout, + } + ) + ); + ++scannedDirectories; + continue; + } } - ++enqueuedDirs; + fileTargets.push_back(entry.target); } - if (!fileEntries.empty()) - { - const auto fileCount = fileEntries.size(); - // Parent of the batch, surfaced on the card so it can read "Deleting - // items in " before any per-file progress arrives. Captured before - // fileEntries is moved into the operation below. - const auto commonParent = fileEntries.front().path.parent_path(); - // recursive=true so DeleteOperation walks the prefilled entries_ - // (its non-recursive branch ignores entries_ and only deletes the - // top-level remotePath). remotePath is left empty; not used in - // recursive mode. - auto operation = std::make_unique( - sftp, - DeleteOperation::DeleteOperationOptions{ - .filesRemovedProgress = - [weak = weak_from_this(), bulkOperationId, name = rpcName("onDeleteProgress")]( - auto const& path, std::uint64_t filesDeleted, std::uint64_t totalFiles - ) - { - auto self = weak.lock(); - if (!self) - return; - self->hub_->callRemote( - name, - SharedData::BulkDeleteProgress{ - .operationId = bulkOperationId, - .currentFile = path, - .filesDeleted = filesDeleted, - .totalFiles = totalFiles, - } - ); - }, - .remotePath = {}, - .futureTimeout = resolvedTimeout, - .recursive = true, - } - ); - operation->setScanResult(std::move(fileEntries), 0); - targetQueue.emplace_back(bulkOperationId, std::move(operation)); + const auto queued = fileTargets.size() + scannedDirectories; + if (queued == 0) + return 0; - hub_->callRemote( - rpcName("onOperationAdded"), - SharedData::OperationAdded{ - .operationId = bulkOperationId, - .type = SharedData::OperationType::BulkDelete, - .mode = request.mode, - .insertRefresh = request.insertRefresh, - .totalBytes = static_cast(fileCount), - .remotePath = commonParent, - } - ); - } + // Parent of the first item, so the card can read "Deleting items in " before any progress arrives. + const auto commonParent = request.entries.front().target.parent().value_or(Paths::RemotePath::root()); + const auto fileCount = fileTargets.size(); + operation->setTargets(std::move(fileTargets)); + targetQueue.emplace_back(bulkOperationId, std::move(operation)); - Log::info( - "Bulk delete: {} file entries (one card) + {} directory entries", - request.entries.size() - enqueuedDirs, - enqueuedDirs + hub_->callRemote( + rpcName("onOperationAdded"), + SharedData::OperationAdded{ + .operationId = bulkOperationId, + .type = SharedData::OperationType::BulkDelete, + .mode = request.mode, + .insertRefresh = request.insertRefresh, + .totalBytes = static_cast(fileCount), + .remotePath = commonParent, + .scanPhase = scannedDirectories > 0, + } ); + Log::info("Bulk delete: {} files and {} scanned directories in one operation", fileCount, scannedDirectories); if (!request.entries.empty()) { - bulkResumeStash_.insert_or_assign( - bulkOperationId, BulkResumeEntry{.kind = BulkResumeEntry::Kind::BulkDelete, .request = request} - ); + bulkResumeStash_.insert_or_assign(bulkOperationId, BulkResumeEntry{.request = request}); } - return request.entries.size(); + return queued; } std::expected OperationQueue::addRenameOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, - std::filesystem::path const& sourcePath, - std::filesystem::path const& destinationPath, + Paths::RemotePath const& sourcePath, + Paths::RemotePath const& destinationPath, SharedData::OperationMode mode ) { @@ -1796,8 +1988,8 @@ std::expected OperationQueue::addRenameOperation( .operationId = operationId, .type = SharedData::OperationType::Rename, .mode = mode, - .localPath = sourcePath, - .remotePath = destinationPath, + .source = sourcePath, + .destination = destinationPath, } ); @@ -1809,8 +2001,8 @@ void OperationQueue::addSyncScanOperation( Ids::SyncSessionId syncSessionId, Ids::OperationId remoteScanId, Ids::OperationId localScanId, - std::filesystem::path const& remotePath, - std::filesystem::path const& localPath, + Paths::RemotePath const& remotePath, + Paths::LocalPath const& localPath, bool respectIgnoreFiles, bool recursive, bool ignoreHidden @@ -1831,15 +2023,24 @@ void OperationQueue::addSyncScanOperation( // Scan completion callbacks hand the ScanNode trees into the session on its // own strand, then emit onSyncScanPhaseDone to the frontend. syncScanCallbacks_[remoteScanId.value()] = - [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}](SharedData::Sync::ScanNode tree) + [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}]( + std::expected tree + ) { auto self = weak.lock(); auto sess = weakSession.lock(); if (!self || !sess) return; + if (!tree) + { + self->hub_->callRemote( + self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/false, tree.error() + ); + return; + } boost::asio::post( sess->strand(), - [sess, tree = std::move(tree)]() mutable + [sess, tree = std::move(*tree)]() mutable { sess->setRemoteTreeInStrand(std::move(tree)); } @@ -1848,15 +2049,24 @@ void OperationQueue::addSyncScanOperation( }; syncScanCallbacks_[localScanId.value()] = - [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}](SharedData::Sync::ScanNode tree) + [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}]( + std::expected tree + ) { auto self = weak.lock(); auto sess = weakSession.lock(); if (!self || !sess) return; + if (!tree) + { + self->hub_->callRemote( + self->rpcName("onSyncScanFailed"), syncSessionId, /*isLocal=*/true, tree.error() + ); + return; + } boost::asio::post( sess->strand(), - [sess, tree = std::move(tree)]() mutable + [sess, tree = std::move(*tree)]() mutable { sess->setLocalTreeInStrand(std::move(tree)); } @@ -1865,6 +2075,16 @@ void OperationQueue::addSyncScanOperation( }; // Queue remote scan — build-tree mode. + // Temp files of transfers on either side, a sync neither copies nor deletes them. + std::vector temporarySuffixes{ + Persistence::effectiveTempFileSuffix( + sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}).tempFileSuffix + ), + Persistence::effectiveTempFileSuffix( + sftpOpts_.downloadOptions.value_or(Persistence::DownloadOptions{}).tempFileSuffix + ), + }; + auto remoteScan = std::make_unique( sftp, ScanOperation::ScanOperationOptions{ @@ -1875,6 +2095,7 @@ void OperationQueue::addSyncScanOperation( .recursive = recursive, .ignoreHidden = ignoreHidden, .buildTree = true, + .excludedSuffixes = temporarySuffixes, } ); priorityOperations_.emplace_back(remoteScanId, std::move(remoteScan)); @@ -1897,6 +2118,7 @@ void OperationQueue::addSyncScanOperation( .recursive = recursive, .ignoreHidden = ignoreHidden, .buildTree = true, + .excludedSuffixes = std::move(temporarySuffixes), }); priorityOperations_.emplace_back(localScanId, std::move(localScan)); diff --git a/backend/source/backend/sftp/rename_operation.cpp b/backend/source/backend/sftp/rename_operation.cpp index 2ea6d42e..8fac293a 100644 --- a/backend/source/backend/sftp/rename_operation.cpp +++ b/backend/source/backend/sftp/rename_operation.cpp @@ -30,7 +30,13 @@ std::expected RenameOperati } case (Running): { - auto fut = sftp_->rename(sourcePath_, destinationPath_); + // A queued rename mirrors a local move and must not replace what is at the destination. + auto fut = sftp_->performPromise( + [this]() + { + return sftp_->renameNoReplaceInStrand(sourcePath_, destinationPath_); + } + ); if (fut.wait_for(futureTimeout_) != std::future_status::ready) return enterErrorState({.type = ErrorType::FutureTimeout}); @@ -39,7 +45,7 @@ std::expected RenameOperati return enterErrorState({ .type = ErrorType::RenameFailure, .sftpError = result.error(), - .extraInfo = sourcePath_.generic_string(), + .extraInfo = sourcePath_.value(), }); state_ = Completed; @@ -79,11 +85,7 @@ std::expected RenameOperation::cancel(bool adoptCa { if (adoptCancelState) { - Log::info( - "RenameOperation: Rename of '{}' to '{}' canceled.", - sourcePath_.generic_string(), - destinationPath_.generic_string() - ); + Log::info("RenameOperation: Rename of '{}' to '{}' canceled.", sourcePath_.value(), destinationPath_.value()); state_ = OperationState::Canceled; } return {}; diff --git a/backend/source/backend/sftp/resume_check.cpp b/backend/source/backend/sftp/resume_check.cpp new file mode 100644 index 00000000..eef6100e --- /dev/null +++ b/backend/source/backend/sftp/resume_check.cpp @@ -0,0 +1,36 @@ +#include + +#include +#include +#include + +namespace ResumeCheck +{ + std::expected + tailsMatchInStrand(SecureShell::IFileStream& remote, std::filesystem::path const& local, std::uint64_t size) + { + const auto count = std::min(size, comparedBytes); + const auto offset = size - count; + + std::string localTail(static_cast(count), '\0'); + std::ifstream localFile{local, std::ios::binary}; + if (!localFile.seekg(static_cast(offset)) || + !localFile.read(localTail.data(), static_cast(count))) + return false; + + if (auto seeked = remote.seekInStrand(static_cast(offset)); !seeked) + return std::unexpected(seeked.error()); + std::string remoteTail(static_cast(count), '\0'); + std::size_t filled = 0; + while (filled < remoteTail.size()) + { + const auto read = remote.readSomeInStrand(remoteTail.data() + filled, remoteTail.size() - filled); + if (!read) + return std::unexpected(read.error()); + if (*read == 0) + return false; + filled += *read; + } + return localTail == remoteTail; + } +} diff --git a/backend/source/backend/sftp/scan_operation.cpp b/backend/source/backend/sftp/scan_operation.cpp index b5cc675d..7dc3c004 100644 --- a/backend/source/backend/sftp/scan_operation.cpp +++ b/backend/source/backend/sftp/scan_operation.cpp @@ -16,6 +16,7 @@ ScanOperation::ScanOperation(SecureShell::SftpSession& sftp, ScanOperationOption , recursive_{options.recursive} , ignoreHidden_{options.ignoreHidden} , buildTree_{options.buildTree} + , excludedSuffixes_{std::move(options.excludedSuffixes)} {} ScanOperation::~ScanOperation() = default; @@ -23,9 +24,10 @@ ScanOperation::~ScanOperation() = default; namespace { std::optional - readRemoteFile(SecureShell::SftpSession& sftp, std::filesystem::path const& path, std::chrono::seconds timeout) + readRemoteFile(SecureShell::SftpSession& sftp, Paths::RemotePath const& path, std::chrono::seconds timeout) { - auto openFut = sftp.openFile(path, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + auto openFut = + sftp.openFile(path, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::owner_read); if (openFut.wait_for(timeout) != std::future_status::ready) return std::nullopt; auto openResult = openFut.get(); @@ -54,12 +56,13 @@ namespace } std::expected, ScanOperation::Error> -ScanOperation::scanner(std::filesystem::path const& path) +ScanOperation::scanner(Paths::RelativePath const& relative) { if (!recursive_ && rootScanned_) return std::vector{}; rootScanned_ = true; + const auto path = remotePath_ / relative; auto fut = sftp_->listDirectory(path); fut.wait_for(futureTimeout_); if (fut.wait_for(futureTimeout_) != std::future_status::ready) @@ -67,9 +70,22 @@ ScanOperation::scanner(std::filesystem::path const& path) auto result = fut.get(); if (!result.has_value()) + { + // A directory below the root that vanished or denies listing has unknown content, a sync tree + // keeps it as unreadable instead of failing or treating it as empty. + const auto sftpError = result.error().sftpError; + const bool deniedOrVanished = sftpError == SSH_FX_PERMISSION_DENIED || sftpError == SSH_FX_NO_SUCH_FILE; + if (buildTree_ && !relative.empty() && deniedOrVanished) + { + Log::warn("ScanOperation: Cannot list '{}': {}", path.value(), result.error().toString()); + return std::unexpected( + Error{.type = ErrorType::DirectoryUnreadable, .sftpError = result.error(), .extraInfo = path.value()} + ); + } return enterErrorState>( {.type = ErrorType::SftpError, .sftpError = result.error()} ); + } auto entries = std::move(result).value(); @@ -77,8 +93,7 @@ ScanOperation::scanner(std::filesystem::path const& path) { entries.erase( std::remove_if(entries.begin(), entries.end(), [](SharedData::DirectoryEntry const& entry) { - const auto name = entry.path.filename().string(); - return !name.empty() && name.front() == '.'; + return !entry.name.empty() && entry.name.front() == '.'; }), entries.end() ); @@ -87,23 +102,15 @@ ScanOperation::scanner(std::filesystem::path const& path) if (!respectIgnoreFiles_) return {std::move(entries)}; - const auto relDir = [&]() -> std::filesystem::path { - std::error_code errc{}; - auto rel = std::filesystem::relative(path, remotePath_, errc); - if (errc || rel.empty() || rel == std::filesystem::path{"."}) - return {}; - return rel; - }(); - for (auto const& entry : entries) { - const auto name = entry.path.generic_string(); + auto const& name = entry.name; if (name != ".gitignore" && name != ".ignore") continue; if (entry.type != SharedData::DirectoryEntry::FileType::Regular) continue; - if (auto content = readRemoteFile(*sftp_, path / entry.path, futureTimeout_)) - ignoreMatcher_.addFile(relDir, *content); + if (auto content = readRemoteFile(*sftp_, path / *Paths::PathSegment::make(name), futureTimeout_)) + ignoreMatcher_.addFile(relative, *content); } if (ignoreMatcher_.empty()) @@ -113,10 +120,10 @@ ScanOperation::scanner(std::filesystem::path const& path) filtered.reserve(entries.size()); for (auto& entry : entries) { - const auto childRel = - relDir.empty() ? std::filesystem::path{entry.path} : (relDir / entry.path); + // A name that is no segment cannot match a rule, the walker rejects it. + const auto segment = Paths::PathSegment::make(entry.name); const bool isDir = entry.type == SharedData::DirectoryEntry::FileType::Directory; - if (ignoreMatcher_.isIgnored(childRel, isDir)) + if (segment && ignoreMatcher_.isIgnored(relative / *segment, isDir)) continue; filtered.push_back(std::move(entry)); } @@ -136,8 +143,15 @@ std::expected ScanOperation::wo { case (NotStarted): { + if (remotePath_.empty()) + { + Log::error("ScanOperation: No root to scan."); + return enterErrorState( + {.type = ErrorType::ImplementationError, .extraInfo = "The scan has no root."} + ); + } state_ = Running; - Log::info("ScanOperation: Starting scan of '{}'.", remotePath_.generic_string()); + Log::info("ScanOperation: Starting scan of '{}'.", remotePath_.value()); progressCallback_(0, 0, 0); return WorkStatus::MoreWork; } @@ -150,7 +164,7 @@ std::expected ScanOperation::wo { if (walker.completed()) { - Log::info("ScanOperation: Scan of '{}' completed.", remotePath_.generic_string()); + Log::info("ScanOperation: Scan of '{}' completed.", remotePath_.value()); state_ = Completed; return WorkStatus::Complete; } @@ -204,7 +218,7 @@ std::expected ScanOperation::cancel(bool adoptCancel { if (adoptCancelState) { - Log::info("ScanOperation: Scan of '{}' canceled.", remotePath_.generic_string()); + Log::info("ScanOperation: Scan of '{}' canceled.", remotePath_.value()); enterState(OperationState::Canceled); } return {}; diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index acbd2d6e..c6fc4033 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -1,8 +1,11 @@ #include +#include +#include #include #include #include +#include #include #include @@ -16,6 +19,10 @@ UploadOperation::UploadOperation(SecureShell::SftpSession& sftp, UploadOperation , localFile_{} { options_.tempFileSuffix = Persistence::effectiveTempFileSuffix(options_.tempFileSuffix); + remotePath_ = options_.remotePath; + if (!options_.localPath.empty()) + localNativePath_ = options_.localPath.native(); + tempPath_ = remotePath_.withFilenameSuffix(options_.tempFileSuffix).value_or(Paths::RemotePath{}); } UploadOperation::~UploadOperation() @@ -76,12 +83,11 @@ std::expected UploadOperati Log::error("UploadOperation: Invalid local path."); return enterErrorState({.type = ErrorType::InvalidPath}); } - const auto localStatus = std::filesystem::symlink_status(options_.localPath); + const auto localStatus = std::filesystem::symlink_status(localNativePath_); if (!std::filesystem::is_regular_file(localStatus) && !std::filesystem::is_symlink(localStatus)) { Log::error( - "UploadOperation: Local path '{}' is not a regular file or symlink.", - options_.localPath.generic_string() + "UploadOperation: Local path '{}' is not a regular file or symlink.", options_.localPath.toUtf8() ); return enterErrorState({.type = ErrorType::NotAFile}); } @@ -92,23 +98,23 @@ std::expected UploadOperati } else { - if (!std::filesystem::is_regular_file(std::filesystem::status(options_.localPath))) + if (!std::filesystem::is_regular_file(std::filesystem::status(localNativePath_))) { Log::error( "UploadOperation: Local path '{}' is a directory, but a file was expected.", - options_.localPath.generic_string() + options_.localPath.toUtf8() ); return enterErrorState({.type = ErrorType::NotAFile}); } - if (!std::filesystem::exists(options_.localPath)) + if (!std::filesystem::exists(localNativePath_)) { - Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.generic_string()); + Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); return enterErrorState({.type = ErrorType::FileNotFound}); } - localFile_.open(options_.localPath, std::ios::binary); + localFile_.open(localNativePath_, std::ios::binary); if (!localFile_.is_open()) { - Log::error("UploadOperation: Failed to open local file: {}", options_.localPath.generic_string()); + Log::error("UploadOperation: Failed to open local file: {}", options_.localPath.toUtf8()); return enterErrorState({.type = ErrorType::OpenFailure}); } localFile_.seekg(0, std::ios::end); @@ -117,9 +123,7 @@ std::expected UploadOperati localFile_.seekg(0, std::ios::beg); if (!localFile_.good()) { - Log::error( - "UploadOperation: Failed to seek in local file: {}", options_.localPath.generic_string() - ); + Log::error("UploadOperation: Failed to seek in local file: {}", options_.localPath.toUtf8()); return enterErrorState({.type = ErrorType::FileSeekFailure}); } const auto prepareResult = prepareInStrand(); @@ -158,7 +162,7 @@ std::expected UploadOperati return enterErrorState({.type = ErrorType::FileStreamExpired}); } const auto contextResult = stream->writeAsyncInStrand( - totalSize_, + static_cast(leftToUpload_), buffer_.data(), buffer_.size(), [this](SecureShell::IFileStream::SignedSizeType bytesRead) @@ -202,6 +206,9 @@ std::expected UploadOperati { if (asyncTransferContext_->hasEnded()) { + if (asyncTransferContext_->outcome() != SecureShell::AsyncTransferContext::Outcome::Completed) + return enterErrorState(asyncTransferError(*asyncTransferContext_)); + leftToUpload_ = 0; options_.progressCallback(0ull, totalSize_, totalSize(), asyncTransferContext_->bytesPerSecond()); state_ = Finalizing; [[fallthrough]]; @@ -211,7 +218,8 @@ std::expected UploadOperati options_.progressCallback( 0ull, totalSize_, - asyncTransferContext_->bytesTransferred(), + totalSize_ - leftToUpload_ + + static_cast(asyncTransferContext_->bytesTransferred()), asyncTransferContext_->bytesPerSecond() ); return WorkStatus::MoreWork; @@ -297,48 +305,45 @@ std::expected UploadOperation::handleSymlinkInStra { if (options_.symlinkHandling == Persistence::SymlinkHandling::SkipSymlink) { - Log::info("UploadOperation: Skipping symlink as per options: {}.", options_.localPath.string()); + Log::info("UploadOperation: Skipping symlink as per options: {}.", options_.localPath.toUtf8()); return {}; } - const auto target = std::filesystem::read_symlink(options_.localPath); - - // sftp_symlink fails if the link path already exists. If we're allowed to overwrite, - // remove whatever is there first (file or link — directories are left alone since - // that almost certainly isn't what the user wants). - if (options_.mayOverwrite) - { - const auto statResult = sftp_->lstatInStrand(options_.remotePath); - if (statResult.has_value() && statResult->type != SharedData::FileType::Directory) - { - const auto rmResult = sftp_->removeFileInStrand(options_.remotePath); - if (!rmResult.has_value()) - { - Log::warn( - "UploadOperation: Could not remove existing '{}' before symlink: {}.", - options_.remotePath.generic_string(), - rmResult.error().message - ); - } - } - } + const auto target = std::filesystem::read_symlink(localNativePath_); if (options_.createMissingDirectories) { - if (auto res = ensureRemoteDirectoryExistsInStrand(options_.remotePath.parent_path()); !res.has_value()) + if (auto res = ensureRemoteDirectoryExistsInStrand(*remotePath_.parent()); !res.has_value()) return std::unexpected(res.error()); } - const auto result = sftp_->createSymLinkInStrand(target, options_.remotePath); + // The link is created under the temp name and renamed into place, so an existing destination is + // never removed before its replacement exists. + std::ignore = sftp_->removeFileInStrand(tempPath_); + const auto result = sftp_->createSymLinkInStrand(Paths::nativeToUtf8(target), tempPath_); if (!result.has_value()) { Log::error( - "UploadOperation: Failed to create symlink at '{}': {}.", - options_.remotePath.generic_string(), - result.error().message + "UploadOperation: Failed to create symlink at '{}': {}.", tempPath_.value(), result.error().message ); return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = result.error()}); } + + const auto renameResult = options_.mayOverwrite + ? sftp_->renameReplaceInStrand(tempPath_, remotePath_, options_.tempFileSuffix + ".old") + : sftp_->renameNoReplaceInStrand(tempPath_, remotePath_); + if (!renameResult.has_value()) + { + std::ignore = sftp_->removeFileInStrand(tempPath_); + if (renameResult.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) + return std::unexpected(Error{.type = ErrorType::FileExists}); + Log::error( + "UploadOperation: Failed to move symlink to '{}': {}.", + options_.remotePath.value(), + renameResult.error().message + ); + return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = renameResult.error()}); + } return {}; } @@ -386,6 +391,11 @@ std::expected UploadOperation::writeOnceInStrand() Log::error("UploadOperation: Read more data than expected."); return enterErrorState({.type = ErrorType::ImplementationError}); } + if (readCount == 0 && leftToUpload_ > 0) + { + Log::error("UploadOperation: The local file ended before its expected size was read."); + return enterErrorState({.type = ErrorType::SourceFileNotGood}); + } leftToUpload_ -= readCount; const auto result = stream->writeInStrand(std::string_view{buffer_.data(), readCount}); @@ -402,7 +412,7 @@ std::expected UploadOperation::writeOnceInStrand() std::expected UploadOperation::openOrAdoptFileInStrand() { - const auto result = sftp_->lstatInStrand(options_.remotePath); + const auto result = sftp_->lstatInStrand(remotePath_); if (!result.has_value()) { const auto err = result.error(); @@ -419,19 +429,18 @@ std::expected UploadOperation::openOrAdoptFileInSt { Log::warn( "UploadOperation: Remote file '{}' already exists and may not be overwritten.", - options_.remotePath.generic_string() + options_.remotePath.value() ); return std::unexpected(Error{.type = ErrorType::FileExists}); } } - auto tempPath = options_.remotePath; - tempPath += options_.tempFileSuffix; - const auto tempResult = sftp_->lstatInStrand(tempPath); + const auto tempResult = sftp_->lstatInStrand(tempPath_); - // local file perms - const auto status = std::filesystem::symlink_status(options_.localPath); - const auto perms = determinePerms(status.permissions()); + // Permissions of what is uploaded: the target of a followed symlink, not the link. + std::error_code statusError{}; + const auto status = std::filesystem::status(localNativePath_, statusError); + const auto perms = determinePerms(statusError ? std::filesystem::perms::unknown : Utility::localPermissions(status)); // Adoption mechanic for temp file parts: if (tempResult.has_value()) @@ -441,11 +450,17 @@ std::expected UploadOperation::openOrAdoptFileInSt Log::debug("UploadOperation: Remote temp file is larger than local file, do not adopt file."); // Do not adopt file } + else if (options_.tryContinue && !tempBelongsToSourceInStrand(tempResult->size)) + { + Log::info( + "UploadOperation: Remote temp file '{}' is not a part of this source, starting over.", tempPath_.value() + ); + } else if (options_.tryContinue) { Log::debug("UploadOperation: Continuing upload to existing temp file."); - const auto openResult = sftp_->openFileInStrand(tempPath, SecureShell::SftpSession::OpenType::Write, perms); + const auto openResult = sftp_->openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Write, perms); if (!openResult.has_value()) { Log::error("Failed to open remote sftp file for continue: {}", openResult.error().message); @@ -476,14 +491,14 @@ std::expected UploadOperation::openOrAdoptFileInSt if (options_.createMissingDirectories) { - if (auto res = ensureRemoteDirectoryExistsInStrand(tempPath.parent_path()); !res.has_value()) + if (auto res = ensureRemoteDirectoryExistsInStrand(*tempPath_.parent()); !res.has_value()) return std::unexpected(res.error()); } // Open temp file part regularly, freshly: - Log::debug("UploadOperation: Starting new upload to '{}'.", tempPath.generic_string()); + Log::debug("UploadOperation: Starting new upload to '{}'.", tempPath_.value()); const auto openResult = sftp_->openFileInStrand( - tempPath, + tempPath_, SecureShell::SftpSession::OpenType::Write | SecureShell::SftpSession::OpenType::Create | SecureShell::SftpSession::OpenType::Truncate, perms @@ -499,10 +514,27 @@ std::expected UploadOperation::openOrAdoptFileInSt return {}; } +bool UploadOperation::tempBelongsToSourceInStrand(std::uint64_t tempSize) +{ + // Another program's partial file or a part of an older version of the source must not be continued. + if (tempSize == 0) + return false; + const auto opened = + sftp_->openFileInStrand(tempPath_, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::none); + if (!opened) + return false; + auto stream = opened->lock(); + if (!stream) + return false; + const auto matches = ResumeCheck::tailsMatchInStrand(*stream, localNativePath_, tempSize); + stream->closeInStrand(); + return matches.value_or(false); +} + std::expected -UploadOperation::ensureRemoteDirectoryExistsInStrand(std::filesystem::path const& dir) +UploadOperation::ensureRemoteDirectoryExistsInStrand(Paths::RemotePath const& dir) { - if (dir.empty() || dir == dir.root_path() || dir == std::filesystem::path{"/"}) + if (dir.isRoot()) return {}; const auto statResult = sftp_->lstatInStrand(dir); @@ -511,7 +543,7 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(std::filesystem::path const if (statResult->type == SharedData::FileType::Directory) return {}; Log::error( - "UploadOperation: Cannot create parent '{}': path exists and is not a directory.", dir.generic_string() + "UploadOperation: Cannot create parent '{}': path exists and is not a directory.", dir.value() ); return std::unexpected( Error{ @@ -526,7 +558,8 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(std::filesystem::path const if (statResult.error().sftpError != SSH_FX_NO_SUCH_FILE) return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = statResult.error()}); - if (auto parent = ensureRemoteDirectoryExistsInStrand(dir.parent_path()); !parent.has_value()) + if (auto parent = ensureRemoteDirectoryExistsInStrand(dir.parent().value_or(Paths::RemotePath::root())); + !parent.has_value()) return parent; const auto dirPerms = options_.directoryPermissions.value_or( @@ -537,7 +570,7 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(std::filesystem::path const if (!mkdirResult.has_value()) { Log::error( - "UploadOperation: Failed to create parent dir '{}': {}.", dir.generic_string(), mkdirResult.error().message + "UploadOperation: Failed to create parent dir '{}': {}.", dir.value(), mkdirResult.error().message ); return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = mkdirResult.error()}); } @@ -551,12 +584,15 @@ std::expected UploadOperation::prepare() Log::error("UploadOperation: Invalid local path."); return enterErrorState({.type = ErrorType::InvalidPath}); } - const auto localStatus = std::filesystem::symlink_status(options_.localPath); + if (remotePath_.empty() || remotePath_.isRoot() || tempPath_.empty()) + { + Log::error("UploadOperation: Invalid remote path '{}'.", options_.remotePath.value()); + return enterErrorState({.type = ErrorType::InvalidPath, .extraInfo = options_.remotePath.value()}); + } + const auto localStatus = std::filesystem::symlink_status(localNativePath_); if (!std::filesystem::is_regular_file(localStatus) && !std::filesystem::is_symlink(localStatus)) { - Log::error( - "UploadOperation: Local path '{}' is not a regular file or symlink.", options_.localPath.generic_string() - ); + Log::error("UploadOperation: Local path '{}' is not a regular file or symlink.", options_.localPath.toUtf8()); return enterErrorState({.type = ErrorType::NotAFile}); } @@ -567,26 +603,25 @@ std::expected UploadOperation::prepare() return {}; } - if (!std::filesystem::is_regular_file(std::filesystem::status(options_.localPath))) + if (!std::filesystem::is_regular_file(std::filesystem::status(localNativePath_))) { Log::error( - "UploadOperation: Local path '{}' is a directory, but a file was expected.", - options_.localPath.generic_string() + "UploadOperation: Local path '{}' is a directory, but a file was expected.", options_.localPath.toUtf8() ); return enterErrorState({.type = ErrorType::NotAFile}); } // Initial check. Check again later before rename - if (!std::filesystem::exists(options_.localPath)) + if (!std::filesystem::exists(localNativePath_)) { - Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.generic_string()); + Log::error("UploadOperation: Local file '{}' does not exist.", options_.localPath.toUtf8()); return enterErrorState({.type = ErrorType::FileNotFound}); } - localFile_.open(options_.localPath, std::ios::binary); + localFile_.open(localNativePath_, std::ios::binary); if (!localFile_.is_open()) { - Log::error("UploadOperation: Failed to open local file: {}", options_.localPath.generic_string()); + Log::error("UploadOperation: Failed to open local file: {}", options_.localPath.toUtf8()); return enterErrorState({.type = ErrorType::OpenFailure}); } @@ -597,7 +632,7 @@ std::expected UploadOperation::prepare() if (!localFile_.good()) { - Log::error("UploadOperation: Failed to seek in local file: {}", options_.localPath.generic_string()); + Log::error("UploadOperation: Failed to seek in local file: {}", options_.localPath.toUtf8()); return enterErrorState({.type = ErrorType::FileSeekFailure}); } @@ -620,9 +655,7 @@ std::expected UploadOperation::prepare() } Log::debug( - "UploadOperation: Prepared upload of '{}' to '{}'.", - options_.localPath.generic_string(), - options_.remotePath.generic_string() + "UploadOperation: Prepared upload of '{}' to '{}'.", options_.localPath.toUtf8(), options_.remotePath.value() ); return {}; @@ -663,8 +696,8 @@ std::expected UploadOperation::cancel(bool adoptCa { Log::info( "UploadOperation: Upload of '{}' to '{}' canceled.", - options_.remotePath.generic_string(), - options_.localPath.generic_string() + options_.remotePath.value(), + options_.localPath.toUtf8() ); state_ = OperationState::Canceled; } @@ -712,9 +745,7 @@ std::expected UploadOperation::finalize() return result; Log::debug( - "UploadOperation: Finalized upload of '{}' to '{}'.", - options_.remotePath.generic_string(), - options_.localPath.generic_string() + "UploadOperation: Finalized upload of '{}' to '{}'.", options_.remotePath.value(), options_.localPath.toUtf8() ); return {}; } @@ -728,38 +759,37 @@ std::expected UploadOperation::finalizeInStrand() return std::unexpected(Error{.type = ErrorType::FileStreamExpired}); } - stream->closeInStrand(); - - // Recheck if the destination already exists. + // Every byte must have been written before the temp file may replace the destination. + if (leftToUpload_ != 0) { - const auto result = sftp_->statInStrand(options_.remotePath); - if (!result.has_value()) - { - // Interpret error as ok — either does not exist or something else is wrong; - // try to rename anyway. - } - else - { - if (!options_.mayOverwrite) - { - Log::warn( - "UploadOperation: Remote file '{}' already exists and may not be overwritten.", - options_.remotePath.generic_string() - ); - return std::unexpected(Error{.type = ErrorType::FileExists}); - } - else - { - std::ignore = sftp_->removeFileInStrand(options_.remotePath); + Log::error("UploadOperation: {} bytes were not uploaded.", leftToUpload_); + return std::unexpected( + Error{ + .type = ErrorType::SizeMismatch, + .extraInfo = fmt::format( + "Sent {} of {} bytes for {}", totalSize_ - leftToUpload_, totalSize_, options_.localPath.toUtf8() + ), } - } + ); } - auto tempPath = options_.remotePath; - tempPath += options_.tempFileSuffix; - const auto renameResult = sftp_->renameInStrand(tempPath, options_.remotePath); + stream->closeInStrand(); + + // The destination is never deleted ahead of the rename: replacing is atomic or moves it aside, and + // without permission to overwrite the rename fails instead of replacing. + const auto renameResult = options_.mayOverwrite + ? sftp_->renameReplaceInStrand(tempPath_, remotePath_, options_.tempFileSuffix + ".old") + : sftp_->renameNoReplaceInStrand(tempPath_, remotePath_); if (!renameResult.has_value()) { + if (renameResult.error().sftpError == SSH_FX_FILE_ALREADY_EXISTS) + { + Log::warn( + "UploadOperation: Remote file '{}' already exists and may not be overwritten.", + options_.remotePath.value() + ); + return std::unexpected(Error{.type = ErrorType::FileExists}); + } Log::error("Failed to rename remote sftp file: {}", renameResult.error().message); return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = renameResult.error()}); } @@ -768,7 +798,7 @@ std::expected UploadOperation::finalizeInStrand() // don't see the file as modified (best-effort; a failure is only a warning). { std::error_code errc{}; - const auto ftime = std::filesystem::last_write_time(options_.localPath, errc); + const auto ftime = std::filesystem::last_write_time(localNativePath_, errc); if (!errc) { const auto sctime = std::chrono::file_clock::to_sys(ftime); @@ -779,12 +809,12 @@ std::expected UploadOperation::finalizeInStrand() attrs.flags = SSH_FILEXFER_ATTR_ACMODTIME; attrs.atime = mtimeSecs; attrs.mtime = mtimeSecs; - const auto mtimeResult = sftp_->statInStrand(options_.remotePath, &attrs); + const auto mtimeResult = sftp_->statInStrand(remotePath_, &attrs); if (!mtimeResult.has_value()) { Log::warn( "UploadOperation: Failed to set mtime on '{}': {}.", - options_.remotePath.generic_string(), + options_.remotePath.value(), mtimeResult.error().message ); } @@ -792,9 +822,7 @@ std::expected UploadOperation::finalizeInStrand() else { Log::warn( - "UploadOperation: Could not read local mtime for '{}': {}.", - options_.localPath.generic_string(), - errc.message() + "UploadOperation: Could not read local mtime for '{}': {}.", options_.localPath.toUtf8(), errc.message() ); } } @@ -807,7 +835,10 @@ std::filesystem::perms UploadOperation::determinePerms(std::filesystem::perms lo { if (options_.inheritPermissions) { - return localPerms; + return Utility::sanitizePermissions(localPerms).value_or( + std::filesystem::perms::owner_read | std::filesystem::perms::group_read | + std::filesystem::perms::others_read + ); } else { diff --git a/backend/source/backend/sync/sync_session.cpp b/backend/source/backend/sync/sync_session.cpp index 1393fab6..083de7b9 100644 --- a/backend/source/backend/sync/sync_session.cpp +++ b/backend/source/backend/sync/sync_session.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -64,16 +65,32 @@ ScanTrees SyncSession::scanTrees() const return ScanTrees{ .local = localTree_ ? &*localTree_ : nullptr, .remote = remoteTree_ ? &*remoteTree_ : nullptr, - .localRoot = options_.localRoot, - .remoteRoot = options_.remoteRoot, }; } +std::expected +SyncSession::resolve(Paths::RelativePath const& relative) const +{ + return options_.localRoot.join(relative).transform( + [this, &relative](Paths::LocalPath local) + { + return ResolvedPaths{.local = std::move(local), .remote = options_.remoteRoot / relative}; + } + ); +} + DiffSummary SyncSession::recomputeDiffInStrand( - DiffOptions const& options, + DiffOptions const& requestedOptions, std::function onProgress ) { + auto options = requestedOptions; +#if defined(_WIN32) || defined(__APPLE__) + // NTFS and APFS fold case by default. + options.localCaseInsensitive = true; +#else + options.localCaseInsensitive = false; +#endif uploads_.clear(options); downloads_.clear(options); deletes_.clear(options); @@ -105,6 +122,8 @@ DiffSummary SyncSession::recomputeDiffInStrand( summary.deletes = deletes_.summary(); summary.entriesCompared = sink.lastCompared(); summary.cancelled = sink.cancelled(); + summary.blockedCount = sink.blockedCount(); + summary.blockedSamples = sink.blockedSamples(); return summary; } @@ -157,3 +176,26 @@ SyncSession::buildEnqueuePlanInStrand(DiffSection section, std::unordered_set const& selectedRelKeys) const +{ + ResolvedPlan result; + for (auto& entry : buildEnqueuePlanInStrand(section, selectedRelKeys)) + { + auto paths = Paths::RelativePath::make(entry.relKey).and_then( + [this](Paths::RelativePath const& relative) + { + return resolve(relative); + } + ); + if (!paths) + { + Log::warn("SyncSession: '{}' has no path on both sides: {}", entry.relKey, paths.error().toString()); + result.refused.push_back(std::move(entry.relKey)); + continue; + } + result.entries.push_back(ResolvedPlanEntry{.entry = std::move(entry), .paths = *std::move(paths)}); + } + return result; +} diff --git a/backend/source/backend/windows/main_windows.cpp b/backend/source/backend/windows/main_windows.cpp index 93e79ea8..8ce15627 100644 --- a/backend/source/backend/windows/main_windows.cpp +++ b/backend/source/backend/windows/main_windows.cpp @@ -1,5 +1,44 @@ #include +#include + +#include +#include + Main::PlatformSpecifics::PlatformSpecifics(Nui::Window& window, Nui::RpcHub& hub) : enableDragDrop{window, hub} -{} \ No newline at end of file +{} + +namespace +{ + /** + * @brief A standard handle that already leads somewhere: a file or pipe it was redirected to. + */ + bool isUsable(DWORD standardHandle) + { + const auto handle = GetStdHandle(standardHandle); + return handle != nullptr && handle != INVALID_HANDLE_VALUE && GetFileType(handle) != FILE_TYPE_UNKNOWN; + } +} + +void attachParentConsole() +{ + // Redirected streams keep their file or pipe, only the ones without a target go to the console. + const bool outputUsable = isUsable(STD_OUTPUT_HANDLE); + const bool errorUsable = isUsable(STD_ERROR_HANDLE); + const bool inputUsable = isUsable(STD_INPUT_HANDLE); + if (outputUsable && errorUsable && inputUsable) + return; + if (!AttachConsole(ATTACH_PARENT_PROCESS)) + return; + FILE* stream = nullptr; + if (!outputUsable) + freopen_s(&stream, "CONOUT$", "w", stdout); + if (!errorUsable) + freopen_s(&stream, "CONOUT$", "w", stderr); + if (!inputUsable) + freopen_s(&stream, "CONIN$", "r", stdin); + std::cout.clear(); + std::cerr.clear(); + std::cin.clear(); +} diff --git a/backend/source/backend/windows/opener_windows.cpp b/backend/source/backend/windows/opener_windows.cpp index 4470877d..bcfa3a71 100644 --- a/backend/source/backend/windows/opener_windows.cpp +++ b/backend/source/backend/windows/opener_windows.cpp @@ -47,13 +47,22 @@ namespace } } - std::expected checkExtensionPolicy(std::filesystem::path const& path) - { - auto const ext = path.extension().wstring(); - if (AssocIsDangerous(ext.c_str())) - return std::unexpected{"backend.opener.windows.blockedByPolicy"}; - return {}; - } +} + +std::optional Opener::riskOf(std::filesystem::path const& path) const +{ + std::error_code statusError{}; + if (std::filesystem::is_directory(path, statusError)) + return std::nullopt; + // Content based: a renamed program is still a program. + DWORD binaryType{}; + if (GetBinaryTypeW(path.c_str(), &binaryType)) + return std::string{"backend.opener.risk.program"}; + // The shell's own list of types that run code: scripts, installers, shortcuts and the like. + const auto extension = path.extension().wstring(); + if (!extension.empty() && AssocIsDangerous(extension.c_str())) + return std::string{"backend.opener.risk.dangerousType"}; + return std::nullopt; } struct Opener::Implementation @@ -74,17 +83,7 @@ std::expected Opener::openFile(std::filesystem::path const& p const auto pathStrU16 = path.native(); const auto pathWstr = std::wstring{pathStrU16.begin(), pathStrU16.end()}; - // Refuse to open binary executables (content-based, not extension-based) - DWORD binaryType{}; - if (GetBinaryTypeW(pathWstr.c_str(), &binaryType)) - return std::unexpected{"backend.opener.windows.binaryExecutable"}; - - // Windows attachment policy (respects system/zone policy) - // If I keep this, it prevents opening of folders: - - // if (auto result = checkExtensionPolicy(path); !result) - // return result; - + // Programs, scripts and shortcuts are judged by riskOf and the user's policy before this is called. if (openWith) { if (!std::filesystem::exists(path) || !path.is_absolute()) diff --git a/backend/source/backend/windows/windows_drag_drop.cpp b/backend/source/backend/windows/windows_drag_drop.cpp index 04db9dc2..2cbe6297 100644 --- a/backend/source/backend/windows/windows_drag_drop.cpp +++ b/backend/source/backend/windows/windows_drag_drop.cpp @@ -1,10 +1,10 @@ #include +#include #include #include #include -#include #include @@ -94,7 +94,7 @@ EnableWindowsDragDrop::EnableWindowsDragDrop(Nui::Window& wnd, Nui::RpcHub& hub) objectsCollection->get_Count(&length); // Array of file paths to be sent back to the webview as JSON - std::vector entries; + std::vector entries; for (unsigned int i = 0; i < length; i++) { winrt::com_ptr object{ @@ -123,9 +123,8 @@ EnableWindowsDragDrop::EnableWindowsDragDrop(Nui::Window& wnd, Nui::RpcHub& hub) // Add the file to message to be sent back to webview CoTaskMemString pathStr{nullptr}; file->get_Path(pathStr.recepticle()); - entries.emplace_back(leanDirectoryEntryFromPath( - Utility::pathToUtf8Generic(std::filesystem::path{pathStr.toStandardString()}) - )); + if (auto item = droppedItemFromPath(std::filesystem::path{pathStr.toStandardString()}); item) + entries.push_back(*std::move(item)); } } diff --git a/backend/test/backend/CMakeLists.txt b/backend/test/backend/CMakeLists.txt index 8d970b5e..b0ef58b1 100644 --- a/backend/test/backend/CMakeLists.txt +++ b/backend/test/backend/CMakeLists.txt @@ -32,7 +32,10 @@ target_compile_definitions(nui-sftp-backend-test PRIVATE NUI_SFTP_TEST_DOCKER_DI set_target_properties(nui-sftp-backend-test PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/test") -gtest_discover_tests(nui-sftp-backend-test) +# The docker shell suites prepare their images per process and start a container per test, so they run one at a +# time; everything else runs in parallel (ctest -j). +gtest_discover_tests(nui-sftp-backend-test TEST_FILTER "-Shells/*:Bash/*") +gtest_discover_tests(nui-sftp-backend-test TEST_FILTER "Shells/*:Bash/*" PROPERTIES RESOURCE_LOCK docker) # If msys2, copy dynamic libraries to executable directory, visual studio does this automatically. # And there is no need on linux. diff --git a/backend/test/backend/archive_test_helpers.hpp b/backend/test/backend/archive_test_helpers.hpp index 191b6282..352459dd 100644 --- a/backend/test/backend/archive_test_helpers.hpp +++ b/backend/test/backend/archive_test_helpers.hpp @@ -32,25 +32,20 @@ namespace Test std::string contents{}; }; - inline SharedData::DirectoryEntry - makeFileEntry(std::filesystem::path const& fullPath, std::uint64_t size) + inline SharedData::ArchiveDownloadRoot makeFileEntry(std::string_view path, std::uint64_t size) { - SharedData::DirectoryEntry entry{}; - entry.path = fullPath; - entry.fullPath = fullPath; - entry.type = SharedData::FileType::Regular; - entry.size = size; - return entry; + return SharedData::ArchiveDownloadRoot{ + .path = remote(path), + .entry = SharedData::DirectoryEntry{.type = SharedData::FileType::Regular, .size = size}, + }; } - inline SharedData::DirectoryEntry - makeDirectoryEntry(std::filesystem::path const& fullPath) + inline SharedData::ArchiveDownloadRoot makeDirectoryEntry(std::string_view path) { - SharedData::DirectoryEntry entry{}; - entry.path = fullPath; - entry.fullPath = fullPath; - entry.type = SharedData::FileType::Directory; - return entry; + return SharedData::ArchiveDownloadRoot{ + .path = remote(path), + .entry = SharedData::DirectoryEntry{.type = SharedData::FileType::Directory}, + }; } /** @@ -80,8 +75,8 @@ namespace Test auto entry = std::move(**nextResult); RecoveredEntry recovered{}; - recovered.path = entry.directoryEntry().path.generic_string(); - recovered.type = entry.directoryEntry().type; + recovered.path = entry.header().path; + recovered.type = entry.header().type; if (recovered.type == SharedData::FileType::Regular) { diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index 9bf84d0f..d405f5a2 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -1,8 +1,10 @@ +#include #include "nui_env.hpp" #include "test_download_operation.hpp" #include "test_upload_operation.hpp" #include "test_scan_operation.hpp" +#include "test_local_delete_guard.hpp" #include "test_local_scan_operation.hpp" #include "test_bulk_download_operation.hpp" #include "test_bulk_upload_operation.hpp" @@ -21,6 +23,10 @@ #include "test_file_change_filter.hpp" #include "test_password_request.hpp" #include "test_sync_session.hpp" +#include "test_transfer_failures.hpp" +#include "test_operation_queue.hpp" +#include "test_existence_check.hpp" +#include "test_opener_risk.hpp" #include @@ -29,6 +35,8 @@ #include std::filesystem::path programDirectory; +/** @brief The temporary directory of this process alone, so tests running in parallel share no files. */ +std::filesystem::path temporaryRoot; int main(int argc, char** argv) { @@ -39,6 +47,9 @@ int main(int argc, char** argv) // instanceDir (from a relative programDirectory) doubles the path. programDirectory = std::filesystem::weakly_canonical(std::filesystem::path{argv[0]}.parent_path()); + const Utility::ProcessTemporaryDirectory processTemporary{programDirectory / "temp"}; + temporaryRoot = processTemporary.path(); + ::testing::InitGoogleTest(&argc, argv); ::testing::AddGlobalTestEnvironment(new Test::NuiEnvGuard{}); return RUN_ALL_TESTS(); diff --git a/backend/test/backend/real_server_tests.hpp b/backend/test/backend/real_server_tests.hpp index e2a5979b..7b9c939a 100644 --- a/backend/test/backend/real_server_tests.hpp +++ b/backend/test/backend/real_server_tests.hpp @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include @@ -13,6 +15,23 @@ using namespace std::chrono_literals; using namespace std::string_literals; extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; + +/** + * @brief A remote path from a test literal, which is known to be valid. + */ +inline Paths::RemotePath remote(std::string_view path) +{ + return Paths::RemotePath::make(path).value(); +} + +/** + * @brief A local path below the test directory, which is known to be valid. + */ +inline Paths::LocalPath toLocal(std::filesystem::path const& path) +{ + return Paths::LocalPath::make(path).value(); +} #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wc23-extensions" @@ -42,7 +61,7 @@ class RealServerTests : public ::testing::Test protected: void SetUp() override { - std::ofstream privateKeyFile{programDirectory / "temp" / "key.private", std::ios_base::binary}; + std::ofstream privateKeyFile{temporaryRoot / "key.private", std::ios_base::binary}; privateKeyFile.write(privateKey, std::strlen(privateKey)); } @@ -121,7 +140,7 @@ class RealServerTests : public ::testing::Test } protected: - Utility::TemporaryDirectory isolateDirectory_{programDirectory / "temp", false}; + Utility::TemporaryDirectory isolateDirectory_{temporaryRoot, false}; boost::asio::thread_pool pool_{1}; nlohmann::json defaultPackageJson_ = nlohmann::json({ {"name", "test"}, diff --git a/backend/test/backend/test_archive_download_operation.hpp b/backend/test/backend/test_archive_download_operation.hpp index 8e4cf794..ee2599cc 100644 --- a/backend/test/backend/test_archive_download_operation.hpp +++ b/backend/test/backend/test_archive_download_operation.hpp @@ -11,6 +11,8 @@ #include #include +extern std::filesystem::path temporaryRoot; + namespace Test { class ArchiveDownloadOperationTests : public RealServerTests @@ -22,7 +24,7 @@ namespace Test } protected: - Utility::TemporaryDirectory archiveDir_{programDirectory / "temp", true}; + Utility::TemporaryDirectory archiveDir_{temporaryRoot, true}; /** * @brief Drive an ArchiveDownloadOperation to completion or until it errors. @@ -57,7 +59,7 @@ namespace Test { ScanOperation scan{ sftp, - ScanOperation::ScanOperationOptions{.remotePath = path}, + ScanOperation::ScanOperationOptions{.remotePath = remote(path.generic_string())}, }; while (true) { @@ -92,7 +94,7 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, .localArchivePath = {}, }}; auto result = operation.work(); @@ -100,6 +102,25 @@ namespace Test EXPECT_EQ(result.error().type, ArchiveDownloadOperation::ErrorType::InvalidPath); } + TEST_F(ArchiveDownloadOperationTests, RootWithoutNameIsRefused) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + ArchiveDownloadOperation operation{ + *sftp, + { + .roots = {makeDirectoryEntry("/")}, + .localArchivePath = toLocal(archiveDir_.path() / "root.tar"), + } + }; + operation.setScanResultForRoot(remote("/"), {SharedData::DirectoryEntry{.type = SharedData::FileType::Directory}}, 0u); + const auto result = runToCompletion(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, ArchiveDownloadOperation::ErrorType::InvalidPath); + EXPECT_FALSE(std::filesystem::exists(archiveDir_.path() / "root.tar")); + } + TEST_F(ArchiveDownloadOperationTests, NoEntriesFails) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -108,9 +129,10 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {}, - .localArchivePath = archiveDir_.path() / "empty.tar", - }}; + .roots = {}, + .localArchivePath = toLocal(archiveDir_.path() / "empty.tar"), + } + }; auto result = operation.work(); ASSERT_FALSE(result.has_value()); EXPECT_EQ(result.error().type, ArchiveDownloadOperation::ErrorType::ImplementationError); @@ -125,10 +147,11 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archivePath, + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; + } + }; auto result = runToCompletion(operation); ASSERT_TRUE(result.has_value()); EXPECT_EQ(*result, ArchiveDownloadOperation::WorkStatus::Complete); @@ -150,10 +173,11 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archivePath, + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archivePath), // Auto: detected from extension. - }}; + } + }; auto result = runToCompletion(operation); ASSERT_TRUE(result.has_value()); EXPECT_EQ(*result, ArchiveDownloadOperation::WorkStatus::Complete); @@ -172,9 +196,10 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archivePath, - }}; + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archivePath), + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); const auto entries = recoverArchiveEntries(archivePath); @@ -191,9 +216,10 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archivePath, - }}; + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archivePath), + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); const auto entries = recoverArchiveEntries(archivePath); @@ -210,14 +236,15 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = + .roots = { makeFileEntry("/home/test/file1.txt", 17u), makeFileEntry("/home/test/file2.txt", 17u), }, - .localArchivePath = archivePath, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); const auto entries = recoverArchiveEntries(archivePath); @@ -238,11 +265,12 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeDirectoryEntry(rootPath)}, - .localArchivePath = archivePath, + .roots = {makeDirectoryEntry(rootPath.generic_string())}, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; - operation.setScanResultForRoot(rootPath, std::move(scan.entries), scan.totalBytes); + } + }; + operation.setScanResultForRoot(remote(rootPath.generic_string()), std::move(scan.entries), scan.totalBytes); ASSERT_TRUE(runToCompletion(operation).has_value()); const auto entries = recoverArchiveEntries(archivePath); @@ -254,6 +282,36 @@ namespace Test EXPECT_EQ(doc1->contents, "Document 1 content"); } + TEST_F(ArchiveDownloadOperationTests, EntryShorterThanScannedFailsTheArchive) + { + // A file that shrank after the scan must fail the archive instead of being zero padded. + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const std::filesystem::path rootPath = "/home/test/Documents"; + auto scan = runScan(*sftp, rootPath); + for (auto& entry : scan.entries) + { + if (entry.name == "doc1.txt") + entry.size += 100; + } + + const auto archivePath = archiveDir_.path() / "short.tar"; + ArchiveDownloadOperation operation{ + *sftp, + { + .roots = {makeDirectoryEntry(rootPath.generic_string())}, + .localArchivePath = toLocal(archivePath), + .compression = TarArchive::Compression::None, + } + }; + operation.setScanResultForRoot(remote(rootPath.generic_string()), std::move(scan.entries), scan.totalBytes + 100); + const auto result = runToCompletion(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, ArchiveDownloadOperation::ErrorType::SizeMismatch); + EXPECT_FALSE(std::filesystem::exists(archivePath)); + } + TEST_F(ArchiveDownloadOperationTests, DirectoryRootWithoutPrescanErrors) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -263,10 +321,11 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeDirectoryEntry("/home/test/Documents")}, - .localArchivePath = archivePath, + .roots = {makeDirectoryEntry("/home/test/Documents")}, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; + } + }; // Intentionally skip setScanResultForRoot — the op must not // self-recurse; it has to surface an ImplementationError instead. const auto result = runToCompletion(operation); @@ -288,12 +347,13 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeDirectoryEntry("/home/test/Documents")}, - .localArchivePath = archivePath, + .roots = {makeDirectoryEntry("/home/test/Documents")}, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; + } + }; operation.setScanResultForRoot( - "/home/test/Pictures", std::move(pictures.entries), pictures.totalBytes + remote("/home/test/Pictures"), std::move(pictures.entries), pictures.totalBytes ); const auto result = runToCompletion(operation); ASSERT_FALSE(result.has_value()); @@ -312,19 +372,20 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = + .roots = { makeDirectoryEntry("/home/test/Documents"), makeDirectoryEntry("/home/test/Pictures"), }, - .localArchivePath = archivePath, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; + } + }; operation.setScanResultForRoot( - "/home/test/Documents", std::move(documents.entries), documents.totalBytes + remote("/home/test/Documents"), std::move(documents.entries), documents.totalBytes ); operation.setScanResultForRoot( - "/home/test/Pictures", std::move(pictures.entries), pictures.totalBytes + remote("/home/test/Pictures"), std::move(pictures.entries), pictures.totalBytes ); ASSERT_TRUE(runToCompletion(operation).has_value()); @@ -346,19 +407,20 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = + .roots = { makeFileEntry("/home/test/file1.txt", 17u), makeDirectoryEntry("/home/test/Documents"), }, - .localArchivePath = archivePath, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; + } + }; // Regular-file roots never receive a setScanResultForRoot call — only // the directory root does. This asserts the op still succeeds in that // mixed configuration and doesn't treat file1.txt as missing-scan. operation.setScanResultForRoot( - "/home/test/Documents", std::move(documents.entries), documents.totalBytes + remote("/home/test/Documents"), std::move(documents.entries), documents.totalBytes ); ASSERT_TRUE(runToCompletion(operation).has_value()); @@ -385,11 +447,12 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeDirectoryEntry("/var")}, - .localArchivePath = archivePath, + .roots = {makeDirectoryEntry("/var")}, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; - operation.setScanResultForRoot("/var", std::move(scan.entries), scan.totalBytes); + } + }; + operation.setScanResultForRoot(remote("/var"), std::move(scan.entries), scan.totalBytes); ASSERT_TRUE(runToCompletion(operation).has_value()); const auto entries = recoverArchiveEntries(archivePath); @@ -410,10 +473,11 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/large.txt", 65536u)}, - .localArchivePath = archivePath, + .roots = {makeFileEntry("/home/test/large.txt", 65536u)}, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation, 2000).has_value()); const auto entries = recoverArchiveEntries(archivePath); @@ -431,10 +495,11 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archivePath, + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); EXPECT_TRUE(std::filesystem::exists(archivePath)); @@ -455,10 +520,11 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archivePath, + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archivePath), .mayOverwrite = false, - }}; + } + }; auto result = operation.work(); ASSERT_FALSE(result.has_value()); EXPECT_EQ(result.error().type, ArchiveDownloadOperation::ErrorType::FileExists); @@ -478,11 +544,12 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archivePath, + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); const auto entries = recoverArchiveEntries(archivePath); @@ -498,10 +565,11 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archiveDir_.path() / "finished.tar", + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archiveDir_.path() / "finished.tar"), .compression = TarArchive::Compression::None, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); auto again = operation.work(); @@ -522,14 +590,15 @@ namespace Test { .progressCallback = [&](std::uint64_t, std::uint64_t max, std::uint64_t current, std::make_signed_t) - { - lastMax = max; - lastCurrent = current; - }, - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archiveDir_.path() / "with_progress.tar", + { + lastMax = max; + lastCurrent = current; + }, + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archiveDir_.path() / "with_progress.tar"), .compression = TarArchive::Compression::None, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); EXPECT_EQ(lastMax, 17u); @@ -547,11 +616,12 @@ namespace Test ArchiveDownloadOperation operation{ *sftp, { - .entries = {makeFileEntry("/home/test/file1.txt", 17u)}, - .localArchivePath = archivePath, + .roots = {makeFileEntry("/home/test/file1.txt", 17u)}, + .localArchivePath = toLocal(archivePath), .compression = TarArchive::Compression::None, .createMissingDirectories = true, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); EXPECT_TRUE(std::filesystem::exists(archivePath)); } diff --git a/backend/test/backend/test_archive_upload_operation.hpp b/backend/test/backend/test_archive_upload_operation.hpp index f503c1d7..5b38f90f 100644 --- a/backend/test/backend/test_archive_upload_operation.hpp +++ b/backend/test/backend/test_archive_upload_operation.hpp @@ -17,6 +17,8 @@ #include #include +extern std::filesystem::path temporaryRoot; + namespace Test { class ArchiveUploadOperationTests : public RealServerTests @@ -34,8 +36,8 @@ namespace Test } protected: - Utility::TemporaryDirectory srcDir_{programDirectory / "temp", true}; - Utility::TemporaryDirectory pulledDir_{programDirectory / "temp", true}; + Utility::TemporaryDirectory srcDir_{temporaryRoot, true}; + Utility::TemporaryDirectory pulledDir_{temporaryRoot, true}; static void writeFile(std::filesystem::path const& path, std::string_view contents) { @@ -71,7 +73,7 @@ namespace Test static LocalScanBundle runLocalScan(std::filesystem::path const& path) { LocalScanOperation scan{ - LocalScanOperation::ScanOperationOptions{.localPath = path}, + LocalScanOperation::ScanOperationOptions{.localPath = toLocal(path)}, }; while (true) { @@ -101,10 +103,11 @@ namespace Test DownloadOperation download{ sftp, { - .remotePath = remotePath, - .localPath = localPath, + .remotePath = remote(remotePath.generic_string()), + .localPath = toLocal(localPath), .mayOverwrite = true, - }}; + } + }; for (int i = 0; i < 500; ++i) { auto result = download.work(); @@ -139,9 +142,10 @@ namespace Test ArchiveUploadOperation operation{ *sftp, { - .localPaths = {srcDir_.path() / "alpha.txt"}, + .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, .remoteArchivePath = {}, - }}; + } + }; auto result = operation.work(); ASSERT_FALSE(result.has_value()); EXPECT_EQ(result.error().type, ArchiveUploadOperation::ErrorType::InvalidPath); @@ -156,8 +160,9 @@ namespace Test *sftp, { .localPaths = {}, - .remoteArchivePath = "/home/test/nothing.tar", - }}; + .remoteArchivePath = remote("/home/test/nothing.tar"), + } + }; auto result = operation.work(); ASSERT_FALSE(result.has_value()); EXPECT_EQ(result.error().type, ArchiveUploadOperation::ErrorType::InvalidPath); @@ -172,11 +177,12 @@ namespace Test ArchiveUploadOperation operation{ *sftp, { - .localPaths = {srcDir_.path() / "alpha.txt"}, - .remoteArchivePath = remoteArchive, + .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, + .remoteArchivePath = remote(remoteArchive.generic_string()), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); const auto pulled = pullRemoteArchive(*sftp, remoteArchive, "single.tar"); @@ -195,10 +201,11 @@ namespace Test ArchiveUploadOperation operation{ *sftp, { - .localPaths = {srcDir_.path() / "alpha.txt"}, - .remoteArchivePath = remoteArchive, + .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, + .remoteArchivePath = remote(remoteArchive.generic_string()), .mayOverwrite = true, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); const auto pulled = pullRemoteArchive(*sftp, remoteArchive, "alpha.tar.gz"); @@ -216,10 +223,11 @@ namespace Test ArchiveUploadOperation operation{ *sftp, { - .localPaths = {srcDir_.path() / "alpha.txt"}, - .remoteArchivePath = remoteArchive, + .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, + .remoteArchivePath = remote(remoteArchive.generic_string()), .mayOverwrite = true, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); const auto pulled = pullRemoteArchive(*sftp, remoteArchive, "alpha.tar.zst"); @@ -239,13 +247,14 @@ namespace Test { .localPaths = { - srcDir_.path() / "alpha.txt", - srcDir_.path() / "beta.txt", + toLocal(srcDir_.path() / "alpha.txt"), + toLocal(srcDir_.path() / "beta.txt"), }, - .remoteArchivePath = remoteArchive, + .remoteArchivePath = remote(remoteArchive.generic_string()), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); const auto pulled = pullRemoteArchive(*sftp, remoteArchive, "pair.tar"); @@ -270,12 +279,13 @@ namespace Test ArchiveUploadOperation operation{ *sftp, { - .localPaths = {srcDir_.path()}, - .remoteArchivePath = remoteArchive, + .localPaths = {toLocal(srcDir_.path())}, + .remoteArchivePath = remote(remoteArchive.generic_string()), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; - operation.setScanResultForRoot(srcDir_.path(), std::move(scan.entries), scan.totalBytes); + } + }; + operation.setScanResultForRoot(toLocal(srcDir_.path()), std::move(scan.entries), scan.totalBytes); ASSERT_TRUE(runToCompletion(operation).has_value()); const auto pulled = pullRemoteArchive(*sftp, remoteArchive, "tree.tar"); @@ -299,11 +309,12 @@ namespace Test ArchiveUploadOperation operation{ *sftp, { - .localPaths = {srcDir_.path()}, - .remoteArchivePath = "/home/test/missing_scan.tar", + .localPaths = {toLocal(srcDir_.path())}, + .remoteArchivePath = remote("/home/test/missing_scan.tar"), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; + } + }; // No setScanResultForRoot — the op must not self-recurse; it has to // surface an ImplementationError like its download counterpart. const auto result = runToCompletion(operation); @@ -319,20 +330,21 @@ namespace Test // Create a second directory tree distinct from srcDir_. The scan is // keyed on that path; the archive asks to pack srcDir_ instead, so // the real root is still missing its scan and must error. - Utility::TemporaryDirectory otherDir{programDirectory / "temp", true}; + Utility::TemporaryDirectory otherDir{temporaryRoot, true}; writeFile(otherDir.path() / "unrelated.txt", "unrelated"); auto otherScan = runLocalScan(otherDir.path()); ArchiveUploadOperation operation{ *sftp, { - .localPaths = {srcDir_.path()}, - .remoteArchivePath = "/home/test/wrong_key.tar", + .localPaths = {toLocal(srcDir_.path())}, + .remoteArchivePath = remote("/home/test/wrong_key.tar"), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; + } + }; operation.setScanResultForRoot( - otherDir.path(), std::move(otherScan.entries), otherScan.totalBytes + toLocal(otherDir.path()), std::move(otherScan.entries), otherScan.totalBytes ); const auto result = runToCompletion(operation); ASSERT_FALSE(result.has_value()); @@ -344,8 +356,8 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - Utility::TemporaryDirectory firstDir{programDirectory / "temp", true}; - Utility::TemporaryDirectory secondDir{programDirectory / "temp", true}; + Utility::TemporaryDirectory firstDir{temporaryRoot, true}; + Utility::TemporaryDirectory secondDir{temporaryRoot, true}; writeFile(firstDir.path() / "one.txt", "one payload"); writeFile(secondDir.path() / "two.txt", "two payload"); @@ -356,16 +368,17 @@ namespace Test ArchiveUploadOperation operation{ *sftp, { - .localPaths = {firstDir.path(), secondDir.path()}, - .remoteArchivePath = remoteArchive, + .localPaths = {toLocal(firstDir.path()), toLocal(secondDir.path())}, + .remoteArchivePath = remote(remoteArchive.generic_string()), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; + } + }; operation.setScanResultForRoot( - firstDir.path(), std::move(firstScan.entries), firstScan.totalBytes + toLocal(firstDir.path()), std::move(firstScan.entries), firstScan.totalBytes ); operation.setScanResultForRoot( - secondDir.path(), std::move(secondScan.entries), secondScan.totalBytes + toLocal(secondDir.path()), std::move(secondScan.entries), secondScan.totalBytes ); ASSERT_TRUE(runToCompletion(operation).has_value()); @@ -392,12 +405,13 @@ namespace Test // A regular-file root (no scan needed) alongside a directory // root (which does need a scan) — the op must succeed and // must not mis-flag the file as missing-scan. - .localPaths = {topFile, srcDir_.path()}, - .remoteArchivePath = remoteArchive, + .localPaths = {toLocal(topFile), toLocal(srcDir_.path())}, + .remoteArchivePath = remote(remoteArchive.generic_string()), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; - operation.setScanResultForRoot(srcDir_.path(), std::move(scan.entries), scan.totalBytes); + } + }; + operation.setScanResultForRoot(toLocal(srcDir_.path()), std::move(scan.entries), scan.totalBytes); ASSERT_TRUE(runToCompletion(operation).has_value()); const auto pulled = pullRemoteArchive(*sftp, remoteArchive, "mixed_up.tar"); @@ -422,12 +436,13 @@ namespace Test ArchiveUploadOperation operation{ *sftp, { - .localPaths = {srcDir_.path()}, - .remoteArchivePath = remoteArchive, + .localPaths = {toLocal(srcDir_.path())}, + .remoteArchivePath = remote(remoteArchive.generic_string()), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; - operation.setScanResultForRoot(srcDir_.path(), std::move(scan.entries), scan.totalBytes); + } + }; + operation.setScanResultForRoot(toLocal(srcDir_.path()), std::move(scan.entries), scan.totalBytes); ASSERT_TRUE(runToCompletion(operation).has_value()); const auto pulled = pullRemoteArchive(*sftp, remoteArchive, "nested_up.tar"); @@ -447,11 +462,12 @@ namespace Test ArchiveUploadOperation operation{ *sftp, { - .localPaths = {srcDir_.path() / "alpha.txt"}, - .remoteArchivePath = "/home/test/done.tar", + .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, + .remoteArchivePath = remote("/home/test/done.tar"), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); auto again = operation.work(); @@ -472,15 +488,16 @@ namespace Test { .progressCallback = [&](std::uint64_t, std::uint64_t max, std::uint64_t current, std::make_signed_t) - { - lastMax = max; - lastCurrent = current; - }, - .localPaths = {srcDir_.path() / "alpha.txt"}, - .remoteArchivePath = "/home/test/progress.tar", + { + lastMax = max; + lastCurrent = current; + }, + .localPaths = {toLocal(srcDir_.path() / "alpha.txt")}, + .remoteArchivePath = remote("/home/test/progress.tar"), .compression = TarArchive::Compression::None, .mayOverwrite = true, - }}; + } + }; ASSERT_TRUE(runToCompletion(operation).has_value()); EXPECT_EQ(lastMax, std::string{"alpha payload"}.size()); diff --git a/backend/test/backend/test_bulk_download_operation.hpp b/backend/test/backend/test_bulk_download_operation.hpp index 99142c05..e3871039 100644 --- a/backend/test/backend/test_bulk_download_operation.hpp +++ b/backend/test/backend/test_bulk_download_operation.hpp @@ -8,11 +8,14 @@ #include #include +#include #include #include #include #include +extern std::filesystem::path temporaryRoot; + namespace Test { class BulkDownloadOperationTests : public RealServerTests @@ -39,7 +42,7 @@ namespace Test ScanBundle runScan(SecureShell::SftpSession& sftp, std::filesystem::path const& remotePath, bool recursive = true) { - ScanOperation scan{sftp, {.remotePath = remotePath, .recursive = recursive}}; + ScanOperation scan{sftp, {.remotePath = remote(remotePath.generic_string()), .recursive = recursive}}; for (int iteration = 0; iteration < 500; ++iteration) { auto workResult = scan.work(); @@ -71,7 +74,7 @@ namespace Test } protected: - Utility::TemporaryDirectory downloadDir_{programDirectory / "temp", true}; + Utility::TemporaryDirectory downloadDir_{temporaryRoot, true}; }; TEST_F(BulkDownloadOperationTests, CanCreateBulkDownloadOperation) @@ -97,17 +100,29 @@ namespace Test EXPECT_FALSE(operation.isBarrier()); } - TEST_F(BulkDownloadOperationTests, EmptyEntriesCompletesImmediately) + TEST_F(BulkDownloadOperationTests, EmptyScanResultFails) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {.localPath = downloadDir_.path() / "empty"}}; + BulkDownloadOperation operation{*sftp, {.localPath = toLocal(downloadDir_.path() / "empty")}}; + // A scanned tree always holds its root, an empty one means the scan never delivered. operation.setScanResult({}, 0); auto workResult = operation.work(); - ASSERT_TRUE(workResult.has_value()); - EXPECT_EQ(workResult.value(), BulkDownloadOperation::WorkStatus::Complete); + ASSERT_FALSE(workResult.has_value()); + EXPECT_EQ(workResult.error().type, SharedData::OperationErrorType::ScanIncomplete); + } + + TEST_F(BulkDownloadOperationTests, StartWithoutEntriesFails) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + BulkDownloadOperation operation{*sftp, {}}; + auto workResult = operation.work(); + ASSERT_FALSE(workResult.has_value()); + EXPECT_EQ(workResult.error().type, SharedData::OperationErrorType::ImplementationError); } TEST_F(BulkDownloadOperationTests, FirstEntryMustBeDirectory) @@ -118,12 +133,14 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {.localPath = downloadDir_.path() / "bad"}}; + BulkDownloadOperation operation{ + *sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "bad")} + }; std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test/file1.txt", + .name = "/home/test/file1.txt", .type = SharedData::FileType::Regular, .size = 17, } @@ -145,7 +162,9 @@ namespace Test const auto localRoot = downloadDir_.path() / "Documents"; - BulkDownloadOperation operation{*sftp, {.localPath = localRoot}}; + BulkDownloadOperation operation{ + *sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} + }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -168,7 +187,9 @@ namespace Test const auto localRoot = downloadDir_.path() / "newly" / "created" / "root"; ASSERT_FALSE(std::filesystem::exists(localRoot)); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot}}; + BulkDownloadOperation operation{ + *sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(localRoot)} + }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -188,27 +209,26 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test", .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "Documents", + .name = "Documents", .type = SharedData::FileType::Directory, .parent = 0, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "doc1.txt", + .name = "doc1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Document 1 content"}.size(), .parent = 1, } ); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot}}; + BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), std::string{"Document 1 content"}.size()); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -230,13 +250,12 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test", .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "does_not_exist.txt", + .name = "does_not_exist.txt", .type = SharedData::FileType::Regular, .size = 10, .parent = 0, @@ -244,14 +263,16 @@ namespace Test ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Fake file content"}.size(), .parent = 0, } ); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot, .failFast = false}}; + BulkDownloadOperation operation{ + *sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = false} + }; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -262,7 +283,110 @@ namespace Test const auto failed = operation.getFailed(); ASSERT_EQ(failed.size(), 1u); - EXPECT_EQ(failed.front().first.filename().generic_string(), "does_not_exist.txt"); + EXPECT_TRUE(failed.front().first.ends_with("/does_not_exist.txt")) << failed.front().first; + } + + TEST_F(BulkDownloadOperationTests, ScannedTreeWithoutRootsFails) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localRoot = downloadDir_.path() / "rootless"; + BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + auto scan = runScan(*sftp, "/home/test/Documents"); + operation.setScanResult(std::move(scan.entries), scan.totalBytes); + + auto workResult = operation.work(); + ASSERT_FALSE(workResult.has_value()); + EXPECT_EQ(workResult.error().type, BulkDownloadOperation::ErrorType::ImplementationError); + EXPECT_FALSE(std::filesystem::exists(localRoot)); + } + + TEST_F(BulkDownloadOperationTests, NamesThatAreNoPathSegmentAreRecordedAsFailed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localRoot = downloadDir_.path() / "hostile" / "root"; + std::vector entries; + entries.push_back(SharedData::DirectoryEntry{.type = SharedData::FileType::Directory}); + entries.push_back( + SharedData::DirectoryEntry{.name = "x/..", .type = SharedData::FileType::Regular, .size = 3, .parent = 0} + ); + entries.push_back( + SharedData::DirectoryEntry{ + .name = "file1.txt", + .type = SharedData::FileType::Regular, + .size = std::string{"Fake file content"}.size(), + .parent = 0, + } + ); + + BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + operation.setScanResult(std::move(entries), 0); + ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); + + const auto failed = operation.getFailed(); + ASSERT_EQ(failed.size(), 1u); + EXPECT_EQ(failed.front().second.type, SharedData::OperationErrorType::InvalidName); + EXPECT_EQ(readLocalFile(localRoot / "file1.txt"), "Fake file content"); + EXPECT_EQ( + std::distance( + std::filesystem::directory_iterator{localRoot.parent_path()}, std::filesystem::directory_iterator{} + ), + 1 + ); + } + + TEST_F(BulkDownloadOperationTests, NeverWritesThroughALinkInsideTheLocalTree) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto outside = downloadDir_.path() / "outside"; + const auto localRoot = downloadDir_.path() / "linked"; + std::filesystem::create_directories(outside); + std::filesystem::create_directories(localRoot); +#ifdef _WIN32 + const auto command = + "cmd /c mklink /J \"" + (localRoot / "Documents").string() + "\" \"" + outside.string() + "\" >nul"; + ASSERT_EQ(std::system(command.c_str()), 0); +#else + std::filesystem::create_directory_symlink(outside, localRoot / "Documents"); +#endif + + std::vector entries; + entries.push_back(SharedData::DirectoryEntry{.type = SharedData::FileType::Directory}); + entries.push_back( + SharedData::DirectoryEntry{.name = "Documents", .type = SharedData::FileType::Directory, .parent = 0} + ); + entries.push_back( + SharedData::DirectoryEntry{ + .name = "doc1.txt", + .type = SharedData::FileType::Regular, + .size = std::string{"Document 1 content"}.size(), + .parent = 1, + } + ); + entries.push_back( + SharedData::DirectoryEntry{ + .name = "file1.txt", + .type = SharedData::FileType::Regular, + .size = std::string{"Fake file content"}.size(), + .parent = 0, + } + ); + + BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; + operation.setScanResult(std::move(entries), 0); + ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); + + EXPECT_TRUE(std::filesystem::is_empty(outside)) << "nothing may be written through the link"; + EXPECT_EQ(readLocalFile(localRoot / "file1.txt"), "Fake file content"); + const auto failed = operation.getFailed(); + ASSERT_EQ(failed.size(), 2u); + for (auto const& [source, error] : failed) + EXPECT_EQ(error.type, SharedData::OperationErrorType::LinkInLocalPath) << source; } TEST_F(BulkDownloadOperationTests, FailFastAbortsWholeOperationOnRecoverableError) @@ -275,13 +399,12 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test", .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "does_not_exist.txt", + .name = "does_not_exist.txt", .type = SharedData::FileType::Regular, .size = 10, .parent = 0, @@ -289,14 +412,16 @@ namespace Test ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Fake file content"}.size(), .parent = 0, } ); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot, .failFast = true}}; + BulkDownloadOperation operation{ + *sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = true} + }; operation.setScanResult(std::move(entries), 0); std::expected workResult{ @@ -327,27 +452,26 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test", .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "weird", + .name = "weird", .type = SharedData::FileType::Fifo, .parent = 0, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Fake file content"}.size(), .parent = 0, } ); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot}}; + BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -365,19 +489,18 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test", .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "link_to_file1.txt", + .name = "link_to_file1.txt", .type = SharedData::FileType::Symlink, .parent = 0, } ); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot}}; + BulkDownloadOperation operation{*sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot)}}; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -402,7 +525,8 @@ namespace Test { ++callbackInvocations; }, - .localPath = downloadDir_.path() / "progress", + .remotePath = remote("/home/test/Documents"), + .localPath = toLocal(downloadDir_.path() / "progress"), } }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -418,7 +542,10 @@ namespace Test auto scan = runScan(*sftp, "/home/test/Documents"); - BulkDownloadOperation operation{*sftp, {.localPath = downloadDir_.path() / "completed"}}; + BulkDownloadOperation operation{ + *sftp, + {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(downloadDir_.path() / "completed")} + }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -433,12 +560,14 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkDownloadOperation operation{*sftp, {.localPath = downloadDir_.path() / "failed"}}; + BulkDownloadOperation operation{ + *sftp, {.remotePath = remote("/home/test/file1.txt"), .localPath = toLocal(downloadDir_.path() / "failed")} + }; std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test/file1.txt", + .name = "/home/test/file1.txt", .type = SharedData::FileType::Regular, } ); @@ -466,17 +595,17 @@ namespace Test std::vector files; files.push_back({ - .remoteSrc = "/home/test/Documents/doc1.txt", - .localDst = localRoot / "doc1.txt", + .source = remote("/home/test/Documents/doc1.txt"), + .destination = toLocal(localRoot / "doc1.txt"), .sizeBytes = std::string{"Document 1 content"}.size(), }); files.push_back({ - .remoteSrc = "/home/test/Documents/doc2.txt", - .localDst = localRoot / "doc2.txt", + .source = remote("/home/test/Documents/doc2.txt"), + .destination = toLocal(localRoot / "doc2.txt"), .sizeBytes = std::string{"Document 2 content"}.size(), }); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot}}; + BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -488,9 +617,44 @@ namespace Test EXPECT_TRUE(operation.getFailed().empty()); } + TEST_F(BulkDownloadOperationTests, PrescannedFilesReplaceOnlyApprovedDestinations) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localRoot = downloadDir_.path() / "approval"; + std::filesystem::create_directories(localRoot); + // doc1.txt appeared after the existence check, doc2.txt existed and the user agreed to replace it. + std::ofstream{localRoot / "doc1.txt"} << "appeared"; + std::ofstream{localRoot / "doc2.txt"} << "approved"; + + std::vector files; + files.push_back({ + .source = remote("/home/test/Documents/doc1.txt"), + .destination = toLocal(localRoot / "doc1.txt"), + .sizeBytes = std::string{"Document 1 content"}.size(), + }); + files.push_back({ + .source = remote("/home/test/Documents/doc2.txt"), + .destination = toLocal(localRoot / "doc2.txt"), + .sizeBytes = std::string{"Document 2 content"}.size(), + .mayOverwrite = true, + }); + + BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; + operation.setPrescannedFileList(std::move(files)); + ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); + + EXPECT_EQ(readLocalFile(localRoot / "doc1.txt"), "appeared"); + EXPECT_EQ(readLocalFile(localRoot / "doc2.txt"), "Document 2 content"); + const auto failed = operation.getFailed(); + ASSERT_EQ(failed.size(), 1u); + EXPECT_EQ(failed.front().second.type, SharedData::OperationErrorType::FileExists); + } + TEST_F(BulkDownloadOperationTests, SetPrescannedFileListPreservesMtimeOnDownloadedFiles) { - // Regression: BulkAddEntry → PrescannedFile → DirectoryEntry must + // Regression: BulkDownloadEntry → PrescannedFile → DirectoryEntry must // carry mtime through so each downloaded local file ends up with the // remote's mtime (otherwise a follow-up sync would see all downloaded // files as "modified" and re-enqueue them forever). @@ -505,21 +669,21 @@ namespace Test std::vector files; files.push_back({ - .remoteSrc = "/home/test/Documents/doc1.txt", - .localDst = localRoot / "doc1.txt", + .source = remote("/home/test/Documents/doc1.txt"), + .destination = toLocal(localRoot / "doc1.txt"), .sizeBytes = std::string{"Document 1 content"}.size(), .mtime = mtimeDoc1, .mtimeNsec = 0, }); files.push_back({ - .remoteSrc = "/home/test/Documents/doc2.txt", - .localDst = localRoot / "doc2.txt", + .source = remote("/home/test/Documents/doc2.txt"), + .destination = toLocal(localRoot / "doc2.txt"), .sizeBytes = std::string{"Document 2 content"}.size(), .mtime = mtimeDoc2, .mtimeNsec = 0, }); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot}}; + BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot)}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -548,12 +712,12 @@ namespace Test const auto localRoot = downloadDir_.path() / "prescanned_parents"; std::vector files; files.push_back({ - .remoteSrc = "/home/test/Documents/doc1.txt", - .localDst = localRoot / "new" / "deeper" / "doc1.txt", + .source = remote("/home/test/Documents/doc1.txt"), + .destination = toLocal(localRoot / "new" / "deeper" / "doc1.txt"), .sizeBytes = std::string{"Document 1 content"}.size(), }); - BulkDownloadOperation::BulkDownloadOperationOptions options{.localPath = localRoot}; + BulkDownloadOperation::BulkDownloadOperationOptions options{.localPath = toLocal(localRoot)}; options.individualOptions.createMissingDirectories = true; BulkDownloadOperation operation{*sftp, std::move(options)}; operation.setPrescannedFileList(std::move(files)); @@ -578,20 +742,20 @@ namespace Test std::vector files; files.push_back({ - .remoteSrc = "/home/test/Documents/doc1.txt", - .localDst = localRoot / "blocker" / "doc1.txt", + .source = remote("/home/test/Documents/doc1.txt"), + .destination = toLocal(localRoot / "blocker" / "doc1.txt"), .sizeBytes = std::string{"Document 1 content"}.size(), }); files.push_back({ - .remoteSrc = "/home/test/Documents/doc2.txt", - .localDst = localRoot / "doc2.txt", + .source = remote("/home/test/Documents/doc2.txt"), + .destination = toLocal(localRoot / "doc2.txt"), .sizeBytes = std::string{"Document 2 content"}.size(), }); for (const bool createMissing : {false, true}) { std::filesystem::remove(localRoot / "doc2.txt"); - BulkDownloadOperation::BulkDownloadOperationOptions options{.localPath = localRoot}; + BulkDownloadOperation::BulkDownloadOperationOptions options{.localPath = toLocal(localRoot)}; options.individualOptions.createMissingDirectories = createMissing; BulkDownloadOperation operation{*sftp, std::move(options)}; operation.setPrescannedFileList(files); @@ -609,7 +773,9 @@ namespace Test auto scan = runScan(*sftp, "/home/test/Documents"); - BulkDownloadOperation operation{*sftp, {.localPath = downloadDir_.path() / "cancel"}}; + BulkDownloadOperation operation{ + *sftp, {.remotePath = remote("/home/test/Documents"), .localPath = toLocal(downloadDir_.path() / "cancel")} + }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); auto firstWork = operation.work(); @@ -635,22 +801,22 @@ namespace Test std::vector files; files.push_back({ - .remoteSrc = "/home/test/Documents/doc1.txt", - .localDst = localRoot / "doc1.txt", + .source = remote("/home/test/Documents/doc1.txt"), + .destination = toLocal(localRoot / "doc1.txt"), .sizeBytes = std::string{"Document 1 content"}.size(), }); files.push_back({ - .remoteSrc = "/home/test/Documents/doc2.txt", - .localDst = localRoot / "doc2.txt", + .source = remote("/home/test/Documents/doc2.txt"), + .destination = toLocal(localRoot / "doc2.txt"), .sizeBytes = std::string{"Document 2 content"}.size(), }); files.push_back({ - .remoteSrc = "/home/test/file1.txt", - .localDst = localRoot / "file1.txt", + .source = remote("/home/test/file1.txt"), + .destination = toLocal(localRoot / "file1.txt"), .sizeBytes = std::string{"Fake file content"}.size(), }); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot, .concurrency = concurrency}}; + BulkDownloadOperation operation{*sftp, {.localPath = toLocal(localRoot), .concurrency = concurrency}}; operation.setPrescannedFileList(std::move(files)); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -676,13 +842,12 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test", .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Fake file content"}.size(), .parent = 0, @@ -690,14 +855,14 @@ namespace Test ); entries.push_back( SharedData::DirectoryEntry{ - .path = "Documents", + .name = "Documents", .type = SharedData::FileType::Directory, .parent = 0, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "doc1.txt", + .name = "doc1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Document 1 content"}.size(), .parent = 2, @@ -705,14 +870,16 @@ namespace Test ); entries.push_back( SharedData::DirectoryEntry{ - .path = "doc2.txt", + .name = "doc2.txt", .type = SharedData::FileType::Regular, .size = std::string{"Document 2 content"}.size(), .parent = 2, } ); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot, .concurrency = 3}}; + BulkDownloadOperation operation{ + *sftp, {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .concurrency = 3} + }; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -734,13 +901,12 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test", .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "does_not_exist.txt", + .name = "does_not_exist.txt", .type = SharedData::FileType::Regular, .size = 10, .parent = 0, @@ -748,14 +914,17 @@ namespace Test ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Fake file content"}.size(), .parent = 0, } ); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot, .failFast = false, .concurrency = 2}}; + BulkDownloadOperation operation{ + *sftp, + {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = false, .concurrency = 2} + }; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); @@ -765,7 +934,7 @@ namespace Test const auto failed = operation.getFailed(); ASSERT_EQ(failed.size(), 1u); - EXPECT_EQ(failed.front().first.filename().generic_string(), "does_not_exist.txt"); + EXPECT_TRUE(failed.front().first.ends_with("/does_not_exist.txt")) << failed.front().first; } TEST_F(BulkDownloadOperationTests, FailFastAbortsWholeOperationWithConcurrency) @@ -778,13 +947,12 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = "/home/test", .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "does_not_exist.txt", + .name = "does_not_exist.txt", .type = SharedData::FileType::Regular, .size = 10, .parent = 0, @@ -792,14 +960,17 @@ namespace Test ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Fake file content"}.size(), .parent = 0, } ); - BulkDownloadOperation operation{*sftp, {.localPath = localRoot, .failFast = true, .concurrency = 2}}; + BulkDownloadOperation operation{ + *sftp, + {.remotePath = remote("/home/test"), .localPath = toLocal(localRoot), .failFast = true, .concurrency = 2} + }; operation.setScanResult(std::move(entries), 0); std::expected workResult{ @@ -834,18 +1005,18 @@ namespace Test constexpr std::uint64_t largeSize = 65536u; std::vector files; files.push_back({ - .remoteSrc = "/home/test/large.txt", - .localDst = localRoot / "large_a.txt", + .source = remote("/home/test/large.txt"), + .destination = toLocal(localRoot / "large_a.txt"), .sizeBytes = largeSize, }); files.push_back({ - .remoteSrc = "/home/test/large.txt", - .localDst = localRoot / "large_b.txt", + .source = remote("/home/test/large.txt"), + .destination = toLocal(localRoot / "large_b.txt"), .sizeBytes = largeSize, }); files.push_back({ - .remoteSrc = "/home/test/file1.txt", - .localDst = localRoot / "file1.txt", + .source = remote("/home/test/file1.txt"), + .destination = toLocal(localRoot / "file1.txt"), .sizeBytes = std::string{"Fake file content"}.size(), }); @@ -865,7 +1036,7 @@ namespace Test lastCompleted = update.fileCurrentIndex; last = update; }, - .localPath = localRoot, + .localPath = toLocal(localRoot), .concurrency = concurrency, .progressEmitInterval = std::chrono::milliseconds{0}, } @@ -894,13 +1065,13 @@ namespace Test constexpr std::uint64_t largeSize = 65536u; std::vector files; files.push_back({ - .remoteSrc = "/home/test/large.txt", - .localDst = localRoot / "large_a.txt", + .source = remote("/home/test/large.txt"), + .destination = toLocal(localRoot / "large_a.txt"), .sizeBytes = largeSize, }); files.push_back({ - .remoteSrc = "/home/test/large.txt", - .localDst = localRoot / "large_b.txt", + .source = remote("/home/test/large.txt"), + .destination = toLocal(localRoot / "large_b.txt"), .sizeBytes = largeSize, }); @@ -913,7 +1084,7 @@ namespace Test { maxInFlight = std::max(maxInFlight, update.inFlight.size()); }, - .localPath = localRoot, + .localPath = toLocal(localRoot), .concurrency = 3, .progressEmitInterval = std::chrono::milliseconds{0}, } @@ -930,4 +1101,31 @@ namespace Test EXPECT_EQ(maxInFlight, 1u); EXPECT_EQ(operation.parallelWorkDoable(10), 1) << "nothing pending, still answers at least one"; } + +#ifndef _WIN32 + TEST_F(BulkDownloadOperationTests, InheritedPermissionsNeverMakeTheRootWorldWritableOrSpecial) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + // The scan root carries no permissions, applying "unknown" used to chmod it to 07777. + const auto localRoot = downloadDir_.path() / "inherited"; + BulkDownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/Documents"), + .localPath = toLocal(localRoot), + .individualOptions = {.inheritPermissions = true}, + } + }; + auto scan = runScan(*sftp, "/home/test/Documents"); + operation.setScanResult(std::move(scan.entries), scan.totalBytes); + ASSERT_EQ(runBulkToCompletion(operation), BulkDownloadOperation::WorkStatus::Complete); + + const auto mode = std::filesystem::status(localRoot).permissions(); + EXPECT_EQ(mode & (std::filesystem::perms::set_uid | std::filesystem::perms::set_gid | std::filesystem::perms::sticky_bit), std::filesystem::perms::none); + EXPECT_EQ(mode & std::filesystem::perms::others_write, std::filesystem::perms::none); + EXPECT_TRUE(std::filesystem::exists(localRoot / "doc1.txt")); + } +#endif } diff --git a/backend/test/backend/test_bulk_upload_operation.hpp b/backend/test/backend/test_bulk_upload_operation.hpp index f2e89f67..e56fa64e 100644 --- a/backend/test/backend/test_bulk_upload_operation.hpp +++ b/backend/test/backend/test_bulk_upload_operation.hpp @@ -15,6 +15,8 @@ #include #include +extern std::filesystem::path temporaryRoot; + namespace Test { class BulkUploadOperationTests : public RealServerTests @@ -43,7 +45,7 @@ namespace Test ScanBundle runLocalScan(std::filesystem::path const& localPath, bool recursive = true) { - LocalScanOperation scan{{.localPath = localPath, .recursive = recursive}}; + LocalScanOperation scan{{.localPath = toLocal(localPath), .recursive = recursive}}; for (int iteration = 0; iteration < 500; ++iteration) { auto workResult = scan.work(); @@ -80,7 +82,7 @@ namespace Test std::string const& filename ) { - auto fut = sftp.listDirectory(remoteDir); + auto fut = sftp.listDirectory(remote(remoteDir.generic_string())); if (fut.wait_for(5s) != std::future_status::ready) return false; auto result = fut.get(); @@ -91,13 +93,13 @@ namespace Test result.value().end(), [&](auto const& entry) { - return entry.path.filename().generic_string() == filename; + return entry.name == filename; } ); } protected: - Utility::TemporaryDirectory uploadSrc_{programDirectory / "temp", true}; + Utility::TemporaryDirectory uploadSrc_{temporaryRoot, true}; }; TEST_F(BulkUploadOperationTests, CanCreateBulkUploadOperation) @@ -123,17 +125,31 @@ namespace Test EXPECT_FALSE(operation.isBarrier()); } - TEST_F(BulkUploadOperationTests, EmptyEntriesCompletesImmediately) + TEST_F(BulkUploadOperationTests, EmptyScanResultFails) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {.remotePath = "/home/test/empty_bulk", .localPath = uploadSrc_.path()}}; + BulkUploadOperation operation{ + *sftp, {.remotePath = remote("/home/test/empty_bulk"), .localPath = toLocal(uploadSrc_.path())} + }; + // A scanned tree always holds its root, an empty one means the scan never delivered. operation.setScanResult({}, 0); auto workResult = operation.work(); - ASSERT_TRUE(workResult.has_value()); - EXPECT_EQ(workResult.value(), BulkUploadOperation::WorkStatus::Complete); + ASSERT_FALSE(workResult.has_value()); + EXPECT_EQ(workResult.error().type, SharedData::OperationErrorType::ScanIncomplete); + } + + TEST_F(BulkUploadOperationTests, StartWithoutEntriesFails) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + BulkUploadOperation operation{*sftp, {}}; + auto workResult = operation.work(); + ASSERT_FALSE(workResult.has_value()); + EXPECT_EQ(workResult.error().type, SharedData::OperationErrorType::ImplementationError); } TEST_F(BulkUploadOperationTests, FirstEntryMustBeDirectory) @@ -141,12 +157,14 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {.remotePath = "/home/test/bad_bulk", .localPath = uploadSrc_.path()}}; + BulkUploadOperation operation{ + *sftp, {.remotePath = remote("/home/test/bad_bulk"), .localPath = toLocal(uploadSrc_.path())} + }; std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = uploadSrc_.path() / "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, } ); @@ -167,7 +185,9 @@ namespace Test const auto remoteRoot = std::filesystem::path{"/home/test/bulk_uploaded"}; - BulkUploadOperation operation{*sftp, {.remotePath = remoteRoot, .localPath = uploadSrc_.path()}}; + BulkUploadOperation operation{ + *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); ASSERT_EQ(runBulkToCompletion(operation), BulkUploadOperation::WorkStatus::Complete); @@ -187,20 +207,19 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = uploadSrc_.path(), .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "subdir", + .name = "subdir", .type = SharedData::FileType::Directory, .parent = 0, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Bulk upload file one"}.size(), .parent = 1, @@ -211,7 +230,9 @@ namespace Test std::filesystem::create_directory(uploadSrc_.path() / "subdir"); writeFile(uploadSrc_.path() / "subdir" / "file1.txt", "Bulk upload file one"); - BulkUploadOperation operation{*sftp, {.remotePath = remoteRoot, .localPath = uploadSrc_.path()}}; + BulkUploadOperation operation{ + *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + }; operation.setScanResult(std::move(entries), std::string{"Bulk upload file one"}.size()); ASSERT_EQ(runBulkToCompletion(operation), BulkUploadOperation::WorkStatus::Complete); @@ -230,13 +251,12 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = uploadSrc_.path(), .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "does_not_exist_locally.txt", + .name = "does_not_exist_locally.txt", .type = SharedData::FileType::Regular, .size = 10, .parent = 0, @@ -244,7 +264,7 @@ namespace Test ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Bulk upload file one"}.size(), .parent = 0, @@ -252,7 +272,10 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, {.remotePath = remoteRoot, .localPath = uploadSrc_.path(), .failFast = false} + *sftp, + {.remotePath = remote(remoteRoot.generic_string()), + .localPath = toLocal(uploadSrc_.path()), + .failFast = false} }; operation.setScanResult(std::move(entries), 0); @@ -263,7 +286,7 @@ namespace Test const auto failed = operation.getFailed(); ASSERT_EQ(failed.size(), 1u); - EXPECT_EQ(failed.front().first.filename().generic_string(), "does_not_exist_locally.txt"); + EXPECT_TRUE(failed.front().first.ends_with("/does_not_exist_locally.txt")) << failed.front().first; } TEST_F(BulkUploadOperationTests, FailFastAbortsWholeOperationOnRecoverableError) @@ -276,13 +299,12 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = uploadSrc_.path(), .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "does_not_exist_locally.txt", + .name = "does_not_exist_locally.txt", .type = SharedData::FileType::Regular, .size = 10, .parent = 0, @@ -290,7 +312,7 @@ namespace Test ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Bulk upload file one"}.size(), .parent = 0, @@ -298,7 +320,10 @@ namespace Test ); BulkUploadOperation operation{ - *sftp, {.remotePath = remoteRoot, .localPath = uploadSrc_.path(), .failFast = true} + *sftp, + {.remotePath = remote(remoteRoot.generic_string()), + .localPath = toLocal(uploadSrc_.path()), + .failFast = true} }; operation.setScanResult(std::move(entries), 0); @@ -328,27 +353,28 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = uploadSrc_.path(), .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "weird", + .name = "weird", .type = SharedData::FileType::Fifo, .parent = 0, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, .size = std::string{"Bulk upload file one"}.size(), .parent = 0, } ); - BulkUploadOperation operation{*sftp, {.remotePath = remoteRoot, .localPath = uploadSrc_.path()}}; + BulkUploadOperation operation{ + *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + }; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkUploadOperation::WorkStatus::Complete); @@ -375,25 +401,26 @@ namespace Test std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = uploadSrc_.path(), .type = SharedData::FileType::Directory, } ); entries.push_back( SharedData::DirectoryEntry{ - .path = "the_link", + .name = "the_link", .type = SharedData::FileType::Symlink, - .linkTarget = targetFile, + .linkTarget = targetFile.generic_string(), .parent = 0, } ); - BulkUploadOperation operation{*sftp, {.remotePath = remoteRoot, .localPath = uploadSrc_.path()}}; + BulkUploadOperation operation{ + *sftp, {.remotePath = remote(remoteRoot.generic_string()), .localPath = toLocal(uploadSrc_.path())} + }; operation.setScanResult(std::move(entries), 0); ASSERT_EQ(runBulkToCompletion(operation), BulkUploadOperation::WorkStatus::Complete); - auto lstatFut = sftp->lstat((remoteRoot / "the_link").generic_string()); + auto lstatFut = sftp->lstat(remote((remoteRoot / "the_link").generic_string())); ASSERT_EQ(lstatFut.wait_for(5s), std::future_status::ready); auto entry = lstatFut.get(); ASSERT_TRUE(entry.has_value()); @@ -416,8 +443,8 @@ namespace Test { ++callbackInvocations; }, - .remotePath = "/home/test/progress_bulk", - .localPath = uploadSrc_.path(), + .remotePath = remote("/home/test/progress_bulk"), + .localPath = toLocal(uploadSrc_.path()), } }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -434,7 +461,7 @@ namespace Test auto scan = runLocalScan(uploadSrc_.path()); BulkUploadOperation operation{ - *sftp, {.remotePath = "/home/test/completed_bulk", .localPath = uploadSrc_.path()} + *sftp, {.remotePath = remote("/home/test/completed_bulk"), .localPath = toLocal(uploadSrc_.path())} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -450,12 +477,14 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - BulkUploadOperation operation{*sftp, {.remotePath = "/home/test/failed_bulk", .localPath = uploadSrc_.path()}}; + BulkUploadOperation operation{ + *sftp, {.remotePath = remote("/home/test/failed_bulk"), .localPath = toLocal(uploadSrc_.path())} + }; std::vector entries; entries.push_back( SharedData::DirectoryEntry{ - .path = uploadSrc_.path() / "file1.txt", + .name = "file1.txt", .type = SharedData::FileType::Regular, } ); @@ -476,7 +505,9 @@ namespace Test auto scan = runLocalScan(uploadSrc_.path()); - BulkUploadOperation operation{*sftp, {.remotePath = "/home/test/cancel_bulk", .localPath = uploadSrc_.path()}}; + BulkUploadOperation operation{ + *sftp, {.remotePath = remote("/home/test/cancel_bulk"), .localPath = toLocal(uploadSrc_.path())} + }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); auto firstWork = operation.work(); @@ -506,8 +537,8 @@ namespace Test const std::string content(fileSize, static_cast('a' + index)); writeFile(uploadSrc_.path() / name, content); files.push_back({ - .localSrc = uploadSrc_.path() / name, - .remoteDst = std::filesystem::path{"/home/test"} / name, + .source = toLocal(uploadSrc_.path() / name), + .destination = remote("/home/test/" + name), .sizeBytes = content.size(), }); } @@ -521,8 +552,8 @@ namespace Test { maxInFlight = std::max(maxInFlight, update.inFlight.size()); }, - .remotePath = "/home/test", - .localPath = uploadSrc_.path(), + .remotePath = remote("/home/test"), + .localPath = toLocal(uploadSrc_.path()), .concurrency = 3, .progressEmitInterval = std::chrono::milliseconds{0}, } @@ -558,7 +589,10 @@ namespace Test const std::filesystem::path remoteRoot = "/home/test/tree_concurrent"; BulkUploadOperation operation{ - *sftp, {.remotePath = remoteRoot, .localPath = uploadSrc_.path(), .concurrency = 3} + *sftp, + {.remotePath = remote(remoteRoot.generic_string()), + .localPath = toLocal(uploadSrc_.path()), + .concurrency = 3} }; operation.setScanResult(std::move(scan.entries), scan.totalBytes); @@ -583,18 +617,22 @@ namespace Test std::vector files; files.push_back({ - .localSrc = uploadSrc_.path() / "missing.txt", - .remoteDst = "/home/test/missing_concurrent.txt", + .source = toLocal(uploadSrc_.path() / "missing.txt"), + .destination = remote("/home/test/missing_concurrent.txt"), .sizeBytes = 10, }); files.push_back({ - .localSrc = uploadSrc_.path() / "file1.txt", - .remoteDst = "/home/test/present_concurrent.txt", + .source = toLocal(uploadSrc_.path() / "file1.txt"), + .destination = remote("/home/test/present_concurrent.txt"), .sizeBytes = std::string{"Bulk upload file one"}.size(), }); BulkUploadOperation operation{ - *sftp, {.remotePath = "/home/test", .localPath = uploadSrc_.path(), .failFast = false, .concurrency = 2} + *sftp, + {.remotePath = remote("/home/test"), + .localPath = toLocal(uploadSrc_.path()), + .failFast = false, + .concurrency = 2} }; operation.setPrescannedFileList(std::move(files)); @@ -603,6 +641,6 @@ namespace Test EXPECT_TRUE(remoteHasEntry(*sftp, "/home/test", "present_concurrent.txt")); const auto failed = operation.getFailed(); ASSERT_EQ(failed.size(), 1u); - EXPECT_EQ(failed.front().first.filename().generic_string(), "missing.txt"); + EXPECT_TRUE(failed.front().first.ends_with("/missing.txt")) << failed.front().first; } } diff --git a/backend/test/backend/test_download_operation.hpp b/backend/test/backend/test_download_operation.hpp index 66bbc145..761f73b6 100644 --- a/backend/test/backend/test_download_operation.hpp +++ b/backend/test/backend/test_download_operation.hpp @@ -8,6 +8,8 @@ #include #include +extern std::filesystem::path temporaryRoot; + namespace Test { class DownloadOperationTests : public RealServerTests @@ -26,7 +28,7 @@ namespace Test protected: // Isolated subdirectory per test run so downloaded files do not collide across tests. - Utility::TemporaryDirectory downloadDir_{programDirectory / "temp", true}; + Utility::TemporaryDirectory downloadDir_{temporaryRoot, true}; }; TEST_F(DownloadOperationTests, CanCreateServer) @@ -60,15 +62,39 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, - }}; + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + } + }; auto prepareResult = operation.prepare(); ASSERT_TRUE(prepareResult.has_value()); EXPECT_TRUE(std::filesystem::exists(localPath.generic_string() + ".filepart")); } + TEST_F(DownloadOperationTests, InvalidRemotePathsAreRefusedBeforeTheServer) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = downloadDir_.path() / "file1.txt"; + // Relative and ".." paths cannot be built as a RemotePath at all, unset and the root remain. + for (auto const& remotePath : {Paths::RemotePath{}, Paths::RemotePath::root()}) + { + DownloadOperation operation{ + *sftp, + { + .remotePath = remotePath, + .localPath = toLocal(localPath), + } + }; + auto prepareResult = operation.prepare(); + ASSERT_FALSE(prepareResult.has_value()) << remotePath.value(); + EXPECT_EQ(prepareResult.error().type, Operation::ErrorType::InvalidPath) << remotePath.value(); + } + EXPECT_FALSE(std::filesystem::exists(localPath)); + } + TEST_F(DownloadOperationTests, CanDownloadFile) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -79,9 +105,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, - }}; + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + } + }; DownloadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -114,8 +141,7 @@ namespace Test constexpr std::uint32_t expectedMtimeNsec = 500000000u; // 0.5s SharedData::DirectoryEntry entry{}; - entry.path = "/home/test/file1.txt"; - entry.fullPath = "/home/test/file1.txt"; + entry.name = "file1.txt"; entry.type = SharedData::FileType::Regular; entry.size = std::string{"Fake file content"}.size(); entry.mtime = expectedMtime; @@ -124,10 +150,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .entry = entry, - }}; + } + }; DownloadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -160,9 +187,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, - }}; + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + } + }; DownloadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -191,10 +219,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .mayOverwrite = false, - }}; + } + }; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); @@ -213,9 +242,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, - }}; + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + } + }; DownloadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -234,10 +264,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .mayOverwrite = true, - }}; + } + }; DownloadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -263,9 +294,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/does_not_exist.txt", - .localPath = downloadDir_.path() / "does_not_exist.txt", - }}; + .remotePath = remote("/home/test/does_not_exist.txt"), + .localPath = toLocal(downloadDir_.path() / "does_not_exist.txt"), + } + }; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); @@ -280,9 +312,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", + .remotePath = remote("/home/test/file1.txt"), .localPath = {}, - }}; + } + }; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); @@ -297,9 +330,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = downloadDir_.path() / "file1.txt", - }}; + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(downloadDir_.path() / "file1.txt"), + } + }; DownloadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -325,9 +359,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/does_not_exist.txt", - .localPath = downloadDir_.path() / "does_not_exist.txt", - }}; + .remotePath = remote("/home/test/does_not_exist.txt"), + .localPath = toLocal(downloadDir_.path() / "does_not_exist.txt"), + } + }; auto firstResult = operation.work(); ASSERT_FALSE(firstResult.has_value()); @@ -347,10 +382,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .doCleanup = true, - }}; + } + }; auto prepareResult = operation.prepare(); ASSERT_TRUE(prepareResult.has_value()); @@ -370,10 +406,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .doCleanup = false, - }}; + } + }; auto prepareResult = operation.prepare(); ASSERT_TRUE(prepareResult.has_value()); @@ -400,10 +437,49 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .tryContinue = true, + } + }; + + DownloadOperation::WorkStatus workStatus; + for (int workTimes = 0; workTimes < 100; ++workTimes) + { + auto workResult = operation.work(); + ASSERT_TRUE(workResult.has_value()); + workStatus = workResult.value(); + if (workStatus == DownloadOperation::WorkStatus::Complete) + break; + } + ASSERT_EQ(workStatus, DownloadOperation::WorkStatus::Complete); + + ASSERT_TRUE(std::filesystem::exists(localPath)); + EXPECT_EQ(readLocalFile(localPath), "Fake file content"); + } + + TEST_F(DownloadOperationTests, PartialFileOfAnotherSourceIsNotContinued) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = downloadDir_.path() / "file1.txt"; + const auto filepartPath = localPath.generic_string() + ".filepart"; + + // 8 bytes that are not the start of "Fake file content": a leftover of another program or version. + { + std::ofstream partial{filepartPath, std::ios::binary}; + partial << "Wrong fi"; + } + + DownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .tryContinue = true, - }}; + } + }; DownloadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -437,10 +513,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .tryContinue = true, - }}; + } + }; DownloadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -467,9 +544,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/large.txt", - .localPath = localPath, - }}; + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + } + }; // large.txt is 1 MiB: needs ~64 work() calls with a 16384-byte buffer DownloadOperation::WorkStatus workStatus; @@ -525,9 +603,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/link_to_file1.txt", - .localPath = localPath, - }}; + .remotePath = remote("/home/test/link_to_file1.txt"), + .localPath = toLocal(localPath), + } + }; EXPECT_EQ(runDownloadToCompletion(operation), DownloadOperation::WorkStatus::Complete); @@ -547,9 +626,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/link_to_documents", - .localPath = localPath, - }}; + .remotePath = remote("/home/test/link_to_documents"), + .localPath = toLocal(localPath), + } + }; EXPECT_EQ(runDownloadToCompletion(operation), DownloadOperation::WorkStatus::Complete); @@ -568,10 +648,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/link_to_file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/link_to_file1.txt"), + .localPath = toLocal(localPath), .symlinkHandling = Persistence::SymlinkHandling::SkipSymlink, - }}; + } + }; EXPECT_EQ(runDownloadToCompletion(operation), DownloadOperation::WorkStatus::Complete); @@ -600,10 +681,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .createMissingDirectories = true, - }}; + } + }; EXPECT_EQ(runDownloadToCompletion(operation), DownloadOperation::WorkStatus::Complete); ASSERT_TRUE(std::filesystem::exists(localPath)); @@ -621,10 +703,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .createMissingDirectories = true, - }}; + } + }; EXPECT_EQ(runDownloadToCompletion(operation), DownloadOperation::WorkStatus::Complete); EXPECT_TRUE(std::filesystem::is_directory(downloadDir_.path() / "a")); @@ -645,10 +728,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .createMissingDirectories = false, - }}; + } + }; auto first = operation.work(); // The error can surface on any iteration before completion. @@ -670,10 +754,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), .createMissingDirectories = true, - }}; + } + }; EXPECT_EQ(runDownloadToCompletion(operation), DownloadOperation::WorkStatus::Complete); EXPECT_EQ(readLocalFile(localPath), "Fake file content"); @@ -689,10 +774,11 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/link_to_file1.txt", - .localPath = localPath, + .remotePath = remote("/home/test/link_to_file1.txt"), + .localPath = toLocal(localPath), .createMissingDirectories = true, - }}; + } + }; EXPECT_EQ(runDownloadToCompletion(operation), DownloadOperation::WorkStatus::Complete); EXPECT_TRUE(std::filesystem::is_symlink(std::filesystem::symlink_status(localPath))); @@ -711,9 +797,10 @@ namespace Test DownloadOperation operation{ *sftp, { - .remotePath = "/home/test/dangling_link", - .localPath = localPath, - }}; + .remotePath = remote("/home/test/dangling_link"), + .localPath = toLocal(localPath), + } + }; EXPECT_EQ(runDownloadToCompletion(operation), DownloadOperation::WorkStatus::Complete); EXPECT_TRUE(std::filesystem::is_symlink(std::filesystem::symlink_status(localPath))); diff --git a/backend/test/backend/test_existence_check.hpp b/backend/test/backend/test_existence_check.hpp new file mode 100644 index 00000000..a0aec4f6 --- /dev/null +++ b/backend/test/backend/test_existence_check.hpp @@ -0,0 +1,157 @@ +#pragma once + +#include +#include "real_server_tests.hpp" + +#include +#include + +#include + +#ifndef _WIN32 +# include +#endif + +#include +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class ExistenceCheckTests : public RealServerTests + { + protected: + static constexpr std::chrono::milliseconds timeout{10'000}; + + std::string localPath(std::filesystem::path const& relative) const + { + return Paths::nativeToUtf8(root_.path() / relative); + } + + Utility::TemporaryDirectory root_{temporaryRoot, false}; + }; + + TEST_F(ExistenceCheckTests, LocalMissingPathIsAbsentAndExistingPathsExist) + { + std::ofstream{root_.path() / "file.txt"} << "content"; + std::filesystem::create_directory(root_.path() / "directory"); + + const auto result = + localPathsExist({localPath("file.txt"), localPath("directory"), localPath("missing.txt"), localPath("no/x")}); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(*result, (std::vector{true, true, false, false})); + } + + TEST_F(ExistenceCheckTests, LocalDanglingSymlinkExists) + { + std::error_code errorCode{}; + std::filesystem::create_symlink(root_.path() / "nowhere.txt", root_.path() / "dangling", errorCode); + if (errorCode) + GTEST_SKIP() << "cannot create symlinks here: " << errorCode.message(); + + const auto result = localPathsExist({localPath("dangling")}); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(*result, (std::vector{true})); + } + +#ifndef _WIN32 + TEST_F(ExistenceCheckTests, LocalPathThatCannotBeCheckedFailsTheCheck) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root can look into any directory"; + const auto locked = root_.path() / "locked"; + std::filesystem::create_directory(locked); + std::ofstream{locked / "inside.txt"} << "content"; + std::filesystem::permissions(locked, std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[&locked]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(locked, std::filesystem::perms::owner_all, errorCode); + }}; + + const auto result = localPathsExist({localPath("missing.txt"), localPath("locked/inside.txt")}); + + EXPECT_FALSE(result.has_value()) << "an unknown destination must never count as absent"; + } +#endif + + TEST_F(ExistenceCheckTests, RemoteFewPathsAreCheckedOneByOne) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto result = remotePathsExist( + *sftp, + {"/home/test/file1.txt", "/home/test/dangling_link", "/home/test/missing.txt", "/home/test/no/x"}, + timeout + ); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(*result, (std::vector{true, true, false, false})); + } + + TEST_F(ExistenceCheckTests, RemoteStatFailureFailsTheCheck) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + ASSERT_TRUE(serverStartResult->command("denyStat", {{"path", "/home/test/file2.txt"}})); + + const auto result = remotePathsExist(*sftp, {"/home/test/missing.txt", "/home/test/file2.txt"}, timeout); + + EXPECT_FALSE(result.has_value()) << "an unknown destination must never count as absent"; + } + + TEST_F(ExistenceCheckTests, RemoteManyPathsAreCheckedByListing) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + std::vector paths{ + "/home/test/file1.txt", + "/home/test/dangling_link", + "/home/test/Documents", + "/home/test/Documents/doc1.txt", + "/home/test/missing/x.txt", + }; + std::vector expected{true, true, true, true, false}; + for (int index = 0; index < 8; ++index) + { + paths.push_back("/home/test/missing" + std::to_string(index) + ".txt"); + expected.push_back(false); + } + + const auto result = remotePathsExist(*sftp, paths, timeout); + + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(*result, expected); + } + + TEST_F(ExistenceCheckTests, RemoteListingFailureFailsTheCheck) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); + + std::vector paths; + for (int index = 0; index < 12; ++index) + paths.push_back("/home/test/Documents/new" + std::to_string(index) + ".txt"); + + const auto result = remotePathsExist(*sftp, paths, timeout); + + EXPECT_FALSE(result.has_value()) << "an unknown destination must never count as absent"; + } + + TEST_F(ExistenceCheckTests, RemoteInvalidPathFailsTheCheck) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + EXPECT_FALSE(remotePathsExist(*sftp, {"relative.txt"}, timeout).has_value()); + EXPECT_FALSE(remotePathsExist(*sftp, {"/home/test/../test/file1.txt"}, timeout).has_value()); + } +} diff --git a/backend/test/backend/test_file_tracking.hpp b/backend/test/backend/test_file_tracking.hpp index ddba7d0d..6ba8d6ac 100644 --- a/backend/test/backend/test_file_tracking.hpp +++ b/backend/test/backend/test_file_tracking.hpp @@ -23,6 +23,7 @@ using namespace FileTracking; using namespace std::chrono_literals; extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace Test { @@ -41,7 +42,7 @@ namespace Test inline fs::path makeTestTempRoot(std::string const& tag = "base") { auto suffix = std::to_string(std::chrono::system_clock::now().time_since_epoch().count()); - auto root = programDirectory / "temp" / ("nui_sftp_ft_" + tag + "_" + suffix); + auto root = temporaryRoot / ("nui_sftp_ft_" + tag + "_" + suffix); fs::create_directories(root); return root; } @@ -367,8 +368,7 @@ namespace Test metaPath = inst->instanceDir() / "metadata.json"; } ASSERT_TRUE(fs::exists(metaPath)); - std::ifstream fin{metaPath}; - auto meta = nlohmann::json::parse(fin); + auto const meta = nlohmann::json::parse(std::ifstream{metaPath}); ASSERT_TRUE(meta.contains("deadAt")); EXPECT_FALSE(meta["deadAt"].is_null()); fs::remove_all(metaPath.parent_path()); diff --git a/backend/test/backend/test_fork_pool.hpp b/backend/test/backend/test_fork_pool.hpp index cdc49a3b..6013b505 100644 --- a/backend/test/backend/test_fork_pool.hpp +++ b/backend/test/backend/test_fork_pool.hpp @@ -23,6 +23,7 @@ # include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace Test { @@ -34,7 +35,7 @@ namespace Test // The worker logs below %state_home2%, keep that inside the build directory. if (auto const* previous = std::getenv("XDG_STATE_HOME")) previousStateHome_ = previous; - ::setenv("XDG_STATE_HOME", (programDirectory / "temp" / "fork_pool_state").c_str(), 1); + ::setenv("XDG_STATE_HOME", (temporaryRoot / "fork_pool_state").c_str(), 1); ioThread_ = std::thread{[this]() { diff --git a/backend/test/backend/test_local_delete_guard.hpp b/backend/test/backend/test_local_delete_guard.hpp new file mode 100644 index 00000000..6dbabe27 --- /dev/null +++ b/backend/test/backend/test_local_delete_guard.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include + +namespace Test +{ + class LocalDeleteGuardTests : public ::testing::Test + { + protected: + // POSIX rules on every host, the Windows rules are covered by the paths tests. + void SetUp() override + { + previous_ = Paths::defaultLocalPlatform(); + Paths::defaultLocalPlatform() = Paths::LocalPlatform::Posix; + } + void TearDown() override + { + Paths::defaultLocalPlatform() = previous_; + } + + private: + Paths::LocalPlatform previous_{}; + }; + + TEST_F(LocalDeleteGuardTests, RefusesRootsHomeAndItsAncestors) + { + const auto home = Paths::LocalPath::fromUtf8("/home/user", Paths::LocalPlatform::Posix); + ASSERT_TRUE(home.has_value()); + const auto check = [&home](std::string const& path) + { + return localDeleteTarget(path, *home).has_value(); + }; + EXPECT_FALSE(check("/")); + EXPECT_FALSE(check("/home")); + EXPECT_FALSE(check("/home/user")); + EXPECT_FALSE(check("/home/user/../other")); + EXPECT_FALSE(check("relative/path")); + EXPECT_FALSE(check("")); + EXPECT_TRUE(check("/home/user/file.txt")); + EXPECT_TRUE(check("/home/username")); + EXPECT_TRUE(check("/tmp/x")); + } + + TEST_F(LocalDeleteGuardTests, WindowsRootsHomeAndBackslashParentsAreRefused) + { + Paths::defaultLocalPlatform() = Paths::LocalPlatform::Windows; + const auto home = Paths::LocalPath::fromUtf8("C:/Users/me"); + ASSERT_TRUE(home.has_value()); + const auto check = [&home](std::string const& path) + { + return localDeleteTarget(path, *home).has_value(); + }; + EXPECT_FALSE(check("C:/")); + EXPECT_FALSE(check("C:/Users")); + EXPECT_FALSE(check("C:\\Users\\me")); + EXPECT_FALSE(check("C:\\Users\\me\\..\\other")); + EXPECT_FALSE(check("/no/drive")); + EXPECT_TRUE(check("C:/Users/me/file.txt")); + EXPECT_TRUE(check("D:/data")); + } + + TEST_F(LocalDeleteGuardTests, WithoutAKnownHomeOnlyRootsAreRefused) + { + EXPECT_FALSE(localDeleteTarget("/", std::nullopt).has_value()); + EXPECT_TRUE(localDeleteTarget("/home/user", std::nullopt).has_value()); + } +} diff --git a/backend/test/backend/test_local_scan_operation.hpp b/backend/test/backend/test_local_scan_operation.hpp index b84d4edf..5c524ac9 100644 --- a/backend/test/backend/test_local_scan_operation.hpp +++ b/backend/test/backend/test_local_scan_operation.hpp @@ -3,15 +3,27 @@ #include #include "real_server_tests.hpp" +#include +#include #include +#include + #include +#include #include #include #include #include #include +extern std::filesystem::path temporaryRoot; + +#ifndef _WIN32 +# include +# include +#endif + namespace Test { class LocalScanOperationTests : public ::testing::Test @@ -61,7 +73,7 @@ namespace Test std::string const& filename) { return std::any_of(entries.begin(), entries.end(), [&](SharedData::DirectoryEntry const& entry) { - return entry.path.filename().generic_string() == filename; + return entry.name == filename; }); } @@ -70,29 +82,29 @@ namespace Test std::string const& filename) { auto iter = std::find_if(entries.begin(), entries.end(), [&](SharedData::DirectoryEntry const& entry) { - return entry.path.filename().generic_string() == filename; + return entry.name == filename; }); return iter == entries.end() ? nullptr : &*iter; } protected: - Utility::TemporaryDirectory scanRoot_{programDirectory / "temp", true}; + Utility::TemporaryDirectory scanRoot_{temporaryRoot, true}; }; TEST_F(LocalScanOperationTests, CanCreateLocalScanOperation) { - LocalScanOperation operation{{.localPath = scanRoot_.path()}}; + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path())}}; } TEST_F(LocalScanOperationTests, LocalScanOperationTypeIsLocalScan) { - LocalScanOperation operation{{.localPath = scanRoot_.path()}}; + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path())}}; EXPECT_EQ(operation.type(), SharedData::OperationType::LocalScan); } TEST_F(LocalScanOperationTests, LocalScanOperationIsBarrier) { - LocalScanOperation operation{{.localPath = scanRoot_.path()}}; + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path())}}; EXPECT_TRUE(operation.isBarrier()); } @@ -100,7 +112,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path(), + .localPath = toLocal(scanRoot_.path()), .recursive = false, }}; @@ -116,7 +128,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path(), + .localPath = toLocal(scanRoot_.path()), .recursive = false, }}; @@ -132,7 +144,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path(), + .localPath = toLocal(scanRoot_.path()), .recursive = true, }}; @@ -150,7 +162,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path(), + .localPath = toLocal(scanRoot_.path()), .recursive = false, }}; @@ -173,7 +185,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path() / "subdir", + .localPath = toLocal(scanRoot_.path() / "subdir"), .recursive = true, }}; @@ -194,7 +206,7 @@ namespace Test [&callbackInvocations](std::uint64_t, std::uint64_t, std::uint64_t) { ++callbackInvocations; }, - .localPath = scanRoot_.path(), + .localPath = toLocal(scanRoot_.path()), .recursive = false, }}; @@ -206,7 +218,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path(), + .localPath = toLocal(scanRoot_.path()), .recursive = false, .ignoreHidden = true, }}; @@ -223,7 +235,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path(), + .localPath = toLocal(scanRoot_.path()), .recursive = false, .ignoreHidden = false, }}; @@ -239,7 +251,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path() / "does_not_exist", + .localPath = toLocal(scanRoot_.path() / "does_not_exist"), .recursive = false, }}; @@ -272,7 +284,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path() / "file1.txt", + .localPath = toLocal(scanRoot_.path() / "file1.txt"), .recursive = false, }}; @@ -304,7 +316,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path(), + .localPath = toLocal(scanRoot_.path()), .recursive = false, }}; @@ -319,7 +331,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path() / "does_not_exist", + .localPath = toLocal(scanRoot_.path() / "does_not_exist"), .recursive = false, }}; @@ -346,7 +358,7 @@ namespace Test { LocalScanOperation operation{ { - .localPath = scanRoot_.path(), + .localPath = toLocal(scanRoot_.path()), .recursive = true, }}; @@ -362,13 +374,13 @@ namespace Test TEST_F(LocalScanOperationTests, ScanStrandIsNullForLocalScan) { - LocalScanOperation operation{{.localPath = scanRoot_.path()}}; + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path())}}; EXPECT_EQ(operation.strand(), nullptr); } TEST_F(LocalScanOperationTests, BuildTreeNestsAndSortsEntries) { - LocalScanOperation operation{{.localPath = scanRoot_.path(), .recursive = true, .buildTree = true}}; + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = true, .buildTree = true}}; ASSERT_EQ(runScanToCompletion(operation), LocalScanOperation::WorkStatus::Complete); const auto tree = operation.ejectScanTree(); @@ -383,9 +395,31 @@ namespace Test EXPECT_EQ(subdir.subtreeByteTotal, std::string_view{"nested content"}.size() + std::string_view{"leaf"}.size()); } + TEST_F(LocalScanOperationTests, BuildTreeExcludesTransferTempFiles) + { + std::ofstream{scanRoot_.path() / "big.iso.filepart"} << "partial"; + std::ofstream{scanRoot_.path() / ".filepart"} << "a name, not a temp file"; + + LocalScanOperation operation{{ + .localPath = toLocal(scanRoot_.path()), + .recursive = true, + .buildTree = true, + .excludedSuffixes = {".filepart"}, + }}; + ASSERT_EQ(runScanToCompletion(operation), LocalScanOperation::WorkStatus::Complete); + + const auto tree = operation.ejectScanTree(); + auto const* temp = SharedData::Sync::findScanNode(tree, "big.iso.filepart"); + ASSERT_NE(temp, nullptr); + EXPECT_EQ(temp->availability, SharedData::Sync::Availability::Excluded); + auto const* bare = SharedData::Sync::findScanNode(tree, ".filepart"); + ASSERT_NE(bare, nullptr); + EXPECT_EQ(bare->availability, SharedData::Sync::Availability::Listed); + } + TEST_F(LocalScanOperationTests, NonRecursiveBuildTreeMarksSubdirectoriesUnknown) { - LocalScanOperation operation{{.localPath = scanRoot_.path(), .recursive = false, .buildTree = true}}; + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = false, .buildTree = true}}; ASSERT_EQ(runScanToCompletion(operation), LocalScanOperation::WorkStatus::Complete); const auto tree = operation.ejectScanTree(); @@ -406,7 +440,7 @@ namespace Test { writeFile(scanRoot_.path() / ".gitignore", "*.md\ndeeper/\n"); LocalScanOperation operation{ - {.localPath = scanRoot_.path(), + {.localPath = toLocal(scanRoot_.path()), .respectIgnoreFiles = true, .recursive = true, .ignoreHidden = true, @@ -422,4 +456,110 @@ namespace Test ASSERT_EQ(tree.children.back().children.size(), 1u); EXPECT_EQ(tree.children.back().children.front().name, "nested.txt"); } + +#ifndef _WIN32 + TEST_F(LocalScanOperationTests, BuildTreeKeepsUnreadableDirectoryAsUnreadable) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root can list any directory"; + const auto locked = scanRoot_.path() / "subdir" / "deeper"; + std::filesystem::permissions(locked, std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[&locked]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(locked, std::filesystem::perms::owner_all, errorCode); + }}; + + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = true, .buildTree = true}}; + ASSERT_EQ(runScanToCompletion(operation), LocalScanOperation::WorkStatus::Complete); + + const auto tree = operation.ejectScanTree(); + auto const* deeper = SharedData::Sync::findScanNode(tree, "subdir/deeper"); + ASSERT_NE(deeper, nullptr); + EXPECT_EQ(deeper->availability, SharedData::Sync::Availability::Unreadable); + EXPECT_FALSE(deeper->childrenKnown) << "unknown content must never look empty"; + } + + TEST_F(LocalScanOperationTests, FlatScanReportsAnUnreadableDirectory) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root can list any directory"; + const auto locked = scanRoot_.path() / "subdir" / "deeper"; + std::filesystem::permissions(locked, std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[&locked]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(locked, std::filesystem::perms::owner_all, errorCode); + }}; + + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = true}}; + ASSERT_EQ(runScanToCompletion(operation), LocalScanOperation::WorkStatus::Complete); + EXPECT_EQ(operation.unreadableDirectories(), (std::vector{Paths::nativeToUtf8(locked)})); + } + + TEST_F(LocalScanOperationTests, UnreadableRootFailsTheScan) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root can list any directory"; + const auto locked = scanRoot_.path() / "subdir"; + std::filesystem::permissions(locked, std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[&locked]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(locked, std::filesystem::perms::owner_all, errorCode); + }}; + + LocalScanOperation operation{{.localPath = toLocal(locked), .recursive = true, .buildTree = true}}; + std::expected result{}; + for (int iteration = 0; iteration < 10 && result.has_value() && + result.value() != LocalScanOperation::WorkStatus::Complete; + ++iteration) + result = operation.work(); + EXPECT_FALSE(result.has_value()); + } + + TEST_F(LocalScanOperationTests, BuildTreeKeepsFifoAsExcludedPlaceholder) + { + const auto fifo = scanRoot_.path() / "pipe"; + ASSERT_EQ(::mkfifo(fifo.c_str(), 0600), 0); + + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = true, .buildTree = true}}; + ASSERT_EQ(runScanToCompletion(operation), LocalScanOperation::WorkStatus::Complete); + const auto tree = operation.ejectScanTree(); + auto const* pipe = SharedData::Sync::findScanNode(tree, "pipe"); + ASSERT_NE(pipe, nullptr); + EXPECT_EQ(pipe->availability, SharedData::Sync::Availability::Excluded); + + LocalScanOperation flat{{.localPath = toLocal(scanRoot_.path()), .recursive = true}}; + ASSERT_EQ(runScanToCompletion(flat), LocalScanOperation::WorkStatus::Complete); + EXPECT_FALSE(hasEntryNamed(flat.ejectEntries(), "pipe")) << "transfers never see untransferable entries"; + } +#endif + +#ifdef _WIN32 + TEST_F(LocalScanOperationTests, JunctionIsExcludedAndNotWalked) + { + const auto outside = scanRoot_.path().parent_path() / ("junction_target_" + scanRoot_.path().filename().string()); + std::filesystem::create_directories(outside); + std::ofstream{outside / "inside.txt"} << "inside"; + auto removeOutside = Nui::ScopeExit{[&outside]() noexcept { + std::error_code errorCode{}; + std::filesystem::remove_all(outside, errorCode); + }}; + const auto link = scanRoot_.path() / "linked"; + const auto command = "cmd /c mklink /J \"" + link.string() + "\" \"" + outside.string() + "\" >nul"; + ASSERT_EQ(std::system(command.c_str()), 0); + + LocalScanOperation operation{{.localPath = toLocal(scanRoot_.path()), .recursive = true, .buildTree = true}}; + ASSERT_EQ(runScanToCompletion(operation), LocalScanOperation::WorkStatus::Complete); + const auto tree = operation.ejectScanTree(); + auto const* linked = SharedData::Sync::findScanNode(tree, "linked"); + ASSERT_NE(linked, nullptr); + EXPECT_EQ(linked->availability, SharedData::Sync::Availability::Excluded); + EXPECT_EQ(SharedData::Sync::findScanNode(tree, "linked/inside.txt"), nullptr) << "a junction is never walked"; + + LocalScanOperation flat{{.localPath = toLocal(scanRoot_.path()), .recursive = true}}; + ASSERT_EQ(runScanToCompletion(flat), LocalScanOperation::WorkStatus::Complete); + const auto entries = flat.ejectEntries(); + EXPECT_FALSE(hasEntryNamed(entries, "linked")); + EXPECT_FALSE(hasEntryNamed(entries, "inside.txt")) << "an upload never leaves its tree through a junction"; + } +#endif } diff --git a/backend/test/backend/test_opener_risk.hpp b/backend/test/backend/test_opener_risk.hpp new file mode 100644 index 00000000..fb2b6043 --- /dev/null +++ b/backend/test/backend/test_opener_risk.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include +#include + +#include + +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class OpenerRiskTests : public ::testing::Test + { + protected: + std::filesystem::path write(std::string const& name) + { + const auto path = directory_.path() / name; + std::ofstream{path} << "content"; + return path; + } + + Opener opener_{}; + Utility::TemporaryDirectory directory_{temporaryRoot, true}; + }; + + TEST_F(OpenerRiskTests, DocumentsAndFoldersAreNoRisk) + { + EXPECT_FALSE(opener_.riskOf(write("notes.txt"))); + EXPECT_FALSE(opener_.riskOf(directory_.path())); + } + +#ifdef _WIN32 + TEST_F(OpenerRiskTests, ProgramsAndScriptsAreRisks) + { + EXPECT_EQ(opener_.riskOf(write("run.bat")), "backend.opener.risk.dangerousType"); + EXPECT_EQ(opener_.riskOf(write("run.vbs")), "backend.opener.risk.dangerousType"); + // Judged by content, a renamed program is still one. + const auto program = directory_.path() / "renamed.txt"; + std::filesystem::copy_file("C:\\Windows\\System32\\whoami.exe", program); + EXPECT_EQ(opener_.riskOf(program), "backend.opener.risk.program"); + } +#else + TEST_F(OpenerRiskTests, ProgramsAndScriptsAreRisks) + { + const auto script = write("tool.txt"); + std::filesystem::permissions(script, std::filesystem::perms::owner_exec, std::filesystem::perm_options::add); + EXPECT_EQ(opener_.riskOf(script), "backend.opener.risk.program") << "the execute bit counts, not the name"; +# ifdef __APPLE__ + EXPECT_EQ(opener_.riskOf(write("start.command")), "backend.opener.risk.dangerousType"); +# else + EXPECT_EQ(opener_.riskOf(write("start.desktop")), "backend.opener.risk.dangerousType"); +# endif + } +#endif +} diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp new file mode 100644 index 00000000..790f255a --- /dev/null +++ b/backend/test/backend/test_operation_queue.hpp @@ -0,0 +1,1392 @@ +#pragma once + +#include +#include +#include "nui_env.hpp" +#include "real_server_tests.hpp" + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +# include +#endif + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class OperationQueueTests : public RealServerTests + { + public: + void SetUp() override + { + if (!NuiEnv::instance().available()) + GTEST_SKIP() << "No display available; skipping OperationQueue tests"; + RealServerTests::SetUp(); + } + + void TearDown() override + { + if (queue_) + runOnStrand([this]() { + queue_->cancelAll(); + }); + queue_.reset(); + } + + std::shared_ptr makeQueue() + { + auto& env = NuiEnv::instance(); + queue_ = std::make_shared( + queuePool_.get_executor(), + strand_, + *env.window, + *env.hub, + Persistence::SftpOptions{}, + Ids::makeSessionId("operation-queue-test"), + 2 + ); + queue_->setCompletionObserver( + [this](SharedData::OperationCompleted const& completed) + { + std::scoped_lock lock{completionsMutex_}; + completions_.push_back(completed); + } + ); + return queue_; + } + + /** + * @brief Runs @p function on the queue's strand and waits for it, so it orders with the + * tasks the queue posts there itself. + */ + template + auto runOnStrand(FunctionT&& function) -> std::invoke_result_t + { + using ResultT = std::invoke_result_t; + auto promise = std::make_shared>(); + auto future = promise->get_future(); + boost::asio::post( + *strand_, + [promise, function = std::forward(function)]() mutable + { + if constexpr (std::is_void_v) + { + function(); + promise->set_value(); + } + else + promise->set_value(function()); + } + ); + return future.get(); + } + + void setPaused(bool pause) + { + queue_->paused(pause); + runOnStrand([]() {}); + } + + /** + * @brief Works the queue until @p expectedCompletions completions arrived. + */ + void workUntilCompletions(std::size_t expectedCompletions) + { + const auto deadline = std::chrono::steady_clock::now() + 20s; + while (std::chrono::steady_clock::now() < deadline) + { + runOnStrand([this]() { + return queue_->work(); + }); + runOnStrand([]() {}); + { + std::scoped_lock lock{completionsMutex_}; + if (completions_.size() >= expectedCompletions) + return; + } + std::this_thread::sleep_for(2ms); + } + ADD_FAILURE() << "Timed out waiting for " << expectedCompletions << " completions"; + } + + std::vector completions() + { + std::scoped_lock lock{completionsMutex_}; + return completions_; + } + + static std::set remoteNames(SecureShell::SftpSession& sftp, std::filesystem::path const& directory) + { + auto future = sftp.listDirectory(remote(directory.generic_string())); + if (future.wait_for(5s) != std::future_status::ready) + return {""}; + auto result = future.get(); + if (!result.has_value()) + return {""}; + std::set names; + for (auto const& entry : *result) + { + const auto name = entry.name; + if (name != "." && name != "..") + names.insert(name); + } + return names; + } + + /** + * @brief Hands the queue's backups to @p registry and drops its operations, as a session teardown does. + */ + void tearDownInto(BulkResumeRegistry& registry) + { + runOnStrand( + [this, ®istry]() + { + queue_->dumpBulkResumes(registry); + queue_->cancelAll(); + } + ); + std::scoped_lock lock{completionsMutex_}; + completions_.clear(); + } + + /** + * @brief A running replacement queue that adopted the backups of @p operationIds. + */ + std::shared_ptr adoptInto( + SecureShell::SftpSession& sftp, + BulkResumeRegistry& registry, + std::vector const& operationIds + ) + { + auto replacement = makeQueue(); + setPaused(false); + runOnStrand( + [&]() + { + for (auto const& operationId : operationIds) + { + auto backup = registry.take(operationId); + EXPECT_TRUE(backup.has_value()) << operationId.value(); + if (backup) + replacement->adoptBulkResume(sftp, operationId, *std::move(backup)); + } + } + ); + return replacement; + } + + /** + * @brief The ids of all completions, after giving the queue a few more cycles to show extra ones. + */ + std::multiset completedIdsAfterSettling() + { + for (int cycle = 0; cycle < 50; ++cycle) + { + runOnStrand( + [this]() + { + return queue_->work(); + } + ); + std::this_thread::sleep_for(2ms); + } + std::multiset ids; + for (auto const& finished : completions()) + { + EXPECT_EQ(finished.reason, SharedData::OperationCompletionReason::Completed); + ids.insert(finished.operationId.value()); + } + return ids; + } + + static std::set localNames(std::filesystem::path const& directory) + { + std::set names; + std::error_code errorCode{}; + for (auto const& entry : std::filesystem::directory_iterator{directory, errorCode}) + names.insert(entry.path().filename().generic_string()); + return names; + } + + protected: + boost::asio::thread_pool queuePool_{2}; + std::shared_ptr> strand_{ + std::make_shared>(queuePool_.get_executor())}; + std::shared_ptr queue_{}; + std::mutex completionsMutex_{}; + std::vector completions_{}; + Utility::TemporaryDirectory localDirectory_{temporaryRoot, true}; + }; + + TEST_F(OperationQueueTests, DirectoryDownloadIsOneUnit) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto operationId = Ids::generateOperationId(); + const auto localPath = localDirectory_.path() / "Documents"; + const auto added = runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, operationId, toLocal(localPath), remote("/home/test/Documents"), false, false, false, false + ); + } + ); + ASSERT_TRUE(added.has_value()) << added.error().toString(); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, operationId); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(localNames(localPath), (std::set{"doc1.txt", "doc2.txt"})); + } + + TEST_F(OperationQueueTests, DirectoryUploadCompletionUsesFrontendId) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "toUpload"; + std::filesystem::create_directories(localPath); + std::ofstream{localPath / "a.txt"} << "a"; + std::ofstream{localPath / "b.txt"} << "b"; + + const auto operationId = Ids::generateOperationId(); + const auto added = runOnStrand( + [&]() + { + return queue->addUploadOperation( + *sftp, operationId, toLocal(localPath), remote("/home/test/uploaded"), false, false, false, false + ); + } + ); + ASSERT_TRUE(added.has_value()) << added.error().toString(); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, operationId); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(remoteNames(*sftp, "/home/test/uploaded"), (std::set{"a.txt", "b.txt"})); + } + +#ifndef _WIN32 + TEST_F(OperationQueueTests, DirectoryUploadReportsAnUnreadableSubdirectory) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root can list any directory"; + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "withLocked"; + std::filesystem::create_directories(localPath / "locked"); + std::ofstream{localPath / "a.txt"} << "a"; + std::ofstream{localPath / "locked" / "hidden.txt"} << "hidden"; + std::filesystem::permissions(localPath / "locked", std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[locked = localPath / "locked"]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(locked, std::filesystem::perms::owner_all, errorCode); + }}; + + const auto operationId = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addUploadOperation( + *sftp, operationId, toLocal(localPath), remote("/home/test/withLocked"), false, false, false, false + ); + }).has_value()); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + ASSERT_EQ(finished.front().failedEntries.size(), 1u) << "an unreadable directory is never a silent success"; + EXPECT_EQ(finished.front().failedEntries.front().first, Paths::nativeToUtf8(localPath / "locked")); + EXPECT_EQ(finished.front().failedEntries.front().second.type, SharedData::OperationErrorType::DirectoryUnreadable); + EXPECT_TRUE(remoteNames(*sftp, "/home/test/withLocked").contains("a.txt")); + } + + TEST_F(OperationQueueTests, ArchiveUploadOfAnUnreadableSubdirectoryFails) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root can list any directory"; + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "archiveLocked"; + std::filesystem::create_directories(localPath / "locked"); + std::ofstream{localPath / "a.txt"} << "a"; + std::filesystem::permissions(localPath / "locked", std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[locked = localPath / "locked"]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(locked, std::filesystem::perms::owner_all, errorCode); + }}; + + const auto operationId = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addArchiveUploadOperation( + *sftp, + operationId, + {toLocal(localPath)}, + remote("/home/test/locked.tar"), + TarArchive::Compression::None, + 0, + false + ); + }).has_value()); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Failed); + EXPECT_FALSE(remoteNames(*sftp, "/home/test").contains("locked.tar")); + } +#endif + + TEST_F(OperationQueueTests, DirectoryUploadCreatesDirectoriesWithTheDefaultMode) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "modes"; + std::filesystem::create_directories(localPath / "sub"); + std::ofstream{localPath / "sub" / "a.txt"} << "a"; + std::filesystem::permissions(localPath / "sub", std::filesystem::perms::all); + + const auto operationId = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addUploadOperation( + *sftp, operationId, toLocal(localPath), remote("/home/test/modes"), false, false, false, false + ); + }).has_value()); + workUntilCompletions(1); + ASSERT_EQ(completions().size(), 1u); + + using std::filesystem::perms; + const auto modeOf = [&](std::string const& path) { + auto future = sftp->stat(remote(path)); + if (future.wait_for(5s) != std::future_status::ready) + return perms::unknown; + const auto information = future.get(); + return information ? information->permissions & perms::mask : perms::unknown; + }; + // Not inherited by default: owner everything, group read and enter, never world writable. + const auto expected = perms::owner_all | perms::group_read | perms::group_exec; + EXPECT_EQ(modeOf("/home/test/modes"), expected); + EXPECT_EQ(modeOf("/home/test/modes/sub"), expected); + } + + TEST_F(OperationQueueTests, BulkDownloadQueuesFilesAsOneCardAndDirectoriesUnderTheirIds) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto root = localDirectory_.path(); + const SharedData::BulkDownloadRequest request{ + .entries = { + {.source = remote("/home/test/file1.txt"), + .destination = toLocal(root / "file1.txt"), + .sizeBytes = std::string{"Fake file content"}.size()}, + {.source = remote("/home/test/Pictures"), + .destination = toLocal(root / "Pictures"), + .isDirectory = true}, + }, + }; + const std::vector entryIds{Ids::generateOperationId(), Ids::generateOperationId()}; + const auto card = Ids::generateOperationId(); + EXPECT_EQ( + runOnStrand( + [&]() + { + return queue->addBulkDownloadOperation( + *sftp, + request, + [&entryIds](std::size_t index) + { + return entryIds[index]; + }, + card + ); + } + ), + 2u + ); + + workUntilCompletions(2); + std::set completed; + for (auto const& finished : completions()) + { + EXPECT_EQ(finished.reason, SharedData::OperationCompletionReason::Completed); + completed.insert(finished.operationId.value()); + } + EXPECT_EQ(completed, (std::set{card.value(), entryIds[1].value()})); + EXPECT_EQ(localNames(root), (std::set{"Pictures", "file1.txt"})); + EXPECT_EQ(localNames(root / "Pictures"), (std::set{"image1.png", "image2.jpg"})); + } + + TEST_F(OperationQueueTests, AdoptedBulkUploadIsQueuedAgainUnderItsCardId) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(true); + + const auto localPath = localDirectory_.path() / "toAdopt"; + std::filesystem::create_directories(localPath); + std::ofstream{localPath / "a.txt"} << "a"; + std::ofstream{localPath / "b.txt"} << "b"; + const SharedData::BulkUploadRequest request{ + .entries = { + {.source = toLocal(localPath / "a.txt"), + .destination = remote("/home/test/adopted_a.txt"), + .sizeBytes = 1}, + {.source = toLocal(localPath / "b.txt"), + .destination = remote("/home/test/adopted_b.txt"), + .sizeBytes = 1}, + }, + }; + const auto card = Ids::generateOperationId(); + runOnStrand( + [&]() + { + return queue->addBulkUploadOperation( + *sftp, + request, + [](std::size_t) + { + return Ids::generateOperationId(); + }, + card + ); + } + ); + + // The session goes away before the bulk ran, a replacement session adopts its backup. + BulkResumeRegistry registry; + runOnStrand( + [&]() + { + queue->dumpBulkResumes(registry); + queue->cancelAll(); + } + ); + auto backup = registry.take(card); + ASSERT_TRUE(backup.has_value()); + ASSERT_TRUE(std::holds_alternative(backup->request)); + { + std::scoped_lock lock{completionsMutex_}; + completions_.clear(); + } + + auto replacement = makeQueue(); + setPaused(false); + runOnStrand( + [&]() + { + replacement->adoptBulkResume(*sftp, card, *std::move(backup)); + } + ); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, card); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + const auto names = remoteNames(*sftp, "/home/test"); + EXPECT_TRUE(names.contains("adopted_a.txt")); + EXPECT_TRUE(names.contains("adopted_b.txt")); + } + + TEST_F(OperationQueueTests, QueuedRenameMovesTheFileButNeverReplacesTheDestination) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto moved = Ids::generateOperationId(); + const auto blocked = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addRenameOperation( + *sftp, moved, remote("/home/test/file1.txt"), remote("/home/test/moved.txt") + ); + } + ).has_value()); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addRenameOperation( + *sftp, blocked, remote("/home/test/Documents/doc2.txt"), remote("/home/test/Documents/doc1.txt") + ); + } + ).has_value()); + + workUntilCompletions(2); + std::map reasons; + for (auto const& finished : completions()) + reasons[finished.operationId.value()] = finished.reason; + EXPECT_EQ(reasons[moved.value()], SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(reasons[blocked.value()], SharedData::OperationCompletionReason::Failed); + const auto names = remoteNames(*sftp, "/home/test"); + EXPECT_TRUE(names.contains("moved.txt")); + EXPECT_FALSE(names.contains("file1.txt")); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Documents"), (std::set{"doc1.txt", "doc2.txt"})); + } + + TEST_F(OperationQueueTests, AdoptedDirectoryDownloadKeepsItsCardId) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + makeQueue(); + setPaused(true); + + const auto localPath = localDirectory_.path() / "Pictures"; + const auto operationId = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue_->addDownloadOperation( + *sftp, operationId, toLocal(localPath), remote("/home/test/Pictures"), false, false, false, false + ); + } + ).has_value()); + + BulkResumeRegistry registry; + tearDownInto(registry); + adoptInto(*sftp, registry, {operationId}); + + workUntilCompletions(1); + EXPECT_EQ(completedIdsAfterSettling(), (std::multiset{operationId.value()})); + EXPECT_EQ(localNames(localPath), (std::set{"image1.png", "image2.jpg"})); + } + + TEST_F(OperationQueueTests, MixedBulkUploadResumesEachCardOnce) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + makeQueue(); + setPaused(true); + + const auto localPath = localDirectory_.path() / "mixed"; + std::filesystem::create_directories(localPath / "folder"); + std::ofstream{localPath / "loose.txt"} << "loose"; + std::ofstream{localPath / "folder" / "inner.txt"} << "inner"; + const SharedData::BulkUploadRequest request{ + .entries = { + {.source = toLocal(localPath / "loose.txt"), + .destination = remote("/home/test/mixed_loose.txt"), + .sizeBytes = 5}, + {.source = toLocal(localPath / "folder"), + .destination = remote("/home/test/mixed_folder"), + .isDirectory = true}, + }, + }; + const std::vector entryIds{Ids::generateOperationId(), Ids::generateOperationId()}; + const auto card = Ids::generateOperationId(); + runOnStrand( + [&]() + { + return queue_->addBulkUploadOperation( + *sftp, + request, + [&entryIds](std::size_t index) + { + return entryIds[index]; + }, + card + ); + } + ); + + // One backup per card: the files under the bulk card, the folder under its own id. + BulkResumeRegistry registry; + tearDownInto(registry); + EXPECT_EQ(registry.size(), 2u); + adoptInto(*sftp, registry, {card, entryIds[1]}); + + workUntilCompletions(2); + EXPECT_EQ(completedIdsAfterSettling(), (std::multiset{card.value(), entryIds[1].value()})); + EXPECT_TRUE(remoteNames(*sftp, "/home/test").contains("mixed_loose.txt")); + EXPECT_EQ(remoteNames(*sftp, "/home/test/mixed_folder"), (std::set{"inner.txt"})); + } + + TEST_F(OperationQueueTests, CancelledRecursiveDeleteDuringScanDeletesNothing) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/victim"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/victim/keep.txt"}, {"content", "keep"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto directoryDelete = Ids::generateOperationId(); + const auto fileDelete = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, directoryDelete, remote("/home/test/victim"), true, false); + }).has_value()); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, fileDelete, remote("/home/test/file2.txt"), false, false); + }).has_value()); + + // Starts the directory delete's scan, then cancels the unit while it scans. + runOnStrand([&]() { + return queue->work(); + }); + queue->cancel(directoryDelete); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, fileDelete); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(remoteNames(*sftp, "/home/test/victim"), (std::set{"keep.txt"})); + EXPECT_FALSE(remoteNames(*sftp, "/home/test").contains("file2.txt")); + } + + TEST_F(OperationQueueTests, BulkDeleteOfDirectoriesAndFilesIsOneUnit) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/victim"}})); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/victim/inner"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/victim/inner/a.txt"}, {"content", "a"}})); + ASSERT_TRUE( + serverStartResult->command("createSymlink", {{"path", "/home/test/linkdir"}, {"target", "/home/test/Documents"}}) + ); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto bulk = Ids::generateOperationId(); + const SharedData::BulkDeleteRequest request{ + .entries = + { + {.target = remote("/home/test/victim"), .isDirectory = true}, + {.target = remote("/home/test/linkdir"), .isDirectory = true}, + {.target = remote("/home/test/file2.txt")}, + }, + }; + EXPECT_EQ(runOnStrand([&]() { + return queue->addBulkDeleteOperation(*sftp, request, bulk); + }), 3u); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, bulk); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + const auto home = remoteNames(*sftp, "/home/test"); + EXPECT_FALSE(home.contains("victim")); + EXPECT_FALSE(home.contains("linkdir")); + EXPECT_FALSE(home.contains("file2.txt")); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Documents"), (std::set{"doc1.txt", "doc2.txt"})); + } + + TEST_F(OperationQueueTests, BulkUploadReplacesOnlyApprovedDestinations) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "approval"; + std::filesystem::create_directories(localPath); + std::ofstream{localPath / "a.txt"} << "a"; + std::ofstream{localPath / "b.txt"} << "b"; + // file1.txt appeared after the existence check, file2.txt existed and the user agreed to replace it. + const SharedData::BulkUploadRequest request{ + .entries = + { + {.source = toLocal(localPath / "a.txt"), .destination = remote("/home/test/file1.txt"), .sizeBytes = 1}, + {.source = toLocal(localPath / "b.txt"), + .destination = remote("/home/test/file2.txt"), + .sizeBytes = 1, + .mayOverwrite = true}, + }, + }; + const auto card = Ids::generateOperationId(); + runOnStrand([&]() { + return queue->addBulkUploadOperation( + *sftp, request, [](std::size_t) { return Ids::generateOperationId(); }, card + ); + }); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + ASSERT_EQ(finished.front().failedEntries.size(), 1u); + EXPECT_EQ(finished.front().failedEntries.front().second.type, SharedData::OperationErrorType::FileExists); + const auto sizeOf = [&](std::string const& path) -> std::uint64_t { + auto future = sftp->stat(remote(path)); + if (future.wait_for(5s) != std::future_status::ready) + return 0; + const auto information = future.get(); + return information ? information->size : 0; + }; + EXPECT_EQ(sizeOf("/home/test/file1.txt"), std::string{"Fake file content"}.size()) << "not approved, kept"; + EXPECT_EQ(sizeOf("/home/test/file2.txt"), 1u) << "approved, replaced"; + } + + TEST_F(OperationQueueTests, FailedUploadReportsItsError) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localFile = localDirectory_.path() / "refused.txt"; + std::ofstream{localFile} << "refused"; + const auto upload = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addUploadOperation( + *sftp, upload, toLocal(localFile), remote("/home/test/file1.txt"), false, false, false, false + ); + }).has_value()); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Failed); + ASSERT_TRUE(finished.front().error.has_value()) << "the card has to say why"; + EXPECT_EQ(finished.front().error->type, SharedData::OperationErrorType::FileExists); + } + + TEST_F(OperationQueueTests, BulkDeleteOfAnAlreadyDeletedItemSucceeds) + { + // A resumed bulk delete meets the items it deleted before the interruption. + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto bulk = Ids::generateOperationId(); + const SharedData::BulkDeleteRequest request{ + .entries = {{.target = remote("/home/test/gone.txt")}, {.target = remote("/home/test/file2.txt")}}, + }; + EXPECT_EQ(runOnStrand([&]() { return queue->addBulkDeleteOperation(*sftp, request, bulk); }), 2u); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(finished.front().failedEntries.empty()); + EXPECT_FALSE(remoteNames(*sftp, "/home/test").contains("file2.txt")); + } + + TEST_F(OperationQueueTests, NonRecursiveDeleteKeepsANonEmptyDirectory) + { + // File tracking mirrors local deletions this way: a deleted local file never takes a remote directory along. + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, remove, remote("/home/test/Documents"), false, false); + }).has_value()); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Failed); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Documents"), (std::set{"doc1.txt", "doc2.txt"})); + } + + TEST_F(OperationQueueTests, BulkDeleteOfListedContentKeepsUnlistedItems) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/victim"}})); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/victim/inner"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/victim/listed.txt"}, {"content", "a"}})); + ASSERT_TRUE( + serverStartResult->command("createFile", {{"path", "/home/test/victim/inner/unlisted.txt"}, {"content", "b"}}) + ); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto bulk = Ids::generateOperationId(); + const SharedData::BulkDeleteRequest request{ + .entries = + { + {.target = remote("/home/test/victim"), .isDirectory = true}, + {.target = remote("/home/test/victim/inner"), .isDirectory = true}, + {.target = remote("/home/test/victim/listed.txt")}, + {.target = remote("/home/test/file2.txt")}, + }, + .listedContentOnly = true, + }; + EXPECT_EQ(runOnStrand([&]() { + return queue->addBulkDeleteOperation(*sftp, request, bulk); + }), 4u); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + std::set failed; + for (auto const& [path, error] : finished.front().failedEntries) + failed.insert(path); + EXPECT_EQ(failed, (std::set{"/home/test/victim", "/home/test/victim/inner"})); + EXPECT_EQ(remoteNames(*sftp, "/home/test/victim/inner"), (std::set{"unlisted.txt"})); + EXPECT_FALSE(remoteNames(*sftp, "/home/test/victim").contains("listed.txt")); + EXPECT_FALSE(remoteNames(*sftp, "/home/test").contains("file2.txt")) << "a failure does not stop the rest"; + } + + TEST_F(OperationQueueTests, LocalDeleteIsOneQueuedUnit) + { + const auto root = localDirectory_.path() / "localDelete"; + std::filesystem::create_directories(root / "directory" / "inner"); + std::ofstream{root / "directory" / "inner" / "a.txt"} << "a"; + std::ofstream{root / "file.txt"} << "file"; + std::ofstream{root / "keep.txt"} << "keep"; + auto queue = makeQueue(); + + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addLocalDeleteOperation( + remove, + {Paths::LocalPath::make(root / "directory").value(), + Paths::LocalPath::make(root / "directory" / "inner").value(), + Paths::LocalPath::make(root / "directory" / "inner" / "a.txt").value(), + Paths::LocalPath::make(root / "file.txt").value()} + ); + }).has_value()); + + // Paused queues hold local deletes like any other operation. + EXPECT_TRUE(std::filesystem::exists(root / "file.txt")); + setPaused(false); + workUntilCompletions(1); + ASSERT_EQ(completions().size(), 1u); + EXPECT_EQ(completions().front().operationId, remove); + EXPECT_EQ(completions().front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_FALSE(std::filesystem::exists(root / "directory")); + EXPECT_FALSE(std::filesystem::exists(root / "file.txt")); + EXPECT_TRUE(std::filesystem::exists(root / "keep.txt")); + } + + TEST_F(OperationQueueTests, LocalDeleteKeepsUnlistedContentAndItsDirectories) + { + const auto root = localDirectory_.path() / "localDeleteUnlisted"; + std::filesystem::create_directories(root / "directory" / "inner"); + std::ofstream{root / "directory" / "listed.txt"} << "listed"; + std::ofstream{root / "directory" / "inner" / "unlisted.txt"} << "unlisted"; + std::ofstream{root / "other.txt"} << "other"; + auto queue = makeQueue(); + + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addLocalDeleteOperation( + remove, + {Paths::LocalPath::make(root / "directory").value(), + Paths::LocalPath::make(root / "directory" / "inner").value(), + Paths::LocalPath::make(root / "directory" / "listed.txt").value(), + Paths::LocalPath::make(root / "other.txt").value()} + ); + }).has_value()); + setPaused(false); + workUntilCompletions(1); + + ASSERT_EQ(completions().size(), 1u); + const auto completion = completions().front(); + EXPECT_EQ(completion.reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(completion.failedEntries.size(), 2u) << "inner and directory are not empty"; + EXPECT_TRUE(std::filesystem::exists(root / "directory" / "inner" / "unlisted.txt")); + EXPECT_FALSE(std::filesystem::exists(root / "directory" / "listed.txt")); + EXPECT_FALSE(std::filesystem::exists(root / "other.txt")) << "a failure does not stop the rest"; + } + + TEST_F(OperationQueueTests, LocalDeleteOfARootOrTheHomeDirectoryIsRefused) + { + auto queue = makeQueue(); + const auto file = localDirectory_.path() / "survivor.txt"; + std::ofstream{file} << "survivor"; + const auto root = Paths::LocalPath::make(std::filesystem::path{file}.root_path()).value(); + const auto home = localHomeDirectory(); + ASSERT_TRUE(home.has_value()); + + for (auto const& target : {root, *home}) + { + const auto result = runOnStrand([&]() { + return queue->addLocalDeleteOperation( + Ids::generateOperationId(), {Paths::LocalPath::make(file).value(), target} + ); + }); + ASSERT_FALSE(result.has_value()) << target.toUtf8(); + EXPECT_EQ(result.error().type, Operation::ErrorType::InvalidPath); + } + EXPECT_TRUE(std::filesystem::exists(file)); + } + + TEST_F(OperationQueueTests, SyncPlanQueuesEveryPartUnderTheFrontendIds) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/syncRemote"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/syncRemote/down.txt"}, {"content", "down"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/syncRemote/gone.txt"}, {"content", "gone"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto localRoot = localDirectory_.path() / "syncExecution"; + std::filesystem::create_directories(localRoot); + std::ofstream{localRoot / "up.txt"} << "up"; + std::ofstream{localRoot / "localGone.txt"} << "gone"; + + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::make(localRoot).value(), + .remoteRoot = remote("/home/test/syncRemote"), + }, + context.get_executor() + }; + using SharedData::Sync::Action; + const auto entry = [&session](std::string relKey, Action action, bool isDirectory, std::uint64_t size = 0) + { + auto paths = session.resolve(Paths::RelativePath::make(relKey).value()).value(); + return SyncSession::ResolvedPlanEntry{ + .entry = {.relKey = std::move(relKey), .action = action, .sizeBytes = size, .isDirectory = isDirectory}, + .paths = std::move(paths), + }; + }; + SyncSession::ResolvedPlan plan{ + .entries = + { + entry("up.txt", Action::Upload, false, 2), + entry("emptyA/emptyB", Action::Upload, true), + entry("down.txt", Action::Download, false, 4), + entry("newLocal/dir", Action::Download, true), + entry("gone.txt", Action::DeleteRemote, false), + entry("localGone.txt", Action::DeleteLocal, false), + }, + .refused = {"CON"}, + }; + const SharedData::Sync::SyncExecutionRequest request{ + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + + const auto result = runOnStrand([&]() { + return queue->executeSyncPlan(*sftp, session, std::move(plan), request); + }); + EXPECT_EQ(result.uploads, (std::vector{"up.txt"})); + EXPECT_EQ(result.downloads, (std::vector{"down.txt"})); + EXPECT_EQ(result.remoteDeletes, (std::vector{"gone.txt"})); + EXPECT_EQ(result.localDeletes, (std::vector{"localGone.txt"})); + EXPECT_EQ(result.remoteDeleteSources, (std::vector{"/home/test/syncRemote/gone.txt"})); + EXPECT_EQ( + result.localDeleteSources, + (std::vector{Paths::LocalPath::make(localRoot / "localGone.txt").value().toUtf8()}) + ); + EXPECT_EQ(result.createdDirectories, (std::vector{"emptyA/emptyB", "newLocal/dir"})); + EXPECT_EQ(result.failed, (std::vector{"CON"})); + + setPaused(false); + workUntilCompletions(4); + std::set finishedIds; + for (auto const& completed : completions()) + { + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Completed) << completed.operationId.value(); + finishedIds.insert(completed.operationId.value()); + } + EXPECT_EQ( + finishedIds, + (std::set{ + request.uploadId.value(), + request.downloadId.value(), + request.remoteDeleteId.value(), + request.localDeleteId.value(), + }) + ); + EXPECT_EQ(remoteNames(*sftp, "/home/test/syncRemote"), (std::set{"down.txt", "emptyA", "up.txt"})); + EXPECT_EQ(remoteNames(*sftp, "/home/test/syncRemote/emptyA"), (std::set{"emptyB"})); + EXPECT_TRUE(std::filesystem::exists(localRoot / "down.txt")); + EXPECT_TRUE(std::filesystem::is_directory(localRoot / "newLocal" / "dir")); + EXPECT_FALSE(std::filesystem::exists(localRoot / "localGone.txt")); + } + + TEST_F(OperationQueueTests, CancelledDirectoryDownloadDuringScanLeavesNextUnitIntact) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto first = Ids::generateOperationId(); + const auto second = Ids::generateOperationId(); + const auto firstLocal = localDirectory_.path() / "first"; + const auto secondLocal = localDirectory_.path() / "second"; + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, first, toLocal(firstLocal), remote("/home/test/Documents"), false, false, false, false + ); + } + ).has_value()); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, second, toLocal(secondLocal), remote("/home/test/Pictures"), false, false, false, false + ); + } + ).has_value()); + + runOnStrand([&]() { + return queue->work(); + }); + queue->cancel(first); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().operationId, second); + EXPECT_TRUE(localNames(firstLocal).empty()); + EXPECT_EQ(localNames(secondLocal), (std::set{"image1.png", "image2.jpg"})); + } + + TEST_F(OperationQueueTests, ReorderedUnitsKeepTheirOwnScans) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto first = Ids::generateOperationId(); + const auto second = Ids::generateOperationId(); + const auto firstLocal = localDirectory_.path() / "first"; + const auto secondLocal = localDirectory_.path() / "second"; + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, first, toLocal(firstLocal), remote("/home/test/Documents"), false, false, false, false + ); + } + ).has_value()); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, second, toLocal(secondLocal), remote("/home/test/Pictures"), false, false, false, false + ); + } + ).has_value()); + + queue->moveOperation(second, 0); + setPaused(false); + + workUntilCompletions(2); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 2u); + EXPECT_EQ(finished[0].operationId, second); + EXPECT_EQ(finished[1].operationId, first); + EXPECT_EQ(localNames(firstLocal), (std::set{"doc1.txt", "doc2.txt"})); + EXPECT_EQ(localNames(secondLocal), (std::set{"image1.png", "image2.jpg"})); + } + + TEST_F(OperationQueueTests, ScanFailureFailsTheUnit) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto download = Ids::generateOperationId(); + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, + download, + toLocal(localDirectory_.path() / "Documents"), + remote("/home/test/Documents"), + false, + false, + false, + false + ); + } + ).has_value()); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, remove, remote("/home/test/Documents"), true, false); + }).has_value()); + const auto archive = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addArchiveDownloadOperation( + *sftp, + archive, + {SharedData::ArchiveDownloadRoot{ + .path = remote("/home/test/Documents"), + .entry = SharedData::DirectoryEntry{.name = "Documents", .type = SharedData::FileType::Directory}, + }}, + toLocal(localDirectory_.path() / "documents.tar"), + TarArchive::Compression::None, + 0, + false + ); + }).has_value()); + + workUntilCompletions(3); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 3u); + for (auto const& completed : finished) + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Failed) << completed.operationId.value(); + } + + TEST_F(OperationQueueTests, RecursiveDeleteOfSymlinkRootLeavesTarget) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE( + serverStartResult->command("createSymlink", {{"path", "/home/test/linkdir"}, {"target", "/home/test/Documents"}}) + ); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, remove, remote("/home/test/linkdir"), true, false); + }).has_value()); + + workUntilCompletions(1); + EXPECT_EQ(completions().front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Documents"), (std::set{"doc1.txt", "doc2.txt"})); + EXPECT_FALSE(remoteNames(*sftp, "/home/test").contains("linkdir")); + } + + TEST_F(OperationQueueTests, DeletesOfTheRootAndTheHomeDirectoryAreRefused) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto before = remoteNames(*sftp, "/home/test"); + for (auto const* target : {"/", "/home", "/home/test"}) + { + const auto result = runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, Ids::generateOperationId(), remote(target), true, false); + }); + ASSERT_FALSE(result.has_value()) << target; + EXPECT_EQ(result.error().type, Operation::ErrorType::InvalidPath) << target; + } + const auto unset = runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, Ids::generateOperationId(), Paths::RemotePath{}, false, false); + }); + EXPECT_FALSE(unset.has_value()); + + const auto allowed = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, allowed, remote("/home/test/file2.txt"), false, false); + }).has_value()); + workUntilCompletions(1); + ASSERT_EQ(completions().size(), 1u); + EXPECT_EQ(completions().front().operationId, allowed); + + auto expected = before; + expected.erase("file2.txt"); + EXPECT_EQ(remoteNames(*sftp, "/home/test"), expected); + } + + TEST_F(OperationQueueTests, RecursiveDeleteRefusesScannedEntriesOutsideItsRoot) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + // A name leaving the root, and a second parentless entry that would stand for the root again. + std::vector> hostileScans(2); + hostileScans[0].resize(3); + hostileScans[0][1].name = "doc1.txt"; + hostileScans[0][1].parent = 0; + hostileScans[0][2].name = "../file1.txt"; + hostileScans[0][2].parent = 0; + hostileScans[1].resize(2); + hostileScans[1][1].name = "doc1.txt"; + + for (auto& entries : hostileScans) + { + DeleteOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/Documents"), + .recursive = true, + }}; + operation.setScanResult(std::move(entries), 0); + + const auto result = operation.work(); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::InvalidPath); + EXPECT_TRUE(remoteNames(*sftp, "/home/test").contains("file1.txt")); + EXPECT_TRUE(remoteNames(*sftp, "/home/test/Documents").contains("doc1.txt")); + } + } + + TEST_F(OperationQueueTests, SyncScansAreDeliveredById) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localRoot = localDirectory_.path() / "syncLocal"; + std::filesystem::create_directories(localRoot); + std::ofstream{localRoot / "local.txt"} << "local"; + + const auto syncSessionId = Ids::makeSyncSessionId("sync-session"); + const auto remoteScan = Ids::generateOperationId(); + const auto localScan = Ids::generateOperationId(); + const auto download = Ids::generateOperationId(); + runOnStrand([&]() { + queue->addSyncScanOperation( + *sftp, + syncSessionId, + remoteScan, + localScan, + remote("/home/test/Music"), + Paths::LocalPath::make(localRoot).value(), + false, + true, + false + ); + }); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, + download, + toLocal(localDirectory_.path() / "Pictures"), + remote("/home/test/Pictures"), + false, + false, + false, + false, + SharedData::OperationMode::PriorityQueued + ); + } + ).has_value()); + + workUntilCompletions(3); + auto session = queue->syncSession(syncSessionId); + ASSERT_TRUE(session); + std::promise ready; + boost::asio::post(session->strand(), [&]() { + ready.set_value(session->bothTreesReadyInStrand()); + }); + EXPECT_TRUE(ready.get_future().get()); + EXPECT_EQ( + localNames(localDirectory_.path() / "Pictures"), (std::set{"image1.png", "image2.jpg"}) + ); + } + + TEST_F(OperationQueueTests, HostileNamesNeverBecomeLocalPaths) + { + CREATE_SERVER_AND_JOINER(sftpServer); + // A file literally named "x/.." would turn into the download root's parent when joined as a path. + ASSERT_TRUE(serverStartResult->command( + "rawName", {{"directory", "/home/test/Documents"}, {"name", "x/.."}, {"content", "hostile"}} + )); + ASSERT_TRUE(serverStartResult->command( + "rawName", {{"directory", "/home/test/Documents"}, {"name", ".."}, {"content", "hostile"}} + )); + ASSERT_TRUE(serverStartResult->command( + "rawName", {{"directory", "/home/test/Documents"}, {"name", "a\\b"}, {"content", "backslash"}} + )); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localPath = localDirectory_.path() / "inner" / "Documents"; + std::filesystem::create_directories(localPath.parent_path()); + const auto download = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand( + [&]() + { + return queue->addDownloadOperation( + *sftp, download, toLocal(localPath), remote("/home/test/Documents"), false, false, false, false + ); + } + ).has_value()); + + workUntilCompletions(1); + const auto finished = completions().front(); + // Partial success: the frontend reads failed entries of a completed bulk as such. On Windows "a\b" cannot be + // stored either. +#ifdef _WIN32 + ASSERT_EQ(finished.failedEntries.size(), 2u); +#else + ASSERT_EQ(finished.failedEntries.size(), 1u); +#endif + for (auto const& [path, error] : finished.failedEntries) + EXPECT_EQ(error.type, SharedData::OperationErrorType::InvalidName) << path; + // Nothing was written next to or above the download root. + EXPECT_EQ(localNames(localPath.parent_path()), (std::set{"Documents"})); + EXPECT_EQ(localNames(localDirectory_.path()), (std::set{"inner"})); +#ifdef _WIN32 + EXPECT_EQ(localNames(localPath), (std::set{"doc1.txt", "doc2.txt"})); +#else + EXPECT_EQ(localNames(localPath), (std::set{"a\\b", "doc1.txt", "doc2.txt"})); +#endif + } + + TEST_F(OperationQueueTests, RecursiveDeleteNeverFollowsHostileNamesOut) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createDirectory", {{"path", "/home/test/victim"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/victim/keep.txt"}, {"content", "x"}})); + ASSERT_TRUE(serverStartResult->command( + "rawName", {{"directory", "/home/test/victim"}, {"name", "x/.."}, {"type", "directory"}} + )); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto before = remoteNames(*sftp, "/home/test"); + const auto remove = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, remove, remote("/home/test/victim"), true, false); + }).has_value()); + + workUntilCompletions(1); + // The hostile entry stays, so the directory cannot be removed, but nothing outside it was touched. + auto after = remoteNames(*sftp, "/home/test"); + after.insert("victim"); + EXPECT_EQ(after, before); + EXPECT_EQ(remoteNames(*sftp, "/home/test/Documents"), (std::set{"doc1.txt", "doc2.txt"})); + } +} diff --git a/backend/test/backend/test_pseudo_terminal.hpp b/backend/test/backend/test_pseudo_terminal.hpp index 1ca96dca..3f46709b 100644 --- a/backend/test/backend/test_pseudo_terminal.hpp +++ b/backend/test/backend/test_pseudo_terminal.hpp @@ -28,6 +28,7 @@ # include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace Test { @@ -129,7 +130,7 @@ namespace Test std::filesystem::path pseudoTerminalCaptureFile(std::string const& name) { - const auto directory = programDirectory / "temp" / "pseudo_terminal"; + const auto directory = temporaryRoot / "pseudo_terminal"; std::filesystem::create_directories(directory); const auto file = directory / name; std::filesystem::remove(file); diff --git a/backend/test/backend/test_scan_operation.hpp b/backend/test/backend/test_scan_operation.hpp index 1424088f..749572a7 100644 --- a/backend/test/backend/test_scan_operation.hpp +++ b/backend/test/backend/test_scan_operation.hpp @@ -3,6 +3,8 @@ #include #include "real_server_tests.hpp" +#include + #include #include @@ -41,7 +43,7 @@ namespace Test std::string const& filename) { return std::any_of(entries.begin(), entries.end(), [&](SharedData::DirectoryEntry const& entry) { - return entry.path.filename().generic_string() == filename; + return entry.name == filename; }); } @@ -50,7 +52,7 @@ namespace Test std::string const& filename) { auto iter = std::find_if(entries.begin(), entries.end(), [&](SharedData::DirectoryEntry const& entry) { - return entry.path.filename().generic_string() == filename; + return entry.name == filename; }); return iter == entries.end() ? nullptr : &*iter; } @@ -87,7 +89,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test/Documents", + .remotePath = remote("/home/test/Documents"), .recursive = false, }}; @@ -106,7 +108,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test", + .remotePath = remote("/home/test"), .recursive = false, }}; @@ -128,7 +130,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test", + .remotePath = remote("/home/test"), .recursive = true, }}; @@ -151,7 +153,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test", + .remotePath = remote("/home/test"), .recursive = false, }}; @@ -186,7 +188,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test/Documents", + .remotePath = remote("/home/test/Documents"), .recursive = true, }}; @@ -211,7 +213,7 @@ namespace Test [&callbackInvocations](std::uint64_t, std::uint64_t, std::uint64_t) { ++callbackInvocations; }, - .remotePath = "/home/test/Documents", + .remotePath = remote("/home/test/Documents"), .recursive = false, }}; @@ -227,7 +229,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test/does_not_exist_dir", + .remotePath = remote("/home/test/does_not_exist_dir"), .recursive = false, }}; @@ -255,7 +257,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test/file1.txt", + .remotePath = remote("/home/test/file1.txt"), .recursive = false, }}; @@ -281,7 +283,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test/Documents", + .remotePath = remote("/home/test/Documents"), .recursive = false, }}; @@ -300,7 +302,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test/does_not_exist_dir", + .remotePath = remote("/home/test/does_not_exist_dir"), .recursive = false, }}; @@ -325,7 +327,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test", + .remotePath = remote("/home/test"), .recursive = true, }}; @@ -350,7 +352,7 @@ namespace Test ScanOperation operation{ *sftp, { - .remotePath = "/home/test/Documents", + .remotePath = remote("/home/test/Documents"), .recursive = false, .ignoreHidden = true, }}; @@ -361,4 +363,65 @@ namespace Test EXPECT_TRUE(hasEntryNamed(entries, "doc1.txt")); EXPECT_TRUE(hasEntryNamed(entries, "doc2.txt")); } + + TEST_F(ScanOperationTests, BuildTreeKeepsDeniedSubdirectoryAsUnreadable) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + ScanOperation operation{*sftp, {.remotePath = remote("/home/test"), .buildTree = true}}; + ASSERT_EQ(runScanToCompletion(operation), ScanOperation::WorkStatus::Complete); + + const auto tree = operation.ejectScanTree(); + auto const* documents = SharedData::Sync::findScanNode(tree, "Documents"); + ASSERT_NE(documents, nullptr); + EXPECT_EQ(documents->availability, SharedData::Sync::Availability::Unreadable); + EXPECT_FALSE(documents->childrenKnown); + auto const* pictures = SharedData::Sync::findScanNode(tree, "Pictures"); + ASSERT_NE(pictures, nullptr); + EXPECT_EQ(pictures->children.size(), 2u); + } + + TEST_F(ScanOperationTests, DeniedSubdirectoryFailsAFlatScan) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + // Transfers and deletes use flat scans, an incomplete listing must fail them. + ScanOperation operation{*sftp, {.remotePath = remote("/home/test")}}; + std::expected result{}; + for (int iteration = 0; iteration < 100 && result.has_value() && + result.value() != ScanOperation::WorkStatus::Complete; + ++iteration) + result = operation.work(); + EXPECT_FALSE(result.has_value()); + } + + TEST_F(ScanOperationTests, HostileNamesAreRejectedOrMarked) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("rawName", {{"directory", "/home/test/Documents"}, {"name", "x/.."}})); + ASSERT_TRUE(serverStartResult->command("rawName", {{"directory", "/home/test/Documents"}, {"name", ".."}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + ScanOperation flat{*sftp, {.remotePath = remote("/home/test/Documents")}}; + ASSERT_EQ(runScanToCompletion(flat), ScanOperation::WorkStatus::Complete); + const auto rejected = flat.rejectedPaths(); + ASSERT_EQ(rejected.size(), 1u); + EXPECT_EQ(rejected.front(), "/home/test/Documents/x/..") << "joined for display only"; + for (auto const& entry : flat.ejectEntries()) + EXPECT_NE(entry.name, "x/.."); + + ScanOperation tree{*sftp, {.remotePath = remote("/home/test/Documents"), .buildTree = true}}; + ASSERT_EQ(runScanToCompletion(tree), ScanOperation::WorkStatus::Complete); + const auto root = tree.ejectScanTree(); + const auto hostile = std::ranges::find_if(root.children, [](auto const& child) { + return child.name == "x/.."; + }); + ASSERT_NE(hostile, root.children.end()); + EXPECT_EQ(hostile->availability, SharedData::Sync::Availability::InvalidName); + EXPECT_EQ(std::ranges::count_if(root.children, [](auto const& child) { return child.name == ".."; }), 0); + } } diff --git a/backend/test/backend/test_shell_integration.hpp b/backend/test/backend/test_shell_integration.hpp index 348359df..440fd54c 100644 --- a/backend/test/backend/test_shell_integration.hpp +++ b/backend/test/backend/test_shell_integration.hpp @@ -32,6 +32,7 @@ extern char** environ; extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace Test { @@ -185,7 +186,7 @@ namespace Test */ void dumpFixture(std::string const& name) const { - const auto directory = programDirectory / "temp" / "shell_integration" / "fixtures"; + const auto directory = temporaryRoot / "shell_integration" / "fixtures"; // Sessions of earlier runs, possibly under names that no longer exist, would be replayed too. static const bool cleared = std::filesystem::remove_all(directory) >= 0; static_cast(cleared); @@ -323,7 +324,7 @@ namespace Test if (executable.empty()) GTEST_SKIP() << launch.shell << " is not installed on the host; the docker configurations cover it"; - const auto home = programDirectory / "temp" / "shell_integration" / launch.name; + const auto home = temporaryRoot / "shell_integration" / launch.name; std::filesystem::remove_all(home); std::filesystem::create_directories(home); @@ -722,7 +723,7 @@ namespace Test if (executable.empty()) GTEST_SKIP() << "bash is not installed"; - const auto home = programDirectory / "temp" / "shell_integration" / "IgnoreDups"; + const auto home = temporaryRoot / "shell_integration" / "IgnoreDups"; std::filesystem::create_directories(home); ShellUnderTest shell{{ .executable = executable, diff --git a/backend/test/backend/test_sync_session.hpp b/backend/test/backend/test_sync_session.hpp index ee1dc4ee..2edea1ec 100644 --- a/backend/test/backend/test_sync_session.hpp +++ b/backend/test/backend/test_sync_session.hpp @@ -79,8 +79,8 @@ namespace Test std::shared_ptr session_{std::make_shared( SyncSession::Options{ .sessionId = Ids::generateSyncSessionId(), - .localRoot = "/local", - .remoteRoot = "/remote" + .localRoot = Paths::LocalPath::fromUtf8("/local", Paths::LocalPlatform::Posix).value(), + .remoteRoot = remote("/remote") }, context_.get_executor() )}; @@ -134,15 +134,16 @@ namespace Test EXPECT_EQ(snapshot.deletes[0].children[0].action, SharedData::Sync::Action::DeleteRemote); } - TEST_F(SyncSessionTests, PlanUsesSessionRoots) + TEST_F(SyncSessionTests, PlanResolvesBelowSessionRoots) { session_->recomputeSnapshotInStrand({}, {}, {}); - const auto plan = session_->buildEnqueuePlanInStrand(SharedData::Sync::DiffSection::Upload, {"src"}); - ASSERT_EQ(plan.size(), 2u); - EXPECT_EQ(plan[0].relKey, "src/main.cpp"); - EXPECT_EQ(plan[0].localAbsPath, "/local/src/main.cpp"); - EXPECT_EQ(plan[0].remoteAbsPath, "/remote/src/main.cpp"); - EXPECT_EQ(plan[1].relKey, "src/util/strings.cpp"); + const auto plan = session_->resolvePlanInStrand(SharedData::Sync::DiffSection::Upload, {"src"}); + ASSERT_EQ(plan.entries.size(), 2u); + EXPECT_TRUE(plan.refused.empty()); + EXPECT_EQ(plan.entries[0].entry.relKey, "src/main.cpp"); + EXPECT_EQ(plan.entries[0].paths.local.toUtf8(), "/local/src/main.cpp"); + EXPECT_EQ(plan.entries[0].paths.remote.value(), "/remote/src/main.cpp"); + EXPECT_EQ(plan.entries[1].entry.relKey, "src/util/strings.cpp"); } TEST_F(SyncSessionTests, CancelOnlyStopsTheRunningWalk) @@ -163,4 +164,26 @@ namespace Test EXPECT_EQ(summary.generation, 1u); EXPECT_EQ(summary.uploads.itemCount, 0u); } + + TEST_F(SyncSessionTests, RowsResolveBelowBothRootsOrNotAtAll) + { + boost::asio::io_context context{}; + const SyncSession session{ + SyncSession::Options{ + .localRoot = Paths::LocalPath::fromUtf8("C:/sync", Paths::LocalPlatform::Windows).value(), + .remoteRoot = remote("/srv/sync"), + }, + context.get_executor() + }; + const auto resolved = session.resolve(Paths::RelativePath::make("dir/a\\b").value()); + EXPECT_FALSE(resolved.has_value()) << "a backslash is a separator in a Windows path"; + + const auto plain = session.resolve(Paths::RelativePath::make("dir/a.txt").value()); + ASSERT_TRUE(plain.has_value()); + EXPECT_EQ(plain->local.toUtf8(), "C:/sync/dir/a.txt"); + EXPECT_EQ(plain->remote.value(), "/srv/sync/dir/a.txt"); + + EXPECT_FALSE(session.resolve(Paths::RelativePath::make("dir/CON").value()).has_value()); + EXPECT_EQ(session.resolve(Paths::RelativePath{})->remote.value(), "/srv/sync"); + } } diff --git a/backend/test/backend/test_transfer_failures.hpp b/backend/test/backend/test_transfer_failures.hpp new file mode 100644 index 00000000..4be7d0d7 --- /dev/null +++ b/backend/test/backend/test_transfer_failures.hpp @@ -0,0 +1,474 @@ +#pragma once + +#include +#include +#include +#include "real_server_tests.hpp" + +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class TransferFailureTests : public RealServerTests + { + public: + static std::string alphabet(std::size_t length) + { + std::string result; + result.reserve(length); + for (std::size_t index = 0; index < length; ++index) + result.push_back(static_cast('a' + (index % 26))); + return result; + } + + static std::string readLocalFile(std::filesystem::path const& path) + { + std::ifstream file{path, std::ios::binary}; + return std::string{std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; + } + + static void writeLocalFile(std::filesystem::path const& path, std::string const& content) + { + std::ofstream file{path, std::ios::binary}; + file << content; + } + + static std::string readRemoteFile(SecureShell::SftpSession& sftp, std::filesystem::path const& path) + { + auto openFuture = sftp.openFile(remote(path.generic_string()), SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::unknown); + if (openFuture.wait_for(5s) != std::future_status::ready) + return ""; + auto openResult = openFuture.get(); + if (!openResult.has_value()) + return ""; + auto file = openResult->lock(); + std::string data; + auto readFuture = file->readAll( + [&data](std::string_view chunk) + { + data.append(chunk); + return true; + } + ); + if (readFuture.wait_for(10s) != std::future_status::ready || !readFuture.get().has_value()) + return ""; + return data; + } + + /** + * @brief Works an operation until it completes or fails. Async transfers run on the strand, + * so MoreWork is retried with a short pause. + */ + template + static std::expected runToEnd(OperationT& operation) + { + const auto deadline = std::chrono::steady_clock::now() + 20s; + while (std::chrono::steady_clock::now() < deadline) + { + auto result = operation.work(); + if (!result.has_value() || result.value() == Operation::WorkStatus::Complete) + return result; + std::this_thread::sleep_for(2ms); + } + return std::unexpected(Operation::Error{.type = Operation::ErrorType::FutureTimeout}); + } + + protected: + Utility::TemporaryDirectory localDirectory_{temporaryRoot, true}; + }; + + TEST_F(TransferFailureTests, AsyncDownloadReadFailureKeepsExistingFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("failReadsAfter", {{"bytes", 20000}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "large.txt"; + writeLocalFile(localPath, "original content"); + + DownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .bigFileOptimized = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(readLocalFile(localPath), "original content"); + EXPECT_FALSE(std::filesystem::exists(localPath.generic_string() + ".filepart")); + } + + TEST_F(TransferFailureTests, AsyncDownloadConnectionDropKeepsExistingFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("dropAfterBytes", {{"bytes", 20000}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "large.txt"; + writeLocalFile(localPath, "original content"); + + DownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .bigFileOptimized = true, + .futureTimeout = std::chrono::seconds{3}, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(readLocalFile(localPath), "original content"); + } + + TEST_F(TransferFailureTests, SyncDownloadReadFailureKeepsExistingFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("failReadsAfter", {{"bytes", 0}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "file1.txt"; + writeLocalFile(localPath, "original content"); + + DownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(readLocalFile(localPath), "original content"); + } + + TEST_F(TransferFailureTests, DownloadOfFileShorterThanListedKeepsExistingFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + for (const bool bigFileOptimized : {false, true}) + { + const auto localPath = localDirectory_.path() / fmt::format("large_{}.txt", bigFileOptimized); + writeLocalFile(localPath, "original content"); + + SharedData::DirectoryEntry entry{}; + entry.name = "large.txt"; + entry.type = SharedData::FileType::Regular; + entry.size = 70000u; + + DownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .bigFileOptimized = bigFileOptimized, + .entry = entry, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()) << "bigFileOptimized: " << bigFileOptimized; + EXPECT_EQ(readLocalFile(localPath), "original content") << "bigFileOptimized: " << bigFileOptimized; + } + } + + TEST_F(TransferFailureTests, AsyncDownloadResumesFromPartialFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "large.txt"; + writeLocalFile(localPath.generic_string() + ".filepart", alphabet(65536).substr(0, 20000)); + + DownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/large.txt"), + .localPath = toLocal(localPath), + .tryContinue = true, + .bigFileOptimized = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), alphabet(65536)); + } + + TEST_F(TransferFailureTests, CompletePartialFileIsFinalizedWithoutReading) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "file1.txt"; + writeLocalFile(localPath.generic_string() + ".filepart", "Fake file content"); + + DownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .tryContinue = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readLocalFile(localPath), "Fake file content"); + } + + TEST_F(TransferFailureTests, FollowedSymlinkDownloadsTargetContent) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "followed.txt"; + DownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/link_to_file1.txt"), + .localPath = toLocal(localPath), + .symlinkHandling = Persistence::SymlinkHandling::FollowSymlink, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_FALSE(std::filesystem::is_symlink(std::filesystem::symlink_status(localPath))); + EXPECT_EQ(readLocalFile(localPath), "Fake file content"); + } + + TEST_F(TransferFailureTests, AsyncUploadWriteFailureKeepsExistingRemoteFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("failWritesAfter", {{"bytes", 20000}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "upload.txt"; + writeLocalFile(localPath, alphabet(65536)); + + UploadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .bigFileOptimized = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "Fake file content"); + } + + TEST_F(TransferFailureTests, SyncUploadWriteFailureKeepsExistingRemoteFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("failWritesAfter", {{"bytes", 0}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "upload.txt"; + writeLocalFile(localPath, "new content"); + + UploadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "Fake file content"); + } + + TEST_F(TransferFailureTests, AsyncUploadResumesFromPartialRemoteFile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + const auto content = alphabet(65536); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/resumed.txt.filepart"}, {"content", content.substr(0, 20000)}} + )); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "resumed.txt"; + writeLocalFile(localPath, content); + + UploadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/resumed.txt"), + .localPath = toLocal(localPath), + .tryContinue = true, + .bigFileOptimized = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/resumed.txt"), content); + } + +#ifndef _WIN32 + TEST_F(TransferFailureTests, DownloadRefusesASymlinkAtTheTempPath) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto outside = localDirectory_.path() / "outside.txt"; + writeLocalFile(outside, "must stay"); + const auto localPath = localDirectory_.path() / "file1.txt"; + std::filesystem::create_symlink(outside, localPath.generic_string() + ".filepart"); + + DownloadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(readLocalFile(outside), "must stay"); + } +#endif + + TEST_F(TransferFailureTests, UploadOverwriteReplacesWithAndWithoutPosixRename) + { + for (const bool posixRename : {false, true}) + { + CREATE_SERVER_AND_JOINER(sftpServer); + if (posixRename) + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "posix-rename@openssh.com"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "upload.txt"; + writeLocalFile(localPath, "new content"); + + UploadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "new content"); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt.filepart.old"), ""); + } + } + + TEST_F(TransferFailureTests, FailedUploadRenameKeepsExistingRemoteFile) + { + for (const bool posixRename : {false, true}) + { + CREATE_SERVER_AND_JOINER(sftpServer); + if (posixRename) + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "posix-rename@openssh.com"}})); + ASSERT_TRUE(serverStartResult->command("failRenames", {{"enabled", true}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "upload.txt"; + writeLocalFile(localPath, "new content"); + + UploadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/file1.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt"), "Fake file content"); + } + } + + TEST_F(TransferFailureTests, FailedArchiveUploadKeepsExistingRemoteArchive) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/backup.tar"}, {"content", "old archive"}})); + ASSERT_TRUE(serverStartResult->command("failWritesAfter", {{"bytes", 0}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "packed.txt"; + writeLocalFile(localPath, alphabet(4096)); + + ArchiveUploadOperation operation{ + *sftp, + { + .localPaths = {toLocal(localPath)}, + .remoteArchivePath = remote("/home/test/backup.tar"), + .compression = TarArchive::Compression::None, + .mayOverwrite = true, + } + }; + + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + ASSERT_TRUE(serverStartResult->command("failWritesAfter", {{"bytes", nullptr}})); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/backup.tar"), "old archive"); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/backup.tar.filepart"), ""); + } + + TEST_F(TransferFailureTests, ArchiveUploadReplacesTheArchiveOnlyWhenComplete) + { + CREATE_SERVER_AND_JOINER(sftpServer); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/backup.tar"}, {"content", "old archive"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + const auto localPath = localDirectory_.path() / "packed.txt"; + writeLocalFile(localPath, alphabet(4096)); + + ArchiveUploadOperation refused{ + *sftp, + { + .localPaths = {toLocal(localPath)}, + .remoteArchivePath = remote("/home/test/backup.tar"), + .compression = TarArchive::Compression::None, + } + }; + const auto refusedResult = runToEnd(refused); + ASSERT_FALSE(refusedResult.has_value()); + EXPECT_EQ(refusedResult.error().type, SharedData::OperationErrorType::FileExists); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/backup.tar"), "old archive"); + + ArchiveUploadOperation replacing{ + *sftp, + { + .localPaths = {toLocal(localPath)}, + .remoteArchivePath = remote("/home/test/backup.tar"), + .compression = TarArchive::Compression::None, + .mayOverwrite = true, + } + }; + const auto replacingResult = runToEnd(replacing); + ASSERT_TRUE(replacingResult.has_value()) << replacingResult.error().toString(); + EXPECT_NE(readRemoteFile(*sftp, "/home/test/backup.tar"), "old archive"); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/backup.tar.filepart"), ""); + } +} diff --git a/backend/test/backend/test_upload_operation.hpp b/backend/test/backend/test_upload_operation.hpp index a01a981b..ace9d501 100644 --- a/backend/test/backend/test_upload_operation.hpp +++ b/backend/test/backend/test_upload_operation.hpp @@ -5,6 +5,8 @@ #include +extern std::filesystem::path temporaryRoot; + namespace Test { class UploadOperationTests : public RealServerTests @@ -16,7 +18,7 @@ namespace Test { RealServerTests::SetUp(); - std::ofstream testFile{programDirectory / "temp" / "testfile.txt", std::ios_base::binary}; + std::ofstream testFile{temporaryRoot / "testfile.txt", std::ios_base::binary}; testFile << testFileContent; } }; @@ -50,29 +52,58 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", - }}; + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; auto prepareResult = operation.prepare(); ASSERT_TRUE(prepareResult.has_value()); - auto fut = sftp->listDirectory("/home/test"); + auto fut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); auto listResult = fut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "testfile.txt.filepart"; + return entry.name == "testfile.txt.filepart"; }); EXPECT_NE(it, listResult.value().end()); } + TEST_F(UploadOperationTests, InvalidRemotePathsAreRefusedBeforeTheServer) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + // Relative and ".." paths cannot be built as a RemotePath at all, unset and the root remain. + for (auto const& remotePath : {Paths::RemotePath{}, Paths::RemotePath::root()}) + { + UploadOperation operation{ + *sftp, + { + .remotePath = remotePath, + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; + auto prepareResult = operation.prepare(); + ASSERT_FALSE(prepareResult.has_value()) << remotePath.value(); + EXPECT_EQ(prepareResult.error().type, Operation::ErrorType::InvalidPath) << remotePath.value(); + } + + auto fut = sftp->listDirectory(remote("/home")); + ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); + auto listResult = fut.get(); + ASSERT_TRUE(listResult.has_value()); + for (auto const& entry : *listResult) + EXPECT_NE(entry.name, "testfile.txt.filepart"); + } + TEST_F(UploadOperationTests, CanPrepareUploadByAdoptingFilepart) { CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto createFuture = sftp->createFile("/home/test/testfile.txt.filepart"); + auto createFuture = sftp->createFile(remote("/home/test/testfile.txt.filepart")); ASSERT_EQ(createFuture.wait_for(5s), std::future_status::ready); auto createResult = createFuture.get(); ASSERT_TRUE(createResult.has_value()); @@ -80,24 +111,61 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), .tryContinue = true, - }}; + } + }; auto prepareResult = operation.prepare(); ASSERT_TRUE(prepareResult.has_value()); - auto fut = sftp->listDirectory("/home/test"); + auto fut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); auto listResult = fut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "testfile.txt.filepart"; + return entry.name == "testfile.txt.filepart"; }); EXPECT_NE(it, listResult.value().end()); } +#ifdef _WIN32 + TEST_F(UploadOperationTests, InheritedWindowsPermissionsAreNeverExecutable) + { + // std::filesystem reports every Windows file as executable, those bits must not reach the server. + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + std::ofstream{temporaryRoot / "windows.txt"} << "text"; + + UploadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/windows.txt"), + .localPath = toLocal(temporaryRoot / "windows.txt"), + .inheritPermissions = true, + } + }; + UploadOperation::WorkStatus workStatus{}; + for (int workTimes = 0; workTimes < 100; ++workTimes) + { + auto workResult = operation.work(); + ASSERT_TRUE(workResult.has_value()); + workStatus = workResult.value(); + if (workStatus == UploadOperation::WorkStatus::Complete) + break; + } + ASSERT_EQ(workStatus, UploadOperation::WorkStatus::Complete); + + auto statFuture = sftp->stat(remote("/home/test/windows.txt")); + ASSERT_EQ(statFuture.wait_for(5s), std::future_status::ready); + const auto information = statFuture.get(); + ASSERT_TRUE(information.has_value()); + using std::filesystem::perms; + EXPECT_EQ(information->permissions & (perms::owner_exec | perms::group_exec | perms::others_exec), perms::none); + } +#endif + TEST_F(UploadOperationTests, CanUploadFile) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -106,9 +174,10 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", - }}; + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -121,12 +190,12 @@ namespace Test } ASSERT_EQ(workStatus, UploadOperation::WorkStatus::Complete); - auto fut = sftp->listDirectory("/home/test"); + auto fut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); auto listResult = fut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "testfile.txt"; + return entry.name == "testfile.txt"; }); EXPECT_NE(it, listResult.value().end()); @@ -134,7 +203,7 @@ namespace Test ASSERT_TRUE(closeResult.has_value()); auto readFut = sftp->openFile( - "/home/test/testfile.txt", SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + remote("/home/test/testfile.txt"), SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(readFut.wait_for(5s), std::future_status::ready); auto readResult = readFut.get(); @@ -163,9 +232,10 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", - }}; + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -187,10 +257,11 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), .mayOverwrite = false, - }}; + } + }; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); @@ -208,9 +279,10 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", - }}; + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -232,10 +304,11 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), .mayOverwrite = true, - }}; + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -261,7 +334,7 @@ namespace Test // First upload, but only part of it { auto openFut = sftp->openFile( - "/home/test/testfile.txt.filepart", + remote("/home/test/testfile.txt.filepart"), SecureShell::SftpSession::OpenType::Write | SecureShell::SftpSession::OpenType::Create, std::filesystem::perms::owner_read | std::filesystem::perms::owner_write); ASSERT_EQ(openFut.wait_for(5s), std::future_status::ready); @@ -283,10 +356,88 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + .tryContinue = true, + } + }; + + UploadOperation::WorkStatus workStatus; + for (int workTimes = 0; workTimes < 100; ++workTimes) + { + auto workResult = operation.work(); + ASSERT_TRUE(workResult.has_value()); + workStatus = workResult.value(); + if (workStatus == UploadOperation::WorkStatus::Complete) + break; + } + ASSERT_EQ(workStatus, UploadOperation::WorkStatus::Complete); + + auto closeResult = operation.cancel(false); + ASSERT_TRUE(closeResult.has_value()); + } + + // Verify final file content + { + auto readFut = sftp->openFile( + remote("/home/test/testfile.txt"), + SecureShell::SftpSession::OpenType::Read, + std::filesystem::perms::owner_read); + ASSERT_EQ(readFut.wait_for(5s), std::future_status::ready); + auto readResult = readFut.get(); + + ASSERT_TRUE(readResult.has_value()); + auto fileWeak = std::move(readResult).value(); + auto file = fileWeak.lock(); + ASSERT_TRUE(file); + + std::string data(testFileContent.size() + 100, '\0'); + auto readAllFut = file->readSome(data.data(), data.size()); + ASSERT_EQ(readAllFut.wait_for(5s), std::future_status::ready); + auto readAllResult = readAllFut.get(); + ASSERT_TRUE(readAllResult.has_value()); + data.resize(readAllResult.value()); + + EXPECT_EQ(data, testFileContent); + } + } + + TEST_F(UploadOperationTests, PartialFileOfAnotherSourceIsNotContinued) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + // First upload, but only part of it + { + auto openFut = sftp->openFile( + remote("/home/test/testfile.txt.filepart"), + SecureShell::SftpSession::OpenType::Write | SecureShell::SftpSession::OpenType::Create, + std::filesystem::perms::owner_read | std::filesystem::perms::owner_write); + ASSERT_EQ(openFut.wait_for(5s), std::future_status::ready); + auto openResult = openFut.get(); + ASSERT_TRUE(openResult.has_value()); + auto fileWeak = std::move(openResult).value(); + auto file = fileWeak.lock(); + ASSERT_TRUE(file); + + // Same length as a real part, but not the source's bytes: a leftover of another program or version. + std::string partialContent(testFileContent.size() / 2, '#'); + auto writeFut = file->write(partialContent); + ASSERT_EQ(writeFut.wait_for(5s), std::future_status::ready); + (void)writeFut.get(); + file->close(); + } + + // Second upload continuing the partial file + { + UploadOperation operation{ + *sftp, + { + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), .tryContinue = true, - }}; + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -306,7 +457,7 @@ namespace Test // Verify final file content { auto readFut = sftp->openFile( - "/home/test/testfile.txt", + remote("/home/test/testfile.txt"), SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(readFut.wait_for(5s), std::future_status::ready); @@ -333,15 +484,16 @@ namespace Test CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); - std::ofstream emptyFile{programDirectory / "temp" / "emptyfile.txt", std::ios_base::binary}; + std::ofstream emptyFile{temporaryRoot / "emptyfile.txt", std::ios_base::binary}; emptyFile.close(); UploadOperation operation{ *sftp, { - .remotePath = "/home/test/emptyfile.txt", - .localPath = programDirectory / "temp" / "emptyfile.txt", - }}; + .remotePath = remote("/home/test/emptyfile.txt"), + .localPath = toLocal(temporaryRoot / "emptyfile.txt"), + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -354,12 +506,12 @@ namespace Test } ASSERT_EQ(workStatus, UploadOperation::WorkStatus::Complete); - auto fut = sftp->listDirectory("/home/test"); + auto fut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); auto listResult = fut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "emptyfile.txt"; + return entry.name == "emptyfile.txt"; }); EXPECT_NE(it, listResult.value().end()); } @@ -372,9 +524,10 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", - }}; + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -387,13 +540,13 @@ namespace Test } ASSERT_EQ(workStatus, UploadOperation::WorkStatus::Complete); - auto fut = sftp->listDirectory("/home/test"); + auto fut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); auto listResult = fut.get(); ASSERT_TRUE(listResult.has_value()); auto filepartIt = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "testfile.txt.filepart"; + return entry.name == "testfile.txt.filepart"; }); EXPECT_EQ(filepartIt, listResult.value().end()); } @@ -406,9 +559,10 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", - }}; + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -434,9 +588,10 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "does_not_exist.txt", - }}; + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "does_not_exist.txt"), + } + }; auto firstResult = operation.work(); ASSERT_FALSE(firstResult.has_value()); @@ -454,9 +609,10 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/nonexistent_dir/file.txt", - .localPath = programDirectory / "temp" / "testfile.txt", - }}; + .remotePath = remote("/home/test/nonexistent_dir/file.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); @@ -471,9 +627,10 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "does_not_exist.txt", - }}; + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "does_not_exist.txt"), + } + }; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); @@ -488,9 +645,10 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory, - }}; + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(programDirectory), + } + }; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); @@ -505,16 +663,17 @@ namespace Test // File larger than the internal buffer (16384 bytes) to exercise multi-chunk upload const std::string largeContent(30000, 'x'); { - std::ofstream largeFile{programDirectory / "temp" / "largefile.txt", std::ios_base::binary}; + std::ofstream largeFile{temporaryRoot / "largefile.txt", std::ios_base::binary}; largeFile << largeContent; } UploadOperation operation{ *sftp, { - .remotePath = "/home/test/largefile.txt", - .localPath = programDirectory / "temp" / "largefile.txt", - }}; + .remotePath = remote("/home/test/largefile.txt"), + .localPath = toLocal(temporaryRoot / "largefile.txt"), + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) @@ -527,12 +686,12 @@ namespace Test } ASSERT_EQ(workStatus, UploadOperation::WorkStatus::Complete); - auto fut = sftp->listDirectory("/home/test"); + auto fut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); auto listResult = fut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "largefile.txt"; + return entry.name == "largefile.txt"; }); EXPECT_NE(it, listResult.value().end()); } @@ -581,12 +740,12 @@ namespace Test TEST_F(UploadOperationTests, UploadLocalSymlinkToRegularFileCreatesRemoteSymlink) { // Set up: a local symlink to a regular file. - const auto targetFile = programDirectory / "temp" / "sym_target_file.txt"; + const auto targetFile = temporaryRoot / "sym_target_file.txt"; { std::ofstream f{targetFile, std::ios::binary}; f << "target content"; } - const auto linkPath = makeFreshSymlink(programDirectory / "temp" / "sym_to_file", targetFile); + const auto linkPath = makeFreshSymlink(temporaryRoot / "sym_to_file", targetFile); CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); @@ -594,16 +753,17 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/uploaded_sym_to_file", - .localPath = linkPath, - }}; + .remotePath = remote("/home/test/uploaded_sym_to_file"), + .localPath = toLocal(linkPath), + } + }; auto status = runToCompletion(operation); ASSERT_TRUE(status.has_value()); EXPECT_EQ(*status, UploadOperation::WorkStatus::Complete); // The remote entry must be a symlink, not a file copy of the target. - auto lstatFut = sftp->lstat("/home/test/uploaded_sym_to_file"); + auto lstatFut = sftp->lstat(remote("/home/test/uploaded_sym_to_file")); ASSERT_EQ(lstatFut.wait_for(5s), std::future_status::ready); auto entry = lstatFut.get(); ASSERT_TRUE(entry.has_value()); @@ -615,14 +775,14 @@ namespace Test { // A symlink pointing at a directory must be recreated as a single remote link; // the directory contents must NOT be recursively uploaded into it. - const auto targetDir = programDirectory / "temp" / "sym_target_dir"; + const auto targetDir = temporaryRoot / "sym_target_dir"; std::error_code ec; std::filesystem::create_directory(targetDir, ec); { std::ofstream f{targetDir / "inside.txt", std::ios::binary}; f << "should not appear on remote"; } - const auto linkPath = makeFreshSymlink(programDirectory / "temp" / "sym_to_dir", targetDir); + const auto linkPath = makeFreshSymlink(temporaryRoot / "sym_to_dir", targetDir); CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); @@ -630,15 +790,16 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/uploaded_sym_to_dir", - .localPath = linkPath, - }}; + .remotePath = remote("/home/test/uploaded_sym_to_dir"), + .localPath = toLocal(linkPath), + } + }; auto status = runToCompletion(operation); ASSERT_TRUE(status.has_value()); EXPECT_EQ(*status, UploadOperation::WorkStatus::Complete); - auto lstatFut = sftp->lstat("/home/test/uploaded_sym_to_dir"); + auto lstatFut = sftp->lstat(remote("/home/test/uploaded_sym_to_dir")); ASSERT_EQ(lstatFut.wait_for(5s), std::future_status::ready); auto entry = lstatFut.get(); ASSERT_TRUE(entry.has_value()); @@ -651,7 +812,7 @@ namespace Test // without trying to open/stat the target. This is the scenario that caused the // "Path exists and is not a directory" and FileStatFailed errors before. const auto linkPath = makeFreshSymlink( - programDirectory / "temp" / "dangling_upload", + temporaryRoot / "dangling_upload", "/nowhere/this/should/not/exist.txt" ); @@ -661,15 +822,16 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/uploaded_dangling", - .localPath = linkPath, - }}; + .remotePath = remote("/home/test/uploaded_dangling"), + .localPath = toLocal(linkPath), + } + }; auto status = runToCompletion(operation); ASSERT_TRUE(status.has_value()) << "dangling symlink upload failed"; EXPECT_EQ(*status, UploadOperation::WorkStatus::Complete); - auto lstatFut = sftp->lstat("/home/test/uploaded_dangling"); + auto lstatFut = sftp->lstat(remote("/home/test/uploaded_dangling")); ASSERT_EQ(lstatFut.wait_for(5s), std::future_status::ready); auto entry = lstatFut.get(); ASSERT_TRUE(entry.has_value()); @@ -678,12 +840,12 @@ namespace Test TEST_F(UploadOperationTests, UploadSymlinkWithSkipSymlinkDoesNotCreateRemoteEntry) { - const auto targetFile = programDirectory / "temp" / "sym_skip_target.txt"; + const auto targetFile = temporaryRoot / "sym_skip_target.txt"; { std::ofstream f{targetFile, std::ios::binary}; f << "content"; } - const auto linkPath = makeFreshSymlink(programDirectory / "temp" / "sym_skip", targetFile); + const auto linkPath = makeFreshSymlink(temporaryRoot / "sym_skip", targetFile); CREATE_SERVER_AND_JOINER(sftpServer); auto [_, sftp] = createSftpSession(serverStartResult->port); @@ -691,16 +853,17 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/uploaded_sym_skip", - .localPath = linkPath, + .remotePath = remote("/home/test/uploaded_sym_skip"), + .localPath = toLocal(linkPath), .symlinkHandling = Persistence::SymlinkHandling::SkipSymlink, - }}; + } + }; auto status = runToCompletion(operation); ASSERT_TRUE(status.has_value()); EXPECT_EQ(*status, UploadOperation::WorkStatus::Complete); - auto lstatFut = sftp->lstat("/home/test/uploaded_sym_skip"); + auto lstatFut = sftp->lstat(remote("/home/test/uploaded_sym_skip")); ASSERT_EQ(lstatFut.wait_for(5s), std::future_status::ready); auto entry = lstatFut.get(); EXPECT_FALSE(entry.has_value()) << "SkipSymlink must not create a remote entry"; @@ -719,30 +882,32 @@ namespace Test UploadOperation first{ *sftp, { - .remotePath = "/home/test/sym_overwrite_target", - .localPath = programDirectory / "temp" / "testfile.txt", - }}; + .remotePath = remote("/home/test/sym_overwrite_target"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; auto s = runToCompletion(first); ASSERT_TRUE(s.has_value()); } // Now overwrite with a symlink. const auto linkPath = makeFreshSymlink( - programDirectory / "temp" / "sym_overwrite_link", - programDirectory / "temp" / "testfile.txt" + temporaryRoot / "sym_overwrite_link", + temporaryRoot / "testfile.txt" ); UploadOperation symOp{ *sftp, { - .remotePath = "/home/test/sym_overwrite_target", - .localPath = linkPath, + .remotePath = remote("/home/test/sym_overwrite_target"), + .localPath = toLocal(linkPath), .mayOverwrite = true, - }}; + } + }; auto status = runToCompletion(symOp); ASSERT_TRUE(status.has_value()) << "symlink-over-file overwrite failed"; EXPECT_EQ(*status, UploadOperation::WorkStatus::Complete); - auto lstatFut = sftp->lstat("/home/test/sym_overwrite_target"); + auto lstatFut = sftp->lstat(remote("/home/test/sym_overwrite_target")); ASSERT_EQ(lstatFut.wait_for(5s), std::future_status::ready); auto entry = lstatFut.get(); ASSERT_TRUE(entry.has_value()); @@ -755,8 +920,8 @@ namespace Test // For symlinks, prepare() must not open a .filepart file on the remote — that path // is only for regular-file uploads. const auto linkPath = makeFreshSymlink( - programDirectory / "temp" / "sym_prepare_check", - programDirectory / "temp" / "testfile.txt" + temporaryRoot / "sym_prepare_check", + temporaryRoot / "testfile.txt" ); CREATE_SERVER_AND_JOINER(sftpServer); @@ -765,19 +930,20 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/sym_prepare_check", - .localPath = linkPath, - }}; + .remotePath = remote("/home/test/sym_prepare_check"), + .localPath = toLocal(linkPath), + } + }; auto prepareResult = operation.prepare(); ASSERT_TRUE(prepareResult.has_value()); - auto fut = sftp->listDirectory("/home/test"); + auto fut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); auto listResult = fut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "sym_prepare_check.filepart"; + return entry.name == "sym_prepare_check.filepart"; }); EXPECT_EQ(it, listResult.value().end()) << ".filepart must not be created for symlink uploads"; } @@ -797,20 +963,21 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/newly_created/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", + .remotePath = remote("/home/test/newly_created/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), .createMissingDirectories = true, - }}; + } + }; auto status = runToCompletion(operation); ASSERT_TRUE(status.has_value()) << "upload should succeed when parent is auto-created"; EXPECT_EQ(*status, UploadOperation::WorkStatus::Complete); - auto lstatFut = sftp->lstat("/home/test/newly_created/testfile.txt"); + auto lstatFut = sftp->lstat(remote("/home/test/newly_created/testfile.txt")); ASSERT_EQ(lstatFut.wait_for(5s), std::future_status::ready); ASSERT_TRUE(lstatFut.get().has_value()) << "uploaded file should exist at the nested path"; - auto dirFut = sftp->lstat("/home/test/newly_created"); + auto dirFut = sftp->lstat(remote("/home/test/newly_created")); ASSERT_EQ(dirFut.wait_for(5s), std::future_status::ready); auto dirEntry = dirFut.get(); ASSERT_TRUE(dirEntry.has_value()); @@ -825,10 +992,11 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/a/b/c/d/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", + .remotePath = remote("/home/test/a/b/c/d/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), .createMissingDirectories = true, - }}; + } + }; auto status = runToCompletion(operation); ASSERT_TRUE(status.has_value()) << "deeply nested upload should succeed"; @@ -836,14 +1004,14 @@ namespace Test for (auto const* path : {"/home/test/a", "/home/test/a/b", "/home/test/a/b/c", "/home/test/a/b/c/d"}) { - auto fut = sftp->lstat(path); + auto fut = sftp->lstat(remote(path)); ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); auto entry = fut.get(); ASSERT_TRUE(entry.has_value()) << "intermediate directory missing: " << path; EXPECT_EQ(entry->type, SharedData::FileType::Directory) << path; } - auto fileFut = sftp->lstat("/home/test/a/b/c/d/testfile.txt"); + auto fileFut = sftp->lstat(remote("/home/test/a/b/c/d/testfile.txt")); ASSERT_EQ(fileFut.wait_for(5s), std::future_status::ready); EXPECT_TRUE(fileFut.get().has_value()); } @@ -857,10 +1025,11 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), .createMissingDirectories = true, - }}; + } + }; auto status = runToCompletion(operation); ASSERT_TRUE(status.has_value()); @@ -876,16 +1045,17 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/still_missing_dir/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", + .remotePath = remote("/home/test/still_missing_dir/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), .createMissingDirectories = false, - }}; + } + }; auto workResult = operation.work(); ASSERT_FALSE(workResult.has_value()); EXPECT_EQ(workResult.error().type, UploadOperation::ErrorType::SftpError); - auto dirFut = sftp->lstat("/home/test/still_missing_dir"); + auto dirFut = sftp->lstat(remote("/home/test/still_missing_dir")); ASSERT_EQ(dirFut.wait_for(5s), std::future_status::ready); EXPECT_FALSE(dirFut.get().has_value()) << "parent dir must NOT have been created"; } @@ -894,8 +1064,8 @@ namespace Test { // Symlink path must honour the flag too: createSymLink fails without the parent. const auto linkPath = makeFreshSymlink( - programDirectory / "temp" / "sym_for_missing_dirs", - programDirectory / "temp" / "testfile.txt" + temporaryRoot / "sym_for_missing_dirs", + temporaryRoot / "testfile.txt" ); CREATE_SERVER_AND_JOINER(sftpServer); @@ -904,16 +1074,17 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/missing/for/symlink/uploaded_link", - .localPath = linkPath, + .remotePath = remote("/home/test/missing/for/symlink/uploaded_link"), + .localPath = toLocal(linkPath), .createMissingDirectories = true, - }}; + } + }; auto status = runToCompletion(operation); ASSERT_TRUE(status.has_value()); EXPECT_EQ(*status, UploadOperation::WorkStatus::Complete); - auto lstatFut = sftp->lstat("/home/test/missing/for/symlink/uploaded_link"); + auto lstatFut = sftp->lstat(remote("/home/test/missing/for/symlink/uploaded_link")); ASSERT_EQ(lstatFut.wait_for(5s), std::future_status::ready); auto entry = lstatFut.get(); ASSERT_TRUE(entry.has_value()); @@ -928,7 +1099,7 @@ namespace Test // First upload, but only part of it { auto openFut = sftp->openFile( - "/home/test/testfile.txt.filepart", + remote("/home/test/testfile.txt.filepart"), SecureShell::SftpSession::OpenType::Write | SecureShell::SftpSession::OpenType::Create, std::filesystem::perms::owner_read | std::filesystem::perms::owner_write); ASSERT_EQ(openFut.wait_for(5s), std::future_status::ready); @@ -950,10 +1121,11 @@ namespace Test UploadOperation operation{ *sftp, { - .remotePath = "/home/test/testfile.txt", - .localPath = programDirectory / "temp" / "testfile.txt", + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), .tryContinue = true, - }}; + } + }; UploadOperation::WorkStatus workStatus; for (int workTimes = 0; workTimes < 100; ++workTimes) diff --git a/command-store/test/command-store/command_store.test.cpp b/command-store/test/command-store/command_store.test.cpp index 9c8d0c38..ec391dbb 100644 --- a/command-store/test/command-store/command_store.test.cpp +++ b/command-store/test/command-store/command_store.test.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -19,6 +20,8 @@ #include std::filesystem::path programDirectory; +/** @brief The temporary directory of this process alone, so tests running in parallel share no files. */ +std::filesystem::path temporaryRoot; namespace { @@ -225,7 +228,7 @@ namespace protected: boost::asio::io_context context_{}; - Utility::TemporaryDirectory temporaryDirectory_{programDirectory / "temp", true}; + Utility::TemporaryDirectory temporaryDirectory_{temporaryRoot, true}; std::optional store_{}; }; @@ -815,6 +818,9 @@ int main(int argc, char** argv) { programDirectory = std::filesystem::path{argv[0]}.parent_path(); + const Utility::ProcessTemporaryDirectory processTemporary{programDirectory / "temp"}; + temporaryRoot = processTemporary.path(); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/frontend/include/frontend/dialog/file_property_dialog.hpp b/frontend/include/frontend/dialog/file_property_dialog.hpp index cdf4afcd..722e5a0c 100644 --- a/frontend/include/frontend/dialog/file_property_dialog.hpp +++ b/frontend/include/frontend/dialog/file_property_dialog.hpp @@ -15,7 +15,10 @@ class FilePropertyDialog Nui::ElementRenderer operator()(); - void open(SharedData::DirectoryEntry const& entry); + /** + * @brief Shows @p entry found at @p path. A symlink's target is shown at @p targetPath, which may be empty. + */ + void open(SharedData::DirectoryEntry const& entry, std::string path, std::string targetPath = {}); private: struct Implementation; diff --git a/frontend/include/frontend/file_explorer/local_side_model.hpp b/frontend/include/frontend/file_explorer/local_side_model.hpp index 856a9cc6..b3abac5f 100644 --- a/frontend/include/frontend/file_explorer/local_side_model.hpp +++ b/frontend/include/frontend/file_explorer/local_side_model.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include @@ -15,13 +15,17 @@ #include #include +#include +#include #include #include #include #include #include +class RemoteSideModel; + class LocalSideModel : public SideModel , public NuiFileExplorer::IPlacesProvider @@ -105,13 +109,15 @@ class LocalSideModel { return openerCaps_; } - void setRemoteModel(SideModel* model); + void setRemoteModel(RemoteSideModel* model); + + /** + * @brief The directory the shown items were listed from, an error before the first listing. + */ + std::expected currentLocalPath() const; bool isComplete() const override; void navigateTo(std::filesystem::path const& path) override; - void onDirectoryListing( - std::optional> directoryEntries, - std::string const& reason = {} - ) override; + bool hasParent(std::filesystem::path const& directory) const override; void generatePathBoxSuggestions( std::filesystem::path const& path, @@ -122,21 +128,30 @@ class LocalSideModel private: void enqueueRefresh(bool otherModel); + /** + * @brief Renames @p from to @p to, asks before replacing an existing item and retries with @p overwrite. + */ + void renameItem(Paths::LocalPath const& from, Paths::LocalPath const& to, bool overwrite); + + /** + * @brief Where @p item is: a listed item below the directory it was listed from, a dropped one at its own + * absolute path. A listed name that is no single path segment, like "..", names no item. + */ + std::expected localPathOf(NuiFileExplorer::Item const& item) const; + + /** + * @brief Uploads @p candidates, asking per existing destination whether to overwrite it. + */ void uploadItemsConfirmed( - std::vector> uploadItems, + std::vector candidates, std::shared_ptr> existsResults, std::size_t index = 0, bool overwriteNever = false, bool overwriteAlways = false, - std::shared_ptr> accepted = nullptr + std::shared_ptr> accepted = nullptr ); - void enqueueSingleUpload( - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem, - bool allowOverwrite, - bool insertRefresh - ); + void enqueueSingleUpload(SharedData::BulkUploadEntry const& entry, bool allowOverwrite, bool insertRefresh); private: /** @@ -147,7 +162,7 @@ class LocalSideModel */ void probeOpenerCapabilities(); - SideModel* remoteModel_{nullptr}; + RemoteSideModel* remoteModel_{nullptr}; NuiFileExplorer::PathSuggestionCache pathSuggestionCache_; std::shared_ptr>> favorites_; std::function)> onFavoritesChanged_; diff --git a/frontend/include/frontend/file_explorer/open_local_file.hpp b/frontend/include/frontend/file_explorer/open_local_file.hpp new file mode 100644 index 00000000..3b8e4510 --- /dev/null +++ b/frontend/include/frontend/file_explorer/open_local_file.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include +#include + +namespace FileExplorer +{ + /** + * @brief Opens the local file @p path with its application. A file that can run code is opened only after the + * user confirmed it, when the settings ask for that; errors are shown in @p dialog. + */ + void openLocalFile(ConfirmDialog& dialog, Paths::LocalPath const& path, bool openWith); +} diff --git a/frontend/include/frontend/file_explorer/remote_side_model.hpp b/frontend/include/frontend/file_explorer/remote_side_model.hpp index e16400eb..b20603ee 100644 --- a/frontend/include/frontend/file_explorer/remote_side_model.hpp +++ b/frontend/include/frontend/file_explorer/remote_side_model.hpp @@ -14,7 +14,9 @@ #include #include +#include +#include #include #include #include @@ -23,6 +25,7 @@ #include class FileTrackingPanel; +class LocalSideModel; class RemoteSideModel : public SideModel @@ -99,7 +102,12 @@ class RemoteSideModel return false; } void navigateTo(std::filesystem::path const& path) override; - void setLocalModel(SideModel* model); + void setLocalModel(LocalSideModel* model); + + /** + * @brief The directory the shown items were listed from, an error before the first listing. + */ + std::expected currentRemotePath() const; void setFileTracking(FileTrackingPanel* fileTracking); bool isComplete() const override; @@ -112,31 +120,45 @@ class RemoteSideModel private: void enqueueRefresh(bool otherModel); + /** + * @brief Renames @p from to @p to, asks before replacing an existing item and retries with @p overwrite. + */ + void renameItem(Paths::RemotePath const& from, Paths::RemotePath const& to, bool overwrite); + + /** + * @brief Where @p item is on the server, below the directory it was listed from. A listed name that is no + * single path segment, like "..", names no item. + */ + std::expected remotePathOf(NuiFileExplorer::Item const& item) const; + + /** + * @brief @p name directly below the directory the shown items were listed from. + */ + std::expected childOfCurrentPath(std::string_view name) const; + + /** + * @brief Downloads @p candidates, asking per existing destination whether to overwrite it. + */ void downloadItemsConfirmed( - std::vector> downloadItems, + std::vector candidates, std::shared_ptr> existsResults, std::size_t index = 0, bool overwriteNever = false, bool overwriteAlways = false, - std::shared_ptr> accepted = nullptr + std::shared_ptr> accepted = nullptr ); - void enqueueSingleDownload( - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem, - bool allowOverwrite, - bool insertRefresh - ); + void enqueueSingleDownload(SharedData::BulkDownloadEntry const& entry, bool allowOverwrite, bool insertRefresh); void askNonEmptyDirectoryDeletions( - std::vector confirmedToDelete, - std::vector nonEmpties, - std::vector filesAndEmptyDirs, + std::vector confirmedToDelete, + std::vector nonEmpties, + std::vector filesAndEmptyDirs, std::size_t currentNonEmpty ); void - enqueueDeletes(std::vector nonEmpties, std::vector filesAndEmptyDirs); + enqueueDeletes(std::vector nonEmpties, std::vector filesAndEmptyDirs); void continueDeletion(); void onDownloadAndOpen(NuiFileExplorer::Item const& item, bool openWith); @@ -150,18 +172,18 @@ class RemoteSideModel ); void onFileDownloadComplete( Ids::InstanceId const& instanceId, - std::filesystem::path const& instanceDir, - std::filesystem::path const& remotePath, - std::filesystem::path const& localPath, + Paths::LocalPath const& instanceDirectory, + Paths::RemotePath const& remotePath, + Paths::LocalPath const& localPath, bool openWith, bool synchronize, bool success ); void onFileWatchAdded( Ids::InstanceId const& instanceId, - std::filesystem::path const& instanceDir, - std::filesystem::path const& remotePath, - std::filesystem::path const& localPath, + Paths::LocalPath const& instanceDirectory, + Paths::RemotePath const& remotePath, + Paths::LocalPath const& localPath, bool openWith ); @@ -185,7 +207,7 @@ class RemoteSideModel * @brief Requests waiting for the places request in flight. */ std::vector)>> pendingPlacesRequests_; - SideModel* localModel_{nullptr}; + LocalSideModel* localModel_{nullptr}; FileTrackingPanel* fileTracking_{nullptr}; NuiFileExplorer::PathSuggestionCache pathSuggestionCache_; std::shared_ptr>> favorites_; diff --git a/frontend/include/frontend/file_explorer/side_model.hpp b/frontend/include/frontend/file_explorer/side_model.hpp index 5e4ee17f..1801f57e 100644 --- a/frontend/include/frontend/file_explorer/side_model.hpp +++ b/frontend/include/frontend/file_explorer/side_model.hpp @@ -10,6 +10,7 @@ #include #include +#include #include @@ -111,7 +112,45 @@ class SideModel : public NuiFileExplorer::ISideModel // Design smell: virtual bool isComplete() const; + /** + * @brief A new name is a single path segment: not empty, not "." or "..", without '/', '\\' or NUL. + */ + static bool isValidItemName(std::string_view name); + + /** + * @brief Tells the user @p name cannot be used as a name. + */ + void showInvalidItemName(std::string const& name); + + /** + * @brief Tells the user an action was refused because of the path in @p error. + */ + void showPathRefused(std::string const& headerText, Paths::PathError const& error); + + /** + * @brief Tells the user a transfer stopped because the destinations could not be checked, @p reason says why. + */ + void showExistenceCheckFailed(std::string const& reason); + + /** + * @brief Asks whether the existing item @p name may be replaced, runs @p onReplace when the user agrees. + */ + void confirmReplaceOnRename(std::string const& name, std::function onReplace); + + /** + * @brief The directory has a parent to offer as "..". + */ + virtual bool hasParent(std::filesystem::path const& directory) const + { + return directory.has_relative_path(); + } + + /** + * @brief Shows the listing of @p directory. A reply for a directory the side has navigated away from is dropped, + * so no item carries the path of another directory. + */ virtual void onDirectoryListing( + std::filesystem::path const& directory, std::optional> directoryEntries, std::string const& reason = {} ); @@ -126,6 +165,8 @@ class SideModel : public NuiFileExplorer::ISideModel std::vector items_{}; OperationQueue* operationQueue_{nullptr}; Nui::Observed currentPath_{}; + /** @brief The directory the shown items were listed from, currentPath_ runs ahead while a listing loads. */ + std::filesystem::path listedPath_{}; std::filesystem::path preNavigatePath_{}; std::function refreshCallback_{nullptr}; bool reapplySelectionOnce_{false}; diff --git a/frontend/include/frontend/resumable_op.hpp b/frontend/include/frontend/resumable_op.hpp index 5b6c47d6..a1d72d0b 100644 --- a/frontend/include/frontend/resumable_op.hpp +++ b/frontend/include/frontend/resumable_op.hpp @@ -1,40 +1,56 @@ #pragma once #include +#include -#include +#include /** - * @brief Description of an operation that was in-flight when the SSH transport - * died and that we want to re-enqueue against the new session with - * tryContinue=true. Scan/sync operations are not resumable and are - * omitted from the snapshot. + * @brief Operations that were in flight when the SSH transport died, re-enqueued against the new session with + * tryContinue=true. Scans, sync operations, archives and local deletes are not resumable. */ -struct ResumableOp +namespace Resumable { - enum class Kind + struct Download { - Download, - Upload, - Delete, - Rename, - BulkDownload, - BulkUpload, - BulkDelete, + Paths::RemotePath source{}; + Paths::LocalPath destination{}; + /** @brief What the interrupted transfer was allowed, a resume must not replace more. */ + bool allowOverwrite{false}; }; - Kind kind; - std::filesystem::path src; - std::filesystem::path dst; - bool recursive{false}; - bool allowOverwrite{false}; - bool createMissingDirectories{false}; + struct Upload + { + Paths::LocalPath source{}; + Paths::RemotePath destination{}; + bool allowOverwrite{false}; + }; + + struct Rename + { + Paths::RemotePath source{}; + Paths::RemotePath destination{}; + }; /** - * @brief Backend-side resume key for bulk kinds. The new session - * passes this id to SessionManager::adoptBulkResume so the - * backend can re-issue the bulk from its own backup of the - * original entry list. Empty / invalid for single-file kinds. + * @brief A delete of one remote item. */ - Ids::OperationId operationId{}; -}; + struct Delete + { + Paths::RemotePath target{}; + /** @brief Only a delete that scanned a directory resumes with its content. */ + bool recursive{false}; + }; + + /** + * @brief A bulk card. The new session passes its id to SessionManager::adoptBulkResume and the backend + * re-issues the bulk from its own backup. + */ + struct Bulk + { + Ids::OperationId operationId{}; + }; +} + +using ResumableOp = + std::variant; diff --git a/frontend/include/frontend/session.hpp b/frontend/include/frontend/session.hpp index d6d677f0..805e7ab6 100644 --- a/frontend/include/frontend/session.hpp +++ b/frontend/include/frontend/session.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include class ProtoSession; @@ -160,7 +161,7 @@ class Session /** @brief Handles files dropped onto the session area (Windows). */ void - onDrop(bool isLocalSide, std::vector entries, std::optional const& subdir); + onDrop(bool isLocalSide, std::vector entries, std::optional const& subdir); /** * @brief Ejects local-shell channels from the aux engine. Used by diff --git a/frontend/include/frontend/session/session_helpers.hpp b/frontend/include/frontend/session/session_helpers.hpp index 3d81b38b..857217e5 100644 --- a/frontend/include/frontend/session/session_helpers.hpp +++ b/frontend/include/frontend/session/session_helpers.hpp @@ -3,6 +3,7 @@ #include #include +#include #include namespace SessionInternal @@ -15,8 +16,24 @@ namespace SessionInternal Nui::ElementRenderer resolveIdentityIcon(std::string const& iconName, bool isLocalShell); /** - * @brief Fires the RPC that writes @p content to @p filePath, with - * user-facing logging consistent with the multi-channel save path. + * @brief Writes @p content to @p filePath, which the user picked in a save dialog that confirmed replacing it. */ void writeChannelContentToFile(std::filesystem::path const& filePath, std::string const& content); + + enum class WriteOutcome + { + Written, + /// The file existed and was left alone. + Exists, + Failed, + }; + + /** + * @brief Writes @p content to the new file @p filePath, an existing file is never replaced. + */ + void writeChannelContentToNewFile( + std::filesystem::path const& filePath, + std::string const& content, + std::function onDone + ); } diff --git a/frontend/include/frontend/session_components/file_explorer_panel.hpp b/frontend/include/frontend/session_components/file_explorer_panel.hpp index f4f9f67c..b4db7699 100644 --- a/frontend/include/frontend/session_components/file_explorer_panel.hpp +++ b/frontend/include/frontend/session_components/file_explorer_panel.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -111,7 +112,7 @@ class FileExplorerPanel */ void onDrop( bool isLocalSide, - std::vector entries, + std::vector entries, std::optional const& subdir ); diff --git a/frontend/include/frontend/session_components/file_tracking.hpp b/frontend/include/frontend/session_components/file_tracking.hpp index 57257be7..d884127c 100644 --- a/frontend/include/frontend/session_components/file_tracking.hpp +++ b/frontend/include/frontend/session_components/file_tracking.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -36,15 +37,15 @@ class FileTrackingPanel * @brief Register a new watched entry after a priority download has been initiated. * * @param instanceId The FileTracking instance ID returned by createInstance. - * @param instanceDir The local temporary directory for this instance. + * @param instanceDirectory The local temporary directory for this instance. * @param remotePath Absolute remote path that was downloaded. - * @param localPath Local path inside instanceDir where the file/dir lives. + * @param localPath Local path inside instanceDirectory where the file/dir lives. */ void startWatching( Ids::InstanceId const& instanceId, - std::filesystem::path const& instanceDir, - std::filesystem::path const& remotePath, - std::filesystem::path const& localPath + Paths::LocalPath const& instanceDirectory, + Paths::RemotePath const& remotePath, + Paths::LocalPath const& localPath ); Nui::ElementRenderer operator()(); diff --git a/frontend/include/frontend/session_components/operation_queue.hpp b/frontend/include/frontend/session_components/operation_queue.hpp index 5d1e9006..9b4d3d4c 100644 --- a/frontend/include/frontend/session_components/operation_queue.hpp +++ b/frontend/include/frontend/session_components/operation_queue.hpp @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -62,11 +63,17 @@ class OperationQueue ROAR_PIMPL_SPECIAL_FUNCTIONS(OperationQueue); void activate(std::shared_ptr fileEngine, Ids::SessionId sessionId); + + /** + * @brief The file engine of the session, empty before @ref activate. + */ + std::shared_ptr fileEngine() const; void deactivate(); void enqueueDownload( - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem, + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t sizeBytes, std::function const&, std::string const& info)> onComplete, bool allowOverwrite, bool insertRefresh, @@ -74,8 +81,9 @@ class OperationQueue SharedData::OperationMode mode = SharedData::OperationMode::Queued ); void enqueueUpload( - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem, + Paths::LocalPath const& source, + Paths::RemotePath const& destination, + std::uint64_t sizeBytes, std::function const&, std::string const& info)> onComplete, bool allowOverwrite, bool insertRefresh, @@ -83,13 +91,13 @@ class OperationQueue SharedData::OperationMode mode = SharedData::OperationMode::Queued ); void enqueueRename( - std::filesystem::path const& oldPath, - std::filesystem::path const& newPath, + Paths::RemotePath const& oldPath, + Paths::RemotePath const& newPath, std::function const&, std::string const& info)> onComplete, SharedData::OperationMode mode = SharedData::OperationMode::Queued ); void enqueueDelete( - std::vector const& paths, + std::vector const& paths, bool recursive, std::function> const&, std::string const& info)> onComplete, SharedData::OperationMode mode = SharedData::OperationMode::Queued @@ -117,7 +125,7 @@ class OperationQueue * transfer-progress callbacks per entry. */ void enqueueBulkDownload( - std::vector entries, + std::vector entries, bool allowOverwrite, bool insertRefresh, SharedData::OperationMode mode, @@ -135,8 +143,8 @@ class OperationQueue * user-facing 1..9 slider value, remapped per-codec on the backend. */ void enqueueArchiveDownload( - std::vector entries, - std::filesystem::path const& localArchivePath, + std::vector roots, + Paths::LocalPath const& localArchivePath, int compressionCodec, int compressionLevel, bool mayOverwrite, @@ -146,8 +154,8 @@ class OperationQueue /** @brief Upload analogue of enqueueArchiveDownload — see its docs. */ void enqueueArchiveUpload( - std::vector localPaths, - std::filesystem::path const& remoteArchivePath, + std::vector localPaths, + Paths::RemotePath const& remoteArchivePath, int compressionCodec, int compressionLevel, bool mayOverwrite, @@ -157,7 +165,7 @@ class OperationQueue /** @brief Upload analogue of enqueueBulkDownload — see its docs. */ void enqueueBulkUpload( - std::vector entries, + std::vector entries, bool allowOverwrite, bool insertRefresh, SharedData::OperationMode mode, @@ -171,7 +179,7 @@ class OperationQueue * @param onEnqueued Invoked synchronously with the aggregate file-bulk * OperationId before the RPC dispatches. */ void enqueueBulkDelete( - std::vector entries, + std::vector entries, bool insertRefresh, SharedData::OperationMode mode, std::function onBulkComplete, @@ -186,7 +194,7 @@ class OperationQueue */ void addFailedEntriesCallback( Ids::OperationId const& opId, - std::function const& failedPaths)> callback + std::function const& failedPaths)> callback ); /** @brief Register a callback that receives progress as a 0.0–1.0 fraction for an upload/download. * Automatically removed when the operation completes. @@ -206,37 +214,8 @@ class OperationQueue /** @brief Observable pause state — observe() it to react to pause/unpause changes. */ Nui::Observed& pausedState(); - /** @brief Enqueues a priority remote scan and a priority local scan for sync comparison. - * All callbacks are registered before the backend RPC is called to avoid missing progress. - * - * @param localPath Local directory root to scan. - * @param remotePath Remote directory root to scan. - * @param onRemoteProgress Called on each remote scan progress update. - * @param onLocalProgress Called on each local scan progress update. - * @param onRemoteComplete Called with the remote scan result when it finishes. - * @param onLocalComplete Called with the local scan result when it finishes. - */ - /** @brief Fires a remote-side createDirectory RPC. Used by the sync dialog in - * non-recursive mode where transferring a missing directory must not - * turn into a recursive bulk upload that could clobber existing files. - * Fire-and-forget (no operation ID, no queue entry). - */ - void createRemoteDirectory( - std::filesystem::path const& path, - std::function onComplete - ); - - /** @brief Fires a local-side createDirectory RPC (mirror of createRemoteDirectory). */ - void createLocalDirectory( - std::filesystem::path const& path, - std::function onComplete - ); - - /** @brief Removes local paths, directories with their content. Not queued, like @ref createLocalDirectory. */ - void removeLocalPaths( - std::vector const& paths, - std::function onComplete - ); + /** @brief Drops every callback registered for @p opId, for an operation that was never queued. */ + void removeCallbacks(Ids::OperationId const& opId); /** @brief Shows a restore button in the queue header for a minimized SyncDialog. * The button pulses once on call (and on subsequent calls) to catch the user's eye. @@ -269,6 +248,7 @@ class OperationQueue std::function onRemoteProgress, std::function onLocalProgress, std::function onScanPhaseDone, + std::function onScanFailed, std::function onDiffProgress ); @@ -298,12 +278,9 @@ class OperationQueue std::function onRejected ); - void buildSyncEnqueuePlan( - Ids::SyncSessionId syncSessionId, - SharedData::Sync::DiffSection section, - std::vector selectedRelKeys, - std::uint64_t generation, - std::function)> onResolved, + void executeSyncSelection( + SharedData::Sync::SyncExecutionRequest request, + std::function onResolved, std::function onRejected ); diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp index 445dc4a6..04e12887 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_bulk_operation.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -19,11 +20,12 @@ struct DisplayedBulkOperation : public OperationCard Ids::OperationId operationId, ConfirmDialog& confirmDialog, SharedData::OperationType type, - std::filesystem::path localPath, - std::filesystem::path remotePath, + Paths::LocalPath localPath, + Paths::RemotePath remotePath, std::function doRemoveSelf, std::shared_ptr> doDeletionCountdown, - std::function onCompleteAction + std::function onCompleteAction, + bool scanning = false ) : OperationCard{ type, @@ -35,6 +37,7 @@ struct DisplayedBulkOperation : public OperationCard } , localPath_{std::move(localPath)} , remotePath_{std::move(remotePath)} + , scanPhase_{scanning} , totalProgressBar_({ .height = std::string{progressHeight}, .min = 0, @@ -49,8 +52,15 @@ struct DisplayedBulkOperation : public OperationCard return true; } + void setScanProgress(std::uint64_t scannedBytes, std::uint64_t scannedEntries) + { + scanPhase_.setProgress(scannedBytes, scannedEntries); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + void setProgress(SharedData::BulkProgress const& progress) { + scanPhase_.end(); // A row keeps its file until that file leaves the in-flight set; new files take the // first free row, so nothing shifts when an earlier file finishes. const auto inFlight = [&progress](std::uint64_t entryIndex) @@ -124,29 +134,33 @@ struct DisplayedBulkOperation : public OperationCard style = "flex-grow: 1; overflow: hidden; min-width: 0;" }( [this]() -> Nui::ElementRenderer { - const auto& srcPath = type_ == SharedData::OperationType::BulkUpload ? localPath_ : remotePath_; - const auto& dstPath = type_ == SharedData::OperationType::BulkUpload ? remotePath_ : localPath_; + const auto localText = localPath_.toUtf8(); + const auto remoteText = remotePath_.value(); + const auto& srcPath = type_ == SharedData::OperationType::BulkUpload ? localText : remoteText; + const auto& dstPath = type_ == SharedData::OperationType::BulkUpload ? remoteText : localText; return div{ class_ = "opq-transfer-route", - alt = fmt::format("{} → {}", srcPath.generic_string(), dstPath.generic_string()) + alt = fmt::format("{} → {}", srcPath, dstPath) }( span{ class_ = "opq-route-segment" - }(srcPath.generic_string()), + }(srcPath), span{ class_ = "opq-route-arrow" }(Svgs::arrowRight()), span{ class_ = "opq-route-segment" - }(dstPath.generic_string()) + }(dstPath) ); }() ), span{ class_ = "opq-status-text opq-status-count" }( - observe(fileCurrentIndex, fileCount, state_), + observe(fileCurrentIndex, fileCount, state_, scanPhase_.scanning, scanPhase_.totalScanned), [this](){ + if (scanPhase_.scanning.value() && !isCompletedState()) + return scanPhase_.text(); // On successful completion, pin the numerator to the // total so a dropped or late terminal progress tick // can't leave the card stuck at (N-1)/N. @@ -213,22 +227,7 @@ struct DisplayedBulkOperation : public OperationCard { if (isCompletedState()) return std::nullopt; - ResumableOp out; - if (type_ == SharedData::OperationType::BulkDownload) - { - out.kind = ResumableOp::Kind::BulkDownload; - out.src = remotePath_; - out.dst = localPath_; - } - else - { - out.kind = ResumableOp::Kind::BulkUpload; - out.src = localPath_; - out.dst = remotePath_; - } - out.allowOverwrite = true; - out.operationId = operationId(); - return out; + return Resumable::Bulk{.operationId = operationId()}; } private: @@ -341,8 +340,9 @@ struct DisplayedBulkOperation : public OperationCard } private: - std::filesystem::path localPath_; - std::filesystem::path remotePath_; + Paths::LocalPath localPath_; + Paths::RemotePath remotePath_; + ScanPhaseStatus scanPhase_; Nui::Observed fileCurrentIndex{0ull}; Nui::Observed fileCount{0ull}; Nui::Observed> bytesPerSecond{0}; diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp index d311eab6..1f4e5f70 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_delete_operation.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -12,10 +13,12 @@ struct DisplayedDeleteOperation : public OperationCard Ids::OperationId operationId, ConfirmDialog& confirmDialog, SharedData::OperationType type, - std::filesystem::path removePath, + std::string displayPath, + Paths::RemotePath target, std::function doRemoveSelf, std::shared_ptr> doDeletionCountdown, - std::function onCompleteAction + std::function onCompleteAction, + bool scanning = false ) : OperationCard{ type, @@ -25,7 +28,10 @@ struct DisplayedDeleteOperation : public OperationCard std::move(doDeletionCountdown), std::move(onCompleteAction) } - , removePath_{std::move(removePath)} + , displayPath_{std::move(displayPath)} + , target_{std::move(target)} + , recursive_{scanning} + , scanPhase_{scanning} , progressBar_({ .height = std::string{progressHeight}, .min = 0, @@ -47,8 +53,15 @@ struct DisplayedDeleteOperation : public OperationCard progressBar_.setZeroAsComplete(); } + void setScanProgress(std::uint64_t scannedBytes, std::uint64_t scannedEntries) + { + scanPhase_.setProgress(scannedBytes, scannedEntries); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + void setProgress(SharedData::BulkDeleteProgress const& progress) { + scanPhase_.end(); if (currentFile.value() != progress.currentFile) currentFile = progress.currentFile; @@ -65,24 +78,15 @@ struct DisplayedDeleteOperation : public OperationCard std::optional resumableDescriptor() const override { - if (isCompletedState()) + // A local delete runs in this process and needs no connection to be redone. + if (isCompletedState() || type_ == SharedData::OperationType::LocalDelete) return std::nullopt; - ResumableOp out; - // BulkDelete reuses this card class; differentiate so the resume - // routes to the backend's saved-entry-list path instead of the - // single-Delete frontend path. + // A bulk delete is re-issued by the backend from its own backup. if (type_ == SharedData::OperationType::BulkDelete) - { - out.kind = ResumableOp::Kind::BulkDelete; - out.operationId = operationId(); - } - else - { - out.kind = ResumableOp::Kind::Delete; - out.recursive = true; - } - out.src = removePath_; - return out; + return Resumable::Bulk{.operationId = operationId()}; + if (target_.empty()) + return std::nullopt; + return Resumable::Delete{.target = target_, .recursive = recursive_}; } Nui::ElementRenderer body() const override @@ -99,7 +103,7 @@ struct DisplayedDeleteOperation : public OperationCard div{}( div{ class_ = "opq-transfer-route", - alt = removePath_.generic_string() + alt = displayPath_ }( span{ class_ = "opq-route-segment" @@ -107,9 +111,11 @@ struct DisplayedDeleteOperation : public OperationCard observe(currentFile), [this](){ if (type_ == SharedData::OperationType::BulkDelete) - return fmt::format(fmt::runtime(language->get("operationQueue", "deletingSelectedIn")), removePath_.generic_string()); + return fmt::format(fmt::runtime(language->get("operationQueue", "deletingSelectedIn")), displayPath_); + if (type_ == SharedData::OperationType::LocalDelete) + return fmt::format(fmt::runtime(language->get("operationQueue", "deletingLocallyIn")), displayPath_); if (currentFile.value().empty()) - return fmt::format(fmt::runtime(language->get("operationQueue", "deletingItem")), removePath_.generic_string()); + return fmt::format(fmt::runtime(language->get("operationQueue", "deletingItem")), displayPath_); return fmt::format(fmt::runtime(language->get("operationQueue", "deletingItem")), currentFile.value()); } ) @@ -117,8 +123,10 @@ struct DisplayedDeleteOperation : public OperationCard span{ class_ = "opq-status-text" }( - observe(filesDeleted_, totalFiles_), + observe(filesDeleted_, totalFiles_, scanPhase_.scanning, scanPhase_.totalScanned), [this](){ + if (scanPhase_.scanning.value()) + return scanPhase_.text(); return fmt::format("{}/{}", filesDeleted_.value(), totalFiles_.value()); } ) @@ -132,6 +140,11 @@ struct DisplayedDeleteOperation : public OperationCard Nui::Observed currentFile{""}; Nui::Observed filesDeleted_{0ull}; Nui::Observed totalFiles_{0ull}; - std::filesystem::path removePath_; + std::string displayPath_; + /** @brief The remote item a single delete removes, unset for local and bulk deletes. */ + Paths::RemotePath target_; + /** @brief The delete scanned a directory, so it removes content and resumes doing the same. */ + bool recursive_{false}; + ScanPhaseStatus scanPhase_; Components::ProgressBar progressBar_; }; \ No newline at end of file diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_operation.hpp index afc4e3ac..c1eec91a 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_operation.hpp @@ -64,7 +64,7 @@ struct DisplayedOperation return dynamic_cast(card_.get()); } - void failedEntries(std::vector> entries) + void failedEntries(std::vector> entries) { if (auto* specificCard = getCardSpecifically()) { diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_rename_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_rename_operation.hpp index fb624613..0a969083 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_rename_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_rename_operation.hpp @@ -9,8 +9,8 @@ class DisplayedRenameOperation : public OperationCard DisplayedRenameOperation( Ids::OperationId operationId, ConfirmDialog& confirmDialog, - std::filesystem::path sourcePath, - std::filesystem::path destinationPath, + Paths::RemotePath sourcePath, + Paths::RemotePath destinationPath, std::function doRemoveSelf, std::shared_ptr> doDeletionCountdown, std::function onCompleteAction @@ -36,11 +36,7 @@ class DisplayedRenameOperation : public OperationCard { if (isCompletedState()) return std::nullopt; - ResumableOp out; - out.kind = ResumableOp::Kind::Rename; - out.src = sourcePath_; - out.dst = destinationPath_; - return out; + return Resumable::Rename{.source = sourcePath_, .destination = destinationPath_}; } Nui::ElementRenderer body() const override @@ -55,8 +51,8 @@ class DisplayedRenameOperation : public OperationCard span{}( fmt::format( fmt::runtime(language->get("operationQueue", "renameCard")), - sourcePath_.filename().string(), - destinationPath_.filename().string() + nameOf(sourcePath_), + nameOf(destinationPath_) ) ), div{}() @@ -65,6 +61,14 @@ class DisplayedRenameOperation : public OperationCard } private: - std::filesystem::path sourcePath_; - std::filesystem::path destinationPath_; + static std::string nameOf(Paths::RemotePath const& path) + { + if (auto name = path.filename(); name) + return name->value(); + return path.value(); + } + + private: + Paths::RemotePath sourcePath_; + Paths::RemotePath destinationPath_; }; diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_scan_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_scan_operation.hpp index 643d1e7d..fc8b4c03 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_scan_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_scan_operation.hpp @@ -9,7 +9,7 @@ class DisplayedScanOperation : public OperationCard DisplayedScanOperation( Ids::OperationId operationId, ConfirmDialog& confirmDialog, - std::filesystem::path remotePath, + std::string displayPath, std::function doRemoveSelf, std::shared_ptr> doDeletionCountdown ) @@ -21,7 +21,7 @@ class DisplayedScanOperation : public OperationCard std::move(doDeletionCountdown), []() {} } - , remotePath_{std::move(remotePath)} + , displayPath_{std::move(displayPath)} {} Nui::ElementRenderer body() const override @@ -34,7 +34,7 @@ class DisplayedScanOperation : public OperationCard // clang-format off return fragment( span{}( - fmt::format(fmt::runtime(language->get("operationQueue", "scanCard")), remotePath_.string()) + fmt::format(fmt::runtime(language->get("operationQueue", "scanCard")), displayPath_) ), // Spanner div{}(), @@ -67,7 +67,7 @@ class DisplayedScanOperation : public OperationCard } private: - std::filesystem::path remotePath_; + std::string displayPath_; Nui::Observed totalBytes_{0ull}; Nui::Observed currentIndex_{0ull}; Nui::Observed totalScanned_{0ull}; diff --git a/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp b/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp index 6bcd1ccf..43ca67b9 100644 --- a/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp +++ b/frontend/include/frontend/session_components/operation_queue/displayed_transfer_operation.hpp @@ -2,6 +2,7 @@ #pragma once #include +#include #include #include @@ -14,11 +15,12 @@ class DisplayedTransferOperation : public OperationCard doRemoveSelf, std::shared_ptr> doDeletionCountdown, - std::function onCompleteAction + std::function onCompleteAction, + bool scanning = false ) : OperationCard{ type, @@ -37,6 +39,7 @@ class DisplayedTransferOperation : public OperationCard bytesPerSecond) - { - return fmt::format("{}/s", Utility::formatBytes(bytesPerSecond, Utility::determineOrderOfMagnitude(bytesPerSecond))); - } - ) + observe(bytesPerSecond_, scanPhase_.scanning, scanPhase_.totalScanned), + [this]() + { + if (scanPhase_.scanning.value() && !isCompletedState()) + return scanPhase_.text(); + const auto bytesPerSecond = bytesPerSecond_.value(); + return fmt::format("{}/s", Utility::formatBytes(bytesPerSecond, Utility::determineOrderOfMagnitude(bytesPerSecond))); + } ) ), progressBar_() @@ -98,8 +105,15 @@ class DisplayedTransferOperation : public OperationCard resumableDescriptor() const override { if (isCompletedState()) return std::nullopt; - ResumableOp out; if (type_ == SharedData::OperationType::Download) - { - out.kind = ResumableOp::Kind::Download; - out.src = remotePath_; - out.dst = localPath_; - } - else - { - out.kind = ResumableOp::Kind::Upload; - out.src = localPath_; - out.dst = remotePath_; - } - out.allowOverwrite = true; - return out; + return Resumable::Download{ + .source = remotePath_, .destination = localPath_, .allowOverwrite = allowOverwrite_ + }; + if (type_ == SharedData::OperationType::Upload) + return Resumable::Upload{.source = localPath_, .destination = remotePath_, .allowOverwrite = allowOverwrite_}; + // An archive card does not know the entries it packs, so it cannot be redone. + return std::nullopt; } private: Components::ProgressBar progressBar_; Nui::Observed> bytesPerSecond_{0}; - std::filesystem::path localPath_; - std::filesystem::path remotePath_; + /** @brief Empty for an archive upload. */ + Paths::LocalPath localPath_; + bool allowOverwrite_{false}; + /** @brief Unset for an archive download. */ + Paths::RemotePath remotePath_; + ScanPhaseStatus scanPhase_; }; \ No newline at end of file diff --git a/frontend/include/frontend/session_components/operation_queue/operation_card.hpp b/frontend/include/frontend/session_components/operation_queue/operation_card.hpp index 547f89ae..f1d30bde 100644 --- a/frontend/include/frontend/session_components/operation_queue/operation_card.hpp +++ b/frontend/include/frontend/session_components/operation_queue/operation_card.hpp @@ -176,7 +176,8 @@ class OperationCard : public OperationCardInterface return Ui5Icons::edit(); } else if (type_ == SharedData::OperationType::Delete || - type_ == SharedData::OperationType::BulkDelete) + type_ == SharedData::OperationType::BulkDelete || + type_ == SharedData::OperationType::LocalDelete) { return Ui5Icons::delete_(); } @@ -237,7 +238,7 @@ class OperationCard : public OperationCardInterface { transformedItems.reserve(failedEntries_.value().size()); for (const auto& [path, entryError] : failedEntries_.value()) - transformedItems.push_back({.text = fmt::format("{}: {}", path.string(), entryError.toString())}); + transformedItems.push_back({.text = fmt::format("{}: {}", path, entryError.toString())}); } confirmDialog_->open(ConfirmDialog::OpenOptions{ @@ -342,7 +343,7 @@ class OperationCard : public OperationCardInterface // return error; } - void failedEntries(std::vector> entries) override + void failedEntries(std::vector> entries) override { failedEntries_ = std::move(entries); } @@ -364,6 +365,6 @@ class OperationCard : public OperationCardInterface std::function doRemoveSelf_; std::shared_ptr> doDeletionCountdown_; std::function onCompleteAction_; - Nui::Observed>> failedEntries_; + Nui::Observed>> failedEntries_; std::function kickToTopHandler_; }; \ No newline at end of file diff --git a/frontend/include/frontend/session_components/operation_queue/operation_card_interface.hpp b/frontend/include/frontend/session_components/operation_queue/operation_card_interface.hpp index 996e5ab0..f7935bdf 100644 --- a/frontend/include/frontend/session_components/operation_queue/operation_card_interface.hpp +++ b/frontend/include/frontend/session_components/operation_queue/operation_card_interface.hpp @@ -35,7 +35,7 @@ class OperationCardInterface virtual std::chrono::steady_clock::time_point completionTime() const = 0; virtual Nui::ElementRenderer operator()() const = 0; virtual void setError(SharedData::OperationError const& error) = 0; - virtual void failedEntries(std::vector> entries) = 0; + virtual void failedEntries(std::vector> entries) = 0; /** * @brief Install a click handler for the kick-to-top affordance. The diff --git a/frontend/include/frontend/session_components/operation_queue/scan_phase_status.hpp b/frontend/include/frontend/session_components/operation_queue/scan_phase_status.hpp new file mode 100644 index 00000000..ce4d2e48 --- /dev/null +++ b/frontend/include/frontend/session_components/operation_queue/scan_phase_status.hpp @@ -0,0 +1,51 @@ +#pragma once + +#include +#include + +#include + +#include + +#include +#include + +/** + * @brief Scan progress of a card whose operation scans before its own work. + */ +struct ScanPhaseStatus +{ + explicit ScanPhaseStatus(bool isScanning) + : scanning{isScanning} + {} + + void setProgress(std::uint64_t scannedBytes, std::uint64_t scannedEntries) + { + // Scan progress can arrive after the operation already moved on. + if (!scanning.value()) + return; + if (totalBytes.value() != scannedBytes) + totalBytes = scannedBytes; + if (totalScanned.value() != scannedEntries) + totalScanned = scannedEntries; + } + + void end() + { + if (scanning.value()) + scanning = false; + } + + std::string text() const + { + return fmt::format( + fmt::runtime(language->get("operationQueue", "scanningItems")), + totalScanned.value(), + Utility::formatBytes(static_cast(totalBytes.value())) + ); + } + + Nui::Observed scanning; + Nui::Observed totalBytes{0u}; + Nui::Observed totalScanned{0u}; +}; diff --git a/frontend/include/frontend/settings/general_settings.hpp b/frontend/include/frontend/settings/general_settings.hpp index 89008066..dd0ab14c 100644 --- a/frontend/include/frontend/settings/general_settings.hpp +++ b/frontend/include/frontend/settings/general_settings.hpp @@ -80,6 +80,7 @@ struct GeneralSettings BoolSetting<> preventCreateDirectory; PathSetting homeOverride; PathSetting temporaryDownloadsDirectory; + ComboSetting openRiskyFiles; } localFilesystemOptions; struct FileTrackingOptions @@ -87,6 +88,7 @@ struct GeneralSettings BoolSetting<> autoReupload; BoolSetting<> moveRemoteOnLocalMove; BoolSetting<> deleteRemoteOnLocalDelete; + BoolSetting<> checkRemoteChanges; } fileTrackingOptions; LogOptions logOptions; diff --git a/frontend/include/frontend/settings/sftp_options.hpp b/frontend/include/frontend/settings/sftp_options.hpp index 5eb26e19..04b7dd84 100644 --- a/frontend/include/frontend/settings/sftp_options.hpp +++ b/frontend/include/frontend/settings/sftp_options.hpp @@ -41,6 +41,7 @@ struct SftpOptions : public GroupKeys NumberSetting customDirectoryPermissions; BoolSetting reserveSpace; BoolSetting doCleanup; + BoolSetting markAsDownloaded; ComboSetting symlinkHandling; BoolSetting failFast; }; diff --git a/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp b/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp index 265d3ae4..fdc62019 100644 --- a/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp +++ b/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include @@ -55,7 +55,8 @@ class BackendSyncProvider /** * @brief Kicks off both scans. @p onBothListed fires after both sides have - * reported phase-done (the order between them isn't significant). + * reported phase-done (the order between them isn't significant). @p onScanFailed fires instead, once, + * when a scan fails or is cancelled. */ void open( std::filesystem::path localPath, @@ -66,6 +67,7 @@ class BackendSyncProvider std::function onLocalListing, std::function onRemoteListing, std::function onBothListed, + std::function onScanFailed, std::function onDiffProgress ); @@ -91,14 +93,12 @@ class BackendSyncProvider ); /** - * @brief Asks the backend to collapse @p selectedRelKeys into a minimal - * enqueue plan for @p section. Uses the cached @ref generation_ so - * stale requests (after a recompute) are rejected by the server. + * @brief Has the backend plan and queue @p request for this session. Uses the cached @ref generation_ so a + * request against a recomputed diff is refused by the backend. */ - void buildEnqueuePlan( - SharedData::Sync::DiffSection section, - std::vector selectedRelKeys, - std::function)> onResolved, + void executeSelection( + SharedData::Sync::SyncExecutionRequest request, + std::function onResolved, std::function onRejected ); diff --git a/frontend/include/frontend/sync_dialog/sync_dialog.hpp b/frontend/include/frontend/sync_dialog/sync_dialog.hpp index 073f3468..61109863 100644 --- a/frontend/include/frontend/sync_dialog/sync_dialog.hpp +++ b/frontend/include/frontend/sync_dialog/sync_dialog.hpp @@ -9,6 +9,8 @@ #include #include +#include +#include #include #include @@ -45,15 +47,22 @@ class SyncDialog * served lazily from @p provider). * @param remotePath Remote scan root. * @param scanned Settings the scans of @p provider ran with. + * @param lastSyncTime When this pair was last synchronized, every recompute compares against it. */ void open( - BackendSyncProvider* provider, + std::shared_ptr provider, SharedData::Sync::DiffSnapshot snapshot, std::filesystem::path localPath, std::filesystem::path remotePath, - ScanSettings scanned + ScanSettings scanned, + std::optional lastSyncTime ); + /** + * @brief Called when the user synchronized the whole selection, so the session can record the sync. + */ + void setOnSynchronized(std::function callback); + /** * @brief Scan + diff settings snapshot handed to the Recompare callback. */ diff --git a/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp b/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp index 49a88ea3..7e9aeebb 100644 --- a/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp +++ b/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp @@ -38,7 +38,7 @@ class SyncProgressDialog * @param onDone Called exactly once on success. Not invoked on cancel. */ void open( - BackendSyncProvider* provider, + std::shared_ptr provider, std::filesystem::path localPath, std::filesystem::path remotePath, bool respectIgnoreFiles, diff --git a/frontend/include/frontend/terminal/file_engine.hpp b/frontend/include/frontend/terminal/file_engine.hpp index 8804049c..8efb16e1 100644 --- a/frontend/include/frontend/terminal/file_engine.hpp +++ b/frontend/include/frontend/terminal/file_engine.hpp @@ -2,12 +2,14 @@ #include #include -#include +#include +#include +#include #include #include #include #include -#include +#include #include #include #include @@ -57,8 +59,8 @@ class FileEngine ); void - createDirectory(std::filesystem::path const& path, std::function onComplete); - void createFile(std::filesystem::path const& path, std::function onComplete); + createDirectory(Paths::RemotePath const& path, std::function onComplete); + void createFile(Paths::RemotePath const& path, std::function onComplete); /** * @brief Opens a backend-resident SyncSession and queues both scans. The scans @@ -107,17 +109,12 @@ class FileEngine ); /** - * @brief Asks the backend to collapse the selected relKey SPARSE set in @p section - * into a minimal enqueue plan. An entry X in @p selectedRelKeys means "X - * and every descendant selected"; the backend-resident minimizer handles - * bulk-dir / structural-dir rules. + * @brief Has the backend plan and queue the selected rows of one section, see + * @ref SharedData::Sync::SyncExecutionRequest. */ - void buildSyncEnqueuePlan( - Ids::SyncSessionId syncSessionId, - SharedData::Sync::DiffSection section, - std::vector selectedRelKeys, - std::uint64_t generation, - std::function)> onResolved, + void executeSyncSelection( + SharedData::Sync::SyncExecutionRequest request, + std::function onResolved, std::function onRejected ); @@ -128,8 +125,9 @@ class FileEngine void closeSyncSession(Ids::SyncSessionId syncSessionId); void addDownload( - NuiFileExplorer::Item const& remotePath, - NuiFileExplorer::Item const& localPath, + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t sizeBytes, std::function, std::string const& info)> onOperationCreated, bool allowOverwrite, bool insertRefresh, @@ -137,8 +135,9 @@ class FileEngine SharedData::OperationMode mode = SharedData::OperationMode::Queued ); void addUpload( - NuiFileExplorer::Item const& remotePath, - NuiFileExplorer::Item const& localPath, + Paths::LocalPath const& source, + Paths::RemotePath const& destination, + std::uint64_t sizeBytes, std::function, std::string const& info)> onOperationCreated, bool allowOverwrite, bool insertRefresh, @@ -164,7 +163,7 @@ class FileEngine * before the RPC returns). */ void addBulkDownload( - std::vector entries, + std::vector entries, std::vector operationIds, bool allowOverwrite, bool insertRefresh, @@ -181,8 +180,8 @@ class FileEngine * codec on the backend). */ void addArchiveDownload( - std::vector entries, - std::filesystem::path const& localArchivePath, + std::vector roots, + Paths::LocalPath const& localArchivePath, int compressionCodec, int compressionLevel, bool mayOverwrite, @@ -196,8 +195,8 @@ class FileEngine * addArchiveDownload with source/destination reversed. */ void addArchiveUpload( - std::vector localPaths, - std::filesystem::path const& remoteArchivePath, + std::vector localPaths, + Paths::RemotePath const& remoteArchivePath, int compressionCodec, int compressionLevel, bool mayOverwrite, @@ -207,7 +206,7 @@ class FileEngine /** @brief Upload analogue of addBulkDownload — see its docs. */ void addBulkUpload( - std::vector entries, + std::vector entries, std::vector operationIds, bool allowOverwrite, bool insertRefresh, @@ -219,26 +218,25 @@ class FileEngine * @brief Bulk-delete: collapse N file deletions into a single bulk-delete * card on the frontend. One RPC, one OperationId for the * aggregate file card; directory entries get their own - * Scan+Delete cards on the backend. `dst` and `sizeBytes` of - * each entry are ignored — only `src` (target path) and - * `isDirectory` matter. + * Scan+Delete cards on the backend. */ void addBulkDelete( - std::vector entries, + std::vector entries, Ids::OperationId bulkOperationId, bool insertRefresh, SharedData::OperationMode mode, std::function onBulkCreated ); + /** + * @brief Asks the server which of @p directories are not empty and hands the split to + * @p onNonEmptyDirectoriesFound: files and empty directories first, non-empty directories second. + */ void remove( - std::vector const& files, - std::vector const& directories, + std::vector files, + std::vector directories, std::function onComplete, - std::function, /* regular files & empty */ - std::vector /* non empties */ - )> onNonEmptyDirectoriesFound + std::function, std::vector)> onNonEmptyDirectoriesFound ); /** @@ -249,21 +247,26 @@ class FileEngine * @param onComplete */ void removeOnQueueUnchecked( - std::vector const& paths, + std::vector const& paths, bool recursive, std::function> const&, std::string const& info)> onComplete, SharedData::OperationMode mode = SharedData::OperationMode::Queued ); + /** + * @brief Renames on the server. Fails with "backend.rename.targetExists" when @p newPath exists and + * @p overwrite is false. + */ void rename( - std::filesystem::path const& oldPath, - std::filesystem::path const& newPath, + Paths::RemotePath const& oldPath, + Paths::RemotePath const& newPath, + bool overwrite, std::function onComplete ); void addRename( - std::filesystem::path const& sourcePath, - std::filesystem::path const& destinationPath, + Paths::RemotePath const& sourcePath, + Paths::RemotePath const& destinationPath, std::function, std::string const& info)> onOperationCreated, SharedData::OperationMode mode = SharedData::OperationMode::Queued ); @@ -286,7 +289,7 @@ class FileEngine * diagnostic. */ void existsBatchRemote( - std::vector const& paths, + std::vector const& paths, std::function exists, std::string const& info)> onComplete ); @@ -294,11 +297,6 @@ class FileEngine private: void lazyOpen(std::function const&, std::string const& info)> const& onOpen); - void performDelete( - std::vector files, - std::vector directoriesEmpty, - std::function onComplete - ); private: struct Implementation; diff --git a/frontend/source/frontend/CMakeLists.txt b/frontend/source/frontend/CMakeLists.txt index 2e1da979..02c61618 100644 --- a/frontend/source/frontend/CMakeLists.txt +++ b/frontend/source/frontend/CMakeLists.txt @@ -162,6 +162,7 @@ target_sources( file_explorer/side_model.cpp file_explorer/remote_side_model.cpp file_explorer/local_side_model.cpp + file_explorer/open_local_file.cpp file_explorer/file_explorer_texts.cpp file_explorer/place_names.cpp "${CMAKE_BINARY_DIR}/include/build_environment.hpp" diff --git a/frontend/source/frontend/dialog/file_property_dialog.cpp b/frontend/source/frontend/dialog/file_property_dialog.cpp index 934df463..f2471927 100644 --- a/frontend/source/frontend/dialog/file_property_dialog.cpp +++ b/frontend/source/frontend/dialog/file_property_dialog.cpp @@ -50,17 +50,22 @@ struct FilePropertyDialog::Implementation { std::string id; Nui::Observed entry; + Nui::Observed path; Nui::Observed targetEntry; + Nui::Observed targetPath; std::shared_ptr> carouselPage; std::unique_ptr carousel; Snc::Dialog dialog; - Nui::ElementRenderer renderEntrySection(Nui::Observed& obs); + Nui::ElementRenderer + renderEntrySection(Nui::Observed& obs, Nui::Observed& pathObs); Implementation(std::string ident) : id{std::move(ident)} , entry{} + , path{} , targetEntry{} + , targetPath{} , carouselPage{std::make_shared>(0)} , carousel{[this]() { @@ -69,8 +74,8 @@ struct FilePropertyDialog::Implementation [this](int page) -> Nui::ElementRenderer { if (page == 0) - return renderEntrySection(entry); - return renderEntrySection(targetEntry); + return renderEntrySection(entry, path); + return renderEntrySection(targetEntry, targetPath); }, 1 ); @@ -93,7 +98,9 @@ struct FilePropertyDialog::Implementation }; Nui::ElementRenderer FilePropertyDialog::Implementation::renderEntrySection( - Nui::Observed& obs) + Nui::Observed& obs, + Nui::Observed& pathObs +) { using namespace Nui::Elements; using namespace Nui::Attributes; @@ -107,8 +114,8 @@ Nui::ElementRenderer FilePropertyDialog::Implementation::renderEntrySection( div{}( span{}(language->getObserved("filePropertyDialog", "path")), Snc::textInput(Snc::TextInputOptions{ - .value = observe(obs).generate([](SharedData::DirectoryEntry const& entry) { - return entry.path.generic_string(); + .value = observe(pathObs).generate([](std::string const& path) { + return path; }), .attributes = {readOnly = true}, }) @@ -206,9 +213,11 @@ FilePropertyDialog::FilePropertyDialog(std::string id) : impl_{std::make_unique(std::move(id))} {} -void FilePropertyDialog::open(SharedData::DirectoryEntry const& entry) +void FilePropertyDialog::open(SharedData::DirectoryEntry const& entry, std::string path, std::string targetPath) { impl_->entry = entry; + impl_->path = std::move(path); + impl_->targetPath = std::move(targetPath); *impl_->carouselPage = 0; if (entry.resolvedTarget) { diff --git a/frontend/source/frontend/file_explorer/local_side_model.cpp b/frontend/source/frontend/file_explorer/local_side_model.cpp index 9d73b32d..f8189763 100644 --- a/frontend/source/frontend/file_explorer/local_side_model.cpp +++ b/frontend/source/frontend/file_explorer/local_side_model.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -50,11 +52,10 @@ LocalSideModel::LocalSideModel( Nui::val args = Nui::val::object(); args.set("path", dirPath.generic_string()); - args.set("fileNameOnly", false); Nui::RpcClient::callWithBackChannel( "RpcFilesystem::listFiles", - [onResultsAvailable = std::move(onResultsAvailable)](Nui::val val) + [dirPath, onResultsAvailable = std::move(onResultsAvailable)](Nui::val val) { if (!val.hasOwnProperty("success")) { @@ -81,7 +82,7 @@ LocalSideModel::LocalSideModel( if (static_cast(file["type"].as()) == std::filesystem::file_type::directory) { - listing.emplace_back(file["path"].as()); + listing.emplace_back(dirPath / file["name"].as()); } } @@ -347,13 +348,9 @@ void LocalSideModel::onActivateItem(NuiFileExplorer::Item const& item) return; } - if (item.path == ".") - return; - if (item.path == "..") - return navigateTo(currentPath_.value().parent_path()); - - Log::debug("Local onActivateItem: {}.", item.path.generic_string()); - navigateTo(currentPath_.value() / item.path); + // The item's own path, the side may already be loading another directory. + Log::debug("Local onActivateItem: {}.", item.displayName()); + navigateTo(item.fullPath.empty() ? currentPath_.value() / item.name : item.fullPath); } void LocalSideModel::onNewItem(NuiFileExplorer::Item::Type type) { @@ -374,17 +371,27 @@ void LocalSideModel::onNewItem(NuiFileExplorer::Item::Type type) if (!name) return; - if (name->find('/') != std::string::npos) - { - Log::error("Invalid item name (cannot contain slashes): {}", *name); - return; - } - Log::info("Creating new item: {}", *name); + if (!isValidItemName(*name)) + return showInvalidItemName(*name); + auto path = currentLocalPath().and_then( + [&name](Paths::LocalPath const& directory) + { + return Paths::LocalPathSegment::make(*name).transform( + [&directory](Paths::LocalPathSegment const& segment) + { + return directory / segment; + } + ); + } + ); + if (!path) + return showPathRefused(language->get("localSideModel", "createItemFailedTitle"), path.error()); + Log::info("Creating new item: {}", path->toUtf8()); if (type == NuiFileExplorer::Item::Type::Directory) { Nui::val args = Nui::val::object(); - args.set("path", (currentPath_.value() / *name).generic_string()); + args.set("path", path->toUtf8()); Nui::RpcClient::callWithBackChannel( "RpcFilesystem::createDirectory", @@ -426,7 +433,7 @@ void LocalSideModel::onNewItem(NuiFileExplorer::Item::Type type) else if (type == NuiFileExplorer::Item::Type::Regular) { Nui::val args = Nui::val::object(); - args.set("path", (currentPath_.value() / *name).generic_string()); + args.set("filePath", path->toUtf8()); Nui::RpcClient::callWithBackChannel( "RpcFilesystem::createFile", @@ -483,17 +490,23 @@ void LocalSideModel::onDelete(std::vector const& items) return; std::vector listItems; + bool anyDirectory = false; for (const auto& item : items) { - listItems.push_back({item.path.generic_string(), ""}); + listItems.push_back({item.name, item.fullPath.generic_string()}); + anyDirectory = anyDirectory || item.type == NuiFileExplorer::Item::Type::Directory; } + // Local directories go with their content in one step, unlike remote ones there is no second question. + auto text = language->get("localSideModel", "deleteSelectedConfirm"); + if (anyDirectory) + text += "\n" + language->get("localSideModel", "deleteDirectoriesWithContent"); confirmDialog_->open( - {.styleVariant = ScriptNuiComponents::StyleVariant::Primary, - .headerText = language->get("remoteSideModel", "deleteItemsQuestion"), - .text = language->get("localSideModel", "deleteSelectedConfirm"), + {.styleVariant = ScriptNuiComponents::StyleVariant::Warning, + .headerText = language->get("localSideModel", "deleteLocalItemsQuestion"), + .text = std::move(text), .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, - .focusButton = ConfirmDialog::Button::Yes, + .focusButton = ConfirmDialog::Button::No, .listItems = listItems, .onClose = [this, items](std::optional button) { @@ -509,7 +522,10 @@ void LocalSideModel::onDelete(std::vector const& items) for (const auto& item : items) { - args["paths"].call("push", (*currentPath_ / item.path).generic_string()); + auto path = localPathOf(item); + if (!path) + return showPathRefused(language->get("localSideModel", "deleteFilesFailedTitle"), path.error()); + args["paths"].call("push", path->toUtf8()); } Nui::RpcClient::callWithBackChannel( @@ -539,6 +555,8 @@ void LocalSideModel::onDelete(std::vector const& items) .text = error, .buttons = ConfirmDialog::Button::Ok, }); + // The other items may be gone, the listing has to show it. + navigateTo(currentPath_.value()); return; } @@ -575,25 +593,58 @@ void LocalSideModel::onTransfer( Log::info("Upload items requested: {}", items.size()); - auto destinationDir = remoteModel_->currentPath().value(); - if (subDir) - destinationDir /= *subDir; - const auto sourceDir = currentPath_.value(); + const auto refuse = [this](Paths::PathError const& error) + { + showPathRefused(language->get("localSideModel", "transferFailedTitle"), error); + }; + const auto destinationDirectory = remoteModel_->currentRemotePath().and_then( + [&subDir](Paths::RemotePath const& directory) -> std::expected + { + if (!subDir) + return directory; + return Paths::PathSegment::make(*subDir).transform( + [&directory](Paths::PathSegment const& segment) + { + return directory / segment; + } + ); + } + ); + if (!destinationDirectory) + return refuse(destinationDirectory.error()); + + std::vector candidates; + candidates.reserve(items.size()); + for (auto const& item : items) + { + auto source = localPathOf(item); + if (!source) + return refuse(source.error()); + const auto name = source->filename(); + if (!name) + return refuse(Paths::PathError{.reason = Paths::PathRejection::Empty, .input = source->toUtf8()}); + candidates.push_back( + SharedData::BulkUploadEntry{ + .source = *source, + .destination = *destinationDirectory / *name, + .sizeBytes = item.size, + .isDirectory = item.isDirectory(), + } + ); + } - const auto itemsSize = items.size(); + const auto itemsSize = candidates.size(); std::string confirmText = fmt::format( fmt::runtime( language->get("localSideModel", itemsSize > 1 ? "uploadMultipleConfirmText" : "uploadSingleConfirmText") ), - itemsSize > 1 ? std::to_string(itemsSize) : items.front().path.filename().generic_string(), - destinationDir.generic_string() + itemsSize > 1 ? std::to_string(itemsSize) : std::string{candidates.front().destination.filename()->view()}, + destinationDirectory->value() ); std::vector listItems; - for (const auto& item : items) - { - listItems.push_back({item.path.generic_string(), ""}); - } + for (const auto& candidate : candidates) + listItems.push_back({candidate.source.toUtf8(), ""}); confirmDialog_->open( {.styleVariant = ScriptNuiComponents::StyleVariant::Primary, @@ -602,7 +653,7 @@ void LocalSideModel::onTransfer( .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, .focusButton = ConfirmDialog::Button::Yes, .listItems = listItems, - .onClose = [this, items, destinationDir, sourceDir](std::optional button) + .onClose = [this, candidates = std::move(candidates)](std::optional button) mutable { if (!button || *button != ConfirmDialog::Button::Yes) { @@ -610,58 +661,27 @@ void LocalSideModel::onTransfer( return; } - // pair - std::vector> uploadItems; - std::transform( - items.begin(), - items.end(), - std::back_inserter(uploadItems), - [destinationDir, sourceDir](auto const& item) - { - auto fullSourcePath = [&]() - { - // if the item has a slash in it, assume its a full path: - if (item.path.has_parent_path()) - return item.path; - return sourceDir / item.path; - }; - auto localItem = item; - auto remoteItem = item; - localItem.path = fullSourcePath(); - localItem.fullPath = localItem.path; - remoteItem.path = destinationDir / item.path.filename(); - remoteItem.fullPath = remoteItem.path; - return std::make_pair(remoteItem, localItem); - } - ); - Log::info("Uploading items"); - // Batched remote-existence probe — one RPC for the whole - // batch instead of N per-file SFTP stats. - std::vector remoteDestPaths; - remoteDestPaths.reserve(uploadItems.size()); - for (auto const& pair : uploadItems) - remoteDestPaths.push_back(pair.first.path); + // Batched remote-existence probe, one RPC for the whole batch instead of N per-file SFTP stats. + std::vector destinations; + destinations.reserve(candidates.size()); + for (auto const& candidate : candidates) + destinations.push_back(candidate.destination); fileEngine_->existsBatchRemote( - remoteDestPaths, + destinations, [this, - uploadItems = std::move(uploadItems)](std::vector exists, std::string const& info) mutable + candidates = std::move(candidates)](std::vector exists, std::string const& info) mutable { - auto existsResults = std::make_shared>(); - if (exists.empty() && !uploadItems.empty()) - { - Log::warn("sftp::existsBatch failed: {}; assuming nothing exists yet", info); - existsResults->assign(uploadItems.size(), false); - } - else - { - *existsResults = std::move(exists); - while (existsResults->size() < uploadItems.size()) - existsResults->push_back(false); - } - uploadItemsConfirmed(std::move(uploadItems), std::move(existsResults)); + // An unknown destination is never taken for absent, the batch overwrites what it accepts. + if (exists.size() != candidates.size()) + return showExistenceCheckFailed( + exists.empty() ? info : fmt::format("{} of {} answers", exists.size(), candidates.size()) + ); + uploadItemsConfirmed( + std::move(candidates), std::make_shared>(std::move(exists)) + ); } ); }} @@ -691,70 +711,92 @@ void LocalSideModel::onDropExternal( onTransfer(items, subDir); } -void LocalSideModel::onRename(NuiFileExplorer::Item const& item) +void LocalSideModel::renameItem(Paths::LocalPath const& from, Paths::LocalPath const& to, bool overwrite) { - Log::info("Rename item requested: {}", item.path.generic_string()); + Nui::val args = Nui::val::object(); + args.set("oldPath", from.toUtf8()); + args.set("newPath", to.toUtf8()); + args.set("overwrite", overwrite); - auto doRename = [item, this](std::string const& newName) - { - Nui::val args = Nui::val::object(); - args.set("oldPath", (*currentPath_ / item.path).generic_string()); - args.set("newPath", (*currentPath_ / newName).generic_string()); + Nui::RpcClient::callWithBackChannel( + "RpcFilesystem::rename", + [this, from, to, overwrite](Nui::val val) + { + if (!val.hasOwnProperty("success")) + { + Log::error("Invalid response from RpcFilesystem::rename"); + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = language->get("localSideModel", "renameFailed"), + .text = language->get("localSideModel", "invalidResponseFromBackend"), + .buttons = ConfirmDialog::Button::Ok, + }); + return; + } - Nui::RpcClient::callWithBackChannel( - "RpcFilesystem::rename", - [this](Nui::val val) + if (!val["success"].as()) { - if (!val.hasOwnProperty("success")) + const auto errorKey = val["error"].as(); + if (errorKey == "backend.rename.targetExists" && !overwrite) { - Log::error("Invalid response from RpcFilesystem::rename"); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = language->get("localSideModel", "renameFailed"), - .text = language->get("localSideModel", "invalidResponseFromBackend"), - .buttons = ConfirmDialog::Button::Ok, - }); + confirmReplaceOnRename( + std::string{to.filename()->view()}, + [this, from, to]() + { + renameItem(from, to, true); + } + ); return; } + const auto error = language->translate(errorKey); + Log::error("Failed to rename item: {}", error); + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = language->get("localSideModel", "renameFailed"), + .text = error, + .buttons = ConfirmDialog::Button::Ok, + }); + return; + } - const auto success = val["success"].as(); - if (!success) - { - const auto error = language->translate(val["error"].as()); - Log::error("Failed to rename item: {}", error); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = language->get("localSideModel", "renameFailed"), - .text = error, - .buttons = ConfirmDialog::Button::Ok, - }); - return; - } + Log::info("Item renamed successfully"); + onRefresh(); + }, + args + ); +} - Log::info("Item renamed successfully"); - // Refresh list: - onRefresh(); - }, - args - ); - }; +void LocalSideModel::onRename(NuiFileExplorer::Item const& item) +{ + Log::info("Rename item requested: {}", item.name); + + // The item's own path: the explorer may already show another directory. + auto from = localPathOf(item); + if (!from) + return showPathRefused(language->get("localSideModel", "renameFailed"), from.error()); inputDialog_->open({ .whatFor = language->get("remoteSideModel", "rename"), - .prompt = fmt::format(fmt::runtime(language->get("remoteSideModel", "renamePrompt")), item.path.filename().string()), + .prompt = fmt::format(fmt::runtime(language->get("remoteSideModel", "renamePrompt")), item.name), .headerText = - fmt::format(fmt::runtime(language->get("remoteSideModel", "renameWithItem")), item.path.filename().string()), - .initialValue = item.path.filename().string(), + fmt::format(fmt::runtime(language->get("remoteSideModel", "renameWithItem")), item.name), + .initialValue = item.name, .isPassword = false, - .onConfirm = [doRename, item](std::optional const& name) + .onConfirm = [this, from = *std::move(from), item](std::optional const& name) { if (!name || name->empty()) return; - if (*name == item.path.filename().string()) + if (*name == item.name) return; + if (!isValidItemName(*name)) + return showInvalidItemName(*name); + const auto segment = Paths::LocalPathSegment::make(*name); + const auto directory = from.parent(); + if (!segment || !directory) + return showInvalidItemName(*name); Log::info("Renaming item to: {}", *name); - doRename(*name); + renameItem(from, *directory / *segment, false); }, }); } @@ -762,17 +804,24 @@ void LocalSideModel::onProperties(NuiFileExplorer::Item const& item) { // listFiles only carries the lean field set; fetch the full entry (atime, birthtime, acl, ...) // on demand so the dialog can display everything without inflating every directory listing. + auto fallback = static_cast(item); + auto path = localPathOf(item); + if (!path) + { + Log::warn("Properties of '{}' use the listed entry: {}", item.name, path.error().toString()); + filePropertyDialog_->open(fallback, item.displayName()); + return; + } Nui::val args = Nui::val::object(); - args.set("path", item.path.generic_string()); + args.set("path", path->toUtf8()); - auto fallback = static_cast(item); Nui::RpcClient::callWithBackChannel( "RpcFilesystem::properties", - [this, fallback = std::move(fallback)](Nui::val val) mutable + [this, fallback = std::move(fallback), displayPath = path->toUtf8()](Nui::val val) mutable { - const auto useFallback = [this, &fallback]() + const auto useFallback = [this, &fallback, &displayPath]() { - filePropertyDialog_->open(fallback); + filePropertyDialog_->open(fallback, displayPath); }; if (!val.hasOwnProperty("success") || !val["success"].as()) @@ -818,7 +867,7 @@ void LocalSideModel::onProperties(NuiFileExplorer::Item const& item) }; SharedData::DirectoryEntry out{ - .path = source["path"].template as(), + .name = source["name"].template as(), .type = SharedData::fileTypeFromStdFilesystemType( static_cast(source["type"].template as()) ), @@ -840,7 +889,7 @@ void LocalSideModel::onProperties(NuiFileExplorer::Item const& item) { const auto link = source["linkTarget"]; if (!link.isNull() && !link.isUndefined()) - out.linkTarget = std::filesystem::path{link.template as()}; + out.linkTarget = link.template as(); } return out; }; @@ -854,7 +903,10 @@ void LocalSideModel::onProperties(NuiFileExplorer::Item const& item) full.resolvedTarget = std::make_shared(extractFull(target)); } - filePropertyDialog_->open(full); + std::string resolvedPath{}; + if (val.hasOwnProperty("resolvedPath") && val["resolvedPath"].isString()) + resolvedPath = val["resolvedPath"].as(); + filePropertyDialog_->open(full, displayPath, std::move(resolvedPath)); }, args ); @@ -871,23 +923,17 @@ void LocalSideModel::onError(NuiFileExplorer::Error const& fileExplorerError) }); } -void LocalSideModel::onDirectoryListing( - std::optional> directoryEntries, - std::string const& reason -) +bool LocalSideModel::hasParent(std::filesystem::path const& directory) const { - if (currentPath_.value().has_parent_path()) - { - directoryEntries->insert( - directoryEntries->begin(), - SharedData::DirectoryEntry{ - .path = "..", - .type = SharedData::FileType::Directory, - .size = 0, + // A drive root like "C:/" has no parent either. + return Paths::LocalPath::fromUtf8(directory.generic_string()) + .transform( + [](Paths::LocalPath const& path) + { + return !path.isRoot(); } - ); - } - SideModel::onDirectoryListing(directoryEntries, reason); + ) + .value_or(false); } void LocalSideModel::navigateTo(std::filesystem::path const& path) @@ -899,13 +945,17 @@ void LocalSideModel::navigateTo(std::filesystem::path const& path) Nui::val args = Nui::val::object(); args.set("path", currentPath_.value().generic_string()); - args.set("fileNameOnly", true); Nui::RpcClient::callWithBackChannel( "RpcFilesystem::listFiles", - [this](Nui::val val) + [this, directory = currentPath_.value()](Nui::val val) { Log::debug("Received directory listing response"); + if (directory != currentPath_.value()) + { + Log::debug("Dropping the listing of '{}', the side moved on.", directory.generic_string()); + return; + } auto revertNavigation = [this]() { if (currentPath_.value() != preNavigatePath_) @@ -979,7 +1029,7 @@ void LocalSideModel::navigateTo(std::filesystem::path const& path) }; SharedData::DirectoryEntry entry{ - .path = source["path"].template as(), + .name = source["name"].template as(), .type = SharedData::fileTypeFromStdFilesystemType( static_cast(source["type"].template as()) ), @@ -1000,7 +1050,7 @@ void LocalSideModel::navigateTo(std::filesystem::path const& path) { const auto link = source["linkTarget"]; if (!link.isNull() && !link.isUndefined()) - entry.linkTarget = std::filesystem::path{link.template as()}; + entry.linkTarget = link.template as(); } return entry; }; @@ -1024,19 +1074,32 @@ void LocalSideModel::navigateTo(std::filesystem::path const& path) directoryEntries.push_back(std::move(entry)); } - onDirectoryListing(directoryEntries); + onDirectoryListing(directory, std::move(directoryEntries)); }, args ); } +namespace +{ + /** + * @brief @p entry with the user's consent to replace its existing destination. + */ + template + EntryT approvedForOverwrite(EntryT entry) + { + entry.mayOverwrite = true; + return entry; + } +} + void LocalSideModel::uploadItemsConfirmed( - std::vector> uploadItems, + std::vector candidates, std::shared_ptr> existsResults, std::size_t index, bool overwriteNever, bool overwriteAlways, - std::shared_ptr> accepted + std::shared_ptr> accepted ) { // Mirrors RemoteSideModel::downloadItemsConfirmed — see that for the @@ -1044,28 +1107,10 @@ void LocalSideModel::uploadItemsConfirmed( // keep WASM stack depth bounded by conflicts not by file count). if (!accepted) - accepted = std::make_shared>(); - - auto pushEntry = [](std::vector& bucket, - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem) - { - bucket.push_back( - SharedData::BulkAddEntry{ - // For uploads, src is local and dst is remote — opposite of - // download (the bulk RPC is symmetric on field naming). - .src = !localItem.fullPath.empty() ? localItem.fullPath : localItem.path, - .dst = !remoteItem.fullPath.empty() ? remoteItem.fullPath : remoteItem.path, - .sizeBytes = localItem.size, - .isDirectory = localItem.isDirectory(), - } - ); - }; + accepted = std::make_shared>(); - // Every entry in `accepted` is either a non-existing destination or an - // item the user explicitly approved for overwrite (Yes / All); the "No" - // and "None" branches skip the push. So allowOverwrite=true at flush - // time is semantically correct regardless of the overwriteAlways flag. + // Only entries the user approved (Yes / All) may replace their destination. The others did not exist when + // probed, one that appears before the transfer runs fails instead of being replaced. auto flushAccepted = [this, &accepted]() { if (accepted->empty()) @@ -1076,22 +1121,9 @@ void LocalSideModel::uploadItemsConfirmed( // the bulk path because they need the Scan+Bulk pair. if (accepted->size() == 1 && !accepted->front().isDirectory) { - auto const& entry = accepted->front(); - // For upload, BulkAddEntry.src is the LOCAL path (source) and - // .dst is the REMOTE path (destination) — see the pushEntry - // lambda above. enqueueSingleUpload's parameter order is - // (remoteItem, localItem) so build accordingly. - SharedData::DirectoryEntry localEntry{}; - localEntry.path = entry.src; - localEntry.size = entry.sizeBytes; - localEntry.type = SharedData::FileType::Regular; - SharedData::DirectoryEntry remoteEntry{}; - remoteEntry.path = entry.dst; - remoteEntry.type = SharedData::FileType::Regular; enqueueSingleUpload( - NuiFileExplorer::Item{remoteEntry}, - NuiFileExplorer::Item{localEntry}, - /*allowOverwrite=*/true, + accepted->front(), + /*allowOverwrite=*/accepted->front().mayOverwrite, /*insertRefresh=*/true ); accepted->clear(); @@ -1099,7 +1131,7 @@ void LocalSideModel::uploadItemsConfirmed( } operationQueue_->enqueueBulkUpload( std::move(*accepted), - /*allowOverwrite*/ true, + /*allowOverwrite*/ false, /*insertRefresh*/ true, SharedData::OperationMode::Queued, /*onEachComplete*/ {}, @@ -1119,7 +1151,7 @@ void LocalSideModel::uploadItemsConfirmed( ); }; - if (index == uploadItems.size()) + if (index == candidates.size()) { flushAccepted(); return; @@ -1127,20 +1159,21 @@ void LocalSideModel::uploadItemsConfirmed( if (overwriteAlways) { - for (; index < uploadItems.size(); ++index) - pushEntry(*accepted, uploadItems[index].first, uploadItems[index].second); + for (; index < candidates.size(); ++index) + accepted->push_back(approvedForOverwrite(candidates[index])); flushAccepted(); return; } - while (index < uploadItems.size()) + while (index < candidates.size()) { - auto const& item = uploadItems[index]; - const bool exists = (existsResults && index < existsResults->size()) ? (*existsResults)[index] : false; + auto const& candidate = candidates[index]; + // Without an answer the destination counts as existing, so the user is asked. + const bool exists = !existsResults || index >= existsResults->size() || (*existsResults)[index]; if (!exists) { - pushEntry(*accepted, item.first, item.second); + accepted->push_back(candidate); ++index; continue; } @@ -1152,14 +1185,14 @@ void LocalSideModel::uploadItemsConfirmed( break; } - if (index >= uploadItems.size()) + if (index >= candidates.size()) { flushAccepted(); return; } - auto const& item = uploadItems[index]; - Log::info("Uploading '{}' to '{}'.", item.first.path.generic_string(), item.second.path.generic_string()); + auto const& candidate = candidates[index]; + Log::info("Uploading '{}' to '{}'.", candidate.source.toUtf8(), candidate.destination.value()); confirmDialog_->open( {.styleVariant = ScriptNuiComponents::StyleVariant::Primary, @@ -1168,21 +1201,20 @@ void LocalSideModel::uploadItemsConfirmed( .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No | ConfirmDialog::Button::All | ConfirmDialog::Button::None | ConfirmDialog::Button::Cancel, .focusButton = ConfirmDialog::Button::No, - .listItems = {{.text = item.second.path.generic_string(), .description = language->get("fileExplorer", "fileAlreadyExists")}}, + .listItems = {{.text = candidate.destination.value(), .description = language->get("fileExplorer", "fileAlreadyExists")}}, .onClose = [this, - uploadItems = std::move(uploadItems), + candidates = std::move(candidates), existsResults, index, overwriteNever, overwriteAlways, - accepted, - pushEntry](std::optional button) mutable + accepted](std::optional button) mutable { if (button && *button == ConfirmDialog::Button::Yes) { - pushEntry(*accepted, uploadItems[index].first, uploadItems[index].second); + accepted->push_back(approvedForOverwrite(candidates[index])); uploadItemsConfirmed( - std::move(uploadItems), + std::move(candidates), std::move(existsResults), index + 1, overwriteNever, @@ -1192,9 +1224,9 @@ void LocalSideModel::uploadItemsConfirmed( } else if (button && *button == ConfirmDialog::Button::No) { - Log::info("Skipping upload of existing file: {}", uploadItems[index].second.path.generic_string()); + Log::info("Skipping upload of existing file: {}", candidates[index].destination.value()); uploadItemsConfirmed( - std::move(uploadItems), + std::move(candidates), std::move(existsResults), index + 1, overwriteNever, @@ -1205,9 +1237,9 @@ void LocalSideModel::uploadItemsConfirmed( else if (button && *button == ConfirmDialog::Button::All) { Log::info("Overwriting all existing files from now on."); - pushEntry(*accepted, uploadItems[index].first, uploadItems[index].second); + accepted->push_back(approvedForOverwrite(candidates[index])); uploadItemsConfirmed( - std::move(uploadItems), + std::move(candidates), std::move(existsResults), index + 1, overwriteNever, @@ -1219,7 +1251,7 @@ void LocalSideModel::uploadItemsConfirmed( { Log::info("Skipping all existing files from now on."); uploadItemsConfirmed( - std::move(uploadItems), + std::move(candidates), std::move(existsResults), index + 1, /*overwriteNever*/ true, @@ -1229,9 +1261,9 @@ void LocalSideModel::uploadItemsConfirmed( } else { - const auto terminalIndex = uploadItems.size(); + const auto terminalIndex = candidates.size(); uploadItemsConfirmed( - std::move(uploadItems), + std::move(candidates), std::move(existsResults), terminalIndex, overwriteNever, @@ -1244,16 +1276,16 @@ void LocalSideModel::uploadItemsConfirmed( } void LocalSideModel::enqueueSingleUpload( - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem, + SharedData::BulkUploadEntry const& entry, bool allowOverwrite, bool insertRefresh ) { operationQueue_->enqueueUpload( - remoteItem, - localItem, - [this, localItem](std::optional const& opId, std::string const& info) + entry.source, + entry.destination, + entry.sizeBytes, + [this, source = entry.source](std::optional const& opId, std::string const& info) { if (!opId) { @@ -1262,9 +1294,7 @@ void LocalSideModel::enqueueSingleUpload( .styleVariant = ScriptNuiComponents::StyleVariant::Danger, .headerText = "", .text = fmt::format( - fmt::runtime(language->get("localSideModel", "failedToEnqueueUpload")), - localItem.path.generic_string(), - info + fmt::runtime(language->get("localSideModel", "failedToEnqueueUpload")), source.toUtf8(), info ), .buttons = ConfirmDialog::Button::Ok, }); @@ -1313,7 +1343,7 @@ LocalSideModel::contextMenuItems(std::vector const& selec items.push_back(Snc::PopupMenu::separator()); } - if (singleItem && selectedItems.front().isDirectoryLike() && selectedItems.front().path.filename() != "..") + if (singleItem && selectedItems.front().isDirectoryLike() && !selectedItems.front().isParentLink) { const auto& selPath = selectedItems.front().fullPath; const auto& favs = favorites_->value(); @@ -1398,48 +1428,20 @@ LocalSideModel::contextMenuItems(std::vector const& selec void LocalSideModel::onOpen(NuiFileExplorer::Item const& item, bool openWith) { - Log::info("Open item requested: {}. Open with: {}", item.path.generic_string(), openWith); - - Nui::RpcClient::callWithBackChannel( - "RpcFilesystem::open", - [this](Nui::val val) - { - if (!val.hasOwnProperty("success")) - { - Log::error("Invalid response from RpcFilesystem::open"); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = language->get("localSideModel", "failedToOpenFile"), - .text = language->get("localSideModel", "invalidResponseFromBackend"), - .buttons = ConfirmDialog::Button::Ok, - }); - return; - } - - const auto success = val["success"].as(); - if (!success) - { - const auto error = language->translate(val["error"].as()); - Log::error("Failed to open file: {}", error); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = language->get("localSideModel", "failedToOpenFile"), - .text = error, - .buttons = ConfirmDialog::Button::Ok, - }); - return; - } + Log::info("Open item requested: {}. Open with: {}", item.name, openWith); + auto path = localPathOf(item); + if (!path) + return showPathRefused(language->get("localSideModel", "failedToOpenFile"), path.error()); - Log::info("File opened successfully"); - }, - (*currentPath_ / item.path).generic_string(), - openWith - ); + FileExplorer::openLocalFile(*confirmDialog_, *path, openWith); } void LocalSideModel::onOpenInFileManager(NuiFileExplorer::Item const& item) { - const auto absolutePath = (*currentPath_ / item.path).generic_string(); + auto path = localPathOf(item); + if (!path) + return showPathRefused(language->get("localSideModel", "openInFileManagerFailed"), path.error()); + const auto absolutePath = path->toUtf8(); Log::info("Open in file manager requested: {}", absolutePath); Nui::RpcClient::callWithBackChannel( @@ -1476,11 +1478,29 @@ void LocalSideModel::onOpenInFileManager(NuiFileExplorer::Item const& item) ); } -void LocalSideModel::setRemoteModel(SideModel* model) +void LocalSideModel::setRemoteModel(RemoteSideModel* model) { remoteModel_ = model; } +std::expected LocalSideModel::currentLocalPath() const +{ + return Paths::LocalPath::fromUtf8(listedPath_.generic_string()); +} + +std::expected LocalSideModel::localPathOf(NuiFileExplorer::Item const& item) const +{ + if (item.isParentLink) + return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::DotOrDotDot, .input = ".."}); + return Paths::LocalPathSegment::make(item.name) + .and_then( + [&item](Paths::LocalPathSegment const&) + { + return Paths::LocalPath::fromUtf8(item.fullPath.generic_string()); + } + ); +} + bool LocalSideModel::isComplete() const { return SideModel::isComplete(); @@ -1532,16 +1552,18 @@ void LocalSideModel::onTransferAsArchive(std::vector cons // Files go straight in; directories are flattened on the backend via // std::filesystem::recursive_directory_iterator inside the upload op's // prepareInStrand step. - std::vector localPaths; + std::vector localPaths; localPaths.reserve(items.size()); for (auto const& item : items) - localPaths.push_back(currentPath_.value() / item.path); - - if (localPaths.empty()) - return; + { + auto localPath = localPathOf(item); + if (!localPath) + return showPathRefused(language->get("localSideModel", "archiveUploadFailed"), localPath.error()); + localPaths.push_back(*std::move(localPath)); + } const auto defaultStem = - items.size() == 1 ? items.front().path.filename().generic_string() : language->get("fileExplorer", "archiveDefaultName"); + items.size() == 1 ? items.front().name : language->get("fileExplorer", "archiveDefaultName"); archiveTransferDialog_->open({ .headerText = language->get("fileExplorer", "contextMenu", "uploadAsArchive"), @@ -1559,7 +1581,20 @@ void LocalSideModel::onTransferAsArchive(std::vector cons return; const std::string filename = result->fileStem + ".tar" + archiveCodecExtension(result->codec); - const auto remotePath = remoteModel_->currentPath().value() / filename; + auto archivePath = remoteModel_->currentRemotePath().and_then( + [&filename](Paths::RemotePath const& directory) + { + return Paths::PathSegment::make(filename).transform( + [&directory](Paths::PathSegment const& segment) + { + return directory / segment; + } + ); + } + ); + if (!archivePath) + return showPathRefused(language->get("localSideModel", "archiveUploadFailed"), archivePath.error()); + const auto remotePath = *std::move(archivePath); operationQueue_->enqueueArchiveUpload( std::move(paths), @@ -1581,9 +1616,7 @@ void LocalSideModel::onTransferAsArchive(std::vector cons }); return; } - Log::info( - "Archive upload queued as {} (destination {})", opId->value(), remotePath.generic_string() - ); + Log::info("Archive upload queued as {} (destination {})", opId->value(), remotePath.value()); } ); }, diff --git a/frontend/source/frontend/file_explorer/open_local_file.cpp b/frontend/source/frontend/file_explorer/open_local_file.cpp new file mode 100644 index 00000000..a202dc6d --- /dev/null +++ b/frontend/source/frontend/file_explorer/open_local_file.cpp @@ -0,0 +1,67 @@ +#include +#include +#include + +#include +#include + +#include + +#include + +namespace FileExplorer +{ + namespace + { + void requestOpen(ConfirmDialog& dialog, Paths::LocalPath const& path, bool openWith, bool confirmed) + { + Nui::RpcClient::callWithBackChannel( + "RpcFilesystem::open", + [&dialog, path, openWith](Nui::val reply) + { + if (reply.hasOwnProperty("needsConfirmation") && reply["needsConfirmation"].as()) + { + const auto risk = language->translate(reply["risk"].as()); + dialog.open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Warning, + .headerText = language->get("fileExplorer", "openRiskyHeader"), + .text = fmt::format( + fmt::runtime(language->get("fileExplorer", "openRiskyText")), path.toUtf8(), risk + ), + .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, + .focusButton = ConfirmDialog::Button::No, + .onClose = + [&dialog, path, openWith](std::optional button) + { + if (button == ConfirmDialog::Button::Yes) + requestOpen(dialog, path, openWith, /*confirmed=*/true); + }, + }); + return; + } + if (reply.hasOwnProperty("success") && reply["success"].as()) + return; + + const auto error = reply.hasOwnProperty("error") + ? language->translate(reply["error"].as()) + : language->get("localSideModel", "invalidResponseFromBackend"); + Log::error("Failed to open '{}': {}", path.toUtf8(), error); + dialog.open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = language->get("localSideModel", "failedToOpenFile"), + .text = error, + .buttons = ConfirmDialog::Button::Ok, + }); + }, + path.toUtf8(), + openWith, + confirmed + ); + } + } + + void openLocalFile(ConfirmDialog& dialog, Paths::LocalPath const& path, bool openWith) + { + requestOpen(dialog, path, openWith, /*confirmed=*/false); + } +} diff --git a/frontend/source/frontend/file_explorer/remote_side_model.cpp b/frontend/source/frontend/file_explorer/remote_side_model.cpp index f85e839c..91a53bd9 100644 --- a/frontend/source/frontend/file_explorer/remote_side_model.cpp +++ b/frontend/source/frontend/file_explorer/remote_side_model.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -73,7 +75,7 @@ RemoteSideModel::RemoteSideModel( for (auto&& entry : std::move(*entries)) { if (entry.isDirectory()) - listing.emplace_back(std::move(entry).path); + listing.emplace_back(std::move(entry).name); } onResultsAvailable(listing); @@ -149,7 +151,7 @@ RemoteSideModel::contextMenuItems(std::vector const& sele std::vector items; - if (singleItem && selectedItems.front().isDirectoryLike() && selectedItems.front().path.filename() != "..") + if (singleItem && selectedItems.front().isDirectoryLike() && !selectedItems.front().isParentLink) { const auto& selPath = selectedItems.front().fullPath; const auto& favs = favorites_->value(); @@ -265,7 +267,7 @@ RemoteSideModel::contextMenuItems(std::vector const& sele return items; } -void RemoteSideModel::setLocalModel(SideModel* model) +void RemoteSideModel::setLocalModel(LocalSideModel* model) { localModel_ = model; } @@ -275,6 +277,39 @@ void RemoteSideModel::setFileTracking(FileTrackingPanel* fileTracking) fileTracking_ = fileTracking; } +std::expected RemoteSideModel::currentRemotePath() const +{ + return Paths::RemotePath::make(listedPath_.generic_string()); +} + +std::expected RemoteSideModel::remotePathOf(NuiFileExplorer::Item const& item) const +{ + if (item.isParentLink) + return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::DotOrDotDot, .input = ".."}); + return Paths::PathSegment::make(item.name) + .and_then( + [&item](Paths::PathSegment const&) + { + return Paths::RemotePath::make(item.fullPath.generic_string()); + } + ); +} + +std::expected RemoteSideModel::childOfCurrentPath(std::string_view name) const +{ + return currentRemotePath().and_then( + [name](Paths::RemotePath const& directory) + { + return Paths::PathSegment::make(name).transform( + [&directory](Paths::PathSegment const& segment) + { + return directory / segment; + } + ); + } + ); +} + bool RemoteSideModel::isComplete() const { return localModel_ != nullptr && SideModel::isComplete(); @@ -292,13 +327,9 @@ void RemoteSideModel::onActivateItem(NuiFileExplorer::Item const& item) return; } - if (item.path == ".") - return; - if (item.path == "..") - return navigateTo(currentPath_.value().parent_path()); - - Log::debug("Remote onActivateItem: {}.", item.path.generic_string()); - navigateTo(currentPath_.value() / item.path); + // The item's own path, the side may already be loading another directory. + Log::debug("Remote onActivateItem: {}.", item.displayName()); + navigateTo(item.fullPath.empty() ? currentPath_.value() / item.name : item.fullPath); } void RemoteSideModel::onNewItem(NuiFileExplorer::Item::Type type) @@ -318,14 +349,14 @@ void RemoteSideModel::onNewItem(NuiFileExplorer::Item::Type type) if (!name) return; - Log::info("Creating new directory: {}", *name); - if (name->find('/') != std::string::npos) - { - Log::error("Invalid directory name (cannot contain slashes): {}", *name); - return; - } + if (!isValidItemName(*name)) + return showInvalidItemName(*name); + auto path = childOfCurrentPath(*name); + if (!path) + return showPathRefused(language->get("remoteSideModel", "createNewDirectory"), path.error()); + Log::info("Creating new directory: {}", path->value()); fileEngine_->createDirectory( - currentPath_.value() / *name, + *path, [this](bool success, std::string const& info) { if (!success) @@ -358,14 +389,14 @@ void RemoteSideModel::onNewItem(NuiFileExplorer::Item::Type type) if (!name) return; - Log::info("Creating new file: {}", *name); - if (name->find('/') != std::string::npos) - { - Log::error("Invalid file name (cannot contain slashes): {}", *name); - return; - } + if (!isValidItemName(*name)) + return showInvalidItemName(*name); + auto path = childOfCurrentPath(*name); + if (!path) + return showPathRefused(language->get("remoteSideModel", "createNewFile"), path.error()); + Log::info("Creating new file: {}", path->value()); fileEngine_->createFile( - currentPath_.value() / *name, + *path, [this](bool success, std::string const& info) { if (!success) @@ -405,9 +436,9 @@ void RemoteSideModel::onError(NuiFileExplorer::Error const& fileExplorerError) } void RemoteSideModel::askNonEmptyDirectoryDeletions( - std::vector confirmedToDelete, - std::vector nonEmpties, - std::vector filesAndEmptyDirs, + std::vector confirmedToDelete, + std::vector nonEmpties, + std::vector filesAndEmptyDirs, std::size_t currentNonEmpty ) { @@ -422,7 +453,7 @@ void RemoteSideModel::askNonEmptyDirectoryDeletions( confirmDialog_->open( {.styleVariant = ScriptNuiComponents::StyleVariant::Warning, .headerText = language->get("remoteSideModel", "nonEmptyDirectoriesFound"), - .text = fmt::format(fmt::runtime(language->get("remoteSideModel", "nonEmptyDeleteAsk")), dir.string()), + .text = fmt::format(fmt::runtime(language->get("remoteSideModel", "nonEmptyDeleteAsk")), dir.value()), .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No | ConfirmDialog::Button::None | ConfirmDialog::Button::All | ConfirmDialog::Button::Cancel, .focusButton = ConfirmDialog::Button::No, @@ -479,38 +510,20 @@ void RemoteSideModel::askNonEmptyDirectoryDeletions( } void RemoteSideModel::enqueueDeletes( - std::vector nonEmpties, - std::vector filesAndEmptyDirs + std::vector nonEmpties, + std::vector filesAndEmptyDirs ) { // Combine into a single bulk-delete request. File / empty-dir entries // become one bulk-delete card; non-empty directories become individual // Scan+Delete pairs (their isDirectory=true triggers the recursive flow // backend-side). One RPC for the whole user action. - std::vector entries; + std::vector entries; entries.reserve(filesAndEmptyDirs.size() + nonEmpties.size()); - for (auto const& path : filesAndEmptyDirs) - { - entries.push_back( - SharedData::BulkAddEntry{ - .src = path, - .dst = {}, - .sizeBytes = 0, - .isDirectory = false, - } - ); - } - for (auto const& path : nonEmpties) - { - entries.push_back( - SharedData::BulkAddEntry{ - .src = path, - .dst = {}, - .sizeBytes = 0, - .isDirectory = true, - } - ); - } + for (auto& path : filesAndEmptyDirs) + entries.push_back({.target = std::move(path), .isDirectory = false}); + for (auto& path : nonEmpties) + entries.push_back({.target = std::move(path), .isDirectory = true}); if (entries.empty()) return; @@ -558,21 +571,21 @@ void RemoteSideModel::onDelete(std::vector const& items) confirmText = fmt::format(fmt::runtime(language->get("remoteSideModel", "deleteMultipleConfirm")), itemSize); else confirmText = fmt::format( - fmt::runtime(language->get("remoteSideModel", "deleteSingleConfirm")), items.front().path.generic_string() + fmt::runtime(language->get("remoteSideModel", "deleteSingleConfirm")), items.front().name ); std::vector listItems; for (const auto& item : items) { - listItems.push_back({item.path.generic_string(), ""}); + listItems.push_back({item.name, item.fullPath.generic_string()}); } confirmDialog_->open( - {.styleVariant = ScriptNuiComponents::StyleVariant::Primary, - .headerText = language->get("remoteSideModel", "deleteItemsQuestion"), + {.styleVariant = ScriptNuiComponents::StyleVariant::Warning, + .headerText = language->get("remoteSideModel", "deleteRemoteItemsQuestion"), .text = confirmText, .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, - .focusButton = ConfirmDialog::Button::Yes, + .focusButton = ConfirmDialog::Button::No, .listItems = listItems, .onClose = [this, items](std::optional button) { @@ -582,21 +595,36 @@ void RemoteSideModel::onDelete(std::vector const& items) return; } - std::vector directories; - std::vector files; - for (auto item : items) + std::vector directories; + std::vector files; + for (auto const& item : items) { - item.path = currentPath_.value() / item.path; + // A name like "x/.." or ".." must not turn into another delete target. + auto path = remotePathOf(item); + if (!path) + { + Log::error("Refusing to delete '{}': not an item of the current directory.", item.name); + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = language->get("remoteSideModel", "deleteFailed"), + .text = fmt::format( + fmt::runtime(language->get("remoteSideModel", "failedToDeleteItems")), + item.name + ), + .buttons = ConfirmDialog::Button::Ok, + }); + return; + } if (item.type == NuiFileExplorer::Item::Type::Directory) - directories.push_back(item); + directories.push_back(*std::move(path)); else - files.push_back(item); + files.push_back(*std::move(path)); } Log::info("Deleting items"); fileEngine_->remove( - files, - directories, + std::move(files), + std::move(directories), [this](bool success, std::string const& info) { if (!success) @@ -615,10 +643,7 @@ void RemoteSideModel::onDelete(std::vector const& items) // Refresh list from server: onRefresh(); }, - [this]( - std::vector filesAndEmptyDirs, - std::vector nonEmpties - ) + [this](std::vector filesAndEmptyDirs, std::vector nonEmpties) { askNonEmptyDirectoryDeletions({}, std::move(nonEmpties), std::move(filesAndEmptyDirs), 0); } @@ -628,15 +653,15 @@ void RemoteSideModel::onDelete(std::vector const& items) } void RemoteSideModel::enqueueSingleDownload( - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem, + SharedData::BulkDownloadEntry const& entry, bool allowOverwrite, bool insertRefresh ) { operationQueue_->enqueueDownload( - remoteItem, - localItem, + entry.source, + entry.destination, + entry.sizeBytes, [this](std::optional const& opId, std::string const& info) { if (!opId) @@ -659,13 +684,26 @@ void RemoteSideModel::enqueueSingleDownload( ); } +namespace +{ + /** + * @brief @p entry with the user's consent to replace its existing destination. + */ + template + EntryT approvedForOverwrite(EntryT entry) + { + entry.mayOverwrite = true; + return entry; + } +} + void RemoteSideModel::downloadItemsConfirmed( - std::vector> downloadItems, + std::vector candidates, std::shared_ptr> existsResults, std::size_t index, bool overwriteNever, bool overwriteAlways, - std::shared_ptr> accepted + std::shared_ptr> accepted ) { // Lazily allocate the accumulator on the first (top-level) call. All @@ -673,28 +711,10 @@ void RemoteSideModel::downloadItemsConfirmed( // confirmations land in one batch that's flushed as a single bulk RPC at // the terminal recursion. if (!accepted) - accepted = std::make_shared>(); + accepted = std::make_shared>(); - auto pushEntry = [](std::vector& bucket, - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem) - { - bucket.push_back( - SharedData::BulkAddEntry{ - .src = !remoteItem.fullPath.empty() ? remoteItem.fullPath : remoteItem.path, - .dst = !localItem.fullPath.empty() ? localItem.fullPath : localItem.path, - .sizeBytes = remoteItem.size, - .isDirectory = remoteItem.isDirectory(), - .mtime = remoteItem.mtime, - .mtimeNsec = remoteItem.mtimeNsec, - } - ); - }; - - // Every entry in `accepted` is either a non-existing destination or an - // item the user explicitly approved for overwrite (Yes / All); the "No" - // and "None" branches skip the push. So allowOverwrite=true at flush - // time is semantically correct regardless of the overwriteAlways flag. + // Only entries the user approved (Yes / All) may replace their destination. The others did not exist when + // probed, one that appears before the transfer runs fails instead of being replaced. auto flushAccepted = [this, &accepted]() { if (accepted->empty()) @@ -705,20 +725,9 @@ void RemoteSideModel::downloadItemsConfirmed( // the bulk path because they need the Scan+Bulk pair. if (accepted->size() == 1 && !accepted->front().isDirectory) { - auto const& entry = accepted->front(); - SharedData::DirectoryEntry remoteEntry{}; - remoteEntry.path = entry.src; - remoteEntry.size = entry.sizeBytes; - remoteEntry.type = SharedData::FileType::Regular; - remoteEntry.mtime = entry.mtime; - remoteEntry.mtimeNsec = entry.mtimeNsec; - SharedData::DirectoryEntry localEntry{}; - localEntry.path = entry.dst; - localEntry.type = SharedData::FileType::Regular; enqueueSingleDownload( - NuiFileExplorer::Item{remoteEntry}, - NuiFileExplorer::Item{localEntry}, - /*allowOverwrite=*/true, + accepted->front(), + /*allowOverwrite=*/accepted->front().mayOverwrite, /*insertRefresh=*/true ); accepted->clear(); @@ -730,7 +739,7 @@ void RemoteSideModel::downloadItemsConfirmed( // RPCs with one. operationQueue_->enqueueBulkDownload( std::move(*accepted), - /*allowOverwrite*/ true, + /*allowOverwrite*/ false, /*insertRefresh*/ true, SharedData::OperationMode::Queued, /*onEachComplete*/ {}, @@ -752,7 +761,7 @@ void RemoteSideModel::downloadItemsConfirmed( ); }; - if (index == downloadItems.size()) + if (index == candidates.size()) { flushAccepted(); return; @@ -760,8 +769,8 @@ void RemoteSideModel::downloadItemsConfirmed( if (overwriteAlways) { - for (; index < downloadItems.size(); ++index) - pushEntry(*accepted, downloadItems[index].first, downloadItems[index].second); + for (; index < candidates.size(); ++index) + accepted->push_back(approvedForOverwrite(candidates[index])); flushAccepted(); return; } @@ -769,19 +778,18 @@ void RemoteSideModel::downloadItemsConfirmed( // Iterate synchronously until we hit either end-of-list or a conflict // that requires an async confirmation dialog. Recursing per-item would // blow WASM's ~64KB stack at ~175 deep for a 1000-file batch, since each - // frame holds a moved copy of downloadItems. - while (index < downloadItems.size()) + // frame holds a moved copy of the candidates. + while (index < candidates.size()) { - auto const& item = downloadItems[index]; + auto const& candidate = candidates[index]; // existsResults was filled by a single batched RpcFilesystem::existsBatch // call before the loop started, so this adds zero RPC round-trips. - // Defensive: if results are missing or shorter than expected, treat - // the file as not-existing. - const bool exists = (existsResults && index < existsResults->size()) ? (*existsResults)[index] : false; + // Without an answer the destination counts as existing, so the user is asked. + const bool exists = !existsResults || index >= existsResults->size() || (*existsResults)[index]; if (!exists) { - pushEntry(*accepted, item.first, item.second); + accepted->push_back(candidate); ++index; continue; } @@ -799,18 +807,18 @@ void RemoteSideModel::downloadItemsConfirmed( break; } - if (index >= downloadItems.size()) + if (index >= candidates.size()) { // All items processed without needing further prompts — flush. flushAccepted(); return; } - auto const& item = downloadItems[index]; - Log::info("Downloading '{}' to '{}'.", item.first.path.generic_string(), item.second.path.generic_string()); + auto const& candidate = candidates[index]; + Log::info("Downloading '{}' to '{}'.", candidate.source.value(), candidate.destination.toUtf8()); { - Log::info("File already exists: {}", item.second.path.generic_string()); + Log::info("File already exists: {}", candidate.destination.toUtf8()); confirmDialog_->open( {.styleVariant = ScriptNuiComponents::StyleVariant::Primary, .headerText = language->get("remoteSideModel", "fileAlreadyExistsOverwrite"), @@ -818,21 +826,20 @@ void RemoteSideModel::downloadItemsConfirmed( .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No | ConfirmDialog::Button::All | ConfirmDialog::Button::None | ConfirmDialog::Button::Cancel, .focusButton = ConfirmDialog::Button::No, - .listItems = {{.text = item.second.path.generic_string(), .description = language->get("fileExplorer", "fileAlreadyExists")}}, + .listItems = {{.text = candidate.destination.toUtf8(), .description = language->get("fileExplorer", "fileAlreadyExists")}}, .onClose = [this, - downloadItems = std::move(downloadItems), + candidates = std::move(candidates), existsResults, index, overwriteNever, overwriteAlways, - accepted, - pushEntry](std::optional button) mutable + accepted](std::optional button) mutable { if (button && button == ConfirmDialog::Button::Yes) { - pushEntry(*accepted, downloadItems[index].first, downloadItems[index].second); + accepted->push_back(approvedForOverwrite(candidates[index])); downloadItemsConfirmed( - std::move(downloadItems), + std::move(candidates), std::move(existsResults), index + 1, overwriteNever, @@ -843,10 +850,10 @@ void RemoteSideModel::downloadItemsConfirmed( else if (button && button == ConfirmDialog::Button::No) { Log::info( - "Skipping download of existing file: {}", downloadItems[index].second.path.generic_string() + "Skipping download of existing file: {}", candidates[index].destination.toUtf8() ); downloadItemsConfirmed( - std::move(downloadItems), + std::move(candidates), std::move(existsResults), index + 1, overwriteNever, @@ -857,9 +864,9 @@ void RemoteSideModel::downloadItemsConfirmed( else if (button && button == ConfirmDialog::Button::All) { Log::info("Overwriting all existing files from now on."); - pushEntry(*accepted, downloadItems[index].first, downloadItems[index].second); + accepted->push_back(approvedForOverwrite(candidates[index])); downloadItemsConfirmed( - std::move(downloadItems), + std::move(candidates), std::move(existsResults), index + 1, overwriteNever, @@ -871,7 +878,7 @@ void RemoteSideModel::downloadItemsConfirmed( { Log::info("Skipping all existing files from now on."); downloadItemsConfirmed( - std::move(downloadItems), + std::move(candidates), std::move(existsResults), index + 1, /*overwriteNever*/ true, @@ -883,9 +890,9 @@ void RemoteSideModel::downloadItemsConfirmed( { // Cancel — flush whatever the user has already // accepted so far and stop iterating. - const auto terminalIndex = downloadItems.size(); + const auto terminalIndex = candidates.size(); downloadItemsConfirmed( - std::move(downloadItems), + std::move(candidates), std::move(existsResults), terminalIndex, overwriteNever, @@ -925,35 +932,86 @@ void RemoteSideModel::onTransfer( return; } - const auto itemsSize = items.size(); + if (!localModel_) + { + Log::error("Cannot download items: local model is not set"); + return; + } - auto destinationDir = localModel_->currentPath().value(); - if (subDir) - destinationDir /= *subDir; + const auto refuse = [this](Paths::PathError const& error) + { + showPathRefused(language->get("remoteSideModel", "downloadFailed"), error); + }; + const auto destinationDirectory = localModel_->currentLocalPath().and_then( + [&subDir](Paths::LocalPath const& directory) -> std::expected + { + if (!subDir) + return directory; + return Paths::LocalPathSegment::make(*subDir).transform( + [&directory](Paths::LocalPathSegment const& segment) + { + return directory / segment; + } + ); + } + ); + if (!destinationDirectory) + return refuse(destinationDirectory.error()); + + std::vector candidates; + candidates.reserve(items.size()); + for (auto const& item : items) + { + // A name this platform cannot hold is refused before anything is written. + auto source = remotePathOf(item); + auto destination = source.and_then( + [&destinationDirectory](Paths::RemotePath const& path) + { + return Paths::LocalPathSegment::make(*path.filename()) + .transform( + [&destinationDirectory](Paths::LocalPathSegment const& segment) + { + return *destinationDirectory / segment; + } + ); + } + ); + if (!destination) + return refuse(destination.error()); + candidates.push_back( + SharedData::BulkDownloadEntry{ + .source = *std::move(source), + .destination = *std::move(destination), + .sizeBytes = item.size, + .isDirectory = item.isDirectory(), + .mtime = item.mtime, + .mtimeNsec = item.mtimeNsec, + } + ); + } + const auto itemsSize = candidates.size(); std::string confirmText; if (itemsSize > 1) { confirmText = fmt::format( fmt::runtime(language->get("remoteSideModel", "downloadConfirmIfMultiple")), itemsSize, - destinationDir.generic_string() + destinationDirectory->toUtf8() ); } else { confirmText = fmt::format( fmt::runtime(language->get("remoteSideModel", "downloadConfirmIfSingle")), - items.front().path.generic_string(), - destinationDir.generic_string() + candidates.front().source.value(), + destinationDirectory->toUtf8() ); } std::vector listItems; - for (const auto& item : items) - { - listItems.push_back({item.path.generic_string(), ""}); - } + for (const auto& candidate : candidates) + listItems.push_back({candidate.source.value(), ""}); confirmDialog_->open( {.styleVariant = ScriptNuiComponents::StyleVariant::Primary, @@ -962,7 +1020,7 @@ void RemoteSideModel::onTransfer( .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, .focusButton = ConfirmDialog::Button::Yes, .listItems = listItems, - .onClose = [this, items, destinationDir](std::optional button) + .onClose = [this, candidates = std::move(candidates)](std::optional button) mutable { if (!button || *button != ConfirmDialog::Button::Yes) { @@ -970,61 +1028,45 @@ void RemoteSideModel::onTransfer( return; } - std::vector> downloadItems; - std::transform( - items.begin(), - items.end(), - std::back_inserter(downloadItems), - [this, destinationDir](auto const& item) - { - NuiFileExplorer::Item localItem = item; - NuiFileExplorer::Item remoteItem = item; - localItem.path = destinationDir / item.path.filename(); - localItem.fullPath = localItem.path; - remoteItem.path = currentPath_.value() / item.path; - remoteItem.fullPath = remoteItem.path; - return std::make_pair(remoteItem, localItem); - } - ); - Log::info("Downloading items."); - // Batched existence probe: one RPC for the whole batch - // instead of N. The recursion below then walks the cached - // result vector with zero further round-trips. - std::vector destPathStrings; - destPathStrings.reserve(downloadItems.size()); - for (auto const& pair : downloadItems) - destPathStrings.push_back(pair.second.path.generic_string()); + // Batched existence probe: one RPC for the whole batch instead of N. The recursion below then walks + // the cached result vector with zero further round-trips. + std::vector destinations; + destinations.reserve(candidates.size()); + for (auto const& candidate : candidates) + destinations.push_back(candidate.destination.toUtf8()); Nui::RpcClient::callWithBackChannel( "RpcFilesystem::existsBatch", - [this, downloadItems = std::move(downloadItems)](Nui::val response) mutable + [this, candidates = std::move(candidates)](Nui::val response) mutable { - auto existsResults = std::make_shared>(); - + // An unknown destination is never taken for absent, the batch overwrites what it accepts. if (!response.hasOwnProperty("success") || !response["success"].as() || !response.hasOwnProperty("exists")) { - Log::warn("RpcFilesystem::existsBatch failed; assuming nothing exists yet"); - existsResults->assign(downloadItems.size(), false); + return showExistenceCheckFailed( + response.hasOwnProperty("error") ? language->translate(response["error"].as()) + : std::string{"RpcFilesystem::existsBatch"} + ); } - else + + auto existsResults = std::make_shared>(); + auto arr = response["exists"]; + const auto length = arr["length"].as(); + existsResults->reserve(static_cast(length)); + for (long long idx = 0; idx < length; ++idx) + existsResults->push_back(arr[static_cast(idx)].as()); + if (existsResults->size() != candidates.size()) { - auto arr = response["exists"]; - const auto length = arr["length"].as(); - existsResults->reserve(static_cast(length)); - for (long long idx = 0; idx < length; ++idx) - existsResults->push_back(arr[static_cast(idx)].as()); - // Defensive padding in case the backend returned - // fewer entries than requested. - while (existsResults->size() < downloadItems.size()) - existsResults->push_back(false); + return showExistenceCheckFailed( + fmt::format("{} of {} answers", existsResults->size(), candidates.size()) + ); } - downloadItemsConfirmed(std::move(downloadItems), std::move(existsResults)); + downloadItemsConfirmed(std::move(candidates), std::move(existsResults)); }, - destPathStrings + destinations ); }} ); @@ -1076,30 +1118,28 @@ void RemoteSideModel::onTransferAsArchive(std::vector con Log::error("Cannot download as archive: local model is not wired"); return; } - if (!localModel_->currentPath().value().empty() == false) - { - Log::error("Local side has no active path; cannot pick a destination"); - return; - } // Root entries: files go straight in, directories are flattened on the // backend via a recursive SFTP listDirectory in the archive op's // prepareInStrand step. Symlinks and other non-regular / non-directory // types are logged and dropped there too. - std::vector rootEntries; - rootEntries.reserve(items.size()); + std::vector roots; + roots.reserve(items.size()); for (auto const& item : items) { - SharedData::DirectoryEntry entry = item; - entry.fullPath = currentPath_.value() / item.path; - rootEntries.push_back(std::move(entry)); + auto path = remotePathOf(item); + if (!path) + return showPathRefused(language->get("remoteSideModel", "archiveDownloadFailed"), path.error()); + roots.push_back( + SharedData::ArchiveDownloadRoot{ + .path = *std::move(path), + .entry = static_cast(item), + } + ); } - if (rootEntries.empty()) - return; - const auto defaultStem = - items.size() == 1 ? items.front().path.filename().generic_string() : language->get("fileExplorer", "archiveDefaultName"); + items.size() == 1 ? items.front().name : language->get("fileExplorer", "archiveDefaultName"); archiveTransferDialog_->open({ .headerText = language->get("fileExplorer", "contextMenu", "downloadAsArchive"), @@ -1107,7 +1147,7 @@ void RemoteSideModel::onTransferAsArchive(std::vector con .initialCodec = ArchiveCodec::Gzip, .initialCompressionLevel = 5, .onConfirm = - [this, entries = std::move(rootEntries)](std::optional const& result) mutable + [this, roots = std::move(roots)](std::optional const& result) mutable { if (!result) { @@ -1118,10 +1158,23 @@ void RemoteSideModel::onTransferAsArchive(std::vector con return; const std::string filename = result->fileStem + ".tar" + archiveCodecExtension(result->codec); - const auto localPath = localModel_->currentPath().value() / filename; + auto archivePath = localModel_->currentLocalPath().and_then( + [&filename](Paths::LocalPath const& directory) + { + return Paths::LocalPathSegment::make(filename).transform( + [&directory](Paths::LocalPathSegment const& segment) + { + return directory / segment; + } + ); + } + ); + if (!archivePath) + return showPathRefused(language->get("remoteSideModel", "archiveDownloadFailed"), archivePath.error()); + const auto localPath = *std::move(archivePath); operationQueue_->enqueueArchiveDownload( - std::move(entries), + std::move(roots), localPath, backendCompressionCode(result->codec), result->compressionLevel, @@ -1140,77 +1193,89 @@ void RemoteSideModel::onTransferAsArchive(std::vector con }); return; } - Log::info( - "Archive download queued as {} (destination {})", opId->value(), localPath.generic_string() - ); + Log::info("Archive download queued as {} (destination {})", opId->value(), localPath.toUtf8()); } ); }, }); } +void RemoteSideModel::renameItem(Paths::RemotePath const& from, Paths::RemotePath const& to, bool overwrite) +{ + fileEngine_->rename( + from, + to, + overwrite, + [this, from, to, overwrite](bool success, std::string const& info) + { + if (!success) + { + if (info == "backend.rename.targetExists" && !overwrite) + { + confirmReplaceOnRename( + std::string{to.filename()->view()}, + [this, from, to]() + { + renameItem(from, to, true); + } + ); + return; + } + Log::error("Failed to rename item '{}' to '{}': {}", from.value(), to.value(), info); + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = "", + .text = fmt::format( + fmt::runtime(language->get("remoteSideModel", "failedToRenameItem")), + from.value(), + to.value(), + language->translate(info) + ), + .buttons = ConfirmDialog::Button::Ok, + }); + return; + } + Log::info("Renamed item '{}' to '{}'", from.value(), to.value()); + onRefresh(); + } + ); +} + void RemoteSideModel::onRename(NuiFileExplorer::Item const& item) { CHECK_COMPLETE(); - Log::info("Rename item requested: {}", item.path.generic_string()); + Log::info("Rename item requested: {}", item.name); + + // The item's own path: the explorer may already show another directory. + auto from = remotePathOf(item); + if (!from) + return showPathRefused(language->get("remoteSideModel", "rename"), from.error()); inputDialog_->open({ .whatFor = language->get("remoteSideModel", "rename"), .prompt = - fmt::format(fmt::runtime(language->get("remoteSideModel", "renamePrompt")), item.path.filename().string()), + fmt::format(fmt::runtime(language->get("remoteSideModel", "renamePrompt")), item.name), .headerText = fmt::format( - fmt::runtime(language->get("remoteSideModel", "renameWithItem")), item.path.filename().string() + fmt::runtime(language->get("remoteSideModel", "renameWithItem")), item.name ), - .initialValue = item.path.filename().string(), + .initialValue = item.name, .isPassword = false, - .onConfirm = [this, item](std::optional const& name) + .onConfirm = [this, from = *std::move(from), item](std::optional const& name) { if (!name || name->empty()) return; - if (*name == item.path.filename().string()) + if (*name == item.name) return; - if (name->find('/') != std::string::npos) - { - Log::error("Invalid name (cannot contain slashes): {}", *name); - return; - } + if (!isValidItemName(*name)) + return showInvalidItemName(*name); + const auto segment = Paths::PathSegment::make(*name); + const auto directory = from.parent(); + if (!segment || !directory) + return showInvalidItemName(*name); Log::info("Renaming item to: {}", *name); - - std::filesystem::path fullFrom = currentPath_.value() / item.path; - std::filesystem::path fullTo = currentPath_.value() / *name; - - fileEngine_->rename( - fullFrom, - fullTo, - [this, fullFrom, fullTo](bool success, std::string const& info) - { - if (!success) - { - Log::error( - "Failed to rename item '{}' to '{}': {}", - fullFrom.generic_string(), - fullTo.generic_string(), - info - ); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = "", - .text = fmt::format( - fmt::runtime(language->get("remoteSideModel", "failedToRenameItem")), - fullFrom.generic_string(), - fullTo.generic_string(), - info - ), - .buttons = ConfirmDialog::Button::Ok, - }); - return; - } - Log::info("Renamed item '{}' to '{}'", fullFrom.generic_string(), fullTo.generic_string()); - onRefresh(); - } - ); + renameItem(from, *directory / *segment, false); }, }); } @@ -1219,9 +1284,14 @@ void RemoteSideModel::onProperties(NuiFileExplorer::Item const& item) { CHECK_COMPLETE(); - Log::info("Properties requested: {}", item.path.generic_string()); + Log::info("Properties requested: {}", item.name); - filePropertyDialog_->open(static_cast(item)); + const auto path = remotePathOf(item); + filePropertyDialog_->open( + static_cast(item), + path ? path->value() : item.displayName(), + item.linkTarget.value_or(std::string{}) + ); } void RemoteSideModel::navigateTo(std::filesystem::path const& path) @@ -1234,7 +1304,12 @@ void RemoteSideModel::navigateTo(std::filesystem::path const& path) currentPath_ = lexicallyNormal; fileEngine_->listDirectory( currentPath_.value(), - std::bind(&RemoteSideModel::onDirectoryListing, this, std::placeholders::_1, std::placeholders::_2) + [this, directory = currentPath_.value()]( + std::optional> const& entries, std::string const& info + ) + { + onDirectoryListing(directory, entries, info); + } ); } @@ -1338,32 +1413,41 @@ void RemoteSideModel::onFileTrackingInstanceCreated( std::string const& instanceDirStr ) { - std::string normalizedDirStr = instanceDirStr; - std::replace(normalizedDirStr.begin(), normalizedDirStr.end(), '\\', '/'); - std::filesystem::path instanceDir{normalizedDirStr}; - Log::info("FileTracking: createInstance succeeded, id={}, dir={}", instanceId.value(), instanceDirStr); - std::filesystem::path remotePath = currentPath_.value() / item.path; - std::filesystem::path localPath = instanceDir / item.path.filename(); - - NuiFileExplorer::Item remoteItem = item; - remoteItem.path = remotePath; - remoteItem.fullPath = remotePath; - NuiFileExplorer::Item localItem = item; - localItem.path = localPath; - // FileEngine::addDownload prefers `fullPath` over `path` when serializing - // the destination. Without overwriting it here, `localItem.fullPath` would - // still hold the source item's *remote* fullPath (carried over by the - // copy-construction above), and the backend would receive the remote path - // as both source and destination — making it try to write - // `.filepart` next to the remote file (permission denied). - localItem.fullPath = localPath; + const auto instanceDirectory = Paths::LocalPath::fromUtf8(instanceDirStr); + const auto remotePath = remotePathOf(item); + const auto localPath = remotePath.and_then( + [&instanceDirectory](Paths::RemotePath const& path) + { + return Paths::LocalPathSegment::make(*path.filename()) + .and_then( + [&instanceDirectory](Paths::LocalPathSegment const& segment) + { + return instanceDirectory.transform( + [&segment](Paths::LocalPath const& directory) + { + return directory / segment; + } + ); + } + ); + } + ); + if (!localPath) + return showPathRefused(language->get("remoteSideModel", "downloadFailed"), localPath.error()); operationQueue_->enqueueDownload( - remoteItem, - localItem, - [this, instanceId, instanceDir, remotePath, localPath, openWith, synchronize]( + *remotePath, + *localPath, + item.size, + [this, + instanceId, + instanceDirectory = *instanceDirectory, + remotePath = *remotePath, + localPath = *localPath, + openWith, + synchronize]( std::optional const& opId, std::string const& info ) { @@ -1374,10 +1458,10 @@ void RemoteSideModel::onFileTrackingInstanceCreated( } operationQueue_->addCompletionCallback( *opId, - [this, instanceId, instanceDir, remotePath, localPath, openWith, synchronize](bool success) + [this, instanceId, instanceDirectory, remotePath, localPath, openWith, synchronize](bool success) { onFileDownloadComplete( - instanceId, instanceDir, remotePath, localPath, openWith, synchronize, success + instanceId, instanceDirectory, remotePath, localPath, openWith, synchronize, success ); } ); @@ -1391,9 +1475,9 @@ void RemoteSideModel::onFileTrackingInstanceCreated( void RemoteSideModel::onFileDownloadComplete( Ids::InstanceId const& instanceId, - std::filesystem::path const& instanceDir, - std::filesystem::path const& remotePath, - std::filesystem::path const& localPath, + Paths::LocalPath const& instanceDirectory, + Paths::RemotePath const& remotePath, + Paths::LocalPath const& localPath, bool openWith, bool synchronize, bool success @@ -1401,30 +1485,21 @@ void RemoteSideModel::onFileDownloadComplete( { if (!success) { - Log::error("FileTracking: download failed for {}", localPath.generic_string()); + Log::error("FileTracking: download failed for {}", localPath.toUtf8()); return; } - Log::info("FileTracking: download completed for {}", localPath.generic_string()); + Log::info("FileTracking: download completed for {}", localPath.toUtf8()); if (!synchronize) { - Log::info("FileTracking: opening {} (openWith={}, no sync)", localPath.generic_string(), openWith); - Nui::RpcClient::callWithBackChannel( - "RpcFilesystem::open", - [](Nui::val result) - { - if (!result.hasOwnProperty("success") || !result["success"].as()) - Log::error("RpcFilesystem::open failed"); - }, - localPath.generic_string(), - openWith - ); + Log::info("FileTracking: opening {} (openWith={}, no sync)", localPath.toUtf8(), openWith); + FileExplorer::openLocalFile(*confirmDialog_, localPath, openWith); return; } Nui::RpcClient::callWithBackChannel( "FileTracking::addWatch", - [this, instanceId, instanceDir, remotePath, localPath, openWith](Nui::val watchResponse) + [this, instanceId, instanceDirectory, remotePath, localPath, openWith](Nui::val watchResponse) { if (!watchResponse.hasOwnProperty("success") || !watchResponse["success"].as()) { @@ -1432,10 +1507,10 @@ void RemoteSideModel::onFileDownloadComplete( return; } Log::info("FileTracking: addWatch succeeded for instance {}", instanceId.value()); - onFileWatchAdded(instanceId, instanceDir, remotePath, localPath, openWith); + onFileWatchAdded(instanceId, instanceDirectory, remotePath, localPath, openWith); }, instanceId.value(), - instanceDir.generic_string(), + instanceDirectory.toUtf8(), true, fileTracking_ ? fileTracking_->downloadTempSuffix() : std::string{Persistence::defaultTempFileSuffix} ); @@ -1443,26 +1518,17 @@ void RemoteSideModel::onFileDownloadComplete( void RemoteSideModel::onFileWatchAdded( Ids::InstanceId const& instanceId, - std::filesystem::path const& instanceDir, - std::filesystem::path const& remotePath, - std::filesystem::path const& localPath, + Paths::LocalPath const& instanceDirectory, + Paths::RemotePath const& remotePath, + Paths::LocalPath const& localPath, bool openWith ) { - fileTracking_->startWatching(instanceId, instanceDir, remotePath, localPath); + fileTracking_->startWatching(instanceId, instanceDirectory, remotePath, localPath); - Log::info("FileTracking: opening {} (openWith={})", localPath.generic_string(), openWith); + Log::info("FileTracking: opening {} (openWith={})", localPath.toUtf8(), openWith); - Nui::RpcClient::callWithBackChannel( - "RpcFilesystem::open", - [](Nui::val result) - { - if (!result.hasOwnProperty("success") || !result["success"].as()) - Log::error("RpcFilesystem::open failed"); - }, - localPath.generic_string(), - openWith - ); + FileExplorer::openLocalFile(*confirmDialog_, localPath, openWith); } // --- IPlacesProvider --- @@ -1540,7 +1606,7 @@ void RemoteSideModel::placesFromHomeListing() { const bool isDirectory = entry.isDirectory() || (entry.resolvedTarget != nullptr && entry.resolvedTarget->isDirectory()); - return isDirectory && entry.path.filename().generic_string() == place.directoryName; + return isDirectory && entry.name == place.directoryName; } ); if (exists) diff --git a/frontend/source/frontend/file_explorer/side_model.cpp b/frontend/source/frontend/file_explorer/side_model.cpp index 697fa028..f464e03c 100644 --- a/frontend/source/frontend/file_explorer/side_model.cpp +++ b/frontend/source/frontend/file_explorer/side_model.cpp @@ -21,6 +21,62 @@ std::shared_ptr SideModel::engine() return fileEngine_; } +bool SideModel::isValidItemName(std::string_view name) +{ + if (name.empty() || name == "." || name == "..") + return false; + return name.find_first_of(std::string_view{"/\\\0", 3}) == std::string_view::npos; +} + +void SideModel::showInvalidItemName(std::string const& name) +{ + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = language->get("fileExplorer", "invalidItemNameHeader"), + .text = fmt::format(fmt::runtime(language->get("fileExplorer", "invalidItemName")), name), + .buttons = ConfirmDialog::Button::Ok, + }); +} + +void SideModel::showPathRefused(std::string const& headerText, Paths::PathError const& error) +{ + Log::error("{}: {}", headerText, error.toString()); + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = headerText, + .text = error.toString(), + .buttons = ConfirmDialog::Button::Ok, + }); +} + +void SideModel::showExistenceCheckFailed(std::string const& reason) +{ + Log::error("Transfer stopped, the existence check failed: {}", reason); + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = language->get("fileExplorer", "existenceCheckFailedHeader"), + .text = fmt::format(fmt::runtime(language->get("fileExplorer", "existenceCheckFailed")), reason), + .buttons = ConfirmDialog::Button::Ok, + }); +} + +void SideModel::confirmReplaceOnRename(std::string const& name, std::function onReplace) +{ + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Warning, + .headerText = language->get("fileExplorer", "renameTargetExistsHeader"), + .text = fmt::format(fmt::runtime(language->get("fileExplorer", "renameTargetExists")), name), + .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, + .focusButton = ConfirmDialog::Button::No, + .onClose = + [onReplace = std::move(onReplace)](std::optional button) + { + if (button == ConfirmDialog::Button::Yes) + onReplace(); + }, + }); +} + bool SideModel::isComplete() const { return operationQueue_ != nullptr && fileEngine_ != nullptr; @@ -36,10 +92,17 @@ const std::vector& SideModel::items() const } void SideModel::onDirectoryListing( + std::filesystem::path const& directory, std::optional> directoryEntries, std::string const& reason ) { + if (directory != currentPath_.value()) + { + Log::debug("Dropping the listing of '{}', the side moved on.", directory.generic_string()); + return; + } + if (!directoryEntries) { Log::error("Failed to list directory: {}", reason); @@ -64,28 +127,26 @@ void SideModel::onDirectoryListing( return; } + // Servers send "." and "..", the parent link below stands for the parent instead. std::erase_if( *directoryEntries, [](auto const& entry) { - return entry.path.filename() == "."; + return entry.name == "." || entry.name == ".."; } ); - - for (auto& entry : *directoryEntries) - { - if (entry.fullPath.empty()) - entry.fullPath = *currentPath_ / entry.path; - } + listedPath_ = directory; std::vector items{}; + if (hasParent(directory)) + items.push_back(NuiFileExplorer::Item::parentLink(directory.parent_path(), "nui://app.example/icons/folder.svg")); std::transform( begin(*directoryEntries), end(*directoryEntries), std::back_inserter(items), - [this](auto const& entry) + [this, &directory](auto const& entry) { - return NuiFileExplorer::Item{ + auto item = NuiFileExplorer::Item{ entry, [&entry, this]() -> std::string { @@ -95,15 +156,15 @@ void SideModel::onDirectoryListing( if (type == NuiFileExplorer::Item::Type::BlockDevice) return "nui://app.example/icons/hard-drive.svg"; - if (uiOptions_.fileGridExtensionIcons.contains(entry.path.extension().string())) - { - return "nui://app.example/" + - uiOptions_.fileGridExtensionIcons.at(entry.path.extension().string()); - } + const auto extension = std::filesystem::path{entry.name}.extension().string(); + if (uiOptions_.fileGridExtensionIcons.contains(extension)) + return "nui://app.example/" + uiOptions_.fileGridExtensionIcons.at(extension); return "nui://app.example/icons/file.svg"; }() }; + item.fullPath = directory / entry.name; + return item; } ); diff --git a/frontend/source/frontend/main_page.cpp b/frontend/source/frontend/main_page.cpp index fb100cb4..59c05814 100644 --- a/frontend/source/frontend/main_page.cpp +++ b/frontend/source/frontend/main_page.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -132,6 +133,13 @@ void MainPage::onSetupComplete() "Main::getVersion", [this](Nui::val response) { + // Local paths are parsed by the rules of the machine the backend runs on, not WebAssembly's. + if (response.hasOwnProperty("localPlatform")) + { + Paths::defaultLocalPlatform() = response["localPlatform"].as() == "windows" + ? Paths::LocalPlatform::Windows + : Paths::LocalPlatform::Posix; + } impl_->events->appVersion = AppVersion{ .version = response["version"].as(), .gitTag = response["gitTag"].as(), @@ -237,4 +245,4 @@ Nui::ElementRenderer MainPage::render() ) ); // clang-format on -} \ No newline at end of file +} diff --git a/frontend/source/frontend/session.cpp b/frontend/source/frontend/session.cpp index ee436125..4a3ea8fc 100644 --- a/frontend/source/frontend/session.cpp +++ b/frontend/source/frontend/session.cpp @@ -86,10 +86,14 @@ struct Session::Implementation // Holds the active BackendSyncProvider between progress-dialog open and // sync-dialog close. Destroyed (→ provider.close() → backend session // released) when replaced or explicitly reset. - std::unique_ptr syncProvider_; + std::shared_ptr syncProvider_; // Last local+remote paths given to startSyncFlow; used by the dialog's // Recompare path to re-issue the flow without asking the user again. std::filesystem::path lastSyncLocalPath_{}; + /** @brief Names the sync pair of the running flow in State::syncBaselines, unset without a saved session. */ + std::optional syncBaselineKey_{}; + /** @brief When the running flow started its compare, recorded as the baseline once it is synchronized. */ + std::uint64_t syncFlowStart_{0}; std::filesystem::path lastSyncRemotePath_{}; std::unique_ptr connectionLossOverlay; std::unique_ptr snapshotManager; @@ -188,6 +192,23 @@ struct Session::Implementation // settings, reusing the cached local+remote roots. The old provider is // replaced by a fresh one; the backend session it held is closed on // destruction. + // A synchronized selection makes the time its compare started the baseline of the next sync. + syncDialog.setOnSynchronized( + [this]() + { + if (!syncBaselineKey_ || !this->stateHolder) + return; + this->stateHolder->loadModifySave( + [key = *syncBaselineKey_, time = syncFlowStart_](Persistence::State& state) + { + if (!state.syncBaselines) + state.syncBaselines.emplace(); + (*state.syncBaselines)[key] = time; + } + ); + } + ); + syncDialog.setOnRecompareRequested( [this](SyncDialog::RecompareRequest req) { @@ -219,21 +240,41 @@ struct Session::Implementation lastSyncLocalPath_ = localPath; lastSyncRemotePath_ = remotePath; + // Only a saved session has a stable identity to remember the last sync of. + syncBaselineKey_.reset(); + std::optional lastSyncTime{}; + if (persistenceName && !persistenceName->empty()) + { + syncBaselineKey_ = + fmt::format("{}\n{}\n{}", *persistenceName, localPath.generic_string(), remotePath.generic_string()); + if (auto const& baselines = stateHolder->stateCache().syncBaselines; baselines) + { + if (const auto found = baselines->find(*syncBaselineKey_); found != baselines->end()) + lastSyncTime = found->second; + } + } + syncFlowStart_ = static_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() + ); + initialDiffOptions.lastSyncTime = lastSyncTime; + // Replace the old provider before opening the progress dialog so its // backend session (if any) is closed before the new one opens. - syncProvider_ = std::make_unique(&operationQueue); + syncProvider_ = std::make_shared(&operationQueue); syncProgressDialog.open( - syncProvider_.get(), + syncProvider_, localPath, remotePath, scan.respectIgnoreFiles, scan.recursive, scan.ignoreHidden, initialDiffOptions, - [this, localPath, remotePath, scan](SharedData::Sync::DiffSnapshot snapshot) + [this, provider = syncProvider_, localPath, remotePath, scan, lastSyncTime]( + SharedData::Sync::DiffSnapshot snapshot + ) { - syncDialog.open(syncProvider_.get(), std::move(snapshot), localPath, remotePath, scan); + syncDialog.open(provider, std::move(snapshot), localPath, remotePath, scan, lastSyncTime); Nui::globalEventContext.executeActiveEventsImmediately(); } ); @@ -253,7 +294,7 @@ std::string Session::layoutId() const void Session::onDrop( bool isLocalSide, - std::vector entries, + std::vector entries, std::optional const& subdir ) { diff --git a/frontend/source/frontend/session/session_helpers.cpp b/frontend/source/frontend/session/session_helpers.cpp index 7f3b6e94..15d13359 100644 --- a/frontend/source/frontend/session/session_helpers.cpp +++ b/frontend/source/frontend/session/session_helpers.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -23,22 +24,51 @@ namespace SessionInternal "RpcFilesystem::writeFile", [filePath](Nui::val response) { - if (!response.hasOwnProperty("success")) + if (!response.hasOwnProperty("success") || !response["success"].as()) { Log::error( - "Invalid response from RpcFilesystem::writeFile for file '{}'", filePath.generic_string() + "Failed to write file '{}': {}", + filePath.generic_string(), + response.hasOwnProperty("error") ? response["error"].as() : std::string{} ); return; } + Log::info("Successfully wrote file '{}'", filePath.generic_string()); + }, + filePath.generic_string(), + content + ); + } + + void writeChannelContentToNewFile( + std::filesystem::path const& filePath, + std::string const& content, + std::function onDone + ) + { + Nui::RpcClient::callWithBackChannel( + "RpcFilesystem::writeNewFile", + [filePath, onDone = std::move(onDone)](Nui::val response) + { + if (!response.hasOwnProperty("success")) + { + Log::error( + "Invalid response from RpcFilesystem::writeNewFile for file '{}'", filePath.generic_string() + ); + return onDone(WriteOutcome::Failed); + } if (!response["success"].as()) { const auto error = response["error"].as(); + if (Utility::parseLocalizedMessage(error).key == "backend.rpcFilesystem.fileExists") + return onDone(WriteOutcome::Exists); Log::error("Failed to write file '{}': {}", filePath.generic_string(), error); - return; + return onDone(WriteOutcome::Failed); } Log::info("Successfully wrote file '{}'", filePath.generic_string()); + onDone(WriteOutcome::Written); }, - filePath, + filePath.generic_string(), content ); } diff --git a/frontend/source/frontend/session_area.cpp b/frontend/source/frontend/session_area.cpp index 1a84a54f..747a0081 100644 --- a/frontend/source/frontend/session_area.cpp +++ b/frontend/source/frontend/session_area.cpp @@ -238,7 +238,7 @@ void SessionArea::registerRpc() session->onDrop( isLeft, - json.value("entries", nlohmann::json::array()).get>(), + json.value("entries", nlohmann::json::array()).get>(), subdir ); } diff --git a/frontend/source/frontend/session_components/file_explorer_panel.cpp b/frontend/source/frontend/session_components/file_explorer_panel.cpp index dd9e781d..42bc4914 100644 --- a/frontend/source/frontend/session_components/file_explorer_panel.cpp +++ b/frontend/source/frontend/session_components/file_explorer_panel.cpp @@ -266,7 +266,7 @@ Nui::ElementRenderer FileExplorerPanel::makeFileExplorerElement() void FileExplorerPanel::onDrop( bool isLocalSide, - std::vector entries, + std::vector entries, std::optional const& subdir ) { @@ -290,9 +290,9 @@ void FileExplorerPanel::onDrop( std::make_move_iterator(begin(entries)), std::make_move_iterator(end(entries)), std::back_inserter(items), - [](SharedData::DirectoryEntry const& entry) + [](SharedData::DroppedItem const& dropped) { - return NuiFileExplorer::Item{entry}; + return NuiFileExplorer::Item::dropped(dropped.entry, dropped.path.toUtf8()); } ); diff --git a/frontend/source/frontend/session_components/file_tracking.cpp b/frontend/source/frontend/session_components/file_tracking.cpp index f61eda2d..63519961 100644 --- a/frontend/source/frontend/session_components/file_tracking.cpp +++ b/frontend/source/frontend/session_components/file_tracking.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include +#include #include #include #include @@ -30,13 +32,30 @@ using namespace std::string_literals; struct TrackedEntry { Ids::InstanceId instanceId; - std::filesystem::path instanceDir; - std::filesystem::path remotePath; - std::filesystem::path localPath; + Paths::LocalPath instanceDirectory; + Paths::RemotePath remotePath; + Paths::LocalPath localPath; bool autoReupload{true}; bool moveRemoteOnLocalMove{false}; bool deleteRemote{false}; std::shared_ptr> uploading{std::make_shared>(false)}; + /** + * @brief Changed files whose upload is being queued, by local path. True when the file changed again meanwhile, + * it is then queued once more. A tracked directory has many files, each is tracked on its own. + */ + std::shared_ptr> uploadsBeingQueued{std::make_shared>()}; + /** @brief Ask before replacing a server copy that changed since it was opened or last uploaded. */ + bool checkRemoteChanges{true}; + struct ServerState + { + std::uint64_t mtime{0}; + std::uint64_t size{0}; + }; + /** + * @brief What the server had for each file, by local path: taken from the download (it keeps the server's + * modification time) and updated after every upload. + */ + std::shared_ptr> serverStates{std::make_shared>()}; }; struct FileTrackingPanel::Implementation @@ -82,6 +101,11 @@ struct FileTrackingPanel::Implementation .initialWidth = 120, .resizeable = false, }, + ScriptNuiComponents::ResizableTable::HeaderTableCell{ + .content = std::string{language->get("fileTracking", "columnCheckRemote")}, + .initialWidth = 120, + .resizeable = false, + }, ScriptNuiComponents::ResizableTable::HeaderTableCell{ .content = std::string{language->get("fileTracking", "columnStatus")}, .initialWidth = 100, @@ -227,8 +251,8 @@ struct FileTrackingPanel::Implementation auto uploadingObs = entry.uploading; return ResizableTable::TableRow{ - entry.localPath.filename().generic_string(), - entry.remotePath.generic_string(), + entry.localPath.filename() ? std::string{entry.localPath.filename()->view()} : entry.localPath.toUtf8(), + entry.remotePath.value(), ResizableTable::TableCell{ [this, instanceIdStr, isChecked = entry.autoReupload](std::unique_ptr) -> Nui::ElementRenderer @@ -290,6 +314,27 @@ struct FileTrackingPanel::Implementation }); }, }, + ResizableTable::TableCell{ + [this, instanceIdStr, isChecked = entry.checkRemoteChanges]( + std::unique_ptr + ) -> Nui::ElementRenderer + { + return ScriptNuiComponents::switch_({ + .isChecked = isChecked, + .onChange = [this, instanceIdStr](bool checked, Nui::WebApi::MouseEvent const&) + { + for (auto& ent : entries.value()) + { + if (ent.instanceId.value() == instanceIdStr) + { + ent.checkRemoteChanges = checked; + break; + } + } + }, + }); + }, + }, ResizableTable::TableCell{ [uploadingObs](std::unique_ptr) -> Nui::ElementRenderer { @@ -367,6 +412,175 @@ struct FileTrackingPanel::Implementation instanceIdStr ); } + + /** + * @brief Queues the upload of the changed @p source of the tracked item @p instanceId. A change that arrives while + * the same file is being queued is not dropped, the file is queued again afterwards. + */ + void queueReupload(Ids::InstanceId const& instanceId, Paths::LocalPath const& source, Paths::RemotePath const& destination) + { + auto entry = std::find_if( + entries.value().begin(), + entries.value().end(), + [&instanceId](TrackedEntry const& tracked) + { + return tracked.instanceId.value() == instanceId.value(); + } + ); + if (entry == entries.value().end() || !operationQueue) + return; + + auto queued = entry->uploadsBeingQueued; + const auto key = source.toUtf8(); + if (auto found = queued->find(key); found != queued->end()) + { + found->second = true; + return; + } + queued->emplace(key, false); + (*entry->uploading) = true; + + Reupload reupload{ + .instanceId = instanceId, + .source = source, + .destination = destination, + .queued = queued, + .uploading = entry->uploading, + .serverStates = entry->serverStates, + }; + auto fileEngine = operationQueue->fileEngine(); + if (!entry->checkRemoteChanges || !fileEngine) + return enqueueReupload(std::move(reupload)); + + fileEngine->stat( + std::filesystem::path{destination.value()}, + [this, reupload]( + std::optional> const& result, std::string const& info + ) mutable + { + const auto known = reupload.serverStates->find(reupload.source.toUtf8()); + // A file the server has but nobody saw here, a vanished one and one with another size or time all + // changed on the server. An unanswered check counts as changed. + bool changed = true; + if (result) + { + auto const& [exists, stat] = *result; + if (known == reupload.serverStates->end()) + changed = exists; + else + changed = !exists || stat.mtime != known->second.mtime || stat.size != known->second.size; + } + else + { + Log::warn("FileTracking: cannot check '{}' on the server: {}", reupload.destination.value(), info); + } + if (!changed) + return enqueueReupload(std::move(reupload)); + + confirmDialog->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Warning, + .headerText = language->get("fileTracking", "serverChangedHeader"), + .text = fmt::format( + fmt::runtime(language->get("fileTracking", "serverChangedText")), reupload.destination.value() + ), + .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, + .focusButton = ConfirmDialog::Button::No, + .onClose = + [this, reupload](std::optional button) mutable + { + if (button == ConfirmDialog::Button::Yes) + return enqueueReupload(std::move(reupload)); + Log::info("FileTracking: kept the server's version of '{}'.", reupload.destination.value()); + finishQueueing(reupload); + }, + }); + } + ); + } + + /** + * @brief A changed tracked file on its way to the server. + */ + struct Reupload + { + Ids::InstanceId instanceId; + Paths::LocalPath source; + Paths::RemotePath destination; + std::shared_ptr> queued; + std::shared_ptr> uploading; + std::shared_ptr> serverStates; + }; + + void enqueueReupload(Reupload reupload) + { + if (!operationQueue) + return finishQueueing(reupload); + operationQueue->enqueueUpload( + reupload.source, + reupload.destination, + /*sizeBytes=*/0, + [this, reupload](std::optional const& opId, std::string const& info) + { + if (!opId) + Log::error( + "FileTracking: failed to enqueue upload for instance {}: {}", reupload.instanceId.value(), info + ); + else + rememberServerStateAfter(*opId, reupload); + finishQueueing(reupload); + }, + true, // allowOverwrite: the tracked file replaces its remote original, checked above + false, // insertRefresh + false, // createMissingDirectories + SharedData::OperationMode::PriorityQueued + ); + } + + /** + * @brief Once the upload finished, what the server has now is the state the next check compares with. + */ + void rememberServerStateAfter(Ids::OperationId const& operationId, Reupload const& reupload) + { + operationQueue->addCompletionCallback( + operationId, + [this, reupload](bool success) + { + auto fileEngine = operationQueue ? operationQueue->fileEngine() : nullptr; + if (!success || !fileEngine) + return; + fileEngine->stat( + std::filesystem::path{reupload.destination.value()}, + [reupload]( + std::optional> const& result, std::string const& + ) + { + if (result && result->first) + { + (*reupload.serverStates)[reupload.source.toUtf8()] = TrackedEntry::ServerState{ + .mtime = result->second.mtime, + .size = result->second.size, + }; + } + } + ); + } + ); + } + + /** + * @brief The file is no longer being queued; when it changed again meanwhile it goes through once more. + */ + void finishQueueing(Reupload const& reupload) + { + const auto key = reupload.source.toUtf8(); + const auto found = reupload.queued->find(key); + const bool changedAgain = found != reupload.queued->end() && found->second; + if (found != reupload.queued->end()) + reupload.queued->erase(found); + (*reupload.uploading) = !reupload.queued->empty(); + if (changedAgain) + queueReupload(reupload.instanceId, reupload.source, reupload.destination); + } }; FileTrackingPanel::FileTrackingPanel( @@ -408,23 +622,52 @@ void FileTrackingPanel::deactivate() impl_->cleanupRunning = false; } +namespace +{ + /** + * @brief The remote path a change at @p changed maps to. The instance directory stands for the directory of + * @p remotePath, a change outside it or of the instance directory itself maps to nothing. + */ + std::expected remotePathOfChange( + Paths::LocalPath const& changed, + Paths::LocalPath const& instanceDirectory, + Paths::RemotePath const& remotePath + ) + { + const auto remoteDirectory = remotePath.parent(); + if (!remoteDirectory) + return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::EscapesRoot, .input = remotePath.value()}); + return changed.relativeTo(instanceDirectory) + .and_then( + [&changed, &remoteDirectory]( + Paths::RelativePath const& relative + ) -> std::expected + { + if (relative.empty()) + return std::unexpected(Paths::PathError{.reason = Paths::PathRejection::Empty, .input = changed.toUtf8()}); + return *remoteDirectory / relative; + } + ); + } +} + void FileTrackingPanel::startWatching( Ids::InstanceId const& instanceId, - std::filesystem::path const& instanceDir, - std::filesystem::path const& remotePath, - std::filesystem::path const& localPath + Paths::LocalPath const& instanceDirectory, + Paths::RemotePath const& remotePath, + Paths::LocalPath const& localPath ) { Log::info( "FileTracking: starting watch for instance {} (remote: {}, local: {})", instanceId.value(), - remotePath.generic_string(), - localPath.generic_string() + remotePath.value(), + localPath.toUtf8() ); impl_->fileChangeListeners[instanceId.value()] = Nui::RpcClient::autoRegisterFunction( fmt::format("FileTracking::{}::onFileChanged", instanceId.value()), - [this, instanceId, remotePath, localPath, instanceDir](Nui::val payload) + [this, instanceId, remotePath, localPath, instanceDirectory](Nui::val payload) { const auto action = payload["action"].as(); const auto directory = payload["directory"].as(); @@ -432,14 +675,18 @@ void FileTrackingPanel::startWatching( Log::debug("FileTracking: file changed in instance {}: {} {}", instanceId.value(), action, filename); + const auto localPathOfChange = [&directory](std::string const& name) + { + return Paths::LocalPath::fromUtf8(fmt::format("{}/{}", directory, name)); + }; // Only the tracked file or directory maps to the remote side, anything else in the instance // directory (Finder's .DS_Store for one) would land next to it on the server. - const auto isTracked = [&localPath](std::filesystem::path const& changedLocalPath) + const auto isTracked = [&localPath](std::expected const& changed) { - const auto relative = changedLocalPath.lexically_relative(localPath); - return !relative.empty() && *relative.begin() != ".."; + return changed && changed->isWithin(localPath); }; - if (!isTracked(std::filesystem::path{directory} / filename)) + const auto changedLocalPath = localPathOfChange(filename); + if (!isTracked(changedLocalPath)) { Log::debug("FileTracking: ignoring change outside the tracked item: {}", filename); return; @@ -461,19 +708,32 @@ void FileTrackingPanel::startWatching( { if (it->deleteRemote && impl_->operationQueue) { - std::filesystem::path changedLocalPath = std::filesystem::path{directory} / filename; - std::filesystem::path relPath = changedLocalPath.lexically_relative(instanceDir); - std::filesystem::path changedRemotePath = remotePath.parent_path() / relPath; + // The deleted item must map to a path below the tracked remote directory, never next to it. + const auto remoteOfChange = remotePathOfChange(*changedLocalPath, instanceDirectory, remotePath); + if (!remoteOfChange) + { + Log::error( + "FileTracking: not mirroring the deletion of '{}': {}", + changedLocalPath->toUtf8(), + remoteOfChange.error().toString() + ); + return; + } + const auto changedRemotePath = *remoteOfChange; Log::info( "FileTracking: enqueueing remote delete for instance {} due to local deletion: {}", instanceId.value(), - changedRemotePath.generic_string() + changedRemotePath.value() ); + // Never recursive: the event does not say whether a file or a directory went, and a deleted + // local file must not take a remote directory of that name with it. A deleted local directory + // reports its content first, so its remote copy is empty by the time it is removed; anything + // only the server has keeps it. impl_->operationQueue->enqueueDelete( {changedRemotePath}, - true, + /*recursive=*/false, [instanceId](std::optional> const& opIds, std::string const& info) { if (!opIds) @@ -489,37 +749,29 @@ void FileTrackingPanel::startWatching( return; } - const auto oldLocalPath = - std::filesystem::path{directory} / payload["oldFilename"].as(); + const auto oldLocalPath = localPathOfChange(payload["oldFilename"].as()); // An atomic save renames a temporary from outside the tracked item over it, a change of its content. if (action == "Moved" && isTracked(oldLocalPath)) { if (!it->moveRemoteOnLocalMove || !impl_->operationQueue) return; - std::filesystem::path newLocalPath = std::filesystem::path{directory} / filename; - - std::filesystem::path oldRelPath = oldLocalPath.lexically_relative(instanceDir); - std::filesystem::path newRelPath = newLocalPath.lexically_relative(instanceDir); - - // Check that newRelPath is inside the instanceDir to avoid arbitrary remote renames - if (newRelPath.empty() || newRelPath.string().starts_with("..")) + // Both ends must lie inside the instance directory to avoid arbitrary remote renames. + auto oldRemotePath = remotePathOfChange(*oldLocalPath, instanceDirectory, remotePath); + auto newRemotePath = remotePathOfChange(*changedLocalPath, instanceDirectory, remotePath); + if (!oldRemotePath || !newRemotePath) { Log::warn( - "FileTracking: ignoring move event for instance {} because new path is outside instance dir: " - "{}", + "FileTracking: not mirroring the move in instance {}: {}", instanceId.value(), - newRelPath.generic_string() + (oldRemotePath ? newRemotePath.error() : oldRemotePath.error()).toString() ); return; } - std::filesystem::path oldRemotePath = remotePath.parent_path() / oldRelPath; - std::filesystem::path newRemotePath = remotePath.parent_path() / newRelPath; - impl_->operationQueue->enqueueRename( - oldRemotePath, - newRemotePath, + *oldRemotePath, + *newRemotePath, [instanceId](std::optional const& opId, std::string const& info) { if (!opId) @@ -534,53 +786,25 @@ void FileTrackingPanel::startWatching( return; } - if (!it->autoReupload || *it->uploading) + if (!it->autoReupload) return; if (!impl_->operationQueue) return; - (*it->uploading) = true; - - std::filesystem::path changedLocalPath = std::filesystem::path{directory} / filename; - std::filesystem::path relPath = changedLocalPath.lexically_relative(instanceDir); - std::filesystem::path changedRemotePath = remotePath.parent_path() / relPath; - - NuiFileExplorer::Item localItem{SharedData::DirectoryEntry{ - .path = changedLocalPath, - .type = SharedData::FileType::Regular, - }}; - NuiFileExplorer::Item remoteItem{SharedData::DirectoryEntry{ - .path = changedRemotePath, - .type = SharedData::FileType::Regular, - }}; - - impl_->operationQueue->enqueueUpload( - remoteItem, - localItem, - [this, instanceId](std::optional const& opId, std::string const& info) - { - if (!opId) - { - Log::error( - "FileTracking: failed to enqueue upload for instance {}: {}", instanceId.value(), info - ); - } - auto& vec = impl_->entries.value(); - for (auto& ent : vec) - { - if (ent.instanceId.value() == instanceId.value()) - { - (*ent.uploading) = false; - break; - } - } - }, - true, // allowOverwrite — re-upload always overwrites - false, // insertRefresh - false, // createMissingDirectories - SharedData::OperationMode::PriorityQueued - ); + const auto source = *changedLocalPath; + const auto destination = remotePathOfChange(source, instanceDirectory, remotePath); + if (!destination) + { + Log::warn( + "FileTracking: not uploading the change in instance {}: {}", + instanceId.value(), + destination.error().toString() + ); + return; + } + + impl_->queueReupload(instanceId, source, *destination); } ); @@ -588,16 +812,44 @@ void FileTrackingPanel::startWatching( impl_->entries.value().push_back( TrackedEntry{ .instanceId = instanceId, - .instanceDir = instanceDir, + .instanceDirectory = instanceDirectory, .remotePath = remotePath, .localPath = localPath, .autoReupload = ftOpts.autoReupload, .moveRemoteOnLocalMove = ftOpts.moveRemoteOnLocalMove, .deleteRemote = ftOpts.deleteRemoteOnLocalDelete, + .checkRemoteChanges = ftOpts.checkRemoteChanges.value_or(true), } ); impl_->table->addRow(impl_->buildRowForEntry(impl_->entries.value().back())); impl_->entries.modifyNow(); + + // The downloaded files carry the server's modification time, so they tell what the server had. + Nui::RpcClient::callWithBackChannel( + "FileTracking::fileStates", + [serverStates = impl_->entries.value().back().serverStates, instanceId](Nui::val reply) + { + if (!reply.hasOwnProperty("files")) + { + Log::warn("FileTracking: no file states for instance {}, every re-upload will ask.", instanceId.value()); + return; + } + const auto files = reply["files"]; + const auto count = files["length"].as(); + for (int index = 0; index < count; ++index) + { + const auto file = files[index]; + serverStates->insert_or_assign( + file["path"].as(), + TrackedEntry::ServerState{ + .mtime = static_cast(file["mtime"].as()), + .size = static_cast(file["size"].as()), + } + ); + } + }, + instanceId.value() + ); } Nui::ElementRenderer FileTrackingPanel::operator()() diff --git a/frontend/source/frontend/session_components/operation_queue.cpp b/frontend/source/frontend/session_components/operation_queue.cpp index 3d4002ab..8ac8f71f 100644 --- a/frontend/source/frontend/session_components/operation_queue.cpp +++ b/frontend/source/frontend/session_components/operation_queue.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -85,7 +86,7 @@ struct OperationQueue::Implementation Nui::TimerHandle autoCleanTimer; std::unordered_map> completionCallbacks; - std::unordered_map const&)>> + std::unordered_map const&)>> failedEntriesCallbacks; std::unordered_map> transferProgressCallbacks; // Keyed by the aggregate bulk operation id (first pre-generated opId for @@ -101,6 +102,7 @@ struct OperationQueue::Implementation { std::array scanIds{}; // [0]=remote, [1]=local std::function onScanPhaseDone; + std::function onScanFailed; std::function onDiffProgress; }; std::unordered_map syncSessionRouting; @@ -447,6 +449,11 @@ void OperationQueue::deactivate() impl_->sessionId = {}; } +std::shared_ptr OperationQueue::fileEngine() const +{ + return impl_->fileEngine; +} + void OperationQueue::activate(std::shared_ptr fileEngine, Ids::SessionId sessionId) { impl_->fileEngine = std::move(fileEngine); @@ -589,6 +596,23 @@ void OperationQueue::activate(std::shared_ptr fileEngine, Ids::Sessi ) ); + impl_->onUpdate.push_back( + Nui::RpcClient::autoRegisterFunction( + fmt::format("OperationQueue::{}::onSyncScanFailed", impl_->sessionId.value()), + [this](std::string const& sessionIdString, bool isLocal, std::string const& reason) + { + const auto routingIt = impl_->syncSessionRouting.find(sessionIdString); + if (routingIt == impl_->syncSessionRouting.end()) + return; + Log::error("Sync scan of the {} side failed: {}", isLocal ? "local" : "remote", reason); + for (auto const& scanId : routingIt->second.scanIds) + impl_->syncScanProgressCallbacks.erase(scanId.value()); + if (routingIt->second.onScanFailed) + routingIt->second.onScanFailed(reason); + } + ) + ); + impl_->onUpdate.push_back( Nui::RpcClient::autoRegisterFunction( fmt::format("OperationQueue::{}::onSyncDiffProgress", impl_->sessionId.value()), @@ -651,7 +675,7 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) ); return {}; } - return std::make_unique( + auto card = std::make_unique( added.operationId, SharedData::OperationType::Download, *impl_->confirmDialog, @@ -665,6 +689,8 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) impl_->autoClean, localRefresh ); + card->allowOverwrite(added.allowOverwrite.value_or(false)); + return card; } else if (added.type == SharedData::OperationType::Upload) { @@ -676,7 +702,7 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) ); return {}; } - return std::make_unique( + auto card = std::make_unique( added.operationId, SharedData::OperationType::Upload, *impl_->confirmDialog, @@ -690,6 +716,8 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) impl_->autoClean, remoteRefresh ); + card->allowOverwrite(added.allowOverwrite.value_or(false)); + return card; } else if (added.type == SharedData::OperationType::Scan) { @@ -703,7 +731,7 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) return std::make_unique( added.operationId, *impl_->confirmDialog, - *added.remotePath, + added.remotePath->value(), [this](OperationCard const& operation) { cancelOperation(operation); @@ -721,7 +749,7 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) return std::make_unique( added.operationId, *impl_->confirmDialog, - *added.localPath, + added.localPath->toUtf8(), [this](OperationCard const& operation) { cancelOperation(operation); @@ -743,19 +771,38 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) "Creating {} operation card for operation id: {}. path: {}", added.type == SharedData::OperationType::BulkDelete ? "bulk delete" : "delete", added.operationId.value(), - added.remotePath ? added.remotePath->generic_string() : "???"s + added.remotePath ? added.remotePath->value() : "???"s ); return std::make_unique( added.operationId, *impl_->confirmDialog, added.type, - added.remotePath ? *added.remotePath : std::filesystem::path{}, + added.remotePath ? added.remotePath->value() : std::string{}, + added.type == SharedData::OperationType::Delete ? added.remotePath.value_or(Paths::RemotePath{}) + : Paths::RemotePath{}, [this](OperationCard const& operation) { cancelOperation(operation); }, impl_->autoClean, - remoteRefresh + remoteRefresh, + added.scanPhase.value_or(false) + ); + } + else if (added.type == SharedData::OperationType::LocalDelete) + { + return std::make_unique( + added.operationId, + *impl_->confirmDialog, + added.type, + added.localPath ? added.localPath->toUtf8() : std::string{}, + Paths::RemotePath{}, + [this](OperationCard const& operation) + { + cancelOperation(operation); + }, + impl_->autoClean, + std::function{} ); } else if (added.type == SharedData::OperationType::BulkDownload) @@ -784,7 +831,8 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) cancelOperation(operation); }, impl_->autoClean, - localRefresh + localRefresh, + added.scanPhase.value_or(false) ); } else if (added.type == SharedData::OperationType::BulkUpload) @@ -813,7 +861,8 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) cancelOperation(operation); }, impl_->autoClean, - remoteRefresh + remoteRefresh, + added.scanPhase.value_or(false) ); } else if (added.type == SharedData::OperationType::ArchiveDownload) @@ -832,13 +881,14 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) *impl_->confirmDialog, added.totalBytes ? static_cast(*added.totalBytes) : 0, *added.localPath, - std::filesystem::path{}, + Paths::RemotePath{}, [this](OperationCard const& operation) { cancelOperation(operation); }, impl_->autoClean, - localRefresh + localRefresh, + added.scanPhase.value_or(false) ); } else if (added.type == SharedData::OperationType::ArchiveUpload) @@ -856,19 +906,20 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) SharedData::OperationType::ArchiveUpload, *impl_->confirmDialog, added.totalBytes ? static_cast(*added.totalBytes) : 0, - std::filesystem::path{}, + Paths::LocalPath{}, *added.remotePath, [this](OperationCard const& operation) { cancelOperation(operation); }, impl_->autoClean, - remoteRefresh + remoteRefresh, + added.scanPhase.value_or(false) ); } else if (added.type == SharedData::OperationType::Rename) { - if (!added.localPath || !added.remotePath) + if (!added.source || !added.destination) { Log::error( "Received OperationAdded for Rename id: {} without source or destination", added.operationId.value() @@ -878,8 +929,8 @@ void OperationQueue::onOperationAdded(SharedData::OperationAdded const& added) return std::make_unique( added.operationId, *impl_->confirmDialog, - *added.localPath, - *added.remotePath, + *added.source, + *added.destination, [this](OperationCard const& operation) { cancelOperation(operation); @@ -950,7 +1001,8 @@ void OperationQueue::onDeleteProgress(SharedData::BulkDeleteProgress const& prog // Both Delete (per-directory recursive) and BulkDelete (the file/empty-dir // aggregate) cards render via DisplayedDeleteOperation and emit onDeleteProgress. if (operation->type() != SharedData::OperationType::Delete && - operation->type() != SharedData::OperationType::BulkDelete) + operation->type() != SharedData::OperationType::BulkDelete && + operation->type() != SharedData::OperationType::LocalDelete) { Log::error("Received delete progress for operation id: {} which is not a delete", progress.operationId.value()); return; @@ -1011,12 +1063,39 @@ void OperationQueue::onScanProgress(SharedData::ScanProgress const& progress) if (!operation) return; // Scan-only operations (for sync) may not be in the display map — that is fine. - if (operation->type() == SharedData::OperationType::Scan || - operation->type() == SharedData::OperationType::LocalScan) + // Bulk, delete and archive operations report their own scan phase under their id. + switch (operation->type()) { - auto* renderer = operation->getCardSpecifically(); - if (renderer) - renderer->setProgress(progress.totalBytes, progress.currentIndex, progress.totalScanned); + case SharedData::OperationType::Scan: + case SharedData::OperationType::LocalScan: + { + if (auto* card = operation->getCardSpecifically(); card) + card->setProgress(progress.totalBytes, progress.currentIndex, progress.totalScanned); + break; + } + case SharedData::OperationType::BulkDownload: + case SharedData::OperationType::BulkUpload: + { + if (auto* card = operation->getCardSpecifically(); card) + card->setScanProgress(progress.totalBytes, progress.totalScanned); + break; + } + case SharedData::OperationType::Delete: + case SharedData::OperationType::BulkDelete: + { + if (auto* card = operation->getCardSpecifically(); card) + card->setScanProgress(progress.totalBytes, progress.totalScanned); + break; + } + case SharedData::OperationType::ArchiveDownload: + case SharedData::OperationType::ArchiveUpload: + { + if (auto* card = operation->getCardSpecifically(); card) + card->setScanProgress(progress.totalBytes, progress.totalScanned); + break; + } + default: + break; } } @@ -1216,7 +1295,7 @@ void OperationQueue::onOperationCompleted(Nui::val val) impl_->failedEntriesCallbacks.erase(failedIt); if (!completed.failedEntries.empty()) { - std::vector failedPaths; + std::vector failedPaths; failedPaths.reserve(completed.failedEntries.size()); for (auto const& entry : completed.failedEntries) failedPaths.push_back(entry.first); @@ -1291,7 +1370,7 @@ void OperationQueue::addCompletionCallback(Ids::OperationId const& opId, std::fu void OperationQueue::addFailedEntriesCallback( Ids::OperationId const& opId, - std::function const&)> callback + std::function const&)> callback ) { impl_->failedEntriesCallbacks[opId.value()] = std::move(callback); @@ -1382,6 +1461,7 @@ void OperationQueue::openSyncSession( std::function onRemoteProgress, std::function onLocalProgress, std::function onScanPhaseDone, + std::function onScanFailed, std::function onDiffProgress ) { @@ -1401,6 +1481,7 @@ void OperationQueue::openSyncSession( impl_->syncSessionRouting[syncSessionId.value()] = Implementation::SyncSessionRouting{ .scanIds = {remoteScanId, localScanId}, .onScanPhaseDone = std::move(onScanPhaseDone), + .onScanFailed = std::move(onScanFailed), .onDiffProgress = std::move(onDiffProgress), }; @@ -1413,15 +1494,20 @@ void OperationQueue::openSyncSession( respectIgnoreFiles, recursive, ignoreHidden, - [remoteScanId, localScanId](bool success, std::string const& info) + [this, syncSessionId, remoteScanId, localScanId](bool success, std::string const& info) { - if (!success) - Log::error( - "Failed to open sync session (remoteId={}, localId={}): {}", - remoteScanId.value(), - localScanId.value(), - info - ); + if (success) + return; + Log::error( + "Failed to open sync session (remoteId={}, localId={}): {}", + remoteScanId.value(), + localScanId.value(), + info + ); + // No scan runs, so neither phase would ever report back. + const auto routingIt = impl_->syncSessionRouting.find(syncSessionId.value()); + if (routingIt != impl_->syncSessionRouting.end() && routingIt->second.onScanFailed) + routingIt->second.onScanFailed(info); } ); } @@ -1467,20 +1553,15 @@ void OperationQueue::loadSyncDiffChildren( ); } -void OperationQueue::buildSyncEnqueuePlan( - Ids::SyncSessionId syncSessionId, - SharedData::Sync::DiffSection section, - std::vector selectedRelKeys, - std::uint64_t generation, - std::function)> onResolved, +void OperationQueue::executeSyncSelection( + SharedData::Sync::SyncExecutionRequest request, + std::function onResolved, std::function onRejected ) { if (!impl_->fileEngine) - return; - impl_->fileEngine->buildSyncEnqueuePlan( - syncSessionId, section, std::move(selectedRelKeys), generation, std::move(onResolved), std::move(onRejected) - ); + return onRejected("No file engine set"); + impl_->fileEngine->executeSyncSelection(std::move(request), std::move(onResolved), std::move(onRejected)); } void OperationQueue::cancelSyncDiff(Ids::SyncSessionId syncSessionId) @@ -1497,69 +1578,11 @@ void OperationQueue::closeSyncSession(Ids::SyncSessionId syncSessionId) impl_->fileEngine->closeSyncSession(syncSessionId); } -void OperationQueue::createRemoteDirectory( - std::filesystem::path const& path, - std::function onComplete -) -{ - if (!impl_->fileEngine) - { - Log::error("No file engine set for operation queue, cannot create remote directory"); - onComplete(false, "No file engine set"); - return; - } - impl_->fileEngine->createDirectory(path, std::move(onComplete)); -} - -void OperationQueue::createLocalDirectory( - std::filesystem::path const& path, - std::function onComplete -) +void OperationQueue::removeCallbacks(Ids::OperationId const& opId) { - Nui::val args = Nui::val::object(); - args.set("path", path.generic_string()); - Nui::RpcClient::callWithBackChannel( - "RpcFilesystem::createDirectory", - [onComplete = std::move(onComplete)](Nui::val val) - { - if (!val.hasOwnProperty("success")) - { - onComplete(false, "Invalid response from backend"); - return; - } - if (!val["success"].as()) - { - onComplete(false, val["error"].as()); - return; - } - onComplete(true, "Success"); - }, - args - ); -} - -void OperationQueue::removeLocalPaths( - std::vector const& paths, - std::function onComplete -) -{ - Nui::val args = Nui::val::object(); - args.set("paths", Nui::val::array()); - args.set("recursive", true); - for (auto const& path : paths) - args["paths"].call("push", path.generic_string()); - Nui::RpcClient::callWithBackChannel( - "RpcFilesystem::removeSome", - [onComplete = std::move(onComplete)](Nui::val val) - { - if (!val.hasOwnProperty("success")) - return onComplete(false, "Invalid response from backend"); - if (!val["success"].as()) - return onComplete(false, language->translate(val["error"].as())); - onComplete(true, "Success"); - }, - args - ); + impl_->completionCallbacks.erase(opId.value()); + impl_->failedEntriesCallbacks.erase(opId.value()); + impl_->bulkProgressCallbacks.erase(opId.value()); } void OperationQueue::showMinimizedSync(std::function onRestore) @@ -2151,8 +2174,9 @@ Nui::ElementRenderer OperationQueue::operator()() } void OperationQueue::enqueueDownload( - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem, + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t sizeBytes, std::function const&, std::string const& info)> onComplete, bool allowOverwrite, bool insertRefresh, @@ -2167,16 +2191,22 @@ void OperationQueue::enqueueDownload( return; } - Log::info( - "Frontend Operation Queue download: {} -> {}", remoteItem.path.generic_string(), localItem.path.generic_string() - ); + Log::info("Frontend Operation Queue download: {} -> {}", source.value(), destination.toUtf8()); impl_->fileEngine->addDownload( - remoteItem, localItem, std::move(onComplete), allowOverwrite, insertRefresh, createMissingDirectories, mode + source, + destination, + sizeBytes, + std::move(onComplete), + allowOverwrite, + insertRefresh, + createMissingDirectories, + mode ); } void OperationQueue::enqueueUpload( - NuiFileExplorer::Item const& remoteItem, - NuiFileExplorer::Item const& localItem, + Paths::LocalPath const& source, + Paths::RemotePath const& destination, + std::uint64_t sizeBytes, std::function const&, std::string const& info)> onComplete, bool allowOverwrite, bool insertRefresh, @@ -2191,16 +2221,21 @@ void OperationQueue::enqueueUpload( return; } - Log::info( - "Frontend Operation Queue upload: {} -> {}", localItem.path.generic_string(), remoteItem.path.generic_string() - ); + Log::info("Frontend Operation Queue upload: {} -> {}", source.toUtf8(), destination.value()); impl_->fileEngine->addUpload( - remoteItem, localItem, std::move(onComplete), allowOverwrite, insertRefresh, createMissingDirectories, mode + source, + destination, + sizeBytes, + std::move(onComplete), + allowOverwrite, + insertRefresh, + createMissingDirectories, + mode ); } void OperationQueue::enqueueArchiveDownload( - std::vector entries, - std::filesystem::path const& localArchivePath, + std::vector roots, + Paths::LocalPath const& localArchivePath, int compressionCodec, int compressionLevel, bool mayOverwrite, @@ -2216,14 +2251,14 @@ void OperationQueue::enqueueArchiveDownload( } Log::info( - "Frontend Operation Queue archive download: {} entries → {} (codec={}, level={})", - entries.size(), - localArchivePath.generic_string(), + "Frontend Operation Queue archive download: {} roots → {} (codec={}, level={})", + roots.size(), + localArchivePath.toUtf8(), compressionCodec, compressionLevel ); impl_->fileEngine->addArchiveDownload( - std::move(entries), + std::move(roots), localArchivePath, compressionCodec, compressionLevel, @@ -2238,8 +2273,8 @@ void OperationQueue::enqueueArchiveDownload( } void OperationQueue::enqueueArchiveUpload( - std::vector localPaths, - std::filesystem::path const& remoteArchivePath, + std::vector localPaths, + Paths::RemotePath const& remoteArchivePath, int compressionCodec, int compressionLevel, bool mayOverwrite, @@ -2257,7 +2292,7 @@ void OperationQueue::enqueueArchiveUpload( Log::info( "Frontend Operation Queue archive upload: {} paths → {} (codec={}, level={})", localPaths.size(), - remoteArchivePath.generic_string(), + remoteArchivePath.value(), compressionCodec, compressionLevel ); @@ -2277,8 +2312,8 @@ void OperationQueue::enqueueArchiveUpload( } void OperationQueue::enqueueRename( - std::filesystem::path const& oldPath, - std::filesystem::path const& newPath, + Paths::RemotePath const& oldPath, + Paths::RemotePath const& newPath, std::function const&, std::string const& info)> onComplete, SharedData::OperationMode mode ) @@ -2290,11 +2325,11 @@ void OperationQueue::enqueueRename( return; } - Log::info("Frontend Operation Queue rename: {} -> {}", oldPath.generic_string(), newPath.generic_string()); + Log::info("Frontend Operation Queue rename: {} -> {}", oldPath.value(), newPath.value()); impl_->fileEngine->addRename(oldPath, newPath, std::move(onComplete), mode); } void OperationQueue::enqueueDelete( - std::vector const& paths, + std::vector const& paths, bool recursive, std::function> const&, std::string const& info)> onComplete, SharedData::OperationMode mode @@ -2310,7 +2345,7 @@ void OperationQueue::enqueueDelete( } void OperationQueue::enqueueBulkDownload( - std::vector entries, + std::vector entries, bool allowOverwrite, bool insertRefresh, SharedData::OperationMode mode, @@ -2366,7 +2401,7 @@ void OperationQueue::enqueueBulkDownload( } void OperationQueue::enqueueBulkUpload( - std::vector entries, + std::vector entries, bool allowOverwrite, bool insertRefresh, SharedData::OperationMode mode, @@ -2419,112 +2454,90 @@ void OperationQueue::enqueueResumable(ResumableOp const& op) { if (!impl_->fileEngine) { - Log::error( - "OperationQueue::enqueueResumable: no file engine, cannot resume kind={} src={}", - static_cast(op.kind), - op.src.generic_string() - ); + Log::error("OperationQueue::enqueueResumable: no file engine, cannot resume an operation"); return; } - Log::info( - "OperationQueue::enqueueResumable: dispatching kind={} src={} dst={}", - static_cast(op.kind), - op.src.generic_string(), - op.dst.generic_string() - ); - - auto reportSingle = [kind = op.kind, - src = op.src, - dst = op.dst](std::optional const& opId, std::string const& info) + auto report = [](std::string description) { - if (opId) - { - Log::info( - "Resumed operation kind={} src={} dst={} as id {}", - static_cast(kind), - src.generic_string(), - dst.generic_string(), - opId->value() - ); - } - else + return + [description = std::move(description)](std::optional const& opId, std::string const& info) { - Log::error( - "Failed to re-enqueue operation kind={} src={} dst={}: {}", - static_cast(kind), - src.generic_string(), - dst.generic_string(), - info - ); - } + if (opId) + Log::info("Resumed {} as id {}", description, opId->value()); + else + Log::error("Failed to resume {}: {}", description, info); + }; }; - // Build a minimal Item from a path. The size hint biases the backend - // toward the streaming/big-file code path so a partial transfer can - // resume; the small-file path doesn't honour tryContinue. - auto makeItem = [](std::filesystem::path const& path) - { - SharedData::DirectoryEntry entry{}; - entry.path = path; - entry.type = SharedData::FileType::Regular; - entry.size = static_cast(Constants::bigFileCutOff) + 1; - return NuiFileExplorer::Item{entry}; - }; + // Resumes take the big-file path, the small-file path does not honour tryContinue. + constexpr auto resumeSize = static_cast(Constants::bigFileCutOff) + 1; - switch (op.kind) - { - case ResumableOp::Kind::Download: - enqueueDownload( - makeItem(op.src), - makeItem(op.dst), - reportSingle, - op.allowOverwrite, - /*insertRefresh=*/true, - op.createMissingDirectories, - SharedData::OperationMode::Queued - ); - break; - case ResumableOp::Kind::Upload: - enqueueUpload( - makeItem(op.dst), - makeItem(op.src), - reportSingle, - op.allowOverwrite, - /*insertRefresh=*/true, - op.createMissingDirectories, - SharedData::OperationMode::Queued - ); - break; - case ResumableOp::Kind::Rename: - enqueueRename(op.src, op.dst, reportSingle, SharedData::OperationMode::Queued); - break; - case ResumableOp::Kind::Delete: - enqueueDelete( - {op.src}, - op.recursive, - [src = op.src](std::optional> const& ids, std::string const& info) - { - if (ids) - Log::info("Resumed delete of {} ({} op(s) created)", src.generic_string(), ids->size()); - else - Log::error("Failed to re-enqueue delete of {}: {}", src.generic_string(), info); - }, - SharedData::OperationMode::Queued - ); - break; - case ResumableOp::Kind::BulkDownload: - case ResumableOp::Kind::BulkUpload: - case ResumableOp::Kind::BulkDelete: - // Bulk kinds are adopted via SessionManager::adoptBulkResume — - // see Session::applySnapshot. This helper handles only the - // single-file kinds. - break; - } + std::visit( + Utility::overloaded{ + [&](Resumable::Download const& download) + { + enqueueDownload( + download.source, + download.destination, + resumeSize, + report(fmt::format("download {} -> {}", download.source.value(), download.destination.toUtf8())), + download.allowOverwrite, + /*insertRefresh=*/true, + /*createMissingDirectories=*/false, + SharedData::OperationMode::Queued + ); + }, + [&](Resumable::Upload const& upload) + { + enqueueUpload( + upload.source, + upload.destination, + resumeSize, + report(fmt::format("upload {} -> {}", upload.source.toUtf8(), upload.destination.value())), + upload.allowOverwrite, + /*insertRefresh=*/true, + /*createMissingDirectories=*/false, + SharedData::OperationMode::Queued + ); + }, + [&](Resumable::Rename const& rename) + { + enqueueRename( + rename.source, + rename.destination, + report(fmt::format("rename {} -> {}", rename.source.value(), rename.destination.value())), + SharedData::OperationMode::Queued + ); + }, + [&](Resumable::Delete const& remove) + { + enqueueDelete( + {remove.target}, + remove.recursive, + [target = remove.target]( + std::optional> const& ids, std::string const& info + ) + { + if (ids) + Log::info("Resumed delete of {} ({} op(s) created)", target.value(), ids->size()); + else + Log::error("Failed to re-enqueue delete of {}: {}", target.value(), info); + }, + SharedData::OperationMode::Queued + ); + }, + [](Resumable::Bulk const&) + { + // Bulks are adopted through SessionManager::adoptBulkResume, see SessionSnapshotManager::apply. + }, + }, + op + ); } void OperationQueue::enqueueBulkDelete( - std::vector entries, + std::vector entries, bool insertRefresh, SharedData::OperationMode mode, std::function onBulkComplete, diff --git a/frontend/source/frontend/session_components/session_snapshot_manager.cpp b/frontend/source/frontend/session_components/session_snapshot_manager.cpp index 9b60c019..9d1462fd 100644 --- a/frontend/source/frontend/session_components/session_snapshot_manager.cpp +++ b/frontend/source/frontend/session_components/session_snapshot_manager.cpp @@ -256,24 +256,16 @@ void SessionSnapshotManager::apply(SessionSnapshot const& snapshot) std::vector bulkResumeIds; for (auto const& op : snapshot.inFlightOps) { - switch (op.kind) + if (auto const* bulk = std::get_if(&op); bulk) { - case ResumableOp::Kind::Download: - case ResumableOp::Kind::Upload: - case ResumableOp::Kind::Delete: - case ResumableOp::Kind::Rename: - impl_->operationQueue->enqueueResumable(op); - break; - case ResumableOp::Kind::BulkDownload: - case ResumableOp::Kind::BulkUpload: - case ResumableOp::Kind::BulkDelete: - if (op.operationId.isValid()) - { - bulkResumeIds.push_back(op.operationId.value()); - impl_->trackedBulkResumes.push_back(op.operationId); - } - break; + if (bulk->operationId.isValid()) + { + bulkResumeIds.push_back(bulk->operationId.value()); + impl_->trackedBulkResumes.push_back(bulk->operationId); + } + continue; } + impl_->operationQueue->enqueueResumable(op); } if (!bulkResumeIds.empty() && impl_->frontendSessionManager->value() && impl_->frontendSessionManager->value()->engine().engineName() == "ssh") diff --git a/frontend/source/frontend/session_components/terminal_panel.cpp b/frontend/source/frontend/session_components/terminal_panel.cpp index e2d85f98..25c764f8 100644 --- a/frontend/source/frontend/session_components/terminal_panel.cpp +++ b/frontend/source/frontend/session_components/terminal_panel.cpp @@ -142,21 +142,53 @@ namespace * @brief Writes contents[i] to "{stem}_{i}{ext}" alongside @p file for each * entry. Used by the locked-mode S-key save-all path. */ + /** + * @brief Writes each channel's content to {stem}_{index}{extension} next to @p file, skipping names that exist. + */ void saveTerminalContents( std::filesystem::path const& file, std::vector const& contents ) { - int indexIncrement = 0; - const auto basePath = file.parent_path(); - const auto extension = file.extension().string(); - const auto baseName = file.stem().string(); - for (auto const& textContent : contents) + struct SaveState { + std::filesystem::path directory; + std::string stem; + std::string extension; + std::vector contents; + std::size_t next{0}; + int index{0}; + /// Bounds the search for a free name. + int attemptsLeft{1000}; + }; + auto state = std::make_shared(SaveState{ + .directory = file.parent_path(), + .stem = file.stem().string(), + .extension = file.extension().string(), + .contents = contents, + }); + // Each pending reply owns the chain, it ends with the last one. + auto writeNext = std::make_shared>(); + *writeNext = [state, weakWriteNext = std::weak_ptr{writeNext}]() + { + auto self = weakWriteNext.lock(); + if (!self || state->next >= state->contents.size() || state->attemptsLeft-- <= 0) + return; const auto filePath = - basePath / fmt::format("{}_{}{}", baseName, indexIncrement++, extension); - SessionInternal::writeChannelContentToFile(filePath, textContent); - } + state->directory / fmt::format("{}_{}{}", state->stem, state->index++, state->extension); + SessionInternal::writeChannelContentToNewFile( + filePath, + state->contents[state->next], + [state, self](SessionInternal::WriteOutcome outcome) + { + // A taken name is skipped, the same content goes to the next one. + if (outcome != SessionInternal::WriteOutcome::Exists) + ++state->next; + (*self)(); + } + ); + }; + (*writeNext)(); } } diff --git a/frontend/source/frontend/settings/general_settings.cpp b/frontend/source/frontend/settings/general_settings.cpp index 962beeec..eda05310 100644 --- a/frontend/source/frontend/settings/general_settings.cpp +++ b/frontend/source/frontend/settings/general_settings.cpp @@ -239,6 +239,29 @@ GeneralSettings::GeneralSettings( onChange(); }, }, + .openRiskyFiles = ComboSetting{ + {Persistence::RiskyFilePolicy::Warn, Persistence::RiskyFilePolicy::Never, Persistence::RiskyFilePolicy::Allow}, + localFilesystemFactory_.identity({"general", "localFilesystemOptions", "openRiskyFiles"}), + onChange, + [this, onChange]() + { + localFilesystemOptions.openRiskyFiles.value(Persistence::RiskyFilePolicy::Warn); + onChange(); + }, + [](Persistence::RiskyFilePolicy policy) -> std::string + { + switch (policy) + { + case Persistence::RiskyFilePolicy::Allow: + return language->get("settings", "general", "localFilesystemOptions", "openRiskyFilesAllow"); + case Persistence::RiskyFilePolicy::Warn: + return language->get("settings", "general", "localFilesystemOptions", "openRiskyFilesWarn"); + case Persistence::RiskyFilePolicy::Never: + return language->get("settings", "general", "localFilesystemOptions", "openRiskyFilesNever"); + } + return "???"; + }, + }, } , fileTrackingOptions{ .autoReupload = BoolSetting<>{ @@ -270,6 +293,15 @@ GeneralSettings::GeneralSettings( onChange(); }, }, + .checkRemoteChanges = BoolSetting<>{ + fileTrackingFactory_.identity({"general", "fileTrackingOptions", "checkRemoteChanges"}), + onChange, + [this, onChange]() + { + fileTrackingOptions.checkRemoteChanges.value(true); + onChange(); + }, + }, } , logOptions(factory.within({"loggingAndErrorReportingGroupHeader"}), onChange) , availableThemesListener{ @@ -331,6 +363,7 @@ void GeneralSettings::applyToState(Persistence::State& state) const state.fileTrackingOptions.autoReupload = fileTrackingOptions.autoReupload.value(); state.fileTrackingOptions.moveRemoteOnLocalMove = fileTrackingOptions.moveRemoteOnLocalMove.value(); state.fileTrackingOptions.deleteRemoteOnLocalDelete = fileTrackingOptions.deleteRemoteOnLocalDelete.value(); + state.fileTrackingOptions.checkRemoteChanges = fileTrackingOptions.checkRemoteChanges.value(); // Local FS state.localFilesystemOptions.preventDeletion = localFilesystemOptions.preventDeletion.value(); @@ -340,6 +373,7 @@ void GeneralSettings::applyToState(Persistence::State& state) const state.localFilesystemOptions.homeOverride = localFilesystemOptions.homeOverride.value(); state.localFilesystemOptions.temporaryDownloadsDirectory = localFilesystemOptions.temporaryDownloadsDirectory.value(); + state.localFilesystemOptions.openRiskyFiles = localFilesystemOptions.openRiskyFiles.value(); logOptions.applyToState(state.logOptions); } @@ -364,6 +398,7 @@ void GeneralSettings::loadFromState(Persistence::State const& state) fileTrackingOptions.autoReupload.value(state.fileTrackingOptions.autoReupload); fileTrackingOptions.moveRemoteOnLocalMove.value(state.fileTrackingOptions.moveRemoteOnLocalMove); fileTrackingOptions.deleteRemoteOnLocalDelete.value(state.fileTrackingOptions.deleteRemoteOnLocalDelete); + fileTrackingOptions.checkRemoteChanges.value(state.fileTrackingOptions.checkRemoteChanges.value_or(true)); // Local FS localFilesystemOptions.preventDeletion.value(state.localFilesystemOptions.preventDeletion); @@ -371,6 +406,9 @@ void GeneralSettings::loadFromState(Persistence::State const& state) localFilesystemOptions.preventCreateFile.value(state.localFilesystemOptions.preventCreateFile); localFilesystemOptions.preventCreateDirectory.value(state.localFilesystemOptions.preventCreateDirectory); localFilesystemOptions.homeOverride.value(state.localFilesystemOptions.homeOverride); + localFilesystemOptions.openRiskyFiles.value( + state.localFilesystemOptions.openRiskyFiles.value_or(Persistence::RiskyFilePolicy::Warn) + ); localFilesystemOptions.temporaryDownloadsDirectory.value( state.localFilesystemOptions.temporaryDownloadsDirectory.value_or("%temp%/nui-sftp-downloads") ); @@ -429,13 +467,15 @@ Nui::ElementRenderer GeneralSettings::render( localFilesystemOptions.preventCreateFile(), localFilesystemOptions.preventCreateDirectory(), localFilesystemOptions.homeOverride(), - localFilesystemOptions.temporaryDownloadsDirectory() + localFilesystemOptions.temporaryDownloadsDirectory(), + localFilesystemOptions.openRiskyFiles() ); auto fileTrackingOptionsUi = fragment( fileTrackingOptions.autoReupload(), fileTrackingOptions.moveRemoteOnLocalMove(), - fileTrackingOptions.deleteRemoteOnLocalDelete() + fileTrackingOptions.deleteRemoteOnLocalDelete(), + fileTrackingOptions.checkRemoteChanges() ); // clang-format on diff --git a/frontend/source/frontend/settings/sftp_options.cpp b/frontend/source/frontend/settings/sftp_options.cpp index a9a85393..570df2ee 100644 --- a/frontend/source/frontend/settings/sftp_options.cpp +++ b/frontend/source/frontend/settings/sftp_options.cpp @@ -63,6 +63,11 @@ SftpOptions::SftpOptions(SettingFactory const& factory, std::function co onChange, nulloptReset(downloadOptions.doCleanup, onChange), }, + .markAsDownloaded{ + downloadFactory_.identity({"sftpOptions", "downloadOptions", "markAsDownloaded"}), + onChange, + nulloptReset(downloadOptions.markAsDownloaded, onChange), + }, .symlinkHandling{ std::vector{ Persistence::SymlinkHandling::AsSymlink, @@ -208,7 +213,8 @@ void SftpOptions::applyToState(Persistence::SftpOptions& state) const .failFast = downloadOptions.failFast.value(), }, .reserveSpace = downloadOptions.reserveSpace.value(), - .doCleanup = downloadOptions.doCleanup.value() + .doCleanup = downloadOptions.doCleanup.value(), + .markAsDownloaded = downloadOptions.markAsDownloaded.value() }; #pragma clang diagnostic pop } @@ -260,6 +266,7 @@ void SftpOptions::loadFromState(Persistence::SftpOptions const& state, bool) ); downloadOptions.reserveSpace.value(state.downloadOptions->reserveSpace); downloadOptions.doCleanup.value(state.downloadOptions->doCleanup); + downloadOptions.markAsDownloaded.value(state.downloadOptions->markAsDownloaded); downloadOptions.symlinkHandling.value(state.downloadOptions->symlinkHandling); downloadOptions.failFast.value(state.downloadOptions->failFast); } @@ -275,6 +282,7 @@ void SftpOptions::loadFromState(Persistence::SftpOptions const& state, bool) downloadOptions.customDirectoryPermissions.value(std::nullopt); downloadOptions.reserveSpace.value(std::nullopt); downloadOptions.doCleanup.value(std::nullopt); + downloadOptions.markAsDownloaded.value(std::nullopt); downloadOptions.symlinkHandling.value(std::nullopt); downloadOptions.failFast.value(std::nullopt); } @@ -334,6 +342,7 @@ void SftpOptions::assumeDefaultsFrom(Persistence::SftpOptions const& state) ); downloadOptions.reserveSpace.inherit(state.downloadOptions->reserveSpace); downloadOptions.doCleanup.inherit(state.downloadOptions->doCleanup); + downloadOptions.markAsDownloaded.inherit(state.downloadOptions->markAsDownloaded); downloadOptions.symlinkHandling.inherit(state.downloadOptions->symlinkHandling); downloadOptions.failFast.inherit(state.downloadOptions->failFast); } @@ -347,6 +356,7 @@ void SftpOptions::assumeDefaultsFrom(Persistence::SftpOptions const& state) downloadOptions.customDirectoryPermissions.inherit(std::nullopt); downloadOptions.reserveSpace.inherit(std::nullopt); downloadOptions.doCleanup.inherit(std::nullopt); + downloadOptions.markAsDownloaded.inherit(std::nullopt); downloadOptions.symlinkHandling.inherit(std::nullopt); downloadOptions.failFast.inherit(std::nullopt); } @@ -405,6 +415,7 @@ Nui::ElementRenderer SftpOptions::render() downloadOptions.customDirectoryPermissions(), downloadOptions.reserveSpace(), downloadOptions.doCleanup(), + downloadOptions.markAsDownloaded(), downloadOptions.symlinkHandling(), downloadOptions.failFast() ) diff --git a/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp index c8c5e9d7..e8048136 100644 --- a/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp +++ b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp @@ -24,6 +24,7 @@ void BackendSyncProvider::open( std::function onLocalListing, std::function onRemoteListing, std::function onBothListed, + std::function onScanFailed, std::function onDiffProgress ) { @@ -40,9 +41,11 @@ void BackendSyncProvider::open( bool local{false}; bool remote{false}; std::function onBothListed; + std::function onScanFailed; }; auto gate = std::make_shared(); gate->onBothListed = std::move(onBothListed); + gate->onScanFailed = std::move(onScanFailed); queue_->openSyncSession( sessionId_, @@ -63,9 +66,17 @@ void BackendSyncProvider::open( { auto cb = std::move(gate->onBothListed); gate->onBothListed = nullptr; + gate->onScanFailed = nullptr; cb(); } }, + [gate](std::string const& reason) + { + // The first failure ends the listing, the other side's result is of no use any more. + gate->onBothListed = nullptr; + if (auto callback = std::exchange(gate->onScanFailed, nullptr); callback) + callback(reason); + }, std::move(onDiffProgress) ); } @@ -128,30 +139,24 @@ void BackendSyncProvider::loadChildren( ); } -void BackendSyncProvider::buildEnqueuePlan( - SharedData::Sync::DiffSection section, - std::vector selectedRelKeys, - std::function)> onResolved, +void BackendSyncProvider::executeSelection( + SharedData::Sync::SyncExecutionRequest request, + std::function onResolved, std::function onRejected ) { if (!queue_ || closed_) return; - const auto expectedGeneration = generation_; - queue_->buildSyncEnqueuePlan( - sessionId_, - section, - std::move(selectedRelKeys), - expectedGeneration, - [alive = std::weak_ptr{alive_}, - expectedGeneration, - onResolved = std::move(onResolved)](std::vector plan) mutable + request.syncSessionId = sessionId_; + request.generation = generation_; + queue_->executeSyncSelection( + std::move(request), + [alive = std::weak_ptr{alive_}, onResolved = std::move(onResolved)](SharedData::Sync::SyncExecutionResult result) { - const auto self = alive.lock(); - if (!self || expectedGeneration != (*self)->generation_) - return; - onResolved(std::move(plan)); + // The operations are queued either way, the reply only maps them to rows of a dialog that may be gone. + if (alive.lock()) + onResolved(std::move(result)); }, std::move(onRejected) ); diff --git a/frontend/source/frontend/sync_dialog/sync_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_dialog.cpp index 6f3519f5..5ea8c275 100644 --- a/frontend/source/frontend/sync_dialog/sync_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_dialog.cpp @@ -31,12 +31,13 @@ #include #include #include +#include #include #include -#include #include +#include #include #include #include @@ -207,7 +208,8 @@ struct SyncDialog::Implementation std::filesystem::path localPath_{}; std::filesystem::path remotePath_{}; - BackendSyncProvider* provider_{nullptr}; + /** @brief Shared with the session, so a minimized dialog keeps its provider when a new flow starts. */ + std::shared_ptr provider_{}; // Settings Nui::Observed directionStr_{"Both"s}; @@ -277,6 +279,9 @@ struct SyncDialog::Implementation ConfirmDialog* confirmDialog_; OperationQueue* operationQueue_; std::function onRecompareRequested_{}; + std::function onSynchronized_{}; + /** @brief When the shown pair was last synchronized, part of every diff. */ + std::optional lastSyncTime_{}; explicit Implementation(ConfirmDialog* confirmDialog, OperationQueue* operationQueue) : confirmDialog_{confirmDialog} @@ -773,6 +778,7 @@ struct SyncDialog::Implementation .actionDelete = actionDelete_.value(), .recursive = recursive_.value(), .ignoreHidden = ignoreHidden_.value(), + .lastSyncTime = lastSyncTime_, }; } @@ -831,6 +837,23 @@ struct SyncDialog::Implementation ); } + /** @brief Selected items and bytes of one section, from the rows the selection names. */ + std::pair selectedTotalsOf(DiffSection section) const + { + std::uint64_t items = 0; + std::uint64_t bytes = 0; + auto const& rows = stateFor(section).rows; + for (auto const& key : selectionFor(section)->value()) + { + const auto iter = rows.find(key); + if (iter == rows.end()) + continue; + items += SharedData::Sync::itemWeight(iter->second); + bytes += iter->second.descendantByteTotal; + } + return {items, bytes}; + } + /** @brief Selected items and bytes over all sections, from the rows the selection names. */ std::pair selectedTotals() const { @@ -838,19 +861,100 @@ struct SyncDialog::Implementation std::uint64_t bytes = 0; for (const auto section : {DiffSection::Upload, DiffSection::Download, DiffSection::Delete}) { - auto const& rows = stateFor(section).rows; - for (auto const& key : selectionFor(section)->value()) - { - const auto iter = rows.find(key); - if (iter == rows.end()) - continue; - items += SharedData::Sync::itemWeight(iter->second); - bytes += iter->second.descendantByteTotal; - } + const auto [sectionItems, sectionBytes] = selectedTotalsOf(section); + items += sectionItems; + bytes += sectionBytes; } return {items, bytes}; } + /** @brief "on the server" or "locally", for the side the delete section removes from. */ + std::string deleteSideText() const + { + return language->get("syncDialog", direction_ == SyncDirection::Upload ? "deleteSideRemote" : "deleteSideLocal"); + } + + /** @brief Text of the confirmation before synchronizing: names the deletes and their side. */ + std::string synchronizeConfirmText() const + { + auto text = fmt::format( + fmt::runtime(language->get("syncDialog", "confirmText")), + localPath_.filename().string(), + remotePath_.filename().string() + ); + const auto [deleteItems, deleteBytes] = selectedTotalsOf(DiffSection::Delete); + if (deleteItems == 0) + return text; + return fmt::format( + "{} {}", + text, + fmt::format( + fmt::runtime(language->get("syncDialog", "confirmDeletes")), + deleteItems, + Utility::formatBytes(static_cast(deleteBytes)), + deleteSideText() + ) + ); + } + + /** @brief Runs the row arrow, a delete only after the user confirmed it. */ + void enqueueSingleNodeConfirmed(DiffSection section, std::string const& relKey) + { + if (section != DiffSection::Delete) + return enqueueSingleNode(section, relKey); + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = language->get("syncDialog", "confirmDeleteRowHeader"), + .text = fmt::format(fmt::runtime(language->get("syncDialog", "confirmDeleteRow")), relKey, deleteSideText()), + .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, + .focusButton = ConfirmDialog::Button::No, + .onClose = + [this, relKey](std::optional button) + { + if (button == ConfirmDialog::Button::Yes) + enqueueSingleNode(DiffSection::Delete, relKey); + }, + }); + } + + /** @brief Names the entries sync leaves alone, one per line. */ + std::string blockedSamplesText() const + { + auto const& summary = summary_.value(); + std::string text; + for (auto const& entry : summary.blockedSamples) + { + if (!text.empty()) + text.push_back('\n'); + text += fmt::format( + "{} ({})", + entry.relKey, + language->get("syncDialog", "blockReason", Utility::enumToString(entry.reason).c_str()) + ); + } + if (summary.blockedCount > summary.blockedSamples.size()) + { + text += '\n'; + text += fmt::format( + fmt::runtime(language->get("confirmDialog", "andMore")), + summary.blockedCount - summary.blockedSamples.size() + ); + } + return text; + } + + /** @brief The entries sync leaves alone in a dialog, its text scrolls however many there are. */ + void showBlockedEntries() + { + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Warning, + .headerText = language->get("syncDialog", "blockedTitle"), + .text = blockedSamplesText(), + .buttons = ConfirmDialog::Button::Ok, + .focusButton = ConfirmDialog::Button::Ok, + }); + } + /** @brief The arrow of a row: plans just that row (a directory with its content) and runs it. */ void enqueueSingleNode(DiffSection section, std::string const& relKey) { @@ -869,293 +973,210 @@ struct SyncDialog::Implementation } } - /** @brief Asks the backend to turn sparse selection keys into transfers, then dispatches them. */ - void enqueueSelection(DiffSection section, std::vector relKeys, SharedData::OperationMode mode) - { - if (!provider_) - return; - provider_->buildEnqueuePlan( - section, - std::move(relKeys), - [this, section, mode](std::vector plan) - { - dispatchPlan(section, std::move(plan), mode); - Nui::globalEventContext.executeActiveEventsImmediately(); - }, - [](std::string const& msg) - { - Log::error("buildSyncEnqueuePlan failed: {}", msg); - } - ); - } - - /** @brief Files go into one bulk transfer per direction, directories in the plan carry no content - * and are only created, remote deletes become one bulk delete, local deletes one local - * remove call. + /** + * @brief Progress of one operation of a sync execution. Its rows are only known once the backend answers, and + * the operation can finish before that, so an early completion or early failures wait here. */ - void dispatchPlan( - DiffSection section, - std::vector plan, - SharedData::OperationMode mode - ) + struct SyncPart { - if (plan.empty()) - return; - - const auto finishWith = [](std::vector observers) + /** @brief One observer per entry index; a directory delete lists its content as entries of their own. */ + std::vector observers{}; + std::unordered_map indexBySource{}; + std::set seenInFlight{}; + bool mapped{false}; + std::optional completion{}; + std::vector failedSources{}; + + void markFailedSources() { - return [observers = std::move(observers)](bool success, std::string const& info) + for (auto const& source : failedSources) { - if (!success) - Log::error("Sync step failed: {}", info); - for (auto const& observer : observers) - *observer = success ? 1.1 : -1.0; - Nui::globalEventContext.executeActiveEventsImmediately(); - }; - }; - const auto appendCovering = [this, section](std::vector& target, std::string const& relKey) - { - auto covering = progressCovering(section, relKey); - target.insert(target.end(), covering.begin(), covering.end()); - }; - - std::vector uploadEntries; - std::vector uploadObservers; - std::vector downloadEntries; - std::vector downloadObservers; - std::vector remoteDeletes; - std::vector remoteDeleteObservers; - std::vector localDeletes; - std::vector localDeleteObservers; + if (const auto iter = indexBySource.find(source); iter != indexBySource.end()) + *observers[iter->second] = -1.0; + } + failedSources.clear(); + } - for (auto& entry : plan) + void complete() { - auto progress = progressFor(section, entry.relKey); - switch (entry.action) + for (auto const& observer : observers) { - case SharedData::Sync::Action::Upload: - { - if (entry.isDirectory) - { - operationQueue_->createRemoteDirectory( - entry.remoteAbsPath, finishWith(progressCovering(section, entry.relKey)) - ); - break; - } - uploadEntries.push_back( - SharedData::BulkAddEntry{ - .src = entry.localAbsPath, - .dst = entry.remoteAbsPath, - .sizeBytes = entry.sizeBytes, - .isDirectory = false, - .mtime = entry.mtime, - .mtimeNsec = entry.mtimeNsec, - } - ); - uploadObservers.push_back(progress); - break; - } - case SharedData::Sync::Action::Download: - { - if (entry.isDirectory) - { - operationQueue_->createLocalDirectory( - entry.localAbsPath, finishWith(progressCovering(section, entry.relKey)) - ); - break; - } - downloadEntries.push_back( - SharedData::BulkAddEntry{ - .src = entry.remoteAbsPath, - .dst = entry.localAbsPath, - .sizeBytes = entry.sizeBytes, - .isDirectory = false, - .mtime = entry.mtime, - .mtimeNsec = entry.mtimeNsec, - } - ); - downloadObservers.push_back(progress); - break; - } - case SharedData::Sync::Action::DeleteLocal: - { - localDeletes.push_back(entry.localAbsPath); - appendCovering(localDeleteObservers, entry.relKey); - break; - } - case SharedData::Sync::Action::DeleteRemote: - { - remoteDeletes.push_back( - SharedData::BulkAddEntry{ - .src = entry.remoteAbsPath, - .dst = {}, - .sizeBytes = 0, - .isDirectory = entry.isDirectory, - } - ); - appendCovering(remoteDeleteObservers, entry.relKey); - break; - } + // Rows already marked failed stay failed on an otherwise successful operation. + if (!(*completion && observer->value() < 0.0)) + *observer = *completion ? 1.1 : -1.0; } } - if (!uploadEntries.empty()) - hookBulkTransfer(std::move(uploadEntries), std::move(uploadObservers), /*isUpload=*/true, mode); - if (!downloadEntries.empty()) - hookBulkTransfer(std::move(downloadEntries), std::move(downloadObservers), /*isUpload=*/false, mode); - - if (!localDeletes.empty()) + /** @brief The backend answered with the rows of this part. */ + void map() { - operationQueue_->removeLocalPaths( - localDeletes, - [observers = std::move(localDeleteObservers)](bool success, std::string const& info) - { - if (!success) - Log::error("Sync local delete failed: {}", info); - for (auto const& observer : observers) - *observer = success ? 1.1 : -1.0; - Nui::globalEventContext.executeActiveEventsImmediately(); - } - ); + mapped = true; + markFailedSources(); + if (completion) + complete(); } + }; - if (!remoteDeletes.empty()) - { - operationQueue_->enqueueBulkDelete( - std::move(remoteDeletes), - /*insertRefresh=*/false, - mode, - [observers = std::move(remoteDeleteObservers)](bool success) + void watchCompletion(Ids::OperationId const& operationId, std::shared_ptr part) + { + operationQueue_->addCompletionCallback( + operationId, + [part](bool success) + { + part->completion = success; + if (part->mapped) + part->complete(); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ); + } + + /** @brief Files can finish in any order in a bulk, so rows seen in flight are done once they drop out. */ + void watchBulkPart(Ids::OperationId const& operationId, std::shared_ptr part) + { + operationQueue_->addBulkProgressCallback( + operationId, + [part](SharedData::BulkProgress const& progress) + { + if (!part->mapped) + return; + auto& observers = part->observers; + for (const auto entryIndex : progress.failedEntryIndices) { - for (auto const& observer : observers) - *observer = success ? 1.1 : -1.0; - Nui::globalEventContext.executeActiveEventsImmediately(); - }, - [](bool success, std::string const& info) + if (entryIndex < observers.size() && observers[entryIndex]->value() >= 0.0) + *observers[entryIndex] = -1.0; + } + std::set stillInFlight; + for (auto const& file : progress.inFlight) { - if (!success) - Log::error("Sync bulk delete failed: {}", info); + if (file.entryIndex >= observers.size()) + continue; + stillInFlight.insert(file.entryIndex); + part->seenInFlight.insert(file.entryIndex); + if (file.totalBytes > 0) + *observers[file.entryIndex] = + static_cast(file.bytes) / static_cast(file.totalBytes); } - ); - } + std::erase_if( + part->seenInFlight, + [&](std::uint64_t entryIndex) + { + if (stillInFlight.contains(entryIndex)) + return false; + auto& observer = observers[entryIndex]; + if (observer->value() >= 0.0 && observer->value() < 1.0) + *observer = 1.1; + return true; + } + ); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ); + watchFailedEntries(operationId, std::move(part)); } - /** @brief Shared wiring between BulkUpload and BulkDownload of files. Tracks per-file progress via - * the backend's BulkProgress event stream and marks completed rows green. - */ - void hookBulkTransfer( - std::vector entries, - std::vector observers, - bool isUpload, - SharedData::OperationMode mode - ) + /** @brief Failed entries arrive by their source path and mark the rows they belong to; then completion. */ + void watchFailedEntries(Ids::OperationId const& operationId, std::shared_ptr part) { - auto observersShared = std::make_shared>(std::move(observers)); - auto indexBySource = std::make_shared>(); - for (std::size_t index = 0; index < entries.size(); ++index) - indexBySource->emplace(entries[index].src.generic_string(), index); - - // Files can finish in any order now that bulks transfer several at once, so track - // which entries we have seen in flight and mark them done once they drop out. Entries - // the backend gave up on are marked failed first so they never count as done. - auto seenInFlight = std::make_shared>(); - auto onBulkProgress = [observersShared, seenInFlight](SharedData::BulkProgress const& prog) - { - auto& observersRef = *observersShared; - for (const auto entryIndex : prog.failedEntryIndices) - { - if (entryIndex < observersRef.size() && observersRef[entryIndex]->value() >= 0.0) - *observersRef[entryIndex] = -1.0; - } - std::set stillInFlight; - for (auto const& file : prog.inFlight) + operationQueue_->addFailedEntriesCallback( + operationId, + [part](std::vector const& failedPaths) { - if (file.entryIndex >= observersRef.size()) - continue; - stillInFlight.insert(file.entryIndex); - seenInFlight->insert(file.entryIndex); - if (file.totalBytes > 0) - *observersRef[file.entryIndex] = - static_cast(file.bytes) / static_cast(file.totalBytes); + for (auto const& path : failedPaths) + part->failedSources.push_back(path); + if (part->mapped) + part->markFailedSources(); } - std::erase_if( - *seenInFlight, - [&](std::uint64_t entryIndex) - { - if (stillInFlight.contains(entryIndex)) - return false; - auto& observer = observersRef[entryIndex]; - if (observer->value() >= 0.0 && observer->value() < 1.0) - *observer = 1.1; - return true; - } - ); - Nui::globalEventContext.executeActiveEventsImmediately(); - }; + ); + watchCompletion(operationId, std::move(part)); + } - auto onEnqueued = [this, onBulkProgress = std::move(onBulkProgress), observersShared, indexBySource]( - std::vector const& opIds - ) - { - if (opIds.empty()) - return; - operationQueue_->addBulkProgressCallback(opIds.back(), onBulkProgress); - operationQueue_->addFailedEntriesCallback( - opIds.back(), - [observersShared, indexBySource](std::vector const& failedPaths) + /** + * @brief Has the backend plan and queue the sparse selection @p relKeys. The operation ids are chosen here and + * listened to before the request goes out, the answer maps them to rows. + */ + void enqueueSelection(DiffSection section, std::vector relKeys, SharedData::OperationMode mode) + { + if (!provider_) + return; + + const SharedData::Sync::SyncExecutionRequest request{ + .section = section, + .relKeys = std::move(relKeys), + .mode = mode, + .uploadId = Ids::generateOperationId(), + .downloadId = Ids::generateOperationId(), + .remoteDeleteId = Ids::generateOperationId(), + .localDeleteId = Ids::generateOperationId(), + }; + auto uploads = std::make_shared(); + auto downloads = std::make_shared(); + auto remoteDeletes = std::make_shared(); + auto localDeletes = std::make_shared(); + watchBulkPart(request.uploadId, uploads); + watchBulkPart(request.downloadId, downloads); + watchFailedEntries(request.remoteDeleteId, remoteDeletes); + watchFailedEntries(request.localDeleteId, localDeletes); + + provider_->executeSelection( + request, + [this, section, request, uploads, downloads, remoteDeletes, localDeletes]( + SharedData::Sync::SyncExecutionResult result + ) + { + const auto mapBulk = [this, section]( + SyncPart& part, + std::vector const& rows, + std::vector const& sources + ) { - for (auto const& path : failedPaths) + for (std::size_t index = 0; index < rows.size(); ++index) { - if (const auto iter = indexBySource->find(path.generic_string()); iter != indexBySource->end()) - *(*observersShared)[iter->second] = -1.0; + part.observers.push_back(progressFor(section, rows[index])); + if (index < sources.size()) + part.indexBySource.emplace(sources[index], index); } - } - ); - operationQueue_->addCompletionCallback( - opIds.back(), - [observersShared](bool success) + }; + const auto setCovering = [this, section](std::vector const& rows, double value) { - for (auto const& observer : *observersShared) + for (auto const& row : rows) { - // Rows already marked failed stay failed on an otherwise successful bulk. - if (!(success && observer->value() < 0.0)) - *observer = success ? 1.1 : -1.0; + for (auto const& observer : progressCovering(section, row)) + *observer = value; } - Nui::globalEventContext.executeActiveEventsImmediately(); + }; + + mapBulk(*uploads, result.uploads, result.uploadSources); + mapBulk(*downloads, result.downloads, result.downloadSources); + mapBulk(*remoteDeletes, result.remoteDeletes, result.remoteDeleteSources); + mapBulk(*localDeletes, result.localDeletes, result.localDeleteSources); + + // A part without rows was never queued and will not complete. + const std::array parts{ + std::pair{request.uploadId, uploads}, + std::pair{request.downloadId, downloads}, + std::pair{request.remoteDeleteId, remoteDeletes}, + std::pair{request.localDeleteId, localDeletes}, + }; + for (auto const& [operationId, part] : parts) + { + if (part->observers.empty()) + operationQueue_->removeCallbacks(operationId); + else + part->map(); } - ); - }; - - auto onBulkAck = [isUpload](bool success, std::string const& info) - { - if (!success) - Log::error("Sync bulk {} failed: {}", isUpload ? "upload" : "download", info); - }; - - if (isUpload) - { - operationQueue_->enqueueBulkUpload( - std::move(entries), - /*allowOverwrite*/ true, - /*insertRefresh*/ false, - mode, - /*onEachComplete*/ {}, - std::move(onBulkAck), - std::move(onEnqueued) - ); - } - else - { - operationQueue_->enqueueBulkDownload( - std::move(entries), - /*allowOverwrite*/ true, - /*insertRefresh*/ false, - mode, - /*onEachComplete*/ {}, - std::move(onBulkAck), - std::move(onEnqueued) - ); - } + setCovering(result.createdDirectories, 1.1); + setCovering(result.failed, -1.0); + Nui::globalEventContext.executeActiveEventsImmediately(); + }, + [this, request](std::string const& message) + { + Log::error("Sync: the backend did not run the selection: {}", message); + for (auto const& operationId : + {request.uploadId, request.downloadId, request.remoteDeleteId, request.localDeleteId}) + operationQueue_->removeCallbacks(operationId); + } + ); } }; @@ -1218,7 +1239,7 @@ ScriptNuiComponents::Tree::RowContentRenderer SyncDialog::Implementation::makeTr onClick = [this, section, relKey](Nui::val event) { event.call("stopPropagation"); - enqueueSingleNode(section, relKey); + enqueueSingleNodeConfirmed(section, relKey); }, }(std::move(arrowIcon)); @@ -1297,6 +1318,11 @@ SyncDialog::~SyncDialog() SyncDialog::SyncDialog(SyncDialog&&) = default; SyncDialog& SyncDialog::operator=(SyncDialog&&) = default; +void SyncDialog::setOnSynchronized(std::function callback) +{ + impl_->onSynchronized_ = std::move(callback); +} + void SyncDialog::setOnRecompareRequested(std::function callback) { impl_->onRecompareRequested_ = std::move(callback); @@ -1308,14 +1334,16 @@ SyncDialog::RecompareRequest SyncDialog::currentSettings() const } void SyncDialog::open( - BackendSyncProvider* provider, + std::shared_ptr provider, SharedData::Sync::DiffSnapshot snapshot, std::filesystem::path localPath, std::filesystem::path remotePath, - ScanSettings scanned + ScanSettings scanned, + std::optional lastSyncTime ) { - impl_->provider_ = provider; + impl_->provider_ = std::move(provider); + impl_->lastSyncTime_ = lastSyncTime; impl_->localPath_ = std::move(localPath); impl_->remotePath_ = std::move(remotePath); impl_->scanned_ = scanned; @@ -1681,6 +1709,24 @@ Nui::ElementRenderer SyncDialog::operator()() ) ), + // Entries sync leaves alone, listed in a dialog on click. + div{}( + observe(impl_->summary_), + [this]() -> Nui::ElementRenderer { + using Nui::Elements::div; + const auto blocked = impl_->summary_.value().blockedCount; + if (blocked == 0) + return Nui::nil(); + return div{ + class_ = "sync-blocked-notice", + onClick = [this]() { impl_->showBlockedEntries(); } + }( + Ui5Icons::alert(), + span{}(fmt::format(fmt::runtime(language->get("syncDialog", "blockedNotice")), blocked)) + ); + } + ), + // Footer div{class_ = "sync-dialog-footer"}( div{class_ = "sync-footer-actions"}( @@ -1746,18 +1792,19 @@ Nui::ElementRenderer SyncDialog::operator()() .icon = Ui5Icons::synchronize(), .attributes = { onClick = [this](Nui::val) { + const bool deletes = impl_->selectedTotalsOf(DiffSection::Delete).first > 0; impl_->confirmDialog_->open({ - .styleVariant = Snc::StyleVariant::Warning, + .styleVariant = deletes ? Snc::StyleVariant::Danger : Snc::StyleVariant::Warning, .headerText = language->get("syncDialog", "confirmHeader"), - .text = fmt::format( - fmt::runtime(language->get("syncDialog", "confirmText")), - impl_->localPath_.filename().string(), - impl_->remotePath_.filename().string() - ), + .text = impl_->synchronizeConfirmText(), .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, + .focusButton = deletes ? std::optional{ConfirmDialog::Button::No} : std::nullopt, .onClose = [this](std::optional btn) { - if (btn == ConfirmDialog::Button::Yes) - impl_->enqueueOperations(); + if (btn != ConfirmDialog::Button::Yes) + return; + impl_->enqueueOperations(); + if (impl_->onSynchronized_) + impl_->onSynchronized_(); } }); } diff --git a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp index e70292e5..1a729a61 100644 --- a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp @@ -45,6 +45,9 @@ struct SyncProgressDialog::Implementation // Comparing counter — will be populated when the diff step is added Nui::Observed compared_{0ull}; + /** @brief Why a scan failed, shown in the Failed phase. */ + std::string failureReason_{}; + // Hashing progress bar — total becomes known when comparing finishes Components::ProgressBar hashProgressBar_{Components::ProgressBar::Settings{ .height = "10px", @@ -56,7 +59,7 @@ struct SyncProgressDialog::Implementation std::filesystem::path localPath_{}; std::filesystem::path remotePath_{}; - BackendSyncProvider* provider_{nullptr}; + std::shared_ptr provider_{}; SharedData::Sync::DiffOptions initialOptions_{}; std::function onDone_{}; @@ -76,7 +79,7 @@ SyncProgressDialog::SyncProgressDialog(SyncProgressDialog&&) = default; SyncProgressDialog& SyncProgressDialog::operator=(SyncProgressDialog&&) = default; void SyncProgressDialog::open( - BackendSyncProvider* provider, + std::shared_ptr provider, std::filesystem::path localPath, std::filesystem::path remotePath, bool respectIgnoreFiles, @@ -152,6 +155,15 @@ void SyncProgressDialog::open( } ); }, + // onScanFailed + [this, token](std::string const& reason) + { + if (*token) + return; + impl_->failureReason_ = reason; + impl_->phase_ = SharedData::SyncPhase::Failed; + Nui::globalEventContext.executeActiveEventsImmediately(); + }, // onDiffProgress [this, token](std::uint64_t compared) { @@ -250,6 +262,16 @@ Nui::ElementRenderer SyncProgressDialog::operator()() ); } + if (phase == SharedData::SyncPhase::Failed) + { + return div{class_ = "sync-progress-step sync-progress-failed"}( + div{class_ = "sync-progress-step-header"}( + span{class_ = "sync-progress-step-label"}(language->getObserved("syncProgressDialog", "scanFailed")) + ), + div{class_ = "sync-progress-failure-reason"}(impl_->failureReason_) + ); + } + // Comparing phase — placeholder for future diff/hash step if (phase == SharedData::SyncPhase::Comparing) { diff --git a/frontend/source/frontend/terminal/file_engine.cpp b/frontend/source/frontend/terminal/file_engine.cpp index 835fc801..85c59623 100644 --- a/frontend/source/frontend/terminal/file_engine.cpp +++ b/frontend/source/frontend/terminal/file_engine.cpp @@ -204,7 +204,7 @@ void FileEngine::userDirectories( } void FileEngine::createDirectory( - std::filesystem::path const& path, + Paths::RemotePath const& path, std::function onComplete ) { @@ -218,7 +218,7 @@ void FileEngine::createDirectory( return; } - Log::info("Creating directory: {}", path.generic_string()); + Log::info("Creating directory: {}", path.value()); Nui::RpcClient::callWithBackChannel( fmt::format("Session::{}::sftp::createDirectory", impl_->engine->sshSessionId().value()), [onComplete = std::move(onComplete)](Nui::val val) @@ -235,14 +235,14 @@ void FileEngine::createDirectory( onComplete(true, "Success"); }, channelId.value().value(), - path.generic_string() + path.value() ); } ); } void FileEngine::createFile( - std::filesystem::path const& path, + Paths::RemotePath const& path, std::function onComplete ) { @@ -256,7 +256,7 @@ void FileEngine::createFile( return; } - Log::info("Creating file: {}", path.generic_string()); + Log::info("Creating file: {}", path.value()); Nui::RpcClient::callWithBackChannel( fmt::format("Session::{}::sftp::createFile", impl_->engine->sshSessionId().value()), [onComplete = std::move(onComplete)](Nui::val val) @@ -273,7 +273,7 @@ void FileEngine::createFile( onComplete(true, "Success"); }, channelId.value().value(), - path.generic_string() + path.value() ); } ); @@ -417,42 +417,46 @@ void FileEngine::loadSyncDiffChildren( ); } -void FileEngine::buildSyncEnqueuePlan( - Ids::SyncSessionId syncSessionId, - SharedData::Sync::DiffSection section, - std::vector selectedRelKeys, - std::uint64_t generation, - std::function)> onResolved, +void FileEngine::executeSyncSelection( + SharedData::Sync::SyncExecutionRequest request, + std::function onResolved, std::function onRejected ) { - Nui::RpcClient::callWithBackChannel( - fmt::format("Session::{}::sftp::buildSyncEnqueuePlan", impl_->engine->sshSessionId().value()), - [onResolved = std::move(onResolved), onRejected = std::move(onRejected)](Nui::val val) + lazyOpen( + [this, request = std::move(request), onResolved = std::move(onResolved), onRejected = std::move(onRejected)]( + auto const& channelId, std::string const& info + ) mutable { - if (val.hasOwnProperty("error")) + if (!channelId) { - onRejected(val["error"].as()); + onRejected(info); return; } - try - { - const auto json = nlohmann::json::parse(Nui::JSON::stringify(val)); - std::vector plan; - plan.reserve(json.size()); - for (auto const& elem : json) - plan.push_back(elem.get()); - onResolved(std::move(plan)); - } - catch (std::exception const& exc) - { - onRejected(exc.what()); - } - }, - syncSessionId.value(), - section, - selectedRelKeys, - std::to_string(generation) + Nui::RpcClient::callWithBackChannel( + fmt::format("Session::{}::sftp::executeSyncSelection", impl_->engine->sshSessionId().value()), + [onResolved = std::move(onResolved), onRejected = std::move(onRejected)](Nui::val val) + { + if (val.hasOwnProperty("error")) + { + onRejected(val["error"].as()); + return; + } + try + { + onResolved( + nlohmann::json::parse(Nui::JSON::stringify(val)).get() + ); + } + catch (std::exception const& exception) + { + onRejected(exception.what()); + } + }, + channelId.value().value(), + request + ); + } ); } @@ -475,8 +479,9 @@ void FileEngine::closeSyncSession(Ids::SyncSessionId syncSessionId) } void FileEngine::addDownload( - NuiFileExplorer::Item const& remotePath, - NuiFileExplorer::Item const& localPath, + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t sizeBytes, std::function, std::string const& info)> onOperationCreated, bool allowOverwrite, bool insertRefresh, @@ -484,13 +489,12 @@ void FileEngine::addDownload( SharedData::OperationMode mode ) { - Log::info( - "Requesting to add download: {} -> {}", remotePath.path.generic_string(), localPath.path.generic_string() - ); + Log::info("Requesting to add download: {} -> {}", source.value(), destination.toUtf8()); lazyOpen( [this, - remotePath, - localPath, + source, + destination, + sizeBytes, onOperationCreated = std::move(onOperationCreated), allowOverwrite, insertRefresh, @@ -507,10 +511,7 @@ void FileEngine::addDownload( const auto operationId = Ids::generateOperationId(); Log::info( - "Adding download (with ID '{}'): {} -> {}", - operationId.value(), - remotePath.path.generic_string(), - localPath.path.generic_string() + "Adding download (with ID '{}'): {} -> {}", operationId.value(), source.value(), destination.toUtf8() ); Nui::RpcClient::callWithBackChannel( @@ -529,10 +530,10 @@ void FileEngine::addDownload( }, channelId.value().value(), operationId.value(), - (!remotePath.fullPath.empty() ? remotePath.fullPath : remotePath.path).generic_string(), - (!localPath.fullPath.empty() ? localPath.fullPath : localPath.path).generic_string(), + source.value(), + destination.toUtf8(), allowOverwrite, - remotePath.size > Constants::bigFileCutOff, + sizeBytes > Constants::bigFileCutOff, insertRefresh, createMissingDirectories, static_cast(mode) @@ -542,8 +543,8 @@ void FileEngine::addDownload( } void FileEngine::addArchiveDownload( - std::vector entries, - std::filesystem::path const& localArchivePath, + std::vector roots, + Paths::LocalPath const& localArchivePath, int compressionCodec, int compressionLevel, bool mayOverwrite, @@ -552,16 +553,16 @@ void FileEngine::addArchiveDownload( ) { Log::info( - "Requesting archive download: {} entries → {} (codec={}, level={})", - entries.size(), - localArchivePath.generic_string(), + "Requesting archive download: {} roots → {} (codec={}, level={})", + roots.size(), + localArchivePath.toUtf8(), compressionCodec, compressionLevel ); lazyOpen( [this, - entries = std::move(entries), + roots = std::move(roots), localArchivePath, compressionCodec, compressionLevel, @@ -578,11 +579,9 @@ void FileEngine::addArchiveDownload( const auto operationId = Ids::generateOperationId(); - // DirectoryEntry has to_json but no to_val; the RPC wire layer needs + // The roots have to_json but no to_val; the RPC wire layer needs // Nui::val conversion, so stringify here and parse on the backend. - nlohmann::json entriesJson = nlohmann::json::array(); - for (auto const& entry : entries) - entriesJson.push_back(entry); + const nlohmann::json rootsJson = roots; Nui::RpcClient::callWithBackChannel( fmt::format("Session::{}::sftp::addArchiveDownload", impl_->engine->sshSessionId().value()), @@ -599,8 +598,8 @@ void FileEngine::addArchiveDownload( }, channelId.value().value(), operationId.value(), - entriesJson.dump(), - localArchivePath.generic_string(), + rootsJson.dump(), + localArchivePath.toUtf8(), compressionCodec, compressionLevel, mayOverwrite, @@ -611,8 +610,8 @@ void FileEngine::addArchiveDownload( } void FileEngine::addArchiveUpload( - std::vector localPaths, - std::filesystem::path const& remoteArchivePath, + std::vector localPaths, + Paths::RemotePath const& remoteArchivePath, int compressionCodec, int compressionLevel, bool mayOverwrite, @@ -623,7 +622,7 @@ void FileEngine::addArchiveUpload( Log::info( "Requesting archive upload: {} paths → {} (codec={}, level={})", localPaths.size(), - remoteArchivePath.generic_string(), + remoteArchivePath.value(), compressionCodec, compressionLevel ); @@ -650,7 +649,7 @@ void FileEngine::addArchiveUpload( std::vector localPathStrings; localPathStrings.reserve(localPaths.size()); for (auto const& localPath : localPaths) - localPathStrings.push_back(localPath.generic_string()); + localPathStrings.push_back(localPath.toUtf8()); Nui::RpcClient::callWithBackChannel( fmt::format("Session::{}::sftp::addArchiveUpload", impl_->engine->sshSessionId().value()), @@ -668,7 +667,7 @@ void FileEngine::addArchiveUpload( channelId.value().value(), operationId.value(), localPathStrings, - remoteArchivePath.generic_string(), + remoteArchivePath.value(), compressionCodec, compressionLevel, mayOverwrite, @@ -679,7 +678,7 @@ void FileEngine::addArchiveUpload( } void FileEngine::addBulkDownload( - std::vector entries, + std::vector entries, std::vector operationIds, bool allowOverwrite, bool insertRefresh, @@ -719,7 +718,7 @@ void FileEngine::addBulkDownload( for (auto const& id : operationIds) operationIdStrings.push_back(id.value()); - SharedData::BulkAddRequest request{ + SharedData::BulkDownloadRequest request{ .entries = entries, .allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, @@ -748,7 +747,7 @@ void FileEngine::addBulkDownload( } void FileEngine::addBulkUpload( - std::vector entries, + std::vector entries, std::vector operationIds, bool allowOverwrite, bool insertRefresh, @@ -788,7 +787,7 @@ void FileEngine::addBulkUpload( for (auto const& id : operationIds) operationIdStrings.push_back(id.value()); - SharedData::BulkAddRequest request{ + SharedData::BulkUploadRequest request{ .entries = entries, .allowOverwrite = allowOverwrite, .insertRefresh = insertRefresh, @@ -817,7 +816,7 @@ void FileEngine::addBulkUpload( } void FileEngine::addBulkDelete( - std::vector entries, + std::vector entries, Ids::OperationId bulkOperationId, bool insertRefresh, SharedData::OperationMode mode, @@ -841,9 +840,8 @@ void FileEngine::addBulkDelete( return; } - SharedData::BulkAddRequest request{ + SharedData::BulkDeleteRequest request{ .entries = entries, - .allowOverwrite = false, .insertRefresh = insertRefresh, .mode = mode, }; @@ -870,8 +868,9 @@ void FileEngine::addBulkDelete( } void FileEngine::addUpload( - NuiFileExplorer::Item const& remotePath, - NuiFileExplorer::Item const& localPath, + Paths::LocalPath const& source, + Paths::RemotePath const& destination, + std::uint64_t sizeBytes, std::function, std::string const& info)> onOperationCreated, bool allowOverwrite, bool insertRefresh, @@ -879,11 +878,12 @@ void FileEngine::addUpload( SharedData::OperationMode mode ) { - Log::info("Requesting to add upload: {} -> {}", localPath.path.generic_string(), remotePath.path.generic_string()); + Log::info("Requesting to add upload: {} -> {}", source.toUtf8(), destination.value()); lazyOpen( [this, - remotePath, - localPath, + source, + destination, + sizeBytes, onOperationCreated = std::move(onOperationCreated), allowOverwrite, insertRefresh, @@ -900,10 +900,7 @@ void FileEngine::addUpload( const auto operationId = Ids::generateOperationId(); Log::info( - "Adding upload (with ID '{}'): {} -> {}", - operationId.value(), - localPath.path.generic_string(), - remotePath.path.generic_string() + "Adding upload (with ID '{}'): {} -> {}", operationId.value(), source.toUtf8(), destination.value() ); Nui::RpcClient::callWithBackChannel( @@ -922,10 +919,10 @@ void FileEngine::addUpload( }, channelId.value().value(), operationId.value(), - (!localPath.fullPath.empty() ? localPath.fullPath : localPath.path).generic_string(), - (!remotePath.fullPath.empty() ? remotePath.fullPath : remotePath.path).generic_string(), + source.toUtf8(), + destination.value(), allowOverwrite, - remotePath.size > Constants::bigFileCutOff, + sizeBytes > Constants::bigFileCutOff, insertRefresh, createMissingDirectories, static_cast(mode) @@ -935,13 +932,10 @@ void FileEngine::addUpload( } void FileEngine::remove( - std::vector const& files, - std::vector const& directories, + std::vector files, + std::vector directories, std::function onComplete, - std::function, /* regular files & empty */ - std::vector /* non empties */ - )> onNonEmptyDirectoriesFound + std::function, std::vector)> onNonEmptyDirectoriesFound ) { Log::info("Requesting to remove {} items", files.size() + directories.size()); @@ -949,8 +943,8 @@ void FileEngine::remove( lazyOpen( [this, onComplete = std::move(onComplete), - files, - directories, + files = std::move(files), + directories = std::move(directories), onNonEmptyDirectoriesFound = std::move(onNonEmptyDirectoriesFound)](auto const& channelId, std::string const& info) mutable { @@ -962,16 +956,9 @@ void FileEngine::remove( } std::vector transformedDirectories; - transformedDirectories.resize(directories.size()); - std::transform( - directories.begin(), - directories.end(), - transformedDirectories.begin(), - [](auto const& item) - { - return item.path.generic_string(); - } - ); + transformedDirectories.reserve(directories.size()); + for (auto const& directory : directories) + transformedDirectories.push_back(directory.value()); Nui::RpcClient::callWithBackChannel( fmt::format("Session::{}::sftp::preDeleteChecks", impl_->engine->sshSessionId().value()), @@ -994,17 +981,18 @@ void FileEngine::remove( onComplete(false, "no nonEmptyDirectories"); return; } - std::vector nonEmpties; - Nui::convertFromVal(val["nonEmptyDirectories"], nonEmpties); - - std::vector filesAndEmptyDirs; - filesAndEmptyDirs.reserve(files.size() + (directories.size() - nonEmpties.size())); - for (const auto& file : files) - filesAndEmptyDirs.push_back(file.path); - for (const auto& dir : directories) + std::vector nonEmptyNames; + Nui::convertFromVal(val["nonEmptyDirectories"], nonEmptyNames); + + // Only directories that were asked about count, the answer never adds a delete target. + std::vector nonEmpties; + std::vector filesAndEmptyDirs = std::move(files); + for (auto& directory : directories) { - if (std::find(nonEmpties.begin(), nonEmpties.end(), dir.path) == nonEmpties.end()) - filesAndEmptyDirs.push_back(dir.path); + if (std::ranges::find(nonEmptyNames, directory.value()) != nonEmptyNames.end()) + nonEmpties.push_back(std::move(directory)); + else + filesAndEmptyDirs.push_back(std::move(directory)); } // Always route through the queue (bulk delete), never a synchronous @@ -1021,77 +1009,17 @@ void FileEngine::remove( ); } -void FileEngine::performDelete( - std::vector files, - std::vector directoriesEmpty, - std::function onComplete -) -{ - lazyOpen( - [this, - onComplete = std::move(onComplete), - files = std::move(files), - directoriesEmpty = std::move(directoriesEmpty)](auto const& channelId, std::string const& info) mutable - { - if (!channelId) - { - Log::error("Cannot add upload, no channel"); - onComplete(false, info); - return; - } - - std::vector transformedPaths; - transformedPaths.reserve(files.size() + directoriesEmpty.size()); - std::transform( - files.begin(), - files.end(), - std::back_inserter(transformedPaths), - [](auto const& item) - { - return item.path.generic_string(); - } - ); - std::transform( - directoriesEmpty.begin(), - directoriesEmpty.end(), - std::back_inserter(transformedPaths), - [](auto const& path) - { - return path.generic_string(); - } - ); - - Nui::RpcClient::callWithBackChannel( - fmt::format("Session::{}::sftp::deleteFiles", impl_->engine->sshSessionId().value()), - [onComplete = std::move(onComplete)](Nui::val val) - { - Nui::WebApi::Console::log(val); - - if (val.hasOwnProperty("error")) - { - Log::error("(Frontend) Failed to delete files: {}", val["error"].as()); - onComplete(false, val["error"].as()); - return; - } - onComplete(true, "Success"); - }, - channelId.value().value(), - transformedPaths - ); - } - ); -} - void FileEngine::rename( - std::filesystem::path const& oldPath, - std::filesystem::path const& newPath, + Paths::RemotePath const& oldPath, + Paths::RemotePath const& newPath, + bool overwrite, std::function onComplete ) { - Log::info("Requesting to rename file: {} -> {}", oldPath.generic_string(), newPath.generic_string()); + Log::info("Requesting to rename file: {} -> {}", oldPath.value(), newPath.value()); lazyOpen( - [this, oldPath, newPath, onComplete = std::move(onComplete)]( + [this, oldPath, newPath, overwrite, onComplete = std::move(onComplete)]( auto const& channelId, std::string const& info ) mutable { @@ -1115,23 +1043,22 @@ void FileEngine::rename( onComplete(true, "Success"); }, channelId.value().value(), - oldPath.generic_string(), - newPath.generic_string() + oldPath.value(), + newPath.value(), + overwrite ); } ); } void FileEngine::addRename( - std::filesystem::path const& sourcePath, - std::filesystem::path const& destinationPath, + Paths::RemotePath const& sourcePath, + Paths::RemotePath const& destinationPath, std::function, std::string const& info)> onOperationCreated, SharedData::OperationMode mode ) { - Log::info( - "Requesting to add rename operation: {} -> {}", sourcePath.generic_string(), destinationPath.generic_string() - ); + Log::info("Requesting to add rename operation: {} -> {}", sourcePath.value(), destinationPath.value()); lazyOpen( [this, sourcePath, destinationPath, onOperationCreated = std::move(onOperationCreated), mode]( auto const& channelId, std::string const& info @@ -1149,8 +1076,8 @@ void FileEngine::addRename( Log::info( "Adding rename (with ID '{}'): {} -> {}", operationId.value(), - sourcePath.generic_string(), - destinationPath.generic_string() + sourcePath.value(), + destinationPath.value() ); Nui::RpcClient::callWithBackChannel( @@ -1167,8 +1094,8 @@ void FileEngine::addRename( }, channelId.value().value(), operationId.value(), - sourcePath.generic_string(), - destinationPath.generic_string(), + sourcePath.value(), + destinationPath.value(), static_cast(mode) ); } @@ -1176,7 +1103,7 @@ void FileEngine::addRename( } void FileEngine::removeOnQueueUnchecked( - std::vector const& paths, + std::vector const& paths, bool recursive, std::function> const&, std::string const& info)> onComplete, SharedData::OperationMode mode @@ -1194,6 +1121,11 @@ void FileEngine::removeOnQueueUnchecked( return; } + std::vector pathStrings; + pathStrings.reserve(paths.size()); + for (auto const& path : paths) + pathStrings.push_back(path.value()); + Nui::RpcClient::callWithBackChannel( fmt::format("Session::{}::sftp::queuedRemove", impl_->engine->sshSessionId().value()), [onComplete = std::move(onComplete)](Nui::val val) mutable @@ -1213,7 +1145,7 @@ void FileEngine::removeOnQueueUnchecked( onComplete(operationIds, "Success"); }, channelId.value().value(), - paths, + pathStrings, recursive, true, static_cast(mode) @@ -1223,7 +1155,7 @@ void FileEngine::removeOnQueueUnchecked( } void FileEngine::existsBatchRemote( - std::vector const& paths, + std::vector const& paths, std::function exists, std::string const& info)> onComplete ) { @@ -1232,7 +1164,7 @@ void FileEngine::existsBatchRemote( std::vector pathStrings; pathStrings.reserve(paths.size()); for (auto const& path : paths) - pathStrings.push_back(path.generic_string()); + pathStrings.push_back(path.value()); lazyOpen( [this, diff --git a/ids/include/ids/id.hpp b/ids/include/ids/id.hpp index aa74cc67..0f713b37 100644 --- a/ids/include/ids/id.hpp +++ b/ids/include/ids/id.hpp @@ -112,7 +112,7 @@ namespace Ids name() \ : Id{"INVALID_ID"} \ {} \ - name(Id id) \ + explicit name(Id id) \ : Id{std::move(id)} \ {} \ \ diff --git a/nui-file-explorer/include/nui-file-explorer/item.hpp b/nui-file-explorer/include/nui-file-explorer/item.hpp index 5ad52806..c3dc1713 100644 --- a/nui-file-explorer/include/nui-file-explorer/item.hpp +++ b/nui-file-explorer/include/nui-file-explorer/item.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -17,10 +18,46 @@ namespace NuiFileExplorer { using Type = SharedData::DirectoryEntry::FileType; std::string icon; // url or base64 url etc. + /** @brief The absolute path of the item on its side, set by the side model that lists it. */ + std::filesystem::path fullPath{}; + /** @brief The entry leading to the parent directory, shown as "..". It is never selected or acted on. */ + bool isParentLink{false}; explicit Item(SharedData::DirectoryEntry const& entry, std::string icon = {}) : SharedData::DirectoryEntry{entry} , icon{std::move(icon)} {} + + /** + * @brief The entry leading to @p parentDirectory. + */ + static Item parentLink(std::filesystem::path parentDirectory, std::string icon = {}) + { + Item item{SharedData::DirectoryEntry{.type = Type::Directory}, std::move(icon)}; + item.fullPath = std::move(parentDirectory); + item.isParentLink = true; + return item; + } + + /** + * @brief An item dropped from outside, known by its absolute path only. Separators of both platforms + * count, the frontend cannot know which one the path came from. + */ + static Item dropped(SharedData::DirectoryEntry entry, std::string const& absolutePath) + { + std::string_view trimmed{absolutePath}; + while (trimmed.size() > 1 && (trimmed.back() == '/' || trimmed.back() == '\\')) + trimmed.remove_suffix(1); + const auto separator = trimmed.find_last_of("/\\"); + entry.name = std::string{separator == std::string_view::npos ? trimmed : trimmed.substr(separator + 1)}; + Item item{entry}; + item.fullPath = std::filesystem::path{std::string{trimmed}}; + return item; + } + + std::string displayName() const + { + return isParentLink ? std::string{".."} : name; + } }; -} \ No newline at end of file +} diff --git a/nui-file-explorer/include/nui-file-explorer/item_with_internals.hpp b/nui-file-explorer/include/nui-file-explorer/item_with_internals.hpp index a775187f..d402ada1 100644 --- a/nui-file-explorer/include/nui-file-explorer/item_with_internals.hpp +++ b/nui-file-explorer/include/nui-file-explorer/item_with_internals.hpp @@ -40,7 +40,7 @@ namespace NuiFileExplorer explicit ItemWithInternals(Item const& sourceItem) : item{sourceItem} , element{} - , displayFilename{sourceItem.path.filename().string()} + , displayFilename{sourceItem.displayName()} , displaySize{Utility::formatBytes(static_cast(sourceItem.size))} , displayPerms{sourceItem.lsStyleTypePermsUserGroup()} , displayMtime{sourceItem.readableMTime()} @@ -64,7 +64,7 @@ namespace NuiFileExplorer bool isSelectable() const { - return item.path.filename() != ".."; + return !item.isParentLink; } auto observeClassRelevant(auto&& fn) const diff --git a/nui-file-explorer/include/nui-file-explorer/side/selection_manager.hpp b/nui-file-explorer/include/nui-file-explorer/side/selection_manager.hpp index b78ad3cb..669c4974 100644 --- a/nui-file-explorer/include/nui-file-explorer/side/selection_manager.hpp +++ b/nui-file-explorer/include/nui-file-explorer/side/selection_manager.hpp @@ -97,6 +97,17 @@ namespace NuiFileExplorer /// Not called by selectAll / deselectAll / selectRange / ctrlAddRange. void setScrollIntoViewCallback(std::function callback); + /** + * @brief Items for which @p isVisible is false (hidden by a filter) are never selected, not by select-all and + * not inside a shift range. Call @ref refreshSelection when the answer changes. + */ + void setVisibilityPredicate(std::function isVisible); + + /** + * @brief Re-applies the selection model, dropping items that are no longer visible. + */ + void refreshSelection(); + // ------------------------------------------------------------------ interaction void onItemClicked(ItemWithInternals const& item, Nui::WebApi::MouseEvent const& event); @@ -277,6 +288,7 @@ namespace NuiFileExplorer // Scroll-into-view std::function scrollIntoView_{}; + std::function isVisible_{}; std::optional lastScrolledIndex_{}; }; } \ No newline at end of file diff --git a/nui-file-explorer/include/nui-file-explorer/side/side_implementation.hpp b/nui-file-explorer/include/nui-file-explorer/side/side_implementation.hpp index 66649946..919013ff 100644 --- a/nui-file-explorer/include/nui-file-explorer/side/side_implementation.hpp +++ b/nui-file-explorer/include/nui-file-explorer/side/side_implementation.hpp @@ -148,6 +148,15 @@ namespace NuiFileExplorer std::sort(items.begin(), partitionBorder, predicate); if (partitionBorder != items.end()) std::sort(partitionBorder, items.end(), predicate); + // The parent link leads in every order. + std::stable_partition( + items.begin(), + items.end(), + [](auto const& entry) + { + return entry.item.isParentLink; + } + ); } void sortByName(bool ascending) @@ -158,8 +167,8 @@ namespace NuiFileExplorer [](auto const& lhs, auto const& rhs) { return Utility::Algorithm::caseInsensitiveCompare( - lhs.item.path.filename().string(), - rhs.item.path.filename().string()) < 0; + lhs.item.displayName(), + rhs.item.displayName()) < 0; } ); } @@ -169,8 +178,8 @@ namespace NuiFileExplorer [](auto const& lhs, auto const& rhs) { return Utility::Algorithm::caseInsensitiveCompare( - lhs.item.path.filename().string(), - rhs.item.path.filename().string()) > 0; + lhs.item.displayName(), + rhs.item.displayName()) > 0; } ); } diff --git a/nui-file-explorer/source/nui-file-explorer/drop_event_extractor.cpp b/nui-file-explorer/source/nui-file-explorer/drop_event_extractor.cpp index 9efe0de3..f6fca3a0 100644 --- a/nui-file-explorer/source/nui-file-explorer/drop_event_extractor.cpp +++ b/nui-file-explorer/source/nui-file-explorer/drop_event_extractor.cpp @@ -5,7 +5,11 @@ #include #include +#include #include +#include +#include +#include using namespace std::string_literals; @@ -13,20 +17,68 @@ namespace NuiFileExplorer { namespace { - bool looksLikeAPath(std::string const& text) + /** + * @brief Decodes the %XX escapes of a file URL path, other text stays as it is. + */ + std::string percentDecoded(std::string_view text) { - if (text.empty()) - return false; - - if (text.front() == '/') - return true; + const auto hexValue = [](char character) -> int + { + if (character >= '0' && character <= '9') + return character - '0'; + if (character >= 'a' && character <= 'f') + return character - 'a' + 10; + if (character >= 'A' && character <= 'F') + return character - 'A' + 10; + return -1; + }; + std::string decoded; + decoded.reserve(text.size()); + for (std::size_t index = 0; index < text.size(); ++index) + { + if (text[index] == '%' && index + 2 < text.size()) + { + const auto high = hexValue(text[index + 1]); + const auto low = hexValue(text[index + 2]); + if (high >= 0 && low >= 0) + { + decoded.push_back(static_cast(high * 16 + low)); + index += 2; + continue; + } + } + decoded.push_back(text[index]); + } + return decoded; + } + /** + * @brief The absolute local path a dropped text token names: a file URL or an absolute path. Any other text, + * like words with a slash in them, is no dropped file. + */ + std::optional droppedPathOf(std::string const& text) + { + if (text.empty()) + return std::nullopt; #ifdef PATH_MAX if (text.length() > PATH_MAX) - return false; + return std::nullopt; #endif - - return text.find('/') != std::string::npos || text.find('\\') != std::string::npos; + if (text.starts_with("file://")) + { + auto path = percentDecoded(std::string_view{text}.substr(7)); + // file:///C:/x carries the drive after the slash. + if (path.size() > 3 && path[0] == '/' && path[2] == ':') + path.erase(0, 1); + return path; + } + if (text.front() == '/') + return text; + const bool hasDrive = text.size() > 2 && std::isalpha(static_cast(text[0])) && + text[1] == ':' && (text[2] == '\\' || text[2] == '/'); + if (hasDrive) + return text; + return std::nullopt; } std::vector splitByWhitespace(std::string const& text) @@ -64,7 +116,7 @@ namespace NuiFileExplorer return {}; std::string_view fileUrl = html.substr(pos + 1); if (fileUrl.starts_with("file://")) - return std::string{fileUrl.substr(7)}; + return std::string{fileUrl}; return {}; } } @@ -80,7 +132,7 @@ namespace NuiFileExplorer DropEventResult result; if (droppedOnItem && droppedOnItem->type == Item::Type::Directory) - result.internalDropSubdir = droppedOnItem->path.filename().string(); + result.internalDropSubdir = droppedOnItem->name; const auto files = dataTransferOpt->files(); if (files.length() > 0 && STRINGIZE_EXPANDED(BROWSER_ENGINE) == "webview2"s) @@ -203,10 +255,10 @@ namespace NuiFileExplorer auto split = splitByWhitespace(text); for (auto const& part : split) { - if (looksLikeAPath(part)) + if (auto path = droppedPathOf(part); path) { result.externDroppedItems = result.externDroppedItems.value_or(std::vector{}); - result.externDroppedItems->push_back(Item{SharedData::DirectoryEntry{.path = part}}); + result.externDroppedItems->push_back(Item::dropped(SharedData::DirectoryEntry{}, *path)); } } return result; @@ -220,10 +272,10 @@ namespace NuiFileExplorer auto extractedLink = extractFromHtmlLink(html); if (!extractedLink.empty()) { - if (looksLikeAPath(extractedLink)) + if (auto path = droppedPathOf(extractedLink); path) { result.externDroppedItems = result.externDroppedItems.value_or(std::vector{}); - result.externDroppedItems->push_back(Item{SharedData::DirectoryEntry{.path = extractedLink}}); + result.externDroppedItems->push_back(Item::dropped(SharedData::DirectoryEntry{}, *path)); result.issueWebkitWarning = STRINGIZE_EXPANDED(BROWSER_ENGINE) == "webkitgtk"s; return result; } diff --git a/nui-file-explorer/source/nui-file-explorer/side.cpp b/nui-file-explorer/source/nui-file-explorer/side.cpp index 9915b076..0c4f4f0a 100644 --- a/nui-file-explorer/source/nui-file-explorer/side.cpp +++ b/nui-file-explorer/source/nui-file-explorer/side.cpp @@ -59,6 +59,14 @@ namespace NuiFileExplorer // PgUp/PgDn should align with the pagination step so keyboard paging matches the footer. impl_->selectionManager.setPageJumpSize(static_cast(std::max(1, impl_->pageSize.value()))); + // Items a filter hides are never acted on: select-all and shift ranges skip them. + impl_->selectionManager.setVisibilityPredicate( + [impl = impl_.get()](std::size_t index) + { + return impl->filterQuery.value().empty() || + impl->filterMatchPosition.contains(static_cast(index)); + } + ); if (impl_->model->placesProvider() || impl_->model->favoritesProvider() || impl_->model->drivesProvider()) { @@ -177,10 +185,10 @@ namespace NuiFileExplorer { // If only a single item is selected and its a directory, set that as the target subdir for the // transfer. Otherwise, transfer to the current directory. - std::optional targetSubdir = std::nullopt; + std::optional targetSubdir = std::nullopt; auto selectedItems = this->selectedItems(); if (selectedItems.size() == 1 && selectedItems.front().isDirectory()) - targetSubdir = selectedItems.front().path.filename(); + targetSubdir = selectedItems.front().name; impl_->otherSide->model().onTransfer(copiedFiles, targetSubdir); } } @@ -346,12 +354,12 @@ namespace NuiFileExplorer std::views::filter( [showHidden](auto const& item) { - const auto filename = item.path.filename().string(); - return showHidden || filename.empty() || filename[0] != '.' || filename == ".."; + const auto filename = item.displayName(); + return showHidden || item.isParentLink || filename.empty() || filename[0] != '.'; } ); for (auto const& item : visibleItems) - pathSet.insert(item.path); + pathSet.insert(item.fullPath); std::ranges::transform( visibleItems, std::back_inserter(impl_->items.value()), @@ -591,7 +599,7 @@ namespace NuiFileExplorer if (item != nullptr) { - Nui::WebApi::Console::log("Context menu item: ", item->item.path.string()); + Nui::WebApi::Console::log("Context menu item: ", item->item.displayName()); auto selected = selectedItems(); if (std::find_if( @@ -599,7 +607,7 @@ namespace NuiFileExplorer selected.end(), [&item](auto const& sel) { - return sel.path == item->item.path; + return sel.fullPath == item->item.fullPath; } ) == selected.end()) { @@ -623,7 +631,7 @@ namespace NuiFileExplorer clickItems.end(), [](auto const& itm) { - return itm.path.filename() == ".."; + return itm.isParentLink; } ), clickItems.end() @@ -713,7 +721,7 @@ namespace NuiFileExplorer result.begin(), [](auto const& item) { - return item.path; + return item.fullPath; } ); return result; @@ -1107,7 +1115,7 @@ namespace NuiFileExplorer for (auto& item : impl_->items.value()) { item.searchHighlight( - matchesQuery(item.item.path.generic_string(), normalizedQuery, queryTokens) + matchesQuery(item.item.displayName(), normalizedQuery, queryTokens) ? ItemWithInternals::SearchHighlight::Highlight : ItemWithInternals::SearchHighlight::Muted ); @@ -1181,7 +1189,7 @@ namespace NuiFileExplorer auto const& items = impl_->items.value(); for (std::size_t itemIdx = 0; itemIdx < items.size(); ++itemIdx) { - if (matchesQuery(items[itemIdx].item.path.generic_string(), normalizedQuery, queryTokens)) + if (matchesQuery(items[itemIdx].item.displayName(), normalizedQuery, queryTokens)) { const auto absoluteIndex = static_cast(itemIdx); impl_->filterMatchPosition.emplace( @@ -1210,6 +1218,7 @@ namespace NuiFileExplorer impl_->currentPage = 0; impl_->filterQuery = std::move(query); + impl_->selectionManager.refreshSelection(); impl_->items.modifyNow(); Nui::globalEventContext.executeActiveEventsImmediately(); } diff --git a/nui-file-explorer/source/nui-file-explorer/side/icon_flavor.cpp b/nui-file-explorer/source/nui-file-explorer/side/icon_flavor.cpp index d8669fb6..05d0b1bb 100644 --- a/nui-file-explorer/source/nui-file-explorer/side/icon_flavor.cpp +++ b/nui-file-explorer/source/nui-file-explorer/side/icon_flavor.cpp @@ -114,7 +114,7 @@ namespace NuiFileExplorer !(itemRect.right() < boxRect.left() || itemRect.left() > boxRect.right() || itemRect.bottom() < boxRect.top() || itemRect.top() > boxRect.bottom()); - if (intersects && item.item.path.filename() != "..") + if (intersects && !item.item.isParentLink) impl().selectionManager.select(item); } } diff --git a/nui-file-explorer/source/nui-file-explorer/side/selection_manager.cpp b/nui-file-explorer/source/nui-file-explorer/side/selection_manager.cpp index 660ff47e..4a19cdf0 100644 --- a/nui-file-explorer/source/nui-file-explorer/side/selection_manager.cpp +++ b/nui-file-explorer/source/nui-file-explorer/side/selection_manager.cpp @@ -114,7 +114,7 @@ namespace NuiFileExplorer { const auto& vec = items_->value(); for (std::size_t i = 0; i < vec.size(); ++i) - if (vec[i].item.path == path) + if (vec[i].item.fullPath == path) return i; return std::nullopt; } @@ -124,7 +124,17 @@ namespace NuiFileExplorer const auto& vec = items_->value(); if (idx >= vec.size()) return false; - return vec[idx].item.path.filename() != ".."; + return !vec[idx].item.isParentLink && (!isVisible_ || isVisible_(idx)); + } + + void SelectionManager::setVisibilityPredicate(std::function isVisible) + { + isVisible_ = std::move(isVisible); + } + + void SelectionManager::refreshSelection() + { + rebuildSelection(); } // ========================================================================= @@ -317,7 +327,7 @@ namespace NuiFileExplorer std::set result; for (auto const& item : items_->value()) if (item.isSelected()) - result.insert(item.item.path); + result.insert(item.item.fullPath); return result; } diff --git a/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp b/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp index 5d14f579..887e2439 100644 --- a/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp +++ b/nui-file-explorer/source/nui-file-explorer/support/default_places_posix.cpp @@ -34,7 +34,8 @@ namespace NuiFileExplorer if (!file) return {}; const std::string content{std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; - return Utility::parseUserDirectories(content, home); + // POSIX paths are UTF-8 already. + return Utility::parseUserDirectories(content, home.native()); } /** @@ -56,7 +57,7 @@ namespace NuiFileExplorer const auto userDirectory = std::ranges::find(userDirectories, definition.userDirectoryName, &Utility::UserDirectory::name); if (userDirectory != userDirectories.end()) - return userDirectory->path; + return std::filesystem::path{userDirectory->path}; } if (!userDirectories.empty()) return {}; diff --git a/paths/include/paths/local_path.hpp b/paths/include/paths/local_path.hpp new file mode 100644 index 00000000..9533d46b --- /dev/null +++ b/paths/include/paths/local_path.hpp @@ -0,0 +1,242 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef __EMSCRIPTEN__ +# include +#endif + +namespace Paths +{ + /** + * @brief An absolute, normalized path on the local machine, kept as generic UTF-8 ('/' separated). + * + * The frontend runs in WebAssembly where std::filesystem would misread "C:/x", so the type does its own parsing + * for both platforms. The backend converts to a native std::filesystem::path only through @ref native. + * + * A default constructed LocalPath is unset (@ref empty), every operation that touches the disk must refuse it. + */ + class LocalPath + { + public: + LocalPath() = default; + + static std::expected + fromUtf8(std::string_view utf8, LocalPlatform platform = defaultLocalPlatform()) + { + const auto refuse = [utf8](PathRejection reason) + { + return std::unexpected(PathError{.reason = reason, .input = std::string{utf8}}); + }; + if (utf8.empty()) + return refuse(PathRejection::Empty); + if (!isValidUtf8(utf8)) + return refuse(PathRejection::InvalidUtf8); + + std::string generic{utf8}; + if (platform == LocalPlatform::Windows) + std::ranges::replace(generic, '\\', '/'); + + auto rootLength = rootLengthOf(generic, platform); + if (rootLength == 0) + return refuse(PathRejection::NotAbsolute); + + LocalPath result; + result.platform_ = platform; + result.value_ = generic.substr(0, rootLength); + if (result.value_.back() != '/') + result.value_.push_back('/'); + result.rootLength_ = result.value_.size(); + + std::vector segments; + std::string_view rest{generic}; + rest.remove_prefix(rootLength); + std::size_t start = 0; + while (start <= rest.size()) + { + const auto end = std::min(rest.find('/', start), rest.size()); + const auto name = rest.substr(start, end - start); + start = end + 1; + if (name.empty() || name == ".") + continue; + if (name == "..") + { + if (segments.empty()) + return refuse(PathRejection::EscapesRoot); + segments.pop_back(); + continue; + } + segments.push_back(name); + } + for (auto const name : segments) + { + auto segment = LocalPathSegment::make(name, platform); + if (!segment) + return refuse(segment.error().reason); + result = result / *segment; + } + return result; + } + +#ifndef __EMSCRIPTEN__ + /** + * @brief Takes a native path, which has to be absolute. + */ + static std::expected make(std::filesystem::path const& path) + { + const auto utf8 = path.generic_u8string(); + return fromUtf8(std::string_view{reinterpret_cast(utf8.data()), utf8.size()}); + } + + std::filesystem::path native() const + { + return std::filesystem::path{ + std::u8string{reinterpret_cast(value_.data()), value_.size()} + }.make_preferred(); + } +#endif + + LocalPath operator/(LocalPathSegment const& segment) const + { + assert(!empty()); + LocalPath result{*this}; + if (result.value_.back() != '/') + result.value_.push_back('/'); + result.value_ += segment.value(); + return result; + } + + /** + * @brief Joins @p relative, every segment has to be a valid name on this path's platform. + */ + std::expected join(RelativePath const& relative) const + { + LocalPath result{*this}; + for (auto const& segment : relative.segments()) + { + auto local = LocalPathSegment::make(segment, platform_); + if (!local) + return std::unexpected(local.error()); + result = result / *local; + } + return result; + } + + bool empty() const noexcept + { + return value_.empty(); + } + + bool isRoot() const noexcept + { + return !empty() && value_.size() == rootLength_; + } + + std::optional parent() const + { + if (empty() || isRoot()) + return std::nullopt; + LocalPath result{*this}; + const auto slash = value_.rfind('/'); + result.value_ = slash + 1 <= rootLength_ ? value_.substr(0, rootLength_) : value_.substr(0, slash); + return result; + } + + std::optional filename() const + { + if (empty() || isRoot()) + return std::nullopt; + return *PathSegment::make(std::string_view{value_}.substr(value_.rfind('/') + 1)); + } + + /** + * @brief True when this is @p base or lies below it. + */ + bool isWithin(LocalPath const& base) const + { + if (empty() || base.empty() || platform_ != base.platform_) + return false; + if (value_ == base.value_) + return true; + if (!value_.starts_with(base.value_)) + return false; + return base.value_.back() == '/' || value_[base.value_.size()] == '/'; + } + + /** + * @brief The path below @p base, EscapesRoot when this is not within it. + */ + std::expected relativeTo(LocalPath const& base) const + { + if (!isWithin(base)) + return std::unexpected(PathError{.reason = PathRejection::EscapesRoot, .input = value_}); + if (value_ == base.value_) + return RelativePath{}; + auto rest = std::string_view{value_}.substr(base.value_.size()); + if (!rest.empty() && rest.front() == '/') + rest.remove_prefix(1); + return RelativePath::make(rest); + } + + /** + * @brief Generic UTF-8 form, '/' separated on every platform. + */ + std::string const& toUtf8() const noexcept + { + return value_; + } + + LocalPlatform platform() const noexcept + { + return platform_; + } + + auto operator<=>(LocalPath const&) const = default; + + private: + /** + * @brief Length of the root prefix ("/", "C:/" or "//server/share"), 0 when not absolute. + */ + static std::size_t rootLengthOf(std::string_view generic, LocalPlatform platform) + { + if (platform == LocalPlatform::Posix) + return generic.starts_with('/') ? 1u : 0u; + + const auto isLetter = [](char character) + { + return (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'); + }; + if (generic.size() >= 3 && isLetter(generic[0]) && generic[1] == ':' && generic[2] == '/') + return 3u; + if (generic.size() == 2 && isLetter(generic[0]) && generic[1] == ':') + return 2u; + if (generic.starts_with("//")) + { + // UNC: //server/share is the root. + const auto serverEnd = generic.find('/', 2); + if (serverEnd == std::string_view::npos || serverEnd == 2) + return 0u; + const auto shareEnd = std::min(generic.find('/', serverEnd + 1), generic.size()); + if (shareEnd == serverEnd + 1) + return 0u; + return shareEnd; + } + return 0u; + } + + std::string value_{}; + std::size_t rootLength_{0}; + LocalPlatform platform_{LocalPlatform::Posix}; + }; +} diff --git a/paths/include/paths/local_path_segment.hpp b/paths/include/paths/local_path_segment.hpp new file mode 100644 index 00000000..5f345e28 --- /dev/null +++ b/paths/include/paths/local_path_segment.hpp @@ -0,0 +1,128 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace Paths +{ + /** + * @brief Naming rules of the local filesystem. + */ + enum class LocalPlatform : std::uint8_t + { + Posix, + Windows, + }; + + /** + * @brief The platform this binary runs on. + */ + constexpr LocalPlatform compiledLocalPlatform() + { +#ifdef _WIN32 + return LocalPlatform::Windows; +#else + return LocalPlatform::Posix; +#endif + } + + /** + * @brief The local platform paths default to. The frontend runs in WebAssembly and sets the backend's here. + */ + inline LocalPlatform& defaultLocalPlatform() + { + static LocalPlatform platform = compiledLocalPlatform(); + return platform; + } + + /** + * @brief A @ref PathSegment that can be created on the local filesystem of @p platform. + * + * Windows additionally refuses '\\', ':', '<', '>', '"', '|', '?', '*', control characters, device names + * like CON or LPT1 (also with an extension) and a trailing dot or space. A remote name has to pass here + * before it becomes part of a local path. + */ + class LocalPathSegment + { + public: + static std::expected + make(PathSegment const& segment, LocalPlatform platform = defaultLocalPlatform()) + { + if (platform == LocalPlatform::Windows) + { + if (auto reason = windowsRejection(segment.view()); reason) + return std::unexpected(PathError{.reason = *reason, .input = segment.value()}); + } + return LocalPathSegment{segment}; + } + + static std::expected + make(std::string_view utf8, LocalPlatform platform = defaultLocalPlatform()) + { + return PathSegment::make(utf8).and_then( + [platform](PathSegment const& segment) + { + return make(segment, platform); + } + ); + } + + PathSegment const& segment() const noexcept + { + return segment_; + } + + std::string const& value() const noexcept + { + return segment_.value(); + } + + auto operator<=>(LocalPathSegment const&) const = default; + + private: + explicit LocalPathSegment(PathSegment segment) + : segment_{std::move(segment)} + {} + + static std::optional windowsRejection(std::string_view name) + { + for (const auto character : name) + { + const auto byte = static_cast(character); + if (byte < 0x20u || std::string_view{"\\:<>\"|?*"}.find(character) != std::string_view::npos) + return PathRejection::ReservedCharacter; + } + if (name.back() == '.' || name.back() == ' ') + return PathRejection::TrailingDotOrSpace; + + // Device names count with any extension: "nul.txt" is the NUL device too. + const auto stem = name.substr(0, name.find('.')); + std::string upper{stem}; + std::transform( + upper.begin(), + upper.end(), + upper.begin(), + [](unsigned char character) + { + return static_cast(std::toupper(character)); + } + ); + constexpr std::array devices{"CON", "PRN", "AUX", "NUL"}; + if (std::ranges::find(devices, upper) != devices.end()) + return PathRejection::ReservedDeviceName; + if (upper.size() == 4 && (upper.starts_with("COM") || upper.starts_with("LPT")) && upper[3] >= '1' && + upper[3] <= '9') + return PathRejection::ReservedDeviceName; + return std::nullopt; + } + + PathSegment segment_; + }; +} diff --git a/paths/include/paths/native.hpp b/paths/include/paths/native.hpp new file mode 100644 index 00000000..7440378c --- /dev/null +++ b/paths/include/paths/native.hpp @@ -0,0 +1,29 @@ +#pragma once + +#ifndef __EMSCRIPTEN__ +# include +# include +# include + +namespace Paths +{ + /** + * @brief The text of a native path as generic UTF-8. For names and link literals, which are no LocalPath; an + * absolute path goes through LocalPath::make. + */ + inline std::string nativeToUtf8(std::filesystem::path const& path) + { + const auto text = path.generic_u8string(); + return std::string{reinterpret_cast(text.data()), text.size()}; + } + + /** + * @brief A native path from UTF-8 text, the reverse of @ref nativeToUtf8. An absolute path goes through + * LocalPath::fromUtf8, which validates it. + */ + inline std::filesystem::path nativeFromUtf8(std::string_view utf8) + { + return std::filesystem::path{std::u8string_view{reinterpret_cast(utf8.data()), utf8.size()}}; + } +} +#endif diff --git a/paths/include/paths/path_error.hpp b/paths/include/paths/path_error.hpp new file mode 100644 index 00000000..2f7bd12b --- /dev/null +++ b/paths/include/paths/path_error.hpp @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include + +namespace Paths +{ + /** + * @brief Why a string was refused as a path or path segment. + */ + enum class PathRejection : std::uint8_t + { + Empty, + DotOrDotDot, + ContainsSeparator, + ContainsNul, + InvalidUtf8, + NotAbsolute, + /// A character the local platform cannot store in a name, like '\\' or ':' on Windows. + ReservedCharacter, + /// A Windows device name like CON or LPT1, with or without an extension. + ReservedDeviceName, + /// Windows drops a trailing dot or space, the name would refer to another file. + TrailingDotOrSpace, + /// The path leaves the directory it is meant to stay in. + EscapesRoot, + }; + + inline std::string_view toString(PathRejection rejection) + { + switch (rejection) + { + case PathRejection::Empty: + return "is empty"; + case PathRejection::DotOrDotDot: + return "is '.' or '..'"; + case PathRejection::ContainsSeparator: + return "contains a path separator"; + case PathRejection::ContainsNul: + return "contains a NUL character"; + case PathRejection::InvalidUtf8: + return "is not valid UTF-8"; + case PathRejection::NotAbsolute: + return "is not absolute"; + case PathRejection::ReservedCharacter: + return "contains a character the local system does not allow"; + case PathRejection::ReservedDeviceName: + return "is a reserved device name"; + case PathRejection::TrailingDotOrSpace: + return "ends with a dot or space"; + case PathRejection::EscapesRoot: + return "leaves its root directory"; + } + return "is invalid"; + } + + /** + * @brief A refused path, with the input made printable. + */ + struct PathError + { + PathRejection reason{PathRejection::Empty}; + std::string input{}; + + std::string toString() const + { + std::string printable; + printable.reserve(input.size()); + for (const auto character : input) + { + const auto byte = static_cast(character); + if (byte < 0x20u || byte == 0x7Fu) + { + constexpr std::string_view digits = "0123456789ABCDEF"; + printable += "\\x"; + printable.push_back(digits[byte >> 4u]); + printable.push_back(digits[byte & 0x0Fu]); + } + else + printable.push_back(character); + } + return "'" + printable + "' " + std::string{Paths::toString(reason)}; + } + }; +} diff --git a/paths/include/paths/path_segment.hpp b/paths/include/paths/path_segment.hpp new file mode 100644 index 00000000..92944454 --- /dev/null +++ b/paths/include/paths/path_segment.hpp @@ -0,0 +1,113 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace Paths +{ + /** + * @brief True when @p text is well-formed UTF-8 (no overlong forms, no surrogates). + */ + inline bool isValidUtf8(std::string_view text) + { + std::size_t index = 0; + while (index < text.size()) + { + const auto lead = static_cast(text[index]); + std::size_t length = 0; + std::uint32_t codePoint = 0; + if (lead < 0x80u) + { + ++index; + continue; + } + if ((lead & 0xE0u) == 0xC0u) + { + length = 2; + codePoint = lead & 0x1Fu; + } + else if ((lead & 0xF0u) == 0xE0u) + { + length = 3; + codePoint = lead & 0x0Fu; + } + else if ((lead & 0xF8u) == 0xF0u) + { + length = 4; + codePoint = lead & 0x07u; + } + else + return false; + + if (index + length > text.size()) + return false; + for (std::size_t offset = 1; offset < length; ++offset) + { + const auto continuation = static_cast(text[index + offset]); + if ((continuation & 0xC0u) != 0x80u) + return false; + codePoint = (codePoint << 6u) | (continuation & 0x3Fu); + } + constexpr std::uint32_t minimumForLength[] = {0u, 0u, 0x80u, 0x800u, 0x10000u}; + if (codePoint < minimumForLength[length] || codePoint > 0x10FFFFu || + (codePoint >= 0xD800u && codePoint <= 0xDFFFu)) + return false; + index += length; + } + return true; + } + + /** + * @brief One name of a path: never empty, "." or "..", never containing '/' or NUL, always UTF-8. + * + * A remote name with a backslash or colon is a valid segment, whether it can exist locally is decided by + * @ref LocalPathSegment. + */ + class PathSegment + { + public: + static std::expected make(std::string_view utf8) + { + const auto refuse = [utf8](PathRejection reason) + { + return std::unexpected(PathError{.reason = reason, .input = std::string{utf8}}); + }; + if (utf8.empty()) + return refuse(PathRejection::Empty); + if (utf8 == "." || utf8 == "..") + return refuse(PathRejection::DotOrDotDot); + if (utf8.find('/') != std::string_view::npos) + return refuse(PathRejection::ContainsSeparator); + if (utf8.find('\0') != std::string_view::npos) + return refuse(PathRejection::ContainsNul); + if (!isValidUtf8(utf8)) + return refuse(PathRejection::InvalidUtf8); + return PathSegment{std::string{utf8}}; + } + + std::string const& value() const noexcept + { + return value_; + } + + std::string_view view() const noexcept + { + return value_; + } + + auto operator<=>(PathSegment const&) const = default; + + private: + explicit PathSegment(std::string value) + : value_{std::move(value)} + {} + + std::string value_; + }; +} diff --git a/paths/include/paths/paths.hpp b/paths/include/paths/paths.hpp new file mode 100644 index 00000000..0895c872 --- /dev/null +++ b/paths/include/paths/paths.hpp @@ -0,0 +1,104 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#ifdef __EMSCRIPTEN__ +# include +#endif + +#include +#include + +namespace Paths +{ + namespace Detail + { + template + PathT orThrow(std::expected result) + { + if (!result) + throw std::invalid_argument{result.error().toString()}; + return std::move(result).value(); + } + } + + // Paths travel as plain strings, invalid ones are refused when they are read. + + inline void to_json(nlohmann::json& json, PathSegment const& segment) + { + json = segment.value(); + } + inline void from_json(nlohmann::json const& json, PathSegment& segment) + { + segment = Detail::orThrow(PathSegment::make(json.get())); + } + + inline void to_json(nlohmann::json& json, RelativePath const& path) + { + json = path.value(); + } + inline void from_json(nlohmann::json const& json, RelativePath& path) + { + path = Detail::orThrow(RelativePath::make(json.get())); + } + + inline void to_json(nlohmann::json& json, RemotePath const& path) + { + json = path.value(); + } + inline void from_json(nlohmann::json const& json, RemotePath& path) + { + path = Detail::orThrow(RemotePath::make(json.get())); + } + + inline void to_json(nlohmann::json& json, LocalPath const& path) + { + json = path.toUtf8(); + } + inline void from_json(nlohmann::json const& json, LocalPath& path) + { + path = Detail::orThrow(LocalPath::fromUtf8(json.get())); + } + +#ifdef __EMSCRIPTEN__ + inline void to_val(Nui::val& value, PathSegment const& segment) + { + value = Nui::val::u8string(segment.value().c_str()); + } + inline void from_val(Nui::val const& value, PathSegment& segment) + { + segment = Detail::orThrow(PathSegment::make(value.as())); + } + inline void to_val(Nui::val& value, RelativePath const& path) + { + value = Nui::val::u8string(path.value().c_str()); + } + inline void from_val(Nui::val const& value, RelativePath& path) + { + path = Detail::orThrow(RelativePath::make(value.as())); + } + inline void to_val(Nui::val& value, RemotePath const& path) + { + value = Nui::val::u8string(path.value().c_str()); + } + inline void from_val(Nui::val const& value, RemotePath& path) + { + path = Detail::orThrow(RemotePath::make(value.as())); + } + inline void to_val(Nui::val& value, LocalPath const& path) + { + value = Nui::val::u8string(path.toUtf8().c_str()); + } + inline void from_val(Nui::val const& value, LocalPath& path) + { + path = Detail::orThrow(LocalPath::fromUtf8(value.as())); + } +#endif +} diff --git a/paths/include/paths/relative_path.hpp b/paths/include/paths/relative_path.hpp new file mode 100644 index 00000000..4e5fa390 --- /dev/null +++ b/paths/include/paths/relative_path.hpp @@ -0,0 +1,131 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace Paths +{ + /** + * @brief A path below some root: zero or more segments joined by '/'. Empty is the root itself. + * + * It never starts with '/' and never contains "." or "..", so joining it onto a root cannot leave that root. + * This is the only bridge between a local and a remote path. + */ + class RelativePath + { + public: + RelativePath() = default; + + static std::expected make(std::string_view utf8) + { + RelativePath result; + if (utf8.empty()) + return result; + if (utf8.front() == '/') + return std::unexpected(PathError{.reason = PathRejection::EscapesRoot, .input = std::string{utf8}}); + + std::size_t start = 0; + while (start <= utf8.size()) + { + const auto end = std::min(utf8.find('/', start), utf8.size()); + auto segment = PathSegment::make(utf8.substr(start, end - start)); + if (!segment) + return std::unexpected(PathError{.reason = segment.error().reason, .input = std::string{utf8}}); + result = result / *segment; + start = end + 1; + } + return result; + } + + RelativePath operator/(PathSegment const& segment) const + { + RelativePath result{*this}; + if (!result.value_.empty()) + result.value_.push_back('/'); + result.value_ += segment.value(); + return result; + } + + RelativePath operator/(RelativePath const& other) const + { + if (other.empty()) + return *this; + if (empty()) + return other; + RelativePath result{*this}; + result.value_.push_back('/'); + result.value_ += other.value_; + return result; + } + + bool empty() const noexcept + { + return value_.empty(); + } + + std::vector segments() const + { + std::vector result; + std::size_t start = 0; + while (!value_.empty() && start <= value_.size()) + { + const auto end = std::min(value_.find('/', start), value_.size()); + result.push_back(*PathSegment::make(std::string_view{value_}.substr(start, end - start))); + start = end + 1; + } + return result; + } + + std::optional parent() const + { + if (empty()) + return std::nullopt; + const auto slash = value_.rfind('/'); + RelativePath result; + if (slash != std::string::npos) + result.value_ = value_.substr(0, slash); + return result; + } + + std::optional leaf() const + { + if (empty()) + return std::nullopt; + const auto slash = value_.rfind('/'); + return *PathSegment::make(slash == std::string::npos ? value_ : value_.substr(slash + 1)); + } + + /** + * @brief True when this is @p other or lies below it. + */ + bool isWithin(RelativePath const& other) const + { + if (other.empty() || value_ == other.value_) + return true; + return value_.size() > other.value_.size() && value_.starts_with(other.value_) && + value_[other.value_.size()] == '/'; + } + + std::string const& value() const noexcept + { + return value_; + } + + std::string_view view() const noexcept + { + return value_; + } + + auto operator<=>(RelativePath const&) const = default; + + private: + std::string value_{}; + }; +} diff --git a/paths/include/paths/remote_path.hpp b/paths/include/paths/remote_path.hpp new file mode 100644 index 00000000..74926388 --- /dev/null +++ b/paths/include/paths/remote_path.hpp @@ -0,0 +1,217 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace Paths +{ + /** + * @brief An absolute path on the server: UTF-8, '/' separated, normalized. + * + * Never touches std::filesystem, so a backslash or a colon in a server name stays a character on every client + * platform. ".." is refused instead of resolved: the server decides what it means. + * + * A default constructed RemotePath is unset (@ref empty), every operation that touches the server must refuse it. + */ + class RemotePath + { + public: + RemotePath() = default; + + static RemotePath root() + { + return RemotePath{std::string{"/"}}; + } + + /** + * @brief Collapses repeated separators and "." segments and drops a trailing '/'. + */ + static std::expected make(std::string_view utf8) + { + const auto refuse = [utf8](PathRejection reason) + { + return std::unexpected(PathError{.reason = reason, .input = std::string{utf8}}); + }; + if (utf8.empty()) + return refuse(PathRejection::Empty); + if (utf8.front() != '/') + return refuse(PathRejection::NotAbsolute); + + auto result = root(); + std::size_t start = 1; + while (start <= utf8.size()) + { + const auto end = std::min(utf8.find('/', start), utf8.size()); + const auto name = utf8.substr(start, end - start); + start = end + 1; + if (name.empty() || name == ".") + continue; + auto segment = PathSegment::make(name); + if (!segment) + return refuse(segment.error().reason); + result = result / *segment; + } + return result; + } + + RemotePath operator/(PathSegment const& segment) const + { + assert(!empty()); + RemotePath result{*this}; + if (result.value_.back() != '/') + result.value_.push_back('/'); + result.value_ += segment.value(); + return result; + } + + RemotePath operator/(RelativePath const& relative) const + { + if (relative.empty()) + return *this; + assert(!empty()); + RemotePath result{*this}; + if (result.value_.back() != '/') + result.value_.push_back('/'); + result.value_ += relative.value(); + return result; + } + + /** + * @brief This path with @p suffix appended to its last name, like "file.txt" to "file.txt.filepart". + */ + std::expected withFilenameSuffix(std::string_view suffix) const + { + const auto name = filename(); + if (!name) + return std::unexpected(PathError{.reason = PathRejection::Empty, .input = value_}); + return PathSegment::make(name->value() + std::string{suffix}) + .transform( + [this](PathSegment const& segment) + { + return *parent() / segment; + } + ); + } + + /** + * @brief Where a symlink at this path with the content @p linkContent points to. + * + * Relative content is taken from the link's directory. ".." is resolved lexically, which differs from the + * server's view only when a directory on the way is itself a symlink. ".." at the root stays at the root. + */ + std::expected resolveLink(std::string_view linkContent) const + { + const auto refuse = [linkContent](PathRejection reason) + { + return std::unexpected(PathError{.reason = reason, .input = std::string{linkContent}}); + }; + if (linkContent.empty()) + return refuse(PathRejection::Empty); + if (empty()) + return refuse(PathRejection::NotAbsolute); + + auto result = linkContent.front() == '/' ? root() : parent().value_or(root()); + std::size_t start = 0; + while (start <= linkContent.size()) + { + const auto end = std::min(linkContent.find('/', start), linkContent.size()); + const auto name = linkContent.substr(start, end - start); + start = end + 1; + if (name.empty() || name == ".") + continue; + if (name == "..") + { + result = result.parent().value_or(root()); + continue; + } + auto segment = PathSegment::make(name); + if (!segment) + return refuse(segment.error().reason); + result = result / *segment; + } + return result; + } + + bool empty() const noexcept + { + return value_.empty(); + } + + bool isRoot() const noexcept + { + return value_ == "/"; + } + + std::optional parent() const + { + if (empty() || isRoot()) + return std::nullopt; + const auto slash = value_.rfind('/'); + return RemotePath{slash == 0 ? std::string{"/"} : value_.substr(0, slash)}; + } + + std::optional filename() const + { + if (empty() || isRoot()) + return std::nullopt; + return *PathSegment::make(std::string_view{value_}.substr(value_.rfind('/') + 1)); + } + + /** + * @brief True when this is @p base or lies below it. + */ + bool isWithin(RemotePath const& base) const + { + if (empty() || base.empty()) + return false; + if (base.isRoot() || value_ == base.value_) + return true; + return value_.size() > base.value_.size() && value_.starts_with(base.value_) && + value_[base.value_.size()] == '/'; + } + + /** + * @brief The path below @p base, EscapesRoot when this is not within it. + */ + std::expected relativeTo(RemotePath const& base) const + { + if (!isWithin(base)) + return std::unexpected(PathError{.reason = PathRejection::EscapesRoot, .input = value_}); + if (value_ == base.value_) + return RelativePath{}; + const auto offset = base.isRoot() ? 1u : base.value_.size() + 1u; + return RelativePath::make(std::string_view{value_}.substr(offset)); + } + + std::string const& value() const noexcept + { + return value_; + } + + std::string_view view() const noexcept + { + return value_; + } + + char const* c_str() const noexcept + { + return value_.c_str(); + } + + auto operator<=>(RemotePath const&) const = default; + + private: + explicit RemotePath(std::string value) + : value_{std::move(value)} + {} + + std::string value_{}; + }; +} diff --git a/paths/source/paths/CMakeLists.txt b/paths/source/paths/CMakeLists.txt new file mode 100644 index 00000000..4cc13287 --- /dev/null +++ b/paths/source/paths/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(paths INTERFACE) + +target_include_directories(paths INTERFACE "${CMAKE_CURRENT_LIST_DIR}/../../include") +target_link_libraries(paths INTERFACE nlohmann_json::nlohmann_json) diff --git a/paths/test/paths/CMakeLists.txt b/paths/test/paths/CMakeLists.txt new file mode 100644 index 00000000..7579316b --- /dev/null +++ b/paths/test/paths/CMakeLists.txt @@ -0,0 +1,28 @@ +include(GoogleTest) + +add_executable( + nui-sftp-paths-test + main.cpp +) + +target_link_libraries( + nui-sftp-paths-test + PUBLIC + paths + core-target + GTest::gtest + GTest::gtest_main +) + +set_target_properties(nui-sftp-paths-test PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/test") + +gtest_discover_tests(nui-sftp-paths-test) + +# If msys2, copy dynamic libraries to executable directory, visual studio does this automatically. +# And there is no need on linux. +if (DEFINED ENV{MSYSTEM}) + add_custom_command(TARGET nui-sftp-paths-test POST_BUILD + COMMAND bash -c "ldd $" | "grep" "clang" | awk "NF == 4 { system(\"${CMAKE_COMMAND} -E copy \" \$3 \" $\") }" + VERBATIM + ) +endif() diff --git a/paths/test/paths/main.cpp b/paths/test/paths/main.cpp new file mode 100644 index 00000000..2fbe76bb --- /dev/null +++ b/paths/test/paths/main.cpp @@ -0,0 +1,12 @@ +#include "test_path_segment.hpp" +#include "test_remote_path.hpp" +#include "test_local_path.hpp" +#include "test_native.hpp" + +#include + +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/paths/test/paths/test_local_path.hpp b/paths/test/paths/test_local_path.hpp new file mode 100644 index 00000000..1c5fd92a --- /dev/null +++ b/paths/test/paths/test_local_path.hpp @@ -0,0 +1,77 @@ +#pragma once + +#include + +#include + +namespace Paths::Test +{ + TEST(LocalPathTests, ParsesPosixPaths) + { + const auto path = LocalPath::fromUtf8("/home//tim/./sync/", LocalPlatform::Posix); + ASSERT_TRUE(path.has_value()); + EXPECT_EQ(path->toUtf8(), "/home/tim/sync"); + EXPECT_EQ(LocalPath::fromUtf8("/home/tim/../root", LocalPlatform::Posix)->toUtf8(), "/home/root"); + EXPECT_EQ(LocalPath::fromUtf8("/..", LocalPlatform::Posix).error().reason, PathRejection::EscapesRoot); + EXPECT_EQ(LocalPath::fromUtf8("relative", LocalPlatform::Posix).error().reason, PathRejection::NotAbsolute); + // A backslash is a character on POSIX. + EXPECT_EQ(LocalPath::fromUtf8("/a\\b", LocalPlatform::Posix)->filename()->value(), "a\\b"); + } + + TEST(LocalPathTests, ParsesWindowsPaths) + { + EXPECT_EQ(LocalPath::fromUtf8("C:\\Users\\tim\\", LocalPlatform::Windows)->toUtf8(), "C:/Users/tim"); + EXPECT_EQ(LocalPath::fromUtf8("C:/", LocalPlatform::Windows)->toUtf8(), "C:/"); + EXPECT_EQ(LocalPath::fromUtf8("c:", LocalPlatform::Windows)->toUtf8(), "c:/"); + EXPECT_EQ(LocalPath::fromUtf8("\\\\server\\share\\dir", LocalPlatform::Windows)->toUtf8(), "//server/share/dir"); + EXPECT_EQ(LocalPath::fromUtf8("/x", LocalPlatform::Windows).error().reason, PathRejection::NotAbsolute); + EXPECT_EQ(LocalPath::fromUtf8("C:x", LocalPlatform::Windows).error().reason, PathRejection::NotAbsolute); + EXPECT_EQ( + LocalPath::fromUtf8("C:/dir/file.txt:stream", LocalPlatform::Windows).error().reason, + PathRejection::ReservedCharacter + ); + } + + TEST(LocalPathTests, ParentAndFilenameStopAtTheRoot) + { + const auto drive = LocalPath::fromUtf8("C:/a", LocalPlatform::Windows).value(); + EXPECT_EQ(drive.parent()->toUtf8(), "C:/"); + EXPECT_FALSE(drive.parent()->parent().has_value()); + EXPECT_EQ(LocalPath::fromUtf8("/a/b", LocalPlatform::Posix)->parent()->toUtf8(), "/a"); + EXPECT_EQ(LocalPath::fromUtf8("/a", LocalPlatform::Posix)->parent()->toUtf8(), "/"); + } + + TEST(LocalPathTests, JoiningRemoteNamesChecksTheLocalRules) + { + const auto relative = RelativePath::make("dir/a\\b").value(); + const auto posix = LocalPath::fromUtf8("/sync", LocalPlatform::Posix).value(); + EXPECT_EQ(posix.join(relative)->toUtf8(), "/sync/dir/a\\b"); + + const auto windows = LocalPath::fromUtf8("C:/sync", LocalPlatform::Windows).value(); + EXPECT_EQ(windows.join(relative).error().reason, PathRejection::ReservedCharacter); + EXPECT_EQ(windows.join(RelativePath::make("CON").value()).error().reason, PathRejection::ReservedDeviceName); + } + + TEST(LocalPathTests, RelativeToStaysWithinTheBase) + { + const auto base = LocalPath::fromUtf8("/home/tim", LocalPlatform::Posix).value(); + EXPECT_EQ(LocalPath::fromUtf8("/home/tim/a/b", LocalPlatform::Posix)->relativeTo(base)->value(), "a/b"); + EXPECT_EQ( + LocalPath::fromUtf8("/home/timothy", LocalPlatform::Posix)->relativeTo(base).error().reason, + PathRejection::EscapesRoot + ); + const auto root = LocalPath::fromUtf8("/", LocalPlatform::Posix).value(); + EXPECT_EQ(base.relativeTo(root)->value(), "home/tim"); + } + +#ifndef __EMSCRIPTEN__ + TEST(LocalPathTests, ConvertsToAndFromNativePaths) + { + const auto native = std::filesystem::current_path() / "t\xC3\xA4st"; + const auto path = LocalPath::make(native); + ASSERT_TRUE(path.has_value()) << path.error().toString(); + EXPECT_EQ(path->native(), native.lexically_normal()); + EXPECT_FALSE(LocalPath::make(std::filesystem::path{"relative"}).has_value()); + } +#endif +} diff --git a/paths/test/paths/test_native.hpp b/paths/test/paths/test_native.hpp new file mode 100644 index 00000000..3d239379 --- /dev/null +++ b/paths/test/paths/test_native.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include + +#include +#include + +namespace Paths::Test +{ + TEST(NativePathTests, NonAsciiTextSurvivesTheRoundTrip) + { + const std::string text = "ordner/Grüße 東京 🚀.txt"; + EXPECT_EQ(nativeToUtf8(nativeFromUtf8(text)), text); + } + + TEST(NativePathTests, NameOfANativePathIsUtf8) + { + const auto native = nativeFromUtf8("Ärger.txt"); + EXPECT_EQ(nativeToUtf8(native.filename()), "Ärger.txt"); + EXPECT_EQ(native.filename().extension(), nativeFromUtf8(".txt")); + } + +#ifdef _WIN32 + TEST(NativePathTests, BackslashesBecomeForwardSlashes) + { + EXPECT_EQ(nativeToUtf8(std::filesystem::path{L"C:\\Users\\Jürgen"}), "C:/Users/Jürgen"); + } +#endif +} diff --git a/paths/test/paths/test_path_segment.hpp b/paths/test/paths/test_path_segment.hpp new file mode 100644 index 00000000..c78cd828 --- /dev/null +++ b/paths/test/paths/test_path_segment.hpp @@ -0,0 +1,86 @@ +#pragma once + +#include + +#include + +#include + +namespace Paths::Test +{ + TEST(PathSegmentTests, RefusesNamesThatAreNotASingleSegment) + { + for (auto const& [name, reason] : std::initializer_list>{ + {"", PathRejection::Empty}, + {".", PathRejection::DotOrDotDot}, + {"..", PathRejection::DotOrDotDot}, + {"x/..", PathRejection::ContainsSeparator}, + {"a/b", PathRejection::ContainsSeparator}, + {"/", PathRejection::ContainsSeparator}, + {std::string{"a\0b", 3}, PathRejection::ContainsNul}, + {"\xC3\x28", PathRejection::InvalidUtf8}, + {"\xED\xA0\x80", PathRejection::InvalidUtf8}, + {"\xC0\xAF", PathRejection::InvalidUtf8}, + }) + { + const auto segment = PathSegment::make(name); + ASSERT_FALSE(segment.has_value()) << name; + EXPECT_EQ(segment.error().reason, reason) << name; + } + } + + TEST(PathSegmentTests, AcceptsNamesThatAreLegalOnServers) + { + for (auto const* name : {"a", "...", ".hidden", "a\\b", "C:", "con", "trailing.", "\xC3\xA4\xC3\xB6", "a b"}) + EXPECT_TRUE(PathSegment::make(name).has_value()) << name; + } + + TEST(LocalPathSegmentTests, PosixAcceptsEveryPathSegment) + { + for (auto const* name : {"a\\b", "C:", "con", "trailing.", "a?b"}) + EXPECT_TRUE(LocalPathSegment::make(name, LocalPlatform::Posix).has_value()) << name; + } + + TEST(LocalPathSegmentTests, WindowsRefusesNamesItCannotStore) + { + for (auto const& [name, reason] : std::initializer_list>{ + {"a\\b", PathRejection::ReservedCharacter}, + {"..\\..\\x", PathRejection::ReservedCharacter}, + {"C:", PathRejection::ReservedCharacter}, + {"file.txt:stream", PathRejection::ReservedCharacter}, + {"a?b", PathRejection::ReservedCharacter}, + {"tab\there", PathRejection::ReservedCharacter}, + {"trailing.", PathRejection::TrailingDotOrSpace}, + {"trailing ", PathRejection::TrailingDotOrSpace}, + {"CON", PathRejection::ReservedDeviceName}, + {"nul.txt", PathRejection::ReservedDeviceName}, + {"Lpt9", PathRejection::ReservedDeviceName}, + {"com1.tar.gz", PathRejection::ReservedDeviceName}, + }) + { + const auto segment = LocalPathSegment::make(name, LocalPlatform::Windows); + ASSERT_FALSE(segment.has_value()) << name; + EXPECT_EQ(segment.error().reason, reason) << name; + } + for (auto const* name : {"console", "com10", "lpt", "a.b", "nul_"}) + EXPECT_TRUE(LocalPathSegment::make(name, LocalPlatform::Windows).has_value()) << name; + } + + TEST(RelativePathTests, ParsesAndJoinsSegments) + { + const auto path = RelativePath::make("a/b/c.txt"); + ASSERT_TRUE(path.has_value()); + EXPECT_EQ(path->segments().size(), 3u); + EXPECT_EQ(path->leaf()->value(), "c.txt"); + EXPECT_EQ(path->parent()->value(), "a/b"); + EXPECT_EQ(RelativePath{}.value(), ""); + EXPECT_TRUE(path->isWithin(*RelativePath::make("a"))); + EXPECT_FALSE(path->isWithin(*RelativePath::make("a/bb"))); + } + + TEST(RelativePathTests, RefusesAnythingThatCouldLeaveItsRoot) + { + for (auto const* text : {"/a", "..", "a/../b", "a//b", "a/", "./a", "x/.."}) + EXPECT_FALSE(RelativePath::make(text).has_value()) << text; + } +} diff --git a/paths/test/paths/test_remote_path.hpp b/paths/test/paths/test_remote_path.hpp new file mode 100644 index 00000000..300632dc --- /dev/null +++ b/paths/test/paths/test_remote_path.hpp @@ -0,0 +1,99 @@ +#pragma once + +#include + +#include + +namespace Paths::Test +{ + TEST(RemotePathTests, NormalizesSeparatorsAndDots) + { + EXPECT_EQ(RemotePath::make("/home//test/./a/")->value(), "/home/test/a"); + EXPECT_EQ(RemotePath::make("/")->value(), "/"); + EXPECT_EQ(RemotePath::make("//")->value(), "/"); + } + + TEST(RemotePathTests, RefusesRelativeAndParentReferences) + { + EXPECT_EQ(RemotePath::make("home").error().reason, PathRejection::NotAbsolute); + EXPECT_EQ(RemotePath::make("").error().reason, PathRejection::Empty); + EXPECT_EQ(RemotePath::make("/home/..").error().reason, PathRejection::DotOrDotDot); + EXPECT_EQ(RemotePath::make("/a/../b").error().reason, PathRejection::DotOrDotDot); + } + + TEST(RemotePathTests, BackslashesAndColonsAreOrdinaryCharacters) + { + const auto base = RemotePath::make("/home/test").value(); + const auto joined = base / PathSegment::make("a\\..\\b").value(); + EXPECT_EQ(joined.value(), "/home/test/a\\..\\b"); + EXPECT_EQ(joined.parent()->value(), "/home/test"); + EXPECT_EQ(joined.filename()->value(), "a\\..\\b"); + + // A drive-like name never replaces the base like std::filesystem::path("/x") / "C:y" does on Windows. + EXPECT_EQ((base / PathSegment::make("C:notes").value()).value(), "/home/test/C:notes"); + } + + TEST(RemotePathTests, ParentAndFilenameStopAtTheRoot) + { + EXPECT_FALSE(RemotePath::root().parent().has_value()); + EXPECT_FALSE(RemotePath::root().filename().has_value()); + EXPECT_EQ(RemotePath::make("/a")->parent()->value(), "/"); + } + + TEST(RemotePathTests, RelativeToStaysWithinTheBase) + { + const auto base = RemotePath::make("/home/test").value(); + EXPECT_EQ(RemotePath::make("/home/test/a/b")->relativeTo(base)->value(), "a/b"); + EXPECT_EQ(RemotePath::make("/home/test")->relativeTo(base)->value(), ""); + EXPECT_EQ(RemotePath::make("/home/testing")->relativeTo(base).error().reason, PathRejection::EscapesRoot); + EXPECT_EQ(RemotePath::make("/etc")->relativeTo(base).error().reason, PathRejection::EscapesRoot); + EXPECT_EQ(RemotePath::make("/a/b")->relativeTo(RemotePath::root())->value(), "a/b"); + } + + TEST(RemotePathTests, JoiningARelativePathCannotEscape) + { + const auto base = RemotePath::make("/srv/sync").value(); + const auto joined = base / RelativePath::make("x/y").value(); + EXPECT_EQ(joined.value(), "/srv/sync/x/y"); + EXPECT_TRUE(joined.isWithin(base)); + } + + TEST(RemotePathTests, RoundTripsThroughJson) + { + const nlohmann::json json = RemotePath::make("/home/t\xC3\xA4st/a\\b").value(); + EXPECT_EQ(json.get(), "/home/t\xC3\xA4st/a\\b"); + EXPECT_EQ(json.get().value(), "/home/t\xC3\xA4st/a\\b"); + EXPECT_THROW(nlohmann::json{"relative"}.at(0).get(), std::invalid_argument); + } + + TEST(RemotePathTests, FilenameSuffixStaysInTheSameDirectory) + { + EXPECT_EQ(RemotePath::make("/home/a.txt")->withFilenameSuffix(".filepart")->value(), "/home/a.txt.filepart"); + EXPECT_EQ(RemotePath::make("/a")->withFilenameSuffix("~")->value(), "/a~"); + EXPECT_EQ(RemotePath::make("/home/a")->withFilenameSuffix("/../x").error().reason, PathRejection::ContainsSeparator); + EXPECT_FALSE(RemotePath::root().withFilenameSuffix(".old").has_value()); + EXPECT_FALSE(RemotePath{}.withFilenameSuffix(".old").has_value()); + } + + TEST(RemotePathTests, LinksResolveFromTheirDirectory) + { + const auto link = RemotePath::make("/usr/lib/libx.so").value(); + EXPECT_EQ(link.resolveLink("libx.so.1")->value(), "/usr/lib/libx.so.1"); + EXPECT_EQ(link.resolveLink("../share/./x/")->value(), "/usr/share/x"); + EXPECT_EQ(link.resolveLink("/opt/x")->value(), "/opt/x"); + EXPECT_EQ(link.resolveLink("../../../../etc")->value(), "/etc"); + EXPECT_EQ(link.resolveLink("..")->value(), "/usr"); + EXPECT_EQ(link.resolveLink("").error().reason, PathRejection::Empty); + EXPECT_FALSE(RemotePath{}.resolveLink("x").has_value()); + } + + TEST(RemotePathTests, DefaultIsUnset) + { + const RemotePath unset{}; + EXPECT_TRUE(unset.empty()); + EXPECT_FALSE(unset.isRoot()); + EXPECT_FALSE(unset.parent().has_value()); + EXPECT_FALSE(unset.isWithin(RemotePath::root())); + EXPECT_FALSE(RemotePath::root().isWithin(unset)); + } +} diff --git a/persistence/include/persistence/state/file_tracking_options.hpp b/persistence/include/persistence/state/file_tracking_options.hpp index 188bea90..44634a63 100644 --- a/persistence/include/persistence/state/file_tracking_options.hpp +++ b/persistence/include/persistence/state/file_tracking_options.hpp @@ -2,6 +2,8 @@ #include +#include + namespace Persistence { struct FileTrackingOptions : public DefaultMissingMember @@ -9,7 +11,14 @@ namespace Persistence bool autoReupload{true}; bool moveRemoteOnLocalMove{true}; bool deleteRemoteOnLocalDelete{true}; + /// Before a re-upload, ask when the server's copy changed since it was opened. Unset means true, optional so + /// configurations written before it load without a warning. + std::optional checkRemoteChanges{std::nullopt}; }; - BOOST_DESCRIBE_STRUCT(FileTrackingOptions, (), (autoReupload, moveRemoteOnLocalMove, deleteRemoteOnLocalDelete)) + BOOST_DESCRIBE_STRUCT( + FileTrackingOptions, + (), + (autoReupload, moveRemoteOnLocalMove, deleteRemoteOnLocalDelete, checkRemoteChanges) + ) } diff --git a/persistence/include/persistence/state/local_filesystem_options.hpp b/persistence/include/persistence/state/local_filesystem_options.hpp index abee67ee..b4051bf4 100644 --- a/persistence/include/persistence/state/local_filesystem_options.hpp +++ b/persistence/include/persistence/state/local_filesystem_options.hpp @@ -7,6 +7,17 @@ namespace Persistence { + /** + * @brief What happens when a file that can run code (a program, a script, a shortcut) is opened. + */ + enum class RiskyFilePolicy + { + Allow, + Warn, + Never + }; + BOOST_DESCRIBE_ENUM(RiskyFilePolicy, Allow, Warn, Never) + struct LocalFilesystemOptions : public DefaultMissingMember { // by default, dont allow the user to delete files locally. @@ -16,6 +27,8 @@ namespace Persistence bool preventCreateDirectory{false}; std::optional homeOverride{std::nullopt}; std::optional temporaryDownloadsDirectory{"%cache_home%/nui-sftp-downloads"}; + /// Optional so configurations written before it load without a warning; unset means Warn. + std::optional openRiskyFiles{std::nullopt}; }; BOOST_DESCRIBE_STRUCT( @@ -26,6 +39,7 @@ namespace Persistence preventCreateFile, preventCreateDirectory, homeOverride, - temporaryDownloadsDirectory) + temporaryDownloadsDirectory, + openRiskyFiles) ) } \ No newline at end of file diff --git a/persistence/include/persistence/state/sftp_options.hpp b/persistence/include/persistence/state/sftp_options.hpp index a19bb553..41a068fc 100644 --- a/persistence/include/persistence/state/sftp_options.hpp +++ b/persistence/include/persistence/state/sftp_options.hpp @@ -62,8 +62,10 @@ namespace Persistence { std::optional reserveSpace{std::nullopt}; std::optional doCleanup{std::nullopt}; + /// Mark downloaded files as coming from the internet (Mark of the Web on Windows, quarantine on macOS). + std::optional markAsDownloaded{std::nullopt}; }; - BOOST_DESCRIBE_STRUCT(DownloadOptions, (CommonTransferOptions), (reserveSpace, doCleanup)) + BOOST_DESCRIBE_STRUCT(DownloadOptions, (CommonTransferOptions), (reserveSpace, doCleanup, markAsDownloaded)) struct UploadOptions : public CommonTransferOptions {}; diff --git a/persistence/include/persistence/state/state.hpp b/persistence/include/persistence/state/state.hpp index 364ccd30..98db154e 100644 --- a/persistence/include/persistence/state/state.hpp +++ b/persistence/include/persistence/state/state.hpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -58,6 +59,11 @@ namespace Persistence LocalizationOptions localizationOptions{}; std::optional lastDirectConnect{}; std::optional lastDirectConnectInheritables{}; + /** + * @brief When each sync pair (session, local root and remote root) was last synchronized, in seconds since + * the epoch. Sync compares modification times against it to see which side changed since. + */ + std::optional> syncBaselines{}; State fullyResolve() const; @@ -89,6 +95,7 @@ namespace Persistence fileTrackingOptions, localFilesystemOptions, lastDirectConnect, - lastDirectConnectInheritables) + lastDirectConnectInheritables, + syncBaselines) ) } \ No newline at end of file diff --git a/shared_data/benchmark/shared_data/sync_benchmark.cpp b/shared_data/benchmark/shared_data/sync_benchmark.cpp index 56a43e20..c9fdfb3f 100644 --- a/shared_data/benchmark/shared_data/sync_benchmark.cpp +++ b/shared_data/benchmark/shared_data/sync_benchmark.cpp @@ -154,7 +154,7 @@ int main(int argc, char** argv) { for (std::size_t file = files; file-- > 0;) entries.push_back( - {.path = fmt::format("file{:05}.txt", file), .type = SharedData::FileType::Regular, .size = 1} + {.name = fmt::format("file{:05}.txt", file), .type = SharedData::FileType::Regular, .size = 1} ); total += files; return; @@ -162,7 +162,7 @@ int main(int argc, char** argv) for (std::size_t directory = 10; directory-- > 0;) { const auto name = fmt::format("dir{:03}", directory); - entries.push_back({.path = name, .type = SharedData::FileType::Directory}); + entries.push_back({.name = name, .type = SharedData::FileType::Directory}); build(path + "/" + name, depth + 1); } }; @@ -172,12 +172,12 @@ int main(int argc, char** argv) [&] { auto scanner = [&listing]( - std::filesystem::path const& path + Paths::RelativePath const& relative ) -> std::expected, int> { - return listing.at(path.generic_string()); + return listing.at(relative.empty() ? std::string{"/root"} : "/root/" + relative.value()); }; - TreeDirectoryWalker walker{"/root", std::move(scanner)}; + TreeDirectoryWalker walker{std::move(scanner)}; if (!walker.walkAll()) std::abort(); auto tree = std::move(walker).ejectTree(); @@ -267,7 +267,7 @@ int main(int argc, char** argv) std::unordered_set roots; for (std::size_t index = 0; index < count; ++index) roots.insert(fmt::format("project{:06}", index)); - const ScanTrees trees{.local = &sparseLocal, .remote = &sparseRemote, .localRoot = "/l", .remoteRoot = "/r"}; + const ScanTrees trees{.local = &sparseLocal, .remote = &sparseRemote}; const auto planTime = measure( repetitions, [&] @@ -281,7 +281,7 @@ int main(int argc, char** argv) { Stores stores; stores.run(local, remote); - const ScanTrees trees{.local = &local, .remote = &remote, .localRoot = "/l", .remoteRoot = "/r"}; + const ScanTrees trees{.local = &local, .remote = &remote}; std::unordered_set roots; for (auto const& row : stores.uploads.loadChildren("", trees)) roots.insert(row.relKey); @@ -311,7 +311,7 @@ int main(int argc, char** argv) } ); report("diff + finalize: one-sided subtree", oneSided.subtreeFileCount, diffTime); - const ScanTrees trees{.local = &oneSided, .remote = &empty, .localRoot = "/l", .remoteRoot = "/r"}; + const ScanTrees trees{.local = &oneSided, .remote = &empty}; const auto planTime = measure( repetitions, [&] diff --git a/shared_data/include/shared_data/directory_entry.hpp b/shared_data/include/shared_data/directory_entry.hpp index 602dd2b6..8ecdb0f8 100644 --- a/shared_data/include/shared_data/directory_entry.hpp +++ b/shared_data/include/shared_data/directory_entry.hpp @@ -1,14 +1,20 @@ #pragma once +#include + #include #include #include #include +#include #include #include #include +#include #include +#include +#include namespace SharedData { @@ -81,9 +87,12 @@ namespace SharedData { using FileType = SharedData::FileType; - std::filesystem::path path{}; - std::filesystem::path fullPath{}; - std::filesystem::path longName{}; + /** + * @brief The name as listed, UTF-8. A server or a hostile local name can be anything, like "..", "x/.." or + * empty, so it only becomes part of a path through Paths::PathSegment::make. Empty for the root of a + * walk and for stat results. + */ + std::string name{}; std::uint32_t flags{0}; FileType type{FileType::Unknown}; std::uint64_t size{0}; @@ -99,9 +108,9 @@ namespace SharedData std::uint64_t mtime{0}; std::uint32_t mtimeNsec{0}; std::string acl{}; - // For symlinks: raw value returned by readlink (i.e. the link literal, not the resolved target). + // For symlinks: raw value returned by readlink (i.e. the link literal, not the resolved target) as UTF-8. // nullopt when not a symlink or when unavailable. This is what the sync diff compares by. - std::optional linkTarget{}; + std::optional linkTarget{}; // For symlinks: the full entry of the symlink target (if known). nullptr means unknown. std::shared_ptr resolvedTarget{nullptr}; @@ -213,33 +222,31 @@ namespace SharedData } }; - inline std::filesystem::path fullPath(std::vector const& entries, DirectoryEntry const& entry) + /** + * @brief The path of @p entry below the root of its walk, following the parent indices. Empty for the root, + * which is the entry without a parent. Refused when a name on the way is no path segment. + */ + inline std::expected + relativePath(std::vector const& entries, DirectoryEntry const& entry) { - if (entry.parent) - { - const auto parentIndex = entry.parent.value(); - if (parentIndex >= entries.size()) - throw std::out_of_range("Parent index is out of range"); - return fullPath(entries, entries[parentIndex]) / entry.path; - } - else - return entry.path; - } - - inline std::filesystem::path - fullPathRelative(std::vector const& entries, DirectoryEntry const& entry) - { - if (entry.parent) - { - const auto parentIndex = entry.parent.value(); - if (parentIndex >= entries.size()) - throw std::out_of_range("Parent index is out of range"); - if (parentIndex == 0) - return entry.path; - return fullPathRelative(entries, entries[parentIndex]) / entry.path; - } - else - return entry.path; + if (!entry.parent) + return Paths::RelativePath{}; + const auto parentIndex = entry.parent.value(); + if (parentIndex >= entries.size()) + throw std::out_of_range("Parent index is out of range"); + return Paths::PathSegment::make(entry.name) + .and_then( + [&entries, parentIndex](Paths::PathSegment const& segment) + { + return relativePath(entries, entries[parentIndex]) + .transform( + [&segment](Paths::RelativePath const& parent) + { + return parent / segment; + } + ); + } + ); } void to_json(nlohmann::json& j, DirectoryEntry const& entry); diff --git a/shared_data/include/shared_data/dropped_item.hpp b/shared_data/include/shared_data/dropped_item.hpp new file mode 100644 index 00000000..efb05c1c --- /dev/null +++ b/shared_data/include/shared_data/dropped_item.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace SharedData +{ + /** + * @brief A local file or directory dropped onto the window from the desktop. + */ + struct DroppedItem + { + Paths::LocalPath path{}; + /** @brief Type and name of the item, enough for a transfer. */ + DirectoryEntry entry{}; + }; + BOOST_DESCRIBE_STRUCT(DroppedItem, (), (path, entry)) +} diff --git a/shared_data/include/shared_data/file_operations/archive_download_root.hpp b/shared_data/include/shared_data/file_operations/archive_download_root.hpp new file mode 100644 index 00000000..f2ba4da9 --- /dev/null +++ b/shared_data/include/shared_data/file_operations/archive_download_root.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace SharedData +{ + /** + * @brief One selected remote item of an archive download, packed under its own name. A directory is scanned + * and packed with its content. + */ + struct ArchiveDownloadRoot + { + Paths::RemotePath path{}; + /** @brief The listing entry of @ref path: type, size, permissions and times for the tar header. */ + DirectoryEntry entry{}; + }; + BOOST_DESCRIBE_STRUCT(ArchiveDownloadRoot, (), (path, entry)) +} diff --git a/shared_data/include/shared_data/file_operations/bulk_add_request.hpp b/shared_data/include/shared_data/file_operations/bulk_add_request.hpp deleted file mode 100644 index a7bd9013..00000000 --- a/shared_data/include/shared_data/file_operations/bulk_add_request.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include -#include -#include - -namespace SharedData -{ - /** - * @brief One item in a bulk-add request. The frontend is authoritative: - * sizes are trusted and no filesystem probe happens on the backend - * for file entries. Directories are split off into their own Scan - * + Bulk operation pair during backend dispatch. - */ - struct BulkAddEntry - { - std::filesystem::path src{}; - std::filesystem::path dst{}; ///< Unused for delete. - std::uint64_t sizeBytes{0}; ///< Meaningful only when !isDirectory. - bool isDirectory{false}; - /// Source file's mtime in seconds + nanoseconds. Only consumed by - /// bulk download (to restore the local file's mtime after transfer - /// so subsequent syncs don't see a spurious diff). Unused for bulk - /// upload (backend reads the local file's mtime directly) and delete. - std::uint64_t mtime{0}; - std::uint32_t mtimeNsec{0}; - }; - BOOST_DESCRIBE_STRUCT(BulkAddEntry, (), (src, dst, sizeBytes, isDirectory, mtime, mtimeNsec)) - - /** - * @brief Single-RPC payload for adding many file operations at once. - * Backend partitions entries into files (→ one aggregate Bulk - * operation) and directories (→ one Scan+Bulk pair each), and - * creates them all within a single strand dispatch to amortize - * SSH context switching. - */ - struct BulkAddRequest - { - std::vector entries{}; - bool allowOverwrite{false}; - bool insertRefresh{false}; - OperationMode mode{OperationMode::Queued}; - }; - BOOST_DESCRIBE_STRUCT(BulkAddRequest, (), (entries, allowOverwrite, insertRefresh, mode)) -} diff --git a/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp b/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp new file mode 100644 index 00000000..f5ec99ca --- /dev/null +++ b/shared_data/include/shared_data/file_operations/bulk_delete_request.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include + +namespace SharedData +{ + /** + * @brief One remote item of a bulk delete. A directory is scanned and removed with its content. + */ + struct BulkDeleteEntry + { + Paths::RemotePath target{}; + bool isDirectory{false}; + /** @brief Only shown on the card for files. */ + std::uint64_t sizeBytes{0}; + }; + BOOST_DESCRIBE_STRUCT(BulkDeleteEntry, (), (target, isDirectory, sizeBytes)) + + /** + * @brief Single-RPC payload for deleting many remote items. Files become one aggregate delete, every directory + * becomes a delete of its own that scans first. + */ + struct BulkDeleteRequest + { + std::vector entries{}; + bool insertRefresh{false}; + OperationMode mode{OperationMode::Queued}; + /** + * @brief The entries already list every item to delete, a directory before its content: nothing is + * scanned, a directory is only removed once empty. + */ + bool listedContentOnly{false}; + }; + BOOST_DESCRIBE_STRUCT(BulkDeleteRequest, (), (entries, insertRefresh, mode, listedContentOnly)) +} diff --git a/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp b/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp new file mode 100644 index 00000000..aee44469 --- /dev/null +++ b/shared_data/include/shared_data/file_operations/bulk_transfer_request.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include + +namespace SharedData +{ + /** + * @brief One local item of a bulk upload. A directory is scanned and uploaded with its content. + */ + struct BulkUploadEntry + { + Paths::LocalPath source{}; + Paths::RemotePath destination{}; + /** @brief Trusted for files, the backend does not stat them. */ + std::uint64_t sizeBytes{0}; + bool isDirectory{false}; + /** + * @brief The user agreed to replace this existing destination. Without it, or the request's allowOverwrite, + * a destination that appeared after the existence check is not replaced. + */ + bool mayOverwrite{false}; + }; + BOOST_DESCRIBE_STRUCT(BulkUploadEntry, (), (source, destination, sizeBytes, isDirectory, mayOverwrite)) + + /** + * @brief One remote item of a bulk download. A directory is scanned and downloaded with its content. + */ + struct BulkDownloadEntry + { + Paths::RemotePath source{}; + Paths::LocalPath destination{}; + /** @brief Trusted for files, the backend does not stat them. */ + std::uint64_t sizeBytes{0}; + bool isDirectory{false}; + /** @brief Modification time of the remote file, restored on the local copy so the next sync sees no change. */ + std::uint64_t mtime{0}; + std::uint32_t mtimeNsec{0}; + /** @brief See @ref BulkUploadEntry::mayOverwrite. */ + bool mayOverwrite{false}; + }; + BOOST_DESCRIBE_STRUCT( + BulkDownloadEntry, + (), + (source, destination, sizeBytes, isDirectory, mtime, mtimeNsec, mayOverwrite) + ) + + /** + * @brief Single-RPC payload for uploading many items. Files become one aggregate bulk, every directory becomes + * a bulk of its own that scans first. + */ + struct BulkUploadRequest + { + std::vector entries{}; + bool allowOverwrite{false}; + bool insertRefresh{false}; + OperationMode mode{OperationMode::Queued}; + }; + BOOST_DESCRIBE_STRUCT(BulkUploadRequest, (), (entries, allowOverwrite, insertRefresh, mode)) + + /** + * @brief Download analogue of @ref BulkUploadRequest. + */ + struct BulkDownloadRequest + { + std::vector entries{}; + bool allowOverwrite{false}; + bool insertRefresh{false}; + OperationMode mode{OperationMode::Queued}; + }; + BOOST_DESCRIBE_STRUCT(BulkDownloadRequest, (), (entries, allowOverwrite, insertRefresh, mode)) +} diff --git a/shared_data/include/shared_data/file_operations/operation_added.hpp b/shared_data/include/shared_data/file_operations/operation_added.hpp index c230a448..d8c9ca9d 100644 --- a/shared_data/include/shared_data/file_operations/operation_added.hpp +++ b/shared_data/include/shared_data/file_operations/operation_added.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -17,8 +18,30 @@ namespace SharedData OperationMode mode{OperationMode::Queued}; bool insertRefresh{false}; std::optional totalBytes{std::nullopt}; - std::optional localPath{}; - std::optional remotePath{}; + std::optional localPath{}; + std::optional remotePath{}; + /** @brief Rename only: the remote item before the rename. */ + std::optional source{}; + /** @brief Rename only: where the remote item ends up. */ + std::optional destination{}; + /** @brief The operation scans before it starts its own work. */ + std::optional scanPhase{}; + /** @brief Single transfers: whether the destination may be replaced, a resume asks for the same. */ + std::optional allowOverwrite{}; }; - BOOST_DESCRIBE_STRUCT(OperationAdded, (), (operationId, type, mode, totalBytes, localPath, remotePath, insertRefresh)) + BOOST_DESCRIBE_STRUCT( + OperationAdded, + (), + (operationId, + type, + mode, + totalBytes, + localPath, + remotePath, + source, + destination, + insertRefresh, + scanPhase, + allowOverwrite) + ) } \ No newline at end of file diff --git a/shared_data/include/shared_data/file_operations/operation_completed.hpp b/shared_data/include/shared_data/file_operations/operation_completed.hpp index e99f7ac9..5b27fdc3 100644 --- a/shared_data/include/shared_data/file_operations/operation_completed.hpp +++ b/shared_data/include/shared_data/file_operations/operation_completed.hpp @@ -4,13 +4,13 @@ #include #include #include +#include #include #include #include #include -#include #include namespace SharedData @@ -22,10 +22,11 @@ namespace SharedData OperationCompletionReason reason; Ids::OperationId operationId; std::chrono::system_clock::time_point completionTime; - std::optional localPath{std::nullopt}; - std::optional remotePath{std::nullopt}; + std::optional localPath{std::nullopt}; + std::optional remotePath{std::nullopt}; std::optional error{std::nullopt}; - std::vector> failedEntries{}; + /** @brief The skipped entries of a bulk by their source path, a remote path for downloads. */ + std::vector> failedEntries{}; }; BOOST_DESCRIBE_STRUCT( OperationCompleted, diff --git a/shared_data/include/shared_data/file_operations/operation_error.hpp b/shared_data/include/shared_data/file_operations/operation_error.hpp index bfa20fb6..a0315322 100644 --- a/shared_data/include/shared_data/file_operations/operation_error.hpp +++ b/shared_data/include/shared_data/file_operations/operation_error.hpp @@ -6,7 +6,18 @@ namespace SecureShell { - BOOST_DESCRIBE_ENUM(WrapperErrors, None, OwnerNull, SharedPtrDestroyed, ShortWrite, FileNull) + BOOST_DESCRIBE_ENUM( + WrapperErrors, + None, + OwnerNull, + SharedPtrDestroyed, + ShortWrite, + FileNull, + TaskPushFailed, + BufferUnavailable, + UnexpectedEndOfFile, + LocalIoFailure + ) BOOST_DESCRIBE_STRUCT(SftpError, (), (message, sshError, sftpError, wrapperError)) inline void from_json(nlohmann::json const& json, WrapperErrors& error) diff --git a/shared_data/include/shared_data/file_operations/operation_error_type.hpp b/shared_data/include/shared_data/file_operations/operation_error_type.hpp index aa270007..20ce72dd 100644 --- a/shared_data/include/shared_data/file_operations/operation_error_type.hpp +++ b/shared_data/include/shared_data/file_operations/operation_error_type.hpp @@ -42,6 +42,11 @@ namespace SharedData DeleteFailed, CannotCreateSymlink, SymlinkPointsToDirectory, - NotAFile + NotAFile, + SizeMismatch, + ScanIncomplete, + DirectoryUnreadable, + InvalidName, + LinkInLocalPath ); } \ No newline at end of file diff --git a/shared_data/include/shared_data/file_operations/operation_type.hpp b/shared_data/include/shared_data/file_operations/operation_type.hpp index bed77cf1..8324a719 100644 --- a/shared_data/include/shared_data/file_operations/operation_type.hpp +++ b/shared_data/include/shared_data/file_operations/operation_type.hpp @@ -17,6 +17,7 @@ namespace SharedData Rename, Delete, BulkDelete, + LocalDelete, ArchiveDownload, ArchiveUpload, CustomAction diff --git a/shared_data/include/shared_data/ignore_rules.hpp b/shared_data/include/shared_data/ignore_rules.hpp index 43c43f83..be9a922f 100644 --- a/shared_data/include/shared_data/ignore_rules.hpp +++ b/shared_data/include/shared_data/ignore_rules.hpp @@ -1,7 +1,8 @@ #pragma once +#include + #include -#include #include #include #include @@ -20,7 +21,7 @@ namespace SharedData /** @brief Rules parsed from one ignore file, tagged with the directory they apply under. */ struct IgnoreRuleSet { - std::filesystem::path originDir{}; + Paths::RelativePath originDir{}; std::vector patterns{}; }; @@ -46,14 +47,14 @@ namespace SharedData { public: /** @brief Register parsed patterns from one ignore file living at @p relDir. */ - void addFile(std::filesystem::path const& relDir, std::string_view content); + void addFile(Paths::RelativePath const& relDir, std::string_view content); /** @brief True if @p relPath is ignored under any currently-known rule set. * - * @param relPath Entry path relative to the scan root (use '/' separators). + * @param relPath Entry path relative to the scan root. * @param isDirectory Whether the entry is a directory (relevant for directoryOnly rules). */ - bool isIgnored(std::filesystem::path const& relPath, bool isDirectory) const; + bool isIgnored(Paths::RelativePath const& relPath, bool isDirectory) const; bool empty() const noexcept { diff --git a/shared_data/include/shared_data/sync/diff.hpp b/shared_data/include/shared_data/sync/diff.hpp index 68417077..1d9add0d 100644 --- a/shared_data/include/shared_data/sync/diff.hpp +++ b/shared_data/include/shared_data/sync/diff.hpp @@ -1,12 +1,14 @@ #pragma once #include +#include #include #include #include #include +#include #include #include @@ -31,11 +33,17 @@ namespace SharedData::Sync bool recursive{true}; /// When true, entries with any path segment starting with '.' are dropped. bool ignoreHidden{false}; + /// The local filesystem folds case (Windows, macOS), set by the backend. Remote names that differ only in + /// case would land on one local file then. + bool localCaseInsensitive{false}; + /// When this pair was last synchronized (seconds since the epoch); a side whose modification time is later + /// changed since. Unset for the first sync, newest wins then. + std::optional lastSyncTime{}; }; BOOST_DESCRIBE_STRUCT( DiffOptions, (), - (direction, actionUpload, actionDownload, actionDelete, recursive, ignoreHidden) + (direction, actionUpload, actionDownload, actionDelete, recursive, ignoreHidden, localCaseInsensitive, lastSyncTime) ) /** @@ -54,6 +62,12 @@ namespace SharedData::Sync virtual void emitDownload(DiffTreeNode node, std::string const& parentRelKey) = 0; virtual void emitDelete(DiffTreeNode node, std::string const& parentRelKey) = 0; + /** + * @brief An entry the diff leaves alone, so nothing is transferred or deleted for it or below it. + */ + virtual void onBlocked(std::string const& /* relKey */, BlockReason /* reason */) + {} + /** * @brief Called periodically with the running compare count. Implementations * typically translate this into a remote-progress RPC. diff --git a/shared_data/include/shared_data/sync/diff_summary.hpp b/shared_data/include/shared_data/sync/diff_summary.hpp index b77dd5ff..d8d29dac 100644 --- a/shared_data/include/shared_data/sync/diff_summary.hpp +++ b/shared_data/include/shared_data/sync/diff_summary.hpp @@ -5,9 +5,35 @@ #include #include +#include +#include namespace SharedData::Sync { + /** + * @brief Why the diff left an entry alone even though it exists on one or both sides. + */ + BOOST_DEFINE_ENUM_CLASS( + BlockReason, + Unreadable, + Excluded, + InvalidName, + TypeConflict, + CaseConflict, + ChangedOnBothSides, + ChangedOnDestination + ) + + /** + * @brief An entry sync does not act on, with the reason. + */ + struct BlockedEntry + { + std::string relKey{}; + BlockReason reason{BlockReason::Unreadable}; + }; + BOOST_DESCRIBE_STRUCT(BlockedEntry, (), (relKey, reason)) + /** * @brief Aggregate metrics for one section (uploads, downloads, or deletes) of a diff. */ @@ -44,16 +70,45 @@ namespace SharedData::Sync bool heavyCompare{false}; /// True when the walk bailed out early because of a cancel request. bool cancelled{false}; + /// Number of entries sync leaves alone because they are unreadable, excluded or conflicting. + std::uint64_t blockedCount{0}; + /// The first blocked entries, for the dialog to name them. + std::vector blockedSamples{}; }; BOOST_DESCRIBE_STRUCT( DiffSummary, (), - (sessionId, uploads, downloads, deletes, entriesCompared, generation, heavyCompare, cancelled) + (sessionId, + uploads, + downloads, + deletes, + entriesCompared, + generation, + heavyCompare, + cancelled, + blockedCount, + blockedSamples) ) // ADL hooks — nlohmann's adl_serializer looks for to_json/from_json in the type's // namespace, and SharedData::to_json lives one namespace up. These forwarders // let it find the generic described-struct path. + inline void to_json(nlohmann::json& j, BlockReason const& e) + { + SharedData::to_json(j, e); + } + inline void from_json(nlohmann::json const& j, BlockReason& e) + { + SharedData::from_json(j, e); + } + inline void to_json(nlohmann::json& j, BlockedEntry const& s) + { + SharedData::to_json(j, s); + } + inline void from_json(nlohmann::json const& j, BlockedEntry& s) + { + SharedData::from_json(j, s); + } inline void to_json(nlohmann::json& j, SectionSummary const& s) { SharedData::to_json(j, s); @@ -72,6 +127,14 @@ namespace SharedData::Sync } #ifdef NUI_FRONTEND + inline void to_val(Nui::val& v, BlockReason const& e) + { + SharedData::to_val(v, e); + } + inline void from_val(Nui::val const& v, BlockReason& e) + { + SharedData::from_val(v, e); + } // No to_val/from_val forwarders for SectionSummary / DiffSummary — the // described-struct overload in nui's convertToVal handles them natively. #endif diff --git a/shared_data/include/shared_data/sync/diff_tree_store.hpp b/shared_data/include/shared_data/sync/diff_tree_store.hpp index e810636e..de398bfb 100644 --- a/shared_data/include/shared_data/sync/diff_tree_store.hpp +++ b/shared_data/include/shared_data/sync/diff_tree_store.hpp @@ -25,8 +25,6 @@ namespace SharedData::Sync { ScanNode const* local{nullptr}; ScanNode const* remote{nullptr}; - std::filesystem::path localRoot{}; - std::filesystem::path remoteRoot{}; }; /** @@ -97,6 +95,18 @@ namespace SharedData::Sync std::vector& plan ) const; + /** + * @brief A directory delete as what the scan listed below it, the directory before its content. Deleted + * last first and directories only once empty, so unlisted content (unreadable, excluded, hidden or + * ignored) keeps its directories. + */ + void appendDeletePlan( + ScanNode const& directory, + std::string const& relKey, + Action action, + std::vector& plan + ) const; + DiffOptions options_{}; std::unordered_map nodes_{}; std::unordered_map> children_{}; @@ -122,6 +132,7 @@ namespace SharedData::Sync void emitUpload(DiffTreeNode node, std::string const& parentRelKey) override; void emitDownload(DiffTreeNode node, std::string const& parentRelKey) override; void emitDelete(DiffTreeNode node, std::string const& parentRelKey) override; + void onBlocked(std::string const& relKey, BlockReason reason) override; void onProgress(std::uint64_t entriesCompared) override; bool cancelled() const override; @@ -130,6 +141,19 @@ namespace SharedData::Sync return lastCompared_; } + std::uint64_t blockedCount() const + { + return blockedCount_; + } + + std::vector const& blockedSamples() const + { + return blockedSamples_; + } + + /** @brief How many blocked entries are kept by name. */ + static constexpr std::size_t maxBlockedSamples = 10000u; + private: DiffTreeStore* uploads_; DiffTreeStore* downloads_; @@ -137,5 +161,7 @@ namespace SharedData::Sync std::shared_ptr> cancelled_; std::function onProgress_; std::uint64_t lastCompared_{0}; + std::uint64_t blockedCount_{0}; + std::vector blockedSamples_{}; }; } diff --git a/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp b/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp index 21388df6..8c14244c 100644 --- a/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp +++ b/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp @@ -23,15 +23,12 @@ namespace SharedData::Sync * side creates missing directories as needed via the bulk upload/download * flow. * - * Both @ref localAbsPath and @ref remoteAbsPath are always populated (the sides - * the action does not touch simply carry the path that would apply if it did). + * Entries carry no paths: the backend resolves @ref relKey below its own sync roots. */ struct EnqueuePlanEntry { std::string relKey{}; Action action{Action::Upload}; - std::string localAbsPath{}; - std::string remoteAbsPath{}; std::uint64_t sizeBytes{0}; std::uint64_t mtime{0}; std::uint32_t mtimeNsec{0}; @@ -40,7 +37,7 @@ namespace SharedData::Sync BOOST_DESCRIBE_STRUCT( EnqueuePlanEntry, (), - (relKey, action, localAbsPath, remoteAbsPath, sizeBytes, mtime, mtimeNsec, isDirectory) + (relKey, action, sizeBytes, mtime, mtimeNsec, isDirectory) ) inline void to_json(nlohmann::json& j, EnqueuePlanEntry const& e) diff --git a/shared_data/include/shared_data/sync/scan_node.hpp b/shared_data/include/shared_data/sync/scan_node.hpp index 56547ccf..2e68f441 100644 --- a/shared_data/include/shared_data/sync/scan_node.hpp +++ b/shared_data/include/shared_data/sync/scan_node.hpp @@ -10,6 +10,20 @@ namespace SharedData::Sync { + /** + * @brief Whether the scan knows what is behind a node. Sync never acts on a node that is not Listed. + */ + enum class Availability + { + Listed, + /// A directory whose listing failed, its content is unknown. + Unreadable, + /// Present but not transferable: sockets, devices, FIFOs and symlinks on Windows. + Excluded, + /// A name that cannot be a path segment, like ".." or a name containing '/'. + InvalidName, + }; + /** * @brief One node of a scan tree. * @@ -27,7 +41,7 @@ namespace SharedData::Sync std::uint32_t mtimeNsec{0}; std::filesystem::perms permissions{std::filesystem::perms::unknown}; /// For symlinks: the raw link literal. What the diff compares by. - std::optional linkTarget{}; + std::optional linkTarget{}; std::uint32_t uid{0}; std::uint32_t gid{0}; /// Sorted ascending by name (POSIX byte comparison). @@ -39,5 +53,6 @@ namespace SharedData::Sync /// False when the scan did not descend into this directory (non-recursive mode). /// Diff suppresses one-sided delete emissions when this is false. bool childrenKnown{true}; + Availability availability{Availability::Listed}; }; } diff --git a/shared_data/include/shared_data/sync/sync_execution.hpp b/shared_data/include/shared_data/sync/sync_execution.hpp new file mode 100644 index 00000000..fe000bf2 --- /dev/null +++ b/shared_data/include/shared_data/sync/sync_execution.hpp @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace SharedData::Sync +{ + /** + * @brief Asks the backend to plan and queue the selected rows of one section. Paths never travel: the backend + * resolves every row below its own roots. + * + * The operation ids are chosen by the frontend, so it can listen to them before the reply arrives. An id is + * unused when its part of the plan is empty. + */ + struct SyncExecutionRequest + { + Ids::SyncSessionId syncSessionId{}; + std::uint64_t generation{0}; + DiffSection section{DiffSection::Upload}; + /** @brief Sparse: a key selects its row and everything below it. */ + std::vector relKeys{}; + OperationMode mode{OperationMode::Queued}; + Ids::OperationId uploadId{}; + Ids::OperationId downloadId{}; + Ids::OperationId remoteDeleteId{}; + Ids::OperationId localDeleteId{}; + }; + BOOST_DESCRIBE_STRUCT( + SyncExecutionRequest, + (), + (syncSessionId, generation, section, relKeys, mode, uploadId, downloadId, remoteDeleteId, localDeleteId) + ) + + /** + * @brief Which rows each queued operation covers. + */ + struct SyncExecutionResult + { + /** @brief Rows of the bulk upload, in the order of its entry indices. */ + std::vector uploads{}; + /** @brief The source each upload row reports when it fails, in the same order. */ + std::vector uploadSources{}; + std::vector downloads{}; + std::vector downloadSources{}; + /** @brief Every listed item of a directory delete is a row of its own, its content after it. */ + std::vector remoteDeletes{}; + std::vector remoteDeleteSources{}; + std::vector localDeletes{}; + std::vector localDeleteSources{}; + /** @brief Directories without content, created right away. */ + std::vector createdDirectories{}; + /** @brief Rows that were refused or failed right away, nothing was queued for them. */ + std::vector failed{}; + }; + BOOST_DESCRIBE_STRUCT( + SyncExecutionResult, + (), + (uploads, + uploadSources, + downloads, + downloadSources, + remoteDeletes, + remoteDeleteSources, + localDeletes, + localDeleteSources, + createdDirectories, + failed) + ) + + inline void to_json(nlohmann::json& json, SyncExecutionRequest const& request) + { + SharedData::to_json(json, request); + } + inline void from_json(nlohmann::json const& json, SyncExecutionRequest& request) + { + SharedData::from_json(json, request); + } + inline void to_json(nlohmann::json& json, SyncExecutionResult const& result) + { + SharedData::to_json(json, result); + } + inline void from_json(nlohmann::json const& json, SyncExecutionResult& result) + { + SharedData::from_json(json, result); + } +} diff --git a/shared_data/include/shared_data/sync/tree_directory_walker.hpp b/shared_data/include/shared_data/sync/tree_directory_walker.hpp index d2d2e074..5537bf68 100644 --- a/shared_data/include/shared_data/sync/tree_directory_walker.hpp +++ b/shared_data/include/shared_data/sync/tree_directory_walker.hpp @@ -1,8 +1,10 @@ #pragma once #include +#include #include #include +#include #include #include @@ -10,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -20,7 +23,8 @@ namespace SharedData::Sync * @brief Directory walker that produces a sorted @ref ScanNode tree directly. * * Mirrors the scanner contract of @ref Utility::DeepDirectoryWalker — the caller - * supplies a scanner that returns a batch of @ref DirectoryEntry for one directory — + * supplies a scanner that returns a batch of @ref DirectoryEntry for one directory, given + * relative to the root — * but internally stitches results into a nested ScanNode tree with: * * - sibling sort by name applied per directory as entries arrive, and @@ -34,26 +38,40 @@ namespace SharedData::Sync requires std::is_invocable_r_v< std::expected, WalkErrorType>, ScannerT, - std::filesystem::path const&> + Paths::RelativePath const&> class TreeDirectoryWalker : public Utility::BaseDirectoryWalker { public: /** - * @param rootPath Directory the walk starts at. - * @param scanner Lists one directory. + * @param scanner Lists one directory, given relative to the root. * @param recursive When false only the root is listed. Its subdirectories stay in the tree with * @ref ScanNode::childrenKnown set to false so the diff never deletes what nobody saw. + * @param isUnreadable Classifies a scanner error of a directory below the root. A directory it + * accepts stays in the tree as @ref Availability::Unreadable and the walk goes on, every + * other error ends the walk. + * @param requireLocalNames Names must be creatable on the local filesystem, set for remote trees whose + * entries may be downloaded. Others are kept as @ref Availability::InvalidName. + * @param excludedSuffixes Names ending in one of these, the temp files of transfers, are kept as + * @ref Availability::Excluded so a sync neither copies nor deletes them. */ template requires std::is_same_v, ScannerT> - TreeDirectoryWalker(std::filesystem::path rootPath, ForwardingScannerT&& scanner, bool recursive = true) - : rootPath_{std::move(rootPath)} - , scanner_{std::forward(scanner)} + explicit TreeDirectoryWalker( + ForwardingScannerT&& scanner, + bool recursive = true, + std::function isUnreadable = {}, + bool requireLocalNames = false, + std::vector excludedSuffixes = {} + ) + : scanner_{std::forward(scanner)} , recursive_{recursive} + , requireLocalNames_{requireLocalNames} + , excludedSuffixes_{std::move(excludedSuffixes)} + , isUnreadable_{std::move(isUnreadable)} { root_.type = SharedData::FileType::Directory; root_.childrenKnown = true; - queue_.push_back(QueuedDir{.node = &root_, .fullPath = rootPath_, .depth = 0}); + queue_.push_back(QueuedDir{.node = &root_, .relativePath = {}, .depth = 0}); ++totalEntries_; } @@ -83,7 +101,7 @@ namespace SharedData::Sync root_.type = SharedData::FileType::Directory; root_.childrenKnown = true; queue_.clear(); - queue_.push_back(QueuedDir{.node = &root_, .fullPath = rootPath_, .depth = 0}); + queue_.push_back(QueuedDir{.node = &root_, .relativePath = {}, .depth = 0}); currentIndex_ = 0; totalBytes_ = 0; totalEntries_ = 1; @@ -100,18 +118,25 @@ namespace SharedData::Sync const auto pending = queue_.front(); queue_.pop_front(); - auto result = scanner_(pending.fullPath); + auto result = scanner_(pending.relativePath); if (!result) - return std::unexpected(std::move(result).error()); + { + const bool belowRoot = pending.depth > 0; + if (!belowRoot || !isUnreadable_ || !isUnreadable_(result.error())) + return std::unexpected(std::move(result).error()); + // Unknown content must never look like empty content, the diff would delete the other side. + pending.node->availability = Availability::Unreadable; + pending.node->childrenKnown = false; + ++currentIndex_; + return queue_.empty(); + } auto rawChildren = std::move(result).value(); - if constexpr (ScannerIncludesDotAndDotDot) - { - std::erase_if(rawChildren, [](SharedData::DirectoryEntry const& entry) { - return entry.path == "." || entry.path == ".."; - }); - } + // Every "." and "..", a server may send them more than once. + std::erase_if(rawChildren, [](SharedData::DirectoryEntry const& entry) { + return entry.name == "." || entry.name == ".."; + }); std::vector children; children.reserve(rawChildren.size()); @@ -120,9 +145,12 @@ namespace SharedData::Sync if (raw.type == SharedData::FileType::Regular) totalBytes_ += raw.size; + // The listed name as it is, "x/.." stays an invalid name. + std::string nameUtf8 = std::move(raw.name); + const auto availability = nameAvailability(nameUtf8, raw.type); children.push_back( ScanNode{ - .name = raw.path.filename().generic_string(), + .name = std::move(nameUtf8), .type = raw.type, .size = raw.size, .mtime = raw.mtime, @@ -135,6 +163,7 @@ namespace SharedData::Sync .subtreeFileCount = 0, .subtreeByteTotal = 0, .childrenKnown = recursive_ || raw.type != SharedData::FileType::Directory, + .availability = availability, } ); ++totalEntries_; @@ -154,12 +183,13 @@ namespace SharedData::Sync { for (auto& child : pending.node->children) { - if (child.type != SharedData::FileType::Directory) + if (child.type != SharedData::FileType::Directory || child.availability != Availability::Listed) continue; queue_.push_back( QueuedDir{ .node = &child, - .fullPath = pending.fullPath / child.name, + // Listed names are valid segments, see nameAvailability. + .relativePath = pending.relativePath / *Paths::PathSegment::make(child.name), .depth = pending.depth + 1, } ); @@ -192,6 +222,8 @@ namespace SharedData::Sync node.subtreeByteTotal = 0; for (auto& child : node.children) { + if (child.availability != Availability::Listed) + continue; if (child.type == SharedData::FileType::Directory) { finalizeCounts(child); @@ -206,16 +238,41 @@ namespace SharedData::Sync } } + static bool isTransferable(SharedData::FileType type) + { + return type == SharedData::FileType::Regular || type == SharedData::FileType::Directory || + type == SharedData::FileType::Symlink; + } + + Availability nameAvailability(std::string const& name, SharedData::FileType type) const + { + const auto segment = Paths::PathSegment::make(name); + if (!segment) + return Availability::InvalidName; + if (requireLocalNames_ && !Paths::LocalPathSegment::make(*segment).has_value()) + return Availability::InvalidName; + const auto isTempFile = std::ranges::any_of( + excludedSuffixes_, + [&name](std::string const& suffix) + { + return !suffix.empty() && name.size() > suffix.size() && name.ends_with(suffix); + } + ); + return isTransferable(type) && !isTempFile ? Availability::Listed : Availability::Excluded; + } + struct QueuedDir { ScanNode* node; - std::filesystem::path fullPath; + Paths::RelativePath relativePath; std::size_t depth; }; - std::filesystem::path rootPath_; ScannerT scanner_; bool recursive_{true}; + bool requireLocalNames_{false}; + std::vector excludedSuffixes_{}; + std::function isUnreadable_{}; ScanNode root_{}; std::deque queue_{}; std::size_t totalEntries_{0}; diff --git a/shared_data/include/shared_data/sync_phase.hpp b/shared_data/include/shared_data/sync_phase.hpp index 1f621c33..1f68cc4e 100644 --- a/shared_data/include/shared_data/sync_phase.hpp +++ b/shared_data/include/shared_data/sync_phase.hpp @@ -12,6 +12,7 @@ namespace SharedData Listing, Comparing, Hashing, - Done + Done, + Failed ) } diff --git a/shared_data/source/shared_data/CMakeLists.txt b/shared_data/source/shared_data/CMakeLists.txt index 79fa8526..5511a387 100644 --- a/shared_data/source/shared_data/CMakeLists.txt +++ b/shared_data/source/shared_data/CMakeLists.txt @@ -10,7 +10,7 @@ add_library( target_include_directories(shared-data PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../include" "${CMAKE_CURRENT_SOURCE_DIR}/../../../ssh/include") -target_link_libraries(shared-data PRIVATE core-target PUBLIC utility ids fmt::fmt) +target_link_libraries(shared-data PRIVATE core-target PUBLIC utility ids paths fmt::fmt) if (EMSCRIPTEN) target_link_libraries(shared-data PUBLIC nui-frontend) diff --git a/shared_data/source/shared_data/directory_entry.cpp b/shared_data/source/shared_data/directory_entry.cpp index 317a2708..61265b8b 100644 --- a/shared_data/source/shared_data/directory_entry.cpp +++ b/shared_data/source/shared_data/directory_entry.cpp @@ -1,29 +1,17 @@ #include -#include - namespace SharedData { - // Paths cross the RPC boundary as UTF-8. In-memory they are std::filesystem::path (native - // encoding: UTF-16 on Windows, UTF-8 on POSIX), so conversion happens here and nowhere else. - // Default nlohmann serialization of fs::path on MSVC uses path.string(), which goes through - // the active code page — that would corrupt any non-ASCII character, hence the explicit - // Utility::pathToUtf8Generic / pathFromUtf8 calls. - void to_json(nlohmann::json& j, DirectoryEntry const& entry) { j = nlohmann::json{ - {"path", Utility::pathToUtf8Generic(entry.path)}, + {"name", entry.name}, {"type", entry.type}, {"size", entry.size}, {"permissions", entry.permissions}, {"mtime", entry.mtime}, {"mtimeNsec", entry.mtimeNsec}, }; - if (!entry.fullPath.empty()) - j["fullPath"] = Utility::pathToUtf8Generic(entry.fullPath); - if (!entry.longName.empty()) - j["longName"] = Utility::pathToUtf8Generic(entry.longName); if (entry.flags != 0) j["flags"] = entry.flags; if (entry.uid != 0) @@ -45,25 +33,19 @@ namespace SharedData if (!entry.acl.empty()) j["acl"] = entry.acl; if (entry.linkTarget) - j["linkTarget"] = Utility::pathToUtf8Generic(*entry.linkTarget); + j["linkTarget"] = *entry.linkTarget; if (entry.resolvedTarget) j["resolvedTarget"] = *entry.resolvedTarget; } void from_json(nlohmann::json const& j, DirectoryEntry& entry) { - const auto getOptionalPath = [&](char const* key, std::filesystem::path& target) { - if (auto found = j.find(key); found != j.end() && !found->is_null()) - target = Utility::pathFromUtf8(found->get()); - }; const auto getOptional = [&](char const* key, auto& target) { if (auto found = j.find(key); found != j.end() && !found->is_null()) found->get_to(target); }; - entry.path = Utility::pathFromUtf8(j.at("path").get()); + j.at("name").get_to(entry.name); j.at("type").get_to(entry.type); - getOptionalPath("fullPath", entry.fullPath); - getOptionalPath("longName", entry.longName); getOptional("flags", entry.flags); getOptional("size", entry.size); getOptional("uid", entry.uid); @@ -79,7 +61,7 @@ namespace SharedData getOptional("mtimeNsec", entry.mtimeNsec); getOptional("acl", entry.acl); if (auto found = j.find("linkTarget"); found != j.end() && !found->is_null()) - entry.linkTarget = Utility::pathFromUtf8(found->get()); + entry.linkTarget = found->get(); if (auto found = j.find("resolvedTarget"); found != j.end() && !found->is_null()) { DirectoryEntry target{}; diff --git a/shared_data/source/shared_data/ignore_rules.cpp b/shared_data/source/shared_data/ignore_rules.cpp index 12e77b9b..7dcffaba 100644 --- a/shared_data/source/shared_data/ignore_rules.cpp +++ b/shared_data/source/shared_data/ignore_rules.cpp @@ -15,21 +15,6 @@ namespace SharedData return text; } - /** @brief Returns true if @p ancestor is an ancestor (or equal to) @p descendant. */ - bool isAncestorOrEqual(std::filesystem::path const& ancestor, std::filesystem::path const& descendant) - { - auto ancestorIter = ancestor.begin(); - auto descendantIter = descendant.begin(); - for (; ancestorIter != ancestor.end(); ++ancestorIter, ++descendantIter) - { - if (descendantIter == descendant.end()) - return false; - if (*ancestorIter != *descendantIter) - return false; - } - return true; - } - /** @brief True if pattern contains a '/' that is not the final char. */ bool patternIsAnchored(std::string_view pattern) { @@ -133,28 +118,26 @@ namespace SharedData return patterns; } - void IgnoreMatcher::addFile(std::filesystem::path const& relDir, std::string_view content) + void IgnoreMatcher::addFile(Paths::RelativePath const& relDir, std::string_view content) { auto patterns = parseIgnoreFile(content); if (patterns.empty()) return; - ruleSets_.push_back(IgnoreRuleSet{.originDir = relDir.lexically_normal(), .patterns = std::move(patterns)}); + ruleSets_.push_back(IgnoreRuleSet{.originDir = relDir, .patterns = std::move(patterns)}); } - bool IgnoreMatcher::isIgnored(std::filesystem::path const& relPath, bool isDirectory) const + bool IgnoreMatcher::isIgnored(Paths::RelativePath const& relPath, bool isDirectory) const { - const auto normalized = relPath.lexically_normal(); - bool ignored = false; for (auto const& ruleSet : ruleSets_) { - if (!isAncestorOrEqual(ruleSet.originDir, normalized)) + if (relPath == ruleSet.originDir || !relPath.isWithin(ruleSet.originDir)) continue; - auto relToOrigin = normalized.lexically_relative(ruleSet.originDir); - if (relToOrigin.empty() || relToOrigin == std::filesystem::path{"."}) - continue; - const auto relToOriginStr = relToOrigin.generic_string(); + const auto relToOriginStr = ruleSet.originDir.empty() + ? relPath.value() + : relPath.value().substr(ruleSet.originDir.value().size() + 1u); + const auto relToOrigin = *Paths::RelativePath::make(relToOriginStr); for (auto const& pattern : ruleSet.patterns) { @@ -168,9 +151,9 @@ namespace SharedData } else { - for (auto const& component : relToOrigin) + for (auto const& component : relToOrigin.segments()) { - if (globMatch(pattern.pattern, component.generic_string())) + if (globMatch(pattern.pattern, component.view())) { matched = true; break; diff --git a/shared_data/source/shared_data/sync/diff.cpp b/shared_data/source/shared_data/sync/diff.cpp index b5c2350c..fe5a7907 100644 --- a/shared_data/source/shared_data/sync/diff.cpp +++ b/shared_data/source/shared_data/sync/diff.cpp @@ -1,9 +1,13 @@ #include #include +#include #include #include #include +#include +#include +#include #include #include @@ -146,6 +150,42 @@ namespace SharedData::Sync std::uint64_t& comparedCounter ); + std::optional blockReasonOf(ScanNode const& node) + { + switch (node.availability) + { + case Availability::Listed: + return std::nullopt; + case Availability::Unreadable: + return BlockReason::Unreadable; + case Availability::Excluded: + return BlockReason::Excluded; + case Availability::InvalidName: + return BlockReason::InvalidName; + } + return BlockReason::Excluded; + } + + /** + * @brief The latest modification time in @p node and everything listed below it. + */ + std::uint64_t newestModification(ScanNode const& node) + { + auto newest = node.mtime; + for (auto const& child : node.children) + newest = std::max(newest, newestModification(child)); + return newest; + } + + /** + * @brief A delete must not remove what appeared or changed on that side since the last sync, nobody has + * seen it on the other side. + */ + bool changedSinceLastSync(ScanNode const& node, DiffOptions const& options) + { + return options.lastSyncTime && newestModification(node) > *options.lastSyncTime; + } + void handleBothPresent( ScanNode const& localChild, ScanNode const& remoteChild, @@ -156,6 +196,25 @@ namespace SharedData::Sync std::uint64_t& comparedCounter ) { + // Unknown or untransferable on one side: neither side may be changed for this name. + if (auto reason = blockReasonOf(localChild).or_else([&]() { return blockReasonOf(remoteChild); }); reason) + { + sink.onBlocked(relKey, *reason); + return; + } + + // Transferring a directory onto a symlink writes through the link, out of the sync root. + const auto isPair = [&](FileType first, FileType second) + { + return (localChild.type == first && remoteChild.type == second) || + (localChild.type == second && remoteChild.type == first); + }; + if (isPair(FileType::Directory, FileType::Symlink)) + { + sink.onBlocked(relKey, BlockReason::TypeConflict); + return; + } + const bool sameDir = localChild.type == FileType::Directory && remoteChild.type == FileType::Directory; if (sameDir) @@ -168,6 +227,25 @@ namespace SharedData::Sync if (!entriesDiffer(localChild, remoteChild)) return; + // With a known last sync, the side changed since then is the one that holds new data. + if (options.lastSyncTime) + { + const bool localChanged = localChild.mtime > *options.lastSyncTime; + const bool remoteChanged = remoteChild.mtime > *options.lastSyncTime; + const bool destinationChanged = (options.direction == Direction::Upload && remoteChanged) || + (options.direction == Direction::Download && localChanged); + if (options.direction == Direction::Both && localChanged && remoteChanged) + { + sink.onBlocked(relKey, BlockReason::ChangedOnBothSides); + return; + } + if (destinationChanged) + { + sink.onBlocked(relKey, BlockReason::ChangedOnDestination); + return; + } + } + const auto action = chooseTwoSidedAction(localChild, remoteChild, options); if (!action) return; @@ -178,6 +256,77 @@ namespace SharedData::Sync sink.emitDownload(std::move(row), parentRelKey); } + std::string foldedCase(std::string_view name) + { + std::string folded{name}; + for (auto& character : folded) + { + if (character >= 'A' && character <= 'Z') + character = static_cast(character - 'A' + 'a'); + } + return folded; + } + + /** + * @brief Names of one directory that would meet as one file on a case folding filesystem: a local-only and a + * remote-only name that differ only in case, and remote names that differ only in case when the local + * filesystem folds case. Transferring either could overwrite the other and a delete could hit both. + * ASCII letters only. + */ + std::unordered_set caseConflictsOf( + std::vector const& localChildren, + std::vector const& remoteChildren, + DiffOptions const& options + ) + { + const auto hasUpper = [](std::vector const& children) + { + return std::ranges::any_of( + children, + [](ScanNode const& node) + { + return std::ranges::any_of(node.name, [](char character) { return character >= 'A' && character <= 'Z'; }); + } + ); + }; + // Without an upper case letter folding changes nothing, which is the common case. + if (!hasUpper(localChildren) && !hasUpper(remoteChildren)) + return {}; + + struct Variants + { + std::vector local; + std::vector remote; + }; + std::unordered_map byFolded; + for (auto const& node : localChildren) + byFolded[foldedCase(node.name)].local.push_back(node.name); + for (auto const& node : remoteChildren) + byFolded[foldedCase(node.name)].remote.push_back(node.name); + + std::unordered_set conflicts; + for (auto const& [folded, variants] : byFolded) + { + const bool crossSide = std::ranges::any_of( + variants.local, + [&variants](std::string_view local) + { + return std::ranges::any_of( + variants.remote, [local](std::string_view remote) { return remote != local; } + ); + } + ); + const bool remoteVariants = options.localCaseInsensitive && variants.remote.size() > 1; + if (!crossSide && !remoteVariants) + continue; + for (auto const name : variants.local) + conflicts.emplace(name); + for (auto const name : variants.remote) + conflicts.emplace(name); + } + return conflicts; + } + void mergeChildren( std::vector const& localChildren, std::vector const& remoteChildren, @@ -189,6 +338,14 @@ namespace SharedData::Sync { std::size_t li = 0; std::size_t ri = 0; + const auto caseConflicts = caseConflictsOf(localChildren, remoteChildren, options); + const auto blockedForCase = [&](std::string const& name) + { + if (!caseConflicts.contains(name)) + return false; + sink.onBlocked(joinRel(parentRelKey, name), BlockReason::CaseConflict); + return true; + }; const auto bumpAndCheckpoint = [&]() { ++comparedCounter; @@ -213,7 +370,7 @@ namespace SharedData::Sync { auto const& lc = localChildren[li]; auto const& rc = remoteChildren[ri]; - if (!options.ignoreHidden || !isHiddenName(lc.name)) + if ((!options.ignoreHidden || !isHiddenName(lc.name)) && !blockedForCase(lc.name)) { const auto relKey = joinRel(parentRelKey, lc.name); handleBothPresent(lc, rc, relKey, parentRelKey, options, sink, comparedCounter); @@ -225,11 +382,16 @@ namespace SharedData::Sync else if (takeLeft) { auto const& lc = localChildren[li]; - if (!options.ignoreHidden || !isHiddenName(lc.name)) + if ((!options.ignoreHidden || !isHiddenName(lc.name)) && !blockedForCase(lc.name)) { - if (auto action = chooseLocalOnlyAction(lc, options); action) + if (auto reason = blockReasonOf(lc); reason) + sink.onBlocked(joinRel(parentRelKey, lc.name), *reason); + else if (auto action = chooseLocalOnlyAction(lc, options); action) { const auto relKey = joinRel(parentRelKey, lc.name); + if (*action == Action::DeleteLocal && changedSinceLastSync(lc, options)) + sink.onBlocked(relKey, BlockReason::ChangedOnDestination); + else emitOneSided( makeOneSidedRow(lc, relKey, *action, true), *action, parentRelKey, options, sink ); @@ -241,11 +403,16 @@ namespace SharedData::Sync else if (takeRight) { auto const& rc = remoteChildren[ri]; - if (!options.ignoreHidden || !isHiddenName(rc.name)) + if ((!options.ignoreHidden || !isHiddenName(rc.name)) && !blockedForCase(rc.name)) { - if (auto action = chooseRemoteOnlyAction(rc, options); action) + if (auto reason = blockReasonOf(rc); reason) + sink.onBlocked(joinRel(parentRelKey, rc.name), *reason); + else if (auto action = chooseRemoteOnlyAction(rc, options); action) { const auto relKey = joinRel(parentRelKey, rc.name); + if (*action == Action::DeleteRemote && changedSinceLastSync(rc, options)) + sink.onBlocked(relKey, BlockReason::ChangedOnDestination); + else emitOneSided( makeOneSidedRow(rc, relKey, *action, false), *action, parentRelKey, options, sink ); @@ -288,6 +455,26 @@ namespace SharedData::Sync return !name.empty() && name.front() == '.'; } + /** + * @brief Link literals compare with repeated separators collapsed, "a//b" points where "a/b" does. + */ + bool linkTargetsDiffer(std::string_view local, std::string_view remote) + { + const auto collapsed = [](std::string_view text) + { + std::string result; + result.reserve(text.size()); + for (auto const character : text) + { + if (character == '/' && !result.empty() && result.back() == '/') + continue; + result.push_back(character); + } + return result; + }; + return collapsed(local) != collapsed(remote); + } + bool entriesDiffer(ScanNode const& localNode, ScanNode const& remoteNode) { if (localNode.type != remoteNode.type) @@ -295,7 +482,7 @@ namespace SharedData::Sync if (localNode.type == FileType::Symlink) { if (localNode.linkTarget && remoteNode.linkTarget) - return *localNode.linkTarget != *remoteNode.linkTarget; + return linkTargetsDiffer(*localNode.linkTarget, *remoteNode.linkTarget); return false; } if (localNode.size != remoteNode.size) diff --git a/shared_data/source/shared_data/sync/diff_tree_store.cpp b/shared_data/source/shared_data/sync/diff_tree_store.cpp index e31b38f7..1962649d 100644 --- a/shared_data/source/shared_data/sync/diff_tree_store.cpp +++ b/shared_data/source/shared_data/sync/diff_tree_store.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include @@ -192,6 +194,8 @@ namespace SharedData::Sync { if (options_.ignoreHidden && isHiddenName(child.name)) continue; + if (child.availability != Availability::Listed) + continue; auto childKey = joinRelKey(parentRelKey, child.name); nodes_.insert_or_assign(childKey, makeOneSidedRow(child, childKey, parent.action, sourceIsLocal)); order_.push_back(childKey); @@ -213,8 +217,6 @@ namespace SharedData::Sync return EnqueuePlanEntry{ .relKey = key, .action = action, - .localAbsPath = (trees.localRoot / key).generic_string(), - .remoteAbsPath = (trees.remoteRoot / key).generic_string(), .sizeBytes = node.type == FileType::Directory ? 0u : node.size, .mtime = node.mtime, .mtimeNsec = node.mtimeNsec, @@ -222,11 +224,34 @@ namespace SharedData::Sync }; }; + // Downloaded onto a case folding filesystem, names that differ only in case would meet as one file. + std::unordered_map foldedCounts; + const bool foldsCase = action == Action::Download && options_.localCaseInsensitive; + const auto folded = [](std::string name) + { + for (auto& character : name) + { + if (character >= 'A' && character <= 'Z') + character = static_cast(character - 'A' + 'a'); + } + return name; + }; + if (foldsCase) + { + for (auto const& child : directory.children) + ++foldedCounts[folded(child.name)]; + } + bool anyChild = false; for (auto const& child : directory.children) { if (options_.ignoreHidden && isHiddenName(child.name)) continue; + // Unknown or untransferable entries are never planned, not even inside a transferred directory. + if (child.availability != Availability::Listed) + continue; + if (foldsCase && foldedCounts[folded(child.name)] > 1) + continue; anyChild = true; auto childKey = joinRelKey(relKey, child.name); if (child.type == FileType::Directory) @@ -239,6 +264,46 @@ namespace SharedData::Sync plan.push_back(makeEntry(directory, relKey)); } + void DiffTreeStore::appendDeletePlan( + ScanNode const& directory, + std::string const& relKey, + Action action, + std::vector& plan + ) const + { + plan.push_back( + EnqueuePlanEntry{ + .relKey = relKey, + .action = action, + .mtime = directory.mtime, + .mtimeNsec = directory.mtimeNsec, + .isDirectory = true, + } + ); + for (auto const& child : directory.children) + { + if (options_.ignoreHidden && isHiddenName(child.name)) + continue; + if (child.availability != Availability::Listed) + continue; + auto childKey = joinRelKey(relKey, child.name); + if (child.type == FileType::Directory) + { + appendDeletePlan(child, childKey, action, plan); + continue; + } + plan.push_back( + EnqueuePlanEntry{ + .relKey = std::move(childKey), + .action = action, + .sizeBytes = child.size, + .mtime = child.mtime, + .mtimeNsec = child.mtimeNsec, + } + ); + } + } + std::vector DiffTreeStore::buildPlan(std::unordered_set const& selectedRelKeys, ScanTrees const& trees) const { @@ -263,9 +328,12 @@ namespace SharedData::Sync auto const* scan = sourceRoot ? findScanNode(*sourceRoot, node.relKey) : nullptr; const bool isDelete = node.action == Action::DeleteLocal || node.action == Action::DeleteRemote; - if (scan && node.isDirectory && !isDelete && options_.recursive && scan->type == FileType::Directory) + if (scan && node.isDirectory && options_.recursive && scan->type == FileType::Directory) { - appendSubtreePlan(*scan, node.relKey, node.action, trees, plan); + if (isDelete) + appendDeletePlan(*scan, node.relKey, node.action, plan); + else + appendSubtreePlan(*scan, node.relKey, node.action, trees, plan); continue; } @@ -273,8 +341,6 @@ namespace SharedData::Sync EnqueuePlanEntry{ .relKey = node.relKey, .action = node.action, - .localAbsPath = (trees.localRoot / node.relKey).generic_string(), - .remoteAbsPath = (trees.remoteRoot / node.relKey).generic_string(), .sizeBytes = node.isDirectory ? 0u : (scan ? scan->size : (sourceIsLocal ? node.localSize : node.remoteSize)), @@ -316,6 +382,13 @@ namespace SharedData::Sync deletes_->add(std::move(node), parentRelKey); } + void StoreDiffSink::onBlocked(std::string const& relKey, BlockReason reason) + { + ++blockedCount_; + if (blockedSamples_.size() < maxBlockedSamples) + blockedSamples_.push_back(BlockedEntry{.relKey = relKey, .reason = reason}); + } + void StoreDiffSink::onProgress(std::uint64_t entriesCompared) { lastCompared_ = entriesCompared; diff --git a/shared_data/test/shared_data/main.cpp b/shared_data/test/shared_data/main.cpp index 4e35cb03..e83708e2 100644 --- a/shared_data/test/shared_data/main.cpp +++ b/shared_data/test/shared_data/main.cpp @@ -1,3 +1,6 @@ +#include "test_bulk_delete_request.hpp" +#include "test_bulk_transfer_request.hpp" +#include "test_operation_added.hpp" #include "test_diff.hpp" #include "test_diff_tree_store.hpp" #include "test_enqueue_minimizer.hpp" diff --git a/shared_data/test/shared_data/test_bulk_delete_request.hpp b/shared_data/test/shared_data/test_bulk_delete_request.hpp new file mode 100644 index 00000000..98aab0e0 --- /dev/null +++ b/shared_data/test/shared_data/test_bulk_delete_request.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include +#include + +#include + +namespace SharedData::Test +{ + TEST(BulkDeleteRequestTests, RoundTripsThroughJson) + { + const BulkDeleteRequest request{ + .entries = + { + {.target = Paths::RemotePath::make("/home/test/a\\b").value(), .isDirectory = false, .sizeBytes = 3}, + {.target = Paths::RemotePath::make("/home/test/dir").value(), .isDirectory = true}, + }, + .insertRefresh = true, + .mode = OperationMode::PriorityQueued, + }; + const nlohmann::json json = request; + const auto read = json.get(); + ASSERT_EQ(read.entries.size(), 2u); + EXPECT_EQ(read.entries[0].target.value(), "/home/test/a\\b"); + EXPECT_EQ(read.entries[0].sizeBytes, 3u); + EXPECT_TRUE(read.entries[1].isDirectory); + EXPECT_TRUE(read.insertRefresh); + EXPECT_EQ(read.mode, OperationMode::PriorityQueued); + } + + TEST(BulkDeleteRequestTests, RelativeOrParentTargetsAreRefusedOnRead) + { + for (auto const* target : {"home/test/a", "/home/test/../etc", ""}) + { + nlohmann::json json = BulkDeleteRequest{.entries = {{.target = Paths::RemotePath::root()}}}; + json["entries"][0]["target"] = target; + EXPECT_THROW(json.get(), std::exception) << target; + } + } +} diff --git a/shared_data/test/shared_data/test_bulk_transfer_request.hpp b/shared_data/test/shared_data/test_bulk_transfer_request.hpp new file mode 100644 index 00000000..77deef0a --- /dev/null +++ b/shared_data/test/shared_data/test_bulk_transfer_request.hpp @@ -0,0 +1,106 @@ +#pragma once + +#include + +#include +#include + +#include +#include + +namespace SharedData::Test +{ + namespace + { +#ifdef _WIN32 + constexpr char const* localDirectory = "C:/Users/test"; +#else + constexpr char const* localDirectory = "/home/test"; +#endif + + Paths::LocalPath local(std::string const& below) + { + return Paths::LocalPath::fromUtf8(std::string{localDirectory} + "/" + below).value(); + } + + Paths::RemotePath remote(std::string const& path) + { + return Paths::RemotePath::make(path).value(); + } + } + + TEST(BulkTransferRequestTests, UploadRoundTripsThroughJson) + { + const BulkUploadRequest request{ + .entries = + { + {.source = local("a.txt"), .destination = remote("/srv/a\\b"), .sizeBytes = 3}, + {.source = local("dir"), .destination = remote("/srv/dir"), .isDirectory = true}, + }, + .allowOverwrite = true, + .insertRefresh = true, + .mode = OperationMode::PriorityQueued, + }; + const nlohmann::json json = request; + const auto read = json.get(); + ASSERT_EQ(read.entries.size(), 2u); + EXPECT_EQ(read.entries[0].source, local("a.txt")); + EXPECT_EQ(read.entries[0].destination.value(), "/srv/a\\b"); + EXPECT_EQ(read.entries[0].sizeBytes, 3u); + EXPECT_TRUE(read.entries[1].isDirectory); + EXPECT_TRUE(read.allowOverwrite); + EXPECT_TRUE(read.insertRefresh); + EXPECT_EQ(read.mode, OperationMode::PriorityQueued); + } + + TEST(BulkTransferRequestTests, DownloadRoundTripsThroughJson) + { + const BulkDownloadRequest request{ + .entries = + { + {.source = remote("/srv/a.txt"), + .destination = local("a.txt"), + .sizeBytes = 7, + .mtime = 1700000000u, + .mtimeNsec = 5u}, + }, + }; + const nlohmann::json json = request; + const auto read = json.get(); + ASSERT_EQ(read.entries.size(), 1u); + EXPECT_EQ(read.entries[0].source.value(), "/srv/a.txt"); + EXPECT_EQ(read.entries[0].destination, local("a.txt")); + EXPECT_EQ(read.entries[0].sizeBytes, 7u); + EXPECT_EQ(read.entries[0].mtime, 1700000000u); + EXPECT_EQ(read.entries[0].mtimeNsec, 5u); + } + + TEST(BulkTransferRequestTests, InvalidPathsAreRefusedOnRead) + { + for (auto const* remotePath : {"srv/a", "/srv/../etc", ""}) + { + nlohmann::json upload = + BulkUploadRequest{.entries = {{.source = local("a"), .destination = remote("/srv/a")}}}; + upload["entries"][0]["destination"] = remotePath; + EXPECT_THROW(upload.get(), std::exception) << remotePath; + + nlohmann::json download = + BulkDownloadRequest{.entries = {{.source = remote("/srv/a"), .destination = local("a")}}}; + download["entries"][0]["source"] = remotePath; + EXPECT_THROW(download.get(), std::exception) << remotePath; + } + + for (auto const* localPath : {"relative/a", ""}) + { + nlohmann::json upload = + BulkUploadRequest{.entries = {{.source = local("a"), .destination = remote("/srv/a")}}}; + upload["entries"][0]["source"] = localPath; + EXPECT_THROW(upload.get(), std::exception) << localPath; + + nlohmann::json download = + BulkDownloadRequest{.entries = {{.source = remote("/srv/a"), .destination = local("a")}}}; + download["entries"][0]["destination"] = localPath; + EXPECT_THROW(download.get(), std::exception) << localPath; + } + } +} diff --git a/shared_data/test/shared_data/test_diff.hpp b/shared_data/test/shared_data/test_diff.hpp index aa109b64..d24557cc 100644 --- a/shared_data/test/shared_data/test_diff.hpp +++ b/shared_data/test/shared_data/test_diff.hpp @@ -28,6 +28,7 @@ namespace SharedData::Sync::Test std::vector uploads; std::vector downloads; std::vector deletes; + std::vector blocked; std::uint64_t lastProgress{0}; void emitUpload(DiffTreeNode node, std::string const& parentRelKey) override @@ -42,12 +43,41 @@ namespace SharedData::Sync::Test { deletes.push_back(Record{.node = std::move(node), .parentRelKey = parentRelKey}); } + void onBlocked(std::string const& relKey, BlockReason reason) override + { + blocked.push_back(BlockedEntry{.relKey = relKey, .reason = reason}); + } void onProgress(std::uint64_t entriesCompared) override { lastProgress = entriesCompared; } }; + /** + * @brief Builds a directory whose listing failed: its content is unknown. + */ + inline ScanNode makeUnreadableDir(std::string const& name) + { + return ScanNode{ + .name = name, + .type = FileType::Directory, + .childrenKnown = false, + .availability = Availability::Unreadable, + }; + } + + /** + * @brief Builds an entry the scan keeps but never transfers, like a FIFO. + */ + inline ScanNode makeExcluded(std::string const& name, FileType type = FileType::Fifo) + { + return ScanNode{ + .name = name, + .type = type, + .availability = Availability::Excluded, + }; + } + inline bool hasRelKey(std::vector const& records, std::string const& relKey) { return std::any_of(records.begin(), records.end(), [&](CollectingSink::Record const& record) { @@ -80,7 +110,7 @@ namespace SharedData::Sync::Test /** * @brief Builds a symlink node with a given raw link target. */ - inline ScanNode makeSymlink(std::string const& name, std::filesystem::path const& target, std::uint64_t mtime = 0) + inline ScanNode makeSymlink(std::string const& name, std::string const& target, std::uint64_t mtime = 0) { return ScanNode{ .name = name, @@ -634,6 +664,12 @@ namespace SharedData::Sync::Test EXPECT_FALSE(entriesDiffer(localLink, remoteLink)); } + TEST(SyncDiffTests, SymlinkTargetsDifferingOnlyInRepeatedSeparatorsDoNotDiffer) + { + EXPECT_FALSE(entriesDiffer(makeSymlink("link", "../a//b"), makeSymlink("link", "../a/b"))); + EXPECT_TRUE(entriesDiffer(makeSymlink("link", "a/b"), makeSymlink("link", "a/b/c"))); + } + TEST(SyncDiffTests, DirectoryAgainstSymlinkIsATypeConflict) { EXPECT_TRUE(entriesDiffer(makeDir("name", {}), makeSymlink("name", "/elsewhere"))); @@ -759,4 +795,202 @@ namespace SharedData::Sync::Test EXPECT_EQ(row.descendantItemCount, 0u); EXPECT_EQ(row.descendantByteTotal, 0u); } + + TEST(SyncDiffTests, WithoutALastSyncTheNewestSideWins) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("a.txt", 1, 300)}), + makeDir("", {makeFile("a.txt", 2, 200)}), + {.direction = Direction::Both}, + sink + ); + EXPECT_EQ(sink.uploads.size(), 1u); + EXPECT_TRUE(sink.blocked.empty()); + } + + TEST(SyncDiffTests, BothSidesChangedSinceTheLastSyncIsBlocked) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("both.txt", 1, 300), makeFile("local.txt", 1, 300)}), + makeDir("", {makeFile("both.txt", 2, 250), makeFile("local.txt", 2, 50)}), + {.direction = Direction::Both, .lastSyncTime = 100}, + sink + ); + ASSERT_EQ(sink.blocked.size(), 1u); + EXPECT_EQ(sink.blocked.front().relKey, "both.txt"); + EXPECT_EQ(sink.blocked.front().reason, BlockReason::ChangedOnBothSides); + ASSERT_EQ(sink.uploads.size(), 1u) << "only the local side changed, it wins"; + EXPECT_EQ(sink.uploads.front().node.relKey, "local.txt"); + } + + TEST(SyncDiffTests, UploadNeverReplacesWhatChangedOnTheServerSinceTheLastSync) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("a.txt", 1, 50), makeFile("b.txt", 1, 300)}), + makeDir("", {makeFile("a.txt", 2, 200), makeFile("b.txt", 2, 50)}), + {.direction = Direction::Upload, .lastSyncTime = 100}, + sink + ); + ASSERT_EQ(sink.blocked.size(), 1u); + EXPECT_EQ(sink.blocked.front().relKey, "a.txt"); + EXPECT_EQ(sink.blocked.front().reason, BlockReason::ChangedOnDestination); + ASSERT_EQ(sink.uploads.size(), 1u); + EXPECT_EQ(sink.uploads.front().node.relKey, "b.txt"); + } + + TEST(SyncDiffTests, DeleteNeverRemovesWhatAppearedSinceTheLastSync) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {}), + makeDir( + "", + {makeFile("old.txt", 1, 50), + makeFile("new.txt", 1, 200), + makeDir("dir", {makeFile("old", 1, 50), makeFile("fresh", 1, 200)})} + ), + {.direction = Direction::Upload, .actionDelete = true, .lastSyncTime = 100}, + sink + ); + ASSERT_EQ(sink.deletes.size(), 1u); + EXPECT_EQ(sink.deletes.front().node.relKey, "old.txt"); + EXPECT_EQ(sink.blocked.size(), 2u) << "new.txt and the directory holding a new file"; + } + + TEST(SyncDiffTests, NamesDifferingOnlyInCaseAcrossSidesAreBlocked) + { + // On a case insensitive server "Foo" replaces "foo", the delete of "foo" would then remove the upload. + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("Foo", 3, 200), makeFile("other", 1, 100)}), + makeDir("", {makeFile("foo", 3, 100)}), + {.direction = Direction::Upload, .actionDelete = true}, + sink + ); + ASSERT_EQ(sink.uploads.size(), 1u); + EXPECT_EQ(sink.uploads.front().node.relKey, "other"); + EXPECT_TRUE(sink.deletes.empty()); + ASSERT_EQ(sink.blocked.size(), 2u); + for (auto const& blocked : sink.blocked) + EXPECT_EQ(blocked.reason, BlockReason::CaseConflict) << blocked.relKey; + } + + TEST(SyncDiffTests, RemoteCaseVariantsAreBlockedOnlyForACaseFoldingLocalFilesystem) + { + const auto local = makeDir("", {}); + const auto remote = makeDir("", {makeFile("Readme", 1, 100), makeFile("readme", 1, 100)}); + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Download}, sink); + EXPECT_EQ(sink.downloads.size(), 2u) << "a case sensitive filesystem stores both"; + EXPECT_TRUE(sink.blocked.empty()); + } + { + CollectingSink sink; + diffScanTrees(local, remote, {.direction = Direction::Download, .localCaseInsensitive = true}, sink); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_EQ(sink.blocked.size(), 2u); + } + } + + TEST(SyncDiffTests, UnreadableLocalDirectoryNeverDeletesItsRemoteContent) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeUnreadableDir("data")}), + makeDir("", {makeDir("data", {makeFile("a.txt", 1, 100), makeDir("deep", {makeFile("b", 1, 100)})})}), + {.direction = Direction::Upload, .actionDelete = true}, + sink + ); + EXPECT_TRUE(sink.deletes.empty()); + EXPECT_TRUE(sink.uploads.empty()); + ASSERT_EQ(sink.blocked.size(), 1u); + EXPECT_EQ(sink.blocked.front().relKey, "data"); + EXPECT_EQ(sink.blocked.front().reason, BlockReason::Unreadable); + } + + TEST(SyncDiffTests, UnreadableRemoteDirectoryNeverDeletesItsLocalContent) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeDir("data", {makeFile("a.txt", 1, 100)})}), + makeDir("", {makeUnreadableDir("data")}), + {.direction = Direction::Download, .actionDelete = true}, + sink + ); + EXPECT_TRUE(sink.deletes.empty()); + EXPECT_TRUE(sink.downloads.empty()); + ASSERT_EQ(sink.blocked.size(), 1u); + EXPECT_EQ(sink.blocked.front().reason, BlockReason::Unreadable); + } + + TEST(SyncDiffTests, OneSidedUnreadableDirectoryIsNeitherTransferredNorDeleted) + { + for (const auto direction : {Direction::Upload, Direction::Download, Direction::Both}) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeUnreadableDir("local-only")}), + makeDir("", {makeUnreadableDir("remote-only")}), + {.direction = direction, .actionDelete = true}, + sink + ); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_TRUE(sink.deletes.empty()); + EXPECT_EQ(sink.blocked.size(), 2u); + } + } + + TEST(SyncDiffTests, ExcludedLocalEntryKeepsTheRemoteDirectoryOfTheSameName) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeExcluded("assets", FileType::Unknown)}), + makeDir("", {makeDir("assets", {makeFile("logo.png", 10, 100)})}), + {.direction = Direction::Upload, .actionDelete = true}, + sink + ); + EXPECT_TRUE(sink.deletes.empty()); + EXPECT_TRUE(sink.uploads.empty()); + ASSERT_EQ(sink.blocked.size(), 1u); + EXPECT_EQ(sink.blocked.front().reason, BlockReason::Excluded); + } + + TEST(SyncDiffTests, OneSidedExcludedEntryIsNotTransferred) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeExcluded("pipe")}), + makeDir("", {makeExcluded("socket", FileType::Socket)}), + {.direction = Direction::Both}, + sink + ); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_EQ(sink.blocked.size(), 2u); + } + + TEST(SyncDiffTests, DirectoryAgainstSymlinkIsBlockedInBothDirections) + { + for (const auto direction : {Direction::Upload, Direction::Download, Direction::Both}) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeSymlink("x", "/etc", 200), makeDir("y", {makeFile("a", 1, 100)})}), + makeDir("", {makeDir("x", {makeFile("passwd", 1, 100)}), makeSymlink("y", "/root", 200)}), + {.direction = direction, .actionDelete = true}, + sink + ); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_TRUE(sink.deletes.empty()); + ASSERT_EQ(sink.blocked.size(), 2u); + EXPECT_EQ(sink.blocked[0].reason, BlockReason::TypeConflict); + EXPECT_EQ(sink.blocked[1].reason, BlockReason::TypeConflict); + } + } } diff --git a/shared_data/test/shared_data/test_diff_tree_store.hpp b/shared_data/test/shared_data/test_diff_tree_store.hpp index 4d754cd0..df25dc31 100644 --- a/shared_data/test/shared_data/test_diff_tree_store.hpp +++ b/shared_data/test/shared_data/test_diff_tree_store.hpp @@ -41,7 +41,7 @@ namespace SharedData::Sync::Test ScanTrees trees() const { - return ScanTrees{.local = &local, .remote = &remote, .localRoot = "/l", .remoteRoot = "/r"}; + return ScanTrees{.local = &local, .remote = &remote}; } }; @@ -203,8 +203,7 @@ namespace SharedData::Sync::Test EXPECT_EQ(plan[0].sizeBytes, 16u); EXPECT_EQ(plan[0].mtime, 2000u); EXPECT_EQ(plan[0].mtimeNsec, 42u); - EXPECT_EQ(plan[0].localAbsPath, "/l/socket.cpp"); - EXPECT_EQ(plan[0].remoteAbsPath, "/r/socket.cpp"); + EXPECT_EQ(plan[0].relKey, "socket.cpp"); } TEST(DiffTreeStoreTests, PlanForANewerLocalFileCarriesLocalSizeAndMtime) @@ -297,18 +296,50 @@ namespace SharedData::Sync::Test EXPECT_TRUE(plan[0].isDirectory); } - TEST(DiffTreeStoreTests, PlanKeepsDeletedDirectoriesWhole) + TEST(DiffTreeStoreTests, PlanListsADeletedDirectoryBeforeItsContent) { DiffStores stores{ makeDir("", {}), - makeDir("", {makeDir("gone", {makeFile("a", 1, 100), makeFile("b", 1, 100)})}), + makeDir("", {makeDir("gone", {makeFile("a", 1, 100), makeDir("sub", {makeFile("b", 1, 100)})})}), {.direction = Direction::Upload, .actionDelete = true}, }; const auto plan = stores.deletes.buildPlan({"gone"}, stores.trees()); - ASSERT_EQ(plan.size(), 1u); - EXPECT_EQ(plan[0].relKey, "gone"); + EXPECT_EQ(relKeys(plan), (std::vector{"gone", "gone/a", "gone/sub", "gone/sub/b"})); + for (auto const& entry : plan) + EXPECT_EQ(entry.action, Action::DeleteRemote) << entry.relKey; EXPECT_TRUE(plan[0].isDirectory); - EXPECT_EQ(plan[0].action, Action::DeleteRemote); + EXPECT_FALSE(plan[1].isDirectory); + EXPECT_TRUE(plan[2].isDirectory); + } + + TEST(DiffTreeStoreTests, DeletePlanLeavesOutWhatTheScanDidNotList) + { + auto unreadable = makeDir("unreadable", {}); + unreadable.availability = Availability::Unreadable; + unreadable.childrenKnown = false; + auto fifo = makeFile("fifo", 0, 100); + fifo.availability = Availability::Excluded; + DiffStores stores{ + makeDir("", {}), + makeDir( + "", + {makeDir("gone", {makeFile(".hidden", 1, 100), makeFile("a", 1, 100), fifo, unreadable})} + ), + {.direction = Direction::Upload, .actionDelete = true, .ignoreHidden = true}, + }; + EXPECT_EQ( + relKeys(stores.deletes.buildPlan({"gone"}, stores.trees())), (std::vector{"gone", "gone/a"}) + ); + } + + TEST(DiffTreeStoreTests, DownloadPlanSkipsCaseVariantsForACaseFoldingLocalFilesystem) + { + DiffStores stores{ + makeDir("", {}), + makeDir("", {makeDir("new", {makeFile("A.txt", 1, 100), makeFile("a.txt", 1, 100), makeFile("b", 1, 100)})}), + {.direction = Direction::Download, .localCaseInsensitive = true}, + }; + EXPECT_EQ(relKeys(stores.downloads.buildPlan({"new"}, stores.trees())), (std::vector{"new/b"})); } TEST(DiffTreeStoreTests, PlanForAPartlySelectedExpandedDirectory) @@ -388,4 +419,33 @@ namespace SharedData::Sync::Test EXPECT_TRUE(plan[0].isDirectory); EXPECT_EQ(stores.uploads.summary().itemCount, 1u); } + + TEST(DiffTreeStoreTests, PlansAndRowsSkipUnknownAndExcludedEntriesInsideTransferredDirectories) + { + DiffStores stores{ + makeDir("", {makeDir("new", {makeFile("a", 1, 100), makeExcluded("pipe"), makeUnreadableDir("secret")})}), + makeDir("", {}), + }; + + EXPECT_EQ(relKeys(stores.uploads.loadChildren("new", stores.trees())), (std::vector{"new/a"})); + EXPECT_EQ(relKeys(stores.uploads.buildPlan({"new"}, stores.trees())), (std::vector{"new/a"})); + } + + TEST(DiffTreeStoreTests, BlockedEntriesAreCountedWithSamples) + { + DiffTreeStore uploads; + DiffTreeStore downloads; + DiffTreeStore deletes; + StoreDiffSink sink{uploads, downloads, deletes}; + diffScanTrees( + makeDir("", {makeUnreadableDir("a"), makeExcluded("b")}), + makeDir("", {makeDir("a", {makeFile("x", 1, 100)})}), + {.direction = Direction::Upload, .actionDelete = true}, + sink + ); + EXPECT_EQ(sink.blockedCount(), 2u); + ASSERT_EQ(sink.blockedSamples().size(), 2u); + EXPECT_EQ(sink.blockedSamples()[0].relKey, "a"); + EXPECT_EQ(sink.blockedSamples()[1].relKey, "b"); + } } diff --git a/shared_data/test/shared_data/test_operation_added.hpp b/shared_data/test/shared_data/test_operation_added.hpp new file mode 100644 index 00000000..412608cc --- /dev/null +++ b/shared_data/test/shared_data/test_operation_added.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include +#include + +#include + +namespace SharedData::Test +{ + TEST(OperationAddedTests, RenameCarriesRemoteSourceAndDestination) + { + const OperationAdded added{ + .operationId = Ids::generateOperationId(), + .type = OperationType::Rename, + .source = Paths::RemotePath::make("/srv/old name").value(), + .destination = Paths::RemotePath::make("/srv/new name").value(), + }; + const nlohmann::json json = added; + const auto read = json.get(); + ASSERT_TRUE(read.source.has_value()); + ASSERT_TRUE(read.destination.has_value()); + EXPECT_EQ(read.source->value(), "/srv/old name"); + EXPECT_EQ(read.destination->value(), "/srv/new name"); + EXPECT_FALSE(read.localPath.has_value()); + EXPECT_FALSE(read.remotePath.has_value()); + } + + TEST(OperationAddedTests, InvalidPathsAreRefusedOnRead) + { + nlohmann::json json = + OperationAdded{.operationId = Ids::generateOperationId(), .type = OperationType::Download}; + json["remotePath"] = "relative/file"; + EXPECT_THROW(json.get(), std::exception); + } +} diff --git a/shared_data/test/shared_data/test_tree_directory_walker.hpp b/shared_data/test/shared_data/test_tree_directory_walker.hpp index 5a984aeb..e2de0a34 100644 --- a/shared_data/test/shared_data/test_tree_directory_walker.hpp +++ b/shared_data/test/shared_data/test_tree_directory_walker.hpp @@ -24,7 +24,7 @@ namespace SharedData::Sync::Test { directories[directory].push_back( DirectoryEntry{ - .path = name, + .name = name, .type = FileType::Regular, .size = size, .mtime = mtime, @@ -34,20 +34,21 @@ namespace SharedData::Sync::Test void addDirectory(std::string const& directory, std::string const& name) { - directories[directory].push_back(DirectoryEntry{.path = name, .type = FileType::Directory}); + directories[directory].push_back(DirectoryEntry{.name = name, .type = FileType::Directory}); directories[directory + "/" + name]; } }; inline auto makeScanner(FakeFilesystem& filesystem) { - return - [&filesystem](std::filesystem::path const& path) -> std::expected, std::string> + // The walked root is "/root", the walker names directories relative to it. + return [&filesystem](Paths::RelativePath const& relative) -> std::expected, std::string> { - filesystem.listed.push_back(path.generic_string()); - const auto iter = filesystem.directories.find(path.generic_string()); + const auto key = relative.empty() ? std::string{"/root"} : "/root/" + relative.value(); + filesystem.listed.push_back(key); + const auto iter = filesystem.directories.find(key); if (iter == filesystem.directories.end()) - return std::unexpected(std::string{"missing "} + path.generic_string()); + return std::unexpected(std::string{"missing "} + key); return iter->second; }; } @@ -56,8 +57,7 @@ namespace SharedData::Sync::Test inline std::expected walkFake(FakeFilesystem& filesystem, bool recursive = true) { auto scanner = makeScanner(filesystem); - TreeDirectoryWalker walker{ - "/root", std::move(scanner), recursive + TreeDirectoryWalker walker{std::move(scanner), recursive }; auto result = walker.walkAll(); if (!result) @@ -159,7 +159,7 @@ namespace SharedData::Sync::Test TEST(TreeDirectoryWalkerTests, ScannerErrorsAbortTheWalk) { FakeFilesystem filesystem; - filesystem.directories["/root"].push_back(DirectoryEntry{.path = "unlisted", .type = FileType::Directory}); + filesystem.directories["/root"].push_back(DirectoryEntry{.name = "unlisted", .type = FileType::Directory}); const auto tree = walkFake(filesystem); ASSERT_FALSE(tree.has_value()); @@ -174,7 +174,7 @@ namespace SharedData::Sync::Test filesystem.addFile("/root/dir", "b", 50); auto scanner = makeScanner(filesystem); - TreeDirectoryWalker walker{"/root", std::move(scanner)}; + TreeDirectoryWalker walker{std::move(scanner)}; ASSERT_TRUE(walker.walkAll().has_value()); EXPECT_TRUE(walker.completed()); EXPECT_EQ(walker.totalEntries(), 4u) << "root, a, dir, b"; diff --git a/ssh/include/ssh/async/transfer_context.hpp b/ssh/include/ssh/async/transfer_context.hpp index 81cf3df8..5d7580d8 100644 --- a/ssh/include/ssh/async/transfer_context.hpp +++ b/ssh/include/ssh/async/transfer_context.hpp @@ -1,9 +1,12 @@ #pragma once #include +#include -#include #include +#include +#include +#include namespace SecureShell { @@ -14,15 +17,42 @@ namespace SecureShell public: using SignedSizeType = std::make_signed_t; + /** + * @brief How the transfer ended. Only Completed means every byte was transferred. + */ + enum class Outcome + { + Running, + Completed, + Failed, + Canceled, + }; + // may update bytesTransferred_ friend FileStream; bool paused() const; void pause(bool doPause); + /** + * @brief Ends the transfer as canceled unless it already ended. + */ void cancel(); + /** + * @brief Ends the transfer as failed with the given error unless it already ended. + */ + void fail(SftpError error); + /** + * @brief Ends the transfer as completed unless it already ended. + */ + void complete(); SignedSizeType bytesTransferred() const; SignedSizeType bytesPerSecond() const; bool hasEnded() const; + Outcome outcome() const; + /** + * @brief The error a failed transfer ended with. + */ + std::optional error() const; void calculateBytesPerSecond(); /** @@ -34,7 +64,13 @@ namespace SecureShell void addBytesTransferred(SignedSizeType delta); private: + void end(Outcome outcome); + + private: + std::atomic outcome_{Outcome::Running}; std::atomic_bool ended_{false}; + mutable std::mutex errorMutex_{}; + std::optional error_{}; std::atomic_bool paused_{false}; std::atomic bytesTransferred_{0}; std::atomic lastBytesTransferred_{0}; @@ -42,4 +78,4 @@ namespace SecureShell ProcessingThread::PermanentTaskId permanentTaskId_{-1}; std::atomic bytesPerSecond_{0}; }; -} \ No newline at end of file +} diff --git a/ssh/include/ssh/sftp_error.hpp b/ssh/include/ssh/sftp_error.hpp index a08fe9e2..af03b01b 100644 --- a/ssh/include/ssh/sftp_error.hpp +++ b/ssh/include/ssh/sftp_error.hpp @@ -17,6 +17,12 @@ namespace SecureShell FileNull, TaskPushFailed, BufferUnavailable, + // The remote file ended before the expected number of bytes was read. + UnexpectedEndOfFile, + // The local side of a transfer could not be read or written. + LocalIoFailure, + // The remote path is unset or invalid, nothing was sent to the server. + InvalidPath, }; struct SftpError diff --git a/ssh/include/ssh/sftp_session.hpp b/ssh/include/ssh/sftp_session.hpp index d24c3482..49928968 100644 --- a/ssh/include/ssh/sftp_session.hpp +++ b/ssh/include/ssh/sftp_session.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -71,7 +72,7 @@ namespace SecureShell * @return std::future, Error>> */ std::future, Error>> - listDirectory(std::filesystem::path const& path); + listDirectory(Paths::RemotePath const& path); /** * @brief In-strand variant of listDirectory. Must be called from within the processing thread. @@ -79,7 +80,7 @@ namespace SecureShell * @param path * @return std::expected, Error> */ - std::expected, Error> listDirectoryInStrand(std::filesystem::path const& path); + std::expected, Error> listDirectoryInStrand(Paths::RemotePath const& path); /** * @brief Create a directory. @@ -89,7 +90,7 @@ namespace SecureShell * @return std::future> */ std::future> createDirectory( - std::filesystem::path const& path, + Paths::RemotePath const& path, std::filesystem::perms permissions = std::filesystem::perms::owner_all ); @@ -97,7 +98,7 @@ namespace SecureShell * @brief In-strand variant of createDirectory. Must be called from within the processing thread. */ std::expected createDirectoryInStrand( - std::filesystem::path const& path, + Paths::RemotePath const& path, std::filesystem::perms permissions = std::filesystem::perms::owner_all ); @@ -109,7 +110,7 @@ namespace SecureShell * @return std::future> */ std::future> createDirectoryIfItDoesntExist( - std::filesystem::path const& path, + Paths::RemotePath const& path, std::filesystem::perms permissions = std::filesystem::perms::owner_all ); @@ -117,7 +118,7 @@ namespace SecureShell * @brief In-strand variant of createDirectoryIfItDoesntExist. Must be called from within the processing thread. */ std::expected createDirectoryIfItDoesntExistInStrand( - std::filesystem::path const& path, + Paths::RemotePath const& path, std::filesystem::perms permissions = std::filesystem::perms::owner_all ); @@ -129,7 +130,7 @@ namespace SecureShell * @return std::future> */ std::future> createFile( - std::filesystem::path const& path, + Paths::RemotePath const& path, std::filesystem::perms permissions = std::filesystem::perms::owner_read | std::filesystem::perms::owner_write ); @@ -138,7 +139,7 @@ namespace SecureShell * @brief In-strand variant of createFile. Must be called from within the processing thread. */ std::expected createFileInStrand( - std::filesystem::path const& path, + Paths::RemotePath const& path, std::filesystem::perms permissions = std::filesystem::perms::owner_read | std::filesystem::perms::owner_write ); @@ -149,12 +150,12 @@ namespace SecureShell * @param path * @return std::future> */ - std::future> removeFile(std::filesystem::path const& path); + std::future> removeFile(Paths::RemotePath const& path); /** * @brief In-strand variant of removeFile. Must be called from within the processing thread. */ - std::expected removeFileInStrand(std::filesystem::path const& path); + std::expected removeFileInStrand(Paths::RemotePath const& path); /** * @brief Removes a directory. @@ -162,12 +163,12 @@ namespace SecureShell * @param path * @return std::future> */ - std::future> removeDirectory(std::filesystem::path const& path); + std::future> removeDirectory(Paths::RemotePath const& path); /** * @brief In-strand variant of removeDirectory. Must be called from within the processing thread. */ - std::expected removeDirectoryInStrand(std::filesystem::path const& path); + std::expected removeDirectoryInStrand(Paths::RemotePath const& path); /** * @brief Removes everything in the provided paths. @@ -175,7 +176,7 @@ namespace SecureShell * @param path * @return std::future> */ - std::future> removeAll(std::vector paths); + std::future> removeAll(std::vector paths); /** * @brief Checks a list of directories for non-emptiness. @@ -183,8 +184,8 @@ namespace SecureShell * @param path * @return std::future> */ - std::future, Error>> - filterOutEmptyDirectories(std::vector directories); + std::future, Error>> + filterOutEmptyDirectories(std::vector directories); /** * @brief Gets the attributes of a file or directory (resolves links). @@ -192,12 +193,12 @@ namespace SecureShell * @param path * @return std::future> */ - std::future> stat(std::filesystem::path const& path); + std::future> stat(Paths::RemotePath const& path); /** * @brief In-strand variant of stat. Must be called from within the processing thread. */ - std::expected statInStrand(std::filesystem::path const& path); + std::expected statInStrand(Paths::RemotePath const& path); /** * @brief Gets the attributes of a file or directory and if it is a link, the attributes of the link itself. @@ -205,12 +206,12 @@ namespace SecureShell * @param path * @return std::future> */ - std::future> lstat(std::filesystem::path const& path); + std::future> lstat(Paths::RemotePath const& path); /** * @brief In-strand variant of lstat. Must be called from within the processing thread. */ - std::expected lstatInStrand(std::filesystem::path const& path); + std::expected lstatInStrand(Paths::RemotePath const& path); /** * @brief Sets the attributes of a file or directory. @@ -219,12 +220,12 @@ namespace SecureShell * @param attributes * @return std::future> */ - std::future> stat(std::filesystem::path const& path, sftp_attributes attributes); + std::future> stat(Paths::RemotePath const& path, sftp_attributes attributes); /** * @brief In-strand variant of setstat. Must be called from within the processing thread. */ - std::expected statInStrand(std::filesystem::path const& path, sftp_attributes attributes); + std::expected statInStrand(Paths::RemotePath const& path, sftp_attributes attributes); /** * @brief Sets the owner of a file or directory. @@ -234,12 +235,12 @@ namespace SecureShell * @param group * @return std::future> */ - std::future> chown(std::filesystem::path const& path, uid_t owner, gid_t group); + std::future> chown(Paths::RemotePath const& path, uid_t owner, gid_t group); /** * @brief In-strand variant of chown. Must be called from within the processing thread. */ - std::expected chownInStrand(std::filesystem::path const& path, uid_t owner, gid_t group); + std::expected chownInStrand(Paths::RemotePath const& path, uid_t owner, gid_t group); /** * @brief Sets the permissions of a file or directory. @@ -248,24 +249,49 @@ namespace SecureShell * @param mode * @return std::future> */ - std::future> chmod(std::filesystem::path const& path, std::filesystem::perms perms); + std::future> chmod(Paths::RemotePath const& path, std::filesystem::perms perms); /** * @brief In-strand variant of chmod. Must be called from within the processing thread. */ - std::expected chmodInStrand(std::filesystem::path const& path, std::filesystem::perms perms); + std::expected chmodInStrand(Paths::RemotePath const& path, std::filesystem::perms perms); /** * @brief Move a file or directory. */ std::future> - rename(std::filesystem::path const& source, std::filesystem::path const& destination); + rename(Paths::RemotePath const& source, Paths::RemotePath const& destination); /** * @brief In-strand variant of rename. Must be called from within the processing thread. + * Replaces an existing destination when the server supports posix-rename, use + * @ref renameNoReplaceInStrand or @ref renameReplaceInStrand to choose. */ std::expected - renameInStrand(std::filesystem::path const& source, std::filesystem::path const& destination); + renameInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination); + + /** + * @brief Renames without ever replacing @p destination. Fails with SSH_FX_FILE_ALREADY_EXISTS when it + * exists. Regular files move atomically through hardlink@openssh.com where the server has it. + */ + std::expected + renameNoReplaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination); + + /** + * @brief Renames over an existing @p destination without a moment where it is missing: atomically with + * posix-rename@openssh.com, otherwise by moving the destination aside to @p asideSuffix and + * back if the rename fails. + */ + std::expected renameReplaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + std::string_view asideSuffix + ); + + /** + * @brief Whether the server announced the SFTP extension @p name with @p version. + */ + bool supportsExtensionInStrand(char const* name, char const* version = "1") const; enum class OpenType : int { @@ -278,27 +304,27 @@ namespace SecureShell }; std::future, Error>> - openFile(std::filesystem::path const& path, OpenType openType, std::filesystem::perms permissions); + openFile(Paths::RemotePath const& path, OpenType openType, std::filesystem::perms permissions); /** * @brief In-strand variant of openFile. Must be called from within the processing thread. */ std::expected, Error> - openFileInStrand(std::filesystem::path const& path, OpenType openType, std::filesystem::perms permissions); + openFileInStrand(Paths::RemotePath const& path, OpenType openType, std::filesystem::perms permissions); /** * @brief Determines the directory the server starts the user in, which is the home directory on conforming * servers. Falls back to /home/ when the server does not answer with an absolute path, or to / when that * does not exist either. * - * @return std::future> + * @return std::future> */ - std::future> homeDirectory(); + std::future> homeDirectory(); /** * @brief In-strand variant of homeDirectory. Must be called from within the processing thread. */ - std::expected homeDirectoryInStrand(); + std::expected homeDirectoryInStrand(); /** * @brief Reads the XDG user directories from /.config/user-dirs.dirs. A missing file yields no @@ -308,13 +334,13 @@ namespace SecureShell * @return std::future, Error>> */ std::future, Error>> - userDirectories(std::filesystem::path const& home); + userDirectories(Paths::RemotePath const& home); /** * @brief In-strand variant of userDirectories. Must be called from within the processing thread. */ std::expected, Error> - userDirectoriesInStrand(std::filesystem::path const& home); + userDirectoriesInStrand(Paths::RemotePath const& home); std::future> limits(); @@ -351,33 +377,43 @@ namespace SecureShell struct DeepLinkResult { - std::filesystem::path linkTarget; - // target may not exist: + /** @brief The content of the link at the given path, as stored on the server. */ + std::string linkContent; + /** @brief The end of the link chain. */ + Paths::RemotePath linkTarget; + /** @brief The attributes of @ref linkTarget, empty when it does not exist. */ std::optional targetInfo; }; std::future> - readLinkDeep(std::filesystem::path const& path, int maxDepth = 10); + readLinkDeep(Paths::RemotePath const& path, int maxDepth = 10); /** * @brief In-strand variant of readLinkDeep. Must be called from within the processing thread. */ - std::expected readLinkDeepInStrand(std::filesystem::path const& path, int maxDepth = 10); + std::expected readLinkDeepInStrand(Paths::RemotePath const& path, int maxDepth = 10); + /** + * @brief Creates a symlink at @p linkPath with the content @p target, which may be relative. + */ std::future> - createSymLink(std::filesystem::path const& target, std::filesystem::path const& linkPath); + createSymLink(std::string const& target, Paths::RemotePath const& linkPath); /** * @brief In-strand variant of createSymLink. Must be called from within the processing thread. */ - std::expected - createSymLinkInStrand(std::filesystem::path const& target, std::filesystem::path const& linkPath); + std::expected createSymLinkInStrand(std::string const& target, Paths::RemotePath const& linkPath); private: + /** + * @brief Refuses an unset path before it reaches the server. + */ + static std::expected requireSet(Paths::RemotePath const& path); + /** * @brief Reads a file into a string, stopping after maximumSize bytes. */ std::expected - readSmallFileInStrand(std::filesystem::path const& path, std::size_t maximumSize); + readSmallFileInStrand(Paths::RemotePath const& path, std::size_t maximumSize); void fileStreamRemoveItself(FileStream* stream, bool isBackElement); void removeAllFileStreams(); diff --git a/ssh/include/ssh/u8_path.hpp b/ssh/include/ssh/u8_path.hpp deleted file mode 100644 index 9156781f..00000000 --- a/ssh/include/ssh/u8_path.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -#include -#include -#include - -#ifdef _WIN32 -# define WIN32_LEAN_AND_MEAN -# include -#endif - -/** - * @brief Converts a path to a UTF-8 string (lossless on Windows). - * @param path The host-native path. - */ -inline std::string u8Path(std::filesystem::path const& path) -{ - const auto u8Str = path.generic_u8string(); - return std::string(u8Str.begin(), u8Str.end()); -} - -/** - * @brief Constructs a std::filesystem::path from a UTF-8 string. - * @param utf8 The UTF-8 encoded input (as received from libssh or RPC). - * - * On Windows this transcodes UTF-8 to UTF-16 before constructing the path, - * so non-ASCII bytes are preserved. On POSIX paths are already UTF-8. - */ -inline std::filesystem::path pathFromU8(std::string_view utf8) -{ -#ifdef _WIN32 - if (utf8.empty()) - return std::filesystem::path{}; - const int sizeNeeded = ::MultiByteToWideChar( - CP_UTF8, 0, utf8.data(), static_cast(utf8.size()), nullptr, 0 - ); - if (sizeNeeded <= 0) - return std::filesystem::path{std::string{utf8}}; - std::wstring wide(static_cast(sizeNeeded), L'\0'); - ::MultiByteToWideChar( - CP_UTF8, 0, utf8.data(), static_cast(utf8.size()), wide.data(), sizeNeeded - ); - return std::filesystem::path{wide}; -#else - return std::filesystem::path{std::string{utf8}}; -#endif -} diff --git a/ssh/source/ssh/async/transfer_context.cpp b/ssh/source/ssh/async/transfer_context.cpp index a6017717..dc30247b 100644 --- a/ssh/source/ssh/async/transfer_context.cpp +++ b/ssh/source/ssh/async/transfer_context.cpp @@ -2,16 +2,50 @@ namespace SecureShell { - void AsyncTransferContext::cancel() + void AsyncTransferContext::end(Outcome outcome) { + auto expected = Outcome::Running; + outcome_.compare_exchange_strong(expected, outcome); ended_.store(true); } + void AsyncTransferContext::cancel() + { + end(Outcome::Canceled); + } + + void AsyncTransferContext::fail(SftpError error) + { + if (outcome_.load() == Outcome::Running) + { + std::scoped_lock lock{errorMutex_}; + if (!error_) + error_ = std::move(error); + } + end(Outcome::Failed); + } + + void AsyncTransferContext::complete() + { + end(Outcome::Completed); + } + bool AsyncTransferContext::hasEnded() const { return ended_.load(); } + AsyncTransferContext::Outcome AsyncTransferContext::outcome() const + { + return outcome_.load(); + } + + std::optional AsyncTransferContext::error() const + { + std::scoped_lock lock{errorMutex_}; + return error_; + } + AsyncTransferContext::SignedSizeType AsyncTransferContext::bytesPerSecond() const { return bytesPerSecond_.load(); diff --git a/ssh/source/ssh/file_information.cpp b/ssh/source/ssh/file_information.cpp index 1d33c061..ea1016d8 100644 --- a/ssh/source/ssh/file_information.cpp +++ b/ssh/source/ssh/file_information.cpp @@ -6,8 +6,7 @@ namespace SecureShell { return FileInformation{ SharedData::DirectoryEntry{ - .path = attributes->name ? std::string{attributes->name} : std::string{}, - .longName = attributes->longname ? std::string{attributes->longname} : std::string{}, + .name = attributes->name ? std::string{attributes->name} : std::string{}, .flags = attributes->flags, .type = static_cast(attributes->type), .size = attributes->size, diff --git a/ssh/source/ssh/file_stream.cpp b/ssh/source/ssh/file_stream.cpp index 9cc76dec..73ce8dec 100644 --- a/ssh/source/ssh/file_stream.cpp +++ b/ssh/source/ssh/file_stream.cpp @@ -39,6 +39,21 @@ namespace SecureShell { return std::chrono::steady_clock::now() - turnStart > limit; } + + SftpError expiredError(std::string_view what) + { + return SftpError{ + .message = fmt::format("{} expired during the transfer", what), + .wrapperError = WrapperErrors::SharedPtrDestroyed, + }; + } + + SftpError transferError(SftpSession const& sftp, std::string_view what) + { + auto error = sftp.lastError(); + error.message = std::string{what}; + return error; + } } #define VERIFY_FILE_STREAM() \ @@ -265,17 +280,19 @@ namespace SecureShell void onRead(std::make_signed_t amount) { + // The callback runs before the promise is set: the waiting caller may destroy what it captured. if (amount < 0) { - promise.set_value(std::unexpected(stream.lastError())); + const auto error = stream.lastError(); callback({}); + promise.set_value(std::unexpected(error)); return; } if (amount == 0) { - promise.set_value(totalRead); callback({}); + promise.set_value(totalRead); return; } @@ -474,14 +491,14 @@ namespace SecureShell auto stream = weakStream.lock(); if (!stream) { - context->cancel(); + context->fail(expiredError("File stream")); return false; } auto sftp = stream->sftp_.lock(); if (!sftp) { - context->cancel(); + context->fail(expiredError("SFTP session")); return false; } const auto target = sftp->chunkController().targetCallDuration(); @@ -504,15 +521,18 @@ namespace SecureShell const auto result = sftp_read(stream->file_.get(), buffer, requested); if (result < 0) { - context->cancel(); + context->fail(transferError(*sftp, "Reading the remote file failed")); return false; } if (result == 0) { - // The remote file is shorter than listed; treat it as the end like the - // synchronous path does instead of polling the EOF forever. + // The remote file is shorter than listed. Finalizing would replace the + // destination with a truncated copy. context->calculateBytesPerSecond(); - context->ended_ = true; + context->fail(SftpError{ + .message = "The remote file ended before the expected size was read", + .wrapperError = WrapperErrors::UnexpectedEndOfFile, + }); return false; } // Report against the unclamped chunk: a file tail asks for less and must not @@ -521,7 +541,10 @@ namespace SecureShell remainingRead -= result; if (!onRead(result)) { - context->cancel(); + context->fail(SftpError{ + .message = "Storing the received data failed", + .wrapperError = WrapperErrors::LocalIoFailure, + }); return false; } context->bytesTransferred_ += result; @@ -531,7 +554,7 @@ namespace SecureShell context->calculateBytesPerSecond(); if (remainingRead == 0) { - context->ended_ = true; + context->complete(); return false; } return true; @@ -584,14 +607,14 @@ namespace SecureShell auto stream = weakStream.lock(); if (!stream) { - context->cancel(); + context->fail(expiredError("File stream")); return false; } auto sftp = stream->sftp_.lock(); if (!sftp) { - context->cancel(); + context->fail(expiredError("SFTP session")); return false; } const auto target = sftp->chunkController().targetCallDuration(); @@ -612,7 +635,10 @@ namespace SecureShell const auto amountRead = doRead(std::min(remainingWrite, chunk)); if (amountRead <= 0) { - context->cancel(); + context->fail(SftpError{ + .message = "Reading the local file failed", + .wrapperError = WrapperErrors::LocalIoFailure, + }); return false; } const auto requested = std::min(remainingWrite, amountRead); @@ -620,7 +646,7 @@ namespace SecureShell const auto result = sftp_write(stream->file_.get(), buffer, requested); if (result < 0) { - context->cancel(); + context->fail(transferError(*sftp, "Writing the remote file failed")); return false; } sftp->chunkController().recordDataCall(result, chunk, std::chrono::steady_clock::now() - callStart); @@ -632,7 +658,7 @@ namespace SecureShell context->calculateBytesPerSecond(); if (remainingWrite == 0) { - context->ended_ = true; + context->complete(); return false; } return true; diff --git a/ssh/source/ssh/session.cpp b/ssh/source/ssh/session.cpp index fd5f363f..779f0af8 100644 --- a/ssh/source/ssh/session.cpp +++ b/ssh/source/ssh/session.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include @@ -35,13 +35,13 @@ namespace SecureShell describeKeyImportFailure(int importResult, std::string_view keyKind, std::filesystem::path const& keyFile) { if (importResult == SSH_EOF) - return fmt::format("Cannot open the {} key file '{}'", keyKind, u8Path(keyFile)); + return fmt::format("Cannot open the {} key file '{}'", keyKind, Paths::nativeToUtf8(keyFile)); if (keyKind == "private") return fmt::format( "Cannot load the private key file '{}', it is not a valid key or the passphrase is wrong", - u8Path(keyFile) + Paths::nativeToUtf8(keyFile) ); - return fmt::format("The public key file '{}' is not a valid key", u8Path(keyFile)); + return fmt::format("The public key file '{}' is not a valid key", Paths::nativeToUtf8(keyFile)); } void removeFromContainer(auto& container, auto* ptr, bool isBackElement) @@ -550,7 +550,7 @@ namespace SecureShell { lastAttemptedAppliedOption = "Identity Agent"; return static_cast(*session).setOption( - SSH_OPTIONS_IDENTITY_AGENT, u8Path(sshOptions.identityAgent.value()).c_str() + SSH_OPTIONS_IDENTITY_AGENT, Paths::nativeToUtf8(sshOptions.identityAgent.value()).c_str() ); } return 0; @@ -635,7 +635,7 @@ namespace SecureShell if (maybeSshKeyPublic) { return ssh_pki_import_pubkey_file( - u8Path(*maybeSshKeyPublic).c_str(), &publicKey.underlyingReferenced() + Paths::nativeToUtf8(*maybeSshKeyPublic).c_str(), &publicKey.underlyingReferenced() ); } return SSH_OK; @@ -655,7 +655,7 @@ namespace SecureShell [&sshKeyPrivate, &privateKey, askPass, askPassUserDataKeyPhrase]() { return ssh_pki_import_privkey_file( - u8Path(sshKeyPrivate).c_str(), + Paths::nativeToUtf8(sshKeyPrivate).c_str(), nullptr, askPass, askPassUserDataKeyPhrase, @@ -671,7 +671,7 @@ namespace SecureShell if (result.index == 1 && result.result != SSH_OK) keyFailure = describeKeyImportFailure(result.result, "public", *maybeSshKeyPublic); else if (result.index == 2 && result.result == SSH_AUTH_DENIED) - keyFailure = fmt::format("The server does not accept the public key '{}'", u8Path(*maybeSshKeyPublic)); + keyFailure = fmt::format("The server does not accept the public key '{}'", Paths::nativeToUtf8(*maybeSshKeyPublic)); else if (result.index == 3 && result.result != SSH_OK) keyFailure = describeKeyImportFailure(result.result, "private", sshKeyPrivate); } diff --git a/ssh/source/ssh/sftp_session.cpp b/ssh/source/ssh/sftp_session.cpp index 612ffd8f..c5968330 100644 --- a/ssh/source/ssh/sftp_session.cpp +++ b/ssh/source/ssh/sftp_session.cpp @@ -1,7 +1,9 @@ #include +#include #include #include -#include +#include +#include #include #include @@ -70,16 +72,19 @@ namespace SecureShell } std::expected, SftpSession::Error> - SftpSession::listDirectoryInStrand(std::filesystem::path const& path) + SftpSession::listDirectoryInStrand(Paths::RemotePath const& path) { assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); int closeResult = 0; std::vector entries{}; + std::vector names{}; { std::unique_ptr> dir{ - sftp_opendir(session_, u8Path(path).c_str()), + sftp_opendir(session_, path.c_str()), [&](sftp_dir_struct* dir) { if (dir != nullptr) @@ -101,6 +106,7 @@ namespace SecureShell for (; entry != nullptr; entry.reset(sftp_readdir(session_, dir.get()))) { entries.push_back(fromSftpAttributes(entry.get())); + names.emplace_back(entry->name != nullptr ? entry->name : ""); } } @@ -114,11 +120,16 @@ namespace SecureShell return std::unexpected(lastError()); } - for (auto& entry : entries) + for (std::size_t index = 0; index < entries.size(); ++index) { + auto& entry = entries[index]; if (entry.type == SharedData::FileType::Symlink) { - const auto fullPath = u8Path(path / entry.path); + // A name that is no valid segment, like "x/..", must not become a path to look up. + const auto name = Paths::PathSegment::make(names[index]); + if (!name) + continue; + const auto fullPath = path / *name; std::unique_ptr linkLiteral{ sftp_readlink(session_, fullPath.c_str()), ssh_string_free_char @@ -126,7 +137,7 @@ namespace SecureShell if (linkLiteral) { const auto* raw = static_cast(linkLiteral.get()); - entry.linkTarget = pathFromU8(raw ? raw : ""); + entry.linkTarget = std::string{raw ? raw : ""}; } std::unique_ptr targetAttrs{ @@ -142,7 +153,7 @@ namespace SecureShell } std::future, SftpSession::Error>> - SftpSession::listDirectory(std::filesystem::path const& path) + SftpSession::listDirectory(Paths::RemotePath const& path) { return performPromise( [this, path]() @@ -153,11 +164,13 @@ namespace SecureShell } std::expected - SftpSession::createDirectoryInStrand(std::filesystem::path const& path, std::filesystem::perms permissions) + SftpSession::createDirectoryInStrand(Paths::RemotePath const& path, std::filesystem::perms permissions) { assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); auto result = sftp_mkdir( - session_, u8Path(path).c_str(), static_cast(permissions & std::filesystem::perms::mask) + session_, path.c_str(), static_cast(permissions & std::filesystem::perms::mask) ); if (result != SSH_OK) return std::unexpected(lastError()); @@ -165,7 +178,7 @@ namespace SecureShell } std::future> - SftpSession::createDirectory(std::filesystem::path const& path, std::filesystem::perms permissions) + SftpSession::createDirectory(Paths::RemotePath const& path, std::filesystem::perms permissions) { return performPromise( [this, path, permissions]() @@ -176,18 +189,20 @@ namespace SecureShell } std::expected SftpSession::createDirectoryIfItDoesntExistInStrand( - std::filesystem::path const& path, + Paths::RemotePath const& path, std::filesystem::perms permissions ) { assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); std::unique_ptr attributes{ - sftp_lstat(session_, u8Path(path).c_str()), sftp_attributes_free + sftp_lstat(session_, path.c_str()), sftp_attributes_free }; if (!attributes) { auto result = sftp_mkdir( - session_, u8Path(path).c_str(), static_cast(permissions & std::filesystem::perms::mask) + session_, path.c_str(), static_cast(permissions & std::filesystem::perms::mask) ); if (result != SSH_OK) return std::unexpected(lastError()); @@ -209,7 +224,7 @@ namespace SecureShell } std::future> - SftpSession::createDirectoryIfItDoesntExist(std::filesystem::path const& path, std::filesystem::perms permissions) + SftpSession::createDirectoryIfItDoesntExist(Paths::RemotePath const& path, std::filesystem::perms permissions) { return performPromise( [this, path, permissions]() @@ -220,14 +235,17 @@ namespace SecureShell } std::expected - SftpSession::createFileInStrand(std::filesystem::path const& path, std::filesystem::perms permissions) + SftpSession::createFileInStrand(Paths::RemotePath const& path, std::filesystem::perms permissions) { assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); std::unique_ptr> file{ + // O_EXCL: an existing file is an error, not reported as created. sftp_open( session_, - u8Path(path).c_str(), - O_CREAT, + path.c_str(), + O_CREAT | O_EXCL, static_cast(permissions & std::filesystem::perms::mask) ), [&](sftp_file_struct* file) @@ -246,7 +264,7 @@ namespace SecureShell } std::future> - SftpSession::createFile(std::filesystem::path const& path, std::filesystem::perms permissions) + SftpSession::createFile(Paths::RemotePath const& path, std::filesystem::perms permissions) { return performPromise( [this, path, permissions]() @@ -256,16 +274,18 @@ namespace SecureShell ); } - std::expected SftpSession::removeFileInStrand(std::filesystem::path const& path) + std::expected SftpSession::removeFileInStrand(Paths::RemotePath const& path) { assert(strand_->withinProcessingThread()); - auto result = sftp_unlink(session_, u8Path(path).c_str()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); + auto result = sftp_unlink(session_, path.c_str()); if (result != SSH_OK) return std::unexpected(lastError()); return {}; } - std::future> SftpSession::removeFile(std::filesystem::path const& path) + std::future> SftpSession::removeFile(Paths::RemotePath const& path) { return performPromise( [this, path]() @@ -275,16 +295,18 @@ namespace SecureShell ); } - std::expected SftpSession::removeDirectoryInStrand(std::filesystem::path const& path) + std::expected SftpSession::removeDirectoryInStrand(Paths::RemotePath const& path) { assert(strand_->withinProcessingThread()); - auto result = sftp_rmdir(session_, u8Path(path).c_str()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); + auto result = sftp_rmdir(session_, path.c_str()); if (result != SSH_OK) return std::unexpected(lastError()); return {}; } - std::future> SftpSession::removeDirectory(std::filesystem::path const& path) + std::future> SftpSession::removeDirectory(Paths::RemotePath const& path) { return performPromise( [this, path]() @@ -294,18 +316,20 @@ namespace SecureShell ); } - std::future, SftpSession::Error>> - SftpSession::filterOutEmptyDirectories(std::vector directories) + std::future, SftpSession::Error>> + SftpSession::filterOutEmptyDirectories(std::vector directories) { return performPromise( - [this, directories = std::move(directories)]() -> std::expected, Error> + [this, directories = std::move(directories)]() -> std::expected, Error> { int closeResult = 0; - std::vector nonEmpties; + std::vector nonEmpties; for (const auto& path : directories) { + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); std::unique_ptr attributes{ - sftp_lstat(session_, u8Path(path).c_str()), sftp_attributes_free + sftp_lstat(session_, path.c_str()), sftp_attributes_free }; if (attributes == nullptr) @@ -314,7 +338,7 @@ namespace SecureShell if (attributes->type == SSH_FILEXFER_TYPE_DIRECTORY) { std::unique_ptr> dir{ - sftp_opendir(session_, u8Path(path).c_str()), + sftp_opendir(session_, path.c_str()), [&](sftp_dir_struct* dir) { if (dir != nullptr) @@ -366,15 +390,17 @@ namespace SecureShell } std::future> - SftpSession::removeAll(std::vector paths) + SftpSession::removeAll(std::vector paths) { return performPromise( [this, paths = std::move(paths)]() -> std::expected { for (const auto& path : paths) { + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); std::unique_ptr attributes{ - sftp_lstat(session_, u8Path(path).c_str()), sftp_attributes_free + sftp_lstat(session_, path.c_str()), sftp_attributes_free }; if (attributes == nullptr) @@ -382,13 +408,13 @@ namespace SecureShell if (attributes->type == SSH_FILEXFER_TYPE_DIRECTORY) { - auto result = sftp_rmdir(session_, u8Path(path).c_str()); + auto result = sftp_rmdir(session_, path.c_str()); if (result != SSH_OK) return std::unexpected(lastError()); } else { - auto result = sftp_unlink(session_, u8Path(path).c_str()); + auto result = sftp_unlink(session_, path.c_str()); if (result != SSH_OK) return std::unexpected(lastError()); } @@ -398,12 +424,14 @@ namespace SecureShell ); } - std::expected SftpSession::statInStrand(std::filesystem::path const& path) + std::expected SftpSession::statInStrand(Paths::RemotePath const& path) { assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); const auto callStart = std::chrono::steady_clock::now(); std::unique_ptr attributes{ - sftp_stat(session_, u8Path(path).c_str()), sftp_attributes_free + sftp_stat(session_, path.c_str()), sftp_attributes_free }; chunkController_.recordRoundTrip(std::chrono::steady_clock::now() - callStart); if (attributes == nullptr) @@ -412,7 +440,7 @@ namespace SecureShell return fromSftpAttributes(attributes.get()); } - std::future> SftpSession::stat(std::filesystem::path const& path) + std::future> SftpSession::stat(Paths::RemotePath const& path) { return performPromise( [this, path]() @@ -422,12 +450,14 @@ namespace SecureShell ); } - std::expected SftpSession::lstatInStrand(std::filesystem::path const& path) + std::expected SftpSession::lstatInStrand(Paths::RemotePath const& path) { assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); const auto callStart = std::chrono::steady_clock::now(); std::unique_ptr attributes{ - sftp_lstat(session_, u8Path(path).c_str()), sftp_attributes_free + sftp_lstat(session_, path.c_str()), sftp_attributes_free }; chunkController_.recordRoundTrip(std::chrono::steady_clock::now() - callStart); if (attributes == nullptr) @@ -437,7 +467,7 @@ namespace SecureShell } std::future> - SftpSession::lstat(std::filesystem::path const& path) + SftpSession::lstat(Paths::RemotePath const& path) { return performPromise( [this, path]() @@ -448,17 +478,19 @@ namespace SecureShell } std::expected - SftpSession::statInStrand(std::filesystem::path const& path, sftp_attributes attributes) + SftpSession::statInStrand(Paths::RemotePath const& path, sftp_attributes attributes) { assert(strand_->withinProcessingThread()); - auto result = sftp_setstat(session_, u8Path(path).c_str(), attributes); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); + auto result = sftp_setstat(session_, path.c_str(), attributes); if (result != SSH_OK) return std::unexpected(lastError()); return {}; } std::future> - SftpSession::stat(std::filesystem::path const& path, sftp_attributes attributes) + SftpSession::stat(Paths::RemotePath const& path, sftp_attributes attributes) { return performPromise( [this, path, attributes]() @@ -469,13 +501,14 @@ namespace SecureShell } std::expected - SftpSession::renameInStrand(std::filesystem::path const& source, std::filesystem::path const& destination) + SftpSession::renameInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination) { assert(strand_->withinProcessingThread()); - auto s = source.generic_string(); - auto d = destination.generic_string(); - - auto result = sftp_rename(session_, s.c_str(), d.c_str()); + if (auto valid = requireSet(source); !valid) + return std::unexpected(valid.error()); + if (auto valid = requireSet(destination); !valid) + return std::unexpected(valid.error()); + auto result = sftp_rename(session_, source.c_str(), destination.c_str()); if (result != SSH_OK) { return std::unexpected(lastError()); @@ -483,8 +516,104 @@ namespace SecureShell return {}; } + bool SftpSession::supportsExtensionInStrand(char const* name, char const* version) const + { + assert(strand_->withinProcessingThread()); + return sftp_extension_supported(session_, name, version) == 1; + } + + std::expected + SftpSession::renameNoReplaceInStrand(Paths::RemotePath const& source, Paths::RemotePath const& destination) + { + assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(source); !valid) + return std::unexpected(valid.error()); + if (auto valid = requireSet(destination); !valid) + return std::unexpected(valid.error()); + const auto alreadyExists = [&destination]() + { + return SftpError{ + .message = fmt::format("'{}' already exists", destination.value()), + .sftpError = SSH_FX_FILE_ALREADY_EXISTS, + }; + }; + + const auto sourceInfo = lstatInStrand(source); + if (!sourceInfo.has_value()) + return std::unexpected(sourceInfo.error()); + + // A hard link fails when the destination exists, so the move cannot replace anything. + if (sourceInfo->isRegularFile() && supportsExtensionInStrand("hardlink@openssh.com")) + { + if (sftp_hardlink(session_, source.c_str(), destination.c_str()) != SSH_OK) + { + if (lstatInStrand(destination).has_value()) + return std::unexpected(alreadyExists()); + return std::unexpected(lastError()); + } + return removeFileInStrand(source); + } + + // libssh sends posix-rename@openssh.com when the server offers it, which replaces. Plain SFTP v3 rename + // refuses an existing destination. So only a server with posix-rename but without hardlink leaves the + // window between this check and the rename; OpenSSH offers both. + if (lstatInStrand(destination).has_value()) + return std::unexpected(alreadyExists()); + return renameInStrand(source, destination); + } + + std::expected SftpSession::renameReplaceInStrand( + Paths::RemotePath const& source, + Paths::RemotePath const& destination, + std::string_view asideSuffix + ) + { + assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(source); !valid) + return std::unexpected(valid.error()); + if (auto valid = requireSet(destination); !valid) + return std::unexpected(valid.error()); + if (supportsExtensionInStrand("posix-rename@openssh.com")) + return renameInStrand(source, destination); + + // Plain SFTP rename refuses an existing destination: move it aside instead of deleting it first. + const auto destinationInfo = lstatInStrand(destination); + if (!destinationInfo.has_value()) + return renameInStrand(source, destination); + if (destinationInfo->isDirectory()) + { + return std::unexpected( + SftpError{ + .message = fmt::format("'{}' is a directory", destination.value()), + .sftpError = SSH_FX_FAILURE, + } + ); + } + + const auto aside = destination.withFilenameSuffix(asideSuffix); + if (!aside) + { + return std::unexpected( + SftpError{ + .message = fmt::format("Invalid suffix '{}' for '{}'", asideSuffix, destination.value()), + .wrapperError = WrapperErrors::InvalidPath, + } + ); + } + if (auto movedAside = renameInStrand(destination, *aside); !movedAside.has_value()) + return movedAside; + if (auto renamed = renameInStrand(source, destination); !renamed.has_value()) + { + std::ignore = renameInStrand(*aside, destination); + return renamed; + } + if (auto removed = removeFileInStrand(*aside); !removed.has_value()) + Log::warn("SftpSession: Could not remove '{}': {}", aside->value(), removed.error().toString()); + return {}; + } + std::future> - SftpSession::rename(std::filesystem::path const& source, std::filesystem::path const& destination) + SftpSession::rename(Paths::RemotePath const& source, Paths::RemotePath const& destination) { return performPromise( [this, source, destination]() @@ -495,17 +624,19 @@ namespace SecureShell } std::expected - SftpSession::chownInStrand(std::filesystem::path const& path, uid_t owner, gid_t group) + SftpSession::chownInStrand(Paths::RemotePath const& path, uid_t owner, gid_t group) { assert(strand_->withinProcessingThread()); - auto result = sftp_chown(session_, u8Path(path).c_str(), owner, group); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); + auto result = sftp_chown(session_, path.c_str(), owner, group); if (result != SSH_OK) return std::unexpected(lastError()); return {}; } std::future> - SftpSession::chown(std::filesystem::path const& path, uid_t owner, gid_t group) + SftpSession::chown(Paths::RemotePath const& path, uid_t owner, gid_t group) { return performPromise( [this, path, owner, group]() @@ -516,17 +647,19 @@ namespace SecureShell } std::expected - SftpSession::chmodInStrand(std::filesystem::path const& path, std::filesystem::perms perms) + SftpSession::chmodInStrand(Paths::RemotePath const& path, std::filesystem::perms perms) { assert(strand_->withinProcessingThread()); - auto result = sftp_chmod(session_, u8Path(path).c_str(), static_cast(perms)); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); + auto result = sftp_chmod(session_, path.c_str(), static_cast(perms)); if (result != SSH_OK) return std::unexpected(lastError()); return {}; } std::future> - SftpSession::chmod(std::filesystem::path const& path, std::filesystem::perms perms) + SftpSession::chmod(Paths::RemotePath const& path, std::filesystem::perms perms) { return performPromise( [this, path, perms]() @@ -536,14 +669,17 @@ namespace SecureShell ); } - std::expected SftpSession::homeDirectoryInStrand() + std::expected SftpSession::homeDirectoryInStrand() { assert(strand_->withinProcessingThread()); const std::unique_ptr canonical{ sftp_canonicalize_path(session_, "."), ssh_string_free_char }; if (canonical != nullptr && canonical.get()[0] == '/') - return pathFromU8(canonical.get()); + { + if (auto home = Paths::RemotePath::make(canonical.get()); home) + return *std::move(home); + } char* rawUser = nullptr; const auto optionResult = ssh_options_get(session_->session, SSH_OPTIONS_USER, &rawUser); @@ -551,14 +687,16 @@ namespace SecureShell if (optionResult != SSH_OK || user == nullptr || user.get()[0] == '\0') return std::unexpected(Error{.message = "Server did not report a home directory and the user is unknown"}); - const auto guess = pathFromU8(fmt::format("/home/{}", user.get())); - const auto information = statInStrand(guess); + const auto guess = Paths::RemotePath::make(fmt::format("/home/{}", user.get())); + if (!guess) + return Paths::RemotePath::root(); + const auto information = statInStrand(*guess); if (information && information->isDirectory()) - return guess; - return std::filesystem::path{"/"}; + return *guess; + return Paths::RemotePath::root(); } - std::future> SftpSession::homeDirectory() + std::future> SftpSession::homeDirectory() { return performPromise( [this]() @@ -569,11 +707,13 @@ namespace SecureShell } std::expected - SftpSession::readSmallFileInStrand(std::filesystem::path const& path, std::size_t maximumSize) + SftpSession::readSmallFileInStrand(Paths::RemotePath const& path, std::size_t maximumSize) { assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); const std::unique_ptr file{ - sftp_open(session_, u8Path(path).c_str(), O_RDONLY, 0), sftp_close + sftp_open(session_, path.c_str(), O_RDONLY, 0), sftp_close }; if (file == nullptr) return std::unexpected(lastError()); @@ -594,14 +734,16 @@ namespace SecureShell } std::expected, SftpSession::Error> - SftpSession::userDirectoriesInStrand(std::filesystem::path const& home) + SftpSession::userDirectoriesInStrand(Paths::RemotePath const& home) { constexpr std::size_t maximumFileSize = 64u * 1024u; - return readSmallFileInStrand(home / ".config" / "user-dirs.dirs", maximumFileSize) + if (auto valid = requireSet(home); !valid) + return std::unexpected(valid.error()); + return readSmallFileInStrand(home / *Paths::RelativePath::make(".config/user-dirs.dirs"), maximumFileSize) .transform( [&home](std::string const& content) { - return Utility::parseUserDirectories(content, home); + return Utility::parseUserDirectories(content, home.value()); } ) .or_else( @@ -615,7 +757,7 @@ namespace SecureShell } std::future, SftpSession::Error>> - SftpSession::userDirectories(std::filesystem::path const& home) + SftpSession::userDirectories(Paths::RemotePath const& home) { return performPromise( [this, home]() @@ -625,6 +767,19 @@ namespace SecureShell ); } + std::expected SftpSession::requireSet(Paths::RemotePath const& path) + { + if (!path.empty()) + return {}; + return std::unexpected( + Error{ + .message = "The remote path is not set", + .sftpError = SSH_FX_FAILURE, + .wrapperError = WrapperErrors::InvalidPath, + } + ); + } + SftpError SftpSession::lastError() const { const auto result = SftpError{ @@ -655,17 +810,19 @@ namespace SecureShell } std::expected, SftpSession::Error> SftpSession::openFileInStrand( - std::filesystem::path const& path, + Paths::RemotePath const& path, OpenType openType, std::filesystem::perms permissions ) { assert(strand_->withinProcessingThread()); + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); const auto callStart = std::chrono::steady_clock::now(); std::unique_ptr> file{ sftp_open( session_, - u8Path(path).c_str(), + path.c_str(), static_cast(openType), static_cast(permissions & std::filesystem::perms::mask) ), @@ -693,7 +850,7 @@ namespace SecureShell } std::future, SftpSession::Error>> - SftpSession::openFile(std::filesystem::path const& path, OpenType openType, std::filesystem::perms permissions) + SftpSession::openFile(Paths::RemotePath const& path, OpenType openType, std::filesystem::perms permissions) { return performPromise( [this, path, openType, permissions]() @@ -704,21 +861,25 @@ namespace SecureShell } std::expected - SftpSession::readLinkDeepInStrand(std::filesystem::path const& path, int maxDepth) + SftpSession::readLinkDeepInStrand(Paths::RemotePath const& path, int maxDepth) { assert(strand_->withinProcessingThread()); - std::filesystem::path currentPath = path; + if (auto valid = requireSet(path); !valid) + return std::unexpected(valid.error()); + auto currentPath = path; + std::optional linkContent{}; for (int depth = 0; depth < maxDepth; ++depth) { std::unique_ptr attributes{ - sftp_lstat(session_, currentPath.generic_string().c_str()), sftp_attributes_free + sftp_lstat(session_, currentPath.c_str()), sftp_attributes_free }; if (attributes == nullptr) { auto last = lastError(); - if (last.sftpError == 2) + if (last.sftpError == SSH_FX_NO_SUCH_FILE) { return DeepLinkResult{ + .linkContent = linkContent.value_or(""), .linkTarget = currentPath, .targetInfo = std::nullopt, }; @@ -729,18 +890,35 @@ namespace SecureShell if (attributes->type != SSH_FILEXFER_TYPE_SYMLINK) { return DeepLinkResult{ + .linkContent = linkContent.value_or(""), .linkTarget = currentPath, .targetInfo = fromSftpAttributes(attributes.get()), }; } const auto readLinkResult = std::unique_ptr( - sftp_readlink(session_, currentPath.generic_string().c_str()), ssh_string_free_char + sftp_readlink(session_, currentPath.c_str()), ssh_string_free_char ); if (readLinkResult == nullptr) return std::unexpected(lastError()); + if (!linkContent) + linkContent = readLinkResult.get(); - currentPath = readLinkResult.get(); + // Relative content is relative to the link's directory, not to the server's working directory. + auto next = currentPath.resolveLink(readLinkResult.get()); + if (!next) + { + return std::unexpected( + Error{ + .message = fmt::format( + "Symlink '{}' has an unusable target: {}", currentPath.value(), next.error().toString() + ), + .sftpError = SSH_FX_FAILURE, + .wrapperError = WrapperErrors::InvalidPath, + } + ); + } + currentPath = *std::move(next); } return std::unexpected( @@ -753,7 +931,7 @@ namespace SecureShell } std::future> - SftpSession::readLinkDeep(std::filesystem::path const& path, int maxDepth) + SftpSession::readLinkDeep(Paths::RemotePath const& path, int maxDepth) { return performPromise( [this, path, maxDepth]() @@ -764,17 +942,19 @@ namespace SecureShell } std::expected - SftpSession::createSymLinkInStrand(std::filesystem::path const& target, std::filesystem::path const& linkPath) + SftpSession::createSymLinkInStrand(std::string const& target, Paths::RemotePath const& linkPath) { assert(strand_->withinProcessingThread()); - const auto result = sftp_symlink(session_, target.generic_string().c_str(), linkPath.generic_string().c_str()); + if (auto valid = requireSet(linkPath); !valid) + return std::unexpected(valid.error()); + const auto result = sftp_symlink(session_, target.c_str(), linkPath.c_str()); if (result != SSH_OK) return std::unexpected(lastError()); return {}; } std::future> - SftpSession::createSymLink(std::filesystem::path const& target, std::filesystem::path const& linkPath) + SftpSession::createSymLink(std::string const& target, Paths::RemotePath const& linkPath) { return performPromise( [this, target, linkPath]() diff --git a/ssh/test/ssh/CMakeLists.txt b/ssh/test/ssh/CMakeLists.txt index e02fb8d5..463e72ba 100644 --- a/ssh/test/ssh/CMakeLists.txt +++ b/ssh/test/ssh/CMakeLists.txt @@ -32,6 +32,8 @@ set_target_properties(nui-sftp-ssh-test PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${C find_program(NODE node) find_program(NPM npm) +file(GLOB TEST_SSH2_SERVER_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/test_ssh2_servers/source/*.mjs") + target_compile_definitions(nui-sftp-ssh-test PRIVATE -DNODE_EXECUTABLE="${NODE}" -DNPM_EXECUTABLE="${NPM}") add_custom_command( @@ -64,6 +66,7 @@ add_custom_command( "${CMAKE_CURRENT_LIST_DIR}/test_ssh2_servers/node_modules/.package-lock.json" "${CMAKE_CURRENT_LIST_DIR}/test_ssh2_servers/sftp_server.mjs" "${CMAKE_CURRENT_LIST_DIR}/test_ssh2_servers/shell_emulator.mjs" + ${TEST_SSH2_SERVER_SOURCES} COMMENT "Building test_ssh2_servers with webpack" ) add_custom_target( diff --git a/ssh/test/ssh/common_fixture.hpp b/ssh/test/ssh/common_fixture.hpp index 80fe85c4..ebba5e9d 100644 --- a/ssh/test/ssh/common_fixture.hpp +++ b/ssh/test/ssh/common_fixture.hpp @@ -4,6 +4,7 @@ #include #include +#include #include #pragma clang diagnostic push @@ -16,6 +17,7 @@ #include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wc23-extensions" @@ -48,6 +50,14 @@ static const char publicKey[] = { namespace SecureShell::Test { + /** + * @brief A remote path from a test literal, which is known to be valid. + */ + inline Paths::RemotePath remote(std::string_view path) + { + return Paths::RemotePath::make(path).value(); + } + class CommonFixture : public ::testing::Test { public: @@ -172,7 +182,7 @@ namespace SecureShell::Test ); } - Utility::TemporaryDirectory isolateDirectory_{programDirectory / "temp", true}; + Utility::TemporaryDirectory isolateDirectory_{temporaryRoot, true}; boost::asio::thread_pool pool_{1}; nlohmann::json defaultPackageJson_ = nlohmann::json({ {"name", "test"}, diff --git a/ssh/test/ssh/main.cpp b/ssh/test/ssh/main.cpp index c7f160ef..8930f777 100644 --- a/ssh/test/ssh/main.cpp +++ b/ssh/test/ssh/main.cpp @@ -1,3 +1,4 @@ +#include #include "test_buffer_provider.hpp" #include "test_processing_thread.hpp" #include "test_transfer_chunk_controller.hpp" @@ -12,11 +13,16 @@ #include std::filesystem::path programDirectory; +/** @brief The temporary directory of this process alone, so tests running in parallel share no files. */ +std::filesystem::path temporaryRoot; int main(int argc, char** argv) { programDirectory = std::filesystem::path{argv[0]}.parent_path(); + const Utility::ProcessTemporaryDirectory processTemporary{programDirectory / "temp"}; + temporaryRoot = processTemporary.path(); + ::testing::InitGoogleTest(&argc, argv); ::testing::FLAGS_gtest_death_test_style = "threadsafe"; return RUN_ALL_TESTS(); diff --git a/ssh/test/ssh/test_sftp.hpp b/ssh/test/ssh/test_sftp.hpp index 1a22fa11..845b1e8f 100644 --- a/ssh/test/ssh/test_sftp.hpp +++ b/ssh/test/ssh/test_sftp.hpp @@ -83,14 +83,14 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->listDirectory("/home/test"); + auto fut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().toString(); EXPECT_GT(result.value().size(), 0); auto it = std::find_if(result.value().begin(), result.value().end(), [](const auto& entry) { - return entry.path == "file1.txt"; + return entry.name == "file1.txt"; }); EXPECT_NE(it, result.value().end()); } @@ -100,7 +100,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->listDirectory("/home/test/nonexistant"); + auto fut = sftp->listDirectory(remote("/home/test/nonexistant")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); EXPECT_FALSE(result.has_value()); @@ -111,18 +111,18 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->createDirectory("/home/test/newdir"); + auto fut = sftp->createDirectory(remote("/home/test/newdir")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); - auto listFut = sftp->listDirectory("/home/test"); + auto listFut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); auto listResult = listFut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "newdir"; + return entry.name == "newdir"; }); EXPECT_NE(it, listResult.value().end()); } @@ -132,7 +132,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->createDirectory("/home/test"); + auto fut = sftp->createDirectory(remote("/home/test")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); EXPECT_FALSE(result.has_value()); @@ -143,28 +143,44 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->createFile("/home/test/newfile.txt"); + auto fut = sftp->createFile(remote("/home/test/newfile.txt")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); - auto listFut = sftp->listDirectory("/home/test"); + auto listFut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); auto listResult = listFut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "newfile.txt"; + return entry.name == "newfile.txt"; }); EXPECT_NE(it, listResult.value().end()); } + TEST_F(SftpTests, CreatingAnExistingFileFailsAndKeepsIt) + { + CREATE_SERVER_AND_JOINER(Sftp); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto fut = sftp->createFile(remote("/home/test/file1.txt")); + ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); + EXPECT_FALSE(fut.get().has_value()); + + auto statFut = sftp->stat(remote("/home/test/file1.txt")); + ASSERT_EQ(statFut.wait_for(1s), std::future_status::ready); + const auto information = statFut.get(); + ASSERT_TRUE(information.has_value()); + EXPECT_EQ(information->size, std::string{"Fake file content"}.size()); + } + TEST_F(SftpTests, CreatingFileInNonExistantDirectoryMayFail) { CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->createFile("/home/test/nonexistant/newfile.txt"); + auto fut = sftp->createFile(remote("/home/test/nonexistant/newfile.txt")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); EXPECT_FALSE(result.has_value()); @@ -175,18 +191,18 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->removeFile("/home/test/file1.txt"); + auto fut = sftp->removeFile(remote("/home/test/file1.txt")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); - auto listFut = sftp->listDirectory("/home/test"); + auto listFut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); auto listResult = listFut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "file1.txt"; + return entry.name == "file1.txt"; }); EXPECT_EQ(it, listResult.value().end()); } @@ -196,7 +212,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->removeFile("/home/test/nonexistant.txt"); + auto fut = sftp->removeFile(remote("/home/test/nonexistant.txt")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); EXPECT_FALSE(result.has_value()); @@ -207,7 +223,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->stat("/home/test/file1.txt"); + auto fut = sftp->stat(remote("/home/test/file1.txt")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -224,7 +240,7 @@ namespace SecureShell::Test ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); const auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().message; - EXPECT_EQ(result->generic_string(), "/home/test"); + EXPECT_EQ(result->value(), "/home/test"); } TEST_F(SftpTests, HomeDirectoryIsNotAssumedToBeInHome) @@ -237,7 +253,7 @@ namespace SecureShell::Test ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); const auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().message; - EXPECT_EQ(result->generic_string(), "/Users/test"); + EXPECT_EQ(result->value(), "/Users/test"); auto listFut = sftp->listDirectory(*result); ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); @@ -257,7 +273,7 @@ namespace SecureShell::Test ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); const auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().message; - EXPECT_EQ(result->generic_string(), "/home/test"); + EXPECT_EQ(result->value(), "/home/test"); } TEST_F(SftpTests, HomeDirectoryFallsBackWhenServerAnswersEmptyPath) @@ -271,7 +287,7 @@ namespace SecureShell::Test ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); const auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().message; - EXPECT_EQ(result->generic_string(), "/home/test"); + EXPECT_EQ(result->value(), "/home/test"); } TEST_F(SftpTests, HomeDirectoryFallsBackWhenServerFailsToCanonicalize) @@ -285,7 +301,7 @@ namespace SecureShell::Test ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); const auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().message; - EXPECT_EQ(result->generic_string(), "/home/test"); + EXPECT_EQ(result->value(), "/home/test"); auto listFut = sftp->listDirectory(*result); ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); @@ -305,7 +321,7 @@ namespace SecureShell::Test ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); const auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().message; - EXPECT_EQ(result->generic_string(), "/"); + EXPECT_EQ(result->value(), "/"); } TEST_F(SftpTests, CanReadUserDirectories) @@ -313,14 +329,14 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->userDirectories("/home/deutsch"); + auto fut = sftp->userDirectories(remote("/home/deutsch")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); const auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().toString(); std::vector> directories; for (auto const& directory : *result) - directories.emplace_back(directory.name, directory.path.generic_string()); + directories.emplace_back(directory.name, directory.path); EXPECT_EQ( directories, (std::vector>{ @@ -339,13 +355,13 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->userDirectories("/home/test"); + auto fut = sftp->userDirectories(remote("/home/test")); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); const auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().toString(); EXPECT_TRUE(result->empty()); - auto listFut = sftp->listDirectory("/home/test"); + auto listFut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); EXPECT_TRUE(listFut.get().has_value()); } @@ -355,18 +371,18 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->rename("/home/test/file1.txt", "/home/test/file1_renamed.txt"); + auto fut = sftp->rename(remote("/home/test/file1.txt"), remote("/home/test/file1_renamed.txt")); ASSERT_EQ(fut.wait_for(3s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()) << result.error().message; - auto listFut = sftp->listDirectory("/home/test"); + auto listFut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); auto listResult = listFut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "file1_renamed.txt"; + return entry.name == "file1_renamed.txt"; }); EXPECT_NE(it, listResult.value().end()); } @@ -376,18 +392,18 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); - auto fut = sftp->rename("/home/test/Documents", "/home/test/Documents_renamed"); + auto fut = sftp->rename(remote("/home/test/Documents"), remote("/home/test/Documents_renamed")); ASSERT_EQ(fut.wait_for(5s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); - auto listFut = sftp->listDirectory("/home/test"); + auto listFut = sftp->listDirectory(remote("/home/test")); ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); auto listResult = listFut.get(); ASSERT_TRUE(listResult.has_value()); auto it = std::find_if(listResult.value().begin(), listResult.value().end(), [](const auto& entry) { - return entry.path == "Documents_renamed"; + return entry.name == "Documents_renamed"; }); EXPECT_NE(it, listResult.value().end()); } @@ -398,7 +414,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -422,7 +438,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -448,7 +464,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -470,13 +486,44 @@ namespace SecureShell::Test EXPECT_EQ(data, "Fake file content"); } + TEST_F(SftpTests, ReadAllFinishesItsCallbacksBeforeTheResult) + { + CREATE_SERVER_AND_JOINER(Sftp); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto fut = + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); + auto result = fut.get(); + ASSERT_TRUE(result.has_value()); + auto file = result->lock(); + ASSERT_TRUE(file); + + // Callers capture locals by reference and leave once the result is there. + auto finalCallbackDone = std::make_shared(false); + auto readFut = file->readAll( + [finalCallbackDone](std::string_view chunk) + { + if (chunk.empty()) + { + std::this_thread::sleep_for(100ms); + *finalCallbackDone = true; + } + return true; + } + ); + ASSERT_EQ(readFut.wait_for(5s), std::future_status::ready); + EXPECT_TRUE(readFut.get().has_value()); + EXPECT_TRUE(*finalCallbackDone); + } + TEST_F(SftpTests, CanReadBigFileChunkMethod) { CREATE_SERVER_AND_JOINER(Sftp); auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/large.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/large.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -504,7 +551,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -531,7 +578,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -559,7 +606,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -594,7 +641,7 @@ namespace SecureShell::Test { auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -616,7 +663,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -639,7 +686,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/large.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/large.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -683,7 +730,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/large.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/large.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -734,7 +781,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/large.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/large.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -780,7 +827,7 @@ namespace SecureShell::Test auto [_, sftp] = createSftpSession(serverStartResult->port); auto fut = - sftp->openFile("/home/test/large.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + sftp->openFile(remote("/home/test/large.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto result = fut.get(); ASSERT_TRUE(result.has_value()); @@ -815,4 +862,208 @@ namespace SecureShell::Test EXPECT_EQ(callCount.load(), 3); EXPECT_LT(ctx->bytesTransferred(), static_cast(totalSize)); } + + TEST_F(SftpTests, MockServerCanFailReads) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("failReadsAfter", {{"bytes", 0}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto fut = + sftp->openFile(remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); + auto file = fut.get().value().lock(); + ASSERT_TRUE(file); + + std::array buffer{}; + auto readFut = file->readSome(buffer.data(), buffer.size()); + ASSERT_EQ(readFut.wait_for(1s), std::future_status::ready); + EXPECT_FALSE(readFut.get().has_value()); + } + + TEST_F(SftpTests, MockServerCanDenyDirectoryListing) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("denyDirectory", {{"path", "/home/test/Documents"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto listFut = sftp->listDirectory(remote("/home/test/Documents")); + ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); + EXPECT_FALSE(listFut.get().has_value()); + } + + TEST_F(SftpTests, MockServerListsRawNamesVerbatim) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("rawName", {{"directory", "/home/test"}, {"name", "x/.."}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto listFut = sftp->listDirectory(remote("/home/test")); + ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); + const auto listResult = listFut.get(); + ASSERT_TRUE(listResult.has_value()); + const auto iter = std::find_if(listResult->begin(), listResult->end(), [](auto const& entry) { + return entry.name == "x/.."; + }); + EXPECT_NE(iter, listResult->end()); + } + + TEST_F(SftpTests, MockServerPosixRenameExtensionOverwrites) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "posix-rename@openssh.com"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto fut = sftp->rename(remote("/home/test/file1.txt"), remote("/home/test/file2.txt")); + ASSERT_EQ(fut.wait_for(3s), std::future_status::ready); + const auto result = fut.get(); + ASSERT_TRUE(result.has_value()) << result.error().message; + + auto listFut = sftp->listDirectory(remote("/home/test")); + ASSERT_EQ(listFut.wait_for(1s), std::future_status::ready); + const auto listResult = listFut.get(); + ASSERT_TRUE(listResult.has_value()); + EXPECT_EQ(std::count_if(listResult->begin(), listResult->end(), [](auto const& entry) { + return entry.name == "file1.txt" || entry.name == "file2.txt"; + }), 1); + } + + TEST_F(SftpTests, MockServerPlainRenameDoesNotOverwrite) + { + CREATE_SERVER_AND_JOINER(Sftp); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto fut = sftp->rename(remote("/home/test/file1.txt"), remote("/home/test/file2.txt")); + ASSERT_EQ(fut.wait_for(3s), std::future_status::ready); + EXPECT_FALSE(fut.get().has_value()); + } + + namespace + { + std::string readRemote(SftpSession& sftp, std::string_view path) + { + auto openFuture = sftp.openFile(remote(path), SftpSession::OpenType::Read, std::filesystem::perms::owner_read); + if (openFuture.wait_for(2s) != std::future_status::ready) + return ""; + auto opened = openFuture.get(); + if (!opened.has_value()) + return ""; + std::string data; + auto readFuture = opened->lock()->readAll( + [&data](std::string_view chunk) + { + data.append(chunk); + return true; + } + ); + if (readFuture.wait_for(2s) != std::future_status::ready || !readFuture.get().has_value()) + return ""; + return data; + } + + bool remoteExists(SftpSession& sftp, std::string_view path) + { + auto future = sftp.lstat(remote(path)); + return future.wait_for(2s) == std::future_status::ready && future.get().has_value(); + } + } + + TEST_F(SftpTests, RenameNoReplaceRefusesAnExistingFile) + { + for (const bool hardlink : {false, true}) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); + if (hardlink) + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "hardlink@openssh.com"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto future = sftp->performPromise( + [&sftp]() + { + return sftp->renameNoReplaceInStrand(remote("/home/test/source.txt"), remote("/home/test/target.txt")); + } + ); + ASSERT_EQ(future.wait_for(3s), std::future_status::ready); + const auto result = future.get(); + ASSERT_FALSE(result.has_value()) << "hardlink: " << hardlink; + EXPECT_EQ(result.error().sftpError, SSH_FX_FILE_ALREADY_EXISTS); + EXPECT_EQ(readRemote(*sftp, "/home/test/source.txt"), "source"); + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "target"); + } + } + + TEST_F(SftpTests, RenameNoReplaceMovesToAFreeName) + { + for (const bool hardlink : {false, true}) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + if (hardlink) + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "hardlink@openssh.com"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto future = sftp->performPromise( + [&sftp]() + { + return sftp->renameNoReplaceInStrand(remote("/home/test/source.txt"), remote("/home/test/moved.txt")); + } + ); + ASSERT_EQ(future.wait_for(3s), std::future_status::ready); + ASSERT_TRUE(future.get().has_value()) << "hardlink: " << hardlink; + EXPECT_FALSE(remoteExists(*sftp, "/home/test/source.txt")); + EXPECT_EQ(readRemote(*sftp, "/home/test/moved.txt"), "source"); + } + } + + TEST_F(SftpTests, RenameReplaceReplacesWithAndWithoutPosixRename) + { + for (const bool posixRename : {false, true}) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); + if (posixRename) + ASSERT_TRUE(serverStartResult->command("advertise", {{"name", "posix-rename@openssh.com"}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto future = sftp->performPromise( + [&sftp]() + { + return sftp->renameReplaceInStrand(remote("/home/test/source.txt"), remote("/home/test/target.txt"), ".aside"); + } + ); + ASSERT_EQ(future.wait_for(3s), std::future_status::ready); + ASSERT_TRUE(future.get().has_value()) << "posixRename: " << posixRename; + EXPECT_EQ(readRemote(*sftp, "/home/test/target.txt"), "source"); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/source.txt")); + EXPECT_FALSE(remoteExists(*sftp, "/home/test/target.txt.aside")); + } + } + + TEST_F(SftpTests, FailedRenameReplaceRestoresTheDestination) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/source.txt"}, {"content", "source"}})); + ASSERT_TRUE(serverStartResult->command("createFile", {{"path", "/home/test/target.txt"}, {"content", "target"}})); + // Moving the destination aside succeeds, the rename into place fails. + ASSERT_TRUE(serverStartResult->command("failRenames", {{"after", 1}})); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + auto future = sftp->performPromise( + [&sftp]() + { + return sftp->renameReplaceInStrand(remote("/home/test/source.txt"), remote("/home/test/target.txt"), ".aside"); + } + ); + ASSERT_EQ(future.wait_for(3s), std::future_status::ready); + EXPECT_FALSE(future.get().has_value()); + ASSERT_TRUE(serverStartResult->command("failRenames", {{"enabled", false}})); + // The restore is a rename as well and ran while renames failed, so the original sits aside: never lost. + const bool restored = readRemote(*sftp, "/home/test/target.txt") == "target"; + const bool keptAside = readRemote(*sftp, "/home/test/target.txt.aside") == "target"; + EXPECT_TRUE(restored || keptAside); + EXPECT_EQ(readRemote(*sftp, "/home/test/source.txt"), "source"); + } } diff --git a/ssh/test/ssh/test_sftp_in_strand.hpp b/ssh/test/ssh/test_sftp_in_strand.hpp index 685eff57..045a14f0 100644 --- a/ssh/test/ssh/test_sftp_in_strand.hpp +++ b/ssh/test/ssh/test_sftp_in_strand.hpp @@ -31,7 +31,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->listDirectoryInStrand("/home/test"); + return sftp->listDirectoryInStrand(remote("/home/test")); }); ASSERT_TRUE(result.has_value()) << result.error().toString(); EXPECT_GT(result.value().size(), 0u); @@ -42,7 +42,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->createDirectoryInStrand("/home/test/newdir_is"); + return sftp->createDirectoryInStrand(remote("/home/test/newdir_is")); }); EXPECT_TRUE(result.has_value()) << result.error().toString(); } @@ -53,7 +53,7 @@ namespace SecureShell::Test auto [owner, sftp] = createSftpSession(serverStartResult->port); // Existing directory — should be a no-op success. auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->createDirectoryIfItDoesntExistInStrand("/home/test"); + return sftp->createDirectoryIfItDoesntExistInStrand(remote("/home/test")); }); EXPECT_TRUE(result.has_value()) << result.error().toString(); } @@ -63,7 +63,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->createFileInStrand("/home/test/newfile_is.txt"); + return sftp->createFileInStrand(remote("/home/test/newfile_is.txt")); }); EXPECT_TRUE(result.has_value()) << result.error().toString(); } @@ -73,7 +73,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->removeFileInStrand("/home/test/file1.txt"); + return sftp->removeFileInStrand(remote("/home/test/file1.txt")); }); EXPECT_TRUE(result.has_value()) << result.error().toString(); } @@ -84,11 +84,11 @@ namespace SecureShell::Test auto [owner, sftp] = createSftpSession(serverStartResult->port); // First create a directory we can remove. auto mk = runInStrand(sftp, [sftp = sftp]() { - return sftp->createDirectoryInStrand("/home/test/to_remove_is"); + return sftp->createDirectoryInStrand(remote("/home/test/to_remove_is")); }); ASSERT_TRUE(mk.has_value()); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->removeDirectoryInStrand("/home/test/to_remove_is"); + return sftp->removeDirectoryInStrand(remote("/home/test/to_remove_is")); }); EXPECT_TRUE(result.has_value()) << result.error().toString(); } @@ -98,7 +98,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->statInStrand("/home/test/file1.txt"); + return sftp->statInStrand(remote("/home/test/file1.txt")); }); ASSERT_TRUE(result.has_value()) << result.error().toString(); EXPECT_GT(result.value().size, 0u); @@ -109,7 +109,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->lstatInStrand("/home/test/file1.txt"); + return sftp->lstatInStrand(remote("/home/test/file1.txt")); }); ASSERT_TRUE(result.has_value()) << result.error().toString(); } @@ -119,7 +119,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->renameInStrand("/home/test/file1.txt", "/home/test/file1_renamed_is.txt"); + return sftp->renameInStrand(remote("/home/test/file1.txt"), remote("/home/test/file1_renamed_is.txt")); }); EXPECT_TRUE(result.has_value()) << result.error().toString(); } @@ -129,7 +129,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->chmodInStrand("/home/test/file1.txt", std::filesystem::perms::owner_all); + return sftp->chmodInStrand(remote("/home/test/file1.txt"), std::filesystem::perms::owner_all); }); EXPECT_TRUE(result.has_value()) << result.error().toString(); } @@ -150,7 +150,7 @@ namespace SecureShell::Test auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { return sftp->openFileInStrand( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); }); ASSERT_TRUE(result.has_value()) << result.error().toString(); @@ -163,17 +163,56 @@ namespace SecureShell::Test auto [owner, sftp] = createSftpSession(serverStartResult->port); // Non-symlink regular file is a valid target. auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->readLinkDeepInStrand("/home/test/file1.txt"); + return sftp->readLinkDeepInStrand(remote("/home/test/file1.txt")); }); EXPECT_TRUE(result.has_value()) << result.error().toString(); } + TEST_F(SftpInStrandTests, ReadLinkDeepResolvesRelativeLinksFromTheLinkDirectory) + { + CREATE_SERVER_AND_JOINER(Sftp); + ASSERT_TRUE(serverStartResult->command( + "createSymlink", {{"path", "/home/test/Documents/relative_link"}, {"target", "../link_to_file1.txt"}} + )); + auto [owner, sftp] = createSftpSession(serverStartResult->port); + auto result = runInStrand(sftp, [sftp = sftp]() { + return sftp->readLinkDeepInStrand(remote("/home/test/Documents/relative_link")); + }); + ASSERT_TRUE(result.has_value()) << result.error().toString(); + EXPECT_EQ(result->linkContent, "../link_to_file1.txt"); + EXPECT_EQ(result->linkTarget.value(), "/home/test/file1.txt"); + ASSERT_TRUE(result->targetInfo.has_value()); + EXPECT_TRUE(result->targetInfo->isRegularFile()); + } + + TEST_F(SftpInStrandTests, UnsetPathsNeverReachTheServer) + { + CREATE_SERVER_AND_JOINER(Sftp); + auto [owner, sftp] = createSftpSession(serverStartResult->port); + auto removed = runInStrand(sftp, [sftp = sftp]() { + return sftp->removeFileInStrand(Paths::RemotePath{}); + }); + ASSERT_FALSE(removed.has_value()); + EXPECT_EQ(removed.error().wrapperError, WrapperErrors::InvalidPath); + + auto renamed = runInStrand(sftp, [sftp = sftp]() { + return sftp->renameReplaceInStrand(remote("/home/test/file1.txt"), Paths::RemotePath{}, ".old"); + }); + ASSERT_FALSE(renamed.has_value()); + EXPECT_EQ(renamed.error().wrapperError, WrapperErrors::InvalidPath); + + auto stillThere = runInStrand(sftp, [sftp = sftp]() { + return sftp->lstatInStrand(remote("/home/test/file1.txt")); + }); + EXPECT_TRUE(stillThere.has_value()); + } + TEST_F(SftpInStrandTests, CreateSymLinkInStrandHappyPath) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto result = runInStrand(sftp, [sftp = sftp]() { - return sftp->createSymLinkInStrand("/home/test/file1.txt", "/home/test/link_is.txt"); + return sftp->createSymLinkInStrand("/home/test/file1.txt", remote("/home/test/link_is.txt")); }); EXPECT_TRUE(result.has_value()) << result.error().toString(); } @@ -186,7 +225,7 @@ namespace SecureShell::Test auto [owner, sftp] = createSftpSession(serverStartResult->port); auto fut = sftp->openFile( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto opened = fut.get(); @@ -217,7 +256,7 @@ namespace SecureShell::Test auto [owner, sftp] = createSftpSession(serverStartResult->port); auto fut = sftp->openFile( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto opened = fut.get(); @@ -242,56 +281,56 @@ namespace SecureShell::Test { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->listDirectoryInStrand("/home/test"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->listDirectoryInStrand(remote("/home/test")); }, ".*"); } TEST_F(SftpInStrandDeathTests, CreateDirectoryInStrandAssertsOutsideStrand) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->createDirectoryInStrand("/home/test/x_dt"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->createDirectoryInStrand(remote("/home/test/x_dt")); }, ".*"); } TEST_F(SftpInStrandDeathTests, CreateDirectoryIfItDoesntExistInStrandAssertsOutsideStrand) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->createDirectoryIfItDoesntExistInStrand("/home/test/x_dt2"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->createDirectoryIfItDoesntExistInStrand(remote("/home/test/x_dt2")); }, ".*"); } TEST_F(SftpInStrandDeathTests, CreateFileInStrandAssertsOutsideStrand) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->createFileInStrand("/home/test/x_dt.txt"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->createFileInStrand(remote("/home/test/x_dt.txt")); }, ".*"); } TEST_F(SftpInStrandDeathTests, RemoveFileInStrandAssertsOutsideStrand) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->removeFileInStrand("/home/test/file1.txt"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->removeFileInStrand(remote("/home/test/file1.txt")); }, ".*"); } TEST_F(SftpInStrandDeathTests, RemoveDirectoryInStrandAssertsOutsideStrand) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->removeDirectoryInStrand("/home/test/Documents"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->removeDirectoryInStrand(remote("/home/test/Documents")); }, ".*"); } TEST_F(SftpInStrandDeathTests, StatInStrandAssertsOutsideStrand) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->statInStrand("/home/test/file1.txt"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->statInStrand(remote("/home/test/file1.txt")); }, ".*"); } TEST_F(SftpInStrandDeathTests, LstatInStrandAssertsOutsideStrand) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->lstatInStrand("/home/test/file1.txt"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->lstatInStrand(remote("/home/test/file1.txt")); }, ".*"); } TEST_F(SftpInStrandDeathTests, SetstatInStrandAssertsOutsideStrand) @@ -299,21 +338,21 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); sftp_attributes_struct attrs{}; - ASSERT_DEBUG_DEATH({ (void)sftp->statInStrand("/home/test/file1.txt", &attrs); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->statInStrand(remote("/home/test/file1.txt"), &attrs); }, ".*"); } TEST_F(SftpInStrandDeathTests, RenameInStrandAssertsOutsideStrand) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->renameInStrand("/home/test/file1.txt", "/home/test/whatever.txt"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->renameInStrand(remote("/home/test/file1.txt"), remote("/home/test/whatever.txt")); }, ".*"); } TEST_F(SftpInStrandDeathTests, ChownInStrandAssertsOutsideStrand) { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->chownInStrand("/home/test/file1.txt", 0, 0); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->chownInStrand(remote("/home/test/file1.txt"), 0, 0); }, ".*"); } TEST_F(SftpInStrandDeathTests, ChmodInStrandAssertsOutsideStrand) @@ -321,7 +360,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); ASSERT_DEBUG_DEATH( - { (void)sftp->chmodInStrand("/home/test/file1.txt", std::filesystem::perms::owner_all); }, ".*" + { (void)sftp->chmodInStrand(remote("/home/test/file1.txt"), std::filesystem::perms::owner_all); }, ".*" ); } @@ -339,7 +378,7 @@ namespace SecureShell::Test ASSERT_DEBUG_DEATH( { (void)sftp->openFileInStrand( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); }, ".*" @@ -350,7 +389,7 @@ namespace SecureShell::Test { CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); - ASSERT_DEBUG_DEATH({ (void)sftp->readLinkDeepInStrand("/home/test/file1.txt"); }, ".*"); + ASSERT_DEBUG_DEATH({ (void)sftp->readLinkDeepInStrand(remote("/home/test/file1.txt")); }, ".*"); } TEST_F(SftpInStrandDeathTests, CreateSymLinkInStrandAssertsOutsideStrand) @@ -358,7 +397,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); ASSERT_DEBUG_DEATH( - { (void)sftp->createSymLinkInStrand("/home/test/file1.txt", "/home/test/link_dt.txt"); }, ".*" + { (void)sftp->createSymLinkInStrand("/home/test/file1.txt", remote("/home/test/link_dt.txt")); }, ".*" ); } @@ -372,7 +411,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto fut = sftp->openFile( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto file = fut.get().value().lock(); @@ -385,7 +424,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto fut = sftp->openFile( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto file = fut.get().value().lock(); @@ -398,7 +437,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto fut = sftp->openFile( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto file = fut.get().value().lock(); @@ -411,7 +450,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto fut = sftp->openFile( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto file = fut.get().value().lock(); @@ -424,7 +463,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto fut = sftp->openFile( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto file = fut.get().value().lock(); @@ -438,7 +477,7 @@ namespace SecureShell::Test CREATE_SERVER_AND_JOINER(Sftp); auto [owner, sftp] = createSftpSession(serverStartResult->port); auto fut = sftp->openFile( - "/home/test/file1.txt", SftpSession::OpenType::Read, std::filesystem::perms::owner_read + remote("/home/test/file1.txt"), SftpSession::OpenType::Read, std::filesystem::perms::owner_read ); ASSERT_EQ(fut.wait_for(1s), std::future_status::ready); auto file = fut.get().value().lock(); diff --git a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs index 7af20eb2..c84a80c9 100644 --- a/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs +++ b/ssh/test/ssh/test_ssh2_servers/sftp_server.mjs @@ -161,6 +161,80 @@ let defaultPath = '/home/test'; // 'conforming', 'relative', 'empty' or 'failure'. Emulates servers answering REALPATH "." incorrectly. let realpathDotMode = 'conforming'; +// Fault injection and server behaviour switches, configured over the command line interface. +const faults = { + failReadsAfter: null, + failWritesAfter: null, + dropAfterBytes: null, + failRenames: false, + // Renames that still succeed before failRenames applies, null for none. + renamesBeforeFailure: null, + deniedDirectories: new Set(), + deniedStats: new Set(), + bytesRead: 0, + bytesWritten: 0, +}; +// Extension name to extension data, announced in the VERSION packet. +const advertisedExtensions = {}; + +// True when this rename has to fail. Counts down the renames allowed before failing. +const renameFails = () => { + if (!faults.failRenames) + return false; + if (faults.renamesBeforeFailure !== null && faults.renamesBeforeFailure > 0) { + --faults.renamesBeforeFailure; + return false; + } + return true; +}; + +const lengthPrefixed = (text) => { + const body = Buffer.from(text, 'utf8'); + const header = Buffer.alloc(4); + header.writeUInt32BE(body.length); + return Buffer.concat([header, body]); +}; + +const readLengthPrefixedStrings = (data, count) => { + const strings = []; + let offset = 0; + for (let index = 0; index < count; ++index) { + if (!data || offset + 4 > data.length) + return undefined; + const length = data.readUInt32BE(offset); + offset += 4; + if (offset + length > data.length) + return undefined; + strings.push(data.toString('utf8', offset, offset + length)); + offset += length; + } + return strings; +}; + +// ssh2 always answers INIT with a VERSION packet without extensions, so the packet is replaced +// on its way out when extensions are advertised. +const advertiseExtensions = (sftpStream) => { + const names = Object.keys(advertisedExtensions); + if (names.length === 0) + return; + const protocol = sftpStream._protocol; + const originalChannelData = protocol.channelData; + protocol.channelData = function (id, data) { + const versionPacketType = 2; + if (id === sftpStream.outgoing.id && data.length === 9 && data[4] === versionPacketType) { + protocol.channelData = originalChannelData; + const parts = [Buffer.from([versionPacketType, 0, 0, 0, 3])]; + for (const name of names) + parts.push(lengthPrefixed(name), lengthPrefixed(advertisedExtensions[name])); + const body = Buffer.concat(parts); + const header = Buffer.alloc(4); + header.writeUInt32BE(body.length); + return originalChannelData.call(protocol, id, Buffer.concat([header, body])); + } + return originalChannelData.call(protocol, id, data); + }; +}; + const resolveFileName = (path) => { if (path === '.') { return defaultPath; @@ -226,12 +300,18 @@ const server = new Server({ const handles = new Map(); const sftpStream = accept(); + advertiseExtensions(sftpStream); sftpStream.on('OPEN', (reqid, filename, flags, attrs) => { filename = resolveFileName(filename); logMessage('OPEN', filename, flags, attrs); - const result = fakeFilesystem.find(filename); + let result = fakeFilesystem.find(filename); + // OPEN follows symlinks like a real server. + if (result !== undefined && result.type === 'symlink') { + filename = result.target; + result = fakeFilesystem.find(filename); + } if (result === undefined) { const anyWriteFlag = flags & (OPEN_MODE.WRITE | OPEN_MODE.TRUNC | OPEN_MODE.CREAT | OPEN_MODE.APPEND); if (anyWriteFlag) { @@ -250,6 +330,11 @@ const server = new Server({ return sftpStream.status(reqid, STATUS_CODE.NO_SUCH_FILE); } } else { + // Like a real server: CREAT with EXCL refuses anything that exists. + if ((flags & OPEN_MODE.CREAT) && (flags & OPEN_MODE.EXCL)) { + logMessage('Exists, EXCL given'); + return sftpStream.status(reqid, STATUS_CODE.FAILURE); + } if (result.type !== 'file') { logMessage('Not a file'); return sftpStream.status(reqid, STATUS_CODE.FAILURE); @@ -292,6 +377,16 @@ const server = new Server({ return sftpStream.status(reqid, STATUS_CODE.EOF); } + if (faults.dropAfterBytes !== null && faults.bytesRead + faults.bytesWritten >= faults.dropAfterBytes) { + logMessage('Fault: dropping the connection during WRITE'); + return client.end(); + } + if (faults.failWritesAfter !== null && faults.bytesWritten >= faults.failWritesAfter) { + logMessage('Fault: failing WRITE'); + return sftpStream.status(reqid, STATUS_CODE.FAILURE); + } + faults.bytesWritten += data.length; + // Simulate writing data to the file at the specified offset. // Keep content as Buffer end-to-end; converting to UTF-8 strings // here would corrupt any byte > 0x7F (replacement char), which @@ -340,7 +435,17 @@ const server = new Server({ // result.content is a Buffer; slice() returns a Buffer view — // send it directly without UTF-8 round-tripping. + if (faults.dropAfterBytes !== null && faults.bytesRead + faults.bytesWritten >= faults.dropAfterBytes) { + logMessage('Fault: dropping the connection during READ'); + return client.end(); + } + if (faults.failReadsAfter !== null && faults.bytesRead >= faults.failReadsAfter) { + logMessage('Fault: failing READ'); + return sftpStream.status(reqid, STATUS_CODE.FAILURE); + } + const data = result.content.slice(offset, Math.min(offset + length, size)); + faults.bytesRead += data.length; logMessage(`Read ${data.length} bytes from offset ${offset}`); sftpStream.data(reqid, data); @@ -418,6 +523,11 @@ const server = new Server({ logMessage('OPENDIR', path); + if (faults.deniedDirectories.has(path)) { + logMessage('Fault: permission denied'); + return sftpStream.status(reqid, STATUS_CODE.PERMISSION_DENIED); + } + const result = fakeFilesystem.find(path); if (result === undefined || result.type !== 'directory') { logMessage('Not a directory'); @@ -500,6 +610,11 @@ const server = new Server({ logMessage('RENAME', oldPath, newPath); + if (renameFails()) { + logMessage('Fault: failing RENAME'); + return sftpStream.status(reqid, STATUS_CODE.FAILURE); + } + const oldResult = fakeFilesystem.find(oldPath); if (oldResult === undefined) { logMessage('Old file not found'); @@ -516,7 +631,51 @@ const server = new Server({ sftpStream.status(reqid, STATUS_CODE.OK); }); - sftpStream.on('MKDIR', (reqid, path) => { + sftpStream.on('EXTENDED', (reqid, name, data) => { + logMessage('EXTENDED', name); + + if (!Object.hasOwn(advertisedExtensions, name)) + return sftpStream.status(reqid, STATUS_CODE.OP_UNSUPPORTED); + + const paths = readLengthPrefixedStrings(data, 2); + if (paths === undefined) + return sftpStream.status(reqid, STATUS_CODE.BAD_MESSAGE); + const oldPath = resolveFileName(paths[0]); + const newPath = resolveFileName(paths[1]); + + const oldResult = fakeFilesystem.find(oldPath); + if (oldResult === undefined) + return sftpStream.status(reqid, STATUS_CODE.NO_SUCH_FILE); + const newResult = fakeFilesystem.find(newPath); + + if (name === 'posix-rename@openssh.com') { + if (renameFails()) { + logMessage('Fault: failing posix-rename'); + return sftpStream.status(reqid, STATUS_CODE.FAILURE); + } + if (newResult !== undefined) { + if (newResult.type === 'directory' && (oldResult.type !== 'directory' || newResult.children.length > 0)) + return sftpStream.status(reqid, STATUS_CODE.FAILURE); + newResult.parent.removeChild(newResult.name); + } + renameInFakeFs(oldResult, newPath); + return sftpStream.status(reqid, STATUS_CODE.OK); + } + + if (name === 'hardlink@openssh.com') { + if (newResult !== undefined || oldResult.type !== 'file') + return sftpStream.status(reqid, STATUS_CODE.FAILURE); + const parent = fakeFilesystem.find(pathUtil.dirname(newPath)); + if (parent === undefined || parent.type !== 'directory') + return sftpStream.status(reqid, STATUS_CODE.NO_SUCH_FILE); + parent.insert(file(pathUtil.basename(newPath), oldResult.content, oldResult.stat)); + return sftpStream.status(reqid, STATUS_CODE.OK); + } + + return sftpStream.status(reqid, STATUS_CODE.OP_UNSUPPORTED); + }); + + sftpStream.on('MKDIR', (reqid, path, attrs) => { path = resolveFileName(path); logMessage('MKDIR', path); @@ -534,7 +693,9 @@ const server = new Server({ return sftpStream.status(reqid, STATUS_CODE.FAILURE); } - parent.insert(directory({ name: pathUtil.basename(path) })); + // Like mkdir(2): the requested mode, 0755 when none was sent. + const mode = attrs && attrs.mode !== undefined ? attrs.mode : undefined; + parent.insert(directory({ name: pathUtil.basename(path), stat: { mode } })); sftpStream.status(reqid, STATUS_CODE.OK); }); @@ -688,6 +849,11 @@ const server = new Server({ logMessage('LSTAT', path); + if (faults.deniedStats.has(path)) { + logMessage('Fault: permission denied'); + return sftpStream.status(reqid, STATUS_CODE.PERMISSION_DENIED); + } + const result = fakeFilesystem.find(path); if (result === undefined) { logMessage('File not found'); @@ -736,7 +902,71 @@ cli.on('removeHome', () => { fakeFilesystem.find('/home').removeChild('test'); }); +const nullableBytes = (command) => (command.bytes === undefined || command.bytes === null) ? null : Number(command.bytes); +cli.on('failReadsAfter', (command) => { + faults.failReadsAfter = nullableBytes(command); + faults.bytesRead = 0; +}); +cli.on('failWritesAfter', (command) => { + faults.failWritesAfter = nullableBytes(command); + faults.bytesWritten = 0; +}); +cli.on('dropAfterBytes', (command) => { + faults.dropAfterBytes = nullableBytes(command); + faults.bytesRead = 0; + faults.bytesWritten = 0; +}); +cli.on('failRenames', (command) => { + faults.failRenames = command.enabled !== false; + faults.renamesBeforeFailure = command.after === undefined ? null : Number(command.after); +}); +cli.on('denyDirectory', (command) => { + faults.deniedDirectories.add(command.path); +}); +// LSTAT of the path answers permission denied. +cli.on('denyStat', (command) => { + faults.deniedStats.add(command.path); +}); +cli.on('advertise', (command) => { + advertisedExtensions[command.name] = command.data ?? '1'; +}); +// Inserts an entry under a raw name that may be invalid as a path segment, like '..' or 'a/b'. +cli.on('rawName', (command) => { + const parent = fakeFilesystem.find(command.directory); + if (parent === undefined || parent.type !== 'directory') + return; + if (command.type === 'directory') + parent.insert(directory({ name: command.name }, [])); + else + parent.insert(file(command.name, command.content ?? '')); +}); +cli.on('createFile', (command) => { + const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); + if (parent === undefined || parent.type !== 'directory') + return; + const name = pathUtil.posix.basename(command.path); + parent.removeChild(name); + const content = command.size !== undefined + ? Buffer.alloc(Number(command.size), command.fill ?? 'x') + : (command.content ?? ''); + parent.insert(file(name, content)); +}); +cli.on('createDirectory', (command) => { + const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); + if (parent === undefined || parent.type !== 'directory') + return; + parent.insert(directory({ name: pathUtil.posix.basename(command.path) }, [])); +}); +cli.on('createSymlink', (command) => { + const parent = fakeFilesystem.find(pathUtil.posix.dirname(command.path)); + if (parent === undefined || parent.type !== 'directory') + return; + parent.insert(symlink(pathUtil.posix.basename(command.path), command.target)); +}); + cli.on('exit', () => { logMessage('').end(); server.close(); + // Exits even when a handle stays open, like stdin on Windows, which held every test until the kill timer. + setTimeout(() => process.exit(0), 200).unref(); }); \ No newline at end of file diff --git a/ssh/test/ssh/test_ssh2_servers/source/cli.mjs b/ssh/test/ssh/test_ssh2_servers/source/cli.mjs index ce8b37cc..2f176b4e 100644 --- a/ssh/test/ssh/test_ssh2_servers/source/cli.mjs +++ b/ssh/test/ssh/test_ssh2_servers/source/cli.mjs @@ -18,16 +18,21 @@ class CommandLineInterface { const command = JSON.parse(input); if (Object.hasOwnProperty.call(command, 'command')) { + // Exit runs once: a second run wrote to the closed log and the crash waited for stdin on Windows. if (command.command === 'exit') { if (this.eventMap.has('exit')) { this.eventMap.get('exit')(); } this.close(); + return; } if (this.eventMap.has(command.command)) { this.eventMap.get(command.command)(command); } + if (command.ack === true) { + process.stdout.write(JSON.stringify({ ack: command.command }) + '\n'); + } } }); } diff --git a/ssh/test/ssh/test_ssh2_servers/source/fake_file_system.mjs b/ssh/test/ssh/test_ssh2_servers/source/fake_file_system.mjs index 6ca7b7cd..bf2fc4a5 100644 --- a/ssh/test/ssh/test_ssh2_servers/source/fake_file_system.mjs +++ b/ssh/test/ssh/test_ssh2_servers/source/fake_file_system.mjs @@ -75,7 +75,7 @@ const directory = ({ name, stat }, children) => { stat: { ...defaultStat, ...defined(stat), - mode: 0o755 | S_IFDIR + mode: (stat && stat.mode !== undefined ? stat.mode & 0o7777 : 0o755) | S_IFDIR }, children: children || [], insert: function (entry) { diff --git a/ssh/test/ssh/test_ssh_session.hpp b/ssh/test/ssh/test_ssh_session.hpp index d4ae3ee5..5118fc86 100644 --- a/ssh/test/ssh/test_ssh_session.hpp +++ b/ssh/test/ssh/test_ssh_session.hpp @@ -2,7 +2,7 @@ #include "common_fixture.hpp" #include -#include +#include #include #include @@ -10,6 +10,7 @@ #include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; using namespace std::chrono_literals; using namespace std::string_literals; @@ -24,7 +25,7 @@ namespace SecureShell::Test { std::filesystem::rename( isolateDirectory_.path() / "log.txt", - programDirectory / "temp" / + temporaryRoot / ("log_"s + ::testing::UnitTest::GetInstance()->current_test_info()->test_case_name() + "_"s + ::testing::UnitTest::GetInstance()->current_test_info()->name() + ".txt") ); @@ -266,7 +267,7 @@ namespace SecureShell::Test processThread.join(); }}; - const auto missingKey = programDirectory / "temp" / "missing_key"; + const auto missingKey = temporaryRoot / "missing_key"; auto options = getSessionOptions(result->port); options.users.front().sshKeyPrivate = missingKey; options.sshOptions->usePasswordAuth = false; @@ -286,7 +287,7 @@ namespace SecureShell::Test ASSERT_FALSE(session.has_value()); const auto& error = session.error(); EXPECT_NE(error.find("Cannot open the private key file"), std::string::npos) << error; - EXPECT_NE(error.find(u8Path(missingKey)), std::string::npos) << error; + EXPECT_NE(error.find(Paths::nativeToUtf8(missingKey)), std::string::npos) << error; EXPECT_EQ(error.find("Unknown authentication result"), std::string::npos) << error; } diff --git a/static/assets/languages/de_DE.yaml b/static/assets/languages/de_DE.yaml index 73a5d559..82c752cf 100644 --- a/static/assets/languages/de_DE.yaml +++ b/static/assets/languages/de_DE.yaml @@ -9,6 +9,14 @@ confirmDialog: neverShowAgain: "Nicht mehr anzeigen" andMore: "... und {} weitere" fileExplorer: + openRiskyHeader: "Eine Datei öffnen, die Code ausführen kann?" + openRiskyText: "'{}' {}. Beim Öffnen kann sie ausgeführt werden, nur tun, wenn ihrer Herkunft vertraut wird.\n\nTrotzdem öffnen?" + invalidItemNameHeader: "Ungültiger Name" + invalidItemName: "'{}' kann nicht als Name verwendet werden. Ein Name darf nicht leer, '.' oder '..' sein und weder '/' noch '\\' enthalten." + renameTargetExistsHeader: "Eintrag existiert" + renameTargetExists: "'{}' existiert bereits. Ersetzen?" + existenceCheckFailedHeader: "Existenzprüfung fehlgeschlagen" + existenceCheckFailed: "Es konnte nicht geprüft werden, welche Ziele bereits existieren, es wurde nichts übertragen: {}" contextMenu: download: "Herunterladen" upload: "Hochladen" @@ -86,7 +94,7 @@ remoteSideModel: failedToCreateDeleteOperation: "Löschoperation konnte nicht erstellt werden: {}" deleteMultipleConfirm: "Sind Sie sicher, dass Sie die folgenden {} Elemente löschen möchten?" deleteSingleConfirm: "Sind Sie sicher, dass Sie dieses Element '{}' löschen möchten?" - deleteItemsQuestion: "Elemente löschen?" + deleteRemoteItemsQuestion: "Auf dem Server löschen?" failedToDeleteItems: "Fehler beim Löschen einer oder mehrerer Dateien / Verzeichnisse: {}." downloadFailed: "Download fehlgeschlagen" failedToCreateDownloadOperation: "Download-Operation konnte nicht erstellt werden: {}." @@ -124,6 +132,8 @@ localSideModel: portalUnavailableText: "Das Desktop-Portal zum Öffnen von Dateien in anderen Anwendungen ist nicht verfügbar. \"Öffnen\", \"Öffnen mit\" und \"Im Dateimanager öffnen\" wurden deaktiviert.\n\n{}" archiveUploadFailed: "Archiv-Upload fehlgeschlagen" deleteSelectedConfirm: "Sind Sie sicher, dass Sie die ausgewählten Elemente löschen möchten?" + deleteLocalItemsQuestion: "Auf diesem Computer löschen?" + deleteDirectoriesWithContent: "Ordner werden mit ihrem gesamten Inhalt gelöscht. Dies kann nicht rückgängig gemacht werden." remoteSideUnavailable: "Die Remote-Seite ist nicht verfügbar" transferFailedTitle: "Dateiübertragung fehlgeschlagen" operationQueue: @@ -142,12 +152,14 @@ operationQueue: cancelOperationText: "Sind Sie sicher, dass Sie die Operation abbrechen möchten?" deletingItem: "Lösche: '{}'" deletingSelectedIn: "Lösche ausgewählte Elemente in {}" + deletingLocallyIn: "Lösche lokale Elemente in {}" entriesFailed: "{} Einträge fehlgeschlagen:\n" errorButton: "Fehler!" livePage: "Live" noError: "Kein Fehler" renameCard: "Umbenennen: '{}' -> '{}'" scanCard: "Durchsuche '{}'" + scanningItems: "Durchsuche... {} Einträge ({})" unknownError: "Unbekannter Fehler" syncDialog: titleFormat: "Synchronisieren: {} / {}" @@ -180,6 +192,21 @@ syncDialog: synchronize: "Synchronisieren" confirmHeader: "Verzeichnisse synchronisieren" confirmText: "{} mit {} synchronisieren?" + confirmDeletes: "Dabei werden {} Einträge ({}) {} gelöscht." + deleteSideRemote: "auf dem Server" + deleteSideLocal: "lokal" + confirmDeleteRowHeader: "Jetzt löschen" + confirmDeleteRow: "'{}' {} jetzt löschen?" + blockedNotice: "{} Einträge bleiben unverändert, weil sie nicht lesbar oder nicht übertragbar sind, ein Verzeichnis auf einen symbolischen Link trifft sich ihre Namen nur in der Groß- und Kleinschreibung unterscheiden oder sie sich seit der letzten Synchronisation so geändert haben, dass Daten verloren gingen. Zum Anzeigen klicken." + blockedTitle: "Unveränderte Einträge" + blockReason: + Unreadable: "nicht lesbar" + Excluded: "nicht übertragbar" + InvalidName: "ungültiger Name" + TypeConflict: "Verzeichnis gegen symbolischen Link" + CaseConflict: "Namen unterscheiden sich nur in der Groß- und Kleinschreibung" + ChangedOnBothSides: "seit der letzten Synchronisation auf beiden Seiten geändert" + ChangedOnDestination: "seit der letzten Synchronisation auf der Seite geändert, die ersetzt oder gelöscht würde" tree: loading: "Lädt…" loadFailed: "Fehlgeschlagen: {}" @@ -196,6 +223,7 @@ syncProgressDialog: differencesFound: "Unterschiede gefunden" hashing: "Hashes werden berechnet..." cancel: "Abbrechen" + scanFailed: "Auflisten fehlgeschlagen, es wurde nichts verglichen." reconnectDialog: attempt: "Versuch {}" retryIn: "Neuer Versuch in {} s" @@ -446,6 +474,9 @@ fileTracking: columnAutoReupload: "Auto-Hochladen" columnMoveRemote: "Remote verschieben" columnDeleteRemote: "Remote löschen" + columnCheckRemote: "Server prüfen" + serverChangedHeader: "Auf dem Server geändert" + serverChangedText: "'{}' wurde auf dem Server geändert, seit sie hier geöffnet wurde. Die Version auf dem Server durch die lokale ersetzen?\n\nNein behält die Version auf dem Server, beim nächsten Speichern wird erneut gefragt." columnStatus: "Status" stopButton: "Stopp" statusWatching: "Beobachtend" @@ -692,6 +723,8 @@ settings: reserveSpaceHelpText: "Platz auf dem Zielsystem für Downloads reservieren." doCleanup: "Aufräumen durchführen" doCleanupHelpText: "Temporäre Dateien nach dem Download entfernen." + markAsDownloaded: "Als heruntergeladen markieren" + markAsDownloadedHelpText: "Heruntergeladene Dateien als aus dem Internet stammend markieren, damit Windows (Mark of the Web) und macOS (Quarantäne) vor dem Ausführen warnen. Standardmäßig an." symlinkHandlingHelpText: "Wie symbolische Links während der Übertragung behandelt werden. 'AsSymlink' überträgt symbolische Links als symbolische Links (sofern vom Dateisystem unterstützt), 'FollowSymlink' folgt dem symbolischen Link und überträgt die Zieldatei oder das Zielverzeichnis, und 'SkipSymlink' überspringt die Übertragung von symbolischen Links vollständig." symlinkHandling: "Behandlung von symbolischen Links" failFast: "Schnell fehlschlagen" @@ -980,6 +1013,11 @@ settings: preventCreateDirectory: "Verzeichniserstellung verhindern" homeOverrideHelpText: "Wenn gesetzt, überschreibt dies den Standardpfad des Heimatverzeichnisses der Anwendung." homeOverride: "Heimatverzeichnis überschreiben" + openRiskyFilesHelpText: "Was passiert, wenn eine Datei, die Code ausführen kann (Programm, Skript, Installer oder Verknüpfung), aus dem Dateiexplorer geöffnet wird, auch vom Server geöffnete Dateien." + openRiskyFiles: "Öffnen von Dateien, die Code ausführen können" + openRiskyFilesAllow: "Immer erlauben" + openRiskyFilesWarn: "Vorher fragen" + openRiskyFilesNever: "Nie" temporaryDownloadsDirectoryHelpText: "Das Verzeichnis, in dem heruntergeladene Dateien temporär gespeichert werden. Standardmäßig %temp%/nui-sftp-downloads." temporaryDownloadsDirectory: "Temporäres Download-Verzeichnis" fileTrackingOptions: @@ -989,6 +1027,8 @@ settings: moveRemoteOnLocalMoveHelpText: "Standardwert für neue überwachte Einträge: Wenn eine überwachte Datei lokal verschoben oder umbenannt wird, die gleiche Aktion auf dem Remote ausführen." deleteRemoteOnLocalDelete: "Remote löschen bei lokalem Löschen" deleteRemoteOnLocalDeleteHelpText: "Standardwert für neue überwachte Einträge: Wenn eine überwachte Datei lokal gelöscht wird, auch auf dem Remote löschen." + checkRemoteChanges: "Server vor erneutem Hochladen prüfen" + checkRemoteChangesHelpText: "Standard für neue verfolgte Einträge: Vor dem erneuten Hochladen einer bearbeiteten Datei prüfen, ob sich die Kopie auf dem Server seit dem Öffnen geändert hat, und vor dem Ersetzen fragen." colorSetting: pickButton: "Wählen" persistence: @@ -1111,12 +1151,17 @@ terminalPanel: saveDialogTitle: "Terminalinhalt speichern" saveToFile: "Terminal in Datei speichern" backend: + rename: + targetExists: "Ein Eintrag mit diesem Namen existiert bereits." + invalidName: "Der neue Name muss ein einfacher Name im selben Verzeichnis sein." rpcFilesystem: + fileExists: "'{}' existiert bereits und wurde nicht ersetzt." deletionPrevented: "Das Löschen von Dateien ist durch die Konfiguration verhindert." renamePrevented: "Das Umbenennen von Dateien ist durch die Konfiguration verhindert." createFilePrevented: "Das Erstellen von Dateien ist durch die Konfiguration verhindert." createDirectoryPrevented: "Das Erstellen von Verzeichnissen ist durch die Konfiguration verhindert." - removeFailed: "Pfad '{}' konnte nicht entfernt werden: {}" + removeSomeFailed: "{} Elemente konnten nicht gelöscht werden, die übrigen wurden gelöscht:\n{}" + removeRefused: "'{}' wird nicht gelöscht: Der Pfad ist nicht absolut, ist ein Wurzelverzeichnis oder enthält Ihr Home-Verzeichnis." opener: sessionBusUnavailable: "Der D-Bus-Sitzungsbus ist nicht verfügbar." sessionBusUnavailableOpenFile: "Der Sitzungsbus ist nicht verfügbar, Dateien können nicht über xdg-desktop-portal geöffnet werden. Das passiert typischerweise bei Ausführung als root oder ohne D-Bus-Benutzersitzung." @@ -1124,9 +1169,13 @@ backend: openFileFailed: "Datei konnte nicht geöffnet werden: {}" openUriUnavailable: "Die OpenURI-Schnittstelle von xdg-desktop-portal ist nicht verfügbar." documentPortalUnavailable: "xdg-document-portal ist nicht verfügbar, das Öffnen über Dateideskriptoren ist in diesem Flatpak nicht möglich." + risk: + program: "ist ein Programm" + application: "ist eine Anwendung" + dangerousType: "ist ein Dateityp, der Code ausführt, etwa ein Skript, ein Installer oder eine Verknüpfung" + refused: "'{}' wurde nicht geöffnet: Das Öffnen von Dateien, die Code ausführen können, ist in den Einstellungen abgeschaltet." mac: fileNotFound: "Datei nicht gefunden" - executable: "Die Datei ist ein Programm oder eine Anwendung und wird von hier nicht geöffnet" openFailed: "Keine Anwendung konnte '{}' öffnen" windows: outOfResources: "Nicht genügend Speicher oder Ressourcen" @@ -1143,8 +1192,6 @@ backend: dllNotFound: "DLL nicht gefunden" unknownShellError: "Unbekannter Shell-Fehler (Code {})" shellExecuteExFailed: "ShellExecuteExW fehlgeschlagen: {} (Code {})" - blockedByPolicy: "Der Dateityp ist durch die Anlagenrichtlinie gesperrt" - binaryExecutable: "Die Datei ist eine ausführbare Binärdatei" sessionManager: connectFailedUnspecified: "Verbindung zum SSH-Server aus unbekanntem Grund fehlgeschlagen." persistence: diff --git a/static/assets/languages/en_US.yaml b/static/assets/languages/en_US.yaml index 6cffc50b..bfa5ed6e 100644 --- a/static/assets/languages/en_US.yaml +++ b/static/assets/languages/en_US.yaml @@ -9,6 +9,14 @@ confirmDialog: neverShowAgain: "Never show again" andMore: "... and {} more" fileExplorer: + openRiskyHeader: "Open a file that can run code?" + openRiskyText: "'{}' {}. Opening it may run it, only do this if you trust where it came from.\n\nOpen it anyway?" + invalidItemNameHeader: "Invalid Name" + invalidItemName: "'{}' cannot be used as a name. A name may not be empty, '.' or '..', and may not contain '/' or '\\'." + renameTargetExistsHeader: "Item Exists" + renameTargetExists: "'{}' already exists. Replace it?" + existenceCheckFailedHeader: "Existence Check Failed" + existenceCheckFailed: "Could not check which destinations already exist, nothing was transferred: {}" contextMenu: download: "Download" upload: "Upload" @@ -84,12 +92,14 @@ operationQueue: cancelOperationText: "Are you sure you want to cancel the operation?" deletingItem: "Deleting: '{}'" deletingSelectedIn: "Deleting selected items in {}" + deletingLocallyIn: "Deleting local items in {}" entriesFailed: "{} entries failed:\n" errorButton: "Error!" livePage: "Live" noError: "No Error" renameCard: "Rename: '{}' -> '{}'" scanCard: "Scan '{}'" + scanningItems: "Scanning... {} items ({})" unknownError: "Unknown Error" syncDialog: titleFormat: "Synchronize: {} / {}" @@ -122,6 +132,21 @@ syncDialog: synchronize: "Synchronize" confirmHeader: "Synchronize Directories" confirmText: "Synchronize {} with {}?" + confirmDeletes: "This deletes {} items ({}) {}." + deleteSideRemote: "on the server" + deleteSideLocal: "locally" + confirmDeleteRowHeader: "Delete Now" + confirmDeleteRow: "Delete '{}' {} now?" + blockedNotice: "{} entries are left alone because they cannot be read, cannot be transferred, a directory faces a symlink, their names differ only in case, or they changed since the last sync where that would lose data. Click to see them." + blockedTitle: "Entries left alone" + blockReason: + Unreadable: "unreadable" + Excluded: "not transferable" + InvalidName: "invalid name" + TypeConflict: "directory against symlink" + CaseConflict: "names differ only in case" + ChangedOnBothSides: "changed on both sides since the last sync" + ChangedOnDestination: "changed on the side it would replace or delete since the last sync" tree: loading: "Loading…" loadFailed: "Failed: {}" @@ -138,6 +163,7 @@ syncProgressDialog: differencesFound: "Differences found" hashing: "Hashing..." cancel: "Cancel" + scanFailed: "Listing failed, nothing was compared." reconnectDialog: attempt: "Attempt {}" retryIn: "Retrying in {} s" @@ -162,7 +188,7 @@ remoteSideModel: failedToCreateDeleteOperation: "Failed to create delete operation: {}" deleteMultipleConfirm: "Are you sure you want to delete the following {} items?" deleteSingleConfirm: "Are you sure you want to delete this item '{}'?" - deleteItemsQuestion: "Delete Items?" + deleteRemoteItemsQuestion: "Delete on the server?" failedToDeleteItems: "Failed to delete one or more files / directories: {}." downloadFailed: "Download Failed" failedToCreateDownloadOperation: "Failed to create download operation: {}." @@ -200,6 +226,8 @@ localSideModel: portalUnavailableText: "The desktop portal used to open files in other applications is not available. \"Open\", \"Open With\", and \"Open in File Manager\" have been disabled.\n\n{}" archiveUploadFailed: "Archive Upload Failed" deleteSelectedConfirm: "Are you sure you want to delete the selected items?" + deleteLocalItemsQuestion: "Delete on this computer?" + deleteDirectoriesWithContent: "Folders are deleted with everything inside them. This cannot be undone." remoteSideUnavailable: "Remote side is not available" transferFailedTitle: "File Transfer Failed" sessionFrontend: @@ -446,6 +474,9 @@ fileTracking: columnAutoReupload: "Auto Reupload" columnMoveRemote: "Move Remote" columnDeleteRemote: "Delete Remote" + columnCheckRemote: "Check Server" + serverChangedHeader: "Changed on the Server" + serverChangedText: "'{}' was changed on the server since it was opened here. Replace the server's version with your local one?\n\nNo keeps the server's version, a later save asks again." columnStatus: "Status" stopButton: "Stop" statusWatching: "Watching" @@ -720,6 +751,8 @@ settings: doCleanup: "Do Cleanup" doCleanupHelpText: "Perform cleanup after failed downloads (Prevents continuation of previously failed downloads)." + markAsDownloaded: "Mark as Downloaded" + markAsDownloadedHelpText: "Mark downloaded files as coming from the internet, so Windows (Mark of the Web) and macOS (quarantine) warn before running them. On by default." symlinkHandlingHelpText: "How to handle symbolic links during transfers. 'AsSymlink' will transfer symlinks as symlinks (if supported by the filesystem), 'FollowSymlink' will follow the symlink and transfer the target file or directory, and 'SkipSymlink' will skip transferring symlinks entirely." symlinkHandling: "Symlink Handling" failFastHelpText: "Fail fast on errors during download operations instead of trying to continue." @@ -982,6 +1015,11 @@ settings: preventCreateDirectory: "Prevent Create Directory" homeOverrideHelpText: "If set, overrides the default home directory path used by the application." homeOverride: "Home Directory Override" + openRiskyFilesHelpText: "What happens when a file that can run code (a program, script, installer or shortcut) is opened from the file explorer, including files opened from the server." + openRiskyFiles: "Opening Files That Can Run Code" + openRiskyFilesAllow: "Always allow" + openRiskyFilesWarn: "Ask first" + openRiskyFilesNever: "Never" temporaryDownloadsDirectoryHelpText: "The directory where downloaded files are temporarily stored. Defaults to %temp%/nui-sftp-downloads." temporaryDownloadsDirectory: "Temporary Downloads Directory" fileTrackingOptions: @@ -991,6 +1029,8 @@ settings: moveRemoteOnLocalMoveHelpText: "Default for new tracked entries: when a tracked file is moved or renamed locally, perform the same move on the remote." deleteRemoteOnLocalDelete: "Delete Remote on Local Delete" deleteRemoteOnLocalDeleteHelpText: "Default for new tracked entries: when a tracked file is deleted locally, also delete it on the remote." + checkRemoteChanges: "Check Server Before Re-upload" + checkRemoteChangesHelpText: "Default for new tracked entries: before an edited file is uploaded again, check whether the copy on the server changed since it was opened and ask before replacing it." colorSetting: pickButton: "Pick" persistence: @@ -1113,12 +1153,17 @@ terminalPanel: saveDialogTitle: "Save terminal contents" saveToFile: "Save terminal to file" backend: + rename: + targetExists: "An item with this name already exists." + invalidName: "The new name must be a plain name in the same directory." rpcFilesystem: + fileExists: "'{}' already exists and was not replaced." deletionPrevented: "File deletion is prevented by configuration." renamePrevented: "File renaming is prevented by configuration." createFilePrevented: "File creation is prevented by configuration." createDirectoryPrevented: "Directory creation is prevented by configuration." - removeFailed: "Failed to remove path '{}': {}" + removeSomeFailed: "{} items could not be deleted, the others were:\n{}" + removeRefused: "Refusing to delete '{}': it is not an absolute path, it is a root directory, or it contains your home directory." opener: sessionBusUnavailable: "D-Bus session bus is unavailable." sessionBusUnavailableOpenFile: "Session bus is unavailable; cannot open files via xdg-desktop-portal. This typically happens when running as root or without a user D-Bus session." @@ -1126,9 +1171,13 @@ backend: openFileFailed: "Failed to open file: {}" openUriUnavailable: "xdg-desktop-portal OpenURI interface is unavailable." documentPortalUnavailable: "xdg-document-portal is unavailable; file-descriptor-based opening is not possible in this flatpak." + risk: + program: "is a program" + application: "is an application" + dangerousType: "is a type that runs code, like a script, an installer or a shortcut" + refused: "'{}' was not opened: opening files that can run code is turned off in the settings." mac: fileNotFound: "File not found" - executable: "The file is a program or application and is not opened from here" openFailed: "No application could open '{}'" windows: outOfResources: "Out of memory or resources" @@ -1145,8 +1194,6 @@ backend: dllNotFound: "DLL not found" unknownShellError: "Unknown shell error (code {})" shellExecuteExFailed: "ShellExecuteExW failed: {} (code {})" - blockedByPolicy: "File type is blocked by attachment policy" - binaryExecutable: "File is a binary executable" sessionManager: connectFailedUnspecified: "Failed to connect to ssh server for unspecified reason." persistence: diff --git a/static/styles/sync_dialog.css b/static/styles/sync_dialog.css index debf5f98..f5d6b874 100644 --- a/static/styles/sync_dialog.css +++ b/static/styles/sync_dialog.css @@ -448,6 +448,27 @@ border-color: var(--warning-color, #f5a623); } +.sync-blocked-notice { + display: flex; + align-items: center; + gap: 0.4rem; + padding: 0.25rem 0.75rem; + font-size: 0.8rem; + color: var(--warning-color, #f5a623); + cursor: pointer; +} + +.sync-blocked-notice:hover span { + text-decoration: underline; +} + +.sync-blocked-notice svg { + width: 1rem; + height: 1rem; + flex-shrink: 0; + fill: currentColor; +} + .sync-footer-summary { margin-left: auto; font-size: 0.8rem; @@ -515,6 +536,11 @@ font-size: 0.95rem; } +.sync-progress-failure-reason { + color: var(--script-nui-components-toast-error, #d9534f); + overflow-wrap: anywhere; +} + .sync-progress-counters { display: flex; gap: 24px; diff --git a/tar-archive/include/tar_archive/entry_reader.hpp b/tar-archive/include/tar_archive/entry_reader.hpp index 9eebaa4e..41dc74a4 100644 --- a/tar-archive/include/tar_archive/entry_reader.hpp +++ b/tar-archive/include/tar_archive/entry_reader.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include @@ -21,7 +21,7 @@ namespace TarArchive * * Lifecycle: Reader::nextEntry consumes the 512-byte USTAR record (plus any leading * PAX extended-header records) from the decompressed byte stream before constructing - * the EntryReader. By the time the caller receives the handle, directoryEntry() is + * the EntryReader. By the time the caller receives the handle, header() is * fully populated and the underlying stream is positioned at the first payload byte. * No further stream I/O happens until the caller calls read() or skip(). */ @@ -42,7 +42,7 @@ namespace TarArchive * Available immediately on handle receipt; populated from the header records that * Reader::nextEntry already consumed before returning the handle. */ - SharedData::DirectoryEntry const& directoryEntry() const noexcept { return meta_; } + MemberHeader const& header() const noexcept { return meta_; } /** * @brief Read up to buf.size() bytes of this entry's payload. @@ -71,10 +71,10 @@ namespace TarArchive private: friend class Reader; - EntryReader(Reader* owner, SharedData::DirectoryEntry meta, std::uint64_t size) noexcept; + EntryReader(Reader* owner, MemberHeader meta, std::uint64_t size) noexcept; Reader* owner_{nullptr}; - SharedData::DirectoryEntry meta_{}; + MemberHeader meta_{}; std::uint64_t size_{0}; std::uint64_t bytesRead_{0}; bool drained_{true}; diff --git a/tar-archive/include/tar_archive/header.hpp b/tar-archive/include/tar_archive/header.hpp index 07f1cde7..7bb9eec1 100644 --- a/tar-archive/include/tar_archive/header.hpp +++ b/tar-archive/include/tar_archive/header.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include @@ -96,7 +96,7 @@ namespace TarArchive * @brief Build the on-disk tar records for a DirectoryEntry, emitting PAX extended * headers when the entry's path, link target, or size exceed plain USTAR limits. */ - std::expected buildRecords(SharedData::DirectoryEntry const& meta); + std::expected buildRecords(MemberHeader const& meta); /** * @brief Parsed representation of a single USTAR file record. @@ -160,8 +160,8 @@ namespace TarArchive /** * @brief Apply PAX overrides on top of a parsed USTAR record and produce a populated - * SharedData::DirectoryEntry to hand out to the caller. + * MemberHeader to hand out to the caller. */ - SharedData::DirectoryEntry - assembleDirectoryEntry(ParsedUstar const& ustar, PaxOverrides const& overrides); + MemberHeader + assembleMemberHeader(ParsedUstar const& ustar, PaxOverrides const& overrides); } diff --git a/tar-archive/include/tar_archive/member_header.hpp b/tar-archive/include/tar_archive/member_header.hpp new file mode 100644 index 00000000..141acfef --- /dev/null +++ b/tar-archive/include/tar_archive/member_header.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace TarArchive +{ + /** + * @brief What a tar record says about one member. + */ + struct MemberHeader + { + /** + * @brief The member path, '/' separated and relative. Read from an archive it is untrusted text that can + * hold "..", an absolute path or anything else. + */ + std::string path{}; + SharedData::FileType type{SharedData::FileType::Regular}; + std::uint64_t size{0}; + std::uint32_t uid{0}; + std::uint32_t gid{0}; + std::string owner{}; + std::string group{}; + std::filesystem::perms permissions{std::filesystem::perms::unknown}; + std::uint64_t mtime{0}; + /** @brief For symlinks: the link literal as UTF-8. */ + std::optional linkTarget{}; + }; + + /** + * @brief The header for @p entry stored under @p path. + */ + inline MemberHeader memberHeaderOf(std::string path, SharedData::DirectoryEntry const& entry) + { + return MemberHeader{ + .path = std::move(path), + .type = entry.type, + .size = entry.size, + .uid = entry.uid, + .gid = entry.gid, + .owner = entry.owner, + .group = entry.group, + .permissions = entry.permissions, + .mtime = entry.mtime, + .linkTarget = entry.linkTarget, + }; + } +} diff --git a/tar-archive/include/tar_archive/writer.hpp b/tar-archive/include/tar_archive/writer.hpp index 4a67780e..c2134488 100644 --- a/tar-archive/include/tar_archive/writer.hpp +++ b/tar-archive/include/tar_archive/writer.hpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include @@ -39,7 +39,7 @@ namespace TarArchive * produced by this Writer is still alive, or with WriteAfterFinalize if the Writer * has already been finalised. */ - std::expected beginEntry(SharedData::DirectoryEntry const& meta); + std::expected beginEntry(MemberHeader const& meta); /** * @brief Flush pending compression buffers, write two trailing zero records and diff --git a/tar-archive/source/tar_archive/entry_reader.cpp b/tar-archive/source/tar_archive/entry_reader.cpp index dcaf2e13..4d9d645e 100644 --- a/tar-archive/source/tar_archive/entry_reader.cpp +++ b/tar-archive/source/tar_archive/entry_reader.cpp @@ -5,7 +5,7 @@ namespace TarArchive { - EntryReader::EntryReader(Reader* owner, SharedData::DirectoryEntry meta, std::uint64_t size) noexcept + EntryReader::EntryReader(Reader* owner, MemberHeader meta, std::uint64_t size) noexcept : owner_{owner} , meta_{std::move(meta)} , size_{size} diff --git a/tar-archive/source/tar_archive/header.cpp b/tar-archive/source/tar_archive/header.cpp index 9ce18e9e..7189f707 100644 --- a/tar-archive/source/tar_archive/header.cpp +++ b/tar-archive/source/tar_archive/header.cpp @@ -227,9 +227,8 @@ namespace TarArchive return std::nullopt; } - std::string normaliseArchivePath(std::filesystem::path const& source) + std::string normaliseArchivePath(std::string value) { - std::string value = source.generic_string(); while (!value.empty() && value.front() == '/') value.erase(value.begin()); std::string collapsed; @@ -271,7 +270,7 @@ namespace TarArchive RawRecord& record, std::string const& name, std::string const& prefix, - SharedData::DirectoryEntry const& meta, + MemberHeader const& meta, TypeFlag typeflag, std::uint64_t payloadSize, std::string const& linkTarget @@ -576,17 +575,16 @@ namespace TarArchive return overrides; } - SharedData::DirectoryEntry - assembleDirectoryEntry(ParsedUstar const& ustar, PaxOverrides const& overrides) + MemberHeader + assembleMemberHeader(ParsedUstar const& ustar, PaxOverrides const& overrides) { - SharedData::DirectoryEntry entry{}; + MemberHeader entry{}; std::string chosenPath = overrides.path.value_or(ustar.fullName()); if (ustar.typeflag == TypeFlag::Directory && !chosenPath.empty() && chosenPath.back() == '/') chosenPath.pop_back(); - entry.path = std::filesystem::path{chosenPath}; - entry.fullPath = entry.path; + entry.path = std::move(chosenPath); entry.type = fileTypeFromTypeFlag(ustar.typeflag); entry.size = overrides.size.value_or(ustar.size); entry.uid = overrides.uid.value_or(ustar.uid); @@ -599,12 +597,12 @@ namespace TarArchive if (ustar.typeflag == TypeFlag::SymbolicLink) { const std::string linkText = overrides.linkPath.value_or(ustar.linkName); - entry.linkTarget = std::filesystem::path{linkText}; + entry.linkTarget = linkText; } return entry; } - std::expected buildRecords(SharedData::DirectoryEntry const& meta) + std::expected buildRecords(MemberHeader const& meta) { const auto typeflagOrError = typeFlagFromFileType(meta.type); if (!typeflagOrError) @@ -613,7 +611,7 @@ namespace TarArchive const bool isDirectory = (typeflag == TypeFlag::Directory); const std::string normalisedPath = - normaliseArchivePath(meta.fullPath.empty() ? meta.path : meta.fullPath); + normaliseArchivePath(meta.path); if (normalisedPath.empty()) return std::unexpected(makeError(TarErrorCode::InvalidHeader, "entry path is empty")); @@ -624,7 +622,7 @@ namespace TarArchive return std::unexpected( makeError(TarErrorCode::InvalidHeader, "symlink entry has no linkTarget set") ); - linkTargetText = meta.linkTarget->generic_string(); + linkTargetText = *meta.linkTarget; } const std::uint64_t payloadSize = isDirectory ? 0u : meta.size; @@ -658,7 +656,7 @@ namespace TarArchive if (!paxEntries.empty()) { const std::string paxPayload = buildPaxPayload(paxEntries); - SharedData::DirectoryEntry paxMeta{}; + MemberHeader paxMeta{}; paxMeta.permissions = std::filesystem::perms{0644u}; paxMeta.mtime = meta.mtime; paxMeta.owner = meta.owner; diff --git a/tar-archive/source/tar_archive/reader.cpp b/tar-archive/source/tar_archive/reader.cpp index 4d44118d..c8908f15 100644 --- a/tar-archive/source/tar_archive/reader.cpp +++ b/tar-archive/source/tar_archive/reader.cpp @@ -188,8 +188,8 @@ namespace TarArchive continue; } - SharedData::DirectoryEntry meta = - assembleDirectoryEntry(*parsed, overrides); + MemberHeader meta = + assembleMemberHeader(*parsed, overrides); const std::uint64_t payloadSize = (parsed->typeflag == TypeFlag::Directory || diff --git a/tar-archive/source/tar_archive/writer.cpp b/tar-archive/source/tar_archive/writer.cpp index 2438d683..e9fed7f0 100644 --- a/tar-archive/source/tar_archive/writer.cpp +++ b/tar-archive/source/tar_archive/writer.cpp @@ -80,7 +80,7 @@ namespace TarArchive } std::expected - Writer::beginEntry(SharedData::DirectoryEntry const& meta) + Writer::beginEntry(MemberHeader const& meta) { if (impl_ == nullptr || impl_->finalized) return std::unexpected( diff --git a/tar-archive/test/tar_archive/main.cpp b/tar-archive/test/tar_archive/main.cpp index e5ff5eb4..dcce4655 100644 --- a/tar-archive/test/tar_archive/main.cpp +++ b/tar-archive/test/tar_archive/main.cpp @@ -1,3 +1,4 @@ +#include #include "test_header.hpp" #include "test_compression.hpp" #include "test_roundtrip.hpp" @@ -11,11 +12,16 @@ #include std::filesystem::path programDirectory; +/** @brief The temporary directory of this process alone, so tests running in parallel share no files. */ +std::filesystem::path temporaryRoot; int main(int argc, char** argv) { programDirectory = std::filesystem::path{argv[0]}.parent_path(); + const Utility::ProcessTemporaryDirectory processTemporary{programDirectory / "temp"}; + temporaryRoot = processTemporary.path(); + ::testing::InitGoogleTest(&argc, argv); ::testing::FLAGS_gtest_death_test_style = "threadsafe"; return RUN_ALL_TESTS(); diff --git a/tar-archive/test/tar_archive/test_compression.hpp b/tar-archive/test/tar_archive/test_compression.hpp index b8343e82..01ea9d66 100644 --- a/tar-archive/test/tar_archive/test_compression.hpp +++ b/tar-archive/test/tar_archive/test_compression.hpp @@ -13,6 +13,7 @@ #include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace TarArchive::Test { @@ -20,7 +21,7 @@ namespace TarArchive::Test { std::filesystem::path compressionScratchDirectory() { - const auto directory = programDirectory / "temp" / "tar_archive_compression"; + const auto directory = temporaryRoot / "tar_archive_compression"; std::filesystem::create_directories(directory); return directory; } diff --git a/tar-archive/test/tar_archive/test_gnu_interop.hpp b/tar-archive/test/tar_archive/test_gnu_interop.hpp index 3180bda2..d2664abf 100644 --- a/tar-archive/test/tar_archive/test_gnu_interop.hpp +++ b/tar-archive/test/tar_archive/test_gnu_interop.hpp @@ -10,12 +10,14 @@ #include #include #include +#include #include #include #include #include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace TarArchive::Test { @@ -23,7 +25,7 @@ namespace TarArchive::Test { std::filesystem::path gnuInteropDirectory() { - const auto directory = programDirectory / "temp" / "tar_archive_gnu_interop"; + const auto directory = temporaryRoot / "tar_archive_gnu_interop"; std::filesystem::create_directories(directory); return directory; } @@ -73,13 +75,12 @@ namespace TarArchive::Test return std::system(full.c_str()); } - SharedData::DirectoryEntry regularEntryForPath( + TarArchive::MemberHeader regularEntryForPath( std::string const& archiveName, std::uint64_t size ) { - SharedData::DirectoryEntry entry{}; - entry.path = std::filesystem::path{archiveName}; - entry.fullPath = entry.path; + TarArchive::MemberHeader entry{}; + entry.path = archiveName; entry.type = SharedData::FileType::Regular; entry.size = size; entry.permissions = static_cast(0644u); @@ -109,6 +110,15 @@ namespace TarArchive::Test tarChecked_ = true; tarAvailable_ = !tarBinary_.empty() && std::filesystem::exists(tarBinary_); + // GNU tar reads "C:/x" as the file x on a remote host named C unless told otherwise. + const auto versionOutput = gnuInteropDirectory() / "tar_version.txt"; + if (tarAvailable_ && runShell(tarBinary_ + " --version", versionOutput) == 0) + { + std::ifstream stream{versionOutput}; + const std::string version{std::istreambuf_iterator{stream}, std::istreambuf_iterator{}}; + if (version.find("GNU tar") != std::string::npos) + tarBinary_ += " --force-local"; + } } void SetUp() override @@ -149,7 +159,7 @@ namespace TarArchive::Test } const int status = runShell( - tarBinary_ + " -tf '" + archivePath.string() + "'", listOutput + tarBinary_ + " -tf '" + archivePath.generic_string() + "'", listOutput ); ASSERT_EQ(status, 0) << "tar -tf exited with non-zero status"; @@ -184,8 +194,8 @@ namespace TarArchive::Test } const std::string command = - tarBinary_ + " -xf '" + archivePath.string() + - "' -C '" + extractDir.string() + "'"; + tarBinary_ + " -xf '" + archivePath.generic_string() + + "' -C '" + extractDir.generic_string() + "'"; const int status = runShell(command, directory / "library_content.log"); ASSERT_EQ(status, 0) << "tar -xf exited with non-zero status"; @@ -218,8 +228,8 @@ namespace TarArchive::Test const auto archivePath = directory / "tar_produced.tar"; - std::string createCommand = tarBinary_ + " -cf '" + archivePath.string() + "' -C '" + - sourceTree.string() + "'"; + std::string createCommand = tarBinary_ + " -cf '" + archivePath.generic_string() + "' -C '" + + sourceTree.generic_string() + "'"; for (auto const& fixture : fixtures) createCommand += " " + fixture.name; @@ -237,8 +247,8 @@ namespace TarArchive::Test ASSERT_TRUE(next->has_value()) << "premature end of archive"; auto entry = std::move(**next); - EXPECT_EQ(entry.directoryEntry().path.generic_string(), fixture.name); - EXPECT_EQ(entry.directoryEntry().size, fixture.payload.size()); + EXPECT_EQ(entry.header().path, fixture.name); + EXPECT_EQ(entry.header().size, fixture.payload.size()); std::vector accumulator; std::array scratch{}; @@ -283,7 +293,7 @@ namespace TarArchive::Test } const int status = runShell( - tarBinary_ + " -tzf '" + archivePath.string() + "'", listOutput + tarBinary_ + " -tzf '" + archivePath.generic_string() + "'", listOutput ); ASSERT_EQ(status, 0) << "tar -tzf exited with non-zero status"; diff --git a/tar-archive/test/tar_archive/test_header.hpp b/tar-archive/test/tar_archive/test_header.hpp index b6c0ab81..292e59d1 100644 --- a/tar-archive/test/tar_archive/test_header.hpp +++ b/tar-archive/test/tar_archive/test_header.hpp @@ -14,11 +14,10 @@ namespace TarArchive::Test { namespace { - SharedData::DirectoryEntry makeRegularEntry(std::string const& path, std::uint64_t size) + TarArchive::MemberHeader makeRegularEntry(std::string const& path, std::uint64_t size) { - SharedData::DirectoryEntry entry{}; - entry.path = std::filesystem::path{path}; - entry.fullPath = entry.path; + TarArchive::MemberHeader entry{}; + entry.path = path; entry.type = SharedData::FileType::Regular; entry.size = size; entry.permissions = static_cast(0644u); @@ -66,9 +65,8 @@ namespace TarArchive::Test TEST(TarHeader, BuildRecordsForDirectoryHasTrailingSlashAndNoSize) { - SharedData::DirectoryEntry entry{}; + TarArchive::MemberHeader entry{}; entry.path = "var/log"; - entry.fullPath = "var/log"; entry.type = SharedData::FileType::Directory; entry.permissions = static_cast(0755u); @@ -114,9 +112,8 @@ namespace TarArchive::Test TEST(TarHeader, RejectsUnsupportedFileType) { - SharedData::DirectoryEntry entry{}; + TarArchive::MemberHeader entry{}; entry.path = "foo"; - entry.fullPath = "foo"; entry.type = SharedData::FileType::Unknown; const auto built = buildRecords(entry); ASSERT_FALSE(built.has_value()); @@ -125,11 +122,10 @@ namespace TarArchive::Test TEST(TarHeader, SymlinkPopulatesLinkNameField) { - SharedData::DirectoryEntry entry{}; + TarArchive::MemberHeader entry{}; entry.path = "link"; - entry.fullPath = "link"; entry.type = SharedData::FileType::Symlink; - entry.linkTarget = std::filesystem::path{"target.txt"}; + entry.linkTarget = "target.txt"; entry.permissions = static_cast(0777u); const auto built = buildRecords(entry); @@ -155,9 +151,8 @@ namespace TarArchive::Test TEST(TarHeader, LargeSizeTriggersPaxExtendedHeader) { - SharedData::DirectoryEntry entry{}; + TarArchive::MemberHeader entry{}; entry.path = "big.bin"; - entry.fullPath = "big.bin"; entry.type = SharedData::FileType::Regular; entry.size = 1ull << 34; entry.permissions = static_cast(0644u); diff --git a/tar-archive/test/tar_archive/test_misuse.hpp b/tar-archive/test/tar_archive/test_misuse.hpp index 679b6c31..0cb0aca2 100644 --- a/tar-archive/test/tar_archive/test_misuse.hpp +++ b/tar-archive/test/tar_archive/test_misuse.hpp @@ -12,6 +12,7 @@ #include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace TarArchive::Test { @@ -19,16 +20,15 @@ namespace TarArchive::Test { std::filesystem::path misuseDirectory() { - const auto directory = programDirectory / "temp" / "tar_archive_misuse"; + const auto directory = temporaryRoot / "tar_archive_misuse"; std::filesystem::create_directories(directory); return directory; } - SharedData::DirectoryEntry smallRegular(std::uint64_t size) + TarArchive::MemberHeader smallRegular(std::uint64_t size) { - SharedData::DirectoryEntry entry{}; + TarArchive::MemberHeader entry{}; entry.path = "x.bin"; - entry.fullPath = "x.bin"; entry.type = SharedData::FileType::Regular; entry.size = size; entry.permissions = static_cast(0644u); @@ -120,9 +120,8 @@ namespace TarArchive::Test ASSERT_TRUE(entry->write(std::span{bytes}).has_value()); ASSERT_TRUE(std::move(*entry).close().has_value()); - SharedData::DirectoryEntry second = smallRegular(0u); + TarArchive::MemberHeader second = smallRegular(0u); second.path = "second.bin"; - second.fullPath = "second.bin"; auto secondEntry = writer->beginEntry(second); ASSERT_TRUE(secondEntry.has_value()); ASSERT_TRUE(std::move(*secondEntry).close().has_value()); diff --git a/tar-archive/test/tar_archive/test_pax.hpp b/tar-archive/test/tar_archive/test_pax.hpp index a70060de..dceb3caf 100644 --- a/tar-archive/test/tar_archive/test_pax.hpp +++ b/tar-archive/test/tar_archive/test_pax.hpp @@ -11,6 +11,7 @@ #include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace TarArchive::Test { @@ -18,7 +19,7 @@ namespace TarArchive::Test { std::filesystem::path paxDirectory() { - const auto directory = programDirectory / "temp" / "tar_archive_pax"; + const auto directory = temporaryRoot / "tar_archive_pax"; std::filesystem::create_directories(directory); return directory; } @@ -29,9 +30,8 @@ namespace TarArchive::Test const std::string longName = std::string(180u, 'a') + "/" + std::string(200u, 'b') + "/file.bin"; const std::string contents = "payload for long path entry"; - SharedData::DirectoryEntry meta{}; - meta.path = std::filesystem::path{longName}; - meta.fullPath = meta.path; + TarArchive::MemberHeader meta{}; + meta.path = longName; meta.type = SharedData::FileType::Regular; meta.size = contents.size(); meta.permissions = static_cast(0644u); @@ -59,18 +59,17 @@ namespace TarArchive::Test ASSERT_TRUE(next.has_value()); ASSERT_TRUE(next->has_value()); auto entry = std::move(**next); - EXPECT_EQ(entry.directoryEntry().path.generic_string(), longName); - EXPECT_EQ(entry.directoryEntry().size, contents.size()); + EXPECT_EQ(entry.header().path, longName); + EXPECT_EQ(entry.header().size, contents.size()); } TEST(TarPax, LongSymlinkTargetRoundTripsViaExtendedHeader) { const std::string linkText = std::string(250u, 'x'); - SharedData::DirectoryEntry meta{}; + TarArchive::MemberHeader meta{}; meta.path = "link"; - meta.fullPath = "link"; meta.type = SharedData::FileType::Symlink; - meta.linkTarget = std::filesystem::path{linkText}; + meta.linkTarget = linkText; meta.permissions = static_cast(0777u); const auto path = paxDirectory() / "longlink.tar"; @@ -90,8 +89,8 @@ namespace TarArchive::Test ASSERT_TRUE(next.has_value()); ASSERT_TRUE(next->has_value()); auto entry = std::move(**next); - EXPECT_EQ(entry.directoryEntry().type, SharedData::FileType::Symlink); - ASSERT_TRUE(entry.directoryEntry().linkTarget.has_value()); - EXPECT_EQ(entry.directoryEntry().linkTarget->generic_string(), linkText); + EXPECT_EQ(entry.header().type, SharedData::FileType::Symlink); + ASSERT_TRUE(entry.header().linkTarget.has_value()); + EXPECT_EQ(*entry.header().linkTarget, linkText); } } diff --git a/tar-archive/test/tar_archive/test_roundtrip.hpp b/tar-archive/test/tar_archive/test_roundtrip.hpp index f3b29dec..ba16fe05 100644 --- a/tar-archive/test/tar_archive/test_roundtrip.hpp +++ b/tar-archive/test/tar_archive/test_roundtrip.hpp @@ -13,6 +13,7 @@ #include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace TarArchive::Test { @@ -20,16 +21,15 @@ namespace TarArchive::Test { std::filesystem::path roundTripDirectory() { - const auto directory = programDirectory / "temp" / "tar_archive_roundtrip"; + const auto directory = temporaryRoot / "tar_archive_roundtrip"; std::filesystem::create_directories(directory); return directory; } - SharedData::DirectoryEntry regular(std::string const& path, std::uint64_t size) + TarArchive::MemberHeader regular(std::string const& path, std::uint64_t size) { - SharedData::DirectoryEntry entry{}; - entry.path = std::filesystem::path{path}; - entry.fullPath = entry.path; + TarArchive::MemberHeader entry{}; + entry.path = path; entry.type = SharedData::FileType::Regular; entry.size = size; entry.permissions = static_cast(0644u); @@ -41,11 +41,10 @@ namespace TarArchive::Test return entry; } - SharedData::DirectoryEntry directory(std::string const& path) + TarArchive::MemberHeader directory(std::string const& path) { - SharedData::DirectoryEntry entry{}; - entry.path = std::filesystem::path{path}; - entry.fullPath = entry.path; + TarArchive::MemberHeader entry{}; + entry.path = path; entry.type = SharedData::FileType::Directory; entry.permissions = static_cast(0755u); entry.owner = "alice"; @@ -64,7 +63,7 @@ namespace TarArchive::Test struct WrittenEntry { - SharedData::DirectoryEntry meta; + TarArchive::MemberHeader meta; std::vector payload; }; @@ -117,7 +116,7 @@ namespace TarArchive::Test ASSERT_TRUE(writer.has_value()) << writer.error().toString(); for (auto const& entry : entries) ASSERT_TRUE(writeEntry(*writer, entry).has_value()) - << "failed on " << entry.meta.path.string(); + << "failed on " << entry.meta.path; ASSERT_TRUE(writer->finalize().has_value()); } @@ -130,10 +129,10 @@ namespace TarArchive::Test ASSERT_TRUE(next.has_value()) << next.error().toString(); ASSERT_TRUE(next->has_value()); auto entry = std::move(**next); - EXPECT_EQ(entry.directoryEntry().path, expected.meta.path); - EXPECT_EQ(entry.directoryEntry().type, expected.meta.type); - EXPECT_EQ(entry.directoryEntry().size, expected.meta.size); - EXPECT_EQ(entry.directoryEntry().owner, expected.meta.owner); + EXPECT_EQ(entry.header().path, expected.meta.path); + EXPECT_EQ(entry.header().type, expected.meta.type); + EXPECT_EQ(entry.header().size, expected.meta.size); + EXPECT_EQ(entry.header().owner, expected.meta.owner); if (expected.meta.type == SharedData::FileType::Regular) { const auto payload = readPayload(entry); diff --git a/tar-archive/test/tar_archive/test_size_boundaries.hpp b/tar-archive/test/tar_archive/test_size_boundaries.hpp index 81aff565..fada8e49 100644 --- a/tar-archive/test/tar_archive/test_size_boundaries.hpp +++ b/tar-archive/test/tar_archive/test_size_boundaries.hpp @@ -13,6 +13,7 @@ #include extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace TarArchive::Test { @@ -20,7 +21,7 @@ namespace TarArchive::Test { std::filesystem::path sizeBoundaryDirectory() { - const auto directory = programDirectory / "temp" / "tar_archive_sizes"; + const auto directory = temporaryRoot / "tar_archive_sizes"; std::filesystem::create_directories(directory); return directory; } @@ -33,11 +34,10 @@ namespace TarArchive::Test return payload; } - SharedData::DirectoryEntry regularAtSize(std::uint64_t size) + TarArchive::MemberHeader regularAtSize(std::uint64_t size) { - SharedData::DirectoryEntry entry{}; + TarArchive::MemberHeader entry{}; entry.path = "payload.bin"; - entry.fullPath = "payload.bin"; entry.type = SharedData::FileType::Regular; entry.size = size; entry.permissions = static_cast(0644u); diff --git a/utility/include/utility/directory_traversal.hpp b/utility/include/utility/directory_traversal.hpp index 5727078c..c71faabb 100644 --- a/utility/include/utility/directory_traversal.hpp +++ b/utility/include/utility/directory_traversal.hpp @@ -1,5 +1,10 @@ #pragma once +#include +#include + +#include +#include #include #include #include @@ -30,19 +35,22 @@ namespace Utility std::uint64_t totalBytes_{0}; }; + /** + * @brief Walks a directory tree breadth first. The walker never sees the root path: the scanner lists the + * directory at a path relative to the root, and entries are named relative to their parent. The root + * is entry 0 and has an empty name. + */ template requires std:: - is_invocable_r_v, WalkErrorType>, ScannerT, std::filesystem::path const&> + is_invocable_r_v, WalkErrorType>, ScannerT, Paths::RelativePath const&> class DeepDirectoryWalker : public BaseDirectoryWalker { public: template requires std::is_same_v, ScannerT> - DeepDirectoryWalker(std::filesystem::path rootPath, ForwardingScannerT&& scanner) - : rootPath_{std::move(rootPath)} - , scanner_{std::forward(scanner)} // root is part of the entries: + explicit DeepDirectoryWalker(ForwardingScannerT&& scanner) + : scanner_{std::forward(scanner)} , entries_{EntryT{ - .path = rootPath_, .type = EntryT::FileType::Directory, }} {} @@ -65,9 +73,9 @@ namespace Utility void reset() { entries_ = {EntryT{ - .path = rootPath_, .type = EntryT::FileType::Directory, }}; + rejectedEntries_.clear(); currentIndex_ = 0; totalBytes_ = 0; } @@ -85,7 +93,7 @@ namespace Utility if (entries_[currentIndex_].isDirectory()) { - auto result = scanner_(fullPath(entries_[currentIndex_])); + auto result = scanner_(relativePath(entries_[currentIndex_])); if (!result) return std::unexpected(std::move(result).error()); @@ -97,7 +105,7 @@ namespace Utility // Log::warn( // "DeepDirectoryWalker: Encountered non-directory entry '{}' during walk at a point where a " // "directory was expected, skipping.", - // fullPath(entries_[currentIndex_]).generic_string() + // relativePath(entries_[currentIndex_]).value() // ); } @@ -114,17 +122,29 @@ namespace Utility return completed(); } - std::filesystem::path fullPath(EntryT const& entry) const + /** + * @brief The path of a walked entry below the root, empty for the root. + */ + Paths::RelativePath relativePath(EntryT const& entry) const { - if (entry.parent) - { - const auto parentIndex = entry.parent.value(); - if (parentIndex >= entries_.size()) - throw std::out_of_range("Parent index is out of range"); - return fullPath(entries_[parentIndex]) / entry.path; - } - else - return entry.path; + if (!entry.parent) + return {}; + const auto parentIndex = entry.parent.value(); + if (parentIndex >= entries_.size()) + throw std::out_of_range("Parent index is out of range"); + // Only valid segments are adopted, see moveEntries. + return relativePath(entries_[parentIndex]) / *Paths::PathSegment::make(nameOf(entry)); + } + + /** + * @brief The path of a rejected entry below the root for display, its own name as listed. + */ + std::string describeRejected(EntryT const& entry) const + { + if (!entry.parent || entry.parent.value() >= entries_.size()) + return nameOf(entry); + const auto parent = relativePath(entries_[entry.parent.value()]); + return parent.empty() ? nameOf(entry) : parent.value() + "/" + nameOf(entry); } std::expected walkAll() @@ -144,90 +164,45 @@ namespace Utility return currentIndex_ >= entries_.size(); } + /** + * @brief Entries whose name is not a valid path segment, like ".." or "a/b" from a server. They are never + * walked or returned by @ref entries, their parent index is set so their full path can be shown. + */ + std::vector const& rejectedEntries() const + { + return rejectedEntries_; + } + private: + /** + * @brief Adopts one listing. Every "." and ".." is dropped and every other name has to be a valid path + * segment: a name like "x/.." would turn into a path outside the walked tree. + */ void moveEntries(std::vector&& newEntries, std::size_t parent) { - if constexpr (ScannerIncludesDotAndDotDot) + for (auto& entry : newEntries) { - auto iter = std::make_move_iterator(begin(newEntries)); - auto endMarker = std::make_move_iterator(end(newEntries)); - bool foundOneWasDot = false; - for (; iter != endMarker; ++iter) - { - auto val{*iter}; - if (val.path == ".") - { - foundOneWasDot = true; - ++iter; - break; - } - else if (val.path == "..") - { - ++iter; - break; - } - val.parent = parent; - entries_.push_back(std::move(val)); - } - - if (foundOneWasDot) - { - for (; iter != endMarker; ++iter) - { - auto val{*iter}; - if (val.path == "..") - { - ++iter; - break; - } - val.parent = parent; - entries_.push_back(std::move(val)); - } - } - else + entry.parent = parent; + const auto name = nameOf(entry); + if (name == "." || name == "..") + continue; + if (!Paths::PathSegment::make(name).has_value()) { - for (; iter != endMarker; ++iter) - { - auto val{*iter}; - if (val.path == ".") - { - ++iter; - break; - } - val.parent = parent; - entries_.push_back(std::move(val)); - } + rejectedEntries_.push_back(std::move(entry)); + continue; } - - std::transform( - iter, - endMarker, - std::back_inserter(entries_), - [parent](EntryT&& entry) - { - entry.parent = parent; - return std::move(entry); - } - ); - } - else - { - std::transform( - std::make_move_iterator(begin(newEntries)), - std::make_move_iterator(end(newEntries)), - std::back_inserter(entries_), - [parent](EntryT&& entry) - { - entry.parent = parent; - return std::move(entry); - } - ); + entries_.push_back(std::move(entry)); } } + static std::string const& nameOf(EntryT const& entry) + { + return entry.name; + } + private: - std::filesystem::path rootPath_; ScannerT scanner_; std::vector entries_{}; + std::vector rejectedEntries_{}; }; } \ No newline at end of file diff --git a/utility/include/utility/download_mark.hpp b/utility/include/utility/download_mark.hpp new file mode 100644 index 00000000..8782ce6d --- /dev/null +++ b/utility/include/utility/download_mark.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +namespace Utility +{ + /** + * @brief Marks @p path as downloaded from the internet, so the system warns before running it: the Mark of the + * Web (Zone.Identifier stream, internet zone) on Windows, the quarantine attribute on macOS. Nothing + * elsewhere. A filesystem without streams or extended attributes reports an error, the file is unchanged. + */ + std::error_code markAsDownloaded(std::filesystem::path const& path); +} diff --git a/utility/include/utility/link_safe_remove.hpp b/utility/include/utility/link_safe_remove.hpp new file mode 100644 index 00000000..711d770f --- /dev/null +++ b/utility/include/utility/link_safe_remove.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +namespace Utility +{ + /** + * @brief True for a symlink and, on Windows, for every reparse point such as a junction. std::filesystem may + * report a junction as a plain directory, and walking into it leaves the tree it belongs to. + */ + bool isLinkOrJunction(std::filesystem::path const& path, std::error_code& error); + + /** + * @brief Finds the first existing component of @p path below @p root (root excluded, @p path included) that is + * a symlink or junction. Writing through it would leave @p root. + * @return The offending component, or nothing when there is none. @p error is set when @p path is not lexically + * below @p root or a component cannot be inspected. + */ + std::optional + firstLinkBelow(std::filesystem::path const& root, std::filesystem::path const& path, std::error_code& error); + + /** + * @brief Removes @p path and, for a real directory, everything below it. A symlink or junction is removed + * itself and never followed, so nothing outside @p path is touched. + * @return The number of removed entries, 0 when @p path does not exist. + */ + std::uintmax_t removeAllNoFollow(std::filesystem::path const& path, std::error_code& error); +} diff --git a/utility/include/utility/node/node.hpp b/utility/include/utility/node/node.hpp index 5bbe33f0..22547fd5 100644 --- a/utility/include/utility/node/node.hpp +++ b/utility/include/utility/node/node.hpp @@ -6,6 +6,7 @@ #include +#include #include #include #include @@ -25,8 +26,18 @@ namespace SecureShell::Test std::atomic_bool killed = false; unsigned short port = 0; int code = 0; + /** @brief Receives what the process writes to stderr, which is read and dropped. */ + std::array stderrBuffer{}; void command(std::string const& command); + /** + * @brief Sends a command with arguments and blocks until the process acknowledges it. + * + * @param command The command name. + * @param arguments Extra fields merged into the command object. + * @return true if the acknowledgement arrived. + */ + bool command(std::string const& command, nlohmann::json const& arguments); void terminate(); int wait(); }; diff --git a/utility/include/utility/path_utf.hpp b/utility/include/utility/path_utf.hpp deleted file mode 100644 index 701f3b02..00000000 --- a/utility/include/utility/path_utf.hpp +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include - -#include -#include -#include - -namespace Utility -{ - /** - * @brief Constructs a std::filesystem::path from a UTF-8 encoded string. - * @param utf8 The UTF-8 encoded input. - * - * Analogous to ntoh*: converts from the wire/portable encoding (UTF-8, as - * used over RPC and by Emscripten) to the host's native path encoding. - * On Windows the native encoding is UTF-16, so the input is transcoded; - * on POSIX paths are already UTF-8. - */ - inline std::filesystem::path pathFromUtf8(std::string_view utf8) - { -#ifdef _WIN32 - const std::string owned{utf8}; - const auto utf16 = Nui::utf8ToUtf16(owned); - return std::filesystem::path{utf16}; -#else - return std::filesystem::path{std::string{utf8}}; -#endif - } - - /** - * @brief Converts a std::filesystem::path to a UTF-8 encoded string. - * @param path The path in host-native encoding. - * - * Analogous to hton*: converts from the host's native path encoding to - * the wire/portable encoding (UTF-8) used by RPC and Emscripten. - */ - inline std::string pathToUtf8(std::filesystem::path const& path) - { -#ifdef _WIN32 - return Nui::utf16ToUtf8(path.native()); -#else - return path.native(); -#endif - } - - /** - * @brief Like pathToUtf8 but using the generic (forward-slash) form. - * @param path The path in host-native encoding. - */ - inline std::string pathToUtf8Generic(std::filesystem::path const& path) - { -#ifdef _WIN32 - return Nui::utf16ToUtf8(path.generic_wstring()); -#else - return path.generic_string(); -#endif - } -} diff --git a/utility/include/utility/permissions.hpp b/utility/include/utility/permissions.hpp new file mode 100644 index 00000000..56c661fd --- /dev/null +++ b/utility/include/utility/permissions.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +namespace Utility +{ + /** + * @brief Permissions that are safe to apply to a transferred file or directory. + * + * @param permissions Permissions of the source, unknown when the source did not report any. + * @return nullopt for unknown permissions, otherwise the read, write and execute bits only. Setuid, + * setgid and sticky never travel with a transfer. + */ + std::optional sanitizePermissions(std::filesystem::perms permissions); + + /** + * @brief The file mode creation mask of this process, empty where there is none. + */ + std::filesystem::perms processUmask(); + + /** + * @brief @ref sanitizePermissions with the process umask applied, like the system does for a new file. + */ + std::optional inheritablePermissions(std::filesystem::perms permissions); + + /** + * @brief The permissions of a local file as far as they mean anything. Unknown on Windows, where they are made up + * from the read-only attribute and mark every file executable. + */ + std::filesystem::perms localPermissions(std::filesystem::file_status const& status); +} diff --git a/utility/include/utility/process_temporary_directory.hpp b/utility/include/utility/process_temporary_directory.hpp new file mode 100644 index 00000000..2b8622bf --- /dev/null +++ b/utility/include/utility/process_temporary_directory.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace Utility +{ + /** + * @brief The temporary directory of one test process, below @p base, so tests running in parallel share no files. + * A gtest death test runs in a re-executed copy of the test binary that dies on purpose: it reuses its + * parent's directory, handed over through the environment, and only the parent removes it. + */ + class ProcessTemporaryDirectory + { + public: + explicit ProcessTemporaryDirectory(std::filesystem::path const& base) + { + if (char const* inherited = std::getenv(environmentName); inherited != nullptr && *inherited != '\0') + { + path_ = std::filesystem::path{inherited}; + std::filesystem::create_directories(path_); + return; + } + owned_.emplace(base, true); + path_ = owned_->path(); +#ifdef _WIN32 + ::_putenv_s(environmentName, path_.string().c_str()); +#else + ::setenv(environmentName, path_.c_str(), 1); +#endif + } + + std::filesystem::path const& path() const noexcept + { + return path_; + } + + private: + static constexpr char const* environmentName = "NUI_SFTP_TEST_PROCESS_TEMPORARY"; + + std::optional owned_{}; + std::filesystem::path path_{}; + }; +} diff --git a/utility/include/utility/rename.hpp b/utility/include/utility/rename.hpp new file mode 100644 index 00000000..d144470d --- /dev/null +++ b/utility/include/utility/rename.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +namespace Utility +{ + /** + * @brief Renames @p from to @p to and fails with std::errc::file_exists instead of replacing an existing @p to. + * + * Atomic where the platform offers it: renameat2 RENAME_NOREPLACE on Linux, renamex_np RENAME_EXCL on macOS, + * MoveFileExW without MOVEFILE_REPLACE_EXISTING on Windows. A Linux filesystem without RENAME_NOREPLACE falls + * back to link and unlink for files. + */ + std::expected renameNoReplace(std::filesystem::path const& from, std::filesystem::path const& to); +} diff --git a/utility/include/utility/rename_target.hpp b/utility/include/utility/rename_target.hpp new file mode 100644 index 00000000..775a3c5e --- /dev/null +++ b/utility/include/utility/rename_target.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace Utility +{ + /** + * @brief Whether renaming @p oldPath to @p newPath only changes the name: same parent directory and a new name + * that is a plain path segment. Anything else would move the item elsewhere. + */ + inline bool isRenameInPlace(std::filesystem::path const& oldPath, std::filesystem::path const& newPath) + { + const auto newName = newPath.filename(); + if (newName.empty() || newName == "." || newName == "..") + return false; + return !oldPath.parent_path().empty() && newPath.parent_path() == oldPath.parent_path(); + } +} diff --git a/utility/include/utility/temporary_directory.hpp b/utility/include/utility/temporary_directory.hpp index 95db6021..7bde3acc 100644 --- a/utility/include/utility/temporary_directory.hpp +++ b/utility/include/utility/temporary_directory.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -45,8 +47,9 @@ namespace Utility { std::error_code error; + // Never through a link or Windows junction a test left inside. if (!path_.empty()) - std::filesystem::remove_all(path_, error); + removeAllNoFollow(path_, error); } TemporaryDirectory(TemporaryDirectory const&) = delete; diff --git a/utility/include/utility/user_directories.hpp b/utility/include/utility/user_directories.hpp index 898960f9..5dbbefb4 100644 --- a/utility/include/utility/user_directories.hpp +++ b/utility/include/utility/user_directories.hpp @@ -18,7 +18,8 @@ namespace Utility */ std::string name; - std::filesystem::path path; + /** @brief The absolute remote path as UTF-8. */ + std::string path; }; /** @@ -64,5 +65,5 @@ namespace Utility * @param content The content of the file. * @param home The home directory that $HOME expands to. */ - std::vector parseUserDirectories(std::string_view content, std::filesystem::path const& home); + std::vector parseUserDirectories(std::string_view content, std::string_view home); } diff --git a/utility/source/utility/CMakeLists.txt b/utility/source/utility/CMakeLists.txt index 42f7c120..a669f51a 100644 --- a/utility/source/utility/CMakeLists.txt +++ b/utility/source/utility/CMakeLists.txt @@ -21,6 +21,10 @@ else() utility PRIVATE node/node.cpp + permissions.cpp + rename.cpp + link_safe_remove.cpp + download_mark.cpp resources.cpp ) @@ -39,6 +43,7 @@ target_compile_definitions(utility PRIVATE -DCMD_EXECUTABLE="${CMD}" -DMSYS2_BAS target_link_libraries(utility PUBLIC core-target + paths nlohmann_json::nlohmann_json gimo::gimo fmt::fmt diff --git a/utility/source/utility/download_mark.cpp b/utility/source/utility/download_mark.cpp new file mode 100644 index 00000000..a3ef9b61 --- /dev/null +++ b/utility/source/utility/download_mark.cpp @@ -0,0 +1,44 @@ +#include + +#ifdef _WIN32 +# include +#elif defined(__APPLE__) +# include + +# include +# include +# include +# include +#endif + +namespace Utility +{ + std::error_code markAsDownloaded(std::filesystem::path const& path) + { +#ifdef _WIN32 + auto streamPath = path; + streamPath += L":Zone.Identifier"; + std::ofstream stream{streamPath, std::ios::binary | std::ios::trunc}; + if (!stream) + return std::make_error_code(std::errc::operation_not_supported); + // Zone 3 is the internet zone, SmartScreen and Office's protected view apply. + stream << "[ZoneTransfer]\r\nZoneId=3\r\n"; + if (!stream.flush()) + return std::make_error_code(std::errc::io_error); + return {}; +#elif defined(__APPLE__) + const auto seconds = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch() + ).count(); + char value[64]{}; + // Flags 0081: downloaded, not yet approved by the user; then the time in hex and the downloading agent. + const auto length = std::snprintf(value, sizeof(value), "0081;%llx;nui-sftp;", static_cast(seconds)); + if (::setxattr(path.c_str(), "com.apple.quarantine", value, static_cast(length), 0, 0) != 0) + return std::error_code{errno, std::system_category()}; + return {}; +#else + static_cast(path); + return {}; +#endif + } +} diff --git a/utility/source/utility/link_safe_remove.cpp b/utility/source/utility/link_safe_remove.cpp new file mode 100644 index 00000000..b2ed6b2c --- /dev/null +++ b/utility/source/utility/link_safe_remove.cpp @@ -0,0 +1,113 @@ +#include + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# include +#endif + +namespace Utility +{ + bool isLinkOrJunction(std::filesystem::path const& path, std::error_code& error) + { + error.clear(); +#ifdef _WIN32 + const auto attributes = ::GetFileAttributesW(path.c_str()); + if (attributes == INVALID_FILE_ATTRIBUTES) + { + const auto lastError = ::GetLastError(); + if (lastError != ERROR_FILE_NOT_FOUND && lastError != ERROR_PATH_NOT_FOUND) + error = std::error_code{static_cast(lastError), std::system_category()}; + return false; + } + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0; +#else + const auto status = std::filesystem::symlink_status(path, error); + if (status.type() == std::filesystem::file_type::not_found) + error.clear(); + return !error && std::filesystem::is_symlink(status); +#endif + } + + std::optional + firstLinkBelow(std::filesystem::path const& root, std::filesystem::path const& path, std::error_code& error) + { + error.clear(); + const auto relative = path.lexically_relative(root); + if (relative.empty() || *relative.begin() == "..") + { + error = std::make_error_code(std::errc::invalid_argument); + return std::nullopt; + } + auto current = root; + for (auto const& component : relative) + { + if (component == ".") + continue; + current /= component; + const bool isLink = isLinkOrJunction(current, error); + if (error) + return std::nullopt; + if (isLink) + return current; + } + return std::nullopt; + } + + namespace + { + bool removeLink(std::filesystem::path const& path, std::error_code& error) + { +#ifdef _WIN32 + // A junction or directory symlink is removed like an empty directory, its target stays. + const auto attributes = ::GetFileAttributesW(path.c_str()); + const bool removed = (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0 ? ::RemoveDirectoryW(path.c_str()) != 0 + : ::DeleteFileW(path.c_str()) != 0; + if (!removed) + error = std::error_code{static_cast(::GetLastError()), std::system_category()}; + return removed; +#else + return std::filesystem::remove(path, error); +#endif + } + } + + std::uintmax_t removeAllNoFollow(std::filesystem::path const& path, std::error_code& error) + { + error.clear(); + const bool isLink = isLinkOrJunction(path, error); + if (error) + return 0; + if (isLink) + return removeLink(path, error) ? 1u : 0u; + + const auto status = std::filesystem::symlink_status(path, error); + if (status.type() == std::filesystem::file_type::not_found) + { + error.clear(); + return 0; + } + if (error) + return 0; + + std::uintmax_t removed = 0; + if (std::filesystem::is_directory(status)) + { + std::filesystem::directory_iterator iter{path, error}; + if (error) + return removed; + for (; iter != std::filesystem::directory_iterator{}; iter.increment(error)) + { + if (error) + return removed; + removed += removeAllNoFollow(iter->path(), error); + if (error) + return removed; + } + if (error) + return removed; + } + if (std::filesystem::remove(path, error)) + ++removed; + return removed; + } +} diff --git a/utility/source/utility/node/node.cpp b/utility/source/utility/node/node.cpp index 8672cae7..56cd6560 100644 --- a/utility/source/utility/node/node.cpp +++ b/utility/source/utility/node/node.cpp @@ -6,7 +6,9 @@ #pragma clang diagnostic pop #include +#include +#include #include #include #include @@ -46,6 +48,46 @@ namespace SecureShell::Test std::cerr << "Failed to write command: " << ec.message() << std::endl; return; } + // Node on Windows cannot finish exiting while a read on its stdin pipe is pending. + if (command == "exit") + stdinPipe.close(ec); + } + + bool NodeProcessResult::command(std::string const& command, nlohmann::json const& arguments) + { + if (port == 0 || killed || code != 0) + return false; + + nlohmann::json commandObject = arguments.is_object() ? arguments : nlohmann::json::object(); + commandObject["command"] = command; + commandObject["ack"] = true; +#ifdef _WIN32 + const std::string commandWithNewline = commandObject.dump() + "\r\n"; +#else + const std::string commandWithNewline = commandObject.dump() + "\n"; +#endif + + boost::system::error_code ec; + boost::asio::write(stdinPipe, boost::asio::buffer(commandWithNewline), ec); + if (ec) + { + std::cerr << "Failed to write command: " << ec.message() << std::endl; + return false; + } + + std::string received{}; + std::array buffer{}; + while (received.find('\n') == std::string::npos) + { + const auto readAmount = stdoutPipe.read_some(boost::asio::buffer(buffer), ec); + if (ec || readAmount == 0) + { + std::cerr << "Failed to read command acknowledgement: " << ec.message() << std::endl; + return false; + } + received.append(buffer.data(), readAmount); + } + return received.find(fmt::format("\"ack\":\"{}\"", command)) != std::string::npos; } int NodeProcessResult::wait() @@ -118,6 +160,20 @@ namespace SecureShell::Test child.wait(); } + void drainStandardError(std::shared_ptr const& result) + { + result->stderrPipe.async_read_some( + boost::asio::buffer(result->stderrBuffer), + [weak = std::weak_ptr{result}](boost::system::error_code const& ec, std::size_t) + { + if (ec) + return; + if (auto result = weak.lock()) + drainStandardError(result); + } + ); + } + std::shared_ptr nodeProcess( boost::asio::any_io_executor executor, Utility::TemporaryDirectory const& isolateDirectory, @@ -197,6 +253,9 @@ namespace SecureShell::Test return result; } + // Windows writes to pipes synchronously: once nobody reads stderr and its small buffer is full, the + // process blocks for good and only the kill timer ends it. + drainStandardError(result); return result; } } \ No newline at end of file diff --git a/utility/source/utility/permissions.cpp b/utility/source/utility/permissions.cpp new file mode 100644 index 00000000..956c6d12 --- /dev/null +++ b/utility/source/utility/permissions.cpp @@ -0,0 +1,66 @@ +#include + +#include +#include + +#ifndef _WIN32 +# include +#endif + +namespace Utility +{ + namespace + { + std::filesystem::perms readProcessUmask() + { +#ifdef _WIN32 + return std::filesystem::perms::none; +#else +# ifdef __linux__ + // Reading /proc leaves the mask alone, changing it would race other threads creating files. + std::ifstream status{"/proc/self/status"}; + std::string line; + while (std::getline(status, line)) + { + if (line.starts_with("Umask:")) + return static_cast(std::stoul(line.substr(6), nullptr, 8)); + } +# endif + const auto mask = ::umask(0); + ::umask(mask); + return static_cast(mask); +#endif + } + } + + std::optional sanitizePermissions(std::filesystem::perms permissions) + { + if (permissions == std::filesystem::perms::unknown) + return std::nullopt; + return permissions & std::filesystem::perms::all; + } + + std::filesystem::perms processUmask() + { + static const auto mask = readProcessUmask(); + return mask; + } + + std::optional inheritablePermissions(std::filesystem::perms permissions) + { + const auto sanitized = sanitizePermissions(permissions); + if (!sanitized) + return std::nullopt; + return *sanitized & ~processUmask(); + } + + std::filesystem::perms localPermissions(std::filesystem::file_status const& status) + { +#ifdef _WIN32 + static_cast(status); + return std::filesystem::perms::unknown; +#else + return status.permissions(); +#endif + } +} diff --git a/utility/source/utility/rename.cpp b/utility/source/utility/rename.cpp new file mode 100644 index 00000000..184d5c59 --- /dev/null +++ b/utility/source/utility/rename.cpp @@ -0,0 +1,60 @@ +#include + +#ifdef _WIN32 +# include +#else +# include +# include +# include +# include +#endif + +namespace Utility +{ + namespace + { + std::unexpected lastErrno() + { + return std::unexpected(std::error_code{errno, std::generic_category()}); + } + } + + std::expected renameNoReplace(std::filesystem::path const& from, std::filesystem::path const& to) + { +#if defined(_WIN32) + if (::MoveFileExW(from.c_str(), to.c_str(), 0) == 0) + { + const auto error = ::GetLastError(); + if (error == ERROR_ALREADY_EXISTS || error == ERROR_FILE_EXISTS) + return std::unexpected(std::make_error_code(std::errc::file_exists)); + return std::unexpected(std::error_code{static_cast(error), std::system_category()}); + } + return {}; +#elif defined(__APPLE__) + if (::renamex_np(from.c_str(), to.c_str(), RENAME_EXCL) != 0) + return lastErrno(); + return {}; +#else + if (::renameat2(AT_FDCWD, from.c_str(), AT_FDCWD, to.c_str(), RENAME_NOREPLACE) == 0) + return {}; + if (errno != EINVAL && errno != ENOSYS) + return lastErrno(); + + // The filesystem cannot rename without replacing. A hard link fails on an existing target as well. + std::error_code statusError{}; + if (std::filesystem::is_regular_file(std::filesystem::symlink_status(from, statusError))) + { + if (::link(from.c_str(), to.c_str()) != 0) + return lastErrno(); + if (::unlink(from.c_str()) != 0) + return lastErrno(); + return {}; + } + if (std::filesystem::exists(std::filesystem::symlink_status(to, statusError))) + return std::unexpected(std::make_error_code(std::errc::file_exists)); + if (::rename(from.c_str(), to.c_str()) != 0) + return lastErrno(); + return {}; +#endif + } +} diff --git a/utility/source/utility/user_directories.cpp b/utility/source/utility/user_directories.cpp index f4d12f4a..a8024ed4 100644 --- a/utility/source/utility/user_directories.cpp +++ b/utility/source/utility/user_directories.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -51,9 +50,9 @@ namespace Utility } } - std::vector parseUserDirectories(std::string_view content, std::filesystem::path const& home) + std::vector parseUserDirectories(std::string_view content, std::string_view home) { - auto homeString = pathToUtf8Generic(home); + std::string homeString{home}; while (homeString.size() > 1 && homeString.ends_with('/')) homeString.pop_back(); const std::string_view homePrefix = homeString == "/" ? std::string_view{} : std::string_view{homeString}; @@ -90,7 +89,7 @@ namespace Utility directories.push_back({ .name = std::string{key.substr(prefix.size(), key.size() - prefix.size() - suffix.size())}, - .path = pathFromUtf8(*expanded), + .path = *std::move(expanded), }); } return directories; diff --git a/utility/test/utility/main.cpp b/utility/test/utility/main.cpp index 9f6b7096..445e8490 100644 --- a/utility/test/utility/main.cpp +++ b/utility/test/utility/main.cpp @@ -1,3 +1,4 @@ +#include #include "test_command_template.hpp" #include "test_directory_traversal.hpp" #include "test_echo_suppressor.hpp" @@ -5,6 +6,9 @@ #include "test_keyed_diff.hpp" #include "test_localized_message.hpp" #include "test_move_to_front.hpp" +#include "test_permissions.hpp" +#include "test_rename.hpp" +#include "test_link_safe_remove.hpp" #include "test_shell_integration.hpp" #include "test_typed_line_buffer.hpp" #include "test_user_directories.hpp" @@ -15,11 +19,16 @@ #include std::filesystem::path programDirectory; +/** @brief The temporary directory of this process alone, so tests running in parallel share no files. */ +std::filesystem::path temporaryRoot; int main(int argc, char** argv) { programDirectory = std::filesystem::path{argv[0]}.parent_path(); + const Utility::ProcessTemporaryDirectory processTemporary{programDirectory / "temp"}; + temporaryRoot = processTemporary.path(); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } \ No newline at end of file diff --git a/utility/test/utility/test_directory_traversal.hpp b/utility/test/utility/test_directory_traversal.hpp index 15164c17..76e666d2 100644 --- a/utility/test/utility/test_directory_traversal.hpp +++ b/utility/test/utility/test_directory_traversal.hpp @@ -13,6 +13,7 @@ using namespace std::string_literals; extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; namespace Utility::Test { @@ -22,20 +23,27 @@ namespace Utility::Test template auto withWalkerDo(auto&& fn) { - auto scan = [this](std::filesystem::path const& path) + auto scan = [this](Paths::RelativePath const& relative) -> std::expected, std::string> { if (scanner_) - return scanner_(path); + return scanner_(absolutePath(relative)); else return std::unexpected("No Scanner Installed In Test"s); }; using WalkerType = DeepDirectoryWalker; if (!walker_) - walker_ = std::make_unique(isolateDirectory_.path(), std::move(scan)); + walker_ = std::make_unique(std::move(scan)); return fn(static_cast(*walker_)); } + std::filesystem::path absolutePath(Paths::RelativePath const& relative) const + { + if (relative.empty()) + return isolateDirectory_.path(); + return isolateDirectory_.path() / relative.value(); + } + std::filesystem::path createFileIn(std::filesystem::path const& dir, std::size_t size) { const auto path = dir / ("file" + std::to_string(fileCounter_++)); @@ -77,7 +85,7 @@ namespace Utility::Test for (auto const& dirEntry : std::filesystem::directory_iterator(path)) { SharedData::DirectoryEntry entry{}; - entry.path = dirEntry.path().filename(); + entry.name = dirEntry.path().filename().string(); entry.type = dirEntry.is_directory() ? SharedData::FileType::Directory : SharedData::FileType::Regular; entry.size = dirEntry.is_regular_file() ? dirEntry.file_size() : 0; @@ -88,7 +96,7 @@ namespace Utility::Test } protected: - Utility::TemporaryDirectory isolateDirectory_{programDirectory / "temp", true}; + Utility::TemporaryDirectory isolateDirectory_{temporaryRoot, true}; std::unique_ptr walker_; std::function, std::string>(std::filesystem::path const&)> scanner_; @@ -336,11 +344,11 @@ namespace Utility::Test createSomeFilesIn(subSubDir, fileCount); std::vector paths{}; - const auto walkResult = withWalkerDo([&paths](auto& walker) { + const auto walkResult = withWalkerDo([this, &paths](auto& walker) { auto res = walker.walkAll(); auto const& entries = walker.entries(); - std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [&walker](auto const& entry) { - return walker.fullPath(entry); + std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [this, &walker](auto const& entry) { + return absolutePath(walker.relativePath(entry)); }); return res; }); @@ -404,30 +412,30 @@ namespace Utility::Test [](std::filesystem::path const&) -> std::expected, std::string> { return std::vector{ SharedData::DirectoryEntry{ - .path = ".", + .name = ".", .type = SharedData::FileType::Directory, }, SharedData::DirectoryEntry{ - .path = "..", + .name = "..", .type = SharedData::FileType::Directory, }, SharedData::DirectoryEntry{ - .path = "file0", + .name = "file0", .type = SharedData::FileType::Regular, }, SharedData::DirectoryEntry{ - .path = "dir0", + .name = "dir0", .type = SharedData::FileType::Directory, }, }; }; std::vector paths{}; - const auto walkResult = withWalkerDo([&paths](auto& walker) { + const auto walkResult = withWalkerDo([this, &paths](auto& walker) { auto res = walker.walk(); auto const& entries = walker.entries(); - std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [&walker](auto const& entry) { - return walker.fullPath(entry); + std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [this, &walker](auto const& entry) { + return absolutePath(walker.relativePath(entry)); }); return res; }); @@ -448,19 +456,19 @@ namespace Utility::Test { std::vector results{ SharedData::DirectoryEntry{ - .path = "file0", + .name = "file0", .type = SharedData::FileType::Regular, }, SharedData::DirectoryEntry{ - .path = ".", + .name = ".", .type = SharedData::FileType::Directory, }, SharedData::DirectoryEntry{ - .path = "dir0", + .name = "dir0", .type = SharedData::FileType::Directory, }, SharedData::DirectoryEntry{ - .path = "..", + .name = "..", .type = SharedData::FileType::Directory, }, }; @@ -480,12 +488,12 @@ namespace Utility::Test auto doTheWalk = [this, &paths]() { paths.clear(); - const auto walkResult = withWalkerDo([&paths](auto& walker) { + const auto walkResult = withWalkerDo([this, &paths](auto& walker) { walker.reset(); auto res = walker.walk(); auto const& entries = walker.entries(); - std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [&walker](auto const& entry) { - return walker.fullPath(entry); + std::transform(entries.begin(), entries.end(), std::back_inserter(paths), [this, &walker](auto const& entry) { + return absolutePath(walker.relativePath(entry)); }); return res; }); diff --git a/utility/test/utility/test_link_safe_remove.hpp b/utility/test/utility/test_link_safe_remove.hpp new file mode 100644 index 00000000..1dfd4a34 --- /dev/null +++ b/utility/test/utility/test_link_safe_remove.hpp @@ -0,0 +1,132 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include + +extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; + +namespace Utility::Test +{ + class LinkSafeRemoveTests : public ::testing::Test + { + protected: + /** + * @brief A link at @p link to the directory @p target: a junction on Windows, a symlink elsewhere. + */ + static bool makeDirectoryLink(std::filesystem::path const& link, std::filesystem::path const& target) + { +#ifdef _WIN32 + const auto command = "cmd /c mklink /J \"" + link.string() + "\" \"" + target.string() + "\" >nul"; + return std::system(command.c_str()) == 0 && std::filesystem::exists(link); +#else + std::error_code error; + std::filesystem::create_directory_symlink(target, link, error); + return !error; +#endif + } + + TemporaryDirectory root_{temporaryRoot, true}; + }; + + TEST_F(LinkSafeRemoveTests, DirectoryLinkIsALinkAndADirectoryIsNot) + { + const auto target = root_.path() / "target"; + std::filesystem::create_directories(target); + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + + std::error_code error; + EXPECT_TRUE(isLinkOrJunction(root_.path() / "link", error)); + EXPECT_FALSE(error); + EXPECT_FALSE(isLinkOrJunction(target, error)); + EXPECT_FALSE(isLinkOrJunction(root_.path() / "missing", error)); + EXPECT_FALSE(error) << "a missing path is no link and no error"; + } + + TEST_F(LinkSafeRemoveTests, RemovingATreeNeverDeletesThroughALink) + { + const auto outside = root_.path() / "outside"; + std::filesystem::create_directories(outside); + std::ofstream{outside / "keep.txt"} << "keep"; + const auto tree = root_.path() / "tree"; + std::filesystem::create_directories(tree / "sub"); + std::ofstream{tree / "sub" / "own.txt"} << "own"; + ASSERT_TRUE(makeDirectoryLink(tree / "sub" / "link", outside)); + + std::error_code error; + const auto removed = removeAllNoFollow(tree, error); + + EXPECT_FALSE(error) << error.message(); + EXPECT_EQ(removed, 4u) << "tree, sub, own.txt and the link itself"; + EXPECT_FALSE(std::filesystem::exists(tree)); + EXPECT_TRUE(std::filesystem::exists(outside / "keep.txt")) << "the link target must survive"; + } + + TEST_F(LinkSafeRemoveTests, RemovingALinkKeepsItsTarget) + { + const auto target = root_.path() / "target"; + std::filesystem::create_directories(target); + std::ofstream{target / "keep.txt"} << "keep"; + ASSERT_TRUE(makeDirectoryLink(root_.path() / "link", target)); + + std::error_code error; + EXPECT_EQ(removeAllNoFollow(root_.path() / "link", error), 1u); + EXPECT_FALSE(error) << error.message(); + EXPECT_TRUE(std::filesystem::exists(target / "keep.txt")); + EXPECT_FALSE(std::filesystem::exists(root_.path() / "link")); + } + + TEST_F(LinkSafeRemoveTests, FirstLinkBelowFindsALinkedComponent) + { + const auto outside = root_.path() / "outside"; + std::filesystem::create_directories(outside); + const auto tree = root_.path() / "tree"; + std::filesystem::create_directories(tree / "real"); + ASSERT_TRUE(makeDirectoryLink(tree / "real" / "link", outside)); + + std::error_code error; + const auto found = firstLinkBelow(tree, tree / "real" / "link" / "missing" / "file.txt", error); + EXPECT_FALSE(error) << error.message(); + ASSERT_TRUE(found); + EXPECT_EQ(*found, tree / "real" / "link"); + + EXPECT_EQ(firstLinkBelow(tree, tree / "real" / "link", error), tree / "real" / "link") << "the leaf counts"; + } + + TEST_F(LinkSafeRemoveTests, FirstLinkBelowIgnoresTheRootAndMissingComponents) + { + const auto target = root_.path() / "target"; + std::filesystem::create_directories(target / "real"); + const auto linkedRoot = root_.path() / "linkedRoot"; + ASSERT_TRUE(makeDirectoryLink(linkedRoot, target)); + + std::error_code error; + EXPECT_FALSE(firstLinkBelow(linkedRoot, linkedRoot / "real" / "new" / "file.txt", error)); + EXPECT_FALSE(error) << error.message(); + EXPECT_FALSE(firstLinkBelow(linkedRoot, linkedRoot, error)); + EXPECT_FALSE(error) << error.message(); + } + + TEST_F(LinkSafeRemoveTests, FirstLinkBelowRefusesAPathOutsideTheRoot) + { + std::error_code error; + EXPECT_FALSE(firstLinkBelow(root_.path() / "a", root_.path() / "b" / "file.txt", error)); + EXPECT_TRUE(error); + EXPECT_FALSE(firstLinkBelow(root_.path() / "a", root_.path() / "a" / ".." / "b", error)); + EXPECT_TRUE(error); + } + + TEST_F(LinkSafeRemoveTests, MissingPathRemovesNothing) + { + std::error_code error; + EXPECT_EQ(removeAllNoFollow(root_.path() / "missing", error), 0u); + EXPECT_FALSE(error); + } +} diff --git a/utility/test/utility/test_permissions.hpp b/utility/test/utility/test_permissions.hpp new file mode 100644 index 00000000..d0fb0cb7 --- /dev/null +++ b/utility/test/utility/test_permissions.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include + +#include + +namespace Utility::Test +{ + using std::filesystem::perms; + + TEST(PermissionsTests, UnknownPermissionsAreNotApplied) + { + EXPECT_FALSE(sanitizePermissions(perms::unknown).has_value()); + EXPECT_FALSE(inheritablePermissions(perms::unknown).has_value()); + } + + TEST(PermissionsTests, SpecialBitsNeverTravel) + { + const auto sanitized = sanitizePermissions(perms::all | perms::set_uid | perms::set_gid | perms::sticky_bit); + ASSERT_TRUE(sanitized.has_value()); + EXPECT_EQ(*sanitized, perms::all); + } + + TEST(PermissionsTests, InheritedPermissionsHonorTheUmask) + { + const auto inherited = inheritablePermissions(perms::all); + ASSERT_TRUE(inherited.has_value()); + EXPECT_EQ(*inherited, perms::all & ~processUmask()); + EXPECT_EQ(*inherited & processUmask(), perms::none); + } +} diff --git a/utility/test/utility/test_rename.hpp b/utility/test/utility/test_rename.hpp new file mode 100644 index 00000000..f96170f8 --- /dev/null +++ b/utility/test/utility/test_rename.hpp @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include +#include + +extern std::filesystem::path programDirectory; +extern std::filesystem::path temporaryRoot; + +namespace Utility::Test +{ + class RenameNoReplaceTests : public ::testing::Test + { + protected: + static void write(std::filesystem::path const& path, std::string const& content) + { + std::ofstream{path, std::ios::binary} << content; + } + + static std::string read(std::filesystem::path const& path) + { + std::ifstream file{path, std::ios::binary}; + return std::string{std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; + } + + TemporaryDirectory directory_{temporaryRoot, true}; + }; + + TEST_F(RenameNoReplaceTests, RenamesToAFreeName) + { + write(directory_.path() / "a", "a"); + ASSERT_TRUE(renameNoReplace(directory_.path() / "a", directory_.path() / "b").has_value()); + EXPECT_FALSE(std::filesystem::exists(directory_.path() / "a")); + EXPECT_EQ(read(directory_.path() / "b"), "a"); + } + + TEST_F(RenameNoReplaceTests, RefusesToReplaceAFile) + { + write(directory_.path() / "a", "a"); + write(directory_.path() / "b", "b"); + const auto result = renameNoReplace(directory_.path() / "a", directory_.path() / "b"); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), std::errc::file_exists); + EXPECT_EQ(read(directory_.path() / "a"), "a"); + EXPECT_EQ(read(directory_.path() / "b"), "b"); + } + + TEST_F(RenameNoReplaceTests, RefusesToReplaceAnEmptyDirectory) + { + std::filesystem::create_directory(directory_.path() / "source"); + std::filesystem::create_directory(directory_.path() / "target"); + const auto result = renameNoReplace(directory_.path() / "source", directory_.path() / "target"); + ASSERT_FALSE(result.has_value()); + EXPECT_TRUE(std::filesystem::exists(directory_.path() / "source")); + } + + TEST_F(RenameNoReplaceTests, RenamesDirectories) + { + std::filesystem::create_directory(directory_.path() / "source"); + write(directory_.path() / "source" / "inside", "x"); + ASSERT_TRUE(renameNoReplace(directory_.path() / "source", directory_.path() / "target").has_value()); + EXPECT_EQ(read(directory_.path() / "target" / "inside"), "x"); + } + + TEST(RenameTargetTests, OnlyANewNameInTheSameDirectoryIsARenameInPlace) + { + const std::filesystem::path old{"/data/old.txt"}; + EXPECT_TRUE(isRenameInPlace(old, "/data/new.txt")); + EXPECT_FALSE(isRenameInPlace(old, "/elsewhere/new.txt")) << "a move"; + EXPECT_FALSE(isRenameInPlace(old, "/data/sub/new.txt")) << "a move into a subdirectory"; + EXPECT_FALSE(isRenameInPlace(old, "/data/..")) << "the parent"; + EXPECT_FALSE(isRenameInPlace(old, "/data/.")) << "the directory itself"; + EXPECT_FALSE(isRenameInPlace(old, "/data/")) << "no name"; + EXPECT_FALSE(isRenameInPlace("old.txt", "new.txt")) << "relative paths have no parent to compare"; + } +} diff --git a/utility/test/utility/test_user_directories.hpp b/utility/test/utility/test_user_directories.hpp index 55a7de3c..c98d1918 100644 --- a/utility/test/utility/test_user_directories.hpp +++ b/utility/test/utility/test_user_directories.hpp @@ -13,11 +13,11 @@ namespace Utility::Test namespace { std::vector> - parsed(std::string_view content, std::filesystem::path const& home = "/home/user") + parsed(std::string_view content, std::string_view home = "/home/user") { std::vector> result; for (auto const& directory : parseUserDirectories(content, home)) - result.emplace_back(directory.name, directory.path.generic_string()); + result.emplace_back(directory.name, directory.path); return result; }