diff --git a/.gitmodules b/.gitmodules index dcb53aca..652a634c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -30,7 +30,7 @@ url = https://github.com/gabime/spdlog.git [submodule "dependencies/efsw"] path = dependencies/efsw - url = https://github.com/SpartanJ/efsw.git + url = https://github.com/5cript/efsw.git [submodule "dependencies/promise-cpp"] path = dependencies/promise-cpp url = https://github.com/5cript/promise-cpp.git diff --git a/_cmake/dependencies/efsw.cmake b/_cmake/dependencies/efsw.cmake index df1d42ff..605163e5 100755 --- a/_cmake/dependencies/efsw.cmake +++ b/_cmake/dependencies/efsw.cmake @@ -3,8 +3,8 @@ if (FETCH_EFSW) # If you edit here, update work_dependencies.json FetchContent_Declare( efsw - GIT_REPOSITORY https://github.com/SpartanJ/efsw.git - GIT_TAG 87abe599995d5646f5d83cf2e3a225bd73148b3a + GIT_REPOSITORY https://github.com/5cript/efsw.git + GIT_TAG d71ed846811cc67ee8827614341e2777d200231a ) FetchContent_MakeAvailable(efsw) diff --git a/backend/include/backend/file_tracking/batching_remote.hpp b/backend/include/backend/file_tracking/batching_remote.hpp new file mode 100644 index 00000000..14df764c --- /dev/null +++ b/backend/include/backend/file_tracking/batching_remote.hpp @@ -0,0 +1,123 @@ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + /** + * @brief A TrackerRemote that also queues many uploads or deletes as one operation. + */ + class BulkTrackerRemote : public TrackerRemote + { + public: + /** + * @brief Called once on the strand for each entry of a bulk once the bulk ended, with the entry's index. + */ + using OnEntryEnded = std::function; + + /** + * @brief Called once on the strand for each upload of a bulk once the bulk ended, with the entry's index. + */ + using OnUploadEntryEnded = std::function; + + /** + * @brief Queues the uploads of @p requests, regular files that may create the directories above their + * destination, as one operation. + */ + virtual std::expected + uploadAll(std::vector const& requests, OnUploadEntryEnded onEnded) = 0; + + /** + * @brief Queues the deletes of @p paths as one operation, a directory only when it is empty. + */ + virtual std::expected + removeAll(std::vector const& paths, OnEntryEnded onEnded) = 0; + }; + + /** + * @brief Hands the uploads and deletes a Tracker queues within a short time to the server as one bulk each, which + * costs one operation, one card and one listing refresh instead of one per path. Everything else passes + * through at once. The tracker orders its operations per path, so what is gathered here lies on unrelated + * paths and may run side by side. Lives on the strand of the session. + */ + class BatchingRemote : public TrackerRemote + { + public: + struct Limits + { + /** @brief A batch goes once nothing was added to it for this long. */ + std::chrono::milliseconds quiet{100}; + /** @brief A batch goes at the latest this long after its first entry. */ + std::chrono::milliseconds longest{1000}; + /** @brief A batch goes at once when it holds this many entries. */ + std::size_t maxEntries{500}; + /** @brief A file this large goes on its own, an upload that can be stopped, see cancelUpload(). */ + std::uint64_t aloneFromBytes{8u * 1024u * 1024u}; + }; + + BatchingRemote( + boost::asio::strand strand, + std::unique_ptr inner, + Limits limits + ); + BatchingRemote( + boost::asio::strand strand, + std::unique_ptr inner + ); + ~BatchingRemote() override; + BatchingRemote(BatchingRemote const&) = delete; + BatchingRemote& operator=(BatchingRemote const&) = delete; + BatchingRemote(BatchingRemote&&) = delete; + BatchingRemote& operator=(BatchingRemote&&) = delete; + + std::expected, std::string> stat(Paths::RemotePath const& path) override; + std::expected, std::string> linkStat(Paths::RemotePath const& path) override; + std::expected download( + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t sizeBytes, + OnEnded onEnded + ) override; + + /** + * @brief Gathers @p request into the next batch. It goes on its own when it creates no missing directories, + * refreshes the listing, is no regular file, is a large one, continues a partial file or is alone in + * its batch. + */ + std::expected upload(UploadRequest const& request, OnUploadEnded onEnded) override; + + /** + * @brief An upload still gathered ends as cancelled without being sent, one on its way is stopped by the inner + * remote. One in a bulk goes on. + */ + void cancelUpload(Paths::RemotePath const& destination, PartialFile partial) override; + + std::expected + discardPartialUpload(Paths::RemotePath const& destination, OnEnded onEnded) override; + + std::expected + rename(Paths::RemotePath const& source, Paths::RemotePath const& destination, OnEnded onEnded) override; + + /** + * @brief Gathers the delete of @p path into the next batch, alone in it it goes on its own. + */ + std::expected remove(Paths::RemotePath const& path, OnEnded onEnded) override; + + private: + struct Implementation; + std::shared_ptr impl_; + }; +} diff --git a/backend/include/backend/file_tracking/change_filter.hpp b/backend/include/backend/file_tracking/change_filter.hpp index 27c529e6..3ddaa400 100644 --- a/backend/include/backend/file_tracking/change_filter.hpp +++ b/backend/include/backend/file_tracking/change_filter.hpp @@ -1,7 +1,8 @@ #pragma once -#include +#include +#include #include #include @@ -22,6 +23,18 @@ namespace FileTracking */ bool isTransientFile(std::string const& filename, std::string const& downloadTempSuffix); + /** + * @brief Whether a change of @p filename, moved from @p oldFilename when that is not empty, is noise nobody + * follows: it ends at a temporary and does not come from a real file. Cheap enough for the watcher's + * thread, which drops it before anything else looks at it. + * @param downloadTempSuffix The suffix of downloads in progress, see Persistence::effectiveTempFileSuffix. + */ + bool isTransientChange( + std::string const& filename, + std::string const& oldFilename, + std::string const& downloadTempSuffix + ); + /** * @brief Turns a raw watcher event into the change to mirror to the remote side. * @param change The event as the watcher reports it. @@ -31,4 +44,19 @@ namespace FileTracking */ std::optional filterFileChange(FileChange change, bool isDirectory, std::string const& downloadTempSuffix); + + /** + * @brief Whether a change of @p path, moved from @p oldPath when that is not empty, touches @p item or anything + * below it. All paths are spelled alike (canonical). + */ + bool isChangeOfItem( + std::filesystem::path const& item, + std::filesystem::path const& path, + std::filesystem::path const& oldPath + ); + + /** + * @brief Whether @p path is @p base or lies below it, compared lexically. An empty @p path is neither. + */ + bool isAtOrBelow(std::filesystem::path const& base, std::filesystem::path const& path); } diff --git a/backend/include/backend/file_tracking/change_reporter.hpp b/backend/include/backend/file_tracking/change_reporter.hpp new file mode 100644 index 00000000..0edc7891 --- /dev/null +++ b/backend/include/backend/file_tracking/change_reporter.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +#include +#include + +#include +#include + +namespace FileTracking +{ + class SettledChanges; + + /** + * @brief Reports the changes of one watch to its sink. Noise (see filterFileChange) is dropped, content changes of + * files are reported once they settled (see SettledChanges). A Missed change goes after what is held back + * at and below its directory. Lives on its strand, copies share what is held back. + */ + class ChangeReporter + { + public: + /** + * @param strand The strand report() is called on. + * @param sink Receives the changes on @p strand. + * @param downloadTempSuffix The suffix downloads write under until complete, see + * Persistence::effectiveTempFileSuffix. + */ + ChangeReporter( + boost::asio::strand strand, + ChangeSink sink, + std::string const& downloadTempSuffix + ); + + void report(DirectoryChange const& change) const; + + /** + * @brief Whether a change of @p filename is noise that report() drops, see isTransientFile. + */ + bool isNoise(std::string const& filename) const; + + private: + std::shared_ptr settledChanges_; + std::string downloadTempSuffix_; + }; +} diff --git a/backend/include/backend/file_tracking/directory_watch_service.hpp b/backend/include/backend/file_tracking/directory_watch_service.hpp new file mode 100644 index 00000000..66b6d469 --- /dev/null +++ b/backend/include/backend/file_tracking/directory_watch_service.hpp @@ -0,0 +1,88 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + /** + * @brief One change as efsw reports it, the names UTF-8 on every platform. + */ + struct DirectoryEvent + { + FileAction action; + /** + * @brief The directory of the change, efsw ends it with a separator. For Missed the directory whose changes + * were lost, with or without one. + */ + std::string directory; + /** @brief Empty for Missed, which concerns the directory. */ + std::string filename; + /** @brief The previous name of a move, empty otherwise. */ + std::string oldFilename; + }; + + /** + * @brief Receives the events of a subscription on efsw's thread, it must only hand them on (post to a strand). + * Anything slower holds up the events of every watch, and on Windows loses the last changes of a watched + * directory that is deleted. + */ + using DirectoryEventSink = std::function; + + /** + * @brief The efsw watchers of the process. Directories watched without recursion share one watcher and its + * thread, except on macOS: there a watcher is one FSEvents stream, which efsw rebuilds whenever one of its + * watches comes or goes and drops the events on their way meanwhile. A recursive watch gets a watcher of + * its own: efsw holds its lock while it walks the tree to set the watch up and while it delivers an event, + * so a large tree would hold up the events of every other watch. A directory is watched once however many + * subscribe to it, all of them with or all without recursion. Thread-safe. Held in a std::shared_ptr, see + * shared(). + */ + class DirectoryWatchService : public std::enable_shared_from_this + { + public: + /** + * @brief The service, created by the first caller. It lives as long as someone holds it. + */ + static std::shared_ptr shared(); + + DirectoryWatchService(); + ~DirectoryWatchService(); + DirectoryWatchService(DirectoryWatchService const&) = delete; + DirectoryWatchService& operator=(DirectoryWatchService const&) = delete; + DirectoryWatchService(DirectoryWatchService&&) = delete; + DirectoryWatchService& operator=(DirectoryWatchService&&) = delete; + + /** + * @brief Delivers the changes in @p directory, and below it when @p recursive, to @p sink. + * @param directory An existing absolute path spelled like the events should report it. A recursive watch + * walks it before this returns, without blocking other callers. + * @return The handle that ends the subscription, nothing when efsw rejected the watch or @p directory is + * watched the other way (with or without recursion) already. + */ + std::optional subscribe(std::filesystem::path const& directory, bool recursive, DirectoryEventSink sink); + + /** + * @brief Ends the subscription @p subscriptionId, the directory is no longer watched once nobody subscribes. + */ + void unsubscribe(std::int64_t subscriptionId); + + /** + * @brief Tells the subscriptions of the watch of @p directory, or of the recursive one above it, that the + * changes in @p directory were lost, the way efsw does. For tests, efsw cannot be made to lose changes + * on every platform. + */ + void reportMissed(std::filesystem::path const& directory); + + private: + struct Implementation; + std::unique_ptr impl_; + }; +} diff --git a/backend/include/backend/file_tracking/directory_watcher.hpp b/backend/include/backend/file_tracking/directory_watcher.hpp new file mode 100644 index 00000000..8f2f71f7 --- /dev/null +++ b/backend/include/backend/file_tracking/directory_watcher.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace FileTracking +{ + /** + * @brief One change of a watched directory, as it reaches the strand of its DirectoryWatcher. + */ + struct DirectoryChange + { + FileAction action; + /** + * @brief The directory of the change as the watch reports it, it may end with a separator. At and above the + * watched directory it is spelled like that, see DirectoryWatcher::add. + */ + std::filesystem::path directory; + /** @brief UTF-8, spelled like efsw reports it, see eventSpelling. Empty for Missed. */ + std::string filename; + /** @brief The previous name of a move, UTF-8 and spelled like filename, empty otherwise. */ + std::string oldFilename; + /** @brief Whether path() was a directory when the watch reported the change. */ + bool isDirectory; + + /** @brief The changed path, the directory itself for Missed. */ + std::filesystem::path path() const; + + /** @brief Where a move came from, empty for anything else. */ + std::filesystem::path oldPath() const; + }; + + /** + * @brief Watches directories through the DirectoryWatchService of the process and hands their changes to its + * handler on its strand, in their order. What a change names is looked at on a strand of its own first. + * Destroying it stops the handler, the subscriptions end with their handles. + */ + class DirectoryWatcher + { + public: + using ChangeHandler = std::function; + + /** + * @brief Decides on efsw's thread whether an event reaches the strand. Called for every event of every + * directory watched with it, it must be fast and thread safe. Missed events pass untested. + */ + using EventFilter = std::function; + + /** + * @param strand Runs @p onChange. + * @param onChange Receives the changes of every directory added. + */ + DirectoryWatcher(boost::asio::strand strand, ChangeHandler onChange); + ~DirectoryWatcher(); + DirectoryWatcher(DirectoryWatcher const&) = delete; + DirectoryWatcher& operator=(DirectoryWatcher const&) = delete; + DirectoryWatcher(DirectoryWatcher&&) noexcept; + DirectoryWatcher& operator=(DirectoryWatcher&&) noexcept; + + /** + * @brief Watches @p directory, an existing absolute path spelled like the changes should report it, for the + * changes made from now on. Below it the changes keep the spelling of the events, see eventSpelling. On + * macOS a modification of what is gone once it arrives is dropped, efsw reports one before every removal + * there. + * @param filter Drops the events it refuses before they are looked at further, nothing passes them all. + * @return The handle that ends the watch, nothing when it was rejected. + */ + std::optional + add(std::filesystem::path const& directory, bool recursive, EventFilter filter = {}); + + private: + struct Implementation; + std::unique_ptr impl_; + }; +} diff --git a/backend/include/backend/file_tracking/disk_place.hpp b/backend/include/backend/file_tracking/disk_place.hpp new file mode 100644 index 00000000..e58fd370 --- /dev/null +++ b/backend/include/backend/file_tracking/disk_place.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace FileTracking +{ + /** + * @brief Tells a file apart from every other one on the machine, whatever path leads to it. + */ + struct DiskIdentity + { + std::uint64_t device; + std::uint64_t file; + + friend auto operator<=>(DiskIdentity const&, DiskIdentity const&) = default; + }; + + /** + * @brief The file at @p path, nothing when it does not exist or cannot be looked at. + */ + std::optional diskIdentityOf(std::filesystem::path const& path); + + /** + * @brief Whether the file @p identity names was deleted rather than moved elsewhere on its volume, to the trash + * for one. Nothing when that cannot be told: only macOS looks a file up by its identity, on APFS and HFS+. + * @param directory A directory on the same volume that is there, the one the file was in. + */ + std::optional isDeletedFromItsVolume(DiskIdentity const& identity, std::filesystem::path const& directory); + + /** + * @brief Where a path lies on disk: the identities of the path and of the directories above it that exist. Taken + * once, so comparing places costs no further look at the disk. + */ + struct DiskPlace + { + /** @brief The path itself, nothing when it does not exist. */ + std::optional self{}; + /** @brief The path and the directories above it that exist, the path first. */ + std::vector chain{}; + + static DiskPlace of(std::filesystem::path const& path); + + /** + * @brief Whether this place is @p base or lies below it on disk. Nothing that does not exist is. + */ + bool isAtOrBelow(DiskPlace const& base) const; + }; +} diff --git a/backend/include/backend/file_tracking/event_spelling.hpp b/backend/include/backend/file_tracking/event_spelling.hpp new file mode 100644 index 00000000..75f2280b --- /dev/null +++ b/backend/include/backend/file_tracking/event_spelling.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace FileTracking +{ + /** + * @brief @p name spelled like a DirectoryEvent spells it: precomposed (NFC) on macOS, where efsw precomposes every + * name while the file system keeps the spelling a name was created with (Finder's is decomposed) and finds + * it under either. Unchanged elsewhere. + */ + std::string eventSpelling(std::string const& name); +} diff --git a/backend/include/backend/file_tracking/file_action.hpp b/backend/include/backend/file_tracking/file_action.hpp new file mode 100644 index 00000000..7964c554 --- /dev/null +++ b/backend/include/backend/file_tracking/file_action.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include + +namespace FileTracking +{ + /** + * @brief What happened to a watched path, mirrors the efsw action values and its missed file actions. + */ + enum class FileAction + { + Added, + Modified, + Deleted, + Moved, + /** + * @brief The watch lost the changes in a directory: its event queue overflowed, or the directory went while + * it was read. What is there has to be looked at anew. It names the directory, no entry in it. + */ + Missed, + }; + BOOST_DESCRIBE_ENUM(FileAction, Added, Modified, Deleted, Moved, Missed) +} diff --git a/backend/include/backend/file_tracking/file_states.hpp b/backend/include/backend/file_tracking/file_states.hpp new file mode 100644 index 00000000..32dd5b92 --- /dev/null +++ b/backend/include/backend/file_tracking/file_states.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace FileTracking +{ + /** + * @brief What a watched path holds locally. Uploads and downloads carry the modification time along, so after one + * it is what the server holds too. + */ + struct FileState + { + std::filesystem::path path; + bool isDirectory{false}; + /** @brief A symbolic link, not followed: its state is the server's own, see TrackerRemote::linkStat. */ + bool isSymlink{false}; + /** @brief Bytes, zero for a directory and a symbolic link. */ + std::uint64_t size{0}; + /** @brief Seconds, zero for a directory and a symbolic link. */ + std::uint64_t mtime{0}; + + friend bool operator==(FileState const&, FileState const&) = default; + }; + + /** + * @brief The state of @p entry, the path spelled as the entry spells it, canonical when the walk that found it + * started at a canonical path. Nothing for anything but a regular file, a directory or a symbolic link and + * for an entry that cannot be read. + */ + std::optional fileStateOf(std::filesystem::directory_entry const& entry); + + /** + * @brief The states (see fileStateOf) of @p item and of everything below it when it is a directory, the paths + * canonical. Symbolic links below it are not followed. + */ + std::vector itemStatesOf(std::filesystem::path const& item); + + /** + * @brief How many entries @p item is with everything below it when it is a directory, symbolic links not + * followed. Their states are not read. + */ + std::size_t entryCountOf(std::filesystem::path const& item); + + /** + * @brief The state of the regular file at @p path, nothing for anything else, a symbolic link too, and when it + * cannot be read. + */ + std::optional sourceStateOf(std::filesystem::path const& path); +} diff --git a/backend/include/backend/file_tracking/frontend_tracker_view.hpp b/backend/include/backend/file_tracking/frontend_tracker_view.hpp new file mode 100644 index 00000000..929a1ab9 --- /dev/null +++ b/backend/include/backend/file_tracking/frontend_tracker_view.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include +#include + +#include +#include + +namespace Nui +{ + class RpcHub; +} + +namespace FileTracking +{ + /** + * @brief Shows a Tracker in the file tracking panel of its session, which listens to + * @c "FileTracker::::": onEntry (SharedData::TrackedEntry), onEntryRemoved (watch id), + * onEntriesEnded (SharedData::EndedWatches), onQuestion (SharedData::ServerChangedQuestion), + * onOrphanedCount (count) and onProgress (SharedData::TrackerProgress). + */ + class FrontendTrackerView : public TrackerView + { + public: + FrontendTrackerView(Nui::RpcHub& hub, Ids::SessionId sessionId); + + void showEntry(SharedData::TrackedEntry const& entry) override; + void removeEntry(Ids::WatchId const& watchId) override; + void showEnded(SharedData::EndedWatches const& ended) override; + void ask(SharedData::ServerChangedQuestion const& question) override; + void showOrphanedCount(std::size_t count) override; + void showProgress(SharedData::TrackerProgress const& progress) override; + + private: + std::string eventName(std::string_view event) const; + + private: + Nui::RpcHub* hub_; + Ids::SessionId sessionId_; + }; +} diff --git a/backend/include/backend/file_tracking/instance_watch.hpp b/backend/include/backend/file_tracking/instance_watch.hpp index ec716c5c..114bc1d6 100644 --- a/backend/include/backend/file_tracking/instance_watch.hpp +++ b/backend/include/backend/file_tracking/instance_watch.hpp @@ -1,20 +1,20 @@ #pragma once -#include - +#include #include #include namespace FileTracking { + class DirectoryWatchService; + /** - * @brief Move-only RAII handle representing one efsw directory watch. + * @brief Move-only RAII handle representing one subscription of a DirectoryWatchService. * - * Destroying an InstanceWatch or assigning another one to it removes the - * underlying efsw watch via efsw::FileWatcher::removeWatch(). It does not keep - * the watcher alive: once its instance destroyed the watcher there is nothing - * left to remove. Instances of this type are produced by - * TemporaryDirectoryInstance::addWatch(). + * Destroying an InstanceWatch or assigning another one to it ends the + * subscription. It does not keep the service alive: once the service is gone + * there is nothing left to end. Instances of this type are produced by + * DirectoryWatchService::subscribe(). */ class InstanceWatch { @@ -27,15 +27,11 @@ namespace FileTracking /** * @brief Construct a valid watch handle. * - * @param watcher The efsw::FileWatcher that registered this watch. - * @param watchId The watch ID returned by efsw::FileWatcher::addWatch(). + * @param service The service that holds the subscription. + * @param watchId The subscription. * @param path The directory path being watched. */ - InstanceWatch( - std::weak_ptr watcher, - efsw::WatchID watchId, - std::filesystem::path path - ); + InstanceWatch(std::weak_ptr service, std::int64_t watchId, std::filesystem::path path); ~InstanceWatch(); @@ -46,14 +42,14 @@ namespace FileTracking InstanceWatch& operator=(InstanceWatch&&) noexcept; /** - * @brief The path that was passed to addWatch(). + * @brief The path that was passed to subscribe(). */ std::filesystem::path const& path() const; /** - * @brief The raw efsw watch identifier. + * @brief The subscription, negative for an invalid handle. */ - efsw::WatchID watchId() const; + std::int64_t watchId() const; /** * @brief Returns true when this handle references a live watch. @@ -61,7 +57,7 @@ namespace FileTracking bool isValid() const; /** - * @brief Explicitly remove the watch and invalidate this handle. + * @brief Explicitly end the subscription and invalidate this handle. * * Called automatically by the destructor. Safe to call more than once. */ diff --git a/backend/include/backend/file_tracking/local_item_watch.hpp b/backend/include/backend/file_tracking/local_item_watch.hpp new file mode 100644 index 00000000..487ba789 --- /dev/null +++ b/backend/include/backend/file_tracking/local_item_watch.hpp @@ -0,0 +1,95 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + struct DirectoryChange; + + /** + * @brief How long after the watch followed a rename the item appearing at its old name takes the watch back: an + * editor that moves the original aside and writes the new file later than the settle time. + */ + inline constexpr std::chrono::milliseconds renameReturnTime{5000}; + + /** + * @brief Watches a file or directory the user has locally, not a download in a temporary directory: its + * directory without recursion, so atomic saves, renames and the removal of the item show, and the + * item itself recursively when it is a directory. Only changes of the item reach its sink. + * + * The watch follows the item: a rename within its directory is held back for the settle time and then + * reported and followed, unless the item comes back meanwhile (editors that move the original aside to write + * a new one). When the item appears at its old name within renameReturnTime after that, the watch goes back and + * reports a rename back and the new content. The removal of the item is held back for the settle time too, a + * rename of it or the item coming back meanwhile replaces it. A rename onto a path it may not follow, or onto a + * temporary name (see isTransientFile), is reported as the removal of the item. A directory that is created anew + * at the item is watched again. When a watch lost changes (FileAction::Missed) the item is looked at anew: gone, + * its removal is held back like a reported one, else the sink gets the Missed change to compare what is there with + * what it knows. Lives on its strand: every member must be called from it. + */ + class LocalItemWatch + { + public: + /** + * @param watchId Names the watch. + * @param item The file or directory to watch, canonical, see LocalWatchManager::checkWatchable. + * @param sink Receives the changes of the item on @p strand. + * @param downloadTempSuffix The suffix downloads write under until complete, see + * Persistence::effectiveTempFileSuffix. + * @param mayFollow Whether the watch, named by its watchId(), may follow a rename of the item to the path, + * false when that overlaps another watch. Called on the strand. + */ + static std::expected open( + Ids::WatchId watchId, + std::filesystem::path const& item, + boost::asio::strand strand, + ChangeSink sink, + std::string const& downloadTempSuffix, + std::function mayFollow + ); + + ~LocalItemWatch(); + LocalItemWatch(LocalItemWatch const&) = delete; + LocalItemWatch& operator=(LocalItemWatch const&) = delete; + LocalItemWatch(LocalItemWatch&&) noexcept; + LocalItemWatch& operator=(LocalItemWatch&&) noexcept; + + Ids::WatchId const& watchId() const; + + /** @brief The watched item, canonical like the paths of the change events. */ + std::filesystem::path const& item() const; + + /** @brief Where item() lies on disk, taken when the watch started or followed or found it anew. */ + DiskPlace const& place() const; + + /** @brief The directory of item(), canonical. */ + std::filesystem::path directory() const; + + bool isDirectory() const; + + /** + * @brief Handles @p change as if a watched directory reported it. Tests use it for event orders their + * platform does not produce, like Windows reporting a rename as a removal first. + */ + void handleChange(DirectoryChange const& change); + + private: + struct State; + explicit LocalItemWatch(std::shared_ptr state); + + private: + std::shared_ptr state_; + }; +} diff --git a/backend/include/backend/file_tracking/local_watch_manager.hpp b/backend/include/backend/file_tracking/local_watch_manager.hpp new file mode 100644 index 00000000..3f4a8c9d --- /dev/null +++ b/backend/include/backend/file_tracking/local_watch_manager.hpp @@ -0,0 +1,119 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + namespace LocalWatchManagerDetail + { + /** + * @brief A canonical path and where it lies on disk. + */ + struct PlacedPath + { + std::filesystem::path path; + DiskPlace place; + + /** + * @brief Looks @p path up on disk. + */ + PlacedPath(std::filesystem::path path); + PlacedPath(std::filesystem::path path, DiskPlace place); + }; + + /** + * @brief Whether @p first and @p second overlap: they are the same, or one lies below the other. Case is + * ignored on Windows and macOS, whose file systems fold it, and paths that exist are compared on disk + * too, which tells the same file under another spelling anywhere. + */ + bool overlaps(PlacedPath const& first, PlacedPath const& second); + + /** + * @brief Refuses @p item when it overlaps (see overlaps()) one of @p watchedItems or + * @p temporaryDownloadsDirectory, whose downloads Open and Watch watches. Watches never nest, so no + * change has two owners. All paths canonical. + * @return An error built with Utility::localizedMessage. + */ + std::expected refuseOverlap( + PlacedPath const& item, + std::vector const& watchedItems, + PlacedPath const& temporaryDownloadsDirectory + ); + } + + /** + * @brief Owns the LocalItemWatch objects of files and directories the user uploaded and watches ("Upload and + * Watch"). No two watches overlap, and none overlaps the temporary downloads directory that the + * TempDirInstanceManager watches. Lives on its strand: every member must be called from it. + */ + class LocalWatchManager + { + public: + /** + * @param strand Runs the changes of the watches. + * @param temporaryDownloadsDirectory Where the TempDirInstanceManager keeps its instances, no local watch + * may overlap it. + */ + LocalWatchManager( + boost::asio::strand strand, + std::filesystem::path temporaryDownloadsDirectory + ); + ~LocalWatchManager() = default; + LocalWatchManager(LocalWatchManager const&) = delete; + LocalWatchManager& operator=(LocalWatchManager const&) = delete; + LocalWatchManager(LocalWatchManager&&) = delete; + LocalWatchManager& operator=(LocalWatchManager&&) = delete; + + /** + * @brief Whether @p item can be watched now: it exists, is not a filesystem root and overlaps no watch. + * @return The canonical item, an error built with Utility::localizedMessage that says why it cannot be watched. + */ + std::expected checkWatchable(std::filesystem::path const& item) const; + + /** + * @brief Starts watching @p item as @p watchId, its changes go to @p sink. + * @return The new watch, an error when @p item cannot be watched or overlaps another watch. + */ + std::expected watch( + Ids::WatchId const& watchId, + std::filesystem::path const& item, + std::string const& downloadTempSuffix, + ChangeSink sink + ); + + /** + * @brief Ends the watch @p watchId, nothing happens when there is none. + */ + void unwatch(Ids::WatchId const& watchId); + + LocalItemWatch const* find(Ids::WatchId const& watchId) const; + + private: + /** + * @brief Refuses the canonical @p item when it overlaps a watch other than @p ignoredWatchId or the temporary + * downloads directory, see LocalWatchManagerDetail::refuseOverlap. + */ + std::expected refuseOverlapWithWatches( + std::filesystem::path const& item, + std::optional const& ignoredWatchId = std::nullopt + ) const; + + private: + boost::asio::strand strand_; + std::filesystem::path temporaryDownloadsDirectory_; + std::map watches_; + }; +} diff --git a/backend/include/backend/file_tracking/reported_change.hpp b/backend/include/backend/file_tracking/reported_change.hpp new file mode 100644 index 00000000..001b573c --- /dev/null +++ b/backend/include/backend/file_tracking/reported_change.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include +#include + +namespace FileTracking +{ + /** + * @brief A change of a watched path as its watch reports it, once noise is dropped and content changes settled, + * see ChangeReporter. + */ + struct ReportedChange + { + FileAction action; + /** @brief Spelled like the watched directory, canonical when that is. */ + std::filesystem::path path; + /** @brief Where a move came from, empty for every other change. */ + std::filesystem::path oldPath{}; + /** @brief Whether path was a directory when the change was seen. */ + bool isDirectory{false}; + + friend bool operator==(ReportedChange const&, ReportedChange const&) = default; + }; + + /** + * @brief Receives the changes of a watch, called on the strand of the watch. + */ + using ChangeSink = std::function; +} diff --git a/backend/include/backend/file_tracking/settled_changes.hpp b/backend/include/backend/file_tracking/settled_changes.hpp new file mode 100644 index 00000000..1e6b038e --- /dev/null +++ b/backend/include/backend/file_tracking/settled_changes.hpp @@ -0,0 +1,98 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + /** + * @brief How long the content of a file has to stay unchanged before its change is reported. One save shows as + * several events (inotify reports the writes and the close), they become one change. + */ + inline constexpr std::chrono::milliseconds changeSettleTime{300}; + + /** + * @brief Holds back the content changes of files until they settled, so a save is reported once: no event and no + * other size or modification time for the settle time. Windows reports the writes to a file that stays + * open only when someone else looks at it, which an upload does. Only content changes of the same file + * merge. Every other change goes at once, after the held back changes it concerns: those at its path or + * below it, and at or below where it moved from. Lives on its strand: every member must be called from it. + */ + class SettledChanges : public std::enable_shared_from_this + { + public: + /** + * @param send Reports a change, called on the strand. + */ + SettledChanges( + boost::asio::strand strand, + ChangeSink send, + std::chrono::milliseconds settleTime = changeSettleTime + ); + ~SettledChanges() = default; + SettledChanges(SettledChanges const&) = delete; + SettledChanges& operator=(SettledChanges const&) = delete; + SettledChanges(SettledChanges&&) = delete; + SettledChanges& operator=(SettledChanges&&) = delete; + + /** + * @brief Reports @p change. + * @param isFileContent Whether it changed the content of a file: it is held back until nothing changed + * there for the settle time, a later content change of the same file replaces it. An addition stays + * an addition. + */ + void add(ReportedChange change, bool isFileContent); + + /** + * @brief What a held back file looks like from outside. + */ + struct Appearance + { + std::uintmax_t size; + std::filesystem::file_time_type modified; + + friend bool operator==(Appearance const&, Appearance const&) = default; + }; + + private: + struct Pending + { + ReportedChange change; + /** @brief Orders the held back changes by when they last changed. */ + std::uint64_t sequence; + /** @brief Tells the wait that ends from the ones restarting the timer replaced. */ + std::uint64_t generation; + boost::asio::steady_timer timer; + /** @brief At the last event or the last end of the settle time, nothing when the file was not there. */ + std::optional appearance{}; + }; + + /** + * @brief Sends the held back changes below @p base, and at it when @p includingBase, in their order. + */ + void flush(std::filesystem::path const& base, bool includingBase); + void hold(ReportedChange change); + /** + * @brief Sends the change held back at @p key once the settle time passed without a change of the file. + */ + void settle(std::filesystem::path const& key, Pending& pending); + + private: + boost::asio::strand strand_; + ChangeSink send_; + std::chrono::milliseconds settleTime_; + std::map> pending_; + std::uint64_t nextSequence_; + }; +} diff --git a/backend/include/backend/file_tracking/temp_dir_instance_manager.hpp b/backend/include/backend/file_tracking/temp_dir_instance_manager.hpp index 98b2055f..29371fd5 100644 --- a/backend/include/backend/file_tracking/temp_dir_instance_manager.hpp +++ b/backend/include/backend/file_tracking/temp_dir_instance_manager.hpp @@ -1,10 +1,10 @@ #pragma once +#include #include -#include #include -#include +#include #include #include @@ -14,12 +14,6 @@ #include #include -namespace Nui -{ - class Window; - class RpcHub; -} - namespace FileTracking { /** @@ -34,11 +28,13 @@ namespace FileTracking namespace InstanceManagerDetail { /** - * @brief What the reply of FileTracking::fileStates lists: every regular file below @p instanceDirectory as - * {path, size, mtime} (seconds) and every directory as {path, directory: true}, paths canonical and - * UTF-8. Symbolic links and the bookkeeping directory are left out. + * @brief The states (see fileStateOf) of every regular file and directory below @p instanceDirectory, paths + * canonical. Symbolic links and the bookkeeping directory are left out. + * @param below Lists only this path and what lies below it, nothing when it is outside the instance or in its + * bookkeeping. Empty lists everything. */ - nlohmann::json fileStatesOf(std::filesystem::path const& instanceDirectory); + std::vector + fileStatesOf(std::filesystem::path const& instanceDirectory, std::filesystem::path const& below = {}); /** * @brief @p path as content of @p instanceDirectory that may be removed: its parent directory resolved, @@ -66,41 +62,19 @@ namespace FileTracking }; /** - * @brief Owns and tracks all TemporaryDirectoryInstance objects for a session. - * - * Inherits RpcHelper::StrandRpc so that all RPC handlers it registers run on - * the shared strand. The manager also owns the dead-instance cleanup policy: - * the retention scan runs once at start (see scheduleStartupCleanup), the - * frontend's Clean Up removes every orphaned directory at once (forceCleanup). - * - * ### Registered RPC endpoints - * All names are prefixed with @c "FileTracking::". + * @brief Owns and tracks the TemporaryDirectoryInstance objects of every session. * - * | Name | Parameters | Returns | - * |-------------------------|-------------------------------------|--------------------------------| - * | createInstance | — | {success, instanceId, instanceDir} | - * | destroyInstance | instanceId: string | {success} | - * | listInstances | — | {success, instances: [...]} | - * | getInstanceInfo | instanceId: string | {success, instanceId, instanceDir} | - * | fileStates | instanceId: string | {success, files: [...]}, see InstanceManagerDetail::fileStatesOf | - * | discardContent | instanceId: string, path: string | {success, instanceDestroyed: bool}, see discardContent | - * | countOrphaned | — | {success, count: int} | - * | forceCleanup | — | {success, removed: int} | - * - * ### Push events (instance → frontend) - * Each instance emits @c "FileTracking::\::onFileChanged" - * (see TemporaryDirectoryInstance). + * The manager also owns the dead-instance cleanup policy: the retention scan runs once at start (see + * scheduleStartupCleanup), the user's Clean Up removes every orphaned directory at once (removeOrphaned). + * Lives on its strand: every member must be called from it. */ - class TempDirInstanceManager : public RpcHelper::StrandRpc + class TempDirInstanceManager { public: /** * @brief Construct the manager. * - * @param executor Executor for the shared strand. - * @param wnd Nui window forwarded to each instance. - * @param hub RPC hub forwarded to each instance and used - * for manager RPC registrations. + * @param strand Runs the instances' changes and the startup cleanup. * @param tempRootDir Root directory under which instance * subdirectories are created. * @param deadInstanceRetentionHours @@ -108,9 +82,7 @@ namespace FileTracking * before manualCleanup() deletes it. */ TempDirInstanceManager( - boost::asio::any_io_executor executor, - Nui::Window& wnd, - Nui::RpcHub& hub, + boost::asio::strand strand, std::filesystem::path tempRootDir, std::chrono::hours deadInstanceRetentionHours = std::chrono::hours{24} ); @@ -122,18 +94,10 @@ namespace FileTracking TempDirInstanceManager(TempDirInstanceManager&&) = delete; TempDirInstanceManager& operator=(TempDirInstanceManager&&) = delete; - /** - * @brief Register all RPC handlers listed in the class documentation. - * - * Must be called once after construction, from the strand or before the - * strand starts processing. - */ - void registerRpc(); - /** * @brief Runs manualCleanup() once on the strand, for the instance directories that earlier runs left. * - * Called once at start, after registerRpc(), so the scan runs before the frontend's first request. + * Called once at start, so the scan runs before the frontend's first request. */ void scheduleStartupCleanup(); @@ -149,7 +113,7 @@ namespace FileTracking TemporaryDirectoryInstance* createInstance(); /** - * @brief Remove the watches of the instance with the given ID, then destroy it. + * @brief Destroy the instance with the given ID. * * If the instance does not exist this is a no-op. * @@ -158,9 +122,8 @@ namespace FileTracking void destroyInstance(std::string const& instanceId); /** - * @brief Removes @p path, content of the instance @p instanceId that the user decided not to keep, after - * ending the instance's watches, so the removal is reported as no change. A link is removed itself, - * never followed. When nothing else is left in the instance directory, the instance is destroyed + * @brief Removes @p path, content of the instance @p instanceId that the user decided not to keep. End its + * watch first, or the removal is reported as a change. A link is removed itself, never followed. When nothing else is left in the instance directory, the instance is destroyed * and, unlike destroyInstance(), its directory is removed as well (bookkeeping included, after its * lock is released), so no orphan stays behind. * @@ -170,6 +133,24 @@ namespace FileTracking std::expected discardContent(std::string const& instanceId, std::filesystem::path const& path); + /** + * @brief Ends the instance @p instanceId and removes its directory with everything in it, for an instance + * whose content is of no use, like the remains of a failed download. Nothing happens when there is none. + */ + void removeInstance(std::string const& instanceId); + + /** + * @brief How many instance directories below the root no live process holds. + */ + std::size_t countOrphaned() const; + + /** + * @brief Removes every instance directory below the root that no live process holds, regardless of the + * retention period. + * @return How many were removed. + */ + int removeOrphaned(); + /** * @brief Look up an instance by ID. * @@ -199,10 +180,9 @@ namespace FileTracking void manualCleanup(); private: + boost::asio::strand strand_; std::filesystem::path tempRootDir_; std::chrono::hours deadInstanceRetentionHours_; - Nui::Window* wnd_; std::map> instances_; - std::map> watches_; }; } diff --git a/backend/include/backend/file_tracking/temp_dir_instancing.hpp b/backend/include/backend/file_tracking/temp_dir_instancing.hpp index e0658ca0..bb38d696 100644 --- a/backend/include/backend/file_tracking/temp_dir_instancing.hpp +++ b/backend/include/backend/file_tracking/temp_dir_instancing.hpp @@ -1,9 +1,10 @@ #pragma once +#include #include #include +#include #include -#include #include #include @@ -13,29 +14,8 @@ #include #include -namespace Nui -{ - class Window; - class RpcHub; -} - namespace FileTracking { - /** - * @brief Mirrors the efsw action values; used to produce the "action" field - * in onFileChanged RPC events. - */ - enum class FileAction - { - Added, - Modified, - Deleted, - Moved, - }; - BOOST_DESCRIBE_ENUM(FileAction, Added, Modified, Deleted, Moved) - - // ------------------------------------------------------------------------- - /** * @brief Manages one temporary directory instance used for file-change tracking. * @@ -49,21 +29,10 @@ namespace FileTracking * by TempDirInstanceManager) can reclaim the space after the configured * retention period. * - * File-change events inside the instance directory are forwarded to the frontend - * via @c Nui::RpcHub under the event name - * @c "FileTracking::::onFileChanged" with a JSON payload: - * @code - * { - * "action": "Added" | "Modified" | "Deleted" | "Moved", - * "directory": "/abs/path/to/dir/", - * "filename": "changed-file.txt", - * "oldFilename": "previous-name.txt", // only populated for Moved - * "isDirectory": false // the changed path is a directory now - * } - * @endcode + * File-change events inside the instance directory go to the sink of its first watch, see addWatch(). * * The class is non-copyable and non-movable because it owns OS-level resources - * (lock file descriptor, efsw watcher thread). + * (lock file descriptor, its directory watches). */ class TemporaryDirectoryInstance { @@ -82,23 +51,16 @@ namespace FileTracking * and write the initial metadata. * * @param config Directory root configuration. - * @param strand Strand used to serialise RPC event dispatch from the efsw - * callback thread. - * @param wnd Nui window (reserved for future use). - * @param hub RPC hub used to push file-change events to the frontend. + * @param strand Strand the changes of the watched directories are handled + * and reported on. * * @throws std::runtime_error if the instance directory cannot be created or * the lock file cannot be acquired. */ - TemporaryDirectoryInstance( - Config config, - boost::asio::strand strand, - Nui::Window& wnd, - Nui::RpcHub& hub - ); + TemporaryDirectoryInstance(Config config, boost::asio::strand strand); /** - * @brief Stops the efsw watcher and writes the death timestamp to the + * @brief Stops reporting changes and writes the death timestamp to the * metadata file, but does **not** delete the instance directory. */ ~TemporaryDirectoryInstance(); @@ -119,24 +81,26 @@ namespace FileTracking std::filesystem::path instanceDir() const; /** - * @brief Register an efsw watch and return a move-only RAII handle. + * @brief Watch a directory and return a move-only RAII handle. * * The watch is removed automatically when the returned InstanceWatch is - * destroyed or release() is called on it. The efsw watcher is (re)started - * after a successful watch is added. + * destroyed or release() is called on it. * * @param path Absolute path or path relative to instanceDir(). Must * resolve to a location inside instanceDir(). * @param recursive Whether to recurse into subdirectories. * @param downloadTempSuffix The suffix downloads into the instance write under until complete, those * files and their final rename are not mirrored to the remote side. + * @param sink Receives the changes. The first watch decides the suffix and the sink, later ones share + * its event stream. * @return A valid InstanceWatch on success, or std::nullopt if the path is * outside instanceDir(), does not exist, or efsw rejected the watch. */ std::optional addWatch( std::filesystem::path const& path, bool recursive = true, - std::string downloadTempSuffix = std::string{Persistence::defaultTempFileSuffix} + std::string downloadTempSuffix = std::string{Persistence::defaultTempFileSuffix}, + ChangeSink sink = [](ReportedChange const&) {} ); /** diff --git a/backend/include/backend/file_tracking/tracker.hpp b/backend/include/backend/file_tracking/tracker.hpp new file mode 100644 index 00000000..095faa3c --- /dev/null +++ b/backend/include/backend/file_tracking/tracker.hpp @@ -0,0 +1,183 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + class LocalWatchManager; + + /** + * @brief How long a downloaded copy that was deleted or renamed away has to come back before its watch ends: + * editors that delete the file or move it aside before they write the new one. + */ + inline constexpr std::chrono::milliseconds copyReturnTime{5000}; + + /** + * @brief What the trackers of the sessions work with, both managers on the strand of the sessions. + */ + struct TrackerServices + { + TempDirInstanceManager* instances{nullptr}; + LocalWatchManager* localWatches{nullptr}; + /** @brief The options a new watch starts with, called on the strand. */ + std::function defaultOptions{}; + }; + + /** + * @brief Organizes the file tracking of one session: holds its watches, decides what each change of a watched + * item does on the server and queues that, and asks the user when the server's copy changed meanwhile. + * Its view shows the outcome, the frontend only forwards the user's input. No two watches upload to the + * same remote path or one below the other. When a watch lost changes, what is there is compared with what + * the server is known to have: what is gone below a directory that is there is deleted like reported, and + * what changed is uploaded. Lives on the strand of the session: every member must be called from it. + * + * @par Deleted or moved away + * A directory deleted locally is deleted on the server, one that left with its content (moved out of the + * item, to the trash) keeps its server copy. A deletion reports the content first, so a directory still known + * to have content when its removal is reported left as a whole. Where a watch loses those deletions, the + * directory goes with its whole known tree: on Windows, which ends the read of a deleted watched directory + * but follows a moved one, and on macOS for the watched directory itself, which FSEvents stops reporting + * below once it is removed with its content (`rm -rf`). There it was deleted when its identity is found + * nowhere on its volume, see isDeletedFromItsVolume. + * + * @par Accepted limitation on macOS + * Only APFS and HFS+ volumes find a directory by its identity. On other volumes a watched directory removed + * with its content keeps its server tree, but for the entries FSEvents reported deleted before. + */ + class Tracker + { + public: + /** + * @brief A downloaded copy, ready to be opened, see open(). + */ + struct OpenedCopy + { + Ids::InstanceId instanceId; + Paths::LocalPath localPath; + }; + + /** + * @brief Why open() has no copy: a message built with Utility::localizedMessage, nothing when the download + * did not end well, which its card in the queue shows. + */ + using OpenFailure = std::optional; + + /** + * @brief Where uploadAndWatch() would upload an item to, see checkUploadAndWatch(). + */ + struct UploadTarget + { + /** @brief Canonical, as the watch reports it. */ + Paths::LocalPath item; + Paths::RemotePath destination; + }; + + /** + * @param strand The strand of the session, the changes of the watches are handled on it. + * @param instances Holds the instances of the downloaded copies, on @p strand. Outlives the tracker. + * @param localWatches Holds the watches of the uploaded originals, on @p strand. Outlives the tracker. + * @param downloadTempSuffix The suffix the session's downloads write under until complete, see + * Persistence::effectiveTempFileSuffix. The watches ignore those files. + * @param defaultOptions The options a new watch starts with. + */ + Tracker( + boost::asio::strand strand, + TempDirInstanceManager& instances, + LocalWatchManager& localWatches, + std::unique_ptr remote, + std::unique_ptr view, + std::string downloadTempSuffix, + std::function defaultOptions + ); + ~Tracker(); + Tracker(Tracker const&) = delete; + Tracker& operator=(Tracker const&) = delete; + Tracker(Tracker&&) = delete; + Tracker& operator=(Tracker&&) = delete; + + /** + * @brief Downloads @p remotePath into an instance directory of its own, and watches it there when @p watch + * is set: its changes go back to @p remotePath from then on. @p onOpened gets the copy once the watch + * started, or why there is none. A watch that would overlap another one on the server is refused + * before anything is downloaded. The copy never moves or deletes the server's file: a copy deleted or + * renamed away ends its watch unless it is back within copyReturnTime, the view shows why. + */ + void open( + Paths::RemotePath const& remotePath, + std::uint64_t sizeBytes, + bool watch, + std::function const&)> onOpened + ); + + /** + * @brief Where uploadAndWatch() would upload @p item when it goes into @p directory: under the name of the + * canonical item, so a symbolic link goes under its target's name. + * @return Why it cannot be watched now, built with Utility::localizedMessage: it overlaps another watch + * locally or on the server, or cannot be watched at all. + */ + std::expected + checkUploadAndWatch(Paths::LocalPath const& item, Paths::RemotePath const& directory) const; + + /** + * @brief Watches @p item, a canonical local file or directory, uploads it to @p destination, which may be + * replaced, and mirrors its changes from then on. + * @return Why it cannot be watched, see checkUploadAndWatch(). Nothing is uploaded then. + */ + std::expected + uploadAndWatch(Paths::LocalPath const& item, Paths::RemotePath const& destination); + + /** + * @brief Ends the watch @p watchId: a downloaded copy's instance ends with it, an uploaded original stays. + */ + void stop(Ids::WatchId const& watchId); + + void setOption(Ids::WatchId const& watchId, SharedData::WatchOption option, bool value); + + /** + * @brief Removes @p path, a download of the instance @p instanceId the user decided not to open, and ends + * its watch when it has one. Once nothing else is left in the instance, it ends and its directory is + * removed, so it leaves no orphan. + */ + std::expected + discard(Ids::InstanceId const& instanceId, Paths::LocalPath const& path); + + /** + * @brief The user's answer to the question @p questionId, see TrackerView::ask(). + */ + void answer(Ids::QuestionId const& questionId, SharedData::ServerChangedAnswer answer); + + /** + * @brief Ends every watch without telling the view, for a frontend that is gone or a session that closes. The + * instances of the copies it downloaded, watched or only opened, end too: they count as orphaned then, + * or go when nothing is left in them. A download on its way is dropped, it opens nothing. + */ + void stopAll(); + + /** + * @brief The session lost its connection: every watch ends like in stopAll(), either side may change until it + * is back, and the view shows why. The user watches anew. + */ + void connectionLost(); + + private: + struct Implementation; + std::shared_ptr impl_; + }; +} diff --git a/backend/include/backend/file_tracking/tracker_remote.hpp b/backend/include/backend/file_tracking/tracker_remote.hpp new file mode 100644 index 00000000..2dbc545c --- /dev/null +++ b/backend/include/backend/file_tracking/tracker_remote.hpp @@ -0,0 +1,164 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + /** + * @brief What a Tracker last knew the server to have at the destination of an upload. + */ + struct ExpectedDestination + { + /** @brief The file the server had there, nothing when it had none. */ + std::optional file{}; + }; + + /** + * @brief An upload a Tracker queues, it replaces what the server has at the destination. + */ + struct UploadRequest + { + Paths::LocalPath source; + Paths::RemotePath destination; + /** @brief Refresh the remote listing once it is done. */ + bool refreshListing{false}; + /** @brief Create the directories above the destination that the server lacks. */ + bool createMissingDirectories{false}; + /** + * @brief Replace a regular file's destination only while the server still has what was expected there, see + * UploadEnd::changedOnServer. Unset, whatever it has is replaced. + */ + std::optional expectedDestination{}; + /** + * @brief The destination of a stopped upload of the same file, renamed since: this upload continues the + * partial file it kept, see TrackerRemote::cancelUpload(). + */ + std::optional continuesPartialOf{}; + }; + + /** + * @brief What becomes of the partial file of an upload that is stopped, see TrackerRemote::cancelUpload(). + */ + enum class PartialFile + { + /** @brief It stays for an upload that continues it, see UploadRequest::continuesPartialOf. */ + Kept, + Discarded, + }; + + /** + * @brief A local file an upload wrote to the server. + */ + struct SentFile + { + Paths::LocalPath path; + /** @brief Its state from before it was read until the upload ended, nothing when it changed meanwhile. */ + std::optional state{}; + }; + + /** + * @brief How a queued upload ended. + */ + struct UploadEnd + { + SharedData::OperationCompletionReason reason{SharedData::OperationCompletionReason::Failed}; + /** @brief The destination no longer held the request's expectedDestination, nothing replaced it. */ + bool changedOnServer{false}; + /** + * @brief What the server has at the destination after a completed upload of anything but a directory, nothing + * when it has nothing there or could not be asked. + */ + std::optional destinationState{}; + /** + * @brief The regular files a completed upload wrote to the server, those below a directory too. + */ + std::vector sentFiles{}; + }; + + /** + * @brief The server side of a Tracker: the operation queue of its session, whose operations go ahead of the + * queue, and the server's files. Called on the strand of the session. + */ + class TrackerRemote + { + public: + /** + * @brief Called once on the strand when a queued operation ended. + */ + using OnEnded = std::function; + + /** + * @brief Called once on the strand when a queued upload ended. + */ + using OnUploadEnded = std::function; + + TrackerRemote() = default; + virtual ~TrackerRemote() = default; + TrackerRemote(TrackerRemote const&) = delete; + TrackerRemote& operator=(TrackerRemote const&) = delete; + TrackerRemote(TrackerRemote&&) = delete; + TrackerRemote& operator=(TrackerRemote&&) = delete; + + /** + * @brief What the server has at @p path now, nothing when it has nothing there. + * @return An error when the server could not be asked. + */ + virtual std::expected, std::string> stat(Paths::RemotePath const& path) = 0; + + /** + * @brief What the server has at @p path itself, a symbolic link not followed, see stat(). + */ + virtual std::expected, std::string> linkStat(Paths::RemotePath const& path) = 0; + + /** + * @brief Queues the download of @p source to @p destination, which must not exist yet. + */ + virtual std::expected download( + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t sizeBytes, + OnEnded onEnded + ) = 0; + + /** + * @brief Queues the upload, a source that is gone by the time it runs ends it without a transfer: its + * deletion is mirrored apart. + */ + virtual std::expected upload(UploadRequest const& request, OnUploadEnded onEnded) = 0; + + /** + * @brief Stops the upload to @p destination that is on its way. It still ends once, never before this call + * returned: as cancelled unless it ended first. The partial file it wrote stays for an upload that + * continues it or goes, see @p partial. Nothing happens to an upload that is not on its way or cannot + * be stopped. + */ + virtual void cancelUpload(Paths::RemotePath const& destination, PartialFile partial) = 0; + + /** + * @brief Removes the partial file a stopped upload to @p destination kept, see cancelUpload(), @p onEnded + * hears when that is done. Nothing is removed when there is none. + */ + virtual std::expected + discardPartialUpload(Paths::RemotePath const& destination, OnEnded onEnded) = 0; + + /** + * @brief Queues the rename of @p source to @p destination, which never replaces anything. + */ + virtual std::expected + rename(Paths::RemotePath const& source, Paths::RemotePath const& destination, OnEnded onEnded) = 0; + + /** + * @brief Queues the delete of @p path, a directory only when it is empty. + */ + virtual std::expected remove(Paths::RemotePath const& path, OnEnded onEnded) = 0; + }; +} diff --git a/backend/include/backend/file_tracking/tracker_view.hpp b/backend/include/backend/file_tracking/tracker_view.hpp new file mode 100644 index 00000000..01c53cc9 --- /dev/null +++ b/backend/include/backend/file_tracking/tracker_view.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace FileTracking +{ + /** + * @brief What the user sees of a Tracker: its entries, its questions, its progress and the orphaned instances. Called on the + * strand of the tracker. + */ + class TrackerView + { + public: + TrackerView() = default; + virtual ~TrackerView() = default; + TrackerView(TrackerView const&) = delete; + TrackerView& operator=(TrackerView const&) = delete; + TrackerView(TrackerView&&) = delete; + TrackerView& operator=(TrackerView&&) = delete; + + /** + * @brief Shows @p entry, new or changed. + */ + virtual void showEntry(SharedData::TrackedEntry const& entry) = 0; + + virtual void removeEntry(Ids::WatchId const& watchId) = 0; + + /** + * @brief Tells the user why the watches of @p ended ended, they did not stop them. Their entries are removed + * before, see removeEntry(). + */ + virtual void showEnded(SharedData::EndedWatches const& ended) = 0; + + /** + * @brief Asks @p question, the tracker waits for the answer before it asks the next one. + */ + virtual void ask(SharedData::ServerChangedQuestion const& question) = 0; + + /** + * @brief Shows how many instance directories no live process holds any more. + */ + virtual void showOrphanedCount(std::size_t count) = 0; + + /** + * @brief Shows how far the tracker is with the changes on their way, at most a few times a second. + */ + virtual void showProgress(SharedData::TrackerProgress const& progress) = 0; + }; +} diff --git a/backend/include/backend/file_tracking/tracking_state.hpp b/backend/include/backend/file_tracking/tracking_state.hpp new file mode 100644 index 00000000..39ae509a --- /dev/null +++ b/backend/include/backend/file_tracking/tracking_state.hpp @@ -0,0 +1,860 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + /** + * @brief Size and modification time (seconds) of a file on the server. + */ + struct ServerState + { + std::uint64_t mtime{0}; + std::uint64_t size{0}; + + friend bool operator==(ServerState const&, ServerState const&) = default; + }; + + /** + * @brief Size and modification time of a local file, as exact as its file system keeps them: the state an upload + * sent it in. + */ + struct SourceState + { + std::uint64_t size{0}; + std::filesystem::file_time_type modified{}; + + friend bool operator==(SourceState const&, SourceState const&) = default; + }; + + /** + * @brief Whether the server's copy of a file changed since @p known was taken. A file the server has but nobody + * saw here, a vanished one and one with another size or time all changed. + * @param known What the server had when it was last seen, nothing when it had no file there. + * @param current What the server has now, nothing when it has no file there. + */ + inline bool serverChanged(std::optional const& known, std::optional const& current) + { + if (!known) + return current.has_value(); + return !current || *current != *known; + } + + /** + * @brief The remote path a change at @p changed maps to: @p item stands for @p remotePath, whatever either is + * called, a change outside it maps to nothing. + */ + inline std::expected remotePathOfChange( + Paths::LocalPath const& changed, + Paths::LocalPath const& item, + Paths::RemotePath const& remotePath + ) + { + return changed.relativeTo(item).transform( + [&remotePath](Paths::RelativePath const& relative) + { + return remotePath / relative; + } + ); + } + + /** + * @brief What a move inside a tracked item does on the server. + */ + enum class MoveMeaning + { + /** @brief The server's copy of the source is renamed. */ + Rename, + /** + * @brief The destination's content changed and the source is gone: an editor saving through a temporary + * file, or a move over an existing file. A rename on the server would fail on the existing + * destination or find no source. + */ + ReplaceDestination, + }; + + /** + * @brief What a move means when the server has the destination (@p destinationOnServer) or the source + * (@p sourceOnServer), as far as file tracking knows. + */ + inline MoveMeaning meaningOfMove(bool destinationOnServer, bool sourceOnServer) + { + return destinationOnServer || !sourceOnServer ? MoveMeaning::ReplaceDestination : MoveMeaning::Rename; + } + + /** + * @brief @p path moved along with @p from to @p to, nothing when it is not at or below @p from. + */ + inline std::optional + movedPath(Paths::LocalPath const& path, Paths::LocalPath const& from, Paths::LocalPath const& to) + { + const auto relative = path.relativeTo(from); + if (!relative) + return std::nullopt; + if (relative->empty()) + return to; + return to.join(*relative) + .transform( + [](Paths::LocalPath moved) + { + return std::optional{std::move(moved)}; + } + ) + .value_or(std::nullopt); + } + + /** + * @brief Orders local paths by their text, so the paths below one are found by the text they start with. + */ + struct LocalPathText + { + using is_transparent = void; + + static std::string_view textOf(Paths::LocalPath const& path) + { + return path.toUtf8(); + } + + static std::string_view textOf(std::string_view text) + { + return text; + } + + bool operator()(auto const& first, auto const& second) const + { + return textOf(first) < textOf(second); + } + }; + + /** + * @brief What the server is known to have of one tracked item, by local path: what the download brought and + * what was uploaded since. Lookups below a path cost a search, not a scan. + */ + class KnownServerContent + { + public: + void rememberFile(Paths::LocalPath const& path, ServerState state) + { + files_.insert_or_assign(path, state); + sent_.erase(path); + } + + /** + * @brief The server's copy of the known file at @p path holds what the file had locally in @p state, an + * upload sent it so. Forgotten once the file is remembered anew. + */ + void rememberSent(Paths::LocalPath const& path, SourceState state) + { + if (files_.contains(path)) + sent_.insert_or_assign(path, state); + } + + /** + * @brief The local state the server's copy of the file at @p path was sent in, see rememberSent(). Nothing + * when it is not known. + */ + std::optional sentFrom(Paths::LocalPath const& path) const + { + const auto found = sent_.find(path); + if (found == sent_.end()) + return std::nullopt; + return found->second; + } + + void rememberDirectory(Paths::LocalPath const& path) + { + directories_.insert(path); + } + + /** + * @brief The known state of the file at @p path, nothing when no file is known there. + */ + std::optional fileState(Paths::LocalPath const& path) const + { + const auto found = files_.find(path); + if (found == files_.end()) + return std::nullopt; + return found->second; + } + + /** + * @brief Whether the server is known to have @p path or anything below it. + */ + bool has(Paths::LocalPath const& path) const + { + return files_.contains(path) || directories_.contains(path) || hasBelow(path); + } + + /** + * @brief The known path that names @p path on a file system that ignores Unicode normalization (macOS): + * @p path when it is known, else the one known path the watcher spells alike, see eventSpelling. Nothing + * when none or several do. Elsewhere only @p path itself. + */ + std::optional knownSpellingOf(Paths::LocalPath const& path) const + { + if (files_.contains(path) || directories_.contains(path)) + return path; + const auto utf8 = path.toUtf8(); + // A spelling that differs in normalization only has a byte outside ASCII, and so has its match. + const auto firstOutsideAscii = std::ranges::find_if(utf8, isOutsideAscii); + if (firstOutsideAscii == utf8.end()) + return std::nullopt; + // Every spelling agrees up to the letter ahead of it, which a combining mark may join. + const auto agreed = std::max(firstOutsideAscii - utf8.begin() - 1, std::ptrdiff_t{0}); + const auto prefix = std::string_view{utf8}.substr(0, static_cast(agreed)); + const auto spelling = eventSpelling(utf8); + std::optional found{}; + bool ambiguous{false}; + const auto consider = [&](Paths::LocalPath const& known) + { + const auto knownUtf8 = known.toUtf8(); + if (std::ranges::none_of(knownUtf8, isOutsideAscii) || eventSpelling(knownUtf8) != spelling) + return; + ambiguous = ambiguous || found.has_value(); + found = known; + }; + for (auto const& file : startingWith(files_, prefix)) + consider(pathOf(file)); + for (auto const& directory : startingWith(directories_, prefix)) + consider(pathOf(directory)); + if (ambiguous) + return std::nullopt; + return found; + } + + /** + * @brief Whether the server is known to have anything below @p path. + */ + bool hasBelow(Paths::LocalPath const& path) const + { + return !belowOf(files_, path).empty() || !belowOf(directories_, path).empty(); + } + + bool hasDirectory(Paths::LocalPath const& path) const + { + return directories_.contains(path); + } + + /** + * @brief Forgets @p path and everything below it. + */ + void forget(Paths::LocalPath const& path) + { + files_.erase(path); + directories_.erase(path); + kept_.erase(path); + sent_.erase(path); + eraseBelow(files_, path); + eraseBelow(directories_, path); + eraseBelow(kept_, path); + eraseBelow(sent_, path); + } + + /** + * @brief Moves what is known at @p from and below it to @p to, after the server renamed it. + */ + void move(Paths::LocalPath const& from, Paths::LocalPath const& to) + { + moveIn(files_, from, to); + moveIn(directories_, from, to); + moveIn(kept_, from, to); + moveIn(sent_, from, to); + } + + /** + * @brief The directory @p path left locally with its content, its server copy stays (see keptDirectories). + */ + void keep(Paths::LocalPath const& path) + { + kept_.insert(path); + } + + /** + * @brief The directories that left with their content while the server kept its copies: moved to the trash, + * out of the item, or within it as a removal and an addition (see holdsAllBelow). + */ + std::vector keptDirectories() const + { + return {kept_.begin(), kept_.end()}; + } + + /** + * @brief The kept directories above @p path that nothing is known below anymore, closest first: their + * content was removed after they were. + */ + std::vector emptiedKeptAbove(Paths::LocalPath const& path) const + { + std::vector emptied; + for (auto parent = path.parent(); parent && !hasBelow(*parent); parent = parent->parent()) + if (kept_.contains(*parent)) + emptied.push_back(*parent); + return emptied; + } + + /** + * @brief Whether everything known below @p original is known below @p copy too, at the same relative path + * and with the same state. Moves carry the modification times along. + */ + bool holdsAllBelow(Paths::LocalPath const& copy, Paths::LocalPath const& original) const + { + const auto holdsFile = [this, ©, &original](auto const& file) + { + const auto relative = file.first.relativeTo(original); + if (!relative) + return false; + const auto atCopy = copy.join(*relative); + return atCopy && fileState(*atCopy) == file.second; + }; + const auto holdsDirectory = [this, ©, &original](Paths::LocalPath const& directory) + { + const auto relative = directory.relativeTo(original); + if (!relative) + return false; + const auto atCopy = copy.join(*relative); + return atCopy && directories_.contains(*atCopy); + }; + const auto files = belowOf(files_, original); + const auto directories = belowOf(directories_, original); + return (!files.empty() || !directories.empty()) && std::ranges::all_of(files, holdsFile) && + std::ranges::all_of(directories, holdsDirectory); + } + + /** + * @brief What is known at and below @p path, deepest first: the order to remove it entry by entry. + */ + std::vector deepestFirst(Paths::LocalPath const& path) const + { + std::vector paths; + if (files_.contains(path) || directories_.contains(path)) + paths.push_back(path); + for (auto const& file : belowOf(files_, path)) + paths.push_back(file.first); + for (auto const& directory : belowOf(directories_, path)) + paths.push_back(directory); + // A path sorts before everything below it. + std::ranges::sort(paths, std::ranges::greater{}, &Paths::LocalPath::toUtf8); + return paths; + } + + private: + static bool isOutsideAscii(char character) + { + return static_cast(character) >= 0x80u; + } + + template + static Paths::LocalPath const& pathOf(std::pair const& file) + { + return file.first; + } + + static Paths::LocalPath const& pathOf(Paths::LocalPath const& directory) + { + return directory; + } + + /** + * @brief The entries of @p container whose path starts with @p prefix. + */ + template + static std::ranges::subrange().begin())> + startingWith(ContainerT& container, std::string_view prefix) + { + const auto first = container.lower_bound(prefix); + auto last = first; + while (last != container.end() && pathOf(*last).toUtf8().starts_with(prefix)) + ++last; + return std::ranges::subrange{first, last}; + } + + /** + * @brief The entries of @p container strictly below @p path. + */ + template + static std::ranges::subrange().begin())> + belowOf(ContainerT& container, Paths::LocalPath const& path) + { + const auto prefix = path.isRoot() ? path.toUtf8() : path.toUtf8() + '/'; + auto below = startingWith(container, prefix); + // A root ends with its separator, it is its own first match. + if (!below.empty() && pathOf(*below.begin()) == path) + below.advance(1); + return below; + } + + template + static void eraseBelow(ContainerT& container, Paths::LocalPath const& path) + { + const auto below = belowOf(container, path); + container.erase(below.begin(), below.end()); + } + + /** + * @brief Moves the entries at and below @p from in @p container to @p to, they replace what is known there. + */ + template + static void moveIn(ContainerT& container, Paths::LocalPath const& from, Paths::LocalPath const& to) + { + std::vector nodes; + if (auto node = container.extract(from)) + nodes.push_back(std::move(node)); + const auto below = belowOf(container, from); + for (auto entry = below.begin(); entry != below.end();) + nodes.push_back(container.extract(entry++)); + for (auto& node : nodes) + { + auto& path = [&node]() -> Paths::LocalPath& + { + if constexpr (requires { node.key(); }) + return node.key(); + else + return node.value(); + }(); + path = movedPath(path, from, to).value_or(path); + container.erase(path); + container.insert(std::move(node)); + } + } + + private: + std::map files_; + std::set directories_; + std::set kept_; + /** @brief The local states of known files whose server copies an upload sent, see rememberSent(). */ + std::map sent_; + }; + + /** + * @brief Keeps the server operations of a tracker in the order it decided them: one waits until every earlier + * one on the same path, a path above it or one below it ended. The operation queue runs operations side by + * side, a delete could otherwise remove the upload that followed it, or a directory's delete come before + * the deletes of its content. + */ + class ServerOperationOrder + { + public: + using Ticket = std::uint64_t; + + /** + * @brief Registers an operation on @p paths. + * @return Its ticket and whether it may start now. One that waits starts once leave() returns its ticket. + */ + std::pair enter(std::vector paths) + { + const auto ticket = nextTicket_++; + const bool startsNow = std::ranges::none_of( + operations_, + [&paths](Operation const& earlier) + { + return overlaps(earlier.paths, paths); + } + ); + operations_.push_back({.ticket = ticket, .paths = std::move(paths), .started = startsNow}); + return {ticket, startsNow}; + } + + /** + * @brief The operation @p ticket ended, or never starts. + * @return The tickets that may start now, in the order they entered. + */ + std::vector leave(Ticket ticket) + { + const auto left = std::ranges::find(operations_, ticket, &Operation::ticket); + if (left == operations_.end()) + return {}; + const auto leftPaths = std::move(left->paths); + operations_.erase(left); + std::vector starting; + for (auto waiting = operations_.begin(); waiting != operations_.end(); ++waiting) + { + if (waiting->started || !overlaps(waiting->paths, leftPaths)) + continue; + const bool blocked = std::ranges::any_of( + operations_.begin(), + waiting, + [&waiting](Operation const& earlier) + { + return overlaps(earlier.paths, waiting->paths); + } + ); + if (blocked) + continue; + waiting->started = true; + starting.push_back(waiting->ticket); + } + return starting; + } + + bool empty() const + { + return operations_.empty(); + } + + private: + struct Operation + { + Ticket ticket; + std::vector paths; + bool started; + }; + + static bool overlaps(std::vector const& first, std::vector const& second) + { + return std::ranges::any_of( + first, + [&second](Paths::RemotePath const& one) + { + return std::ranges::any_of( + second, + [&one](Paths::RemotePath const& other) + { + return one.isWithin(other) || other.isWithin(one); + } + ); + } + ); + } + + private: + std::deque operations_{}; + Ticket nextTicket_{0}; + }; + + /** + * @brief A changed path of a tracked item. + */ + struct LocalChange + { + Paths::LocalPath path; + bool isDirectory{false}; + + friend bool operator==(LocalChange const&, LocalChange const&) = default; + }; + + /** + * @brief The uploads of one tracked item that are on their way, from the change until the server's state after + * the upload is known. One path has at most one, and nothing below a directory being uploaded starts its + * own, so two uploads never write the same remote file at once. + */ + class PendingUploads + { + public: + /** + * @brief What became of the uploads of a path when its upload ended. + */ + struct Finished + { + /** @brief The path changed again meanwhile and has to go once more. */ + bool changedAgain{false}; + /** @brief The path was deleted or moved away meanwhile, its server copy has to go. */ + bool removeRemote{false}; + /** @brief Whether the path was a directory at its latest change. */ + bool isDirectory{false}; + /** @brief Changes below the uploaded directory that arrived meanwhile, in arrival order. */ + std::vector changedBelow{}; + /** @brief Where the path moved meanwhile, see moveWhenDone and moveBelow, nothing when it stayed. */ + std::optional movedTo{}; + /** + * @brief Where its server copy is now by local path, when a directory above took it along (see + * moveBelow), nothing when it is where the upload started. + */ + std::optional serverCopyAt{}; + /** @brief The changes it counted as, see changeCount(), without the one more of changedAgain. */ + std::size_t counted{1}; + /** @brief Where the upload of a file was handed on to, see sent(), nothing before that. */ + std::optional sentTo{}; + /** @brief The partial file of the stopped upload stays for one that continues it, see keepsPartial(). */ + bool partialKept{false}; + }; + + /** + * @brief Starts the upload of @p change unless one of its path or of a directory above it is on its way, the + * change then goes again once that one finished, as the kind it has now (see Finished::isDirectory). + * @return Whether the upload starts. + */ + bool start(LocalChange const& change) + { + if (const auto found = std::ranges::find_if(pending_, IsAt{change.path}); found != pending_.end()) + { + found->second.changedAgain = true; + found->second.removeRemote = false; + found->second.isDirectory = change.isDirectory; + return false; + } + const auto directoryAbove = std::ranges::find_if( + pending_, + [&change](auto const& pending) + { + return pending.second.isDirectory && change.path.isWithin(currentPath(pending)); + } + ); + if (directoryAbove != pending_.end()) + { + if (!std::ranges::contains(directoryAbove->second.changedBelow, change)) + directoryAbove->second.changedBelow.push_back(change); + return false; + } + pending_.emplace(change.path, Pending{.isDirectory = change.isDirectory}); + return true; + } + + /** + * @brief The content of @p path as it is now is being handled, by the upload reading it or by the answer to + * a question about it: the changes that arrived so far need nothing more. + */ + void changesHandled(Paths::LocalPath const& path) + { + if (const auto found = pending_.find(path); found != pending_.end()) + { + // The changes below go along with it, they still count. + found->second.carried = countedOf(found->second); + found->second.changedAgain = false; + found->second.changedBelow.clear(); + } + } + + /** + * @brief The upload of @p path carries @p entries entries, the path and what lies below it: it counts as at + * least that many changes, see changeCount(). + */ + void carries(Paths::LocalPath const& path, std::size_t entries) + { + if (const auto found = pending_.find(path); found != pending_.end()) + found->second.carried = std::max(found->second.carried, entries); + } + + /** + * @brief Whether an upload of @p path is on its way. + */ + bool isPending(Paths::LocalPath const& path) const + { + return std::ranges::any_of(pending_, IsAt{path}); + } + + /** + * @brief Whether an upload of @p path or of a directory above it is on its way, a change of it then goes + * once that one finished. + */ + bool covers(Paths::LocalPath const& path) const + { + return std::ranges::any_of( + pending_, + [&path](auto const& pending) + { + auto const& current = currentPath(pending); + return current == path || (pending.second.isDirectory && path.isWithin(current)); + } + ); + } + + /** + * @brief The path of a pending upload was deleted or moved away: its server copy goes once the upload + * ended, and it does not go again. + * @return Whether an upload of @p path is on its way. + */ + bool removeRemoteWhenDone(Paths::LocalPath const& path) + { + const auto found = std::ranges::find_if(pending_, IsAt{path}); + if (found == pending_.end()) + return false; + found->second.removeRemote = true; + found->second.changedAgain = false; + return true; + } + + /** + * @brief The path of a pending upload moved from @p from to @p to: its server copy follows once the upload + * ended (see Finished::movedTo), the changes at and below @p to wait for it like those of the path did. + * @return Whether an upload of @p from is on its way. + */ + bool moveWhenDone(Paths::LocalPath const& from, Paths::LocalPath const& to) + { + const auto found = std::ranges::find_if(pending_, IsAt{from}); + if (found == pending_.end()) + return false; + found->second.movedTo = to; + for (auto& change : found->second.changedBelow) + change.path = movedPath(change.path, from, to).value_or(change.path); + return true; + } + + /** + * @brief A directory above pending uploads moved from @p from to @p to: they read the paths below @p to now, + * see Finished::movedTo, and so do the changes waiting for them. + * @param serverCopiesFollow Whether the server copies below @p from are now those below @p to, because the + * server's directory was renamed too or the item that maps to the server was. + */ + void moveBelow(Paths::LocalPath const& from, Paths::LocalPath const& to, bool serverCopiesFollow) + { + const auto movedBelow = [&from, &to](Paths::LocalPath const& path) -> std::optional + { + if (path == from) + return std::nullopt; + return movedPath(path, from, to); + }; + for (auto& [path, pending] : pending_) + { + if (const auto local = movedBelow(currentPathOf(path, pending))) + pending.movedTo = *local; + if (const auto server = movedBelow(pending.serverCopyAt.value_or(path)); server && serverCopiesFollow) + pending.serverCopyAt = *server; + for (auto& change : pending.changedBelow) + change.path = movedBelow(change.path).value_or(change.path); + } + } + + /** + * @brief The upload that started at @p path was handed on to go to @p destination, as a directory when + * @p isDirectory is set. + */ + void sent(Paths::LocalPath const& path, Paths::RemotePath destination, bool isDirectory) + { + if (const auto found = pending_.find(path); found != pending_.end()) + { + found->second.sentTo = std::move(destination); + found->second.sentAsDirectory = isDirectory; + } + } + + /** + * @brief Where the upload of the file whose path is @p path now was handed on to, nothing for one handed on + * as a directory and for an upload that was not handed on yet. A later change may tell another kind: + * a directory removed meanwhile can no longer be told from a file. + */ + std::optional sentFileTo(Paths::LocalPath const& path) const + { + const auto found = std::ranges::find_if(pending_, IsAt{path}); + if (found == pending_.end() || found->second.sentAsDirectory) + return std::nullopt; + return found->second.sentTo; + } + + /** + * @brief The upload of the path that is @p path now is stopped and its partial file kept, an upload that + * continues it follows, see Finished::partialKept. + */ + void keepsPartial(Paths::LocalPath const& path) + { + if (const auto found = std::ranges::find_if(pending_, IsAt{path}); found != pending_.end()) + found->second.partialKept = true; + } + + /** + * @brief Whether the path of the pending upload that started at @p path is no longer there, see moveWhenDone + * and moveBelow: reading it would fail. + */ + bool movedAway(Paths::LocalPath const& path) const + { + const auto found = pending_.find(path); + return found != pending_.end() && found->second.movedTo.has_value(); + } + + /** + * @brief Ends the upload of @p path, the path it started with. + */ + Finished finish(Paths::LocalPath const& path) + { + const auto found = pending_.find(path); + if (found == pending_.end()) + return {}; + const auto counted = countedOf(found->second); + auto finished = Finished{ + .changedAgain = found->second.changedAgain, + .removeRemote = found->second.removeRemote, + .isDirectory = found->second.isDirectory, + .changedBelow = std::move(found->second.changedBelow), + .movedTo = std::move(found->second.movedTo), + .serverCopyAt = std::move(found->second.serverCopyAt), + .counted = counted, + .sentTo = std::move(found->second.sentTo), + .partialKept = found->second.partialKept, + }; + pending_.erase(found); + return finished; + } + + bool empty() const + { + return pending_.empty(); + } + + /** + * @brief The changes on their way: each pending upload with the entries it carries or the changes waiting + * below its directory, whichever are more since those mostly are the same, and the one more a file + * takes when it changed again. What a directory that changed again lacks counts once it goes. + */ + std::size_t changeCount() const + { + std::size_t count = 0; + for (auto const& pending : std::views::values(pending_)) + count += countedOf(pending) + (pending.changedAgain && !pending.isDirectory ? 1 : 0); + return count; + } + + private: + struct Pending + { + bool isDirectory{false}; + bool changedAgain{false}; + bool removeRemote{false}; + std::vector changedBelow{}; + std::optional movedTo{}; + std::optional serverCopyAt{}; + /** @brief The entries the upload carries, see carries(). */ + std::size_t carried{1}; + std::optional sentTo{}; + bool sentAsDirectory{false}; + bool partialKept{false}; + }; + + static std::size_t countedOf(Pending const& pending) + { + return std::max(pending.carried, 1 + pending.changedBelow.size()); + } + + static Paths::LocalPath const& currentPathOf(Paths::LocalPath const& path, Pending const& pending) + { + return pending.movedTo ? *pending.movedTo : path; + } + + /** + * @brief Where the path of @p pending is now, it is keyed by the one its upload started with. + */ + static Paths::LocalPath const& currentPath(std::pair const& pending) + { + return currentPathOf(pending.first, pending.second); + } + + /** + * @brief Matches the pending upload whose path is @p path now. + */ + struct IsAt + { + Paths::LocalPath const& path; + + bool operator()(std::pair const& pending) const + { + return currentPath(pending) == path; + } + }; + + std::map pending_; + }; +} diff --git a/backend/include/backend/main.hpp b/backend/include/backend/main.hpp index 4f13657c..ac6e0dfd 100644 --- a/backend/include/backend/main.hpp +++ b/backend/include/backend/main.hpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -74,7 +73,6 @@ class Main std::unique_ptr rpcSystem_; ProcessStore processes_; PasswordPrompter prompter_; - std::unique_ptr tempDirInstanceManager_; std::optional commandStore_; /** @brief StoreRpc, or UnavailableStoreRpc when the store could not be opened. */ std::unique_ptr commandStoreRpc_; diff --git a/backend/include/backend/session.hpp b/backend/include/backend/session.hpp index df21957e..a325c854 100644 --- a/backend/include/backend/session.hpp +++ b/backend/include/backend/session.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -42,7 +43,8 @@ class Session Nui::RpcHub& hub, Persistence::SftpOptions const& sftpOptions, BulkResumeRegistry* bulkResumeRegistry, - std::shared_ptr localOperationQueue + std::shared_ptr localOperationQueue, + FileTracking::TrackerServices fileTracking = {} ); Session(const Session&) = delete; @@ -63,7 +65,44 @@ class Session */ void adoptBulkResumes(std::vector const& operationIds); + /** + * @brief Ends the watches of this session without telling the frontend, which is gone, and releases the instances + * of its copies, see FileTracking::Tracker::stopAll(). Must be called from the strand. + */ + void stopFileTracking(); + + /** + * @brief The connection of this session was lost: its watches end and the frontend shows why, see + * FileTracking::Tracker::connectionLost(). Must be called from the strand, before stop(). + */ + void onConnectionLost(); + private: + /** + * @brief The server side of the file tracker: this session's operation queue and SFTP channels. + */ + class FileTrackingRemote; + + /** + * @brief Creates the file tracker of this session, the strand runs it. + */ + void startFileTracking(); + + /** + * @brief Handles the requests of the file tracking panel, "FileTracker::{id}::..." with the paths as UTF-8: + * open (remote path, size, watch) answers once the copy is there with {instanceId, localPath}, an error to + * show or neither, checkUploadAndWatch (local item, remote directory) with {item, destination}, + * uploadAndWatch (local item, remote destination), stop (watch id), setOption (watch id, option, value), + * discard (instance id, local path) with {instanceDestroyed} and answer (question id, answer). See + * FileTracking::Tracker. + */ + void registerRpcFileTracker(); + + /** + * @brief An open SFTP channel of this session, nothing when there is none. + */ + std::shared_ptr anySftpChannel() const; + /** * Handles calls from the frontend to create a new channel with the following payload: * { @@ -370,4 +409,8 @@ class Session std::string remoteTempFileSuffix_; /** @brief Runs the local deletes of this session, which go on when it ends. */ std::shared_ptr localOperationQueue_; + FileTracking::TrackerServices fileTrackingServices_; + /** @brief Suffix of this session's downloads in progress, the file tracker ignores them. */ + std::string downloadTempFileSuffix_; + std::unique_ptr tracker_{}; }; \ No newline at end of file diff --git a/backend/include/backend/session_manager.hpp b/backend/include/backend/session_manager.hpp index 430c7282..ebedcecd 100644 --- a/backend/include/backend/session_manager.hpp +++ b/backend/include/backend/session_manager.hpp @@ -1,12 +1,16 @@ #pragma once #include +#include +#include #include #include #include #include +#include #include #include +#include #include #include @@ -16,6 +20,7 @@ #include #include +#include #include #include #include @@ -46,6 +51,17 @@ class SessionManager */ void preventLocalDeletion(bool prevent); + /** + * @brief Starts file tracking on the strand of the sessions: the instances of temporary downloads below + * @p temporaryDownloadsDirectory and the watches of local items. Call once, before the first session. + */ + void startFileTracking(std::filesystem::path const& temporaryDownloadsDirectory); + + /** + * @brief The options a new watch starts with from now on. Safe to call from any thread. + */ + void fileTrackingDefaults(Persistence::FileTrackingOptions const& options); + void addPasswordProvider(int priority, PasswordProvider* provider); void joinSessionAdder(); void addSession( @@ -85,8 +101,11 @@ class SessionManager */ void registerRpcSessionDisconnect(); - /// Removes a session and closes all its channels. Safe to call from any thread. - void removeSession(Ids::SessionId sessionId); + /** + * @brief Removes a session and closes all its channels. Safe to call from any thread. + * @param connectionLost The session lost its connection, its watches tell the frontend why they end. + */ + void removeSession(Ids::SessionId sessionId, bool connectionLost = false); /** * @brief Handles "Session::sftp::adoptBulkResume" — pulls saved bulk @@ -104,6 +123,18 @@ class SessionManager */ void registerRpcDiscardBulkResume(); + /** + * @brief Handles "FileTracking::countOrphaned" with {count} and "FileTracking::forceCleanup" with {removed}: the + * instance directories no live process holds, which the latter removes. + */ + void registerRpcOrphanedInstances(); + + /** + * @brief Handles "FileTracker::stopAll": a frontend that loads ends the watches of the one before it (a reload), + * nobody is left to see them. + */ + void registerRpcStopFileTracking(); + /** * @brief Makes a new request the pending one, so that shutdown() can cancel it. * @@ -119,6 +150,11 @@ class SessionManager private: std::atomic_bool updateDispatchRunning_{false}; Persistence::StateHolder* stateHolder_{}; + /** @brief Outlive the sessions, which watch through them. */ + std::unique_ptr temporaryInstances_{}; + std::unique_ptr localWatches_{}; + /** @brief Read and written on the strand. */ + SharedData::WatchOptions fileTrackingDefaults_{}; std::unordered_map, Ids::IdHash> sessions_{}; /** @brief Read by the local operation queue on each local delete the explorer asks for. */ std::shared_ptr localDeletionPrevented_{std::make_shared(true)}; diff --git a/backend/include/backend/sftp/bulk_upload_operation.hpp b/backend/include/backend/sftp/bulk_upload_operation.hpp index 44151e98..62db68e3 100644 --- a/backend/include/backend/sftp/bulk_upload_operation.hpp +++ b/backend/include/backend/sftp/bulk_upload_operation.hpp @@ -35,6 +35,12 @@ class BulkUploadOperation : public BulkTransferOperation SharedData::OperationType type() const override; + /** + * @brief Completes without a transfer when the scanned directory is gone before the scan and the individual + * options let the source vanish, see UploadOperation::UploadOperationOptions::sourceMayVanish. + */ + std::expected work() override; + /** @brief See BulkDownloadOperation::PrescannedFile. */ struct PrescannedFile { diff --git a/backend/include/backend/sftp/operation.hpp b/backend/include/backend/sftp/operation.hpp index 55abc65a..75a1ce8c 100644 --- a/backend/include/backend/sftp/operation.hpp +++ b/backend/include/backend/sftp/operation.hpp @@ -187,6 +187,15 @@ class Operation return Error{.type = ErrorType::RenameFailure}; } + /** + * @brief The level a failure with @p error is logged at: a destination that changed since its compare is no error, + * the compare kept it on purpose and the owner of the operation reports it. + */ + static Log::Level failureLevelOf(Error const& error) + { + return error.type == ErrorType::ChangedSinceCompare ? Log::Level::Warning : Log::Level::Error; + } + protected: void enterState(OperationState newState) { diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index 34760186..45d9b6c3 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -28,9 +29,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -56,9 +59,38 @@ class OperationQueue int parallelism = 1 ); + /** + * @brief Cancels every queued operation, each completes as cancelled. + */ void cancelAll(); + + /** + * @brief Drops every queued operation without completing any, for a session that goes: a reconnect may adopt + * their backups, see dumpBulkResumes(), and the frontend keeps their cards until then. + */ + void dropAll(); + + /** + * @brief Cancels the operation @p id, it completes as cancelled. + */ void cancel(Ids::OperationId id); + /** + * @brief Cancels the upload @p id like cancel(), the temporary file it wrote to stays for a later upload or goes, + * see @p partial. + */ + void cancelUpload(Ids::OperationId id, UploadOperation::PartialFile partial); + + /** + * @brief Removes the temporary file an upload to @p remotePath left on @p sftp, a cancelled one that kept it, and + * calls @p onDone on the strand then. Nothing is removed when there is none. + */ + void discardPartialUpload( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& remotePath, + std::function onDone + ); + /** * @brief Move @p operationId within the regular `operations_` queue to * the absolute index @p newIndex. Dispatched through the strand @@ -97,6 +129,19 @@ class OperationQueue SharedData::OperationMode mode = SharedData::OperationMode::Queued ); + /** + * @brief Queues the upload of the file, link or directory @p localPath to @p remotePath. + * @param sourceMayVanish A source that is gone by the time the upload starts completes it without a transfer, see + * UploadOperation::UploadOperationOptions::sourceMayVanish. One that is gone already is refused as + * FileNotFound, which is no error worth logging. + * @param comparedDestination What a file or link must find at @p remotePath to replace it, see + * UploadOperation::UploadOperationOptions::comparedDestination. A directory ignores it. + * @param onSent Hears of each regular file sent unchanged, the files below a directory too, see + * UploadOperation::UploadOperationOptions::onSent. + * @param adoptPartialOf The destination of a cancelled upload of the file under its old name, whose temporary + * file this one continues, see UploadOperation::UploadOperationOptions::adoptPartialOf. A directory + * ignores it. + */ std::expected addUploadOperation( SecureShell::SftpSession& sftp, Ids::OperationId operationId, @@ -106,7 +151,11 @@ class OperationQueue bool isBigFile, bool insertRefresh, bool createMissingDirectories, - SharedData::OperationMode mode = SharedData::OperationMode::Queued + SharedData::OperationMode mode = SharedData::OperationMode::Queued, + bool sourceMayVanish = false, + std::optional comparedDestination = std::nullopt, + UploadOperation::OnSent onSent = {}, + std::optional adoptPartialOf = std::nullopt ); /** @@ -203,12 +252,16 @@ class OperationQueue /** * @brief Upload analogue of @ref addBulkDownloadOperation. + * @param sourceMayVanish Each entry may be gone by the time it is uploaded, see addUploadOperation. + * @param onSent Hears of each regular file sent unchanged, see addUploadOperation. */ std::size_t addBulkUploadOperation( SecureShell::SftpSession& sftp, SharedData::BulkUploadRequest request, std::function const& operationIdFor, - Ids::OperationId const& bulkCardId + Ids::OperationId const& bulkCardId, + bool sourceMayVanish = false, + UploadOperation::OnSent const& onSent = {} ); /** @@ -314,6 +367,26 @@ class OperationQueue */ void setCompletionObserver(std::function observer); + /** + * @brief Calls @p callback on the strand once the operation @p operationId completed, failed or was cancelled, + * after the frontend was told. Must be called from the strand, before the operation can complete. + */ + void whenCompleted(Ids::OperationId const& operationId, std::function callback); + + /** + * @brief Keeps no resume backup of the operation @p operationId, a reconnect does not queue it again. For an owner + * that decides anew what goes to the server. Must be called from the strand. + */ + void discardResumeBackup(Ids::OperationId const& operationId); + + /** + * @brief The owner of the operation @p operationId asks the user about destinations that changed since their + * compare, so such an end is no failure of the queue's: the operation completes as canceled with the error + * that tells the owner why, and a bulk leaves those entries out of its failed ones while its indexed + * failures keep them. Must be called from the strand. + */ + void leaveChangedDestinationsToOwner(Ids::OperationId const& operationId); + bool hasPriorityWork() const; /** @@ -398,6 +471,16 @@ class OperationQueue */ void cancelSiblingScan(Ids::OperationId const& id); + /** + * @brief Takes the operation @p id out of the queue it waits in, nothing when no queue has it. + */ + std::unique_ptr takeOperation(Ids::OperationId const& id); + + /** + * @brief Cancels @p operation, already taken out of its queue, and completes it as cancelled. + */ + void completeCancelled(Ids::OperationId const& id, Operation& operation); + /** * @brief Hands the tree of a completed standalone scan to the sync session registered for its id. */ @@ -423,6 +506,10 @@ class OperationQueue std::deque>> operations_{}; std::atomic_bool paused_{true}; std::function completionObserver_{}; + std::unordered_map, Ids::IdHash> + completionCallbacks_{}; + /** @brief See leaveChangedDestinationsToOwner(). */ + std::unordered_set changedDestinationsLeftToOwner_{}; // Transfer slots handed out per queue tick. int parallelism_{1}; // Upper bound on the slots a single bulk transfer may take, clamped to parallelism_. diff --git a/backend/include/backend/sftp/upload_operation.hpp b/backend/include/backend/sftp/upload_operation.hpp index 91985c2e..3e1af413 100644 --- a/backend/include/backend/sftp/upload_operation.hpp +++ b/backend/include/backend/sftp/upload_operation.hpp @@ -9,14 +9,46 @@ #include #include #include +#include +#include #include -#include +#include +#include #include class UploadOperation : public Operation { public: + /** + * @brief The local state of a regular file an upload sent: it had it from before it was opened until the upload + * ended, so the server's copy holds the content it has in that state. + */ + struct SentSource + { + std::uint64_t size{0}; + std::filesystem::file_time_type modified{}; + + friend bool operator==(SentSource const&, SentSource const&) = default; + }; + + /** + * @brief Hears of the regular file @p source once it replaced the destination, with the state it was sent in + * when it stayed unchanged (see SentSource). A continued upload and a file that changed meanwhile come + * without one. Called on the strand of the transfer. + */ + using OnSent = std::function const& unchanged)>; + + /** + * @brief What a cancelled upload does with the temporary file it wrote to. + */ + enum class PartialFile + { + /** @brief It stays, a later upload of the same source continues it. */ + Kept, + Discarded, + }; + struct UploadOperationOptions { std::function comparedDestination{std::nullopt}; + /** + * @brief A source that is gone when the upload starts is nothing to upload, it completes without a transfer. + * For uploads that mirror local files whose deletions are handled apart, see FileTracking::Tracker. + */ + bool sourceMayVanish{false}; + /** + * @brief Hears of a regular file the upload wrote to the server, see OnSent. A link is not told. + */ + OnSent onSent{}; + /** + * @brief The destination of a cancelled upload of the same source under its old name: its temporary file + * becomes this upload's before it is continued, see tryContinue. Unless this upload has one already. + */ + std::optional adoptPartialOf{}; }; SecureShell::ProcessingStrand* strand() const override; @@ -101,6 +147,12 @@ class UploadOperation : public Operation std::expected cancel(bool adoptCancelState) override; + /** + * @brief What a later cancel(true) does with the temporary file this upload wrote to, kept unless told otherwise. + * A temporary file it did not open itself always stays. + */ + void partialFileOnCancel(PartialFile partial); + void pause(bool doPause) override; std::expected prepare(); @@ -122,8 +174,20 @@ class UploadOperation : public Operation */ std::expected openLocalSource(); + /** + * @brief Whether the local source is not there at all, see UploadOperationOptions::sourceMayVanish. A source that + * cannot be checked is not gone, opening it fails the upload. + */ + bool sourceIsGone() const; + std::expected openOrAdoptFileInStrand(SecureShell::SftpSession& sftp); + /** + * @brief Moves the temporary file of UploadOperationOptions::adoptPartialOf to tempPath_ when this upload has none + * there. A failure leaves the upload to start anew. + */ + void takeOverPartialFileInStrand(SecureShell::SftpSession& sftp); + /// Remote-heavy portion of prepare() — runs under a single strand umbrella. std::expected prepareInStrand(SecureShell::SftpSession& sftp); @@ -157,8 +221,26 @@ class UploadOperation : public Operation */ void applyModificationTimeInStrand(SecureShell::SftpSession& sftp); + /** + * @brief Tells UploadOperationOptions::onSent of the source once it replaced the destination, with its state + * unless it changed since it was opened. + */ + void reportSent() const; + void cleanup(); + /** + * @brief What cancel() does, on the processing thread unless the session is gone: the transfer reads the local + * file and writes the temporary one there. + */ + void cancelInStrand(bool adoptCancelState); + + /** + * @brief Queues the removal of the temporary file this upload wrote to on the strand of the transfer, after the + * file was closed on the server. A failure is only logged. + */ + void removeTemporaryFile(); + private: std::weak_ptr sftp_; std::weak_ptr fileStream_; @@ -169,8 +251,16 @@ class UploadOperation : public Operation std::filesystem::path localNativePath_{}; /** @brief Where the upload is written before it replaces remotePath_. */ Paths::RemotePath tempPath_{}; - std::ifstream localFile_; + /** @brief Lets others rename or delete the file while it uploads, which a std::ifstream refuses on Windows. */ + Utility::SharedInputFileStream localFile_; bool isSymlink_{false}; + /** @brief The state of the source right before it was opened, nothing when it could not be read. */ + std::optional sourceBeforeOpening_{}; + /** @brief The upload continued a temp file an earlier one left, whose content is not known to be the source's. */ + bool continued_{false}; + /** @brief The upload opened tempPath_ to write to it, created anew or continued. */ + bool openedTemporaryFile_{false}; + PartialFile partialFileOnCancel_{PartialFile::Kept}; std::size_t leftToUpload_{0}; std::size_t totalSize_{0}; SecureShell::BufferLease buffer_; diff --git a/backend/source/backend/CMakeLists.txt b/backend/source/backend/CMakeLists.txt index ea7d1e39..17169348 100644 --- a/backend/source/backend/CMakeLists.txt +++ b/backend/source/backend/CMakeLists.txt @@ -3,6 +3,7 @@ add_library( STATIC session_manager.cpp session.cpp + session_file_tracking.cpp rpc_filesystem.cpp local_delete_guard.cpp local_removal.cpp @@ -20,12 +21,24 @@ add_library( process/process.cpp process/process_store.cpp process/environment.cpp + file_tracking/batching_remote.cpp file_tracking/change_filter.cpp + file_tracking/change_reporter.cpp + file_tracking/directory_watch_service.cpp + file_tracking/directory_watcher.cpp + file_tracking/disk_place.cpp + file_tracking/event_spelling.cpp + file_tracking/file_states.cpp + file_tracking/local_item_watch.cpp + file_tracking/local_watch_manager.cpp + file_tracking/settled_changes.cpp file_tracking/instance_bookkeeping.cpp file_tracking/instance_lock.cpp file_tracking/instance_watch.cpp file_tracking/temp_dir_instancing.cpp file_tracking/temp_dir_instance_manager.cpp + file_tracking/frontend_tracker_view.cpp + file_tracking/tracker.cpp sftp/operation_queue.cpp sftp/download_operation.cpp sftp/upload_operation.cpp diff --git a/backend/source/backend/file_tracking/batching_remote.cpp b/backend/source/backend/file_tracking/batching_remote.cpp new file mode 100644 index 00000000..348809a4 --- /dev/null +++ b/backend/source/backend/file_tracking/batching_remote.cpp @@ -0,0 +1,305 @@ +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + namespace + { + struct GatheredUpload + { + UploadRequest request; + TrackerRemote::OnUploadEnded onEnded; + }; + + struct GatheredRemoval + { + Paths::RemotePath path; + TrackerRemote::OnEnded onEnded; + }; + + /** + * @brief Whether @p request may go in a bulk: a regular file smaller than @p aloneFromBytes. Symbolic links + * and directories keep the single upload that handles them, and so does what must not create the + * directories above it, refreshes or continues a partial file. A large file goes alone, so it can be + * stopped. + */ + bool fitsBulk(UploadRequest const& request, std::uint64_t aloneFromBytes) + { + if (!request.createMissingDirectories || request.refreshListing || request.continuesPartialOf) + return false; + std::error_code statusError; + if (std::filesystem::symlink_status(request.source.native(), statusError).type() != + std::filesystem::file_type::regular) + return false; + std::error_code sizeError; + const auto size = std::filesystem::file_size(request.source.native(), sizeError); + return !sizeError && size < aloneFromBytes; + } + + /** + * @brief Ends every one of @p onEnded with @p ended. + */ + template + void endAll(std::vector const& onEnded, EndT const& ended) + { + for (auto const& callback : onEnded) + if (callback) + callback(ended); + } + + /** + * @brief Hands the end of each entry of a bulk to the callback it was gathered with. + */ + template + auto entryEndsOf(std::vector onEnded) + { + return [onEnded = std::move(onEnded)](std::size_t index, auto const& ended) + { + if (index < onEnded.size() && onEnded[index]) + onEnded[index](ended); + }; + } + + const UploadEnd failedUpload{.reason = SharedData::OperationCompletionReason::Failed}; + } + + struct BatchingRemote::Implementation : public std::enable_shared_from_this + { + std::unique_ptr inner; + Limits limits; + boost::asio::steady_timer timer; + std::vector uploads{}; + std::vector removals{}; + std::chrono::steady_clock::time_point batchStart{}; + + Implementation( + boost::asio::strand strand, + std::unique_ptr inner, + Limits limits + ) + : inner{std::move(inner)} + , limits{limits} + , timer{std::move(strand)} + {} + + /** + * @brief Something was gathered: the batch goes once nothing more comes for the quiet time, after the longest + * time or right after this call when it is full. + */ + void gathered() + { + const auto now = std::chrono::steady_clock::now(); + const auto count = uploads.size() + removals.size(); + if (count >= limits.maxEntries) + { + timer.cancel(); + boost::asio::post( + timer.get_executor(), + [weak = weak_from_this(), + full = std::exchange(uploads, {}), + fullRemovals = std::exchange(removals, {})]() mutable + { + if (const auto self = weak.lock()) + self->send(std::move(full), std::move(fullRemovals)); + } + ); + return; + } + if (count == 1) + batchStart = now; + timer.expires_at(std::min(now + limits.quiet, batchStart + limits.longest)); + timer.async_wait( + [weak = weak_from_this()](boost::system::error_code const& error) + { + if (error) + return; + if (const auto self = weak.lock()) + self->send(std::exchange(self->uploads, {}), std::exchange(self->removals, {})); + } + ); + } + + void send(std::vector gatheredUploads, std::vector gatheredRemovals) + { + sendUploads(std::move(gatheredUploads)); + sendRemovals(std::move(gatheredRemovals)); + } + + void sendUploads(std::vector gathered) + { + std::vector bulk; + std::vector bulkEnds; + std::vector single; + for (auto& upload : gathered) + { + if (fitsBulk(upload.request, limits.aloneFromBytes)) + { + bulk.push_back(std::move(upload.request)); + bulkEnds.push_back(std::move(upload.onEnded)); + } + else + single.push_back(std::move(upload)); + } + if (bulk.size() == 1) + single.push_back({.request = std::move(bulk.front()), .onEnded = std::move(bulkEnds.front())}); + else if (!bulk.empty()) + { + Log::info("FileTracking: uploading {} changed files as one operation", bulk.size()); + if (const auto queued = inner->uploadAll(bulk, entryEndsOf(bulkEnds)); !queued) + { + Log::warn("FileTracking: did not upload {} changed files: {}", bulk.size(), queued.error()); + endAll(bulkEnds, failedUpload); + } + } + for (auto& upload : single) + { + if (const auto queued = inner->upload(upload.request, upload.onEnded); !queued) + { + // A temporary file an editor already renamed or removed ends up here. + Log::warn("FileTracking: did not upload '{}': {}", upload.request.source.toUtf8(), queued.error()); + if (upload.onEnded) + upload.onEnded(failedUpload); + } + } + } + + void sendRemovals(std::vector gathered) + { + if (gathered.empty()) + return; + if (gathered.size() == 1) + { + auto& removal = gathered.front(); + if (const auto queued = inner->remove(removal.path, removal.onEnded); !queued) + { + Log::error("FileTracking: did not delete '{}': {}", removal.path.value(), queued.error()); + if (removal.onEnded) + removal.onEnded(SharedData::OperationCompletionReason::Failed); + } + return; + } + std::vector paths; + std::vector ends; + for (auto& removal : gathered) + { + paths.push_back(std::move(removal.path)); + ends.push_back(std::move(removal.onEnded)); + } + Log::info("FileTracking: deleting {} paths as one operation", paths.size()); + if (const auto queued = inner->removeAll(paths, entryEndsOf(ends)); !queued) + { + Log::error("FileTracking: did not delete {} paths: {}", paths.size(), queued.error()); + endAll(ends, SharedData::OperationCompletionReason::Failed); + } + } + + /** + * @brief Takes the gathered upload to @p destination out of the next batch, nothing when none is gathered. + */ + std::optional takeGathered(Paths::RemotePath const& destination) + { + const auto found = std::ranges::find_if( + uploads, + [&destination](GatheredUpload const& upload) + { + return upload.request.destination == destination; + } + ); + if (found == uploads.end()) + return std::nullopt; + auto taken = std::move(*found); + uploads.erase(found); + return taken; + } + }; + + BatchingRemote::BatchingRemote( + boost::asio::strand strand, + std::unique_ptr inner, + Limits limits + ) + : impl_{std::make_shared(std::move(strand), std::move(inner), limits)} + {} + + BatchingRemote::BatchingRemote( + boost::asio::strand strand, + std::unique_ptr inner + ) + : BatchingRemote{std::move(strand), std::move(inner), Limits{}} + {} + + BatchingRemote::~BatchingRemote() = default; + + std::expected, std::string> BatchingRemote::stat(Paths::RemotePath const& path) + { + return impl_->inner->stat(path); + } + + std::expected, std::string> BatchingRemote::linkStat(Paths::RemotePath const& path) + { + return impl_->inner->linkStat(path); + } + + std::expected BatchingRemote::download( + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t sizeBytes, + OnEnded onEnded + ) + { + return impl_->inner->download(source, destination, sizeBytes, std::move(onEnded)); + } + + std::expected BatchingRemote::upload(UploadRequest const& request, OnUploadEnded onEnded) + { + impl_->uploads.push_back({.request = request, .onEnded = std::move(onEnded)}); + impl_->gathered(); + return {}; + } + + void BatchingRemote::cancelUpload(Paths::RemotePath const& destination, PartialFile partial) + { + auto gathered = impl_->takeGathered(destination); + if (!gathered) + return impl_->inner->cancelUpload(destination, partial); + Log::debug("FileTracking: not sending the upload to '{}', it was stopped", destination.value()); + // Ends after the call, whose caller may still be deciding what follows the stop. + boost::asio::post( + impl_->timer.get_executor(), + [onEnded = std::move(gathered->onEnded)]() + { + if (onEnded) + onEnded({.reason = SharedData::OperationCompletionReason::Canceled}); + } + ); + } + + std::expected + BatchingRemote::discardPartialUpload(Paths::RemotePath const& destination, OnEnded onEnded) + { + return impl_->inner->discardPartialUpload(destination, std::move(onEnded)); + } + + std::expected + BatchingRemote::rename(Paths::RemotePath const& source, Paths::RemotePath const& destination, OnEnded onEnded) + { + return impl_->inner->rename(source, destination, std::move(onEnded)); + } + + std::expected BatchingRemote::remove(Paths::RemotePath const& path, OnEnded onEnded) + { + impl_->removals.push_back({.path = path, .onEnded = std::move(onEnded)}); + impl_->gathered(); + return {}; + } +} diff --git a/backend/source/backend/file_tracking/change_filter.cpp b/backend/source/backend/file_tracking/change_filter.cpp index 298c3aff..c3d2811b 100644 --- a/backend/source/backend/file_tracking/change_filter.cpp +++ b/backend/source/backend/file_tracking/change_filter.cpp @@ -47,6 +47,17 @@ namespace FileTracking return isSafeSaveTemporary(name); } + bool isTransientChange( + std::string const& filename, + std::string const& oldFilename, + std::string const& downloadTempSuffix + ) + { + // A real file renamed onto a temporary name is how an editor moves the original aside. + return isTransientFile(filename, downloadTempSuffix) && + (oldFilename.empty() || isTransientFile(oldFilename, downloadTempSuffix)); + } + std::optional filterFileChange(FileChange change, bool isDirectory, std::string const& downloadTempSuffix) { @@ -71,4 +82,21 @@ namespace FileTracking return change; } + + bool isChangeOfItem( + std::filesystem::path const& item, + std::filesystem::path const& path, + std::filesystem::path const& oldPath + ) + { + return isAtOrBelow(item, path) || isAtOrBelow(item, oldPath); + } + + bool isAtOrBelow(std::filesystem::path const& base, std::filesystem::path const& path) + { + if (path.empty()) + return false; + const auto relative = path.lexically_normal().lexically_relative(base.lexically_normal()); + return !relative.empty() && *relative.begin() != ".."; + } } diff --git a/backend/source/backend/file_tracking/change_reporter.cpp b/backend/source/backend/file_tracking/change_reporter.cpp new file mode 100644 index 00000000..0d62eb02 --- /dev/null +++ b/backend/source/backend/file_tracking/change_reporter.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include +#include + +#include + +namespace FileTracking +{ + ChangeReporter::ChangeReporter( + boost::asio::strand strand, + ChangeSink sink, + std::string const& downloadTempSuffix + ) + : settledChanges_{std::make_shared(std::move(strand), std::move(sink))} + , downloadTempSuffix_{Persistence::effectiveTempFileSuffix(downloadTempSuffix)} + {} + + bool ChangeReporter::isNoise(std::string const& filename) const + { + return isTransientFile(filename, downloadTempSuffix_); + } + + void ChangeReporter::report(DirectoryChange const& change) const + { + if (change.action == FileAction::Missed) + return settledChanges_->add( + ReportedChange{.action = FileAction::Missed, .path = change.path(), .isDirectory = change.isDirectory}, + /*isFileContent=*/false + ); + const auto filtered = filterFileChange( + {change.action, change.filename, change.oldFilename}, change.isDirectory, downloadTempSuffix_ + ); + if (!filtered) + return; + + const bool isFileContent = + !change.isDirectory && (filtered->action == FileAction::Added || filtered->action == FileAction::Modified); + settledChanges_->add( + ReportedChange{ + .action = filtered->action, + .path = change.directory / Paths::nativeFromUtf8(filtered->filename), + .oldPath = filtered->oldFilename.empty() + ? std::filesystem::path{} + : change.directory / Paths::nativeFromUtf8(filtered->oldFilename), + .isDirectory = change.isDirectory, + }, + isFileContent + ); + } +} diff --git a/backend/source/backend/file_tracking/directory_watch_service.cpp b/backend/source/backend/file_tracking/directory_watch_service.cpp new file mode 100644 index 00000000..ef36234f --- /dev/null +++ b/backend/source/backend/file_tracking/directory_watch_service.cpp @@ -0,0 +1,415 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + namespace + { + FileAction toFileAction(efsw::Action action) + { + switch (action) + { + case efsw::Actions::Add: + return FileAction::Added; + case efsw::Actions::Modified: + return FileAction::Modified; + case efsw::Actions::Delete: + return FileAction::Deleted; + case efsw::Actions::Moved: + return FileAction::Moved; + default: + return FileAction::Modified; + } + } + + DirectoryEvent eventOf( + std::string const& dir, + std::string const& filename, + efsw::Action action, + std::string const& oldFilename + ) + { + return DirectoryEvent{ + .action = toFileAction(action), + .directory = dir, + .filename = filename, + .oldFilename = oldFilename, + }; + } + + /** + * @brief The event telling that the changes in @p dir were lost, which is worth a line in the log. + */ + DirectoryEvent missedIn(std::string const& dir) + { + Log::warn("DirectoryWatchService: the watch lost the changes in '{}'", dir); + return DirectoryEvent{.action = FileAction::Missed, .directory = dir, .filename = {}, .oldFilename = {}}; + } + + /** + * @brief @p directory ending with a separator, so spellings with and without one compare equal. + */ + std::filesystem::path directoryKey(std::filesystem::path const& directory) + { + return (directory / "").lexically_normal(); + } + + /** + * @brief Whether watches without recursion share one efsw watcher, see DirectoryWatchService. + */ +#ifdef __APPLE__ + constexpr bool sharesWatcher = false; +#else + constexpr bool sharesWatcher = true; +#endif + } + + struct DirectoryWatchService::Implementation : public efsw::FileWatchListener + { + /** + * @brief The efsw watcher of one watch, which delivers to the subscriptions of @p key. + */ + struct OwnWatcher : public efsw::FileWatchListener + { + Implementation& service; + std::filesystem::path key; + /** @brief Declared last, so its thread ends before the listener goes. */ + efsw::FileWatcher watcher{}; + + OwnWatcher(Implementation& owner, std::filesystem::path watchedKey) + : service{owner} + , key{std::move(watchedKey)} + {} + + void handleFileAction( + efsw::WatchID /*watchId*/, + std::string const& dir, + std::string const& filename, + efsw::Action action, + std::string const& oldFilename + ) override + { + deliver(eventOf(dir, filename, action, oldFilename)); + } + + void handleMissedFileActions(efsw::WatchID /*watchId*/, std::string const& dir) override + { + deliver(missedIn(dir)); + } + + void deliver(DirectoryEvent const& event) + { + service.dispatch( + [this](std::map const& watches) -> Watch const* + { + // Until it is registered or after it was discarded the key may belong to another watch. + const auto found = watches.find(key); + return found != watches.end() && found->second.ownWatcher.get() == this ? &found->second + : nullptr; + }, + event + ); + } + }; + + struct Watch + { + efsw::WatchID efswId; + bool recursive; + std::map subscriptions; + /** @brief The watcher of the watch, nothing for one on the shared watcher. */ + std::unique_ptr ownWatcher{}; + }; + + /** @brief Serialises subscribe and unsubscribe. */ + std::mutex registrationMutex{}; + /** + * @brief Guards the maps for efsw's threads. Never held around a call into efsw, which holds its own lock + * while it calls handleFileAction. + */ + std::mutex dispatchMutex{}; + std::map watches{}; + /** @brief The directories of the watches on the shared watcher. */ + std::map directoryOfWatch{}; + std::map directoryOfSubscription{}; + std::int64_t nextSubscriptionId{0}; + /** + * @brief Shared by the watches without recursion. Declared last, so its thread ends before anything it calls + * into goes. + */ + efsw::FileWatcher watcher{}; + + Implementation() = default; + Implementation(Implementation const&) = delete; + Implementation& operator=(Implementation const&) = delete; + Implementation(Implementation&&) = delete; + Implementation& operator=(Implementation&&) = delete; + + ~Implementation() override + { + // Their threads deliver into the maps, so they end while those still exist. + std::vector> ownWatchers; + { + std::scoped_lock lock{dispatchMutex}; + for (auto& watch : std::views::values(watches)) + if (watch.ownWatcher) + ownWatchers.push_back(std::move(watch.ownWatcher)); + } + } + + void handleFileAction( + efsw::WatchID watchId, + std::string const& dir, + std::string const& filename, + efsw::Action action, + std::string const& oldFilename + ) override + { + deliver(watchId, eventOf(dir, filename, action, oldFilename)); + } + + void handleMissedFileActions(efsw::WatchID watchId, std::string const& dir) override + { + deliver(watchId, missedIn(dir)); + } + + /** + * @brief Hands @p event to the subscriptions of the watch @p watchId on the shared watcher. + */ + void deliver(efsw::WatchID watchId, DirectoryEvent const& event) + { + dispatch( + [this, watchId](std::map const& known) -> Watch const* + { + const auto directory = directoryOfWatch.find(watchId); + return directory == directoryOfWatch.end() ? nullptr : &known.at(directory->second); + }, + event + ); + } + + /** + * @brief Hands @p event to the subscriptions of the watch @p findWatch picks, it is called with the dispatch + * mutex held. + */ + void dispatch( + std::function const&)> const& findWatch, + DirectoryEvent const& event + ) + { + std::vector sinks; + { + std::scoped_lock lock{dispatchMutex}; + const auto* watch = findWatch(watches); + if (!watch) + return; + std::ranges::copy(watch->subscriptions | std::views::values, std::back_inserter(sinks)); + } + for (auto const& sink : sinks) + sink(event); + } + + /** + * @brief Watches @p directory with the efsw watcher @p efswWatcher, which reports to @p listener. + * @return The efsw watch, negative when efsw rejected it. + */ + static efsw::WatchID watchWithEfsw( + efsw::FileWatcher& efswWatcher, + efsw::FileWatchListener& listener, + std::filesystem::path const& directory, + bool recursive + ) + { + // A move between two of its directories comes as one move with the absolute old path, where the platform + // tells that, not as a deletion and an addition. + std::vector options; + if (recursive) + options.emplace_back(efsw::Options::ReportCrossDirectoryMoves, 1); + // efsw takes UTF-8, a narrow string is the ANSI code page on Windows. + const auto efswId = efswWatcher.addWatch(Paths::nativeToUtf8(directory), &listener, recursive, options); + if (efswId < 0) + { + Log::warn( + "DirectoryWatchService: efsw rejected the watch of '{}': {}", + Paths::nativeToUtf8(directory), + efsw::Errors::Log::getLastErrorLog() + ); + return efswId; + } + efswWatcher.watch(); + return efswId; + } + + /** + * @brief Subscribes @p sink to the watch of @p key, which exists. Called with the dispatch mutex held. + * @return Nothing when the watch is the other way (with or without recursion). + */ + std::optional joinWatch( + std::weak_ptr service, + std::filesystem::path const& key, + std::filesystem::path const& directory, + bool recursive, + DirectoryEventSink sink + ) + { + auto& watch = watches.at(key); + // Watches never nest (see LocalWatchManager), so nobody needs a directory watched both ways. + if (watch.recursive != recursive) + { + Log::warn( + "DirectoryWatchService: '{}' is watched {} already", + Paths::nativeToUtf8(directory), + watch.recursive ? "recursively" : "without recursion" + ); + return std::nullopt; + } + const auto subscriptionId = nextSubscriptionId++; + watch.subscriptions.emplace(subscriptionId, std::move(sink)); + directoryOfSubscription.emplace(subscriptionId, key); + return InstanceWatch{std::move(service), subscriptionId, directory}; + } + }; + + std::shared_ptr DirectoryWatchService::shared() + { + static std::mutex mutex; + static std::weak_ptr instance; + std::scoped_lock lock{mutex}; + if (auto service = instance.lock()) + return service; + auto service = std::make_shared(); + instance = service; + return service; + } + + DirectoryWatchService::DirectoryWatchService() + : impl_{std::make_unique()} + {} + + DirectoryWatchService::~DirectoryWatchService() = default; + + std::optional DirectoryWatchService::subscribe( + std::filesystem::path const& directory, + bool recursive, + DirectoryEventSink sink + ) + { + auto& impl = *impl_; + const auto key = directoryKey(directory); + { + std::scoped_lock registration{impl.registrationMutex}; + { + std::scoped_lock lock{impl.dispatchMutex}; + if (impl.watches.contains(key)) + return impl.joinWatch(weak_from_this(), key, directory, recursive, std::move(sink)); + } + if (!recursive && sharesWatcher) + { + const auto efswId = Implementation::watchWithEfsw(impl.watcher, impl, directory, recursive); + if (efswId < 0) + return std::nullopt; + std::scoped_lock lock{impl.dispatchMutex}; + impl.watches.emplace( + key, Implementation::Watch{.efswId = efswId, .recursive = recursive, .subscriptions = {}} + ); + impl.directoryOfWatch.emplace(efswId, key); + return impl.joinWatch(weak_from_this(), key, directory, recursive, std::move(sink)); + } + } + + // The walk below the directory takes as long as the tree is large, no lock of the service is held meanwhile. + auto ownWatcher = std::make_unique(impl, key); + const auto efswId = Implementation::watchWithEfsw(ownWatcher->watcher, *ownWatcher, directory, recursive); + if (efswId < 0) + return std::nullopt; + std::optional subscription; + { + std::scoped_lock registration{impl.registrationMutex}; + std::scoped_lock lock{impl.dispatchMutex}; + // Another caller watched the directory meanwhile, its watch serves this one too. + if (!impl.watches.contains(key)) + impl.watches.emplace( + key, + Implementation::Watch{ + .efswId = efswId, + .recursive = recursive, + .subscriptions = {}, + .ownWatcher = std::move(ownWatcher), + } + ); + subscription = impl.joinWatch(weak_from_this(), key, directory, recursive, std::move(sink)); + } + // A watcher nobody took ends here, outside the locks its thread takes. + ownWatcher.reset(); + return subscription; + } + + void DirectoryWatchService::unsubscribe(std::int64_t subscriptionId) + { + auto& impl = *impl_; + std::unique_ptr unusedOwnWatcher; + { + std::scoped_lock registration{impl.registrationMutex}; + std::optional unused; + { + std::scoped_lock lock{impl.dispatchMutex}; + const auto directory = impl.directoryOfSubscription.find(subscriptionId); + if (directory == impl.directoryOfSubscription.end()) + return; + const auto key = directory->second; + impl.directoryOfSubscription.erase(directory); + const auto watch = impl.watches.find(key); + if (watch == impl.watches.end()) + return; + watch->second.subscriptions.erase(subscriptionId); + if (!watch->second.subscriptions.empty()) + return; + if (watch->second.ownWatcher) + unusedOwnWatcher = std::move(watch->second.ownWatcher); + else + { + unused = watch->second.efswId; + impl.directoryOfWatch.erase(watch->second.efswId); + } + impl.watches.erase(watch); + } + if (unused) + impl.watcher.removeWatch(*unused); + } + // Joins its thread, outside the locks that thread takes. + unusedOwnWatcher.reset(); + } + + void DirectoryWatchService::reportMissed(std::filesystem::path const& directory) + { + impl_->dispatch( + [key = directoryKey(directory)]( + std::map const& watches + ) -> Implementation::Watch const* + { + const auto covering = std::ranges::find_if( + watches, + [&key](auto const& watch) + { + return watch.first == key || (watch.second.recursive && isAtOrBelow(watch.first, key)); + } + ); + return covering == watches.end() ? nullptr : &covering->second; + }, + missedIn(Paths::nativeToUtf8(directory)) + ); + } +} diff --git a/backend/source/backend/file_tracking/directory_watcher.cpp b/backend/source/backend/file_tracking/directory_watcher.cpp new file mode 100644 index 00000000..8239d266 --- /dev/null +++ b/backend/source/backend/file_tracking/directory_watcher.cpp @@ -0,0 +1,244 @@ +#include +#include +#include +#include +#include + +#include + +#ifdef __APPLE__ +# include +# include +#endif + +#include +#include +#include +#include +#include + +namespace FileTracking +{ + namespace + { + /** + * @brief Gives the directory of an event the spelling of the watched directory where the event spells it + * differently (see eventSpelling). What lies below the watched directory keeps the event's spelling. + */ + class WatchedSpelling + { + public: + explicit WatchedSpelling(std::filesystem::path const& directory) + : watchedSpelling_{Paths::nativeToUtf8(directory.has_filename() ? directory : directory.parent_path())} + , eventSpelling_{eventSpelling(watchedSpelling_)} + {} + + std::filesystem::path of(std::string const& eventDirectory) const + { + // Only macOS spells events differently, its separator ends the watched part. + const bool spelledDifferently = eventSpelling_ != watchedSpelling_ && + eventDirectory.starts_with(eventSpelling_) && + (eventDirectory.size() == eventSpelling_.size() || eventDirectory[eventSpelling_.size()] == '/'); + if (!spelledDifferently) + return Paths::nativeFromUtf8(eventDirectory); + return Paths::nativeFromUtf8(watchedSpelling_ + eventDirectory.substr(eventSpelling_.size())); + } + + private: + std::string watchedSpelling_; + std::string eventSpelling_; + }; + +#ifdef __APPLE__ + /** + * @brief Tells the changes FSEvents reports although they were made before the watch began: a new stream + * also delivers what happened in the moments before it started. + */ + class ReplayFilter + { + public: + explicit ReplayFilter(std::filesystem::path const& directory) + : watchedSince_{std::chrono::system_clock::now()} + , filtersUntil_{ + hasExactChangeTimes(directory) ? std::chrono::steady_clock::now() + replayTime + : std::chrono::steady_clock::time_point{} + } + {} + + bool isReplay(DirectoryEvent const& event) const + { + // Whatever the watch has to report renews the entry's status change time, renames and moves too. + if (event.action == FileAction::Deleted || std::chrono::steady_clock::now() >= filtersUntil_) + return false; + const auto path = Paths::nativeFromUtf8(event.directory) / Paths::nativeFromUtf8(event.filename); + struct stat status{}; + if (::lstat(path.c_str(), &status) != 0) + return false; + const auto changed = std::chrono::system_clock::time_point{ + std::chrono::duration_cast( + std::chrono::seconds{status.st_ctimespec.tv_sec} + + std::chrono::nanoseconds{status.st_ctimespec.tv_nsec} + ) + }; + if (changed >= watchedSince_) + return false; + Log::debug("FileTracking: '{}' changed before its watch began", Paths::nativeToUtf8(path)); + return true; + } + + private: + /** + * @brief How long after a watch began FSEvents may still report what happened before. + */ + static constexpr std::chrono::seconds replayTime{2}; + + /** + * @brief Whether the volume of @p directory keeps status change times to the nanosecond, FAT and network + * volumes round them or take them from another clock. + */ + static bool hasExactChangeTimes(std::filesystem::path const& directory) + { + struct statfs volume{}; + if (::statfs(directory.c_str(), &volume) != 0) + return false; + const std::string_view type{volume.f_fstypename}; + return type == "apfs" || type == "hfs"; + } + + std::chrono::system_clock::time_point watchedSince_; + std::chrono::steady_clock::time_point filtersUntil_; + }; +#else + /** + * @brief Only FSEvents reports changes made before a watch began. + */ + class ReplayFilter + { + public: + explicit ReplayFilter(std::filesystem::path const& /*directory*/) + {} + + bool isReplay(DirectoryEvent const& /*event*/) const + { + return false; + } + }; +#endif + + /** + * @brief Whether @p action modified what is gone by now (@p status) and means nothing: efsw's FSEvents watcher + * reports a modification before every removal. Elsewhere it can be an edit just before a rename, which + * the tracker uploads to the new name. + */ + bool modifiedWhatIsGone( + [[maybe_unused]] FileAction action, + [[maybe_unused]] std::filesystem::file_status const& status + ) + { +#ifdef __APPLE__ + return action == FileAction::Modified && status.type() == std::filesystem::file_type::not_found; +#else + return false; +#endif + } + + /** + * @brief Hands @p event, in @p directory, to the handler on @p strand once it was looked at what the event + * names. + */ + void handOn( + boost::asio::strand const& strand, + std::weak_ptr const& weakHandler, + DirectoryEvent const& event, + std::filesystem::path const& directory + ) + { + if (weakHandler.expired()) + return; + std::error_code statusError; + const auto status = std::filesystem::status(directory / Paths::nativeFromUtf8(event.filename), statusError); + if (modifiedWhatIsGone(event.action, status)) + return; + boost::asio::post( + strand, + [weakHandler, + change = DirectoryChange{ + .action = event.action, + .directory = directory, + .filename = event.filename, + .oldFilename = event.oldFilename, + .isDirectory = std::filesystem::is_directory(status), + }]() + { + if (const auto onChange = weakHandler.lock()) + (*onChange)(change); + } + ); + } + } + + std::filesystem::path DirectoryChange::path() const + { + if (filename.empty()) + return directory.has_filename() ? directory : directory.parent_path(); + return directory / Paths::nativeFromUtf8(filename); + } + + std::filesystem::path DirectoryChange::oldPath() const + { + return oldFilename.empty() ? std::filesystem::path{} : directory / Paths::nativeFromUtf8(oldFilename); + } + + struct DirectoryWatcher::Implementation + { + std::shared_ptr service; + /** + * @brief Looks at what the events name, in their order. Not on efsw's thread, which Windows outpaces when a + * watched directory is deleted, and not on the strand, which serves the RPCs too. + */ + boost::asio::strand worker; + boost::asio::strand strand; + std::shared_ptr onChange; + }; + + DirectoryWatcher::DirectoryWatcher(boost::asio::strand strand, ChangeHandler onChange) + : impl_{std::make_unique(Implementation{ + .service = DirectoryWatchService::shared(), + .worker = boost::asio::make_strand(strand.get_inner_executor()), + .strand = std::move(strand), + .onChange = std::make_shared(std::move(onChange)), + })} + {} + + DirectoryWatcher::~DirectoryWatcher() = default; + DirectoryWatcher::DirectoryWatcher(DirectoryWatcher&&) noexcept = default; + DirectoryWatcher& DirectoryWatcher::operator=(DirectoryWatcher&&) noexcept = default; + + std::optional + DirectoryWatcher::add(std::filesystem::path const& directory, bool recursive, EventFilter filter) + { + return impl_->service->subscribe( + directory, + recursive, + [worker = impl_->worker, + strand = impl_->strand, + weakHandler = std::weak_ptr{impl_->onChange}, + replays = ReplayFilter{directory}, + filter = std::move(filter), + spelling = WatchedSpelling{directory}](DirectoryEvent const& event) + { + // Before the filter, which may keep state about the events it lets through. A missed change names + // no entry to judge by. + if (event.action != FileAction::Missed && (replays.isReplay(event) || (filter && !filter(event)))) + return; + boost::asio::post( + worker, + [strand, weakHandler, event, changedDirectory = spelling.of(event.directory)]() + { + handOn(strand, weakHandler, event, changedDirectory); + } + ); + } + ); + } +} diff --git a/backend/source/backend/file_tracking/disk_place.cpp b/backend/source/backend/file_tracking/disk_place.cpp new file mode 100644 index 00000000..69008785 --- /dev/null +++ b/backend/source/backend/file_tracking/disk_place.cpp @@ -0,0 +1,114 @@ +#include +#include + +#include + +#include +#include +#include + +#ifdef _WIN32 +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +#else +# include +#endif +#ifdef __APPLE__ +# include +#endif + +namespace FileTracking +{ +#ifdef __APPLE__ + namespace + { + /** + * @brief Whether @p directory lies on the volume @p device and that volume looks files up by their identity + * under /.vol. + */ + bool looksUpByIdentity(std::filesystem::path const& directory, std::uint64_t device) + { + struct stat status{}; + struct statfs volume{}; + if (::stat(directory.c_str(), &status) != 0 || static_cast(status.st_dev) != device || + ::statfs(directory.c_str(), &volume) != 0) + return false; + const std::string_view type{volume.f_fstypename}; + return type == "apfs" || type == "hfs"; + } + } +#endif + + std::optional diskIdentityOf(std::filesystem::path const& path) + { +#ifdef _WIN32 + // Backup semantics open directories too. + HANDLE const opened = CreateFileW( + path.c_str(), + 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + nullptr + ); + if (opened == INVALID_HANDLE_VALUE) + return std::nullopt; + const auto handle = std::unique_ptr{opened, &CloseHandle}; + BY_HANDLE_FILE_INFORMATION information{}; + if (!GetFileInformationByHandle(handle.get(), &information)) + return std::nullopt; + return DiskIdentity{ + .device = information.dwVolumeSerialNumber, + .file = (static_cast(information.nFileIndexHigh) << 32u) | information.nFileIndexLow, + }; +#else + struct stat status{}; + if (::stat(path.c_str(), &status) != 0) + return std::nullopt; + return DiskIdentity{ + .device = static_cast(status.st_dev), + .file = static_cast(status.st_ino), + }; +#endif + } + + std::optional isDeletedFromItsVolume( + [[maybe_unused]] DiskIdentity const& identity, + [[maybe_unused]] std::filesystem::path const& directory + ) + { +#ifdef __APPLE__ + if (!looksUpByIdentity(directory, identity.device)) + return std::nullopt; + // fsgetpath() still finds a deleted directory for a while after the last descriptor of it closed. + const auto status = Utility::checkedSymlinkStatus(fmt::format("/.vol/{}/{}", identity.device, identity.file)); + if (!status) + return std::nullopt; + return !status->has_value(); +#else + return std::nullopt; +#endif + } + + DiskPlace DiskPlace::of(std::filesystem::path const& path) + { + DiskPlace place{.self = diskIdentityOf(path), .chain = {}}; + if (place.self) + place.chain.push_back(*place.self); + for (auto ancestor = path; ancestor.has_relative_path();) + { + ancestor = ancestor.parent_path(); + if (const auto identity = diskIdentityOf(ancestor)) + place.chain.push_back(*identity); + } + return place; + } + + bool DiskPlace::isAtOrBelow(DiskPlace const& base) const + { + return base.self && std::ranges::contains(chain, *base.self); + } +} diff --git a/backend/source/backend/file_tracking/event_spelling.cpp b/backend/source/backend/file_tracking/event_spelling.cpp new file mode 100644 index 00000000..7653c926 --- /dev/null +++ b/backend/source/backend/file_tracking/event_spelling.cpp @@ -0,0 +1,74 @@ +#include + +#ifdef __APPLE__ +# include + +# include +# include +# include +#endif + +namespace FileTracking +{ +#ifdef __APPLE__ + namespace + { + struct CoreFoundationRelease + { + void operator()(void const* object) const + { + CFRelease(object); + } + }; + + template + using CoreFoundationHandle = std::unique_ptr, CoreFoundationRelease>; + + /** + * @brief @p text in NFC the way efsw precomposes it (CFStringNormalize), unchanged when it is not UTF-8. + */ + std::string precomposed(std::string const& text) + { + const CoreFoundationHandle source{CFStringCreateWithBytes( + kCFAllocatorDefault, + reinterpret_cast(text.data()), + static_cast(text.size()), + kCFStringEncodingUTF8, + false + )}; + if (!source) + return text; + const CoreFoundationHandle normalized{ + CFStringCreateMutableCopy(kCFAllocatorDefault, 0, source.get()) + }; + if (!normalized) + return text; + CFStringNormalize(normalized.get(), kCFStringNormalizationFormC); + const auto range = CFRangeMake(0, CFStringGetLength(normalized.get())); + CFIndex byteCount{0}; + CFStringGetBytes(normalized.get(), range, kCFStringEncodingUTF8, 0, false, nullptr, 0, &byteCount); + std::string result(static_cast(byteCount), '\0'); + CFStringGetBytes( + normalized.get(), + range, + kCFStringEncodingUTF8, + 0, + false, + reinterpret_cast(result.data()), + byteCount, + nullptr + ); + return result; + } + } +#endif + + std::string eventSpelling(std::string const& name) + { +#ifdef __APPLE__ + return precomposed(name); +#else + return name; +#endif + } +} diff --git a/backend/source/backend/file_tracking/file_states.cpp b/backend/source/backend/file_tracking/file_states.cpp new file mode 100644 index 00000000..9ef4f60e --- /dev/null +++ b/backend/source/backend/file_tracking/file_states.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include + +#include +#include + +namespace FileTracking +{ + std::optional fileStateOf(std::filesystem::directory_entry const& entry) + { + std::error_code entryError{}; + if (entry.is_symlink(entryError)) + return FileState{.path = entry.path(), .isSymlink = true}; + if (entry.is_directory(entryError)) + return FileState{.path = entry.path(), .isDirectory = true}; + if (!entry.is_regular_file(entryError)) + return std::nullopt; + std::error_code sizeError{}; + const auto size = entry.file_size(sizeError); + const auto modified = entry.last_write_time(entryError); + if (sizeError || entryError) + return std::nullopt; + const auto seconds = + std::chrono::duration_cast(std::chrono::file_clock::to_sys(modified).time_since_epoch()); + return FileState{ + .path = entry.path(), + .size = static_cast(size), + .mtime = static_cast(seconds.count()), + }; + } + + std::vector itemStatesOf(std::filesystem::path const& item) + { + std::vector files; + // Canonical like the watched directories, so the paths match the change events. Below a canonical path the + // walk spells everything canonically. + std::error_code canonicalError{}; + const auto canonicalItem = std::filesystem::weakly_canonical(item, canonicalError); + std::error_code entryError{}; + const std::filesystem::directory_entry itemEntry{canonicalItem, entryError}; + if (auto state = fileStateOf(itemEntry)) + files.push_back(*std::move(state)); + std::error_code typeError{}; + if (!itemEntry.is_directory(typeError) || itemEntry.is_symlink(typeError)) + return files; + + std::error_code walkError{}; + for (std::filesystem::recursive_directory_iterator iter{canonicalItem, walkError}, end; + !walkError && iter != end; + iter.increment(walkError)) + { + if (auto state = fileStateOf(*iter)) + files.push_back(*std::move(state)); + } + if (walkError) + Log::warn( + "FileTracking: walking '{}' failed: {}", Paths::nativeToUtf8(item), Utility::errorMessage(walkError) + ); + return files; + } + + std::size_t entryCountOf(std::filesystem::path const& item) + { + std::size_t count = 1; + std::error_code walkError{}; + for ( + std::filesystem::recursive_directory_iterator iter{item, walkError}, end; !walkError && iter != end; + iter.increment(walkError) + ) + ++count; + return count; + } + + std::optional sourceStateOf(std::filesystem::path const& path) + { + std::error_code statusError{}; + if (!std::filesystem::is_regular_file(std::filesystem::symlink_status(path, statusError))) + return std::nullopt; + std::error_code sizeError{}; + const auto size = std::filesystem::file_size(path, sizeError); + std::error_code timeError{}; + const auto modified = std::filesystem::last_write_time(path, timeError); + if (sizeError || timeError) + return std::nullopt; + return SourceState{.size = static_cast(size), .modified = modified}; + } +} diff --git a/backend/source/backend/file_tracking/frontend_tracker_view.cpp b/backend/source/backend/file_tracking/frontend_tracker_view.cpp new file mode 100644 index 00000000..a59a7445 --- /dev/null +++ b/backend/source/backend/file_tracking/frontend_tracker_view.cpp @@ -0,0 +1,50 @@ +#include +#include + +#include +#include + +#include + +namespace FileTracking +{ + FrontendTrackerView::FrontendTrackerView(Nui::RpcHub& hub, Ids::SessionId sessionId) + : hub_{&hub} + , sessionId_{std::move(sessionId)} + {} + + void FrontendTrackerView::showEntry(SharedData::TrackedEntry const& entry) + { + RpcHelper::callRemoteSafely(*hub_, eventName("onEntry"), entry); + } + + void FrontendTrackerView::removeEntry(Ids::WatchId const& watchId) + { + RpcHelper::callRemoteSafely(*hub_, eventName("onEntryRemoved"), watchId.value()); + } + + void FrontendTrackerView::showEnded(SharedData::EndedWatches const& ended) + { + RpcHelper::callRemoteSafely(*hub_, eventName("onEntriesEnded"), ended); + } + + void FrontendTrackerView::ask(SharedData::ServerChangedQuestion const& question) + { + RpcHelper::callRemoteSafely(*hub_, eventName("onQuestion"), question); + } + + void FrontendTrackerView::showOrphanedCount(std::size_t count) + { + RpcHelper::callRemoteSafely(*hub_, eventName("onOrphanedCount"), count); + } + + void FrontendTrackerView::showProgress(SharedData::TrackerProgress const& progress) + { + RpcHelper::callRemoteSafely(*hub_, eventName("onProgress"), progress); + } + + std::string FrontendTrackerView::eventName(std::string_view event) const + { + return fmt::format("FileTracker::{}::{}", sessionId_.value(), event); + } +} diff --git a/backend/source/backend/file_tracking/instance_watch.cpp b/backend/source/backend/file_tracking/instance_watch.cpp index 842069f8..6e01e713 100644 --- a/backend/source/backend/file_tracking/instance_watch.cpp +++ b/backend/source/backend/file_tracking/instance_watch.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -6,20 +7,20 @@ namespace FileTracking { struct InstanceWatch::Implementation { - std::weak_ptr watcher; - efsw::WatchID watchId; + std::weak_ptr service; + std::int64_t watchId; std::filesystem::path path; }; InstanceWatch::InstanceWatch() = default; InstanceWatch::InstanceWatch( - std::weak_ptr watcher, - efsw::WatchID watchId, + std::weak_ptr service, + std::int64_t watchId, std::filesystem::path path ) : impl_{std::make_unique(Implementation{ - .watcher = std::move(watcher), + .service = std::move(service), .watchId = watchId, .path = std::move(path), })} @@ -49,7 +50,7 @@ namespace FileTracking return impl_->path; } - efsw::WatchID InstanceWatch::watchId() const + std::int64_t InstanceWatch::watchId() const { if (!impl_) return -1; @@ -65,8 +66,8 @@ namespace FileTracking { if (!impl_) return; - if (const auto watcher = impl_->watcher.lock(); watcher && impl_->watchId >= 0) - watcher->removeWatch(impl_->watchId); + if (const auto service = impl_->service.lock(); service && impl_->watchId >= 0) + service->unsubscribe(impl_->watchId); impl_.reset(); } } diff --git a/backend/source/backend/file_tracking/local_item_watch.cpp b/backend/source/backend/file_tracking/local_item_watch.cpp new file mode 100644 index 00000000..4607d64c --- /dev/null +++ b/backend/source/backend/file_tracking/local_item_watch.cpp @@ -0,0 +1,581 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + struct LocalItemWatch::State : public std::enable_shared_from_this + { + /** + * @brief A rename of the item, held back until it is clear the item does not come back. + */ + struct PendingRename + { + DirectoryChange change; + /** @brief The changes at the new name meanwhile, reported once the rename is. */ + std::vector changesAtNewName{}; + }; + + /** + * @brief Where the watch followed the item from: the item appearing there again before @p until takes the + * watch back, see renameReturnTime. + */ + struct FollowedRename + { + std::filesystem::path from; + std::chrono::steady_clock::time_point until; + }; + + /** + * @brief The names in the item's directory that concern the watch, read on efsw's thread and spelled like its + * events (see eventSpelling): the item's and the ones it was renamed to. It only grows, a stale name + * lets an event through that onChange drops. + */ + struct WatchedNames + { + std::mutex mutex{}; + std::set names{}; + /** + * @brief Lets every event of the directory through until the strand settled a rename of the item, see + * isRenamedAgain. + */ + bool renaming{false}; + + bool concerns(DirectoryEvent const& event) + { + std::scoped_lock lock{mutex}; + if (names.contains(event.filename)) + return true; + if (event.action != FileAction::Moved || !names.contains(event.oldFilename)) + return renaming; + // The strand follows the rename later, what happens at the new name meanwhile has to reach it. + names.insert(event.filename); + renaming = true; + return true; + } + + void add(std::string const& name) + { + std::scoped_lock lock{mutex}; + names.insert(name); + } + + void renameSettled() + { + std::scoped_lock lock{mutex}; + renaming = false; + } + }; + + Ids::WatchId watchId; + std::filesystem::path item; + DiskPlace place; + bool isDirectory; + std::function mayFollow; + ChangeReporter reporter; + /** @brief The suffix of downloads in progress, see Persistence::effectiveTempFileSuffix. */ + std::string effectiveDownloadTempSuffix; + std::shared_ptr watchedNames{std::make_shared()}; + /** @brief Ends the hold of pendingRename or pendingRemoval, only one of them is held at a time. */ + boost::asio::steady_timer holdTimer; + /** @brief Tells the end of the current hold from the ones a later hold replaced. */ + std::uint64_t holdGeneration{0}; + /** @brief Ends the item's recursive watch a while after the item was deleted, see endItemWatchSoon. */ + boost::asio::steady_timer itemWatchEndTimer; + std::optional pendingRename{}; + /** + * @brief The removal of the item, held back until it is clear the item does not come back: editors delete + * and write anew, Windows reports a case-only rename as a removal first. + */ + std::optional pendingRemoval{}; + std::optional followedRename{}; + std::optional watcher{}; + std::optional directoryWatch{}; + std::optional itemWatch{}; + + State( + Ids::WatchId id, + std::filesystem::path watchedItem, + bool itemIsDirectory, + std::function mayFollowRename, + boost::asio::strand const& strand, + ChangeSink sink, + std::string const& downloadTempSuffix + ) + : watchId{std::move(id)} + , item{std::move(watchedItem)} + , place{DiskPlace::of(item)} + , isDirectory{itemIsDirectory} + , mayFollow{std::move(mayFollowRename)} + , reporter{strand, std::move(sink), downloadTempSuffix} + , effectiveDownloadTempSuffix{Persistence::effectiveTempFileSuffix(downloadTempSuffix)} + , holdTimer{strand} + , itemWatchEndTimer{strand} + { + watchedNames->names.insert(eventSpelling(Paths::nativeToUtf8(item.filename()))); + } + + /** + * @brief Drops noise on efsw's thread, see isTransientChange. + */ + DirectoryWatcher::EventFilter dropsNoise() const + { + return [suffix = effectiveDownloadTempSuffix](DirectoryEvent const& event) + { + return !isTransientChange(event.filename, event.oldFilename, suffix); + }; + } + + /** + * @brief Lets through on efsw's thread only the events of the item's directory that concern the item, the + * directory can be a busy one like the home directory. + */ + DirectoryWatcher::EventFilter concernsItem() const + { + return [dropsNoise = dropsNoise(), names = watchedNames](DirectoryEvent const& event) + { + return dropsNoise(event) && names->concerns(event); + }; + } + + void onChange(DirectoryChange const& reported) + { + if (reported.action == FileAction::Missed) + return onMissed(reported); + const auto change = spelledLikeKnown(reported); + const auto path = change.path(); + const auto oldPath = change.oldPath(); + const bool appeared = change.action == FileAction::Added || change.action == FileAction::Moved; + if (change.action == FileAction::Moved && oldPath == item && path.parent_path() == item.parent_path()) + return holdRename(change); + if (appeared && path == item) + return onItemAppeared(change); + if (appeared && mayReturnTo(path)) + return returnTo(change); + if (pendingRename && change.action == FileAction::Moved && oldPath == pendingRename->change.path() && + path.parent_path() == item.parent_path()) + { + auto renamedAgain = pendingRename->change; + renamedAgain.filename = change.filename; + return holdRename(renamedAgain); + } + // Windows reports the old name of a rename as removed too. + if (pendingRename && change.action == FileAction::Deleted && path == item) + return; + if (pendingRename && change.action == FileAction::Added && isRenamedAgain(path)) + { + auto renamedAgain = pendingRename->change; + renamedAgain.filename = change.filename; + watchedNames->add(change.filename); + return holdRename(renamedAgain); + } + if (pendingRename && isDirectory && path != item && isAtOrBelow(item, path)) + return pendingRename->changesAtNewName.push_back(movedAlong(change, pendingRename->change.path())); + if (pendingRename && isAtOrBelow(pendingRename->change.path(), path)) + return pendingRename->changesAtNewName.push_back(change); + if (change.action == FileAction::Deleted && path == item) + return holdRemoval(change); + if (isChangeOfItem(item, path, oldPath)) + reporter.report(change); + } + + /** + * @brief A watch lost the changes in the directory of @p change, the item is looked at anew. Gone, it was + * removed, and with its tree when its own watch lost them: the deletions of the last entries are what + * got lost. Otherwise the sink compares what is there with what it knows, below the directory or + * of the whole item. While a rename settles that waits for the new name. + */ + void onMissed(DirectoryChange const& change) + { + const bool belowItem = isAtOrBelow(item, change.path()); + if (pendingRename) + { + const auto renamed = pendingRename->change.path(); + return pendingRename->changesAtNewName.push_back( + belowItem ? movedAlong(change, renamed) : missedAt(renamed) + ); + } + const auto status = Utility::checkedSymlinkStatus(item); + if (!status) + { + Log::warn( + "FileTracking: watch {} lost changes and cannot look at '{}': {}", + watchId.value(), + Paths::nativeToUtf8(item), + Utility::errorMessage(status.error()) + ); + return; + } + if (!*status) + { + if (belowItem) + reporter.report(change); + if (!pendingRemoval) + holdRemoval(DirectoryChange{ + .action = FileAction::Deleted, + .directory = item.parent_path(), + .filename = Paths::nativeToUtf8(item.filename()), + .oldFilename = {}, + .isDirectory = isDirectory, + }); + return; + } + // Back or written anew meanwhile. + if (pendingRemoval || DiskPlace::of(item).self != place.self) + onItemAppeared(DirectoryChange{ + .action = FileAction::Added, + .directory = item.parent_path(), + .filename = Paths::nativeToUtf8(item.filename()), + .oldFilename = {}, + .isDirectory = std::filesystem::is_directory(**status), + }); + reporter.report(belowItem ? change : missedAt(item)); + } + + DirectoryChange missedAt(std::filesystem::path const& path) const + { + return DirectoryChange{ + .action = FileAction::Missed, + .directory = path, + .filename = {}, + .oldFilename = {}, + .isDirectory = isDirectory, + }; + } + + /** + * @brief @p change with the names of the item and of where the watch followed it from spelled like the watch + * knows them, where the event spells them differently (see eventSpelling). + */ + DirectoryChange spelledLikeKnown(DirectoryChange change) const + { + respell(change, item); + if (followedRename) + respell(change, followedRename->from); + return change; + } + + /** + * @brief Spells the name of @p known in @p change like @p known, where the change is in its directory and + * spells the name like the events do. + */ + static void respell(DirectoryChange& change, std::filesystem::path const& known) + { + const auto knownName = Paths::nativeToUtf8(known.filename()); + const auto eventName = eventSpelling(knownName); + if (eventName == knownName || change.path().parent_path() != known.parent_path()) + return; + if (change.filename == eventName) + change.filename = knownName; + if (change.oldFilename == eventName) + change.oldFilename = knownName; + } + + /** + * @brief @p change inside the renamed item as it happened at @p renamed: the old recursive watch follows the + * moved directory but keeps reporting its old path. + */ + DirectoryChange movedAlong(DirectoryChange const& change, std::filesystem::path const& renamed) const + { + auto moved = change; + moved.directory = + (renamed / change.directory.lexically_normal().lexically_relative(item)).lexically_normal(); + // A move between two directories of the item names where it came from in full. + if (const auto oldFilename = Paths::nativeFromUtf8(change.oldFilename); + oldFilename.is_absolute() && isAtOrBelow(item, oldFilename)) + moved.oldFilename = Paths::nativeToUtf8( + (renamed / oldFilename.lexically_normal().lexically_relative(item)).lexically_normal() + ); + std::error_code statusError; + moved.isDirectory = std::filesystem::is_directory(moved.path(), statusError); + return moved; + } + + /** + * @brief Whether @p path, added beside the item while its rename settles, is the item renamed once more: + * FSEvents merges the records of a name renamed to and away again, so efsw reports the last name as + * added. + */ + bool isRenamedAgain(std::filesystem::path const& path) const + { + if (path.parent_path() != item.parent_path() || !place.self) + return false; + return diskIdentityOf(path) == place.self && diskIdentityOf(pendingRename->change.path()) != place.self; + } + + /** + * @brief Ends the item's recursive watch once its last changes had the settle time to arrive: moved out of + * its directory reads as deleted, and the watch would go on reporting the moved directory's content + * under the old path. On macOS every watch has a stream of its own, the removal of the item can arrive + * from its directory's stream before the removals of its content from its own. + */ + void endItemWatchSoon() + { + itemWatchEndTimer.expires_after(changeSettleTime); + itemWatchEndTimer.async_wait( + [weak = weak_from_this()](boost::system::error_code const& error) + { + if (error == boost::asio::error::operation_aborted) + return; + if (const auto self = weak.lock()) + self->itemWatch.reset(); + } + ); + } + + void holdRename(DirectoryChange const& change) + { + pendingRemoval.reset(); + followedRename.reset(); + pendingRename = PendingRename{.change = change}; + hold(); + } + + void holdRemoval(DirectoryChange const& change) + { + endItemWatchSoon(); + pendingRemoval = change; + hold(); + } + + /** + * @brief Acts on the pending rename or removal once nothing replaced it for the settle time. + */ + void hold() + { + const auto generation = ++holdGeneration; + holdTimer.expires_after(changeSettleTime); + holdTimer.async_wait( + [weak = weak_from_this(), generation](boost::system::error_code const& error) + { + if (error == boost::asio::error::operation_aborted) + return; + const auto self = weak.lock(); + // A wait that completed before a later hold restarted the timer is not the last one. + if (!self || self->holdGeneration != generation) + return; + if (self->pendingRename) + return self->followRename(); + self->reportRemoval(); + } + ); + } + + void endHold() + { + ++holdGeneration; + holdTimer.cancel(); + pendingRename.reset(); + pendingRemoval.reset(); + } + + /** + * @brief The item is there again, created, saved over or moved back: an editor moved the original aside or + * removed it to write a new one, rather than the user renaming or deleting it. + */ + void onItemAppeared(DirectoryChange const& change) + { + if (pendingRename) + watchedNames->renameSettled(); + endHold(); + isDirectory = change.isDirectory; + place = DiskPlace::of(item); + watchItemDirectory(); + reporter.report(change); + } + + /** + * @brief Whether the item appearing at @p path takes the watch back to the name it last followed a rename + * from, see renameReturnTime. + */ + bool mayReturnTo(std::filesystem::path const& path) const + { + return followedRename && path == followedRename->from && + std::chrono::steady_clock::now() < followedRename->until && mayFollow(watchId, path); + } + + /** + * @brief The item appeared again at the name the watch followed it from: an editor moved the original aside + * and wrote the new file after the settle time. The watch goes back, a rename back and the new content + * are reported. A held removal of the name it followed to is moot then. + */ + void returnTo(DirectoryChange const& change) + { + Log::info( + "FileTracking: watch {} returns from '{}' to '{}', the item was written there anew", + watchId.value(), + Paths::nativeToUtf8(item), + Paths::nativeToUtf8(change.path()) + ); + const auto followed = std::exchange(item, change.path()); + followedRename.reset(); + reporter.report(DirectoryChange{ + .action = FileAction::Moved, + .directory = item.parent_path(), + .filename = Paths::nativeToUtf8(item.filename()), + .oldFilename = Paths::nativeToUtf8(followed.filename()), + .isDirectory = isDirectory, + }); + onItemAppeared(change); + } + + void reportRemoval() + { + if (!pendingRemoval) + return; + const auto removal = *std::exchange(pendingRemoval, std::nullopt); + reporter.report(removal); + } + + void followRename() + { + if (!pendingRename) + return; + auto rename = *std::exchange(pendingRename, std::nullopt); + watchedNames->renameSettled(); + // Onto another watch, which reports what arrives there, or onto a name whose changes are noise: for this + // one the item is gone. + if (reporter.isNoise(rename.change.filename) || !mayFollow(watchId, rename.change.path())) + { + Log::info( + "FileTracking: watch {} does not follow '{}' to '{}', it overlaps another watch or is temporary", + watchId.value(), + Paths::nativeToUtf8(item), + Paths::nativeToUtf8(rename.change.path()) + ); + // The recursive watch moved along with the directory and would report its content under the old path. + itemWatch.reset(); + return reporter.report(DirectoryChange{ + .action = FileAction::Deleted, + .directory = item.parent_path(), + .filename = Paths::nativeToUtf8(item.filename()), + .oldFilename = {}, + .isDirectory = isDirectory, + }); + } + Log::info( + "FileTracking: watch {} follows '{}' to '{}'", + watchId.value(), + Paths::nativeToUtf8(item), + Paths::nativeToUtf8(rename.change.path()) + ); + followedRename = FollowedRename{ + .from = std::exchange(item, rename.change.path()), + .until = std::chrono::steady_clock::now() + renameReturnTime, + }; + place = DiskPlace::of(item); + watchItemDirectory(); + reporter.report(rename.change); + for (auto const& change : rename.changesAtNewName) + onChange(change); + } + + /** + * @brief Watches the item recursively when it is a directory, anew: the watch of a directory that was + * replaced or renamed reports nothing or the old paths. + */ + void watchItemDirectory() + { + itemWatchEndTimer.cancel(); + itemWatch.reset(); + if (!isDirectory) + return; + itemWatch = watcher->add(item, /*recursive=*/true, dropsNoise()); + if (!itemWatch) + Log::warn("FileTracking: watch {} cannot watch '{}' again", watchId.value(), Paths::nativeToUtf8(item)); + } + }; + + std::expected LocalItemWatch::open( + Ids::WatchId watchId, + std::filesystem::path const& item, + boost::asio::strand strand, + ChangeSink sink, + std::string const& downloadTempSuffix, + std::function mayFollow + ) + { + std::error_code typeError; + const bool isDirectory = std::filesystem::is_directory(item, typeError); + auto state = std::make_shared( + std::move(watchId), item, isDirectory, std::move(mayFollow), strand, std::move(sink), downloadTempSuffix + ); + state->watcher.emplace( + std::move(strand), + [weak = std::weak_ptr{state}](DirectoryChange const& change) + { + if (const auto self = weak.lock()) + self->onChange(change); + } + ); + + state->directoryWatch = state->watcher->add(item.parent_path(), /*recursive=*/false, state->concernsItem()); + if (isDirectory) + state->itemWatch = state->watcher->add(item, /*recursive=*/true, state->dropsNoise()); + if (!state->directoryWatch || (isDirectory && !state->itemWatch)) + return std::unexpected{ + Utility::localizedMessage("backend.fileTracking.watchFailed", Paths::nativeToUtf8(item)) + }; + return LocalItemWatch{std::move(state)}; + } + + LocalItemWatch::LocalItemWatch(std::shared_ptr state) + : state_{std::move(state)} + {} + + LocalItemWatch::~LocalItemWatch() = default; + LocalItemWatch::LocalItemWatch(LocalItemWatch&&) noexcept = default; + LocalItemWatch& LocalItemWatch::operator=(LocalItemWatch&&) noexcept = default; + + Ids::WatchId const& LocalItemWatch::watchId() const + { + return state_->watchId; + } + + std::filesystem::path const& LocalItemWatch::item() const + { + return state_->item; + } + + DiskPlace const& LocalItemWatch::place() const + { + return state_->place; + } + + std::filesystem::path LocalItemWatch::directory() const + { + return state_->item.parent_path(); + } + + bool LocalItemWatch::isDirectory() const + { + return state_->isDirectory; + } + + void LocalItemWatch::handleChange(DirectoryChange const& change) + { + state_->onChange(change); + } +} diff --git a/backend/source/backend/file_tracking/local_watch_manager.cpp b/backend/source/backend/file_tracking/local_watch_manager.cpp new file mode 100644 index 00000000..f75c4b79 --- /dev/null +++ b/backend/source/backend/file_tracking/local_watch_manager.cpp @@ -0,0 +1,191 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace FileTracking +{ + namespace + { + /** + * @brief @p path in lower case where the file systems fold case (Windows, macOS), unchanged elsewhere. + */ + std::filesystem::path foldedSpelling(std::filesystem::path const& path) + { +#if defined(_WIN32) || defined(__APPLE__) + return Paths::nativeFromUtf8(Utility::Algorithm::toLowerCase(Paths::nativeToUtf8(path))); +#else + return path; +#endif + } + } + + namespace LocalWatchManagerDetail + { + PlacedPath::PlacedPath(std::filesystem::path path) + : PlacedPath{path, DiskPlace::of(path)} + {} + + PlacedPath::PlacedPath(std::filesystem::path path, DiskPlace place) + : path{std::move(path)} + , place{std::move(place)} + {} + + bool overlaps(PlacedPath const& first, PlacedPath const& second) + { + const auto firstSpelling = foldedSpelling(first.path); + const auto secondSpelling = foldedSpelling(second.path); + if (isAtOrBelow(firstSpelling, secondSpelling) || isAtOrBelow(secondSpelling, firstSpelling)) + return true; + // A file system that folds case where the platform does not (vfat, case-folding ext4) or two mounts of + // one directory show only on disk. + return first.place.isAtOrBelow(second.place) || second.place.isAtOrBelow(first.place); + } + + std::expected refuseOverlap( + PlacedPath const& item, + std::vector const& watchedItems, + PlacedPath const& temporaryDownloadsDirectory + ) + { + if (overlaps(item, temporaryDownloadsDirectory)) + return std::unexpected{Utility::localizedMessage( + "backend.fileTracking.watchOverlapsDownloads", + Paths::nativeToUtf8(item.path), + Paths::nativeToUtf8(temporaryDownloadsDirectory.path) + )}; + const auto overlapping = std::ranges::find_if( + watchedItems, + [&item](PlacedPath const& watched) + { + return overlaps(item, watched); + } + ); + if (overlapping == watchedItems.end()) + return {}; + return std::unexpected{Utility::localizedMessage( + "backend.fileTracking.watchOverlaps", + Paths::nativeToUtf8(item.path), + Paths::nativeToUtf8(overlapping->path) + )}; + } + } + + LocalWatchManager::LocalWatchManager( + boost::asio::strand strand, + std::filesystem::path temporaryDownloadsDirectory + ) + : strand_{std::move(strand)} + , temporaryDownloadsDirectory_{std::move(temporaryDownloadsDirectory)} + , watches_{} + {} + + std::expected + LocalWatchManager::checkWatchable(std::filesystem::path const& item) const + { + std::error_code canonicalError; + auto canonical = std::filesystem::canonical(item, canonicalError); + if (canonicalError) + return std::unexpected{Utility::localizedMessage( + "backend.fileTracking.watchUnreadable", Paths::nativeToUtf8(item), Utility::errorMessage(canonicalError) + )}; + if (!canonical.has_relative_path()) + return std::unexpected{Utility::localizedMessage("backend.fileTracking.watchRoot", Paths::nativeToUtf8(item))}; + return refuseOverlapWithWatches(canonical) + .transform( + [&canonical]() + { + return std::move(canonical); + } + ); + } + + std::expected LocalWatchManager::refuseOverlapWithWatches( + std::filesystem::path const& item, + std::optional const& ignoredWatchId + ) const + { + const auto watchedItems = watches_ | + std::views::filter( + [&ignoredWatchId](auto const& watched) + { + return watched.first != ignoredWatchId; + } + ) | + std::views::values | + std::views::transform( + [](LocalItemWatch const& watched) + { + // Taken when the watch started or followed its item, nothing else is looked at on disk. + return LocalWatchManagerDetail::PlacedPath{watched.item(), watched.place()}; + } + ) | + std::ranges::to(); + std::error_code temporaryError; + return LocalWatchManagerDetail::refuseOverlap( + item, watchedItems, std::filesystem::weakly_canonical(temporaryDownloadsDirectory_, temporaryError) + ); + } + + std::expected LocalWatchManager::watch( + Ids::WatchId const& watchId, + std::filesystem::path const& item, + std::string const& downloadTempSuffix, + ChangeSink sink + ) + { + return checkWatchable(item) + .and_then( + [this, &watchId, &downloadTempSuffix, &sink](std::filesystem::path const& canonical) + { + // A rename beside itself overlaps the watch's own item only in case, where that is folded. + return LocalItemWatch::open( + watchId, + canonical, + strand_, + std::move(sink), + downloadTempSuffix, + [this](Ids::WatchId const& watchId, std::filesystem::path const& renamed) + { + return refuseOverlapWithWatches(renamed, watchId).has_value(); + } + ); + } + ) + .transform( + [this](LocalItemWatch watched) -> LocalItemWatch const* + { + Log::info( + "FileTracking: watching local '{}' as {}", + Paths::nativeToUtf8(watched.item()), + watched.watchId().value() + ); + auto id = watched.watchId(); + const auto [entry, inserted] = watches_.emplace(std::move(id), std::move(watched)); + return &entry->second; + } + ); + } + + void LocalWatchManager::unwatch(Ids::WatchId const& watchId) + { + if (watches_.erase(watchId) > 0) + Log::info("FileTracking: stopped watching local {}", watchId.value()); + } + + LocalItemWatch const* LocalWatchManager::find(Ids::WatchId const& watchId) const + { + const auto found = watches_.find(watchId); + return found == watches_.end() ? nullptr : &found->second; + } +} diff --git a/backend/source/backend/file_tracking/settled_changes.cpp b/backend/source/backend/file_tracking/settled_changes.cpp new file mode 100644 index 00000000..ad19e841 --- /dev/null +++ b/backend/source/backend/file_tracking/settled_changes.cpp @@ -0,0 +1,133 @@ +#include +#include + +#include + +#include +#include +#include +#include + +namespace FileTracking +{ + namespace + { + std::optional appearanceOf(std::filesystem::path const& path) + { + std::error_code error{}; + const std::filesystem::directory_entry entry{path, error}; + const auto size = entry.file_size(error); + if (error) + return std::nullopt; + const auto modified = entry.last_write_time(error); + if (error) + return std::nullopt; + return SettledChanges::Appearance{.size = size, .modified = modified}; + } + } + + SettledChanges::SettledChanges( + boost::asio::strand strand, + ChangeSink send, + std::chrono::milliseconds settleTime + ) + : strand_{std::move(strand)} + , send_{std::move(send)} + , settleTime_{settleTime} + , pending_{} + , nextSequence_{0} + {} + + void SettledChanges::add(ReportedChange change, bool isFileContent) + { + flush(change.path, /*includingBase=*/!isFileContent); + flush(change.oldPath, /*includingBase=*/true); + if (isFileContent) + return hold(std::move(change)); + send_(change); + } + + void SettledChanges::hold(ReportedChange change) + { + // Normal like the bases flush() looks below. + const auto key = change.path.lexically_normal(); + auto found = pending_.find(key); + if (found == pending_.end()) + found = pending_ + .emplace( + key, + std::make_unique(Pending{ + .change = change, + .sequence = 0, + .generation = 0, + .timer = boost::asio::steady_timer{strand_}, + }) + ) + .first; + else if (found->second->change.action == FileAction::Added) + change.action = FileAction::Added; + auto& pending = *found->second; + pending.change = std::move(change); + pending.sequence = nextSequence_++; + pending.appearance = appearanceOf(key); + settle(key, pending); + } + + void SettledChanges::settle(std::filesystem::path const& key, Pending& pending) + { + const auto generation = ++pending.generation; + pending.timer.expires_after(settleTime_); + pending.timer.async_wait( + [weak = weak_from_this(), key, generation](boost::system::error_code const& error) + { + if (error == boost::asio::error::operation_aborted) + return; + const auto self = weak.lock(); + if (!self) + return; + const auto settled = self->pending_.find(key); + // A wait that completed before a later change restarted the timer is not the last one. + if (settled == self->pending_.end() || settled->second->generation != generation) + return; + // Written on without an event. + if (auto appearance = appearanceOf(key); appearance != settled->second->appearance) + { + settled->second->appearance = std::move(appearance); + return self->settle(key, *settled->second); + } + const auto settledChange = std::move(settled->second->change); + self->pending_.erase(settled); + self->send_(settledChange); + } + ); + } + + void SettledChanges::flush(std::filesystem::path const& base, bool includingBase) + { + if (base.empty()) + return; + // Paths order by their elements, so what lies at and below the base follows it in one run. + const auto normalBase = base.lexically_normal(); + std::vector> flushed; + for (auto pending = pending_.lower_bound(normalBase); + pending != pending_.end() && isAtOrBelow(normalBase, pending->first);) + { + if (!includingBase && pending->first == normalBase) + { + ++pending; + continue; + } + flushed.push_back(std::move(pending->second)); + pending = pending_.erase(pending); + } + std::ranges::sort( + flushed, + [](auto const& first, auto const& second) + { + return first->sequence < second->sequence; + } + ); + for (auto const& pending : flushed) + send_(pending->change); + } +} diff --git a/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp b/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp index ad4a80d4..055b006c 100644 --- a/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp +++ b/backend/source/backend/file_tracking/temp_dir_instance_manager.cpp @@ -1,9 +1,10 @@ #include #include +#include +#include #include #include #include -#include #include #include @@ -174,43 +175,33 @@ namespace FileTracking namespace InstanceManagerDetail { - nlohmann::json fileStatesOf(std::filesystem::path const& instanceDirectory) + std::vector + fileStatesOf(std::filesystem::path const& instanceDirectory, std::filesystem::path const& below) { - nlohmann::json files = nlohmann::json::array(); + std::vector files; + // Below a canonical path the walk spells everything canonically, like the change events. + std::error_code canonicalError{}; + const auto root = std::filesystem::weakly_canonical(instanceDirectory, canonicalError); + if (!below.empty()) + { + const auto canonicalBelow = std::filesystem::weakly_canonical(below, canonicalError); + if (!isAtOrBelow(root, canonicalBelow) || isBookkeepingPath(root, canonicalBelow)) + return files; + if (canonicalBelow.lexically_normal() != root.lexically_normal()) + return itemStatesOf(canonicalBelow); + } std::error_code walkError{}; - for (std::filesystem::recursive_directory_iterator iter{instanceDirectory, walkError}, end; + for (std::filesystem::recursive_directory_iterator iter{root, walkError}, end; !walkError && iter != end; iter.increment(walkError)) { - std::error_code entryError{}; - if (iter->is_symlink(entryError)) - continue; if (iter.depth() == 0 && iter->path().filename() == std::filesystem::path{bookkeepingDirectoryName}) { iter.disable_recursion_pending(); continue; } - // Canonical like the watched directory, so the paths match the change events. - const auto path = Paths::nativeToUtf8(std::filesystem::weakly_canonical(iter->path(), entryError)); - if (iter->is_directory(entryError)) - { - files.push_back({{"path", path}, {"directory", true}}); - continue; - } - if (!iter->is_regular_file(entryError)) - continue; - const auto size = iter->file_size(entryError); - const auto modified = iter->last_write_time(entryError); - if (entryError) - continue; - const auto seconds = std::chrono::duration_cast( - std::chrono::file_clock::to_sys(modified).time_since_epoch() - ); - files.push_back({ - {"path", path}, - {"size", size}, - {"mtime", seconds.count()}, - }); + if (auto state = fileStateOf(*iter)) + files.push_back(*std::move(state)); } if (walkError) Log::warn( @@ -270,181 +261,20 @@ namespace FileTracking // TempDirInstanceManager // ------------------------------------------------------------------------- TempDirInstanceManager::TempDirInstanceManager( - boost::asio::any_io_executor executor, - Nui::Window& wnd, - Nui::RpcHub& hub, + boost::asio::strand strand, std::filesystem::path tempRootDir, std::chrono::hours deadInstanceRetentionHours ) - : StrandRpc{std::move(executor), wnd, hub} + : strand_{std::move(strand)} , tempRootDir_{std::move(tempRootDir)} , deadInstanceRetentionHours_{deadInstanceRetentionHours} - , wnd_{&wnd} , instances_{} {} - void TempDirInstanceManager::registerRpc() - { - registerOnStrand( - "FileTracking::createInstance", - [this](RpcHelper::RpcOnce&& reply) - { - auto* inst = createInstance(); - if (!inst) - { - reply.error("Failed to create instance"); - return; - } - // Canonical like the paths of the change events and of fileStates, which the frontend maps. - std::error_code canonicalError; - const auto canonical = std::filesystem::weakly_canonical(inst->instanceDir(), canonicalError); - reply({ - {"success", true}, - {"instanceId", inst->instanceId()}, - {"instanceDir", Paths::nativeToUtf8(canonicalError ? inst->instanceDir() : canonical)}, - }); - } - ); - - registerOnStrand( - "FileTracking::destroyInstance", - [this](RpcHelper::RpcOnce&& reply, std::string instanceId) - { - if (!findInstance(instanceId)) - { - reply.error(fmt::format("Instance not found: {}", instanceId)); - return; - } - destroyInstance(instanceId); - reply({{"success", true}}); - } - ); - - registerOnStrand( - "FileTracking::addWatch", - [this]( - RpcHelper::RpcOnce&& reply, - std::string instanceId, - std::string path, - bool recursive, - std::string downloadTempSuffix - ) - { - auto* inst = findInstance(instanceId); - if (!inst) - { - reply.error(fmt::format("Instance not found: {}", instanceId)); - return; - } - const auto watchedPath = Paths::LocalPath::fromUtf8(path); - if (!watchedPath) - { - reply.error(fmt::format("Cannot watch '{}': {}", path, watchedPath.error().toString())); - return; - } - auto watch = inst->addWatch(watchedPath->native(), recursive, std::move(downloadTempSuffix)); - if (!watch) - { - reply.error(fmt::format("Failed to add watch on path: {}", path)); - return; - } - watches_[instanceId].push_back(std::move(*watch)); - reply({{"success", true}}); - } - ); - - registerOnStrand( - "FileTracking::listInstances", - [this](RpcHelper::RpcOnce&& reply) - { - auto arr = nlohmann::json::array(); - for (auto const& [instId, inst] : instances_) - { - arr.push_back({ - {"instanceId", instId}, - {"instanceDir", Paths::nativeToUtf8(inst->instanceDir())}, - }); - } - reply({{"success", true}, {"instances", std::move(arr)}}); - } - ); - - registerOnStrand( - "FileTracking::getInstanceInfo", - [this](RpcHelper::RpcOnce&& reply, std::string instanceId) - { - auto* inst = findInstance(instanceId); - if (!inst) - { - reply.error(fmt::format("Instance not found: {}", instanceId)); - return; - } - reply({ - {"success", true}, - {"instanceId", instanceId}, - {"instanceDir", Paths::nativeToUtf8(inst->instanceDir())}, - }); - } - ); - - registerOnStrand( - "FileTracking::fileStates", - [this](RpcHelper::RpcOnce&& reply, std::string instanceId) - { - auto* inst = findInstance(instanceId); - if (!inst) - { - reply.error(fmt::format("Instance not found: {}", instanceId)); - return; - } - reply({{"success", true}, {"files", InstanceManagerDetail::fileStatesOf(inst->instanceDir())}}); - } - ); - - registerOnStrand( - "FileTracking::discardContent", - [this](RpcHelper::RpcOnce&& reply, std::string instanceId, std::string path) - { - const auto content = Paths::LocalPath::fromUtf8(path); - if (!content) - { - reply.error(fmt::format("Cannot discard '{}': {}", path, content.error().toString())); - return; - } - const auto outcome = discardContent(instanceId, content->native()); - if (!outcome) - { - Log::error("FileTracking::discardContent: {}", outcome.error()); - reply.error(outcome.error()); - return; - } - reply({{"success", true}, {"instanceDestroyed", *outcome == DiscardOutcome::InstanceDestroyed}}); - } - ); - - registerOnStrand( - "FileTracking::countOrphaned", - [this](RpcHelper::RpcOnce&& reply) - { - const auto count = orphanedInstanceDirs(tempRootDir_, listInstanceIds()).size(); - reply({{"success", true}, {"count", count}}); - } - ); - - registerOnStrand( - "FileTracking::forceCleanup", - [this](RpcHelper::RpcOnce&& reply) - { - const int removed = forceCleanOrphanedInstanceDirs(tempRootDir_, listInstanceIds()); - reply({{"success", true}, {"removed", removed}}); - } - ); - } - void TempDirInstanceManager::scheduleStartupCleanup() { boost::asio::post( - *strand_, + strand_, [this]() { manualCleanup(); @@ -457,10 +287,7 @@ namespace FileTracking try { auto inst = std::make_unique( - TemporaryDirectoryInstance::Config{.tempRootDir = tempRootDir_}, - *strand_, - *wnd_, - *hub_ + TemporaryDirectoryInstance::Config{.tempRootDir = tempRootDir_}, strand_ ); auto* ptr = inst.get(); instances_.emplace(inst->instanceId(), std::move(inst)); @@ -475,7 +302,6 @@ namespace FileTracking void TempDirInstanceManager::destroyInstance(std::string const& instanceId) { - watches_.erase(instanceId); instances_.erase(instanceId); } @@ -489,7 +315,6 @@ namespace FileTracking if (!content) return std::unexpected{content.error()}; - watches_.erase(instanceId); Log::info("Discarding '{}' of instance '{}'", Paths::nativeToUtf8(*content), instanceId); std::error_code removeError; Utility::removeAllNoFollow(*content, removeError); @@ -505,6 +330,26 @@ namespace FileTracking return DiscardOutcome::InstanceDestroyed; } + void TempDirInstanceManager::removeInstance(std::string const& instanceId) + { + auto* instance = findInstance(instanceId); + if (!instance) + return; + Log::info("Removing instance '{}' with its content", instanceId); + instance->cleanupNow(); + destroyInstance(instanceId); + } + + std::size_t TempDirInstanceManager::countOrphaned() const + { + return orphanedInstanceDirs(tempRootDir_, listInstanceIds()).size(); + } + + int TempDirInstanceManager::removeOrphaned() + { + return forceCleanOrphanedInstanceDirs(tempRootDir_, listInstanceIds()); + } + TemporaryDirectoryInstance* TempDirInstanceManager::findInstance(std::string const& instanceId) const { diff --git a/backend/source/backend/file_tracking/temp_dir_instancing.cpp b/backend/source/backend/file_tracking/temp_dir_instancing.cpp index 57f80ed3..fa07a96b 100644 --- a/backend/source/backend/file_tracking/temp_dir_instancing.cpp +++ b/backend/source/backend/file_tracking/temp_dir_instancing.cpp @@ -1,17 +1,15 @@ #include #include +#include +#include +#include #include -#include #include #include -#include +#include -#include -#include #include -#include #include -#include #include #include @@ -36,94 +34,8 @@ namespace FileTracking throw std::runtime_error{fmt::format("Failed to create instance directory: {}", created.error())}; return dir; } - - FileAction toFileAction(efsw::Action action) - { - switch (action) - { - case efsw::Actions::Add: - return FileAction::Added; - case efsw::Actions::Modified: - return FileAction::Modified; - case efsw::Actions::Delete: - return FileAction::Deleted; - case efsw::Actions::Moved: - return FileAction::Moved; - default: - return FileAction::Modified; - } - } } // namespace - /** - * @brief Receives the efsw events of one instance on efsw's thread and forwards them to the frontend. - */ - class FileChangeListener : public efsw::FileWatchListener - { - public: - /** - * @param watchedInstanceDirectory The instance directory spelled like the watch reports it (canonical). - */ - FileChangeListener( - std::string const& instanceId, - std::filesystem::path watchedInstanceDirectory, - boost::asio::strand strand, - Nui::RpcHub* hub, - std::string downloadTempSuffix - ) - : watchedInstanceDirectory_{std::move(watchedInstanceDirectory)} - , strand_{std::move(strand)} - , hub_{hub} - , eventName_{fmt::format("FileTracking::{}::onFileChanged", instanceId)} - , downloadTempSuffix_{Persistence::effectiveTempFileSuffix(downloadTempSuffix)} - {} - - void handleFileAction( - efsw::WatchID /*watchId*/, - const std::string& dir, - const std::string& filename, - efsw::Action action, - std::string oldFilename - ) override - { - // efsw reports UTF-8 on every platform. - const auto changedPath = Paths::nativeFromUtf8(dir) / Paths::nativeFromUtf8(filename); - if (isBookkeepingPath(watchedInstanceDirectory_, changedPath)) - return; - - std::error_code statusError; - const bool isDirectory = std::filesystem::is_directory(changedPath, statusError); - const auto change = filterFileChange( - {toFileAction(action), filename, std::move(oldFilename)}, isDirectory, downloadTempSuffix_ - ); - if (!change) - return; - - nlohmann::json payload = { - {"action", Utility::enumToString(change->action)}, - {"directory", Paths::nativeToUtf8(Paths::nativeFromUtf8(dir))}, - {"filename", Paths::nativeToUtf8(Paths::nativeFromUtf8(change->filename))}, - {"oldFilename", Paths::nativeToUtf8(Paths::nativeFromUtf8(change->oldFilename))}, - {"isDirectory", isDirectory}, - }; - - boost::asio::post( - strand_, - [hub = hub_, name = eventName_, pld = std::move(payload)]() mutable - { - RpcHelper::callRemoteSafely(*hub, name, pld); - } - ); - } - - private: - std::filesystem::path watchedInstanceDirectory_; - boost::asio::strand strand_; - Nui::RpcHub* hub_; - std::string eventName_; - std::string downloadTempSuffix_; - }; - // ------------------------------------------------------------------------- // Implementation // ------------------------------------------------------------------------- @@ -134,9 +46,7 @@ namespace FileTracking std::filesystem::path instanceDir; std::optional lock; boost::asio::strand strand; - Nui::RpcHub* hub; - std::shared_ptr watcher; - std::unique_ptr listener; + std::optional watcher; bool valid{false}; /** @@ -145,20 +55,13 @@ namespace FileTracking * The directory must exist before the lock file inside it can be created, * which is why prepareInstanceDir() is called in the instanceDir initialiser. */ - Implementation( - Config cfg, - boost::asio::strand strd, - Nui::RpcHub* hubPtr, - std::string instId - ) + Implementation(Config cfg, boost::asio::strand strd, std::string instId) : config{std::move(cfg)} , instanceId{std::move(instId)} , instanceDir{prepareInstanceDir(config.tempRootDir / instanceId)} , lock{std::in_place, bookkeepingPaths(instanceDir).lock} , strand{std::move(strd)} - , hub{hubPtr} - , watcher{std::make_shared()} - , listener{nullptr} + , watcher{} , valid{false} {} }; @@ -168,11 +71,9 @@ namespace FileTracking // ------------------------------------------------------------------------- TemporaryDirectoryInstance::TemporaryDirectoryInstance( Config config, - boost::asio::strand strand, - Nui::Window& /*wnd*/, - Nui::RpcHub& hub + boost::asio::strand strand ) - : impl_{std::make_unique(std::move(config), std::move(strand), &hub, Ids::generateId().value())} + : impl_{std::make_unique(std::move(config), std::move(strand), Ids::generateId().value())} { writeMetadata(); impl_->valid = true; @@ -183,9 +84,8 @@ namespace FileTracking if (!impl_) return; - // Stop the efsw thread before anything else so no more callbacks fire. + // Stop the change handler before anything else so no more changes are reported. impl_->watcher.reset(); - impl_->listener.reset(); if (impl_->valid) writeDeadTimestamp(); @@ -204,7 +104,8 @@ namespace FileTracking std::optional TemporaryDirectoryInstance::addWatch( std::filesystem::path const& path, bool recursive, - std::string downloadTempSuffix + std::string downloadTempSuffix, + ChangeSink sink ) { namespace fs = std::filesystem; @@ -229,30 +130,26 @@ namespace FileTracking return std::nullopt; } - if (impl_->listener) - { - Log::error("Watch already exists for instance '{}'", impl_->instanceId); - return std::nullopt; - } - - impl_->listener = std::make_unique( - impl_->instanceId, - fs::weakly_canonical(impl_->instanceDir), - impl_->strand, - impl_->hub, - std::move(downloadTempSuffix) + // The first watch decides the suffix and the sink, later ones share its event stream. + if (!impl_->watcher) + impl_->watcher.emplace( + impl_->strand, + [reporter = ChangeReporter{impl_->strand, std::move(sink), downloadTempSuffix}, + watchedInstanceDirectory = fs::weakly_canonical(impl_->instanceDir)](DirectoryChange const& change) + { + if (!isBookkeepingPath(watchedInstanceDirectory, change.path())) + reporter.report(change); + } + ); + // A download writes its temporary many times over, those events go no further than efsw's thread. + return impl_->watcher->add( + absPath, + recursive, + [suffix = Persistence::effectiveTempFileSuffix(downloadTempSuffix)](DirectoryEvent const& event) + { + return !isTransientChange(event.filename, event.oldFilename, suffix); + } ); - - // efsw takes UTF-8, a narrow string is the ANSI code page on Windows. - auto watchId = impl_->watcher->addWatch(Paths::nativeToUtf8(absPath), impl_->listener.get(), recursive); - if (watchId < 0) - { - Log::warn("addWatch: efsw rejected watch for '{}'", Paths::nativeToUtf8(absPath)); - return std::nullopt; - } - - impl_->watcher->watch(); - return InstanceWatch{impl_->watcher, watchId, std::move(absPath)}; } void TemporaryDirectoryInstance::cleanupNow() @@ -261,7 +158,6 @@ namespace FileTracking return; impl_->watcher.reset(); - impl_->listener.reset(); impl_->lock.reset(); std::error_code err; diff --git a/backend/source/backend/file_tracking/tracker.cpp b/backend/source/backend/file_tracking/tracker.cpp new file mode 100644 index 00000000..0d220796 --- /dev/null +++ b/backend/source/backend/file_tracking/tracker.cpp @@ -0,0 +1,2291 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + namespace + { + /** + * @brief A watch of a LocalWatchManager that ends with this handle. + */ + class LocalWatchLease + { + public: + LocalWatchLease(LocalWatchManager& manager, Ids::WatchId watchId) + : manager_{&manager} + , watchId_{std::move(watchId)} + {} + ~LocalWatchLease() + { + end(); + } + LocalWatchLease(LocalWatchLease const&) = delete; + LocalWatchLease& operator=(LocalWatchLease const&) = delete; + LocalWatchLease(LocalWatchLease&& other) noexcept + : manager_{std::exchange(other.manager_, nullptr)} + , watchId_{std::move(other.watchId_)} + {} + LocalWatchLease& operator=(LocalWatchLease&& other) noexcept + { + if (this != &other) + { + end(); + manager_ = std::exchange(other.manager_, nullptr); + watchId_ = std::move(other.watchId_); + } + return *this; + } + + private: + void end() + { + if (manager_) + manager_->unwatch(watchId_); + } + + private: + LocalWatchManager* manager_; + Ids::WatchId watchId_; + }; + + /** + * @brief The copy of a download in an instance directory of its own. + */ + struct DownloadedCopy + { + Ids::InstanceId instanceId; + /** @brief Canonical, like the paths of the changes. */ + std::filesystem::path instanceDirectory; + InstanceWatch watch; + /** @brief Runs while the copy was reported deleted or renamed away, see awaitCopyReturn(). */ + std::unique_ptr returnWait{}; + }; + + /** + * @brief An original the user uploaded. + */ + struct UploadedOriginal + { + LocalWatchLease watch; + }; + + /** + * @brief One watched local file or directory. + */ + struct Entry + { + Ids::WatchId watchId; + std::variant source; + Paths::RemotePath remotePath; + /** @brief Stands for remotePath, whatever either is called. */ + Paths::LocalPath localPath; + SharedData::WatchOptions options; + PendingUploads pendingUploads{}; + KnownServerContent serverContent{}; + /** @brief Renames and deletes on the server that are queued and have not ended. */ + std::size_t remoteChangesOnTheirWay{0}; + /** @brief What the view shows, so it hears only of changes. */ + bool shownUploading{false}; + + DownloadedCopy const* downloadedCopy() const + { + return std::get_if(&source); + } + + DownloadedCopy* downloadedCopy() + { + return std::get_if(&source); + } + }; + + /** + * @brief A changed path of an entry on its way to the server. + */ + struct Upload + { + Ids::WatchId watchId; + LocalChange change; + Paths::RemotePath destination; + /** @brief Compare with the server first, when the entry asks for that. */ + bool checkServer{true}; + /** @brief Refresh the remote listing afterwards, for the upload that starts an "Upload and Watch". */ + bool refreshListing{false}; + /** @brief Its place in the server order once it started, see ServerOperationOrder. */ + std::optional ticket{}; + /** + * @brief The destination of a stopped upload of the same file under its old name, whose partial file this + * one continues, see UploadRequest::continuesPartialOf. + */ + std::optional continuesPartialOf{}; + }; + + /** + * @brief How the upload of a change ended. + */ + enum class UploadOutcome + { + Uploaded, + NotUploaded, + /** @brief The user kept the server's version. */ + Declined, + }; + + /** + * @brief Whether a watched directory found gone when its watch lost changes went entry by entry. On Windows + * only a deletion ends the read of a watch, one moved away is followed. Elsewhere it may have left as a + * whole, to the trash, and keeps its server copy like any directory that left with its content. The + * removal of the item itself tells more, see wasDeletedWithItsContent. + */ +#ifdef _WIN32 + constexpr bool goneWatchedDirectoryWasDeleted = true; +#else + constexpr bool goneWatchedDirectoryWasDeleted = false; +#endif + + /** + * @brief What is at @p path itself: its status, nothing when it is gone, an error when that cannot be told. + * Only a path that is gone counts as deleted, see Utility::checkedSymlinkStatus. + */ + std::expected, std::error_code> + localStatusOf(Paths::LocalPath const& path) + { + return Utility::checkedSymlinkStatus(path.native()); + } + + bool isGoneLocally(Paths::LocalPath const& path) + { + const auto status = localStatusOf(path); + return status && !status->has_value(); + } + + bool isSymlink(Paths::LocalPath const& path) + { + const auto status = localStatusOf(path); + return status && *status && std::filesystem::is_symlink(**status); + } + + /** + * @brief Whether @p path is a directory itself, not a symbolic link to one, which goes up as a link. + */ + bool isDirectoryLocally(Paths::LocalPath const& path) + { + const auto status = localStatusOf(path); + return status && *status && std::filesystem::is_directory(**status); + } + + /** + * @brief What the server has of a file an upload sent in @p sent: uploads carry the modification time along, + * in seconds. + */ + ServerState serverStateOf(SourceState const& sent) + { + const auto seconds = std::chrono::duration_cast( + std::chrono::file_clock::to_sys(sent.modified).time_since_epoch() + ); + return ServerState{.mtime = static_cast(seconds.count()), .size = sent.size}; + } + + /** + * @brief Whether @p option may be on for a downloaded copy: removing or renaming a temporary copy never touches + * the server's file. + */ + bool appliesToDownloadedCopies(SharedData::WatchOption option) + { + return option != SharedData::WatchOption::DeleteRemoteOnLocalDelete && + option != SharedData::WatchOption::MoveRemoteOnLocalMove; + } + + SharedData::WatchOptions downloadedCopyOptions(SharedData::WatchOptions options) + { + options.deleteRemoteOnLocalDelete = false; + options.moveRemoteOnLocalMove = false; + return options; + } + + std::string shownNameOf(Paths::LocalPath const& path) + { + return path.filename() ? std::string{path.filename()->view()} : path.toUtf8(); + } + + SharedData::TrackedEntry shownEntryOf(Entry const& entry) + { + return SharedData::TrackedEntry{ + .watchId = entry.watchId, + .name = shownNameOf(entry.localPath), + .remotePath = entry.remotePath, + .options = entry.options, + .uploading = entry.shownUploading, + .downloadedCopy = entry.downloadedCopy() != nullptr, + }; + } + + std::expected + remotePathOf(Entry const& entry, Paths::LocalPath const& local) + { + return remotePathOfChange(local, entry.localPath, entry.remotePath); + } + + /** + * @brief The remote path of @p remotePath's item when it is renamed like the local one to @p newPath. + */ + std::expected + renamedRemotePath(Paths::RemotePath const& remotePath, Paths::LocalPath const& newPath) + { + const auto remoteDirectory = remotePath.parent(); + const auto name = newPath.filename(); + if (!remoteDirectory || !name) + return std::unexpected{ + Paths::PathError{.reason = Paths::PathRejection::EscapesRoot, .input = newPath.toUtf8()}.toString() + }; + return *remoteDirectory / *name; + } + + /** + * @brief @p path spelled like the path @p entry knows the server copy under, where the file system ignores + * Unicode normalization (macOS): efsw reports every name precomposed (NFC), the server copy has the + * spelling its directory listed. Unchanged elsewhere and when nothing known matches. + */ + Paths::LocalPath knownSpelling([[maybe_unused]] Entry const& entry, Paths::LocalPath const& path) + { +#ifdef __APPLE__ + return entry.serverContent.knownSpellingOf(path).value_or(path); +#else + return path; +#endif + } + + /** + * @brief @p path as a local path, nothing when it is empty or unusable. + */ + std::optional localPathOf(std::filesystem::path const& path) + { + if (path.empty()) + return std::nullopt; + return Paths::LocalPath::make(path) + .transform( + [](Paths::LocalPath made) + { + return std::optional{std::move(made)}; + } + ) + .value_or(std::nullopt); + } + + /** + * @brief Where the download of @p remotePath goes in @p instanceDirectory. + */ + std::expected + copyPathOf(std::filesystem::path const& instanceDirectory, Paths::RemotePath const& remotePath) + { + const auto name = remotePath.filename(); + if (!name) + return std::unexpected{ + Paths::PathError{.reason = Paths::PathRejection::Empty, .input = remotePath.value()}.toString() + }; + return Paths::LocalPathSegment::make(*name) + .and_then( + [&instanceDirectory](Paths::LocalPathSegment const& segment) + { + return Paths::LocalPath::make(instanceDirectory) + .transform( + [&segment](Paths::LocalPath const& directory) + { + return directory / segment; + } + ); + } + ) + .transform_error( + [](Paths::PathError const& error) + { + return error.toString(); + } + ); + } + } + + struct Tracker::Implementation : public std::enable_shared_from_this + { + /** @brief How often the view hears of the progress at most, a burst of changes costs a few events. */ + constexpr static std::chrono::milliseconds progressInterval{200}; + + /** + * @brief Holds the remote path of a watch that starts, see claimRemote(), until it is released or goes. + */ + class Claim + { + public: + Claim(std::weak_ptr tracker, std::uint64_t claimId) + : tracker_{std::move(tracker)} + , claimId_{claimId} + {} + ~Claim() + { + release(); + } + Claim(Claim const&) = delete; + Claim& operator=(Claim const&) = delete; + Claim(Claim&&) = delete; + Claim& operator=(Claim&&) = delete; + + void release() + { + if (const auto tracker = std::exchange(tracker_, {}).lock()) + tracker->claims.erase(claimId_); + } + + private: + std::weak_ptr tracker_; + std::uint64_t claimId_; + }; + + /** + * @brief A question the view asks, see TrackerView::ask(). + */ + struct OpenQuestion + { + Ids::QuestionId questionId; + Upload upload; + }; + + using OnOpened = std::function const&)>; + + boost::asio::strand strand; + TempDirInstanceManager* instances; + LocalWatchManager* localWatches; + std::unique_ptr remote; + std::unique_ptr view; + std::string downloadTempSuffix; + std::function defaultOptions; + + std::map entries{}; + /** @brief The instances of downloads on their way, see open(). */ + std::set downloading{}; + /** @brief The instances of copies opened without a watch, they end with the tracker, see stopAll(). */ + std::set openedCopies{}; + /** @brief The remote paths of watches that are starting, by claim. */ + std::map claims{}; + std::uint64_t nextClaimId{0}; + /** @brief Questions waiting for the one the view asks, so no question replaces another. */ + std::deque waitingQuestions{}; + std::optional openQuestion{}; + /** @brief Changes that ended since nothing was on its way any more, see showProgress(). */ + std::uint64_t changesDone{0}; + SharedData::TrackerProgress shownProgress{}; + boost::asio::steady_timer progressTimer; + bool progressScheduled{false}; + /** @brief The view hears of the progress once the current handler ended, see progressChanged(). */ + bool idleProgressPosted{false}; + ServerOperationOrder serverOrder{}; + /** @brief Start the operations that wait in serverOrder, by ticket. */ + std::map> waitingOperations{}; + + Implementation( + boost::asio::strand strand, + TempDirInstanceManager& instances, + LocalWatchManager& localWatches, + std::unique_ptr remote, + std::unique_ptr view, + std::string downloadTempSuffix, + std::function defaultOptions + ) + : strand{std::move(strand)} + , instances{&instances} + , localWatches{&localWatches} + , remote{std::move(remote)} + , view{std::move(view)} + , downloadTempSuffix{std::move(downloadTempSuffix)} + , defaultOptions{std::move(defaultOptions)} + , progressTimer{this->strand} + {} + + Entry* find(Ids::WatchId const& watchId) + { + const auto found = entries.find(watchId); + return found == entries.end() ? nullptr : &found->second; + } + + /** + * @brief Hands the changes of the watch @p watchId to the strand, the watch may end while one is handled. + */ + ChangeSink sinkOf(Ids::WatchId const& watchId) + { + return [strand = strand, weak = weak_from_this(), watchId](ReportedChange const& change) + { + boost::asio::post( + strand, + [weak, watchId, change]() + { + if (const auto self = weak.lock()) + self->onChange(watchId, change); + } + ); + }; + } + + /** + * @brief Runs @p start once the server operations decided before it on @p paths ended, see + * ServerOperationOrder. It gets its ticket, leaveOrder() ends it. + */ + void inOrder(std::vector paths, std::function start) + { + const auto [ticket, startsNow] = serverOrder.enter(std::move(paths)); + if (startsNow) + return start(ticket); + waitingOperations.emplace(ticket, std::move(start)); + } + + void leaveOrder(ServerOperationOrder::Ticket ticket) + { + for (const auto starting : serverOrder.leave(ticket)) + { + auto node = waitingOperations.extract(starting); + // Dropped by stopAll(), it never starts. + if (node.empty()) + leaveOrder(starting); + else + node.mapped()(starting); + } + } + + void leaveOrder(Upload const& upload) + { + if (upload.ticket) + leaveOrder(*upload.ticket); + } + + /** + * @brief Ends @p ticket once the queued operation ended. + */ + TrackerRemote::OnEnded leavesOrderWhenEnded(ServerOperationOrder::Ticket ticket) + { + return [weak = weak_from_this(), ticket](SharedData::OperationCompletionReason) + { + if (const auto self = weak.lock()) + self->leaveOrder(ticket); + }; + } + + /** + * @brief Hands the end of the queued operation to @p ended, then ends @p ticket. + */ + TrackerRemote::OnEnded endsAndLeavesOrder(TrackerRemote::OnEnded ended, ServerOperationOrder::Ticket ticket) + { + return [ended = std::move(ended), leaves = leavesOrderWhenEnded(ticket)]( + SharedData::OperationCompletionReason reason + ) + { + ended(reason); + leaves(reason); + }; + } + + void show(Entry const& entry) + { + view->showEntry(shownEntryOf(entry)); + } + + SharedData::TrackerProgress currentProgress() const + { + std::uint64_t pending = 0; + for (auto const& entry : std::views::values(entries)) + pending += entry.pendingUploads.changeCount() + entry.remoteChangesOnTheirWay; + return { + .pending = pending, + .done = changesDone, + .waitingForAnswer = waitingQuestions.size() + (openQuestion ? 1u : 0u), + }; + } + + /** + * @brief Whether no change is on its way any more, nor waits for an answer. + */ + bool nothingOnItsWay() const + { + return !openQuestion && waitingQuestions.empty() && + std::ranges::all_of( + std::views::values(entries), + [](Entry const& entry) + { + return entry.pendingUploads.empty() && entry.remoteChangesOnTheirWay == 0; + } + ); + } + + /** + * @brief The view hears of the progress once the progress interval passed, or right after the change that + * left nothing on its way, together with the rows that stopped uploading. + */ + void progressChanged() + { + if (nothingOnItsWay() && !idleProgressPosted) + { + idleProgressPosted = true; + // Posted, so the changes the end of an upload starts are on their way by then. + boost::asio::post( + strand, + [weak = weak_from_this()]() + { + const auto self = weak.lock(); + if (!self) + return; + self->idleProgressPosted = false; + if (!self->nothingOnItsWay()) + return; + self->progressTimer.cancel(); + self->showProgress(); + } + ); + } + if (std::exchange(progressScheduled, true)) + return; + progressTimer.expires_after(progressInterval); + progressTimer.async_wait( + [weak = weak_from_this()](boost::system::error_code const& error) + { + if (error) + return; + if (const auto self = weak.lock()) + self->showProgress(); + } + ); + } + + void showProgress() + { + progressScheduled = false; + const auto progress = currentProgress(); + if (progress != shownProgress) + view->showProgress(progress); + shownProgress = progress; + // The next burst counts from zero. + if (progress.pending == 0) + changesDone = 0; + } + + /** + * @param count The changes it counted as, see PendingUploads::changeCount(). + */ + void changeEnded(std::size_t count = 1) + { + changesDone += count; + progressChanged(); + } + + /** + * @brief Hears when a rename or delete on the server for @p entry ended, it is on its way until then. + */ + TrackerRemote::OnEnded remoteChangeOf(Entry& entry) + { + ++entry.remoteChangesOnTheirWay; + progressChanged(); + return [weak = weak_from_this(), watchId = entry.watchId](SharedData::OperationCompletionReason) + { + const auto self = weak.lock(); + if (!self) + return; + if (auto* ended = self->find(watchId); ended && ended->remoteChangesOnTheirWay > 0) + --ended->remoteChangesOnTheirWay; + self->changeEnded(); + }; + } + + void showUploading(Entry& entry) + { + const bool uploading = !entry.pendingUploads.empty(); + if (uploading == entry.shownUploading) + return; + entry.shownUploading = uploading; + show(entry); + } + + /** + * @brief Refuses @p remotePath when it overlaps the remote path of a watch other than @p ignoredWatchId or of + * a claim: it is the same, lies below it or holds it. + */ + std::expected refuseRemoteOverlap( + Paths::RemotePath const& remotePath, + std::optional const& ignoredWatchId = std::nullopt + ) const + { + auto taken = claims | std::views::values | std::ranges::to(); + for (auto const& entry : std::views::values(entries)) + if (entry.watchId != ignoredWatchId) + taken.push_back(entry.remotePath); + const auto overlapping = std::ranges::find_if( + taken, + [&remotePath](Paths::RemotePath const& other) + { + return remotePath.isWithin(other) || other.isWithin(remotePath); + } + ); + if (overlapping == taken.end()) + return {}; + return std::unexpected{Utility::localizedMessage( + "backend.fileTracking.remoteOverlaps", remotePath.value(), overlapping->value() + )}; + } + + std::expected, std::string> claimRemote(Paths::RemotePath const& remotePath) + { + return refuseRemoteOverlap(remotePath) + .transform( + [this, &remotePath]() + { + const auto claimId = nextClaimId++; + claims.emplace(claimId, remotePath); + return std::make_shared(weak_from_this(), claimId); + } + ); + } + + /** + * @brief Shows @p entry and mirrors its changes from now on. + */ + Entry& track(Entry entry) + { + const auto watchId = entry.watchId; + auto& tracked = entries.insert_or_assign(watchId, std::move(entry)).first->second; + show(tracked); + return tracked; + } + + void open(Paths::RemotePath const& remotePath, std::uint64_t sizeBytes, bool watch, OnOpened onOpened) + { + auto claim = std::shared_ptr{}; + if (watch) + { + auto claimed = claimRemote(remotePath); + if (!claimed) + return onOpened(std::unexpected{OpenFailure{claimed.error()}}); + claim = *std::move(claimed); + } + auto* instance = instances->createInstance(); + if (!instance) + return onOpened(std::unexpected{ + OpenFailure{Utility::localizedMessage("backend.fileTracking.instanceCreationFailed")} + }); + const auto instanceId = Ids::makeInstanceId(instance->instanceId()); + downloading.insert(instanceId); + // Canonical like the paths of the changes, which the copy's path is compared with. + std::error_code canonicalError; + const auto canonicalDirectory = std::filesystem::weakly_canonical(instance->instanceDir(), canonicalError); + const auto instanceDirectory = canonicalError ? instance->instanceDir() : canonicalDirectory; + const auto localPath = copyPathOf(instanceDirectory, remotePath); + const auto queued = localPath.and_then( + [&, this](Paths::LocalPath const& destination) + { + return remote->download( + remotePath, + destination, + sizeBytes, + [weak = weak_from_this(), + copy = OpenedCopy{.instanceId = instanceId, .localPath = destination}, + instanceDirectory, + remotePath, + watch, + claim, + onOpened](SharedData::OperationCompletionReason reason) + { + // The watch is tracked or refused by the time anything else runs on the strand. + if (claim) + claim->release(); + if (const auto self = weak.lock()) + self->onDownloaded(copy, instanceDirectory, remotePath, watch, reason, onOpened); + } + ); + } + ); + if (queued) + return; + // The caller shows why, an error would show it a second time. + Log::warn("FileTracking: not downloading '{}': {}", remotePath.value(), queued.error()); + downloading.erase(instanceId); + instances->removeInstance(instanceId.value()); + onOpened(std::unexpected{OpenFailure{queued.error()}}); + } + + void onDownloaded( + OpenedCopy const& copy, + std::filesystem::path const& instanceDirectory, + Paths::RemotePath const& remotePath, + bool watch, + SharedData::OperationCompletionReason reason, + OnOpened const& onOpened + ) + { + // Dropped by stopAll() meanwhile, with its instance. + if (downloading.erase(copy.instanceId) == 0) + return onOpened(std::unexpected{OpenFailure{}}); + if (reason != SharedData::OperationCompletionReason::Completed) + { + Log::warn("FileTracking: the download of '{}' did not complete", remotePath.value()); + instances->removeInstance(copy.instanceId.value()); + return onOpened(std::unexpected{OpenFailure{}}); + } + if (!watch) + { + openedCopies.insert(copy.instanceId); + return onOpened(copy); + } + auto* instance = instances->findInstance(copy.instanceId.value()); + const auto watchId = Ids::generateWatchId(); + auto instanceWatch = instance + ? instance->addWatch(instanceDirectory, /*recursive=*/true, downloadTempSuffix, sinkOf(watchId)) + : std::nullopt; + if (!instanceWatch) + { + Log::error("FileTracking: cannot watch the download '{}'", copy.localPath.toUtf8()); + instances->removeInstance(copy.instanceId.value()); + return onOpened(std::unexpected{OpenFailure{ + Utility::localizedMessage("backend.fileTracking.watchCopyFailed", copy.localPath.toUtf8()) + }}); + } + Log::info( + "FileTracking: watching {} of instance {} as {}", + copy.localPath.toUtf8(), + copy.instanceId.value(), + watchId.value() + ); + auto& entry = track(Entry{ + .watchId = watchId, + .source = + DownloadedCopy{ + .instanceId = copy.instanceId, + .instanceDirectory = instanceDirectory, + .watch = *std::move(instanceWatch), + }, + .remotePath = remotePath, + .localPath = copy.localPath, + .options = downloadedCopyOptions(defaultOptions()), + }); + // The downloaded files carry the server's modification time, so they tell what the server had. + rememberStates(entry, std::nullopt); + onOpened(copy); + } + + std::expected + checkUploadAndWatch(Paths::LocalPath const& item, Paths::RemotePath const& directory) const + { + return localWatches->checkWatchable(item.native()) + .and_then( + [](std::filesystem::path const& canonical) + { + return Paths::LocalPath::make(canonical) + .transform_error( + [](Paths::PathError const& error) + { + return error.toString(); + } + ); + } + ) + .and_then( + [this, &directory](Paths::LocalPath const& canonical) -> std::expected + { + const auto name = canonical.filename(); + if (!name) + return std::unexpected{ + Paths::PathError{.reason = Paths::PathRejection::Empty, .input = canonical.toUtf8()} + .toString() + }; + const auto target = UploadTarget{.item = canonical, .destination = directory / *name}; + return refuseRemoteOverlap(target.destination) + .transform( + [&target]() + { + return target; + } + ); + } + ); + } + + std::expected + uploadAndWatch(Paths::LocalPath const& item, Paths::RemotePath const& destination) + { + Log::info("FileTracking: upload and watch {} to {}", item.toUtf8(), destination.value()); + const auto watchId = Ids::generateWatchId(); + return refuseRemoteOverlap(destination) + .and_then( + [&, this]() + { + return localWatches->watch(watchId, item.native(), downloadTempSuffix, sinkOf(watchId)); + } + ) + .and_then( + [&, this](LocalItemWatch const* watched) -> std::expected + { + auto lease = LocalWatchLease{*localWatches, watchId}; + const auto watchedItem = Paths::LocalPath::make(watched->item()); + if (!watchedItem) + return std::unexpected{watchedItem.error().toString()}; + const bool isDirectory = watched->isDirectory(); + Log::info("FileTracking: watching {} as {}", watchedItem->toUtf8(), watchId.value()); + track(Entry{ + .watchId = watchId, + .source = UploadedOriginal{.watch = std::move(lease)}, + .remotePath = destination, + .localPath = *watchedItem, + .options = defaultOptions(), + }); + // Goes like a change, so what changes while it uploads goes once more afterwards. + queueUpload(Upload{ + .watchId = watchId, + .change = {.path = *watchedItem, .isDirectory = isDirectory}, + .destination = destination, + .checkServer = false, + .refreshListing = true, + }); + return {}; + } + ); + } + + void stop(Ids::WatchId const& watchId) + { + const auto instanceId = forget(watchId); + if (!instanceId) + return; + releaseInstance(*instanceId); + view->showOrphanedCount(instances->countOrphaned()); + } + + /** + * @brief Ends the instance of a copy that is no longer watched or opened: it counts as orphaned then, or goes + * with its directory when nothing is left in it. + */ + void releaseInstance(Ids::InstanceId const& instanceId) + { + const auto* instance = instances->findInstance(instanceId.value()); + if (!instance) + return; + if (InstanceManagerDetail::hasContent(instance->instanceDir())) + instances->destroyInstance(instanceId.value()); + else + instances->removeInstance(instanceId.value()); + } + + /** + * @brief Ends the watch @p watchId and takes its entry off the view. + * @return The instance of a downloaded copy, nothing for an uploaded original or no such watch. + */ + std::optional forget(Ids::WatchId const& watchId) + { + auto node = entries.extract(watchId); + if (node.empty()) + return std::nullopt; + Log::info("FileTracking: stopping watch {}", watchId.value()); + std::vector dropped; + std::erase_if( + waitingQuestions, + [&watchId, &dropped](Upload const& upload) + { + if (upload.watchId != watchId) + return false; + dropped.push_back(upload); + return true; + } + ); + view->removeEntry(watchId); + progressChanged(); + for (auto const& upload : dropped) + leaveOrder(upload); + const auto* copy = node.mapped().downloadedCopy(); + return copy ? std::optional{copy->instanceId} : std::nullopt; + } + + void setOption(Ids::WatchId const& watchId, SharedData::WatchOption option, bool value) + { + auto* entry = find(watchId); + if (!entry) + return; + if (entry->downloadedCopy() && !appliesToDownloadedCopies(option)) + Log::warn( + "FileTracking: watch {} is a downloaded copy, {} stays off", + watchId.value(), + Utility::enumToString(option) + ); + else + entry->options[option] = value; + // Also when nothing changed, so a switch the user flipped shows the option again. + show(*entry); + } + + std::expected + discard(Ids::InstanceId const& instanceId, Paths::LocalPath const& path) + { + Log::info("FileTracking: discarding {} of instance {}", path.toUtf8(), instanceId.value()); + const auto watched = std::ranges::find_if( + entries, + [&instanceId](auto const& entry) + { + const auto* copy = entry.second.downloadedCopy(); + return copy && copy->instanceId == instanceId; + } + ); + if (watched != entries.end()) + forget(watched->first); + const auto discarded = instances->discardContent(instanceId.value(), path.native()); + if (discarded == DiscardOutcome::InstanceDestroyed) + openedCopies.erase(instanceId); + return discarded; + } + + void stopAll() + { + auto copies = std::exchange(openedCopies, {}); + copies.merge(std::exchange(downloading, {})); + for (auto const& entry : std::views::values(entries)) + if (const auto* copy = entry.downloadedCopy()) + copies.insert(copy->instanceId); + entries.clear(); + for (auto const& instanceId : copies) + releaseInstance(instanceId); + claims.clear(); + auto dropped = std::exchange(waitingQuestions, {}); + if (openQuestion) + dropped.push_back(std::move(openQuestion->upload)); + openQuestion.reset(); + const auto waiting = std::exchange(waitingOperations, {}); + for (auto const& upload : dropped) + leaveOrder(upload); + for (auto const& ticket : std::views::keys(waiting)) + leaveOrder(ticket); + progressTimer.cancel(); + progressScheduled = false; + changesDone = 0; + // A frontend that comes next starts with nothing on its way, the idle progress has nothing to tell it. + shownProgress = {}; + } + + void connectionLost() + { + auto ended = SharedData::EndedWatches{.reason = SharedData::WatchEndReason::ConnectionLost, .entries = {}}; + for (auto const& entry : std::views::values(entries)) + ended.entries.push_back(shownEntryOf(entry)); + Log::info("FileTracking: the connection was lost, ending {} watch(es)", ended.entries.size()); + // stopAll() forgets the progress shown, this view still shows it. + const bool progressShown = shownProgress != SharedData::TrackerProgress{}; + stopAll(); + for (auto const& shown : ended.entries) + view->removeEntry(shown.watchId); + if (!ended.entries.empty()) + view->showEnded(ended); + if (progressShown) + view->showProgress(shownProgress); + view->showOrphanedCount(instances->countOrphaned()); + } + + /** + * @brief Records what the server has after a download or an upload: the local states below @p below, or of + * the whole item without it. Transfers carry the modification time along, a symbolic link is asked + * for on the server, which may have skipped it. + * @return How many entries it recorded. + */ + std::size_t rememberStates(Entry& entry, std::optional const& below) + { + std::size_t recorded = 0; + const auto states = [&]() + { + if (const auto* copy = entry.downloadedCopy()) + return InstanceManagerDetail::fileStatesOf( + copy->instanceDirectory, below ? below->native() : std::filesystem::path{} + ); + return itemStatesOf(below.value_or(entry.localPath).native()); + }(); + for (auto const& state : states) + { + const auto path = Paths::LocalPath::make(state.path); + if (!path || (below && !path->isWithin(*below))) + continue; + if (state.isDirectory) + entry.serverContent.rememberDirectory(*path); + else if (state.isSymlink) + rememberLink(entry, *path); + else + entry.serverContent.rememberFile(*path, ServerState{.mtime = state.mtime, .size = state.size}); + ++recorded; + } + return recorded; + } + + void rememberLink(Entry& entry, Paths::LocalPath const& path) + { + const auto remotePath = remotePathOf(entry, path); + if (!remotePath) + return; + if (const auto state = remote->linkStat(*remotePath); state && *state) + entry.serverContent.rememberFile(path, **state); + } + + void onChange(Ids::WatchId const& watchId, ReportedChange const& reported) + { + auto* entry = find(watchId); + if (!entry) + return; + const auto reportedPath = localPathOf(reported.path); + if (!reportedPath) + { + Log::warn( + "FileTracking: ignoring a change in watch {}: '{}'", + watchId.value(), + Paths::nativeToUtf8(reported.path) + ); + return; + } + const auto oldPath = localPathOf(reported.oldPath).transform( + [entry](Paths::LocalPath const& old) + { + return knownSpelling(*entry, old); + } + ); + const auto respelled = knownSpelling(*entry, *reportedPath); + // A rename that changes the normalization only keeps its new spelling, the server copy follows it. + const auto path = reported.action == FileAction::Moved && oldPath == respelled ? *reportedPath : respelled; + Log::debug( + "FileTracking: change in watch {}: {} {}", + watchId.value(), + Utility::enumToString(reported.action), + path.toUtf8() + ); + if (reported.action == FileAction::Missed) + return onMissed(*entry, path); + if (reported.action == FileAction::Moved && oldPath == entry->localPath) + { + // The local watch follows an original renamed beside itself once it is clear the rename stays. + if (!entry->downloadedCopy()) + return onItemMoved(*entry, path); + // A copy renamed away is not what goes to the server any more. + if (path != entry->localPath) + return awaitCopyReturn(*entry); + } + // Only the tracked file or directory maps to the remote side, anything else in its directory (Finder's + // .DS_Store for one) would land next to it on the server. + if (!path.isWithin(entry->localPath)) + return; + // The watch reports a link to a directory as a directory. + const LocalChange change{.path = path, .isDirectory = reported.isDirectory && !isSymlink(path)}; + if (reported.action == FileAction::Deleted) + return onDeleted(*entry, change.path); + // A move from outside the item into it (an atomic save renaming a temporary over it) is a change. + if (reported.action == FileAction::Moved && oldPath && oldPath->isWithin(entry->localPath)) + return onMoved(*entry, *oldPath, change); + onContentChanged(*entry, change); + } + + /** + * @param checkServer Compare with the server first, when the entry asks for that. Changes below a directory + * that was just uploaded skip it, the states recorded after that upload may already be newer. + */ + void onContentChanged(Entry& entry, LocalChange change, bool checkServer = true) + { + if (!entry.options.autoReupload) + return; + // A change that waited below a directory upload was reported before that upload told what the server has. + change.path = knownSpelling(entry, change.path); + const auto destination = remotePathOf(entry, change.path); + if (!destination) + { + Log::warn( + "FileTracking: not uploading the change in watch {}: {}", + entry.watchId.value(), + destination.error().toString() + ); + return; + } + queueUpload(Upload{ + .watchId = entry.watchId, + .change = change, + .destination = *destination, + .checkServer = checkServer, + }); + } + + void onDeleted(Entry& entry, Paths::LocalPath const& path) + { + if (entry.downloadedCopy() && path == entry.localPath) + return awaitCopyReturn(entry); + if (!entry.options.deleteRemoteOnLocalDelete) + return; + // Deleted and created anew before the deletion is handled, the way git rewrites files: the upload of the + // new one replaces the server copy, which a delete before it only makes look changed. + if (isThereAgain(entry, path)) + { + Log::debug("FileTracking: '{}' is there again, not deleting it on the server.", path.toUtf8()); + return; + } + if (entry.pendingUploads.removeRemoteWhenDone(path)) + { + // A deletion reported after the path came back (Windows holds deletions back) stops nothing. + if (isGoneLocally(path)) + stopUpload(entry, path, PartialFile::Discarded); + return; + } + if (!entry.serverContent.has(path)) + { + // The upload of a directory above may have sent it before it went, it is looked at once that ended. + if (entry.pendingUploads.covers(path)) + { + entry.pendingUploads.start({.path = path}); + return progressChanged(); + } + Log::debug("FileTracking: '{}' is not on the server, nothing to delete there.", path.toUtf8()); + return; + } + // Never recursive: a directory deleted entry by entry reports its content first, so only one moved away + // as a whole (to the trash) still has known content, and its server copy keeps it. Until its content + // turns up elsewhere in the item or is deleted after all, see removeMovedAway and removeEmptiedKeptAbove. + if (entry.serverContent.hasBelow(path)) + { + entry.serverContent.keep(path); + if (path == entry.localPath && wasDeletedWithItsContent(entry)) + { + Log::info( + "FileTracking: '{}' was deleted with its content, deleting its server tree too.", path.toUtf8() + ); + return deleteLostTree(entry, path); + } + Log::info( + "FileTracking: not deleting the server's copy of directory '{}', it was removed with its content.", + path.toUtf8() + ); + return; + } + const auto remotePath = remotePathOf(entry, path); + if (!remotePath) + { + Log::error( + "FileTracking: not mirroring the deletion of '{}': {}", path.toUtf8(), remotePath.error().toString() + ); + return; + } + removeRemote(entry, path, *remotePath); + } + + /** + * @brief Whether the watched directory of @p entry, gone from its path, was deleted rather than moved away. + * FSEvents loses the deletions below a watched directory removed with its content, its identity on + * its volume tells instead, see isDeletedFromItsVolume. + */ + bool wasDeletedWithItsContent(Entry const& entry) const + { + auto const* const watch = localWatches->find(entry.watchId); + if (!watch || !watch->place().self) + return false; + return isDeletedFromItsVolume(*watch->place().self, watch->directory()).value_or(false); + } + + /** + * @brief The watch lost the changes in @p directory, what is there now is compared with what the server is + * known to have. A directory around the item, the instance directory of a downloaded copy, stands for + * the item. + */ + void onMissed(Entry& entry, Paths::LocalPath const& directory) + { + const auto scope = entry.localPath.isWithin(directory) ? entry.localPath : directory; + if (!scope.isWithin(entry.localPath)) + return; + const auto status = localStatusOf(scope); + if (!status) + { + Log::error( + "FileTracking: watch {} lost the changes in '{}', which cannot be checked: {}", + entry.watchId.value(), + scope.toUtf8(), + Utility::errorMessage(status.error()) + ); + return; + } + Log::info( + "FileTracking: watch {} lost the changes in '{}', looking at it anew", entry.watchId.value(), scope.toUtf8() + ); + if (!*status) + { + if (entry.downloadedCopy()) + return onDeleted(entry, scope); + if (goneWatchedDirectoryWasDeleted) + return deleteLostTree(entry, scope); + // Like any directory that left with its content. The item's own removal comes from its watch. + if (scope != entry.localPath) + onDeleted(entry, scope); + return; + } + const auto watchId = entry.watchId; + deleteGoneBelow(entry, scope); + auto* stillWatched = find(watchId); + if (!stillWatched) + return; + const bool isDirectory = std::filesystem::is_directory(**status); + if (!isDirectory && isOnTheServerAsItIs(*stillWatched, scope)) + return; + onContentChanged(*stillWatched, {.path = scope, .isDirectory = isDirectory}); + } + + /** + * @brief @p directory was deleted with its tree while its watch lost the changes, the deletions of the last + * entries among them: what is known there is deleted like the watch would have reported it, deepest + * first. The item itself waits for its removal, which its watch holds back (see LocalItemWatch), or + * goes with its last entry when it was kept. + */ + void deleteLostTree(Entry const& entry, Paths::LocalPath const& directory) + { + const auto watchId = entry.watchId; + for (auto const& known : entry.serverContent.deepestFirst(directory)) + { + auto* stillWatched = find(watchId); + if (!stillWatched) + return; + // Removing the last entry of a kept directory removes the directory too. + if (known == stillWatched->localPath || !stillWatched->serverContent.has(known)) + continue; + onDeleted(*stillWatched, known); + } + } + + /** + * @brief Deletes what is known below @p directory and gone locally like the watch would have reported it, a + * directory as a whole: one that left with its content keeps its server copy. + */ + void deleteGoneBelow(Entry const& entry, Paths::LocalPath const& directory) + { + const auto deepestFirst = entry.serverContent.deepestFirst(directory); + std::vector gone; + for (auto const& known : deepestFirst | std::views::reverse) + { + const auto belowGone = std::ranges::any_of( + gone, + [&known](Paths::LocalPath const& removed) + { + return known.isWithin(removed); + } + ); + if (known != directory && !belowGone && isGoneLocally(known)) + gone.push_back(known); + } + const auto watchId = entry.watchId; + for (auto const& removed : gone) + if (auto* stillWatched = find(watchId)) + onDeleted(*stillWatched, removed); + } + + /** + * @brief Whether the server's copy of the file at @p path holds what the file holds now: an upload sent it in + * the state it has, or, for what a download brought, it has the modification time and size. + */ + static bool isOnTheServerAsItIs(Entry const& entry, Paths::LocalPath const& path) + { + if (const auto sent = entry.serverContent.sentFrom(path)) + return sent == sourceStateOf(path.native()); + const auto states = itemStatesOf(path.native()); + return states.size() == 1u && + entry.serverContent.fileState(path) == + ServerState{.mtime = states.front().mtime, .size = states.front().size}; + } + + /** + * @brief Whether @p path, known on the server, is there again locally as the same kind, a directory or not. + * A path that cannot be checked is not, removeRemote() keeps its server copy. + */ + bool isThereAgain(Entry const& entry, Paths::LocalPath const& path) const + { + const auto status = localStatusOf(path); + if (!status || !*status) + return false; + if (entry.serverContent.hasDirectory(path)) + return std::filesystem::is_directory(**status); + return entry.serverContent.fileState(path) && !std::filesystem::is_directory(**status); + } + + /** + * @brief The downloaded copy of @p entry was reported deleted or renamed away: its watch ends unless the copy + * is back within copyReturnTime. + */ + void awaitCopyReturn(Entry& entry) + { + auto* copy = entry.downloadedCopy(); + if (!copy || copy->returnWait) + return; + Log::debug("FileTracking: the copy of watch {} went away, waiting for it", entry.watchId.value()); + copy->returnWait = std::make_unique(strand, copyReturnTime); + copy->returnWait->async_wait( + [weak = weak_from_this(), watchId = entry.watchId](boost::system::error_code const& error) + { + if (error) + return; + if (const auto self = weak.lock()) + self->endUnlessCopyReturned(watchId); + } + ); + } + + void endUnlessCopyReturned(Ids::WatchId const& watchId) + { + auto* entry = find(watchId); + if (!entry || !entry->downloadedCopy()) + return; + entry->downloadedCopy()->returnWait.reset(); + if (!isGoneLocally(entry->localPath)) + return; + Log::info( + "FileTracking: the copy {} of watch {} was deleted or renamed, its watch ends", + entry->localPath.toUtf8(), + watchId.value() + ); + const auto ended = SharedData::EndedWatches{ + .reason = SharedData::WatchEndReason::CopyGone, .entries = {shownEntryOf(*entry)} + }; + stop(watchId); + view->showEnded(ended); + } + + /** + * @brief The watched item itself moved to @p newPath: the entry follows it, the server copy moves along when + * the entry mirrors moves. + */ + void onItemMoved(Entry& entry, Paths::LocalPath const& newPath) + { + const bool mirrorsMoves = entry.options.moveRemoteOnLocalMove; + const auto newRemotePath = mirrorsMoves ? renamedRemotePath(entry.remotePath, newPath) + : std::expected{entry.remotePath}; + if (!newRemotePath) + { + Log::warn( + "FileTracking: not following the move of watch {}: {}", entry.watchId.value(), newRemotePath.error() + ); + return; + } + // The server copy may not follow onto the path of another watch, which writes there. + if (const auto free = refuseRemoteOverlap(*newRemotePath, entry.watchId); !free) + { + Log::warn( + "FileTracking: stopping watch {}, its move overlaps another watch: {}", + entry.watchId.value(), + free.error() + ); + const auto watchId = entry.watchId; + return stop(watchId); + } + Log::info( + "FileTracking: watch {} moved from '{}' to '{}'", + entry.watchId.value(), + entry.localPath.toUtf8(), + newPath.toUtf8() + ); + const auto oldPath = entry.localPath; + // An upload of the item that is on its way writes the old name: it stops and goes on under the new one, + // or its server copy follows once it ended. + if (mirrorsMoves) + stopUpload(entry, oldPath, PartialFile::Kept); + const bool deferred = entry.pendingUploads.moveWhenDone(oldPath, newPath); + // Without mirrored moves the server copy keeps its name, the new local name stands for it. + if (!mirrorsMoves) + entry.serverContent.move(oldPath, newPath); + else if (!deferred && entry.serverContent.has(oldPath)) + renameRemote(entry, oldPath, newPath, entry.remotePath, *newRemotePath); + entry.pendingUploads.moveBelow(oldPath, newPath, /*serverCopiesFollow=*/true); + entry.localPath = newPath; + entry.remotePath = *newRemotePath; + show(entry); + } + + void onMoved(Entry& entry, Paths::LocalPath const& oldPath, LocalChange const& change) + { + const auto oldRemotePath = remotePathOf(entry, oldPath); + const auto newRemotePath = remotePathOf(entry, change.path); + if (!oldRemotePath || !newRemotePath) + { + Log::warn( + "FileTracking: not mirroring the move in watch {}: {}", + entry.watchId.value(), + (oldRemotePath ? newRemotePath.error() : oldRemotePath.error()).toString() + ); + return; + } + + if (meaningOfMoveIn(entry, oldPath, change.path) == MoveMeaning::Rename) + { + // An upload of the source on its way writes the old name: a file stops and goes on under the new one, + // or the server copy follows once the upload ended. + stopUpload(entry, oldPath, PartialFile::Kept); + if (entry.pendingUploads.moveWhenDone(oldPath, change.path)) + return; + const bool renamed = renameRemote(entry, oldPath, change.path, *oldRemotePath, *newRemotePath); + entry.pendingUploads.moveBelow(oldPath, change.path, /*serverCopiesFollow=*/renamed); + return; + } + + Log::info("FileTracking: '{}' replaced '{}', uploading it.", oldPath.toUtf8(), change.path.toUtf8()); + if (entry.options.moveRemoteOnLocalMove) + { + if (entry.pendingUploads.removeRemoteWhenDone(oldPath)) + stopUpload(entry, oldPath, PartialFile::Discarded); + else if (entry.serverContent.has(oldPath) && !entry.serverContent.hasBelow(oldPath)) + removeRemote(entry, oldPath, *oldRemotePath); + } + onContentChanged(entry, change); + } + + /** + * @brief What the move of @p oldPath to @p newPath in @p entry does on the server. With mirrored moves an + * upload on its way counts as the server copy it brings, and a move into a directory whose upload is on + * its way replaces what that upload brings there. + */ + static MoveMeaning + meaningOfMoveIn(Entry const& entry, Paths::LocalPath const& oldPath, Paths::LocalPath const& newPath) + { + auto const& known = entry.serverContent; + auto const& pending = entry.pendingUploads; + if (!entry.options.moveRemoteOnLocalMove) + return meaningOfMove(known.has(newPath), known.has(oldPath)); + return meaningOfMove( + known.has(newPath) || pending.covers(newPath), known.has(oldPath) || pending.isPending(oldPath) + ); + } + + /** + * @brief Stops the upload of the file at @p path that is on its way, once the remote got it: its partial file + * stays for the upload under the new name, see finishMovedUpload(), or goes, see @p partial. Its end + * follows as usual. + */ + void stopUpload(Entry& entry, Paths::LocalPath const& path, PartialFile partial) + { + const auto destination = entry.pendingUploads.sentFileTo(path); + if (!destination) + return; + Log::info( + "FileTracking: stopping the upload of '{}', it was {}.", + path.toUtf8(), + partial == PartialFile::Kept ? "renamed" : "deleted" + ); + if (partial == PartialFile::Kept) + entry.pendingUploads.keepsPartial(path); + remote->cancelUpload(*destination, partial); + } + + /** + * @brief Renames the server copy of @p oldPath, at @p oldRemotePath, to @p newRemotePath for @p newPath, when + * the entry mirrors moves. + * @return Whether the rename goes, in the server order. + */ + bool renameRemote( + Entry& entry, + Paths::LocalPath const& oldPath, + Paths::LocalPath const& newPath, + Paths::RemotePath const& oldRemotePath, + Paths::RemotePath const& newRemotePath + ) + { + if (!entry.options.moveRemoteOnLocalMove) + return false; + inOrder( + {oldRemotePath, newRemotePath}, + [this, watchId = entry.watchId, oldRemotePath, newRemotePath, ended = renameEndOf(entry, newPath)]( + ServerOperationOrder::Ticket ticket + ) + { + const auto queued = + remote->rename(oldRemotePath, newRemotePath, endsAndLeavesOrder(ended, ticket)); + if (queued) + return; + Log::error( + "FileTracking: failed to queue the remote rename for watch {}: {}", + watchId.value(), + queued.error() + ); + ended(SharedData::OperationCompletionReason::Failed); + leaveOrder(ticket); + } + ); + entry.serverContent.move(oldPath, newPath); + return true; + } + + /** + * @brief Hears when the rename of a server copy for @p newPath ended, see remoteChangeOf(). One that failed + * left the copy where it was: what is at @p newPath now goes there, compared with the server first. + */ + TrackerRemote::OnEnded renameEndOf(Entry& entry, Paths::LocalPath newPath) + { + return [weak = weak_from_this(), + watchId = entry.watchId, + newPath = std::move(newPath), + ended = remoteChangeOf(entry)](SharedData::OperationCompletionReason reason) + { + ended(reason); + if (reason == SharedData::OperationCompletionReason::Completed) + return; + if (const auto self = weak.lock()) + self->onRenameFailed(watchId, newPath); + }; + } + + void onRenameFailed(Ids::WatchId const& watchId, Paths::LocalPath const& newPath) + { + auto* entry = find(watchId); + if (!entry) + return; + Log::warn("FileTracking: the server's copy was not renamed to '{}', uploading it there", newPath.toUtf8()); + entry->serverContent.forget(newPath); + const auto status = localStatusOf(newPath); + if (!status || !*status) + return; + onContentChanged(*entry, {.path = newPath, .isDirectory = std::filesystem::is_directory(**status)}); + } + + /** + * @brief Deletes the server copy of @p local, unless @p local cannot be checked: it may still exist. + */ + void removeRemote(Entry& entry, Paths::LocalPath const& local, Paths::RemotePath const& remotePath) + { + if (const auto status = localStatusOf(local); !status) + { + Log::error( + "FileTracking: not deleting '{}' for watch {}, '{}' cannot be checked: {}", + remotePath.value(), + entry.watchId.value(), + local.toUtf8(), + Utility::errorMessage(status.error()) + ); + return; + } + Log::info( + "FileTracking: deleting '{}' for watch {}, it was deleted locally", + remotePath.value(), + entry.watchId.value() + ); + entry.serverContent.forget(local); + inOrder( + {remotePath}, + [this, watchId = entry.watchId, remotePath, ended = remoteChangeOf(entry)]( + ServerOperationOrder::Ticket ticket + ) + { + const auto queued = remote->remove(remotePath, endsAndLeavesOrder(ended, ticket)); + if (queued) + return; + Log::error( + "FileTracking: failed to queue the remote delete for watch {}: {}", + watchId.value(), + queued.error() + ); + ended(SharedData::OperationCompletionReason::Failed); + leaveOrder(ticket); + } + ); + removeEmptiedKeptAbove(entry, local); + } + + /** + * @brief A kept directory above @p removed that nothing is known below anymore goes too once it is no + * directory locally anymore: its deletion arrived before that of its content (FSEvents may report + * them so). + */ + void removeEmptiedKeptAbove(Entry& entry, Paths::LocalPath const& removed) + { + if (!entry.options.deleteRemoteOnLocalDelete) + return; + const auto emptied = entry.serverContent.emptiedKeptAbove(removed); + if (emptied.empty() || isDirectoryLocally(emptied.front())) + return; + // Deleting it looks further up. + if (const auto remotePath = remotePathOf(entry, emptied.front())) + removeRemote(entry, emptied.front(), *remotePath); + } + + /** + * @brief @p uploaded, a directory the server got as a whole, may be one moved within the item that the watch + * reported as a removal and an addition: a kept directory gone locally whose known content it holds + * moved there, its server copy goes entry by entry. + */ + void removeMovedAway(Entry& entry, Paths::LocalPath const& uploaded) + { + if (!entry.options.moveRemoteOnLocalMove) + return; + for (auto const& kept : entry.serverContent.keptDirectories()) + { + if (kept.isWithin(uploaded) || uploaded.isWithin(kept) || !isGoneLocally(kept) || + !entry.serverContent.holdsAllBelow(uploaded, kept)) + continue; + Log::info( + "FileTracking: '{}' moved to '{}', deleting the server's copy at its old place.", + kept.toUtf8(), + uploaded.toUtf8() + ); + removeKnownTree(entry, kept); + } + } + + /** + * @brief Whether the server is known to have the other kind at @p path than an upload of a directory + * (@p isDirectory) or a file would write, and it may be deleted for it. + */ + bool hasOtherKindInTheWay(Entry const& entry, Paths::LocalPath const& path, bool isDirectory) const + { + if (!entry.options.deleteRemoteOnLocalDelete) + return false; + return isDirectory ? entry.serverContent.fileState(path).has_value() + : entry.serverContent.hasDirectory(path); + } + + /** + * @brief Deletes what is known on the server at and below @p path entry by entry, deepest first, which never + * takes anything along the tracker does not know of. + */ + void removeKnownTree(Entry& entry, Paths::LocalPath const& path) + { + for (auto const& known : entry.serverContent.deepestFirst(path)) + { + // Removing the last entry of a kept directory removes the directory too. + if (!entry.serverContent.has(known)) + continue; + if (const auto remotePath = remotePathOf(entry, known)) + removeRemote(entry, known, *remotePath); + } + } + + /** + * @brief Whether the server's copy of the file of @p change holds what the file holds now: an upload sent it + * in the state it has, the change was reported after that. Not while an upload that takes the file + * along is on its way, the change goes after it. + */ + static bool isSentAsItIs(Entry const& entry, LocalChange const& change) + { + if (change.isDirectory || entry.pendingUploads.covers(change.path)) + return false; + const auto sent = entry.serverContent.sentFrom(change.path); + return sent && sent == sourceStateOf(change.path.native()); + } + + /** + * @brief Uploads a change unless the same path, or a directory above it, is already on its way; the change + * then goes once that upload ended. It starts in the server order, after what was decided before on + * its path. A file the server has as it is stays, a directory the server has goes by what it lacks. + */ + void queueUpload(Upload upload) + { + auto* entry = find(upload.watchId); + if (!entry) + return; + if (isSentAsItIs(*entry, upload.change)) + { + Log::debug( + "FileTracking: the server has '{}' as it is, not uploading it.", upload.change.path.toUtf8() + ); + return discardPartial(upload.continuesPartialOf); + } + if (upload.change.isDirectory && !entry->pendingUploads.covers(upload.change.path) && + entry->serverContent.hasDirectory(upload.change.path)) + return uploadWhatTheServerLacks(*entry, upload); + const bool starts = entry->pendingUploads.start(upload.change); + progressChanged(); + if (!starts) + return discardPartial(upload.continuesPartialOf); + showUploading(*entry); + upload.ticket.reset(); + // A continued upload takes the partial file of the old name, after the stopped upload ended. + std::vector paths{upload.destination}; + if (upload.continuesPartialOf) + paths.push_back(*upload.continuesPartialOf); + inOrder( + std::move(paths), + [this, upload = std::move(upload)](ServerOperationOrder::Ticket ticket) mutable + { + upload.ticket = ticket; + checkServerThenUpload(std::move(upload)); + } + ); + } + + /** + * @brief @p upload is of a directory the server has: what is below it goes on its own where the server lacks + * it, a file no upload sent as it is and a directory with its content. A link the server has stays, + * a change of its own retargets it. Nothing is compared with the server, the upload of the whole + * directory would not either: the states recorded after the last one may be newer than what it sent. + */ + void uploadWhatTheServerLacks(Entry& entry, Upload const& upload) + { + Log::debug( + "FileTracking: the server has the directory '{}', uploading what it lacks.", upload.change.path.toUtf8() + ); + const auto watchId = entry.watchId; + std::vector newDirectories; + for (auto const& state : itemStatesOf(upload.change.path.native())) + { + const auto path = Paths::LocalPath::make(state.path); + if (!path || *path == upload.change.path || !path->isWithin(upload.change.path)) + continue; + auto* stillWatched = find(watchId); + if (!stillWatched) + return; + const auto inNewDirectory = std::ranges::any_of( + newDirectories, + [&path](Paths::LocalPath const& directory) + { + return path->isWithin(directory); + } + ); + const bool known = state.isDirectory ? stillWatched->serverContent.hasDirectory(*path) + : state.isSymlink && stillWatched->serverContent.fileState(*path); + if (inNewDirectory || known) + continue; + if (state.isDirectory) + newDirectories.push_back(*path); + if (const auto destination = remotePathOf(*stillWatched, *path)) + queueUpload( + Upload{ + .watchId = watchId, + .change = {.path = *path, .isDirectory = state.isDirectory}, + .destination = *destination, + .checkServer = false, + } + ); + } + } + + void checkServerThenUpload(Upload upload) + { + auto* entry = find(upload.watchId); + // A path that moved away goes again at the new one, see enqueueUpload. + if (entry && !entry->pendingUploads.movedAway(upload.change.path)) + { + const auto status = localStatusOf(upload.change.path); + if (!status) + return onUncheckable(upload, status.error()); + if (!*status) + return onGoneLocally(*entry, upload); + // Turned into the other kind while the change waited, or the server has the other kind there: it goes + // again as what it is now, after the server copy in the way (see followUpUpload). + const bool isDirectory = std::filesystem::is_directory(**status); + if (isDirectory != upload.change.isDirectory || + hasOtherKindInTheWay(*entry, upload.change.path, isDirectory)) + { + entry->pendingUploads.start({.path = upload.change.path, .isDirectory = isDirectory}); + return finishUpload(upload, UploadOutcome::NotUploaded); + } + } + if (!entry || !entry->options.checkRemoteChanges || !upload.checkServer) + return enqueueUpload(std::move(upload)); + // The upload of a regular file compares it, right before it replaces the server's copy. + if (!upload.change.isDirectory && !isSymlink(upload.change.path)) + { + auto expected = ExpectedDestination{.file = entry->serverContent.fileState(upload.change.path)}; + return enqueueUpload(std::move(upload), std::move(expected)); + } + if (serverChangedMeanwhile(*entry, upload)) + return askServerChanged(std::move(upload)); + enqueueUpload(std::move(upload)); + } + + /** + * @brief Whether the server's copy at the destination of @p upload, a directory or a symbolic link, is not + * what this tracker left there. A symbolic link is compared as itself. + */ + bool serverChangedMeanwhile(Entry const& entry, Upload const& upload) + { + const auto current = + isSymlink(upload.change.path) ? remote->linkStat(upload.destination) : remote->stat(upload.destination); + if (!current) + { + Log::warn( + "FileTracking: cannot check '{}' on the server: {}", upload.destination.value(), current.error() + ); + // An unanswered check counts as changed. + return true; + } + if (upload.change.isDirectory) + return current->has_value() && !entry.serverContent.has(upload.change.path); + return serverChanged(entry.serverContent.fileState(upload.change.path), *current); + } + + /** + * @brief The path of @p upload is gone by the time it would go: there is nothing to ask about or to upload, + * and the deletion is mirrored, which the watch may not report (macOS can report a removed directory + * as modified only). The change counts as nothing done, the deletion counts on its own. + */ + void onGoneLocally(Entry const& entry, Upload const& upload) + { + Log::debug("FileTracking: '{}' is gone, not uploading it.", upload.change.path.toUtf8()); + const auto watchId = entry.watchId; + finishUpload(upload, UploadOutcome::NotUploaded, /*carried=*/0u); + if (auto* stillWatched = find(watchId)) + onDeleted(*stillWatched, upload.change.path); + } + + /** + * @brief The path of @p upload cannot be checked (@p error), it may still exist: it is neither uploaded nor + * deleted on the server. + */ + void onUncheckable(Upload const& upload, std::error_code const& error) + { + Log::error( + "FileTracking: not uploading '{}', it cannot be checked: {}", + upload.change.path.toUtf8(), + Utility::errorMessage(error) + ); + finishUpload(upload, UploadOutcome::NotUploaded); + } + + void askServerChanged(Upload upload) + { + waitingQuestions.push_back(std::move(upload)); + askNextServerChanged(); + } + + void askNextServerChanged() + { + while (!openQuestion && !waitingQuestions.empty()) + { + auto upload = std::move(waitingQuestions.front()); + waitingQuestions.pop_front(); + const auto* entry = find(upload.watchId); + if (!entry) + continue; + // Nothing to ask about a path that moved meanwhile, it goes again at the new one. + if (entry->pendingUploads.movedAway(upload.change.path)) + { + enqueueUpload(std::move(upload)); + continue; + } + if (isGoneLocally(upload.change.path)) + { + onGoneLocally(*entry, upload); + continue; + } + openQuestion = OpenQuestion{.questionId = Ids::generateQuestionId(), .upload = std::move(upload)}; + view->ask({.questionId = openQuestion->questionId, .remotePath = openQuestion->upload.destination}); + } + progressChanged(); + } + + void answer(Ids::QuestionId const& questionId, SharedData::ServerChangedAnswer answer) + { + if (!openQuestion || openQuestion->questionId != questionId) + return; + auto upload = std::move(openQuestion->upload); + openQuestion.reset(); + progressChanged(); + switch (answer) + { + case SharedData::ServerChangedAnswer::Replace: + enqueueUpload(std::move(upload)); + break; + case SharedData::ServerChangedAnswer::Keep: + Log::info("FileTracking: kept the server's version of '{}'.", upload.destination.value()); + declineUpload(upload); + break; + case SharedData::ServerChangedAnswer::Dismissed: + return abandonServerChangedQuestions(upload); + } + askNextServerChanged(); + } + + /** + * @brief Another dialog replaced the question for @p upload: it and the waiting questions are asked again on + * the next save, nothing opens now over the dialog that replaced it. + */ + void abandonServerChangedQuestions(Upload const& upload) + { + Log::info( + "FileTracking: the question about '{}' was replaced by another dialog.", upload.destination.value() + ); + declineUpload(upload); + for (auto const& waiting : std::exchange(waitingQuestions, {})) + declineUpload(waiting); + } + + /** + * @brief The change is not uploaded, and the changes that arrived while it was asked about wait for the next + * save. + */ + void declineUpload(Upload const& upload) + { + if (auto* entry = find(upload.watchId)) + entry->pendingUploads.changesHandled(upload.change.path); + finishUpload(upload, UploadOutcome::Declined); + } + + /** + * @param expected What the server must still have at the destination, see UploadRequest::expectedDestination. + */ + void enqueueUpload(Upload upload, std::optional expected = std::nullopt) + { + auto* entry = find(upload.watchId); + // A path that moved meanwhile cannot be read, the upload goes again at the new one once this one ended. + if (!entry || entry->pendingUploads.movedAway(upload.change.path)) + return finishUpload(upload, UploadOutcome::NotUploaded); + // The upload reads the file when it runs, the changes until now are in it. + entry->pendingUploads.changesHandled(upload.change.path); + if (upload.change.isDirectory) + entry->pendingUploads.carries(upload.change.path, entryCountOf(upload.change.path.native())); + entry->pendingUploads.sent(upload.change.path, upload.destination, upload.change.isDirectory); + const auto queued = remote->upload( + UploadRequest{ + .source = upload.change.path, + .destination = upload.destination, + .refreshListing = upload.refreshListing, + // Below the item a directory renamed without mirroring has no server counterpart yet. + .createMissingDirectories = upload.change.path != entry->localPath, + .expectedDestination = std::move(expected), + .continuesPartialOf = upload.continuesPartialOf, + }, + [weak = weak_from_this(), upload](UploadEnd const& ended) + { + if (const auto self = weak.lock()) + self->onUploadEnded(upload, ended); + } + ); + if (queued) + return; + // A temporary file an editor already renamed or removed ends up here. + Log::warn("FileTracking: did not upload '{}': {}", upload.change.path.toUtf8(), queued.error()); + finishUpload(upload, UploadOutcome::NotUploaded); + } + + /** + * @brief Once an upload ended, what the server has now is the state the next check compares with. The upload + * stays on its way until the remote told that state. + */ + void onUploadEnded(Upload const& upload, UploadEnd const& ended) + { + auto* entry = find(upload.watchId); + // The server's copy changed meanwhile and stayed, the upload asks about it now unless the user stopped + // checking since. + if (entry && ended.changedOnServer) + { + if (entry->options.checkRemoteChanges) + return askServerChanged(upload); + return enqueueUpload(upload); + } + if (ended.reason != SharedData::OperationCompletionReason::Completed || !entry) + { + // Nobody continues the partial file of a watch that ended meanwhile. + if (!entry && !upload.change.isDirectory) + discardPartial(upload.destination); + return finishUpload(upload, UploadOutcome::NotUploaded); + } + // Uploads carry the local modification time, so the local states are what the server has now. A link + // to a directory reports itself as one, the walk of its states would follow it. + std::optional carried{}; + if (upload.change.isDirectory && !isSymlink(upload.change.path)) + { + carried = rememberStates(*entry, upload.change.path); + rememberSentButGone(*entry, upload, ended.sentFiles); + removeMovedAway(*entry, upload.change.path); + } + else if (ended.destinationState) + entry->serverContent.rememberFile(upload.change.path, *ended.destinationState); + for (auto const& sent : ended.sentFiles) + if (sent.state && sent.path.isWithin(upload.change.path)) + entry->serverContent.rememberSent(sent.path, *sent.state); + finishUpload(upload, UploadOutcome::Uploaded, carried); + } + + /** + * @brief What the directory upload @p upload wrote that is gone locally by now, which rememberStates() cannot + * see: the files of @p sent with the directories above them, and the directory itself when the server + * has it. Their deletions follow as reported, see onDeleted. + */ + void rememberSentButGone(Entry& entry, Upload const& upload, std::vector const& sent) + { + auto const& directory = upload.change.path; + for (auto const& file : sent) + { + if (!file.path.isWithin(directory) || entry.serverContent.fileState(file.path) || + !isGoneLocally(file.path)) + continue; + const auto state = serverStateOfSent(entry, file); + if (!state) + continue; + entry.serverContent.rememberFile(file.path, *state); + for (auto parent = file.path.parent(); parent && parent->isWithin(directory); parent = parent->parent()) + entry.serverContent.rememberDirectory(*parent); + } + if (entry.serverContent.has(directory) || !isGoneLocally(directory)) + return; + if (const auto state = remote->linkStat(upload.destination); state && *state) + entry.serverContent.rememberDirectory(directory); + } + + /** + * @brief What the server has of @p file, which an upload wrote: the state it was sent in, or else what the + * server tells. Nothing when it has no file there or cannot be asked. + */ + std::optional serverStateOfSent(Entry const& entry, SentFile const& file) + { + if (file.state) + return serverStateOf(*file.state); + const auto remotePath = remotePathOf(entry, file.path); + if (!remotePath) + return std::nullopt; + return remote->linkStat(*remotePath).value_or(std::nullopt); + } + + /** + * @brief The upload of @p upload is no longer on its way, see followUpUpload. The operations waiting for it + * in the server order start then. + * @param carried The entries it left on the server, it counts as done that many: what a completed directory + * upload recorded, the changes below it that arrived meanwhile count when they go. Unset, it counts as + * the changes it stood for. + */ + void + finishUpload(Upload const& upload, UploadOutcome outcome, std::optional carried = std::nullopt) + { + if (auto* entry = find(upload.watchId)) + followUpUpload(*entry, upload, outcome, carried); + leaveOrder(upload); + } + + /** + * @brief What changed while @p upload was on its way goes now: the path itself once more, the changes below an + * uploaded directory, or the removal of a path that went away. + */ + void + followUpUpload(Entry& entry, Upload const& upload, UploadOutcome outcome, std::optional carried) + { + auto finished = entry.pendingUploads.finish(upload.change.path); + showUploading(entry); + changeEnded(carried.value_or(finished.counted)); + // A directory above that moved took the server copy along. + const auto serverPath = finished.serverCopyAt.value_or(upload.change.path); + const auto serverDestination = finished.serverCopyAt + ? remotePathOf(entry, serverPath).value_or(upload.destination) + : upload.destination; + const auto keptPartial = keptPartialOf(upload, outcome, finished); + if (finished.removeRemote) + { + // Deleted while it was on its way but there again by now, the way git flips a path: it goes once + // more instead and the server copy stays. So does one whose stopped upload left nothing. + const bool backUnsent = outcome != UploadOutcome::Uploaded && + !isGoneLocally(finished.movedTo.value_or(upload.change.path)); + if (!isThereAgain(entry, serverPath) && !backUnsent) + { + discardPartial(keptPartial); + // Everything below a directory went along. + if (entry.serverContent.has(serverPath)) + removeKnownTree(entry, serverPath); + else if (outcome == UploadOutcome::Uploaded) + removeRemote(entry, serverPath, serverDestination); + return; + } + finished.changedAgain = true; + } + if (finished.movedTo) + return finishMovedUpload(entry, upload, outcome, finished, serverPath, serverDestination, keptPartial); + if (finished.changedAgain) + { + // Back as the other kind, a directory where a file was or the other way round: the server's copy is + // in the way and goes first. + if (hasOtherKindInTheWay(entry, serverPath, finished.isDirectory)) + removeKnownTree(entry, serverPath); + auto again = upload; + again.change.isDirectory = finished.isDirectory; + again.continuesPartialOf.reset(); + queueUpload(std::move(again)); + } + discardPartial(keptPartial); + for (auto const& change : finished.changedBelow) + onContentChanged(entry, change, /*checkServer=*/false); + } + + /** + * @brief The partial file on the server that the path of @p upload left and no upload continues yet: the one + * a stopped upload kept, or the one @p upload was to continue but never got to. Nothing once it was + * uploaded. + */ + static std::optional + keptPartialOf(Upload const& upload, UploadOutcome outcome, PendingUploads::Finished const& finished) + { + if (outcome == UploadOutcome::Uploaded || upload.change.isDirectory) + return std::nullopt; + if (finished.sentTo) + return finished.partialKept ? finished.sentTo : std::nullopt; + return upload.continuesPartialOf; + } + + /** + * @brief The partial file a stopped upload to @p destination kept goes, no upload continues it. It goes in the + * server order, so it never meets the partial file of a later upload of that path. + */ + void discardPartial(std::optional const& destination) + { + if (!destination) + return; + inOrder( + {*destination}, + [this, destination = *destination](ServerOperationOrder::Ticket ticket) + { + const auto queued = remote->discardPartialUpload(destination, leavesOrderWhenEnded(ticket)); + if (queued) + return; + Log::warn( + "FileTracking: cannot remove the partial upload of '{}': {}", + destination.value(), + queued.error() + ); + leaveOrder(ticket); + } + ); + } + + /** + * @brief The path of @p upload moved while it was on its way (see PendingUploads::moveWhenDone and moveBelow): + * its server copy at @p serverPath follows now, and what the upload could not take or what changed + * meanwhile goes to the new path. + */ + void finishMovedUpload( + Entry& entry, + Upload const& upload, + UploadOutcome outcome, + PendingUploads::Finished const& finished, + Paths::LocalPath const& serverPath, + Paths::RemotePath const& serverDestination, + std::optional const& keptPartial + ) + { + auto const& movedTo = *finished.movedTo; + const auto destination = remotePathOf(entry, movedTo); + if (!destination) + { + Log::warn( + "FileTracking: not following the move of '{}' in watch {}: {}", + upload.change.path.toUtf8(), + entry.watchId.value(), + destination.error().toString() + ); + return discardPartial(keptPartial); + } + // The server copy moved along with a directory above, or the server renames it now. + const bool inPlace = serverPath == movedTo; + const bool renamed = inPlace || + ((outcome == UploadOutcome::Uploaded || entry.serverContent.has(serverPath)) && + renameRemote(entry, serverPath, movedTo, serverDestination, *destination)); + const LocalChange moved{.path = movedTo, .isDirectory = upload.change.isDirectory}; + // The upload read the old path, which may have been gone by then, or it was stopped and goes on under the + // new name. A declined one waits for the next change. + const bool goesAgain = outcome == UploadOutcome::NotUploaded || finished.changedAgain; + // A file changed since it was stopped goes anew: the resume compares only the end of the part sent. + const auto continued = goesAgain && !finished.changedAgain ? keptPartial : std::nullopt; + if (!continued) + discardPartial(keptPartial); + if (goesAgain) + return queueUpload(Upload{ + .watchId = entry.watchId, + .change = moved, + .destination = *destination, + .checkServer = + outcome == UploadOutcome::Declined || (!renamed && outcome != UploadOutcome::Uploaded), + .continuesPartialOf = continued, + }); + // The states taken after the upload are those of the old path, the local ones are the server's now. + if (renamed && outcome == UploadOutcome::Uploaded && (moved.isDirectory || inPlace)) + { + entry.serverContent.forget(upload.change.path); + rememberStates(entry, movedTo); + } + for (auto const& change : finished.changedBelow) + onContentChanged(entry, change, /*checkServer=*/false); + } + }; + + Tracker::Tracker( + boost::asio::strand strand, + TempDirInstanceManager& instances, + LocalWatchManager& localWatches, + std::unique_ptr remote, + std::unique_ptr view, + std::string downloadTempSuffix, + std::function defaultOptions + ) + : impl_{std::make_shared( + std::move(strand), + instances, + localWatches, + std::move(remote), + std::move(view), + std::move(downloadTempSuffix), + std::move(defaultOptions) + )} + {} + + Tracker::~Tracker() = default; + + void Tracker::open( + Paths::RemotePath const& remotePath, + std::uint64_t sizeBytes, + bool watch, + std::function const&)> onOpened + ) + { + impl_->open(remotePath, sizeBytes, watch, std::move(onOpened)); + } + + std::expected + Tracker::checkUploadAndWatch(Paths::LocalPath const& item, Paths::RemotePath const& directory) const + { + return impl_->checkUploadAndWatch(item, directory); + } + + std::expected + Tracker::uploadAndWatch(Paths::LocalPath const& item, Paths::RemotePath const& destination) + { + return impl_->uploadAndWatch(item, destination); + } + + void Tracker::stop(Ids::WatchId const& watchId) + { + impl_->stop(watchId); + } + + void Tracker::setOption(Ids::WatchId const& watchId, SharedData::WatchOption option, bool value) + { + impl_->setOption(watchId, option, value); + } + + std::expected + Tracker::discard(Ids::InstanceId const& instanceId, Paths::LocalPath const& path) + { + return impl_->discard(instanceId, path); + } + + void Tracker::answer(Ids::QuestionId const& questionId, SharedData::ServerChangedAnswer answer) + { + impl_->answer(questionId, answer); + } + + void Tracker::stopAll() + { + impl_->stopAll(); + } + + void Tracker::connectionLost() + { + impl_->connectionLost(); + } +} diff --git a/backend/source/backend/main.cpp b/backend/source/backend/main.cpp index 806829dd..6242aa7b 100644 --- a/backend/source/backend/main.cpp +++ b/backend/source/backend/main.cpp @@ -324,21 +324,18 @@ void Main::registerRpc() [this, riskyFilePolicyOf](Persistence::State const& state) { riskyFilePolicy_.store(riskyFilePolicyOf(state)); + sshSessionManager_->fileTrackingDefaults(state.fileTrackingOptions); } ); opener_ = std::make_unique(portalParentWindow(window_)); rpcSystem_ = std::make_unique(window_.getExecutor(), window_, hub_); - tempDirInstanceManager_ = std::make_unique( - window_.getExecutor(), - window_, - hub_, + const auto temporaryDownloadsDirectory = Nui::resolvePath(holder.stateCache().localFilesystemOptions.temporaryDownloadsDirectory.value_or( Persistence::LocalFilesystemOptions{}.temporaryDownloadsDirectory.value() - )) - ); - tempDirInstanceManager_->registerRpc(); - tempDirInstanceManager_->scheduleStartupCleanup(); + )); + sshSessionManager_->startFileTracking(temporaryDownloadsDirectory); + sshSessionManager_->fileTrackingDefaults(holder.stateCache().fileTrackingOptions); if (error) { diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index ad0c236c..7f814df2 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -71,7 +71,8 @@ Session::Session( Nui::RpcHub& hub, Persistence::SftpOptions const& sftpOptions, BulkResumeRegistry* bulkResumeRegistry, - std::shared_ptr localOperationQueue + std::shared_ptr localOperationQueue, + FileTracking::TrackerServices fileTracking ) : RpcHelper::StrandRpc{executor, std::move(strand), wnd, hub} , id_{std::move(id)} @@ -90,6 +91,10 @@ Session::Session( sftpOptions.uploadOptions ? sftpOptions.uploadOptions->tempFileSuffix : std::nullopt )} , localOperationQueue_{std::move(localOperationQueue)} + , fileTrackingServices_{std::move(fileTracking)} + , downloadTempFileSuffix_{Persistence::effectiveTempFileSuffix( + sftpOptions.downloadOptions ? sftpOptions.downloadOptions->tempFileSuffix : std::nullopt + )} {} Session::~Session() @@ -146,6 +151,8 @@ void Session::start() self->registerRpcSftpPreDeleteChecks(); self->registerRpcQueuedRemoves(); self->operationQueue_->registerRpc(); + self->startFileTracking(); + self->registerRpcFileTracker(); Log::info("Session '{}' connected", self->id_.value()); @@ -179,6 +186,11 @@ void Session::stop() Log::info("Stopping session '{}'", self->id_.value()); + // Before the queue goes, whose cancellations would otherwise reach the tracker. The copies of the session + // count as orphaned from now on. + self->stopFileTracking(); + self->tracker_.reset(); + // Backup any in-flight or queued bulk operations before // teardown so a seamless reconnect can adopt them. Always // happens — the frontend's adopt/discard signals (and the @@ -186,7 +198,7 @@ void Session::stop() if (self->bulkResumeRegistry_) self->operationQueue_->dumpBulkResumes(*self->bulkResumeRegistry_); self->localOperationQueue_->release(self->id_); - self->operationQueue_->cancelAll(); + self->operationQueue_->dropAll(); self->operationQueue_.reset(); // Close all channels @@ -244,15 +256,7 @@ void Session::adoptBulkResumes(std::vector const& operationIds // which expect a channel id. We adopt against the most-recently // opened sftp channel by convention; if none exists, the resume // is dropped back into the registry for a later attempt. - std::shared_ptr sftp; - for (auto const& [channelId, weakChannel] : self->sftpChannels_) - { - if (auto locked = weakChannel.lock(); locked) - { - sftp = locked; - break; - } - } + const auto sftp = self->anySftpChannel(); if (!sftp) { Log::warn( @@ -277,6 +281,16 @@ void Session::adoptBulkResumes(std::vector const& operationIds ); } +std::shared_ptr Session::anySftpChannel() const +{ + for (auto const& [channelId, weakChannel] : sftpChannels_) + { + if (auto locked = weakChannel.lock(); locked) + return locked; + } + return nullptr; +} + void Session::doOperationQueueWork() { within_strand_do_no_recurse( diff --git a/backend/source/backend/session_file_tracking.cpp b/backend/source/backend/session_file_tracking.cpp new file mode 100644 index 00000000..81bc26f0 --- /dev/null +++ b/backend/source/backend/session_file_tracking.cpp @@ -0,0 +1,892 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + /** + * @brief Hands each of the @p count entries of a bulk that ended as @p completed its end, an entry @p skipped + * tells of failed. + */ + void endEntries( + SharedData::OperationCompleted const& completed, + std::size_t count, + std::function const& skipped, + FileTracking::BulkTrackerRemote::OnEntryEnded const& onEnded + ) + { + for (std::size_t index = 0; index < count; ++index) + { + const auto reason = completed.reason == SharedData::OperationCompletionReason::Completed && skipped(index) + ? SharedData::OperationCompletionReason::Failed + : completed.reason; + onEnded(index, reason); + } + } + + /** + * @brief What the server has at uploaded destinations, asked for on the strand of an SFTP channel. Its delivery + * gets the states on the session strand once they are known, or without them when the channel drops the + * request unasked. + */ + class DestinationStates + { + public: + using States = std::vector>; + using Delivery = std::function; + + DestinationStates( + boost::asio::strand strand, + std::vector paths, + Delivery delivery + ) + : strand_{std::move(strand)} + , paths_{std::move(paths)} + , states_(paths_.size()) + , delivery_{std::move(delivery)} + {} + ~DestinationStates() + { + deliver(); + } + DestinationStates(DestinationStates const&) = delete; + DestinationStates& operator=(DestinationStates const&) = delete; + DestinationStates(DestinationStates&&) = delete; + DestinationStates& operator=(DestinationStates&&) = delete; + + void askInStrand(SecureShell::SftpSession& channel) + { + for (std::size_t index = 0; index < paths_.size(); ++index) + { + if (const auto information = channel.lstatInStrand(paths_[index])) + states_[index] = FileTracking::ServerState{.mtime = information->mtime, .size = information->size}; + } + deliver(); + } + + private: + void deliver() + { + if (!delivery_) + return; + boost::asio::post( + strand_, + [states = std::exchange(states_, {}), delivery = std::exchange(delivery_, {})]() + { + delivery(states); + } + ); + } + + private: + boost::asio::strand strand_; + std::vector paths_; + States states_; + Delivery delivery_; + }; + + /** + * @brief Hands only the reason of a completed operation to @p onEnded, nothing for an empty one. + */ + std::function reasonTo(FileTracking::TrackerRemote::OnEnded onEnded) + { + if (!onEnded) + return {}; + return [onEnded = std::move(onEnded)](SharedData::OperationCompleted const& completed) + { + onEnded(completed.reason); + }; + } + + /** + * @brief What an upload must find at its destination to replace it, from what the tracker expects there. A file + * the server lost counts as changed, the tracker asks before it recreates it. + */ + std::optional + comparedStateOf(std::optional const& expected) + { + if (!expected) + return std::nullopt; + if (!expected->file) + return SharedData::ComparedState{}; + return SharedData::ComparedState{ + .exists = true, + .type = SharedData::FileType::Regular, + .size = expected->file->size, + .mtime = expected->file->mtime, + .mustStillExist = true, + }; + } + + bool isDirectoryLocally(Paths::LocalPath const& path) + { + std::error_code statusError; + return std::filesystem::is_directory(std::filesystem::symlink_status(path.native(), statusError)); + } + + /** + * @brief Gathers the files an upload sent unchanged, told on the strands of its transfers, until it ended. + */ + class SentFiles : public std::enable_shared_from_this + { + public: + UploadOperation::OnSent recorder() + { + return [self = shared_from_this()]( + Paths::LocalPath const& source, std::optional const& unchanged + ) + { + const auto state = unchanged.transform( + [](UploadOperation::SentSource const& sent) + { + return FileTracking::SourceState{.size = sent.size, .modified = sent.modified}; + } + ); + std::scoped_lock lock{self->mutex_}; + self->files_.push_back({.path = source, .state = state}); + }; + } + + std::vector take() + { + std::scoped_lock lock{mutex_}; + return std::exchange(files_, {}); + } + + private: + std::mutex mutex_{}; + std::vector files_{}; + }; +} + +class Session::FileTrackingRemote : public FileTracking::BulkTrackerRemote +{ + public: + explicit FileTrackingRemote(std::weak_ptr session) + : session_{std::move(session)} + {} + + std::expected, std::string> stat(Paths::RemotePath const& path) override + { + return stateOf(path, /*followLinks=*/true); + } + + std::expected, std::string> + linkStat(Paths::RemotePath const& path) override + { + return stateOf(path, /*followLinks=*/false); + } + + std::expected download( + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t sizeBytes, + OnEnded onEnded + ) override + { + return queue( + [&](OperationQueue& queue, SecureShell::SftpSession& channel, Ids::OperationId const& operationId) + { + return queue.addDownloadOperation( + channel, + operationId, + destination, + source, + /*allowOverwrite=*/false, + /*isBigFile=*/sizeBytes > Constants::bigFileCutOff, + /*insertRefresh=*/false, + /*createMissingDirectories=*/false, + SharedData::OperationMode::PriorityQueued + ); + }, + reasonTo(std::move(onEnded)) + ); + } + + std::expected upload(FileTracking::UploadRequest const& request, OnUploadEnded onEnded) override + { + const bool isDirectory = isDirectoryLocally(request.source); + const auto sent = std::make_shared(); + std::optional queuedAs{}; + auto queued = queue( + [&request, &sent, &queuedAs]( + OperationQueue& queue, SecureShell::SftpSession& channel, Ids::OperationId const& operationId + ) + { + auto added = queue.addUploadOperation( + channel, + operationId, + request.source, + request.destination, + /*allowOverwrite=*/true, + /*isBigFile=*/false, + request.refreshListing, + request.createMissingDirectories, + SharedData::OperationMode::PriorityQueued, + /*sourceMayVanish=*/true, + comparedStateOf(request.expectedDestination), + sent->recorder(), + request.continuesPartialOf + ); + // The tracker asks the user about a server copy that changed meanwhile. + if (added) + { + queue.leaveChangedDestinationsToOwner(operationId); + queuedAs = operationId; + } + return added; + }, + forgettingSentUpload( + request.destination, endOfUpload(session_, request.destination, isDirectory, sent, std::move(onEnded)) + ) + ); + if (queued && queuedAs) + sentUploads_->insert_or_assign(request.destination.value(), *queuedAs); + return queued; + } + + void cancelUpload(Paths::RemotePath const& destination, FileTracking::PartialFile partial) override + { + const auto found = sentUploads_->find(destination.value()); + const auto session = session_.lock(); + if (found == sentUploads_->end() || !session) + return; + Log::info("FileTracking: stopping the upload to '{}'", destination.value()); + // The stopped upload ends after the tracker's call, which may still decide what follows it. + boost::asio::post( + *session->strand_, + [weak = session_, + operationId = found->second, + partialFile = partial == FileTracking::PartialFile::Kept ? UploadOperation::PartialFile::Kept + : UploadOperation::PartialFile::Discarded]() + { + if (const auto locked = weak.lock(); locked && locked->operationQueue_) + locked->operationQueue_->cancelUpload(operationId, partialFile); + } + ); + } + + std::expected + discardPartialUpload(Paths::RemotePath const& destination, OnEnded onEnded) override + { + return withChannel( + [&destination, + &onEnded](Session& session, SecureShell::SftpSession& channel) -> std::expected + { + session.operationQueue_->discardPartialUpload( + channel, + destination, + [onEnded = std::move(onEnded)]() + { + if (onEnded) + onEnded(SharedData::OperationCompletionReason::Completed); + } + ); + return {}; + } + ); + } + + std::expected + rename(Paths::RemotePath const& source, Paths::RemotePath const& destination, OnEnded onEnded) override + { + return queue( + [&](OperationQueue& queue, SecureShell::SftpSession& channel, Ids::OperationId const& operationId) + { + return queue.addRenameOperation( + channel, operationId, source, destination, SharedData::OperationMode::PriorityQueued + ); + }, + reasonTo(std::move(onEnded)) + ); + } + + std::expected remove(Paths::RemotePath const& path, OnEnded onEnded) override + { + return queue( + [&path](OperationQueue& queue, SecureShell::SftpSession& channel, Ids::OperationId const& operationId) + { + return queue.addDeleteOperation( + channel, + operationId, + path, + /*recursive=*/false, + /*insertRefresh=*/true, + SharedData::OperationMode::PriorityQueued + ); + }, + reasonTo(std::move(onEnded)) + ); + } + + std::expected + uploadAll(std::vector const& requests, OnUploadEntryEnded onEnded) override + { + return withChannel( + [&requests, + &onEnded](Session& session, SecureShell::SftpSession& channel) -> std::expected + { + const auto sent = std::make_shared(); + SharedData::BulkUploadRequest bulk{ + .allowOverwrite = true, + .mode = SharedData::OperationMode::PriorityQueued, + }; + bulk.entries.reserve(requests.size()); + for (auto const& request : requests) + { + std::error_code sizeError; + const auto sizeBytes = std::filesystem::file_size(request.source.native(), sizeError); + bulk.entries.push_back({ + .source = request.source, + .destination = request.destination, + .sizeBytes = sizeError ? 0u : static_cast(sizeBytes), + .mayOverwrite = true, + .comparedDestination = comparedStateOf(request.expectedDestination), + }); + } + auto& queue = *session.operationQueue_; + const auto cardId = Ids::generateOperationId(); + queue.addBulkUploadOperation( + channel, + std::move(bulk), + [](std::size_t) + { + return Ids::generateOperationId(); + }, + cardId, + /*sourceMayVanish=*/true, + sent->recorder() + ); + // The tracker decides anew what goes once the session is back. + queue.discardResumeBackup(cardId); + queue.leaveChangedDestinationsToOwner(cardId); + queue.whenCompleted( + cardId, + [session = session.weak_from_this(), + onEnded = std::move(onEnded), + sent, + sources = requests | std::views::transform(&FileTracking::UploadRequest::source) | + std::ranges::to(), + destinations = requests | std::views::transform(&FileTracking::UploadRequest::destination) | + std::ranges::to()](SharedData::OperationCompleted const& completed) + { + endUploads(session, completed, sources, destinations, sent->take(), onEnded); + } + ); + session.resetQueueThrottle(); + return {}; + } + ); + } + + std::expected + removeAll(std::vector const& paths, OnEntryEnded onEnded) override + { + return withChannel( + [&paths, &onEnded](Session& session, SecureShell::SftpSession& channel) -> std::expected + { + SharedData::BulkDeleteRequest bulk{ + .insertRefresh = true, + .mode = SharedData::OperationMode::PriorityQueued, + .listedContentOnly = true, + }; + bulk.entries.reserve(paths.size()); + for (auto const& path : paths) + bulk.entries.push_back({.target = path}); + auto& queue = *session.operationQueue_; + const auto cardId = Ids::generateOperationId(); + if (queue.addBulkDeleteOperation(channel, bulk, cardId) == 0) + return std::unexpected{std::string{"The server refused every delete"}}; + queue.discardResumeBackup(cardId); + queue.whenCompleted( + cardId, + [onEnded = std::move(onEnded), paths](SharedData::OperationCompleted const& completed) + { + const auto failed = completed.failedEntries | std::views::keys | std::ranges::to(); + endEntries( + completed, + paths.size(), + [&failed, &paths](std::size_t index) + { + return failed.contains(paths[index].value()); + }, + onEnded + ); + } + ); + session.resetQueueThrottle(); + return {}; + } + ); + } + + private: + /** + * @brief Hands the end of an upload to @p onEnded, a completed one of anything but a directory once the server + * told what it has at @p destination, with the files @p sent tells of. Nothing for an empty @p onEnded. + */ + static std::function endOfUpload( + std::weak_ptr session, + Paths::RemotePath destination, + bool isDirectory, + std::shared_ptr sent, + OnUploadEnded onEnded + ) + { + if (!onEnded) + return {}; + return [session = std::move(session), + destination = std::move(destination), + isDirectory, + sent = std::move(sent), + onEnded = std::move(onEnded)](SharedData::OperationCompleted const& completed) + { + const bool uploaded = completed.reason == SharedData::OperationCompletionReason::Completed; + if (!uploaded || isDirectory) + return onEnded({ + .reason = completed.reason, + .changedOnServer = + completed.error && completed.error->type == SharedData::OperationErrorType::ChangedSinceCompare, + .sentFiles = uploaded ? sent->take() : std::vector{}, + }); + askStatesThen( + session, + {destination}, + [onEnded, sentFiles = sent->take()](DestinationStates::States const& states) + { + onEnded({ + .reason = SharedData::OperationCompletionReason::Completed, + .destinationState = states.front(), + .sentFiles = sentFiles, + }); + } + ); + }; + } + + /** + * @brief Hands each upload of a bulk that ended as @p completed its end: a completed one once the server told + * what it has at its destination, of @p destinations by index, with its source when it is among @p sent. + */ + static void endUploads( + std::weak_ptr const& session, + SharedData::OperationCompleted const& completed, + std::vector const& sources, + std::vector const& destinations, + std::vector const& sent, + OnUploadEntryEnded const& onEnded + ) + { + if (completed.reason != SharedData::OperationCompletionReason::Completed) + { + for (std::size_t index = 0; index < destinations.size(); ++index) + onEnded(index, {.reason = completed.reason}); + return; + } + const auto skipped = completed.indexedFailures | + std::views::transform( + [](SharedData::IndexedFailure const& failure) + { + return std::pair{static_cast(failure.index), failure.type}; + } + ) | + std::ranges::to(); + std::vector uploaded; + std::vector uploadedDestinations; + for (std::size_t index = 0; index < destinations.size(); ++index) + { + if (const auto failure = skipped.find(index); failure != skipped.end()) + onEnded( + index, + { + .reason = SharedData::OperationCompletionReason::Failed, + .changedOnServer = failure->second == SharedData::OperationErrorType::ChangedSinceCompare, + } + ); + else + { + uploaded.push_back(index); + uploadedDestinations.push_back(destinations[index]); + } + } + if (uploaded.empty()) + return; + const auto sentBySource = sent | + std::views::transform( + [](FileTracking::SentFile const& file) + { + return std::pair{file.path.toUtf8(), file}; + } + ) | + std::ranges::to(); + const auto sentFilesOf = [&sentBySource, &sources](std::size_t index) + { + const auto found = sentBySource.find(sources[index].toUtf8()); + if (found == sentBySource.end()) + return std::vector{}; + return std::vector{found->second}; + }; + auto uploadedSent = uploaded | std::views::transform(sentFilesOf) | std::ranges::to(); + askStatesThen( + session, + std::move(uploadedDestinations), + [onEnded, + uploaded = std::move(uploaded), + uploadedSent = std::move(uploadedSent)](DestinationStates::States const& states) + { + for (std::size_t position = 0; position < uploaded.size(); ++position) + onEnded( + uploaded[position], + { + .reason = SharedData::OperationCompletionReason::Completed, + .destinationState = states[position], + .sentFiles = uploadedSent[position], + } + ); + } + ); + } + + /** + * @brief Asks the server what it has at @p paths on the strand of one of its SFTP channels, so the session strand + * that runs the queue does not wait for it. @p delivery gets the states on the session strand. + */ + static void askStatesThen( + std::weak_ptr const& session, + std::vector paths, + DestinationStates::Delivery delivery + ) + { + const auto locked = session.lock(); + if (!locked) + return delivery(DestinationStates::States(paths.size())); + auto states = std::make_shared(*locked->strand_, std::move(paths), std::move(delivery)); + // Without a channel the states go unasked once they are dropped here. + if (const auto channel = locked->anySftpChannel()) + channel->perform( + [states, weakChannel = std::weak_ptr{channel}]() + { + if (const auto channel = weakChannel.lock()) + states->askInStrand(*channel); + } + ); + } + + /** + * @brief What the server has at @p path, a symbolic link followed when @p followLinks is set. + */ + std::expected, std::string> + stateOf(Paths::RemotePath const& path, bool followLinks) + { + return withChannel( + [&path, followLinks]( + Session&, SecureShell::SftpSession& channel + ) -> std::expected, std::string> + { + auto future = followLinks ? channel.stat(path) : channel.lstat(path); + if (future.wait_for(Session::futureTimeout) != std::future_status::ready) + return std::unexpected{fmt::format("Asking the server about '{}' timed out", path.value())}; + const auto result = future.get(); + if (!result) + { + if (result.error().sftpError == SSH_FX_NO_SUCH_FILE) + return std::nullopt; + return std::unexpected{result.error().toString()}; + } + return FileTracking::ServerState{.mtime = result->mtime, .size = result->size}; + } + ); + } + + /** + * @brief Calls @p function with the session and one of its SFTP channels, an error when it has none. + */ + template + auto withChannel(FunctionT&& function) const -> std::invoke_result_t + { + const auto session = session_.lock(); + if (!session || !session->operationQueue_) + return std::unexpected{std::string{"The session has ended"}}; + const auto channel = session->anySftpChannel(); + if (!channel) + return std::unexpected{std::string{"The session has no SFTP channel"}}; + return std::forward(function)(*session, *channel); + } + + /** + * @brief Queues an operation with @p add under a new id without a resume backup, @p onCompleted hears when it + * ended. + */ + template + std::expected + queue(AddT&& add, std::function onCompleted) + { + return withChannel( + [&add, + &onCompleted](Session& session, SecureShell::SftpSession& channel) -> std::expected + { + const auto operationId = Ids::generateOperationId(); + const auto added = add(*session.operationQueue_, channel, operationId); + if (!added) + return std::unexpected{added.error().toString()}; + // Like the bulks: the watches end with the connection, a reconnect queues nothing of theirs again. + session.operationQueue_->discardResumeBackup(operationId); + if (onCompleted) + session.operationQueue_->whenCompleted(operationId, std::move(onCompleted)); + session.resetQueueThrottle(); + return {}; + } + ); + } + + /** + * @brief Hands the end of the single upload to @p destination to @p ended, once it can no longer be stopped. + */ + std::function forgettingSentUpload( + Paths::RemotePath const& destination, + std::function ended + ) const + { + return [sentUploads = std::weak_ptr{sentUploads_}, destination = destination.value(), ended = std::move(ended)]( + SharedData::OperationCompleted const& completed + ) + { + if (const auto uploads = sentUploads.lock()) + if (const auto found = uploads->find(destination); + found != uploads->end() && found->second == completed.operationId) + uploads->erase(found); + if (ended) + ended(completed); + }; + } + + private: + std::weak_ptr session_; + /** @brief The single uploads on their way by destination, the tracker may stop them, see cancelUpload(). */ + std::shared_ptr> sentUploads_{ + std::make_shared>() + }; +}; + +void Session::startFileTracking() +{ + if (!fileTrackingServices_.instances || !fileTrackingServices_.localWatches) + { + Log::warn("Session '{}' has no file tracking, it was not started", id_.value()); + return; + } + tracker_ = std::make_unique( + *strand_, + *fileTrackingServices_.instances, + *fileTrackingServices_.localWatches, + std::make_unique( + *strand_, std::make_unique(weak_from_this()) + ), + std::make_unique(*hub_, id_), + downloadTempFileSuffix_, + fileTrackingServices_.defaultOptions + ); +} + +void Session::stopFileTracking() +{ + if (tracker_) + tracker_->stopAll(); +} + +void Session::onConnectionLost() +{ + if (tracker_) + tracker_->connectionLost(); +} + +namespace +{ + /** + * @brief Answers @p reply with {success: true} merged with @p values, or with the error of @p result. + */ + template + void replyWith(RpcHelper::RpcOnce const& reply, std::expected const& result, nlohmann::json values) + { + if (!result) + return reply.error(result.error()); + values["success"] = true; + reply(std::move(values)); + } +} + +void Session::registerRpcFileTracker() +{ + const auto name = [this](std::string_view function) + { + return fmt::format("FileTracker::{}::{}", id_.value(), function); + }; + // The session, which holds the tracker for as long as it is used, or nothing after answering that it is gone. + const auto withTracker = [weak = weak_from_this()](RpcHelper::RpcOnce const& reply) -> std::shared_ptr + { + auto self = weak.lock(); + if (!self || !self->tracker_) + { + reply.error("File tracking is not available in this session"); + return nullptr; + } + return self; + }; + + on(name("open")) + .perform( + [withTracker](RpcHelper::RpcOnce&& reply, std::string const& path, double sizeBytes, bool watch) + { + const auto self = withTracker(reply); + if (!self) + return; + const auto remotePath = RpcHelper::parseRemotePath(path); + if (!remotePath) + return reply.error(remotePath.error()); + self->tracker_->open( + *remotePath, + static_cast(std::max(sizeBytes, 0.0)), + watch, + [reply = std::make_shared(std::move(reply))]( + std::expected const& + opened + ) + { + if (opened) + return (*reply)({ + {"success", true}, + {"instanceId", opened->instanceId.value()}, + {"localPath", opened->localPath.toUtf8()}, + }); + if (opened.error()) + return reply->error(*opened.error()); + (*reply)({{"success", false}}); + } + ); + } + ); + + on(name("checkUploadAndWatch")) + .perform( + [withTracker](RpcHelper::RpcOnce&& reply, std::string const& item, std::string const& directory) + { + const auto self = withTracker(reply); + if (!self) + return; + const auto localItem = RpcHelper::parseLocalPath(item); + const auto remoteDirectory = RpcHelper::parseRemotePath(directory); + if (!localItem || !remoteDirectory) + return reply.error(!localItem ? localItem.error() : remoteDirectory.error()); + const auto target = self->tracker_->checkUploadAndWatch(*localItem, *remoteDirectory); + const auto values = target + ? nlohmann::json{{"item", target->item.toUtf8()}, {"destination", target->destination.value()}} + : nlohmann::json::object(); + replyWith(reply, target, values); + } + ); + + on(name("uploadAndWatch")) + .perform( + [withTracker](RpcHelper::RpcOnce&& reply, std::string const& item, std::string const& destination) + { + const auto self = withTracker(reply); + if (!self) + return; + const auto localItem = RpcHelper::parseLocalPath(item); + const auto remoteDestination = RpcHelper::parseRemotePath(destination); + if (!localItem || !remoteDestination) + return reply.error(!localItem ? localItem.error() : remoteDestination.error()); + replyWith( + reply, self->tracker_->uploadAndWatch(*localItem, *remoteDestination), nlohmann::json::object() + ); + } + ); + + on(name("stop")) + .perform( + [withTracker](RpcHelper::RpcOnce&& reply, std::string const& watchId) + { + if (const auto self = withTracker(reply)) + { + self->tracker_->stop(Ids::makeWatchId(watchId)); + reply({{"success", true}}); + } + } + ); + + on(name("setOption")) + .perform( + [withTracker]( + RpcHelper::RpcOnce&& reply, std::string const& watchId, SharedData::WatchOption option, bool value + ) + { + if (const auto self = withTracker(reply)) + { + self->tracker_->setOption(Ids::makeWatchId(watchId), option, value); + reply({{"success", true}}); + } + } + ); + + on(name("discard")) + .perform( + [withTracker](RpcHelper::RpcOnce&& reply, std::string const& instanceId, std::string const& path) + { + const auto self = withTracker(reply); + if (!self) + return; + const auto localPath = RpcHelper::parseLocalPath(path); + if (!localPath) + return reply.error(localPath.error()); + const auto discarded = self->tracker_->discard(Ids::makeInstanceId(instanceId), *localPath); + if (!discarded) + Log::error("FileTracking: discarding '{}' failed: {}", localPath->toUtf8(), discarded.error()); + replyWith( + reply, + discarded, + {{"instanceDestroyed", discarded == FileTracking::DiscardOutcome::InstanceDestroyed}} + ); + } + ); + + on(name("answer")) + .perform( + [withTracker]( + RpcHelper::RpcOnce&& reply, std::string const& questionId, SharedData::ServerChangedAnswer answer + ) + { + if (const auto self = withTracker(reply)) + { + self->tracker_->answer(Ids::makeQuestionId(questionId), answer); + reply({{"success", true}}); + } + } + ); +} diff --git a/backend/source/backend/session_manager.cpp b/backend/source/backend/session_manager.cpp index f572a535..763d4dde 100644 --- a/backend/source/backend/session_manager.cpp +++ b/backend/source/backend/session_manager.cpp @@ -17,6 +17,7 @@ #include #include #include +#include using namespace Detail; @@ -113,6 +114,49 @@ void SessionManager::preventLocalDeletion(bool prevent) localDeletionPrevented_->store(prevent); } +void SessionManager::startFileTracking(std::filesystem::path const& temporaryDownloadsDirectory) +{ + temporaryInstances_ = + std::make_unique(*strand_, temporaryDownloadsDirectory); + temporaryInstances_->scheduleStartupCleanup(); + localWatches_ = std::make_unique(*strand_, temporaryDownloadsDirectory); + registerRpcOrphanedInstances(); +} + +void SessionManager::registerRpcOrphanedInstances() +{ + on("FileTracking::countOrphaned") + .perform( + [this](RpcHelper::RpcOnce&& reply) + { + reply({{"success", true}, {"count", temporaryInstances_->countOrphaned()}}); + } + ); + on("FileTracking::forceCleanup") + .perform( + [this](RpcHelper::RpcOnce&& reply) + { + reply({{"success", true}, {"removed", temporaryInstances_->removeOrphaned()}}); + } + ); +} + +void SessionManager::fileTrackingDefaults(Persistence::FileTrackingOptions const& options) +{ + within_strand_do( + [this, + defaults = SharedData::WatchOptions{ + .autoReupload = options.autoReupload, + .moveRemoteOnLocalMove = options.moveRemoteOnLocalMove, + .deleteRemoteOnLocalDelete = options.deleteRemoteOnLocalDelete, + .checkRemoteChanges = options.checkRemoteChanges.value_or(true), + }]() + { + fileTrackingDefaults_ = defaults; + } + ); +} + SessionManager::~SessionManager() { for (auto& [id, session] : sessions_) @@ -193,7 +237,7 @@ void SessionManager::addSession( auto self = weak.lock(); if (!self) return; - self->removeSession(sessionId); + self->removeSession(sessionId, /*connectionLost=*/true); // Notify frontend of session loss: self->wnd_->runInJavascriptThread( @@ -231,7 +275,16 @@ void SessionManager::addSession( *hub_, sessionOptions.sftpOptions.value(), &bulkResumeRegistry_, - localOperationQueue_ + localOperationQueue_, + FileTracking::TrackerServices{ + .instances = temporaryInstances_.get(), + .localWatches = localWatches_.get(), + .defaultOptions = + [this]() + { + return fileTrackingDefaults_; + }, + } ); const auto emplaced = sessions_.emplace(sessionId, session); if (!emplaced.second) @@ -253,14 +306,16 @@ void SessionManager::addSession( ); } -void SessionManager::removeSession(Ids::SessionId sessionId) +void SessionManager::removeSession(Ids::SessionId sessionId, bool connectionLost) { within_strand_do( - [this, sessionId]() + [this, sessionId, connectionLost]() { if (auto iter = sessions_.find(sessionId); iter != sessions_.end()) { Log::info("Removing session with id: {}", sessionId.value()); + if (connectionLost) + iter->second->onConnectionLost(); iter->second->stop(); sessions_.erase(iter); } @@ -278,9 +333,23 @@ void SessionManager::registerRpc() registerRpcSessionDisconnect(); registerRpcAdoptBulkResume(); registerRpcDiscardBulkResume(); + registerRpcStopFileTracking(); localOperationQueue_->registerRpc(); } +void SessionManager::registerRpcStopFileTracking() +{ + on("FileTracker::stopAll") + .perform( + [this](RpcHelper::RpcOnce&& reply) + { + for (auto const& session : std::views::values(sessions_)) + session->stopFileTracking(); + reply({{"success", true}}); + } + ); +} + void SessionManager::registerRpcSessionConnect() { on("SessionManager::connect") diff --git a/backend/source/backend/sftp/bulk_transfer_operation.cpp b/backend/source/backend/sftp/bulk_transfer_operation.cpp index 4fb00573..66c83ae1 100644 --- a/backend/source/backend/sftp/bulk_transfer_operation.cpp +++ b/backend/source/backend/sftp/bulk_transfer_operation.cpp @@ -298,8 +298,15 @@ std::expected BulkTransferOperation::stepSlo auto result = slot.child->workInStrand(); if (result.has_value()) { - if (result.value() == WorkStatus::Complete) - finishSlot(slot, true); + if (result.value() != WorkStatus::Complete) + continue; + // A source that may vanish and was gone ends cancelled: no error, but its owner hears it did not go. + const bool transferred = slot.child->state() != OperationState::Canceled; + if (!transferred) + indexedFailures_.push_back( + {.index = static_cast(slot.entryIndex), .type = ErrorType::FileNotFound} + ); + finishSlot(slot, transferred); continue; } @@ -320,7 +327,13 @@ std::expected BulkTransferOperation::stepSlo continue; } - Log::error("{}: Transfer failed for: {}: {}", logName(), slot.paths.remote.value(), error.toString()); + Log::log( + failureLevelOf(error), + "{}: Transfer failed for: {}: {}", + logName(), + slot.paths.remote.value(), + error.toString() + ); if (isSkippableError(error)) { // Not critical for the bulk as a whole: record the entry for the user and move on. diff --git a/backend/source/backend/sftp/bulk_upload_operation.cpp b/backend/source/backend/sftp/bulk_upload_operation.cpp index 5412e406..f5407df1 100644 --- a/backend/source/backend/sftp/bulk_upload_operation.cpp +++ b/backend/source/backend/sftp/bulk_upload_operation.cpp @@ -1,9 +1,12 @@ #include #include #include +#include #include +#include #include +#include #include namespace @@ -37,6 +40,23 @@ SharedData::OperationType BulkUploadOperation::type() const return SharedData::OperationType::BulkUpload; } +std::expected BulkUploadOperation::work() +{ + if (individualOptions_.sourceMayVanish && scanning()) + { + // A root that cannot be checked is not gone, its scan fails. + if (const auto status = Utility::checkedSymlinkStatus(common().localPath.native()); + status && !status->has_value()) + { + Log::info("BulkUploadOperation: '{}' is gone, nothing to upload.", common().localPath.toUtf8()); + // Ends without a transfer, so its owner does not take the destination for uploaded. + enterState(OperationState::Canceled); + return WorkStatus::Complete; + } + } + return BulkTransferOperation::work(); +} + std::expected BulkUploadOperation::prepareRootInStrand(SecureShell::SftpSession& sftp, SharedData::DirectoryEntry const& root) { diff --git a/backend/source/backend/sftp/compared_state_check.cpp b/backend/source/backend/sftp/compared_state_check.cpp index b33e922f..91c15cb7 100644 --- a/backend/source/backend/sftp/compared_state_check.cpp +++ b/backend/source/backend/sftp/compared_state_check.cpp @@ -15,7 +15,7 @@ namespace ComparedStateCheck { Operation::Error changedError(std::string const& path) { - Log::warn("'{}' changed after the sync compared it, it is neither replaced nor deleted.", path); + Log::warn("'{}' changed since it was compared, it is neither replaced nor deleted.", path); return Operation::Error{ .type = Operation::ErrorType::ChangedSinceCompare, .extraInfo = fmt::format("'{}' changed after the compare", path), diff --git a/backend/source/backend/sftp/download_operation.cpp b/backend/source/backend/sftp/download_operation.cpp index 2d38959f..b2705058 100644 --- a/backend/source/backend/sftp/download_operation.cpp +++ b/backend/source/backend/sftp/download_operation.cpp @@ -199,7 +199,11 @@ std::expected DownloadO const auto finalizeResult = finalizeInStrand(); if (!finalizeResult.has_value()) { - Log::error("DownloadOperation: Failed to finalize operation: {}", finalizeResult.error().toString()); + Log::log( + failureLevelOf(finalizeResult.error()), + "DownloadOperation: Failed to finalize operation: {}", + finalizeResult.error().toString() + ); return enterErrorState(finalizeResult.error()); } state_ = Completed; diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 64ac2e08..609ad5c1 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -68,6 +68,29 @@ namespace return parent ? *std::move(parent) : PathT{}; } + bool changedSinceCompare(SharedData::OperationError const& error) + { + return error.type == SharedData::OperationErrorType::ChangedSinceCompare; + } + + /** + * @brief @p completed as it ends an operation whose owner asks about changed destinations itself, see + * OperationQueue::leaveChangedDestinationsToOwner. + */ + void leaveChangesToOwner(OperationQueue::OperationCompleted& completed) + { + if (completed.reason == SharedData::OperationCompletionReason::Failed && completed.error && + changedSinceCompare(*completed.error)) + completed.reason = SharedData::OperationCompletionReason::Canceled; + std::erase_if( + completed.failedEntries, + [](auto const& failed) + { + return changedSinceCompare(failed.second); + } + ); + } + OperationQueue::OperationCompleted makeCompletedOperation( OperationQueue::CompletionReason reason, Ids::OperationId operationId, @@ -189,6 +212,26 @@ namespace .symlinkHandling = opts.symlinkHandling.value_or(def.symlinkHandling), }; } + + /** + * @brief Why the upload of @p localPath is refused, its status (@p status, @p statusError) unreadable. A source + * that may vanish and is gone is nothing that failed. + */ + Operation::Error refusedSource( + Paths::LocalPath const& localPath, + std::filesystem::file_status status, + std::error_code const& statusError, + bool sourceMayVanish + ) + { + if (sourceMayVanish && status.type() == std::filesystem::file_type::not_found) + { + Log::debug("Not uploading '{}', it is gone.", localPath.toUtf8()); + return Operation::Error{.type = Operation::ErrorType::FileNotFound}; + } + Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), Utility::errorMessage(statusError)); + return Operation::Error{.type = Operation::ErrorType::FileStatFailed}; + } } OperationQueue::OperationQueue( @@ -296,6 +339,28 @@ auto OperationQueue::makeBulkProgressCallback(std::string_view eventName, Ids::O } void OperationQueue::cancelAll() +{ + within_strand_do( + [weak = weak_from_this()]() + { + auto self = weak.lock(); + if (!self) + return; + + // Taken out first: an owner hearing of its cancel may queue anew. + auto cancelled = std::exchange(self->priorityOperations_, {}); + std::ranges::move(std::exchange(self->operations_, {}), std::back_inserter(cancelled)); + auto waiting = std::exchange(self->syncScanCallbacks_, {}); + for (auto& [id, callback] : waiting) + callback(std::unexpected(Utility::localizedMessage("backend.sync.scanCancelled"))); + for (auto& [id, operation] : cancelled) + self->completeCancelled(id, *operation); + Log::info("All operations in the queue have been canceled."); + } + ); +} + +void OperationQueue::dropAll() { within_strand_do( [weak = weak_from_this()]() @@ -309,7 +374,7 @@ void OperationQueue::cancelAll() auto waiting = std::exchange(self->syncScanCallbacks_, {}); for (auto& [id, callback] : waiting) callback(std::unexpected(Utility::localizedMessage("backend.sync.scanCancelled"))); - Log::info("All operations in the queue have been canceled."); + Log::info("All operations in the queue have been dropped."); } ); } @@ -323,22 +388,87 @@ void OperationQueue::cancel(Ids::OperationId id) if (!self) return; - auto cancelPredicate = [id](auto& op) - { - bool isMatch = op.first == id; - if (isMatch) - std::ignore = op.second->cancel(true); - return isMatch; - }; - - const auto erasedFromRegular = std::erase_if(self->operations_, cancelPredicate); - if (erasedFromRegular == 0) - std::erase_if(self->priorityOperations_, cancelPredicate); + const auto cancelled = self->takeOperation(id); self->failSyncScan(id, Utility::localizedMessage("backend.sync.scanCancelled")); + if (cancelled) + self->completeCancelled(id, *cancelled); + } + ); +} + +void OperationQueue::cancelUpload(Ids::OperationId id, UploadOperation::PartialFile partial) +{ + within_strand_do( + [weak = weak_from_this(), id = std::move(id), partial]() + { + auto self = weak.lock(); + if (!self) + return; + + const auto cancelled = self->takeOperation(id); + if (!cancelled) + return; + if (cancelled->type() == SharedData::OperationType::Upload) + static_cast(*cancelled).partialFileOnCancel(partial); + self->completeCancelled(id, *cancelled); + } + ); +} + +void OperationQueue::discardPartialUpload( + SecureShell::SftpSession& sftp, + Paths::RemotePath const& remotePath, + std::function onDone +) +{ + const auto suffix = Persistence::effectiveTempFileSuffix( + sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}).tempFileSuffix + ); + const auto partial = remotePath.withFilenameSuffix(suffix); + if (!partial) + { + boost::asio::post(*strand_, std::move(onDone)); + return; + } + sftp.perform( + [weak = sftp.weak_from_this(), partial = *partial, strand = strand_, onDone = std::move(onDone)]() + { + if (const auto session = weak.lock()) + { + if (const auto removed = session->removeFileInStrand(partial); removed.has_value()) + Log::info("OperationQueue: removed the partial upload '{}'.", partial.value()); + } + boost::asio::post(*strand, onDone); } ); } +std::unique_ptr OperationQueue::takeOperation(Ids::OperationId const& id) +{ + for (auto* queue : {&operations_, &priorityOperations_}) + { + const auto found = std::ranges::find_if( + *queue, + [&id](auto const& queued) + { + return queued.first == id; + } + ); + if (found == queue->end()) + continue; + auto operation = std::move(found->second); + queue->erase(found); + return operation; + } + return nullptr; +} + +void OperationQueue::completeCancelled(Ids::OperationId const& id, Operation& operation) +{ + std::ignore = operation.cancel(true); + completeOperation(makeCompletedOperation(OperationQueue::CompletionReason::Canceled, id, operation)); +} + void OperationQueue::moveOperation(Ids::OperationId operationId, std::size_t newIndex) { within_strand_do( @@ -437,8 +567,14 @@ void OperationQueue::completeOperation(SharedData::OperationCompleted&& operatio if (!self) return; + if (self->changedDestinationsLeftToOwner_.erase(operationCompleted.operationId) > 0) + leaveChangesToOwner(operationCompleted); if (operationCompleted.error) - Log::error("Operation failed: {}", operationCompleted.error->toString()); + Log::log( + Operation::failureLevelOf(*operationCompleted.error), + "Operation failed: {}", + operationCompleted.error->toString() + ); // Drop any resume backup tied to this operation, it has run to // completion (success, failure, or cancel) and no longer needs @@ -448,6 +584,8 @@ void OperationQueue::completeOperation(SharedData::OperationCompleted&& operatio if (self->completionObserver_) self->completionObserver_(operationCompleted); RpcHelper::callRemoteSafely(*self->hub_, self->rpcName("onOperationCompleted"), operationCompleted); + if (auto callback = self->completionCallbacks_.extract(operationCompleted.operationId); !callback.empty()) + callback.mapped()(operationCompleted); } ); } @@ -457,6 +595,24 @@ void OperationQueue::setCompletionObserver(std::function callback +) +{ + completionCallbacks_.insert_or_assign(operationId, std::move(callback)); +} + +void OperationQueue::discardResumeBackup(Ids::OperationId const& operationId) +{ + bulkResumeStash_.erase(operationId); +} + +void OperationQueue::leaveChangedDestinationsToOwner(Ids::OperationId const& operationId) +{ + changedDestinationsLeftToOwner_.insert(operationId); +} + void OperationQueue::dumpBulkResumes(BulkResumeRegistry& registry) { if (bulkResumeStash_.empty()) @@ -671,13 +827,20 @@ bool OperationQueue::workQueue(std::deque(index)); + return finished; + }; + bool moreWork = false; for (auto& item : results) { - auto& [id, operation] = queue[item.queueIndex]; - if (!item.result.has_value()) { + const auto [id, operation] = takeFinished(item.queueIndex); failSyncScan( id, Utility::localizedMessage( @@ -689,16 +852,19 @@ bool OperationQueue::workQueue(std::deque(item.queueIndex)); return true; } const auto workStatus = item.result.value(); if (workStatus == Operation::WorkStatus::Complete) { + const auto [id, operation] = takeFinished(item.queueIndex); deliverSyncScan(id, *operation); - completeOperation(makeCompletedOperation(OperationQueue::CompletionReason::Completed, id, *operation)); - queue.erase(queue.begin() + static_cast(item.queueIndex)); + // An upload whose source may vanish and was gone ends cancelled, nothing went. + const auto reason = operation->state() == SharedData::OperationState::Canceled + ? OperationQueue::CompletionReason::Canceled + : OperationQueue::CompletionReason::Completed; + completeOperation(makeCompletedOperation(reason, id, *operation)); return true; } else if (workStatus == Operation::WorkStatus::MoreWork) @@ -1413,7 +1579,11 @@ std::expected OperationQueue::addUploadOperation( bool isBigFile, bool insertRefresh, bool createMissingDirectories, - SharedData::OperationMode mode + SharedData::OperationMode mode, + bool sourceMayVanish, + std::optional comparedDestination, + UploadOperation::OnSent onSent, + std::optional adoptPartialOf ) { // Assumed in strand @@ -1426,10 +1596,7 @@ std::expected OperationQueue::addUploadOperation( const auto symStat = std::filesystem::symlink_status(localNativePath, ec); if (ec) - { - Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), Utility::errorMessage(ec)); - return std::unexpected(Operation::Error{.type = Operation::ErrorType::FileStatFailed}); - } + return std::unexpected(refusedSource(localPath, symStat, ec, sourceMayVanish)); const bool isSymlink = std::filesystem::is_symlink(symStat); std::filesystem::file_status stat = symStat; @@ -1437,10 +1604,7 @@ std::expected OperationQueue::addUploadOperation( { stat = std::filesystem::status(localNativePath, ec); if (ec) - { - Log::error("Failed to stat local file '{}': {}", localPath.toUtf8(), Utility::errorMessage(ec)); - return std::unexpected(Operation::Error{.type = Operation::ErrorType::FileStatFailed}); - } + return std::unexpected(refusedSource(localPath, stat, ec, sourceMayVanish)); } auto transferOptions = sftpOpts_.uploadOptions.value_or(Persistence::UploadOptions{}); @@ -1459,6 +1623,10 @@ std::expected OperationQueue::addUploadOperation( // A dropped or tracked file comes without a size, the slow path reports no speed. opts.bigFileOptimized = isBigFile || (!isSymlink && localFileSize(localNativePath) >= Constants::bigFileCutOff); opts.createMissingDirectories = createMissingDirectories; + opts.sourceMayVanish = sourceMayVanish; + opts.comparedDestination = std::move(comparedDestination); + opts.onSent = std::move(onSent); + opts.adoptPartialOf = std::move(adoptPartialOf); opts.progressCallback = [weak = weak_from_this(), operationId, name = rpcName("onUploadProgress")]( auto min, auto max, auto current, auto bytesPerSecond ) @@ -1510,13 +1678,16 @@ std::expected OperationQueue::addUploadOperation( } else if (stat.type() == std::filesystem::file_type::directory) { + auto individualOptions = resolveUploadOptions(transferOptions, resolvedTimeout); + individualOptions.sourceMayVanish = sourceMayVanish; + individualOptions.onSent = std::move(onSent); auto bulk = std::make_unique( sftp.weak_from_this(), BulkUploadOperation::BulkUploadOperationOptions{ .overallProgressCallback = makeBulkProgressCallback("onBulkUploadProgress", operationId), .remotePath = remotePath, .localPath = localPath, - .individualOptions = resolveUploadOptions(transferOptions, resolvedTimeout), + .individualOptions = std::move(individualOptions), .failFast = transferOptions.failFast.value_or(false), .concurrency = bulkConcurrency_, } @@ -1568,7 +1739,9 @@ std::size_t OperationQueue::addBulkUploadOperation( SecureShell::SftpSession& sftp, SharedData::BulkUploadRequest request, std::function const& operationIdFor, - Ids::OperationId const& bulkCardId + Ids::OperationId const& bulkCardId, + bool sourceMayVanish, + UploadOperation::OnSent const& onSent ) { // Assumed in strand. @@ -1592,7 +1765,10 @@ std::size_t OperationQueue::addBulkUploadOperation( /*isBigFile*/ false, unit.insertRefresh, /*createMissingDirectories*/ true, - unit.mode + unit.mode, + sourceMayVanish, + std::nullopt, + onSent ); if (!result) { @@ -1631,6 +1807,8 @@ std::size_t OperationQueue::addBulkUploadOperation( auto uploadOptions = resolveUploadOptions(transferOptions, resolvedTimeout); uploadOptions.createMissingDirectories = true; + uploadOptions.sourceMayVanish = sourceMayVanish; + uploadOptions.onSent = onSent; auto bulk = std::make_unique( sftp.weak_from_this(), BulkUploadOperation::BulkUploadOperationOptions{ diff --git a/backend/source/backend/sftp/upload_operation.cpp b/backend/source/backend/sftp/upload_operation.cpp index dcc42af6..7048c043 100644 --- a/backend/source/backend/sftp/upload_operation.cpp +++ b/backend/source/backend/sftp/upload_operation.cpp @@ -6,12 +6,34 @@ #include #include +#include #include #include #include #include +#include +#include +#include #include +#include + +namespace +{ + /** + * @brief What the regular file at @p path is like now, nothing when that cannot be read. + */ + std::optional sourceStateOf(std::filesystem::path const& path) + { + std::error_code sizeError{}; + const auto size = std::filesystem::file_size(path, sizeError); + std::error_code timeError{}; + const auto modified = std::filesystem::last_write_time(path, timeError); + if (sizeError || timeError) + return std::nullopt; + return UploadOperation::SentSource{.size = static_cast(size), .modified = modified}; + } +} UploadOperation::UploadOperation(std::weak_ptr sftp, UploadOperationOptions options) : Operation{} @@ -80,6 +102,14 @@ std::expected UploadOperati } case (Preparing): { + if (options_.sourceMayVanish && sourceIsGone()) + { + Log::info("UploadOperation: '{}' is gone, nothing to upload.", options_.localPath.toUtf8()); + // Ends without a transfer, so its owner does not take the destination for uploaded. + state_ = Canceled; + options_.progressCallback(1, 1, 1, 0); + return WorkStatus::Complete; + } if (const auto opened = openLocalSource(); !opened) return enterErrorState(opened.error()); if (!isSymlink_) @@ -87,7 +117,11 @@ std::expected UploadOperati const auto prepareResult = prepareInStrand(*sftp); if (!prepareResult.has_value()) { - Log::error("UploadOperation: Failed to prepare operation: {}", prepareResult.error().toString()); + Log::log( + failureLevelOf(prepareResult.error()), + "UploadOperation: Failed to prepare operation: {}", + prepareResult.error().toString() + ); return enterErrorState(prepareResult.error()); } } @@ -190,7 +224,11 @@ std::expected UploadOperati const auto finalizeResult = finalizeInStrand(*sftp); if (!finalizeResult.has_value()) { - Log::error("UploadOperation: Failed to finalize operation: {}", finalizeResult.error().toString()); + Log::log( + failureLevelOf(finalizeResult.error()), + "UploadOperation: Failed to finalize operation: {}", + finalizeResult.error().toString() + ); return enterErrorState(finalizeResult.error()); } state_ = Completed; @@ -417,6 +455,13 @@ std::expected UploadOperation::openOrAdoptFileInSt } } + // Checked before anything is sent too, not only before the destination is replaced. + if (const auto unchanged = + ComparedStateCheck::requireUnchangedRemoteInStrand(sftp, remotePath_, options_.comparedDestination); + !unchanged) + return std::unexpected(unchanged.error()); + if (options_.adoptPartialOf) + takeOverPartialFileInStrand(sftp); const auto tempResult = sftp.lstatInStrand(tempPath_); // Permissions of what is uploaded: the target of a followed symlink, not the link. @@ -449,6 +494,7 @@ std::expected UploadOperation::openOrAdoptFileInSt return std::unexpected(Error{.type = ErrorType::SftpError, .sftpError = openResult.error()}); } fileStream_ = openResult.value(); + openedTemporaryFile_ = true; auto stream = fileStream_.lock(); if (!stream) @@ -465,6 +511,7 @@ std::expected UploadOperation::openOrAdoptFileInSt } // Success! Continue upload from position. + continued_ = true; leftToUpload_ -= static_cast(tempResult->size); localFile_.seekg(static_cast(tempResult->size)); return {}; @@ -492,10 +539,25 @@ std::expected UploadOperation::openOrAdoptFileInSt } fileStream_ = openResult.value(); + openedTemporaryFile_ = true; return {}; } +void UploadOperation::takeOverPartialFileInStrand(SecureShell::SftpSession& sftp) +{ + const auto partial = options_.adoptPartialOf->withFilenameSuffix(options_.tempFileSuffix); + if (!partial || *partial == tempPath_ || sftp.lstatInStrand(tempPath_).has_value()) + return; + if (options_.createMissingDirectories) + { + if (const auto created = ensureRemoteDirectoryExistsInStrand(sftp, *tempPath_.parent()); !created) + return; + } + if (const auto renamed = sftp.renameNoReplaceInStrand(*partial, tempPath_); renamed.has_value()) + Log::info("UploadOperation: continuing '{}' as '{}'.", partial->value(), tempPath_.value()); +} + bool UploadOperation::tempBelongsToSourceInStrand(SecureShell::SftpSession& sftp, std::uint64_t tempSize) { // Another program's partial file or a part of an older version of the source must not be continued. @@ -559,6 +621,12 @@ UploadOperation::ensureRemoteDirectoryExistsInStrand(SecureShell::SftpSession& s return {}; } +bool UploadOperation::sourceIsGone() const +{ + const auto status = Utility::checkedSymlinkStatus(localNativePath_); + return status && !status->has_value(); +} + std::expected UploadOperation::openLocalSource() { if (options_.localPath.empty()) @@ -607,7 +675,9 @@ std::expected UploadOperation::openLocalSource() return std::unexpected(Error{.type = ErrorType::NotAFile}); } - localFile_.open(localNativePath_, std::ios::binary); + // Taken before the file is opened, so a change while it is read differs from it. + sourceBeforeOpening_ = sourceStateOf(localNativePath_); + localFile_.open(localNativePath_); if (!localFile_.is_open()) { Log::error("UploadOperation: Failed to open local file: {}", options_.localPath.toUtf8()); @@ -708,16 +778,66 @@ std::expected UploadOperation::cancel(bool adoptCa { Log::info( "UploadOperation: Upload of '{}' to '{}' canceled.", - options_.remotePath.value(), - options_.localPath.toUtf8() + options_.localPath.toUtf8(), + options_.remotePath.value() ); - state_ = OperationState::Canceled; } - cleanup(); + // A transfer reads the local file on the processing thread, closing it from elsewhere cuts a read short. + const auto sftp = openSession(sftp_); + if (sftp && !sftp->strand()->withinProcessingThread()) + { + // Only wait, never get(): a finalized strand hands back an exceptional future. + auto cancelled = sftp->performPromise( + [this, adoptCancelState]() + { + cancelInStrand(adoptCancelState); + return true; + } + ); + if (cancelled.wait_for(options_.futureTimeout) != std::future_status::ready) + Log::warn("UploadOperation: Cancelling the upload to '{}' timed out.", options_.remotePath.value()); + } + else + { + cancelInStrand(adoptCancelState); + } return {}; } +void UploadOperation::cancelInStrand(bool adoptCancelState) +{ + if (adoptCancelState) + state_ = OperationState::Canceled; + cleanup(); + const bool openedTemporaryFile = std::exchange(openedTemporaryFile_, false); + if (adoptCancelState && partialFileOnCancel_ == PartialFile::Discarded && openedTemporaryFile) + removeTemporaryFile(); +} + +void UploadOperation::partialFileOnCancel(PartialFile partial) +{ + partialFileOnCancel_ = partial; +} + +void UploadOperation::removeTemporaryFile() +{ + const auto sftp = openSession(sftp_); + if (!sftp) + return; + // Queued behind the close of the file on the server, which cleanup() queued. + sftp->perform( + [weak = sftp_, temporary = tempPath_]() + { + const auto session = openSession(weak); + if (!session) + return; + if (const auto removed = session->removeFileInStrand(temporary); !removed.has_value()) + Log::warn("UploadOperation: Cannot remove '{}': {}", temporary.value(), removed.error().message); + } + ); +} + void UploadOperation::cleanup() { if (asyncTransferContext_) @@ -763,7 +883,7 @@ std::expected UploadOperation::finalize() return result; Log::debug( - "UploadOperation: Finalized upload of '{}' to '{}'.", options_.remotePath.value(), options_.localPath.toUtf8() + "UploadOperation: Finalized upload of '{}' to '{}'.", options_.localPath.toUtf8(), options_.remotePath.value() ); return {}; } @@ -811,13 +931,32 @@ std::expected UploadOperation::finalizeInStrand(Se Log::warn("UploadOperation: Cannot remove '{}': {}", tempPath_.value(), removed.error().message); return std::unexpected(std::move(moved).error()); } + reportSent(); return {}; } +void UploadOperation::reportSent() const +{ + if (!options_.onSent || isSymlink_) + return; + const auto unchanged = [this]() -> std::optional + { + if (continued_ || !sourceBeforeOpening_) + return std::nullopt; + const auto now = sourceStateOf(localNativePath_); + if (now != sourceBeforeOpening_ || now->size != totalSize_) + return std::nullopt; + return now; + }(); + options_.onSent(options_.localPath, unchanged); +} + void UploadOperation::applyModificationTimeInStrand(SecureShell::SftpSession& sftp) { + // As it was when the source was opened: renamed or deleted since, its path has none. std::error_code timeError{}; - const auto localTime = std::filesystem::last_write_time(localNativePath_, timeError); + const auto localTime = sourceBeforeOpening_ ? sourceBeforeOpening_->modified + : std::filesystem::last_write_time(localNativePath_, timeError); if (timeError) { Log::warn( diff --git a/backend/test/backend/error_log_capture.hpp b/backend/test/backend/error_log_capture.hpp new file mode 100644 index 00000000..871bd39d --- /dev/null +++ b/backend/test/backend/error_log_capture.hpp @@ -0,0 +1,82 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace Test +{ + /** + * @brief Records the messages logged at a level or above while it lives. The tests log nothing below it otherwise. + */ + class LogCapture + { + public: + /** + * @param level The lowest level recorded. + */ + explicit LogCapture(spdlog::level::level_enum level) + : sink_{std::make_shared( + [this](spdlog::details::log_msg const& message) + { + std::scoped_lock lock{mutex_}; + messages_.emplace_back(message.payload.data(), message.payload.size()); + } + )} + , previousLevel_{spdlog::get_level()} + { + sink_->set_level(level); + // Added while nothing is logged, so no other thread walks the sinks meanwhile. + spdlog::default_logger()->sinks().push_back(sink_); + spdlog::set_level(level); + } + ~LogCapture() + { + spdlog::set_level(previousLevel_); + std::erase(spdlog::default_logger()->sinks(), sink_); + } + LogCapture(LogCapture const&) = delete; + LogCapture& operator=(LogCapture const&) = delete; + LogCapture(LogCapture&&) = delete; + LogCapture& operator=(LogCapture&&) = delete; + + /** + * @brief How many recorded messages mention @p text. + */ + std::size_t mentioning(std::string_view text) const + { + std::scoped_lock lock{mutex_}; + return static_cast(std::ranges::count_if( + messages_, + [text](std::string const& message) + { + return message.contains(text); + } + )); + } + + private: + mutable std::mutex mutex_{}; + std::vector messages_{}; + std::shared_ptr sink_; + spdlog::level::level_enum previousLevel_; + }; + + /** + * @brief Records the messages logged as errors while it lives. + */ + class ErrorLogCapture : public LogCapture + { + public: + ErrorLogCapture() + : LogCapture{spdlog::level::err} + {} + }; +} diff --git a/backend/test/backend/main.cpp b/backend/test/backend/main.cpp index 3edce237..a6fc788e 100644 --- a/backend/test/backend/main.cpp +++ b/backend/test/backend/main.cpp @@ -28,6 +28,12 @@ #include "test_termios.hpp" #include "test_fork_pool.hpp" #include "test_file_change_filter.hpp" +#include "test_local_watch.hpp" +#include "test_disk_place.hpp" +#include "test_settled_changes.hpp" +#include "test_tracking_state.hpp" +#include "test_tracker.hpp" +#include "test_batching_remote.hpp" #include "test_password_request.hpp" #include "test_sync_session.hpp" #include "test_dropped_entry.hpp" diff --git a/backend/test/backend/test_batching_remote.hpp b/backend/test/backend/test_batching_remote.hpp new file mode 100644 index 00000000..0ea7cd43 --- /dev/null +++ b/backend/test/backend/test_batching_remote.hpp @@ -0,0 +1,520 @@ +#pragma once + +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class BatchingRemoteTest : public ::testing::Test + { + protected: + /** + * @brief Records what reaches the server side, the test ends the operations. + */ + class FakeInner : public FileTracking::BulkTrackerRemote + { + public: + struct Queued + { + std::string kind; + std::vector paths; + OnEnded onEnded{}; + OnEntryEnded onEntryEnded{}; + OnUploadEnded onUploadEnded{}; + OnUploadEntryEnded onUploadEntryEnded{}; + }; + + std::expected, std::string> + stat(Paths::RemotePath const& path) override + { + queued.push_back({.kind = "stat", .paths = {path.value()}}); + return std::nullopt; + } + + std::expected, std::string> + linkStat(Paths::RemotePath const& path) override + { + queued.push_back({.kind = "linkStat", .paths = {path.value()}}); + return std::nullopt; + } + + std::expected download( + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t, + OnEnded onEnded + ) override + { + queued.push_back({ + .kind = "download", + .paths = {source.value(), destination.toUtf8()}, + .onEnded = std::move(onEnded), + }); + return {}; + } + + std::expected + upload(FileTracking::UploadRequest const& request, OnUploadEnded onEnded) override + { + if (failQueueing) + return std::unexpected{std::string{"refused"}}; + queued.push_back({ + .kind = "upload", + .paths = {request.destination.value()}, + .onUploadEnded = std::move(onEnded), + }); + return {}; + } + + void cancelUpload(Paths::RemotePath const& destination, FileTracking::PartialFile partial) override + { + cancels.emplace_back(destination.value(), partial); + } + + std::expected + discardPartialUpload(Paths::RemotePath const& destination, OnEnded onEnded) override + { + queued.push_back({.kind = "discard", .paths = {destination.value()}, .onEnded = std::move(onEnded)}); + return {}; + } + + std::expected + rename(Paths::RemotePath const& source, Paths::RemotePath const& destination, OnEnded onEnded) override + { + queued.push_back({ + .kind = "rename", + .paths = {source.value(), destination.value()}, + .onEnded = std::move(onEnded), + }); + return {}; + } + + std::expected remove(Paths::RemotePath const& path, OnEnded onEnded) override + { + queued.push_back({.kind = "remove", .paths = {path.value()}, .onEnded = std::move(onEnded)}); + return {}; + } + + std::expected + uploadAll(std::vector const& requests, OnUploadEntryEnded onEnded) override + { + if (failQueueing) + return std::unexpected{std::string{"refused"}}; + Queued bulk{.kind = "uploadAll", .paths = {}, .onUploadEntryEnded = std::move(onEnded)}; + for (auto const& request : requests) + bulk.paths.push_back(request.destination.value()); + queued.push_back(std::move(bulk)); + return {}; + } + + std::expected + removeAll(std::vector const& paths, OnEntryEnded onEnded) override + { + Queued bulk{.kind = "removeAll", .paths = {}, .onEnded = {}, .onEntryEnded = std::move(onEnded)}; + for (auto const& path : paths) + bulk.paths.push_back(path.value()); + queued.push_back(std::move(bulk)); + return {}; + } + + std::vector of(std::string const& kind) + { + std::vector found; + for (auto& operation : queued) + if (operation.kind == kind) + found.push_back(&operation); + return found; + } + + std::vector queued{}; + std::vector> cancels{}; + bool failQueueing{false}; + }; + + boost::asio::io_context ioc_; + boost::asio::strand strand_{ioc_.get_executor()}; + std::filesystem::path root_; + FakeInner* inner_{nullptr}; + std::unique_ptr remote_; + /** @brief How each upload or delete of the test ended, by its name. */ + std::map ended_{}; + /** @brief What the server had at the destination of each upload of the test that ended, by its name. */ + std::map> destinationStates_{}; + + void SetUp() override + { + root_ = std::filesystem::canonical(temporaryRoot) / + ("batching_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count())); + std::filesystem::create_directories(root_); + makeRemote( + {.quiet = std::chrono::milliseconds{50}, .longest = std::chrono::milliseconds{400}, .maxEntries = 4} + ); + } + + void TearDown() override + { + remote_.reset(); + if (!root_.empty()) + std::filesystem::remove_all(root_); + } + + void makeRemote(FileTracking::BatchingRemote::Limits limits) + { + auto inner = std::make_unique(); + inner_ = inner.get(); + remote_ = std::make_unique(strand_, std::move(inner), limits); + } + + /** + * @brief A regular file @p name below the test directory. + */ + Paths::LocalPath file(std::string const& name) + { + std::ofstream{root_ / name} << name; + return Paths::LocalPath::make(root_ / name).value(); + } + + static Paths::RemotePath remote(std::string const& path) + { + return Paths::RemotePath::make(path).value(); + } + + FileTracking::TrackerRemote::OnEnded recordAs(std::string name) + { + return [this, name = std::move(name)](SharedData::OperationCompletionReason reason) + { + ended_[name] = reason; + }; + } + + FileTracking::TrackerRemote::OnUploadEnded recordUploadAs(std::string name) + { + return [this, name = std::move(name)](FileTracking::UploadEnd const& ended) + { + ended_[name] = ended.reason; + destinationStates_[name] = ended.destinationState; + }; + } + + void upload(std::string const& name, bool createMissingDirectories = true) + { + ASSERT_TRUE(remote_ + ->upload( + FileTracking::UploadRequest{ + .source = file(name), + .destination = remote("/srv/" + name), + .createMissingDirectories = createMissingDirectories, + }, + recordUploadAs(name) + ) + .has_value()); + } + + void remove(std::string const& name) + { + ASSERT_TRUE(remote_->remove(remote("/srv/" + name), recordAs(name)).has_value()); + } + + void runFor(std::chrono::milliseconds duration) + { + ioc_.restart(); + ioc_.run_for(duration); + } + + bool + runUntil(std::function const& condition, std::chrono::milliseconds timeout = std::chrono::seconds{3}) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) + { + runFor(std::chrono::milliseconds{10}); + if (condition()) + return true; + } + return false; + } + }; + + TEST_F(BatchingRemoteTest, AnUploadAloneGoesOnItsOwnOnceTheBatchIsQuiet) + { + upload("a.txt"); + + EXPECT_TRUE(inner_->queued.empty()); + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->queued.empty(); + } + )); + ASSERT_EQ(inner_->of("upload").size(), 1u); + inner_->of("upload").front()->onUploadEnded({.reason = SharedData::OperationCompletionReason::Completed}); + EXPECT_EQ(ended_.at("a.txt"), SharedData::OperationCompletionReason::Completed); + } + + TEST_F(BatchingRemoteTest, UploadsGatheredTogetherGoAsOneBulkAndEachHearsItsOwnEnd) + { + upload("a.txt"); + upload("b.txt"); + upload("c.txt"); + + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->queued.empty(); + } + )); + const auto bulks = inner_->of("uploadAll"); + ASSERT_EQ(bulks.size(), 1u); + EXPECT_EQ(bulks.front()->paths, (std::vector{"/srv/a.txt", "/srv/b.txt", "/srv/c.txt"})); + EXPECT_TRUE(inner_->of("upload").empty()); + + const FileTracking::ServerState uploaded{.mtime = 7u, .size = 5u}; + bulks.front()->onUploadEntryEnded(1, {.reason = SharedData::OperationCompletionReason::Failed}); + bulks.front()->onUploadEntryEnded( + 0, {.reason = SharedData::OperationCompletionReason::Completed, .destinationState = uploaded} + ); + bulks.front()->onUploadEntryEnded(2, {.reason = SharedData::OperationCompletionReason::Canceled}); + EXPECT_EQ(ended_.at("a.txt"), SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(ended_.at("b.txt"), SharedData::OperationCompletionReason::Failed); + EXPECT_EQ(ended_.at("c.txt"), SharedData::OperationCompletionReason::Canceled); + EXPECT_EQ(destinationStates_.at("a.txt"), uploaded); + EXPECT_FALSE(destinationStates_.at("b.txt").has_value()); + } + + TEST_F(BatchingRemoteTest, UploadsThatMustNotCreateDirectoriesOrAreNoFilesGoOnTheirOwn) + { + upload("a.txt"); + upload("b.txt"); + upload("item.txt", /*createMissingDirectories=*/false); + std::filesystem::create_directory(root_ / "directory"); + ASSERT_TRUE(remote_ + ->upload( + FileTracking::UploadRequest{ + .source = Paths::LocalPath::make(root_ / "directory").value(), + .destination = remote("/srv/directory"), + .createMissingDirectories = true, + }, + recordUploadAs("directory") + ) + .has_value()); + + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->queued.empty(); + } + )); + ASSERT_EQ(inner_->of("uploadAll").size(), 1u); + EXPECT_EQ(inner_->of("uploadAll").front()->paths, (std::vector{"/srv/a.txt", "/srv/b.txt"})); + ASSERT_EQ(inner_->of("upload").size(), 2u); + EXPECT_EQ(inner_->of("upload")[0]->paths.front(), "/srv/item.txt"); + EXPECT_EQ(inner_->of("upload")[1]->paths.front(), "/srv/directory"); + } + + TEST_F(BatchingRemoteTest, DeletesGatheredTogetherGoAsOneBulkAndAloneOnTheirOwn) + { + remove("a.txt"); + remove("b.txt"); + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->queued.empty(); + } + )); + ASSERT_EQ(inner_->of("removeAll").size(), 1u); + EXPECT_EQ(inner_->of("removeAll").front()->paths, (std::vector{"/srv/a.txt", "/srv/b.txt"})); + inner_->of("removeAll").front()->onEntryEnded(0, SharedData::OperationCompletionReason::Completed); + inner_->of("removeAll").front()->onEntryEnded(1, SharedData::OperationCompletionReason::Failed); + EXPECT_EQ(ended_.at("a.txt"), SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(ended_.at("b.txt"), SharedData::OperationCompletionReason::Failed); + + remove("c.txt"); + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->of("remove").empty(); + } + )); + EXPECT_EQ(inner_->of("remove").front()->paths.front(), "/srv/c.txt"); + EXPECT_EQ(inner_->of("removeAll").size(), 1u); + } + + TEST_F(BatchingRemoteTest, AFullBatchGoesAtOnceAndTheNextOneGathersAnew) + { + for (auto const* name : {"a.txt", "b.txt", "c.txt", "d.txt", "e.txt"}) + upload(name); + + runFor(std::chrono::milliseconds{10}); + ASSERT_EQ(inner_->of("uploadAll").size(), 1u); + EXPECT_EQ(inner_->of("uploadAll").front()->paths.size(), 4u); + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->of("upload").empty(); + } + )); + EXPECT_EQ(inner_->of("upload").front()->paths.front(), "/srv/e.txt"); + } + + TEST_F(BatchingRemoteTest, ABatchThatKeepsGrowingGoesAfterTheLongestTime) + { + makeRemote( + {.quiet = std::chrono::milliseconds{200}, .longest = std::chrono::milliseconds{300}, .maxEntries = 100} + ); + const auto start = std::chrono::steady_clock::now(); + for (int index = 0; index < 8 && inner_->queued.empty(); ++index) + { + upload("file" + std::to_string(index) + ".txt"); + runFor(std::chrono::milliseconds{60}); + } + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->queued.empty(); + } + )); + const auto waited = std::chrono::steady_clock::now() - start; + EXPECT_LT(waited, std::chrono::milliseconds{450}); + ASSERT_EQ(inner_->of("uploadAll").size(), 1u); + EXPECT_GE(inner_->of("uploadAll").front()->paths.size(), 4u); + } + + TEST_F(BatchingRemoteTest, ABulkTheQueueRefusesEndsEachOfItsUploadsAsFailed) + { + inner_->failQueueing = true; + upload("a.txt"); + upload("b.txt"); + + ASSERT_TRUE(runUntil( + [this]() + { + return ended_.size() == 2u; + } + )); + EXPECT_EQ(ended_.at("a.txt"), SharedData::OperationCompletionReason::Failed); + EXPECT_EQ(ended_.at("b.txt"), SharedData::OperationCompletionReason::Failed); + } + + TEST_F(BatchingRemoteTest, EverythingElsePassesThroughAtOnce) + { + ASSERT_TRUE(remote_->rename(remote("/srv/a"), remote("/srv/b"), recordAs("rename")).has_value()); + std::ignore = remote_->stat(remote("/srv/a")); + std::ignore = remote_->linkStat(remote("/srv/a")); + ASSERT_TRUE(remote_->download(remote("/srv/a"), file("a"), 1u, recordAs("download")).has_value()); + + ASSERT_EQ(inner_->queued.size(), 4u); + EXPECT_EQ(inner_->queued[0].kind, "rename"); + EXPECT_EQ(inner_->queued[1].kind, "stat"); + EXPECT_EQ(inner_->queued[2].kind, "linkStat"); + EXPECT_EQ(inner_->queued[3].kind, "download"); + } + + TEST_F(BatchingRemoteTest, AGatheredUploadStoppedBeforeItsBatchGoesEndsCanceledAndIsNotSent) + { + upload("a.txt"); + upload("b.txt"); + remote_->cancelUpload(remote("/srv/a.txt"), FileTracking::PartialFile::Kept); + + EXPECT_TRUE(ended_.empty()); + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->queued.empty(); + } + )); + EXPECT_EQ(ended_.at("a.txt"), SharedData::OperationCompletionReason::Canceled); + EXPECT_TRUE(inner_->of("uploadAll").empty()); + ASSERT_EQ(inner_->of("upload").size(), 1u); + EXPECT_EQ(inner_->of("upload").front()->paths.front(), "/srv/b.txt"); + EXPECT_TRUE(inner_->cancels.empty()); + } + + TEST_F(BatchingRemoteTest, AStopOfASentUploadAndADiscardReachTheInnerRemote) + { + upload("a.txt"); + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->queued.empty(); + } + )); + + remote_->cancelUpload(remote("/srv/a.txt"), FileTracking::PartialFile::Discarded); + ASSERT_TRUE(remote_->discardPartialUpload(remote("/srv/b.txt"), recordAs("discard")).has_value()); + + ASSERT_EQ(inner_->cancels.size(), 1u); + EXPECT_EQ(inner_->cancels.front().first, "/srv/a.txt"); + EXPECT_EQ(inner_->cancels.front().second, FileTracking::PartialFile::Discarded); + ASSERT_EQ(inner_->of("discard").size(), 1u); + EXPECT_EQ(inner_->of("discard").front()->paths.front(), "/srv/b.txt"); + inner_->of("discard").front()->onEnded(SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(ended_.at("discard"), SharedData::OperationCompletionReason::Completed); + EXPECT_FALSE(ended_.contains("a.txt")); + } + + TEST_F(BatchingRemoteTest, LargeFilesAndContinuingUploadsGoOnTheirOwn) + { + makeRemote( + {.quiet = std::chrono::milliseconds{50}, + .longest = std::chrono::milliseconds{400}, + .maxEntries = 10, + .aloneFromBytes = 8u} + ); + upload("a.txt"); + upload("b.txt"); + upload("large-file.txt"); + ASSERT_TRUE(remote_ + ->upload( + FileTracking::UploadRequest{ + .source = file("c.txt"), + .destination = remote("/srv/renamed.txt"), + .createMissingDirectories = true, + .continuesPartialOf = remote("/srv/c.txt"), + }, + recordUploadAs("renamed.txt") + ) + .has_value()); + + ASSERT_TRUE(runUntil( + [this]() + { + return !inner_->queued.empty(); + } + )); + ASSERT_EQ(inner_->of("uploadAll").size(), 1u); + EXPECT_EQ(inner_->of("uploadAll").front()->paths, (std::vector{"/srv/a.txt", "/srv/b.txt"})); + ASSERT_EQ(inner_->of("upload").size(), 2u); + EXPECT_EQ(inner_->of("upload")[0]->paths.front(), "/srv/large-file.txt"); + EXPECT_EQ(inner_->of("upload")[1]->paths.front(), "/srv/renamed.txt"); + } + + TEST_F(BatchingRemoteTest, NothingGatheredIsSentOnceTheRemoteIsGone) + { + upload("a.txt"); + upload("b.txt"); + remote_.reset(); + inner_ = nullptr; + + runFor(std::chrono::milliseconds{200}); + + EXPECT_TRUE(ended_.empty()); + } +} diff --git a/backend/test/backend/test_compared_state_check.hpp b/backend/test/backend/test_compared_state_check.hpp index 684eb909..a923c4bd 100644 --- a/backend/test/backend/test_compared_state_check.hpp +++ b/backend/test/backend/test_compared_state_check.hpp @@ -218,6 +218,30 @@ namespace Test EXPECT_EQ(readRemoteFile(*sftp, "/home/test/file1.txt.filepart"), "") << "the temp file is removed"; } + TEST_F(ComparedStateCheckTests, AnUploadThatMustFindItsDestinationDoesNotRecreateOneTheServerLost) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto localPath = localDirectory_.path() / "upload.txt"; + writeLocalFile(localPath, "new content"); + auto compared = remoteNow(*sftp, "/home/test/file1.txt"); + compared.mustStillExist = true; + + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/lost.txt"), + .localPath = toLocal(localPath), + .mayOverwrite = true, + .comparedDestination = compared, + } + }; + const auto result = runToEnd(operation); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().type, Operation::ErrorType::ChangedSinceCompare); + EXPECT_EQ(readRemoteFile(*sftp, "/home/test/lost.txt"), ""); + } + TEST_F(ComparedStateCheckTests, AnUploadReplacesADestinationThatIsAsTheCompareSawIt) { CREATE_SERVER_AND_JOINER(sftpServer); diff --git a/backend/test/backend/test_disk_place.hpp b/backend/test/backend/test_disk_place.hpp new file mode 100644 index 00000000..5e116238 --- /dev/null +++ b/backend/test/backend/test_disk_place.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +# include +#endif + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class DiskPlaceTest : public ::testing::Test + { + protected: + std::filesystem::path root_; + + void SetUp() override + { + root_ = std::filesystem::canonical(temporaryRoot) / + ("disk_place_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count())); + std::filesystem::create_directories(root_ / "watched" / "project" / "src"); + std::filesystem::create_directories(root_ / "trash"); + std::ofstream{root_ / "watched" / "project" / "src" / "main.cpp"} << "int main() {}"; + } + + void TearDown() override + { + std::filesystem::remove_all(root_); + } + + FileTracking::DiskIdentity projectIdentity() const + { + return FileTracking::diskIdentityOf(root_ / "watched" / "project").value(); + } + + std::optional isDeleted(FileTracking::DiskIdentity const& identity) const + { + return FileTracking::isDeletedFromItsVolume(identity, root_ / "watched"); + } + }; + + TEST_F(DiskPlaceTest, ADirectoryMovedElsewhereOnItsVolumeIsNotDeleted) + { + const auto identity = projectIdentity(); + + std::filesystem::rename(root_ / "watched" / "project", root_ / "trash" / "project"); + +#ifdef __APPLE__ + EXPECT_EQ(isDeleted(identity), false); +#else + EXPECT_EQ(isDeleted(identity), std::nullopt); +#endif + } + + TEST_F(DiskPlaceTest, ADirectoryRemovedWithItsContentIsDeletedOnlyWhereItsIdentityCanBeLookedUp) + { + const auto identity = projectIdentity(); + + std::filesystem::remove_all(root_ / "watched" / "project"); + +#ifdef __APPLE__ + EXPECT_EQ(isDeleted(identity), true); +#else + EXPECT_EQ(isDeleted(identity), std::nullopt); +#endif + } + + TEST_F(DiskPlaceTest, AnIdentityOfAnotherVolumeCannotBeTold) + { + auto identity = projectIdentity(); + ++identity.device; + + std::filesystem::remove_all(root_ / "watched" / "project"); + + EXPECT_EQ(isDeleted(identity), std::nullopt); + } + +#ifdef __APPLE__ + TEST_F(DiskPlaceTest, ADirectoryDeletedWhileItWasOpenIsDeletedOnceClosed) + { + const auto identity = projectIdentity(); + const int descriptor = ::open((root_ / "watched" / "project").c_str(), O_RDONLY | O_DIRECTORY); + ASSERT_GE(descriptor, 0); + + std::filesystem::remove_all(root_ / "watched" / "project"); + EXPECT_EQ(isDeleted(identity), true); + ::close(descriptor); + + EXPECT_EQ(isDeleted(identity), true); + } +#endif +} diff --git a/backend/test/backend/test_file_change_filter.hpp b/backend/test/backend/test_file_change_filter.hpp index 42223bdf..194bc2a2 100644 --- a/backend/test/backend/test_file_change_filter.hpp +++ b/backend/test/backend/test_file_change_filter.hpp @@ -45,6 +45,15 @@ namespace Test EXPECT_FALSE(isTransientFile(name, filepart)) << name; } + TEST(FileChangeFilterTests, OnlyChangesEndingAndStartingAtTemporariesAreTransient) + { + EXPECT_TRUE(FileTracking::isTransientChange("report.pdf.filepart", "", filepart)); + EXPECT_TRUE(FileTracking::isTransientChange(".goutputstream-ABC123", ".DS_Store", filepart)); + EXPECT_FALSE(FileTracking::isTransientChange("._project", "project", filepart)); + EXPECT_FALSE(FileTracking::isTransientChange("report.pdf", "report.pdf.filepart", filepart)); + EXPECT_FALSE(FileTracking::isTransientChange("notes.txt", "", filepart)); + } + TEST(FileChangeFilterTests, ChangesOfTransientFilesAreDropped) { EXPECT_FALSE(filterFileChange({FileAction::Modified, ".DS_Store"}, false, filepart).has_value()); diff --git a/backend/test/backend/test_file_tracking.hpp b/backend/test/backend/test_file_tracking.hpp index e610b19b..8ed54a7f 100644 --- a/backend/test/backend/test_file_tracking.hpp +++ b/backend/test/backend/test_file_tracking.hpp @@ -267,7 +267,7 @@ namespace Test } // ========================================================================= - // TemporaryDirectoryInstanceTest — requires a display + // TemporaryDirectoryInstanceTest // ========================================================================= class TemporaryDirectoryInstanceTest : public ::testing::Test @@ -278,8 +278,6 @@ namespace Test void SetUp() override { - if (!NuiEnv::instance().available()) - GTEST_SKIP() << "No display available; skipping TemporaryDirectoryInstance tests"; root_ = makeTestTempRoot("tdi"); } @@ -296,9 +294,8 @@ namespace Test std::unique_ptr makeInstance() { - auto& env = NuiEnv::instance(); return std::make_unique( - TemporaryDirectoryInstance::Config{.tempRootDir = root_}, makeStrand(), *env.window, *env.hub + TemporaryDirectoryInstance::Config{.tempRootDir = root_}, makeStrand() ); } }; @@ -566,7 +563,7 @@ namespace Test } // ========================================================================= - // TempDirInstanceManagerTest — requires a display + // TempDirInstanceManagerTest // ========================================================================= class TempDirInstanceManagerTest : public ::testing::Test @@ -577,8 +574,6 @@ namespace Test void SetUp() override { - if (!NuiEnv::instance().available()) - GTEST_SKIP() << "No display available; skipping TempDirInstanceManager tests"; root_ = makeTestTempRoot("mgr"); } @@ -590,9 +585,8 @@ namespace Test std::unique_ptr makeManager(std::chrono::hours retention = 24h) { - auto& env = NuiEnv::instance(); return std::make_unique( - ioc_.get_executor(), *env.window, *env.hub, root_, retention + boost::asio::make_strand(ioc_.get_executor()), root_, retention ); } }; diff --git a/backend/test/backend/test_instance_bookkeeping.hpp b/backend/test/backend/test_instance_bookkeeping.hpp index 131ce08e..f12b3c8b 100644 --- a/backend/test/backend/test_instance_bookkeeping.hpp +++ b/backend/test/backend/test_instance_bookkeeping.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include extern std::filesystem::path temporaryRoot; @@ -173,24 +174,41 @@ namespace Test writeText(instanceDirectory_ / "tree" / "inner.txt", "12345"); const auto states = FileTracking::InstanceManagerDetail::fileStatesOf(instanceDirectory_); - ASSERT_TRUE(states.is_array()); - const auto find = [&states](std::filesystem::path const& path) -> nlohmann::json + const auto find = [&states](std::filesystem::path const& path) -> std::optional { - const auto wanted = Paths::nativeToUtf8(std::filesystem::weakly_canonical(path)); - const auto found = std::ranges::find_if( - states, - [&wanted](nlohmann::json const& state) - { - return state["path"] == wanted; - } - ); - return found == states.end() ? nlohmann::json{} : *found; + const auto found = + std::ranges::find(states, std::filesystem::weakly_canonical(path), &FileTracking::FileState::path); + return found == states.end() ? std::nullopt : std::optional{*found}; }; EXPECT_EQ(states.size(), 3u); - EXPECT_EQ(find(instanceDirectory_ / "metadata.json")["size"], 7); - EXPECT_EQ(find(instanceDirectory_ / "tree" / "inner.txt")["size"], 5); - EXPECT_TRUE(find(instanceDirectory_ / "tree" / "inner.txt").contains("mtime")); - EXPECT_EQ(find(instanceDirectory_ / "tree")["directory"], true); - EXPECT_TRUE(find(FileTracking::bookkeepingPaths(instanceDirectory_).metadata).is_null()); + EXPECT_EQ(find(instanceDirectory_ / "metadata.json").value_or(FileTracking::FileState{}).size, 7u); + const auto inner = find(instanceDirectory_ / "tree" / "inner.txt"); + ASSERT_TRUE(inner.has_value()); + EXPECT_EQ(inner->size, 5u); + EXPECT_GT(inner->mtime, 0u); + EXPECT_TRUE(find(instanceDirectory_ / "tree").value_or(FileTracking::FileState{}).isDirectory); + EXPECT_FALSE(find(FileTracking::bookkeepingPaths(instanceDirectory_).metadata).has_value()); + } + + TEST_F(InstanceBookkeepingTest, FileStatesBelowAPathListOnlyThatPathAndItsContent) + { + ASSERT_TRUE(FileTracking::createBookkeepingDirectory(instanceDirectory_).has_value()); + writeText(instanceDirectory_ / "metadata.json", "tracked"); + writeText(instanceDirectory_ / "tree" / "inner.txt", "12345"); + const auto canonical = [](std::filesystem::path const& path) + { + return std::filesystem::weakly_canonical(path); + }; + + const auto tree = + FileTracking::InstanceManagerDetail::fileStatesOf(instanceDirectory_, instanceDirectory_ / "tree"); + + ASSERT_EQ(tree.size(), 2u); + EXPECT_EQ(tree[0].path, canonical(instanceDirectory_ / "tree")); + EXPECT_EQ(tree[1].path, canonical(instanceDirectory_ / "tree" / "inner.txt")); + const auto bookkeeping = FileTracking::bookkeepingPaths(instanceDirectory_).metadata; + EXPECT_TRUE(FileTracking::InstanceManagerDetail::fileStatesOf(instanceDirectory_, bookkeeping).empty()); + const auto outside = instanceDirectory_.parent_path(); + EXPECT_TRUE(FileTracking::InstanceManagerDetail::fileStatesOf(instanceDirectory_, outside).empty()); } } diff --git a/backend/test/backend/test_local_watch.hpp b/backend/test/backend/test_local_watch.hpp new file mode 100644 index 00000000..74a7f231 --- /dev/null +++ b/backend/test/backend/test_local_watch.hpp @@ -0,0 +1,1347 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + namespace fs = std::filesystem; + + TEST(LocalWatchScopeTests, AChangeOfTheItemOrBelowItConcernsIt) + { + const fs::path item{"/home/user/project"}; + EXPECT_TRUE(FileTracking::isChangeOfItem(item, "/home/user/project", {})); + EXPECT_TRUE(FileTracking::isChangeOfItem(item, "/home/user/project/src/main.cpp", {})); + EXPECT_TRUE(FileTracking::isChangeOfItem(item, "/home/user/elsewhere", "/home/user/project")); + EXPECT_TRUE(FileTracking::isChangeOfItem(item, "/home/user/project", "/home/user/.project.swp")); + } + + TEST(LocalWatchScopeTests, SiblingsAndPrefixesDoNotConcernTheItem) + { + const fs::path item{"/home/user/project"}; + EXPECT_FALSE(FileTracking::isChangeOfItem(item, "/home/user/project2", {})); + EXPECT_FALSE(FileTracking::isChangeOfItem(item, "/home/user/other.txt", "/home/user/other.tmp")); + EXPECT_FALSE(FileTracking::isChangeOfItem(item, "/home/user", {})); + } + + TEST(LocalWatchOverlapTests, NestedOrEqualItemsAreRefused) + { + const std::vector watched{fs::path{"/home/user/project"}}; + const fs::path downloads{"/var/tmp/nui-sftp"}; + using FileTracking::LocalWatchManagerDetail::refuseOverlap; + EXPECT_FALSE(refuseOverlap(fs::path{"/home/user/project"}, watched, downloads).has_value()); + EXPECT_FALSE(refuseOverlap(fs::path{"/home/user/project/src/main.cpp"}, watched, downloads).has_value()); + EXPECT_FALSE(refuseOverlap(fs::path{"/home/user"}, watched, downloads).has_value()); + EXPECT_FALSE(refuseOverlap(fs::path{"/var/tmp/nui-sftp/instance/file.txt"}, watched, downloads).has_value()); + EXPECT_FALSE(refuseOverlap(fs::path{"/var"}, watched, downloads).has_value()); + } + + TEST(LocalWatchOverlapTests, TheRefusalNamesWhatItOverlaps) + { + using FileTracking::LocalWatchManagerDetail::refuseOverlap; + const auto nested = refuseOverlap( + fs::path{"/home/user/project/src"}, {fs::path{"/home/user/project"}}, fs::path{"/var/tmp/nui-sftp"} + ); + const auto downloads = refuseOverlap(fs::path{"/var/tmp"}, {}, fs::path{"/var/tmp/nui-sftp"}); + + ASSERT_FALSE(nested.has_value()); + EXPECT_EQ( + Utility::parseLocalizedMessage(nested.error()).key, "backend.fileTracking.watchOverlaps" + ); + EXPECT_EQ( + Utility::parseLocalizedMessage(nested.error()).arguments, + (std::vector{"/home/user/project/src", "/home/user/project"}) + ); + ASSERT_FALSE(downloads.has_value()); + EXPECT_EQ( + Utility::parseLocalizedMessage(downloads.error()).key, "backend.fileTracking.watchOverlapsDownloads" + ); + } + + TEST(LocalWatchOverlapTests, SiblingsAndPrefixesAreAccepted) + { + const std::vector watched{fs::path{"/home/user/project"}}; + const fs::path downloads{"/var/tmp/nui-sftp"}; + using FileTracking::LocalWatchManagerDetail::refuseOverlap; + EXPECT_TRUE(refuseOverlap(fs::path{"/home/user/project2"}, watched, downloads).has_value()); + EXPECT_TRUE(refuseOverlap(fs::path{"/home/user/notes.txt"}, watched, downloads).has_value()); + EXPECT_TRUE(refuseOverlap(fs::path{"/home/user/notes.txt"}, {}, downloads).has_value()); + EXPECT_TRUE(refuseOverlap(fs::path{"/var/tmp/nui-sftp2"}, {}, downloads).has_value()); + } + +#if defined(_WIN32) || defined(__APPLE__) + TEST(LocalWatchOverlapTests, CaseIsIgnoredWhereTheFileSystemFoldsIt) + { + EXPECT_TRUE(FileTracking::LocalWatchManagerDetail::overlaps(fs::path{"/Users/Me/Project"}, fs::path{"/users/me/project/src"})); + } +#endif + + TEST(LocalWatchOverlapTests, TheSameDirectoryUnderAnotherSpellingOverlaps) + { + const auto root = fs::canonical(temporaryRoot) / + ("overlap_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count())); + fs::create_directories(root / "project" / "src"); + std::error_code linkError; + fs::create_directory_symlink(root / "project", root / "alias", linkError); + if (linkError) + GTEST_SKIP() << "Cannot create a symbolic link: " << linkError.message(); + + EXPECT_TRUE(FileTracking::LocalWatchManagerDetail::overlaps(root / "alias" / "src", root / "project")); + EXPECT_TRUE(FileTracking::LocalWatchManagerDetail::overlaps(root / "project", root / "alias" / "src")); + EXPECT_FALSE(FileTracking::LocalWatchManagerDetail::overlaps(root / "alias" / "missing", root / "other")); + fs::remove_all(root); + } + + class ItemStatesTest : public ::testing::Test + { + protected: + fs::path root_; + + void SetUp() override + { + root_ = fs::canonical(temporaryRoot) / + ("item_states_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count())); + fs::create_directories(root_ / "tree" / "below"); + std::ofstream{root_ / "tree" / "below" / "file.txt"} << "content"; + std::ofstream{root_ / "sibling.txt"} << "sibling"; + } + + void TearDown() override + { + fs::remove_all(root_); + } + + static std::vector pathsOf(std::vector const& states) + { + std::vector paths; + for (auto const& state : states) + paths.push_back(state.path); + std::ranges::sort(paths); + return paths; + } + }; + + TEST_F(ItemStatesTest, AFileListsOnlyItself) + { + const auto states = FileTracking::itemStatesOf(root_ / "sibling.txt"); + + ASSERT_EQ(states.size(), 1u); + EXPECT_EQ(states[0].path, root_ / "sibling.txt"); + EXPECT_EQ(states[0].size, 7u); + EXPECT_FALSE(states[0].isDirectory); + EXPECT_GT(states[0].mtime, 0u); + } + + TEST_F(ItemStatesTest, ADirectoryListsItselfAndItsContentButNoSiblings) + { + const auto states = FileTracking::itemStatesOf(root_ / "tree"); + + EXPECT_EQ( + pathsOf(states), + (std::vector{ + root_ / "tree", + root_ / "tree" / "below", + root_ / "tree" / "below" / "file.txt", + }) + ); + } + + TEST_F(ItemStatesTest, AMissingItemListsNothing) + { + EXPECT_TRUE(FileTracking::itemStatesOf(root_ / "missing").empty()); + } + + class LocalWatchManagerTest : public ::testing::Test + { + protected: + boost::asio::io_context ioc_; + fs::path root_; + std::unique_ptr manager_; + + void SetUp() override + { + root_ = fs::canonical(temporaryRoot) / + ("local_watch_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count())); + fs::create_directories(root_ / "project" / "src"); + fs::create_directories(root_ / "downloads"); + std::ofstream{root_ / "project" / "src" / "main.cpp"} << "int main() {}"; + std::ofstream{root_ / "notes.txt"} << "notes"; + manager_ = std::make_unique( + boost::asio::make_strand(ioc_.get_executor()), root_ / "downloads" + ); + } + + void TearDown() override + { + manager_.reset(); + if (!root_.empty()) + fs::remove_all(root_); + } + + /** @brief What the watches of recordingSink() reported, in order. */ + std::vector reported_{}; + + auto watch(fs::path const& item) + { + return manager_->watch( + Ids::generateWatchId(), + item, + std::string{Persistence::defaultTempFileSuffix}, + [](FileTracking::ReportedChange const&) {} + ); + } + + FileTracking::ChangeSink recordingSink() + { + return [this](FileTracking::ReportedChange const& change) + { + reported_.push_back(change); + }; + } + + auto watchRecording(fs::path const& item) + { + return manager_->watch( + Ids::generateWatchId(), item, std::string{Persistence::defaultTempFileSuffix}, recordingSink() + ); + } + + /** + * @brief Lets the events from before the watch pass and forgets them, FSEvents delivers those too. + */ + void settleEarlierEvents() + { + runFor(std::chrono::milliseconds{1000}); + reported_.clear(); + } + + /** + * @brief Watches @p item without the manager, so the test can hand it changes, see LocalItemWatch::handleChange. + */ + std::optional openRecording(fs::path const& item) + { + auto opened = FileTracking::LocalItemWatch::open( + Ids::generateWatchId(), + item, + boost::asio::make_strand(ioc_.get_executor()), + recordingSink(), + std::string{Persistence::defaultTempFileSuffix}, + [](Ids::WatchId const&, fs::path const&) + { + return true; + } + ); + if (!opened) + return std::nullopt; + return std::move(*opened); + } + + std::size_t reportedCount(FileTracking::FileAction action, fs::path const& path) const + { + return static_cast(std::ranges::count_if( + reported_, + [&](FileTracking::ReportedChange const& change) + { + return change.action == action && change.path.lexically_normal() == path.lexically_normal(); + } + )); + } + + /** + * @brief How many changes of the content of @p path were reported. FSEvents carries the creation of a file into + * its later events, efsw reports an edit as an addition and a modification then, they settle into the + * addition. + */ + std::size_t reportedContentChanges(fs::path const& path) const + { + return reportedCount(FileTracking::FileAction::Added, path) + + reportedCount(FileTracking::FileAction::Modified, path); + } + + bool reportedRemoval() const + { + return std::ranges::any_of( + reported_, + [](FileTracking::ReportedChange const& change) + { + return change.action == FileTracking::FileAction::Deleted; + } + ); + } + + FileTracking::DirectoryChange changeOf( + FileTracking::FileAction action, + std::string const& filename, + std::string const& oldFilename = {} + ) const + { + return FileTracking::DirectoryChange{ + .action = action, + .directory = root_, + .filename = filename, + .oldFilename = oldFilename, + .isDirectory = false, + }; + } + + /** + * @brief Runs the handlers until @p condition holds, at most for @p timeout. + */ + bool runUntil(std::function const& condition, std::chrono::milliseconds timeout = std::chrono::seconds{5}) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) + { + ioc_.restart(); + ioc_.run_for(std::chrono::milliseconds{20}); + if (condition()) + return true; + } + return false; + } + + void runFor(std::chrono::milliseconds duration) + { + runUntil( + []() + { + return false; + }, + duration + ); + } + }; + + TEST_F(LocalWatchManagerTest, WatchingAFileReportsItAndItsDirectory) + { + const auto watched = watch(root_ / "project" / "src" / ".." / "src" / "main.cpp"); + + ASSERT_TRUE(watched.has_value()) << watched.error(); + EXPECT_EQ((*watched)->item(), root_ / "project" / "src" / "main.cpp"); + EXPECT_EQ((*watched)->directory(), root_ / "project" / "src"); + EXPECT_FALSE((*watched)->isDirectory()); + EXPECT_TRUE((*watched)->watchId().isValid()); + EXPECT_EQ(manager_->find((*watched)->watchId()), *watched); + } + + TEST_F(LocalWatchManagerTest, WatchingADirectoryWorks) + { + const auto watched = watch(root_ / "project"); + + ASSERT_TRUE(watched.has_value()) << watched.error(); + EXPECT_TRUE((*watched)->isDirectory()); + } + + TEST_F(LocalWatchManagerTest, OverlappingWatchesAreRefused) + { + ASSERT_TRUE(watch(root_ / "project").has_value()); + + EXPECT_FALSE(watch(root_ / "project").has_value()); + EXPECT_FALSE(watch(root_ / "project" / "src" / "main.cpp").has_value()); + EXPECT_FALSE(watch(root_).has_value()); + EXPECT_TRUE(watch(root_ / "notes.txt").has_value()); + } + + TEST_F(LocalWatchManagerTest, CheckingTellsWhatAWatchWouldRefuseWithoutWatching) + { + const auto checked = manager_->checkWatchable(root_ / "project" / "." / "src"); + ASSERT_TRUE(checked.has_value()) << checked.error(); + EXPECT_EQ(*checked, root_ / "project" / "src"); + ASSERT_TRUE(watch(root_ / "project").has_value()); + + EXPECT_FALSE(manager_->checkWatchable(root_ / "project" / "src").has_value()); + EXPECT_FALSE(manager_->checkWatchable(root_ / "missing.txt").has_value()); + EXPECT_TRUE(manager_->checkWatchable(root_ / "notes.txt").has_value()); + EXPECT_TRUE(watch(root_ / "notes.txt").has_value()); + } + + TEST_F(LocalWatchManagerTest, TheTemporaryDownloadsDirectoryIsRefused) + { + std::ofstream{root_ / "downloads" / "copy.txt"} << "copy"; + + EXPECT_FALSE(watch(root_ / "downloads" / "copy.txt").has_value()); + EXPECT_FALSE(watch(root_ / "downloads").has_value()); + } + + TEST_F(LocalWatchManagerTest, MissingItemsAndRootsAreRefused) + { + EXPECT_FALSE(watch(root_ / "missing.txt").has_value()); + EXPECT_FALSE(watch(root_.root_path()).has_value()); + } + + TEST_F(LocalWatchManagerTest, ARenamedItemIsFollowed) + { + const auto watched = watch(root_ / "notes.txt"); + ASSERT_TRUE(watched.has_value()) << watched.error(); + const auto watchId = (*watched)->watchId(); + + fs::rename(root_ / "notes.txt", root_ / "renamed.txt"); + + EXPECT_TRUE(runUntil( + [&]() + { + return manager_->find(watchId)->item() == root_ / "renamed.txt"; + } + )); + } + + TEST_F(LocalWatchManagerTest, ARenamedDirectoryIsFollowed) + { + const auto watched = watch(root_ / "project"); + ASSERT_TRUE(watched.has_value()) << watched.error(); + const auto watchId = (*watched)->watchId(); + + fs::rename(root_ / "project", root_ / "renamed"); + + EXPECT_TRUE(runUntil( + [&]() + { + return manager_->find(watchId)->item() == root_ / "renamed"; + } + )); + EXPECT_TRUE(manager_->find(watchId)->isDirectory()); + } + + TEST_F(LocalWatchManagerTest, AnItemRenamedTwiceIsFollowedToItsLastName) + { + const auto watched = watch(root_ / "notes.txt"); + ASSERT_TRUE(watched.has_value()) << watched.error(); + const auto watchId = (*watched)->watchId(); + + fs::rename(root_ / "notes.txt", root_ / "first.txt"); + fs::rename(root_ / "first.txt", root_ / "second.txt"); + + EXPECT_TRUE(runUntil( + [&]() + { + return manager_->find(watchId)->item() == root_ / "second.txt"; + } + )); + } + + TEST_F(LocalWatchManagerTest, ARenameOntoAnotherWatchIsNotFollowed) + { + std::ofstream{root_ / "other.txt"} << "other"; + const auto watched = watch(root_ / "notes.txt"); + const auto other = watch(root_ / "other.txt"); + ASSERT_TRUE(watched.has_value()) << watched.error(); + ASSERT_TRUE(other.has_value()) << other.error(); + const auto watchId = (*watched)->watchId(); + const auto otherId = (*other)->watchId(); + + fs::rename(root_ / "notes.txt", root_ / "other.txt"); + runUntil( + []() + { + return false; + }, + std::chrono::seconds{1} + ); + + EXPECT_EQ(manager_->find(watchId)->item(), root_ / "notes.txt"); + EXPECT_EQ(manager_->find(otherId)->item(), root_ / "other.txt"); + } + + TEST_F(LocalWatchManagerTest, ARenameOntoATemporaryNameIsNotFollowed) + { + const auto watched = watch(root_ / "project"); + ASSERT_TRUE(watched.has_value()) << watched.error(); + const auto watchId = (*watched)->watchId(); + + fs::rename(root_ / "project", root_ / "._project"); + + EXPECT_FALSE(runUntil( + [&]() + { + return manager_->find(watchId)->item() != root_ / "project"; + }, + std::chrono::milliseconds{1000} + )); + } + + TEST_F(LocalWatchManagerTest, AnItemMovedAsideAndWrittenAnewStaysWatched) + { + const auto watched = watch(root_ / "notes.txt"); + ASSERT_TRUE(watched.has_value()) << watched.error(); + const auto watchId = (*watched)->watchId(); + + fs::rename(root_ / "notes.txt", root_ / "notes.txt~"); + std::ofstream{root_ / "notes.txt"} << "written anew"; + + EXPECT_FALSE(runUntil( + [&]() + { + return manager_->find(watchId)->item() != root_ / "notes.txt"; + }, + std::chrono::milliseconds{1000} + )); + } + + TEST_F(LocalWatchManagerTest, AnItemMovedAsideAndWrittenAnewAfterTheSettleTimeTakesTheWatchBack) + { + const auto watched = manager_->watch( + Ids::generateWatchId(), root_ / "notes.txt", std::string{Persistence::defaultTempFileSuffix}, recordingSink() + ); + ASSERT_TRUE(watched.has_value()) << watched.error(); + const auto watchId = (*watched)->watchId(); + + fs::rename(root_ / "notes.txt", root_ / "notes.txt~"); + runUntil( + [&]() + { + return manager_->find(watchId)->item() == root_ / "notes.txt~"; + } + ); + runFor(std::chrono::milliseconds{300}); + std::ofstream{root_ / "notes.txt"} << "written anew"; + fs::remove(root_ / "notes.txt~"); + + EXPECT_TRUE(runUntil( + [&]() + { + return manager_->find(watchId)->item() == root_ / "notes.txt" && + reportedCount(FileTracking::FileAction::Added, root_ / "notes.txt") > 0u; + } + )); + runFor(std::chrono::milliseconds{500}); + EXPECT_EQ(manager_->find(watchId)->item(), root_ / "notes.txt"); + EXPECT_EQ(reportedCount(FileTracking::FileAction::Moved, root_ / "notes.txt~"), 1u); + EXPECT_EQ(reportedCount(FileTracking::FileAction::Moved, root_ / "notes.txt"), 1u); + EXPECT_FALSE(reportedRemoval()); + } + + TEST_F(LocalWatchManagerTest, AFileAtTheOldNameLongAfterAFollowedRenameIsNotTheItem) + { + const auto watched = manager_->watch( + Ids::generateWatchId(), root_ / "notes.txt", std::string{Persistence::defaultTempFileSuffix}, recordingSink() + ); + ASSERT_TRUE(watched.has_value()) << watched.error(); + const auto watchId = (*watched)->watchId(); + + fs::rename(root_ / "notes.txt", root_ / "renamed.txt"); + ASSERT_TRUE(runUntil( + [&]() + { + return manager_->find(watchId)->item() == root_ / "renamed.txt"; + } + )); + runFor(FileTracking::renameReturnTime); + std::ofstream{root_ / "notes.txt"} << "another file"; + runFor(std::chrono::milliseconds{1000}); + + EXPECT_EQ(manager_->find(watchId)->item(), root_ / "renamed.txt"); + EXPECT_EQ(reportedCount(FileTracking::FileAction::Added, root_ / "notes.txt"), 0u); + } + + TEST_F(LocalWatchManagerTest, AnItemRemovedAndWrittenAnewIsNotReportedRemoved) + { + const auto watched = manager_->watch( + Ids::generateWatchId(), root_ / "notes.txt", std::string{Persistence::defaultTempFileSuffix}, recordingSink() + ); + ASSERT_TRUE(watched.has_value()) << watched.error(); + + fs::remove(root_ / "notes.txt"); + std::ofstream{root_ / "notes.txt"} << "written anew"; + + EXPECT_TRUE(runUntil( + [&]() + { + return reportedCount(FileTracking::FileAction::Added, root_ / "notes.txt") > 0u; + } + )); + runFor(std::chrono::milliseconds{500}); + EXPECT_FALSE(reportedRemoval()); + } + + TEST_F(LocalWatchManagerTest, AnItemThatStaysAwayIsReportedRemoved) + { + const auto watched = manager_->watch( + Ids::generateWatchId(), root_ / "notes.txt", std::string{Persistence::defaultTempFileSuffix}, recordingSink() + ); + ASSERT_TRUE(watched.has_value()) << watched.error(); + + fs::remove(root_ / "notes.txt"); + + EXPECT_TRUE(runUntil( + [&]() + { + return reportedCount(FileTracking::FileAction::Deleted, root_ / "notes.txt") == 1u; + } + )); + } + + TEST_F(LocalWatchManagerTest, MissedChangesOfTheItemsDirectoryHaveTheItemLookedAtAnew) + { + auto watched = openRecording(root_ / "notes.txt"); + ASSERT_TRUE(watched.has_value()); + settleEarlierEvents(); + + watched->handleChange(FileTracking::DirectoryChange{ + .action = FileTracking::FileAction::Missed, + .directory = root_, + .filename = {}, + .oldFilename = {}, + .isDirectory = true, + }); + + EXPECT_TRUE(runUntil( + [&]() + { + return reportedCount(FileTracking::FileAction::Missed, root_ / "notes.txt") == 1u; + } + )); + runFor(std::chrono::milliseconds{500}); + EXPECT_FALSE(reportedRemoval()); + } + + TEST_F(LocalWatchManagerTest, AnItemFoundGoneWhenItsOwnWatchMissedChangesIsReportedRemovedOnce) + { + const auto item = root_ / "project"; + auto watched = openRecording(item); + ASSERT_TRUE(watched.has_value()); + settleEarlierEvents(); + + fs::remove_all(item); + watched->handleChange(FileTracking::DirectoryChange{ + .action = FileTracking::FileAction::Missed, + .directory = item, + .filename = {}, + .oldFilename = {}, + .isDirectory = true, + }); + + EXPECT_TRUE(runUntil( + [&]() + { + return reportedCount(FileTracking::FileAction::Deleted, item) > 0u; + } + )); + runFor(std::chrono::milliseconds{500}); + // Windows reports one of its own, the watched directory turned delete-pending. + EXPECT_GE(reportedCount(FileTracking::FileAction::Missed, item), 1u); + EXPECT_EQ(reportedCount(FileTracking::FileAction::Deleted, item), 1u); + } + + TEST_F(LocalWatchManagerTest, ARemovalReportedBeforeARenameOfTheItemIsNotReported) + { + auto watched = openRecording(root_ / "notes.txt"); + ASSERT_TRUE(watched.has_value()); + + // Windows reports a case-only rename as the removal of the old name first. + fs::rename(root_ / "notes.txt", root_ / "NOTES.txt"); + watched->handleChange(changeOf(FileTracking::FileAction::Deleted, "notes.txt")); + watched->handleChange(changeOf(FileTracking::FileAction::Moved, "NOTES.txt", "notes.txt")); + + EXPECT_TRUE(runUntil( + [&]() + { + return watched->item() == root_ / "NOTES.txt"; + } + )); + runFor(std::chrono::milliseconds{500}); + EXPECT_EQ(reportedCount(FileTracking::FileAction::Moved, root_ / "NOTES.txt"), 1u); + EXPECT_FALSE(reportedRemoval()); + } + + TEST_F(LocalWatchManagerTest, ARemovalReportedAfterARenameOfTheItemIsNotReported) + { + auto watched = openRecording(root_ / "notes.txt"); + ASSERT_TRUE(watched.has_value()); + + fs::rename(root_ / "notes.txt", root_ / "NOTES.txt"); + watched->handleChange(changeOf(FileTracking::FileAction::Moved, "NOTES.txt", "notes.txt")); + watched->handleChange(changeOf(FileTracking::FileAction::Deleted, "notes.txt")); + + EXPECT_TRUE(runUntil( + [&]() + { + return watched->item() == root_ / "NOTES.txt"; + } + )); + runFor(std::chrono::milliseconds{500}); + EXPECT_EQ(reportedCount(FileTracking::FileAction::Moved, root_ / "NOTES.txt"), 1u); + EXPECT_FALSE(reportedRemoval()); + } + + TEST_F(LocalWatchManagerTest, UnwatchingFreesTheItem) + { + const auto watched = watch(root_ / "notes.txt"); + ASSERT_TRUE(watched.has_value()); + const auto watchId = (*watched)->watchId(); + + manager_->unwatch(watchId); + + EXPECT_EQ(manager_->find(watchId), nullptr); + EXPECT_TRUE(watch(root_ / "notes.txt").has_value()); + EXPECT_NO_THROW(manager_->unwatch(Ids::makeWatchId("unknown"))); + } + +#ifdef __APPLE__ + TEST_F(LocalWatchManagerTest, ARemovedItemReportsNoChangeOfItsContentFirst) + { + const auto item = root_ / "notes.txt"; + const auto watched = watchRecording(item); + ASSERT_TRUE(watched.has_value()) << watched.error(); + settleEarlierEvents(); + + fs::remove(item); + + EXPECT_TRUE(runUntil( + [&]() + { + return reportedCount(FileTracking::FileAction::Deleted, item) == 1u; + } + )); + EXPECT_EQ(reportedContentChanges(item), 0u); + } + + // Finder names in NFD and APFS keeps that spelling, efsw reports every name precomposed (NFC). + TEST_F(LocalWatchManagerTest, AnItemInADecomposedDirectoryReportsItsEditsUnderItsSpelling) + { + const auto directory = root_ / Paths::nativeFromUtf8("Cafe\xCC\x81"); + const auto item = directory / "notes.txt"; + fs::create_directories(directory); + std::ofstream{item} << "notes"; + const auto watched = watchRecording(item); + ASSERT_TRUE(watched.has_value()) << watched.error(); + ASSERT_EQ((*watched)->item(), item); + settleEarlierEvents(); + + std::ofstream{item, std::ios::app} << " edited"; + + EXPECT_TRUE(runUntil( + [&]() + { + return reportedContentChanges(item) > 0u; + } + )); + } + + TEST_F(LocalWatchManagerTest, AnItemInADecomposedDirectoryIsFollowedWhenRenamed) + { + const auto directory = root_ / Paths::nativeFromUtf8("Cafe\xCC\x81"); + fs::create_directories(directory); + std::ofstream{directory / "notes.txt"} << "notes"; + const auto watched = watchRecording(directory / "notes.txt"); + ASSERT_TRUE(watched.has_value()) << watched.error(); + const auto watchId = (*watched)->watchId(); + settleEarlierEvents(); + + fs::rename(directory / "notes.txt", directory / "renamed.txt"); + + EXPECT_TRUE(runUntil( + [&]() + { + return manager_->find(watchId)->item() == directory / "renamed.txt"; + } + )); + EXPECT_EQ(reportedCount(FileTracking::FileAction::Moved, directory / "renamed.txt"), 1u); + EXPECT_FALSE(reportedRemoval()); + } + + TEST_F(LocalWatchManagerTest, AnItemInADecomposedDirectoryThatStaysAwayIsReportedRemoved) + { + const auto directory = root_ / Paths::nativeFromUtf8("Cafe\xCC\x81"); + const auto item = directory / "notes.txt"; + fs::create_directories(directory); + std::ofstream{item} << "notes"; + const auto watched = watchRecording(item); + ASSERT_TRUE(watched.has_value()) << watched.error(); + settleEarlierEvents(); + + fs::remove(item); + + EXPECT_TRUE(runUntil( + [&]() + { + return reportedCount(FileTracking::FileAction::Deleted, item) == 1u; + } + )); + } + + TEST_F(LocalWatchManagerTest, AnItemNamedDecomposedReportsItsEditsAndIsFollowedWhenRenamed) + { + const auto item = root_ / Paths::nativeFromUtf8("cafe\xCC\x81.txt"); + std::ofstream{item} << "notes"; + const auto watched = watchRecording(item); + ASSERT_TRUE(watched.has_value()) << watched.error(); + ASSERT_EQ((*watched)->item(), item); + const auto watchId = (*watched)->watchId(); + settleEarlierEvents(); + + std::ofstream{item, std::ios::app} << " edited"; + EXPECT_TRUE(runUntil( + [&]() + { + return reportedContentChanges(item) > 0u; + } + )); + fs::rename(item, root_ / "renamed.txt"); + + EXPECT_TRUE(runUntil( + [&]() + { + return manager_->find(watchId)->item() == root_ / "renamed.txt"; + } + )); + EXPECT_EQ(reportedCount(FileTracking::FileAction::Moved, root_ / "renamed.txt"), 1u); + EXPECT_FALSE(reportedRemoval()); + } + + TEST_F(LocalWatchManagerTest, ADecomposedDirectoryReportsChangesBelowItUnderItsSpelling) + { + const auto item = root_ / Paths::nativeFromUtf8("Cafe\xCC\x81"); + fs::create_directories(item / "sub"); + std::ofstream{item / "sub" / "notes.txt"} << "notes"; + const auto watched = watchRecording(item); + ASSERT_TRUE(watched.has_value()) << watched.error(); + ASSERT_TRUE((*watched)->isDirectory()); + settleEarlierEvents(); + + std::ofstream{item / "sub" / "notes.txt", std::ios::app} << " edited"; + + EXPECT_TRUE(runUntil( + [&]() + { + return reportedContentChanges(item / "sub" / "notes.txt") > 0u; + } + )); + } +#endif + + class DirectoryWatchServiceTest : public ::testing::Test + { + protected: + fs::path root_; + std::shared_ptr service_ = FileTracking::DirectoryWatchService::shared(); + + /** + * @brief Collects the names a subscription reports, efsw calls it on its own thread. + */ + struct Received + { + std::mutex mutex{}; + std::vector filenames{}; + std::vector events{}; + + FileTracking::DirectoryEventSink sink() + { + return [this](FileTracking::DirectoryEvent const& event) + { + std::scoped_lock lock{mutex}; + filenames.push_back(event.filename); + events.push_back(event); + }; + } + + /** + * @brief The move reported to the file @p filename, nothing when none was. + */ + std::optional moveTo(std::string const& filename) + { + std::scoped_lock lock{mutex}; + const auto found = std::ranges::find_if( + events, + [&filename](FileTracking::DirectoryEvent const& event) + { + return event.action == FileTracking::FileAction::Moved && + fs::path{event.filename}.filename() == filename; + } + ); + if (found == events.end()) + return std::nullopt; + return *found; + } + + bool has(std::string const& filename) + { + std::scoped_lock lock{mutex}; + return std::ranges::any_of( + filenames, + [&filename](std::string const& received) + { + return fs::path{received}.filename() == filename; + } + ); + } + + /** + * @brief The directories of the Missed events, in their order. + */ + std::vector missed() + { + std::scoped_lock lock{mutex}; + std::vector directories; + for (auto const& event : events) + if (event.action == FileTracking::FileAction::Missed) + directories.push_back(Paths::nativeFromUtf8(event.directory)); + return directories; + } + }; + + void SetUp() override + { + root_ = fs::canonical(temporaryRoot) / + ("watch_service_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count())); + fs::create_directories(root_ / "sub"); + } + + void TearDown() override + { + fs::remove_all(root_); + } + + static bool waitFor(std::function const& condition, std::chrono::milliseconds timeout = std::chrono::seconds{5}) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) + { + if (condition()) + return true; + std::this_thread::sleep_for(std::chrono::milliseconds{20}); + } + return false; + } + }; + + TEST_F(DirectoryWatchServiceTest, SubscribersOfOneDirectoryEachGetItsChanges) + { + Received first; + Received second; + const auto firstWatch = service_->subscribe(root_, false, first.sink()); + const auto secondWatch = service_->subscribe(root_, false, second.sink()); + ASSERT_TRUE(firstWatch.has_value()); + ASSERT_TRUE(secondWatch.has_value()); + + std::ofstream{root_ / "shared.txt"} << "shared"; + + EXPECT_TRUE(waitFor( + [&]() + { + return first.has("shared.txt") && second.has("shared.txt"); + } + )); + } + + TEST_F(DirectoryWatchServiceTest, RecursiveSubscribersOfOneDirectoryEachGetItsChanges) + { + Received first; + Received second; + const auto firstWatch = service_->subscribe(root_, true, first.sink()); + const auto secondWatch = service_->subscribe(root_, true, second.sink()); + ASSERT_TRUE(firstWatch.has_value()); + ASSERT_TRUE(secondWatch.has_value()); + + std::ofstream{root_ / "sub" / "deep.txt"} << "deep"; + + EXPECT_TRUE(waitFor( + [&]() + { + return first.has("deep.txt") && second.has("deep.txt"); + } + )); + } + + TEST_F(DirectoryWatchServiceTest, AMoveBetweenDirectoriesOfARecursiveWatchIsOneMove) + { + fs::create_directories(root_ / "other"); + std::ofstream{root_ / "sub" / "moved.txt"} << "moved"; + Received received; + const auto watch = service_->subscribe(root_, true, received.sink()); + ASSERT_TRUE(watch.has_value()); + + fs::rename(root_ / "sub" / "moved.txt", root_ / "other" / "moved.txt"); + +#ifdef __APPLE__ + // FSEvents pairs the two halves only when they arrive together. + EXPECT_TRUE(waitFor( + [&]() + { + return received.has("moved.txt"); + } + )); +#else + ASSERT_TRUE(waitFor( + [&]() + { + return received.moveTo("moved.txt").has_value(); + } + )); + const auto move = *received.moveTo("moved.txt"); + EXPECT_EQ( + fs::path{Paths::nativeFromUtf8(move.oldFilename)}.lexically_normal(), + (root_ / "sub" / "moved.txt").lexically_normal() + ); +#endif + } + + TEST_F(DirectoryWatchServiceTest, RecursiveAndPlainWatchesOfOtherDirectoriesBothDeliver) + { + fs::create_directories(root_ / "plain"); + Received recursive; + Received plain; + const auto recursiveWatch = service_->subscribe(root_ / "sub", true, recursive.sink()); + const auto plainWatch = service_->subscribe(root_ / "plain", false, plain.sink()); + ASSERT_TRUE(recursiveWatch.has_value()); + ASSERT_TRUE(plainWatch.has_value()); + + fs::create_directories(root_ / "sub" / "nested"); + std::ofstream{root_ / "sub" / "recursive.txt"} << "recursive"; + std::ofstream{root_ / "plain" / "plain.txt"} << "plain"; + + EXPECT_TRUE(waitFor( + [&]() + { + return recursive.has("recursive.txt") && plain.has("plain.txt"); + } + )); + EXPECT_FALSE(recursive.has("plain.txt")); + EXPECT_FALSE(plain.has("recursive.txt")); + } + + TEST_F(DirectoryWatchServiceTest, ADirectoryIsWatchedOneWayOnly) + { + Received plain; + Received recursive; + const auto plainWatch = service_->subscribe(root_, false, plain.sink()); + ASSERT_TRUE(plainWatch.has_value()); + EXPECT_FALSE(service_->subscribe(root_, true, recursive.sink()).has_value()); + + std::ofstream{root_ / "direct.txt"} << "direct"; + + EXPECT_TRUE(waitFor( + [&]() + { + return plain.has("direct.txt"); + } + )); + EXPECT_FALSE(recursive.has("direct.txt")); + } + + TEST_F(DirectoryWatchServiceTest, AnEndedSubscriptionGetsNothingAndTheDirectoryCanBeWatchedAgain) + { + Received ended; + auto endedWatch = service_->subscribe(root_, true, ended.sink()); + ASSERT_TRUE(endedWatch.has_value()); + endedWatch->release(); + + Received again; + const auto againWatch = service_->subscribe(root_, true, again.sink()); + ASSERT_TRUE(againWatch.has_value()); + std::ofstream{root_ / "sub" / "again.txt"} << "again"; + + EXPECT_TRUE(waitFor( + [&]() + { + return again.has("again.txt"); + } + )); + EXPECT_FALSE(ended.has("again.txt")); + } + + TEST_F(DirectoryWatchServiceTest, MissedChangesReachTheSubscriptionsOfTheWatchCoveringTheirDirectory) + { + fs::create_directories(root_ / "sub" / "deep"); + fs::create_directories(root_ / "plain"); + Received recursive; + Received plain; + const auto recursiveWatch = service_->subscribe(root_ / "sub", true, recursive.sink()); + const auto plainWatch = service_->subscribe(root_ / "plain", false, plain.sink()); + ASSERT_TRUE(recursiveWatch.has_value()); + ASSERT_TRUE(plainWatch.has_value()); + + service_->reportMissed(root_ / "sub" / "deep"); + + EXPECT_EQ(recursive.missed(), std::vector{root_ / "sub" / "deep"}); + EXPECT_TRUE(plain.missed().empty()); + } + + TEST_F(DirectoryWatchServiceTest, TheWatchesOfDeletedDirectoriesEnd) + { + fs::create_directories(root_ / "plain" / "inner"); + fs::create_directories(root_ / "recursive" / "inner"); + auto service = std::make_shared(); + auto plain = std::make_shared(); + auto recursive = std::make_shared(); + auto plainWatch = service->subscribe(root_ / "plain", false, plain->sink()); + auto recursiveWatch = service->subscribe(root_ / "recursive", true, recursive->sink()); + ASSERT_TRUE(plainWatch.has_value()); + ASSERT_TRUE(recursiveWatch.has_value()); + + // FSEvents reports nothing below a watched directory removed with its contents, so the inner ones go first. + fs::remove(root_ / "plain" / "inner"); + fs::remove(root_ / "recursive" / "inner"); + EXPECT_TRUE(waitFor( + [&]() + { + return plain->has("inner") && recursive->has("inner"); + } + )); + fs::remove(root_ / "plain"); + fs::remove(root_ / "recursive"); + // Gives the watchers time to find their directories gone. + std::this_thread::sleep_for(std::chrono::milliseconds{500}); + + // efsw 1.7.2 waits forever on Windows for a read that a deleted directory never completes. + auto ended = std::make_shared>(); + auto endedFuture = ended->get_future(); + std::thread{ + [ended, + service = std::move(service), + plain, + recursive, + plainWatch = std::move(plainWatch), + recursiveWatch = std::move(recursiveWatch)]() mutable + { + recursiveWatch.reset(); + plainWatch.reset(); + service.reset(); + ended->set_value(); + } + }.detach(); + EXPECT_EQ(endedFuture.wait_for(std::chrono::seconds{5}), std::future_status::ready); + } + + TEST_F(DirectoryWatchServiceTest, WatchesComingAndGoingCostOthersNoChanges) + { + constexpr int otherCount = 20; + constexpr int fileCount = 100; + for (int index = 0; index < otherCount; ++index) + fs::create_directories(root_ / ("other" + std::to_string(index))); + fs::create_directories(root_ / "watched"); + Received received; + const auto watched = service_->subscribe(root_ / "watched", false, received.sink()); + ASSERT_TRUE(watched.has_value()); + + std::atomic writing{true}; + std::thread comingAndGoing{[&]() + { + while (writing) + { + std::vector> others; + for (int index = 0; index < otherCount; ++index) + others.push_back(service_->subscribe( + root_ / ("other" + std::to_string(index)), + false, + [](FileTracking::DirectoryEvent const&) {} + )); + others.clear(); + std::this_thread::sleep_for(std::chrono::milliseconds{10}); + } + }}; + for (int index = 0; index < fileCount; ++index) + { + std::ofstream{root_ / "watched" / ("file" + std::to_string(index) + ".txt")} << index; + std::this_thread::sleep_for(std::chrono::milliseconds{5}); + } + writing = false; + comingAndGoing.join(); + + const auto missing = [&]() + { + std::string names; + for (int index = 0; index < fileCount; ++index) + if (const auto name = "file" + std::to_string(index) + ".txt"; !received.has(name)) + names += name + " "; + return names; + }; + EXPECT_TRUE(waitFor( + [&]() + { + return missing().empty(); + } + )) << "missing: " << missing(); + } + + class DirectoryWatcherTest : public ::testing::Test + { + protected: + boost::asio::io_context ioc_; + fs::path root_; + std::vector changes_{}; + std::unique_ptr watcher_; + + void SetUp() override + { + root_ = fs::canonical(temporaryRoot) / + ("directory_watcher_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count())); + fs::create_directories(root_); + watcher_ = std::make_unique( + boost::asio::make_strand(ioc_.get_executor()), + [this](FileTracking::DirectoryChange const& change) + { + changes_.push_back(change); + } + ); + } + + void TearDown() override + { + watcher_.reset(); + if (!root_.empty()) + fs::remove_all(root_); + } + + /** + * @brief Runs the handlers until @p condition holds, at most for @p timeout. + */ + bool runUntil(std::function const& condition, std::chrono::milliseconds timeout = std::chrono::seconds{5}) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) + { + ioc_.restart(); + ioc_.run_for(std::chrono::milliseconds{20}); + if (condition()) + return true; + } + return false; + } + + /** + * @brief Whether a change of @p filename came that the trackers act on, which a directory reported as + * modified is not (see filterFileChange). NTFS writes a directory's times into its parent's entry when + * a handle to it closes, so a watch of the parent reports it whenever something opens it, efsw too. + */ + bool reported(std::string const& filename) const + { + return std::ranges::any_of( + changes_, + [&filename](FileTracking::DirectoryChange const& change) + { + return change.filename == filename && + !(change.action == FileTracking::FileAction::Modified && change.isDirectory); + } + ); + } + }; + + TEST_F(DirectoryWatcherTest, ChangesMadeBeforeTheWatchBeganAreNotReported) + { + fs::create_directories(root_ / "fresh" / "inner"); + std::ofstream{root_ / "fresh" / "inner" / "before.txt"} << "before"; + const auto plain = watcher_->add(root_, /*recursive=*/false); + const auto recursive = watcher_->add(root_ / "fresh", /*recursive=*/true); + ASSERT_TRUE(plain.has_value()); + ASSERT_TRUE(recursive.has_value()); + + std::ofstream{root_ / "after.txt"} << "after"; + std::ofstream{root_ / "fresh" / "inner" / "later.txt"} << "later"; + + EXPECT_TRUE(runUntil( + [this]() + { + return reported("after.txt") && reported("later.txt"); + } + )); + runUntil( + []() + { + return false; + }, + std::chrono::milliseconds{500} + ); + EXPECT_FALSE(reported("fresh")); + EXPECT_FALSE(reported("before.txt")); + } + + TEST_F(DirectoryWatcherTest, MissedChangesPassTheFilterAfterTheChangesBeforeThem) + { + const auto watch = watcher_->add( + root_, + /*recursive=*/true, + [](FileTracking::DirectoryEvent const& event) + { + return event.filename == "before.txt"; + } + ); + ASSERT_TRUE(watch.has_value()); + // Subscribed after the watcher, efsw's thread hands an event to it once the watcher has it. + const auto service = FileTracking::DirectoryWatchService::shared(); + std::atomic handedOn{false}; + const auto observer = service->subscribe( + root_, + /*recursive=*/true, + [&handedOn](FileTracking::DirectoryEvent const& event) + { + if (event.filename == "before.txt") + handedOn = true; + } + ); + ASSERT_TRUE(observer.has_value()); + + std::ofstream{root_ / "before.txt"} << "before"; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{5}; + while (!handedOn && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds{10}); + ASSERT_TRUE(handedOn); + service->reportMissed(root_); + + const auto isMissed = [](FileTracking::DirectoryChange const& change) + { + return change.action == FileTracking::FileAction::Missed; + }; + ASSERT_TRUE(runUntil( + [&]() + { + return std::ranges::any_of(changes_, isMissed); + } + )); + const auto missed = std::ranges::find_if(changes_, isMissed); + EXPECT_EQ(missed->path(), root_); + const auto before = std::ranges::find_if( + changes_, + [](FileTracking::DirectoryChange const& change) + { + return change.filename == "before.txt"; + } + ); + EXPECT_LT(before, missed); + } + + TEST_F(DirectoryWatcherTest, AnEntryWrittenBeforeAndRenamedAfterTheWatchBeganIsReported) + { + std::ofstream{root_ / "old.txt"} << "old"; + fs::create_directories(root_ / "folder"); + const auto watch = watcher_->add(root_, /*recursive=*/false); + ASSERT_TRUE(watch.has_value()); + + fs::rename(root_ / "old.txt", root_ / "new.txt"); + fs::rename(root_ / "folder", root_ / "renamed"); + + EXPECT_TRUE(runUntil( + [this]() + { + return reported("new.txt") && reported("renamed"); + } + )); + } +} diff --git a/backend/test/backend/test_operation_queue.hpp b/backend/test/backend/test_operation_queue.hpp index b5cf0f99..6f80cd8c 100644 --- a/backend/test/backend/test_operation_queue.hpp +++ b/backend/test/backend/test_operation_queue.hpp @@ -4,6 +4,7 @@ #include #include #include +#include "error_log_capture.hpp" #include "nui_env.hpp" #include "real_server_tests.hpp" @@ -16,6 +17,7 @@ #include #include +#include #include #include #include @@ -48,7 +50,7 @@ namespace Test { if (queue_) runOnStrand([this]() { - queue_->cancelAll(); + queue_->dropAll(); }); queue_.reset(); localQueue_.reset(); @@ -240,7 +242,7 @@ namespace Test [this, ®istry]() { queue_->dumpBulkResumes(registry); - queue_->cancelAll(); + queue_->dropAll(); } ); std::scoped_lock lock{completionsMutex_}; @@ -306,6 +308,34 @@ namespace Test return names; } + /** + * @brief Queues the delete of the file @p path as @p operationId, what its owner hears of its end goes to + * @p heard on the strand. + */ + std::expected addOwnedDelete( + SecureShell::SftpSession& sftp, + Ids::OperationId const& operationId, + std::string const& path, + std::shared_ptr> const& heard + ) + { + return runOnStrand( + [&]() + { + const auto added = queue_->addDeleteOperation(sftp, operationId, remote(path), false, false); + if (added) + queue_->whenCompleted( + operationId, + [heard](SharedData::OperationCompleted const& completed) + { + heard->push_back(completed); + } + ); + return added; + } + ); + } + protected: boost::asio::thread_pool queuePool_{2}; std::shared_ptr> strand_{ @@ -377,6 +407,153 @@ namespace Test EXPECT_EQ(remoteNames(*sftp, "/home/test/uploaded"), (std::set{"a.txt", "b.txt"})); } + TEST_F(OperationQueueTests, UploadsWhoseSourceMayVanishEndCanceledWithoutATransferOnceItIsGone) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto file = localDirectory_.path() / "vanishing.txt"; + const auto directory = localDirectory_.path() / "vanishingDirectory"; + const auto kept = localDirectory_.path() / "mustExist.txt"; + std::ofstream{file} << "file"; + std::filesystem::create_directories(directory); + std::ofstream{directory / "inside.txt"} << "inside"; + std::ofstream{kept} << "kept"; + const auto fileId = Ids::generateOperationId(); + const auto directoryId = Ids::generateOperationId(); + const auto keptId = Ids::generateOperationId(); + const auto add = [&](Ids::OperationId const& operationId, std::filesystem::path const& path, bool mayVanish) + { + return runOnStrand( + [&]() + { + return queue->addUploadOperation( + *sftp, + operationId, + toLocal(path), + remote("/home/test/" + path.filename().string()), + true, + false, + false, + false, + SharedData::OperationMode::Queued, + mayVanish + ); + } + ); + }; + ASSERT_TRUE(add(fileId, file, true).has_value()); + ASSERT_TRUE(add(directoryId, directory, true).has_value()); + ASSERT_TRUE(add(keptId, kept, false).has_value()); + std::filesystem::remove(file); + std::filesystem::remove_all(directory); + std::filesystem::remove(kept); + setPaused(false); + + workUntilCompletions(3); + std::map reasons; + for (auto const& completed : completions()) + reasons.emplace(completed.operationId.value(), completed.reason); + EXPECT_EQ(reasons[fileId.value()], SharedData::OperationCompletionReason::Canceled); + EXPECT_EQ(reasons[directoryId.value()], SharedData::OperationCompletionReason::Canceled); + EXPECT_EQ(reasons[keptId.value()], SharedData::OperationCompletionReason::Failed); + const auto names = remoteNames(*sftp, "/home/test"); + EXPECT_FALSE(names.contains("vanishing.txt")); + EXPECT_FALSE(names.contains("vanishingDirectory")); + } + + TEST_F(OperationQueueTests, AnUploadWhoseSourceMayVanishAndIsGoneAlreadyIsRefusedWithoutAnError) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + const ErrorLogCapture errors; + + const auto add = [&](std::string const& name, bool mayVanish) + { + return runOnStrand( + [&]() + { + return queue->addUploadOperation( + *sftp, + Ids::generateOperationId(), + toLocal(localDirectory_.path() / name), + remote("/home/test/" + name), + true, + false, + false, + false, + SharedData::OperationMode::Queued, + mayVanish + ); + } + ); + }; + const auto vanished = add("vanished.txt", true); + const auto lost = add("lost.txt", false); + + ASSERT_FALSE(vanished.has_value()); + EXPECT_EQ(vanished.error().type, Operation::ErrorType::FileNotFound); + ASSERT_FALSE(lost.has_value()); + EXPECT_EQ(lost.error().type, Operation::ErrorType::FileStatFailed); + EXPECT_EQ(errors.mentioning("vanished.txt"), 0u); + EXPECT_EQ(errors.mentioning("lost.txt"), 1u); + } + + TEST_F(OperationQueueTests, ABulkUploadWhoseSourcesMayVanishSkipsAGoneFileWithoutAnError) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + const ErrorLogCapture errors; + + const auto localPath = localDirectory_.path() / "gathered"; + std::filesystem::create_directories(localPath); + std::ofstream{localPath / "kept.txt"} << "kept"; + std::ofstream{localPath / "vanished.txt"} << "vanished"; + const SharedData::BulkUploadRequest request{ + .entries = + { + {.source = toLocal(localPath / "kept.txt"), + .destination = remote("/home/test/gathered/kept.txt"), + .sizeBytes = 4}, + {.source = toLocal(localPath / "vanished.txt"), + .destination = remote("/home/test/gathered/vanished.txt"), + .sizeBytes = 8}, + }, + .allowOverwrite = true, + }; + runOnStrand( + [&]() + { + return queue->addBulkUploadOperation( + *sftp, + request, + [](std::size_t) + { + return Ids::generateOperationId(); + }, + Ids::generateOperationId(), + /*sourceMayVanish=*/true + ); + } + ); + std::filesystem::remove(localPath / "vanished.txt"); + setPaused(false); + + workUntilCompletions(1); + const auto finished = completions(); + ASSERT_EQ(finished.size(), 1u); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(finished.front().failedEntries.empty()); + // The entry did not go, which its owner hears by index. + ASSERT_EQ(finished.front().indexedFailures.size(), 1u); + EXPECT_EQ(finished.front().indexedFailures.front().index, 1u); + EXPECT_EQ(remoteNames(*sftp, "/home/test/gathered"), std::set{"kept.txt"}); + EXPECT_EQ(errors.mentioning("vanished.txt"), 0u); + } + #ifndef _WIN32 TEST_F(OperationQueueTests, DirectoryUploadReportsAnUnreadableSubdirectory) { @@ -727,7 +904,7 @@ namespace Test [&]() { queue->dumpBulkResumes(registry); - queue->cancelAll(); + queue->dropAll(); } ); auto backup = registry.take(card); @@ -757,6 +934,48 @@ namespace Test EXPECT_TRUE(names.contains("adopted_b.txt")); } + TEST_F(OperationQueueTests, ADirectoryUploadWhoseBackupWasDiscardedIsNotResumed) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(true); + + const auto localPath = localDirectory_.path() / "watched"; + std::filesystem::create_directories(localPath); + std::ofstream{localPath / "a.txt"} << "a"; + const auto kept = Ids::generateOperationId(); + const auto discarded = Ids::generateOperationId(); + runOnStrand( + [&]() + { + for (auto const& operationId : {kept, discarded}) + { + const auto added = queue->addUploadOperation( + *sftp, + operationId, + toLocal(localPath), + remote("/home/test/watched"), + /*allowOverwrite=*/true, + /*isBigFile=*/false, + /*insertRefresh=*/true, + /*createMissingDirectories=*/false, + SharedData::OperationMode::PriorityQueued, + /*sourceMayVanish=*/true + ); + EXPECT_TRUE(added.has_value()) << added.error().toString(); + } + queue->discardResumeBackup(discarded); + } + ); + + BulkResumeRegistry registry; + tearDownInto(registry); + + EXPECT_TRUE(registry.take(kept).has_value()); + EXPECT_FALSE(registry.take(discarded).has_value()); + } + TEST_F(OperationQueueTests, AResumedBulkUploadLeavesOutTheFilesItFinished) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -1016,6 +1235,42 @@ namespace Test EXPECT_EQ(remoteNames(*sftp, "/home/test/Documents"), (std::set{"doc1.txt", "doc2.txt"})); } + TEST_F(OperationQueueTests, ACompletionCallbackRunsOnceForItsOperationOnly) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto moved = Ids::generateOperationId(); + const auto blocked = Ids::generateOperationId(); + std::vector called; + runOnStrand( + [&]() + { + ASSERT_TRUE(queue->addRenameOperation( + *sftp, moved, remote("/home/test/file1.txt"), remote("/home/test/moved.txt") + ).has_value()); + ASSERT_TRUE(queue->addRenameOperation( + *sftp, blocked, remote("/home/test/Documents/doc2.txt"), remote("/home/test/Documents/doc1.txt") + ).has_value()); + queue->whenCompleted( + blocked, + [&called](SharedData::OperationCompleted const& completed) + { + called.push_back(completed); + } + ); + } + ); + + workUntilCompletions(2); + runOnStrand([]() {}); + ASSERT_EQ(called.size(), 1u); + EXPECT_EQ(called.front().operationId, blocked); + EXPECT_EQ(called.front().reason, SharedData::OperationCompletionReason::Failed); + } + TEST_F(OperationQueueTests, AdoptedDirectoryDownloadKeepsItsCardId) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -1117,11 +1372,13 @@ namespace Test }); queue->cancel(directoryDelete); - workUntilCompletions(1); + workUntilCompletions(2); const auto finished = completions(); - ASSERT_EQ(finished.size(), 1u); - EXPECT_EQ(finished.front().operationId, fileDelete); - EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Completed); + ASSERT_EQ(finished.size(), 2u); + EXPECT_EQ(finished.front().operationId, directoryDelete); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Canceled); + EXPECT_EQ(finished.back().operationId, fileDelete); + EXPECT_EQ(finished.back().reason, SharedData::OperationCompletionReason::Completed); EXPECT_EQ(remoteNames(*sftp, "/home/test/victim"), (std::set{"keep.txt"})); EXPECT_FALSE(remoteNames(*sftp, "/home/test").contains("file2.txt")); } @@ -1233,6 +1490,99 @@ namespace Test EXPECT_EQ(finished.front().error->type, SharedData::OperationErrorType::FileExists); } + TEST_F(OperationQueueTests, AChangedDestinationLeftToItsOwnerIsNoFailure) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + setPaused(false); + + const auto localFile = localDirectory_.path() / "changed.txt"; + std::ofstream{localFile} << "changed"; + // Each expects nothing at file1.txt, which the server has. + const auto add = [&](Ids::OperationId const& operationId, bool leftToOwner) + { + return runOnStrand( + [&]() + { + auto added = queue->addUploadOperation( + *sftp, + operationId, + toLocal(localFile), + remote("/home/test/file1.txt"), + true, + false, + false, + false, + SharedData::OperationMode::PriorityQueued, + false, + SharedData::ComparedState{} + ); + if (added && leftToOwner) + queue->leaveChangedDestinationsToOwner(operationId); + return added; + } + ); + }; + const auto owned = Ids::generateOperationId(); + const auto unowned = Ids::generateOperationId(); + ASSERT_TRUE(add(owned, true).has_value()); + ASSERT_TRUE(add(unowned, false).has_value()); + const SharedData::BulkUploadRequest request{ + .entries = + { + {.source = toLocal(localFile), + .destination = remote("/home/test/file2.txt"), + .sizeBytes = 7, + .mayOverwrite = true, + .comparedDestination = SharedData::ComparedState{}}, + {.source = toLocal(localFile), + .destination = remote("/home/test/fresh.txt"), + .sizeBytes = 7, + .mayOverwrite = true, + .comparedDestination = SharedData::ComparedState{}}, + }, + .mode = SharedData::OperationMode::PriorityQueued, + }; + const auto card = Ids::generateOperationId(); + runOnStrand( + [&]() + { + queue->addBulkUploadOperation( + *sftp, + request, + [](std::size_t) + { + return Ids::generateOperationId(); + }, + card + ); + queue->leaveChangedDestinationsToOwner(card); + } + ); + + workUntilCompletions(3); + std::map finished; + for (auto const& completed : completions()) + finished.emplace(completed.operationId.value(), completed); + ASSERT_EQ(finished.size(), 3u); + auto const& ownedEnd = finished.at(owned.value()); + EXPECT_EQ(ownedEnd.reason, SharedData::OperationCompletionReason::Canceled); + ASSERT_TRUE(ownedEnd.error.has_value()) << "the owner has to learn why"; + EXPECT_EQ(ownedEnd.error->type, SharedData::OperationErrorType::ChangedSinceCompare); + auto const& unownedEnd = finished.at(unowned.value()); + EXPECT_EQ(unownedEnd.reason, SharedData::OperationCompletionReason::Failed); + ASSERT_TRUE(unownedEnd.error.has_value()); + EXPECT_EQ(unownedEnd.error->type, SharedData::OperationErrorType::ChangedSinceCompare); + auto const& bulkEnd = finished.at(card.value()); + EXPECT_EQ(bulkEnd.reason, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(bulkEnd.failedEntries.empty()); + ASSERT_EQ(bulkEnd.indexedFailures.size(), 1u); + EXPECT_EQ(bulkEnd.indexedFailures.front().index, 0u); + EXPECT_EQ(bulkEnd.indexedFailures.front().type, SharedData::OperationErrorType::ChangedSinceCompare); + EXPECT_TRUE(remoteNames(*sftp, "/home/test").contains("fresh.txt")); + } + TEST_F(OperationQueueTests, QueuedOperationsOfAClosedChannelFailWithChannelClosed) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -1972,14 +2322,123 @@ namespace Test }); queue->cancel(first); - workUntilCompletions(1); + workUntilCompletions(2); const auto finished = completions(); - ASSERT_EQ(finished.size(), 1u); - EXPECT_EQ(finished.front().operationId, second); + ASSERT_EQ(finished.size(), 2u); + EXPECT_EQ(finished.front().operationId, first); + EXPECT_EQ(finished.front().reason, SharedData::OperationCompletionReason::Canceled); + EXPECT_EQ(finished.back().operationId, second); EXPECT_TRUE(localNames(firstLocal).empty()); EXPECT_EQ(localNames(secondLocal), (std::set{"image1.png", "image2.jpg"})); } + TEST_F(OperationQueueTests, ACancelledOperationTellsItsOwnerItWasCanceled) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto deletion = Ids::generateOperationId(); + const auto heard = std::make_shared>(); + ASSERT_TRUE(addOwnedDelete(*sftp, deletion, "/home/test/file1.txt", heard).has_value()); + queue->cancel(deletion); + runOnStrand([]() {}); + + ASSERT_EQ(heard->size(), 1u); + EXPECT_EQ(heard->front().operationId, deletion); + EXPECT_EQ(heard->front().reason, SharedData::OperationCompletionReason::Canceled); + ASSERT_EQ(completions().size(), 1u); + EXPECT_EQ(completions().front().reason, SharedData::OperationCompletionReason::Canceled); + EXPECT_TRUE(remoteNames(*sftp, "/home/test").contains("file1.txt")); + } + + TEST_F(OperationQueueTests, CancellingAllTellsEveryWaitingOwner) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto first = Ids::generateOperationId(); + const auto second = Ids::generateOperationId(); + const auto heard = std::make_shared>(); + ASSERT_TRUE(addOwnedDelete(*sftp, first, "/home/test/file1.txt", heard).has_value()); + ASSERT_TRUE(addOwnedDelete(*sftp, second, "/home/test/Documents/doc1.txt", heard).has_value()); + queue->cancelAll(); + runOnStrand([]() {}); + + ASSERT_EQ(heard->size(), 2u); + for (auto const& completed : *heard) + EXPECT_EQ(completed.reason, SharedData::OperationCompletionReason::Canceled); + EXPECT_EQ(completions().size(), 2u); + EXPECT_TRUE(remoteNames(*sftp, "/home/test").contains("file1.txt")); + } + + TEST_F(OperationQueueTests, DroppingAllAtTeardownTellsNobody) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + const auto heard = std::make_shared>(); + ASSERT_TRUE(addOwnedDelete(*sftp, Ids::generateOperationId(), "/home/test/file1.txt", heard).has_value()); + queue->dropAll(); + runOnStrand([]() {}); + setPaused(false); + runOnStrand([&]() { + return queue->work(); + }); + + EXPECT_TRUE(heard->empty()); + EXPECT_TRUE(completions().empty()); + EXPECT_TRUE(remoteNames(*sftp, "/home/test").contains("file1.txt")); + } + + TEST_F(OperationQueueTests, AnOwnerCancellingFromItsCompletionLeavesTheOtherOperationsAlone) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + auto queue = makeQueue(); + + // Takes many steps, so the delete queued behind it completes first. + const auto big = localDirectory_.path() / "big.bin"; + std::ofstream{big, std::ios::binary} << std::string(8u * 1024u * 1024u, 'x'); + const auto upload = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addUploadOperation( + *sftp, upload, toLocal(big), remote("/home/test/big.bin"), true, false, false, false + ); + }).has_value()); + const auto deletion = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + const auto added = queue->addDeleteOperation(*sftp, deletion, remote("/home/test/file1.txt"), false, false); + queue->whenCompleted( + deletion, + [weak = std::weak_ptr{queue}, upload](SharedData::OperationCompleted const&) + { + if (const auto owner = weak.lock()) + owner->cancel(upload); + } + ); + return added; + }).has_value()); + const auto later = Ids::generateOperationId(); + ASSERT_TRUE(runOnStrand([&]() { + return queue->addDeleteOperation(*sftp, later, remote("/home/test/Documents/doc2.txt"), false, false); + }).has_value()); + setPaused(false); + + workUntilCompletions(3); + std::map reasons; + for (auto const& completed : completions()) + reasons.emplace(completed.operationId.value(), completed.reason); + EXPECT_EQ(reasons.size(), 3u); + EXPECT_EQ(reasons[upload.value()], SharedData::OperationCompletionReason::Canceled); + EXPECT_EQ(reasons[deletion.value()], SharedData::OperationCompletionReason::Completed); + EXPECT_EQ(reasons[later.value()], SharedData::OperationCompletionReason::Completed); + EXPECT_FALSE(remoteNames(*sftp, "/home/test").contains("big.bin")); + EXPECT_FALSE(remoteNames(*sftp, "/home/test/Documents").contains("doc2.txt")); + } + TEST_F(OperationQueueTests, ReorderedUnitsKeepTheirOwnScans) { CREATE_SERVER_AND_JOINER(sftpServer); diff --git a/backend/test/backend/test_settled_changes.hpp b/backend/test/backend/test_settled_changes.hpp new file mode 100644 index 00000000..b14b0711 --- /dev/null +++ b/backend/test/backend/test_settled_changes.hpp @@ -0,0 +1,198 @@ +#pragma once + +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class SettledChangesTest : public ::testing::Test + { + protected: + boost::asio::io_context ioc_; + boost::asio::strand strand_{ioc_.get_executor()}; + std::vector sent_; + std::shared_ptr changes_ = std::make_shared( + strand_, + [this](FileTracking::ReportedChange const& change) + { + sent_.push_back(change); + }, + std::chrono::milliseconds{20} + ); + + void runFor(std::chrono::milliseconds duration) + { + ioc_.restart(); + ioc_.run_for(duration); + } + + void add( + FileTracking::FileAction action, + std::filesystem::path const& path, + bool isFileContent, + std::filesystem::path const& oldPath = {} + ) + { + changes_->add( + FileTracking::ReportedChange{.action = action, .path = path, .oldPath = oldPath}, isFileContent + ); + } + }; + + using FileTracking::FileAction; + + TEST_F(SettledChangesTest, ContentChangesOfOneFileInARowAreSentOnce) + { + add(FileAction::Modified, "/watched/notes.txt", true); + add(FileAction::Modified, "/watched/notes.txt", true); + EXPECT_TRUE(sent_.empty()); + + runFor(std::chrono::milliseconds{200}); + + ASSERT_EQ(sent_.size(), 1u); + EXPECT_EQ(sent_[0].path, "/watched/notes.txt"); + EXPECT_EQ(sent_[0].action, FileAction::Modified); + } + + TEST_F(SettledChangesTest, AnAdditionStaysAnAddition) + { + add(FileAction::Added, "/watched/new.txt", true); + add(FileAction::Modified, "/watched/new.txt", true); + + runFor(std::chrono::milliseconds{200}); + + ASSERT_EQ(sent_.size(), 1u); + EXPECT_EQ(sent_[0].action, FileAction::Added); + } + + TEST_F(SettledChangesTest, ARemovalLeavesTheHeldBackChangesNextToItAlone) + { + add(FileAction::Modified, "/watched/tree-copy/file.txt", true); + add(FileAction::Modified, "/watched/tree.txt", true); + add(FileAction::Modified, "/watched/tre/file.txt", true); + add(FileAction::Modified, "/watched/tree/file.txt", true); + add(FileAction::Deleted, "/watched/./tree", false); + + ASSERT_EQ(sent_.size(), 2u); + EXPECT_EQ(sent_[0].path, "/watched/tree/file.txt"); + EXPECT_EQ(sent_[1].path, "/watched/./tree"); + + runFor(std::chrono::milliseconds{200}); + + EXPECT_EQ(sent_.size(), 5u); + } + + TEST_F(SettledChangesTest, ContentChangesOfDifferentFilesAreSentEach) + { + add(FileAction::Modified, "/watched/one.txt", true); + add(FileAction::Modified, "/watched/two.txt", true); + + runFor(std::chrono::milliseconds{200}); + + EXPECT_EQ(sent_.size(), 2u); + } + + TEST_F(SettledChangesTest, ARemovalSendsTheHeldBackChangesAtAndBelowItFirst) + { + add(FileAction::Added, "/watched/tree/file.txt", true); + add(FileAction::Modified, "/watched/kept.txt", true); + add(FileAction::Modified, "/watched/tree/later.txt", true); + add(FileAction::Deleted, "/watched/tree", false); + + ASSERT_EQ(sent_.size(), 3u); + EXPECT_EQ(sent_[0].action, FileAction::Added); + EXPECT_EQ(sent_[1].action, FileAction::Modified); + EXPECT_EQ(sent_[2].action, FileAction::Deleted); + runFor(std::chrono::milliseconds{200}); + + EXPECT_EQ(sent_.size(), 4u); + } + + TEST_F(SettledChangesTest, AMoveSendsTheHeldBackChangeOfWhereItCameFromFirst) + { + add(FileAction::Modified, "/watched/old.txt", true); + add(FileAction::Moved, "/watched/new.txt", false, "/watched/old.txt"); + + ASSERT_EQ(sent_.size(), 2u); + EXPECT_EQ(sent_[0].action, FileAction::Modified); + EXPECT_EQ(sent_[1].action, FileAction::Moved); + } + + TEST_F(SettledChangesTest, AChangeOfADirectorySendsTheHeldBackChangesBelowIt) + { + add(FileAction::Modified, "/watched/tree/file.txt", true); + add(FileAction::Added, "/watched/tree", false); + + ASSERT_EQ(sent_.size(), 2u); + EXPECT_EQ(sent_[0].action, FileAction::Modified); + EXPECT_EQ(sent_[1].action, FileAction::Added); + } + + TEST_F(SettledChangesTest, AnotherClassOfChangeAtTheSameFileSendsTheHeldBackOneFirst) + { + add(FileAction::Modified, "/watched/notes.txt", true); + add(FileAction::Deleted, "/watched/notes.txt", false); + + ASSERT_EQ(sent_.size(), 2u); + EXPECT_EQ(sent_[0].action, FileAction::Modified); + EXPECT_EQ(sent_[1].action, FileAction::Deleted); + runFor(std::chrono::milliseconds{200}); + EXPECT_EQ(sent_.size(), 2u); + } + + TEST_F(SettledChangesTest, NothingIsSentOnceItIsGone) + { + add(FileAction::Modified, "/watched/notes.txt", true); + changes_.reset(); + + runFor(std::chrono::milliseconds{200}); + + EXPECT_TRUE(sent_.empty()); + } + + TEST_F(SettledChangesTest, AFileWrittenOnWithoutEventsIsSentOnceItStaysTheSame) + { + const Utility::TemporaryDirectory directory{temporaryRoot, true}; + const auto file = directory.path() / "big.bin"; + changes_ = std::make_shared( + strand_, + [this](FileTracking::ReportedChange const& change) + { + sent_.push_back(change); + }, + std::chrono::milliseconds{100} + ); + std::ofstream writer{file, std::ios::binary}; + writer << "chunk"; + writer.flush(); + add(FileAction::Added, file, true); + + // Windows reports nothing while the writer keeps the file open. + for (int chunk = 0; chunk < 15; ++chunk) + { + runFor(std::chrono::milliseconds{20}); + writer << "chunk"; + writer.flush(); + } + EXPECT_TRUE(sent_.empty()); + writer.close(); + runFor(std::chrono::milliseconds{400}); + + ASSERT_EQ(sent_.size(), 1u); + EXPECT_EQ(sent_[0].action, FileAction::Added); + } +} diff --git a/backend/test/backend/test_tracker.hpp b/backend/test/backend/test_tracker.hpp new file mode 100644 index 00000000..c2da0f77 --- /dev/null +++ b/backend/test/backend/test_tracker.hpp @@ -0,0 +1,3096 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "error_log_capture.hpp" + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +# include +#endif + +extern std::filesystem::path temporaryRoot; + +namespace Test +{ + class TrackerTest : public ::testing::Test + { + protected: + /** + * @brief Records what the tracker queues, the test ends the operations. + */ + class FakeRemote : public FileTracking::TrackerRemote + { + public: + struct Queued + { + std::string kind; + std::string source; + std::string destination; + FileTracking::UploadRequest upload{}; + OnEnded onEnded{}; + OnUploadEnded onUploadEnded{}; + bool ended{false}; + }; + + std::expected, std::string> + stat(Paths::RemotePath const& path) override + { + ++stats; + const auto found = server.find(path.value()); + if (found == server.end()) + return std::nullopt; + return found->second; + } + + std::expected, std::string> + linkStat(Paths::RemotePath const& path) override + { + return stat(path); + } + + std::expected download( + Paths::RemotePath const& source, + Paths::LocalPath const& destination, + std::uint64_t, + OnEnded onEnded + ) override + { + queued.push_back({ + .kind = "download", + .source = source.value(), + .destination = destination.toUtf8(), + .onEnded = std::move(onEnded), + }); + return {}; + } + + std::expected + upload(FileTracking::UploadRequest const& request, OnUploadEnded onEnded) override + { + queued.push_back({ + .kind = "upload", + .source = request.source.toUtf8(), + .destination = request.destination.value(), + .upload = request, + .onEnded = {}, + .onUploadEnded = std::move(onEnded), + }); + return {}; + } + + std::expected + rename(Paths::RemotePath const& source, Paths::RemotePath const& destination, OnEnded onEnded) override + { + queued.push_back({ + .kind = "rename", + .source = source.value(), + .destination = destination.value(), + .upload = {}, + .onEnded = std::move(onEnded), + }); + return {}; + } + + void cancelUpload(Paths::RemotePath const& destination, FileTracking::PartialFile partial) override + { + cancels.emplace_back(destination.value(), partial); + } + + std::expected + discardPartialUpload(Paths::RemotePath const& destination, OnEnded onEnded) override + { + queued.push_back({ + .kind = "discard", + .source = destination.value(), + .destination = {}, + .upload = {}, + .onEnded = std::move(onEnded), + }); + return {}; + } + + std::expected remove(Paths::RemotePath const& path, OnEnded onEnded) override + { + queued.push_back({ + .kind = "remove", + .source = path.value(), + .destination = {}, + .upload = {}, + .onEnded = std::move(onEnded), + }); + return {}; + } + + std::vector of(std::string const& kind) + { + std::vector found; + for (auto& operation : queued) + if (operation.kind == kind) + found.push_back(&operation); + return found; + } + + /** + * @brief The sources of the operations of @p kind, in the order they were queued. + */ + std::vector sourcesOf(std::string const& kind) + { + std::vector sources; + for (auto const* operation : of(kind)) + sources.push_back(operation->source); + return sources; + } + + std::map server{}; + std::vector queued{}; + std::size_t stats{0}; + /** @brief The uploads the tracker stopped, by destination, in order. */ + std::vector> cancels{}; + }; + + class FakeView : public FileTracking::TrackerView + { + public: + void showEntry(SharedData::TrackedEntry const& entry) override + { + entries.insert_or_assign(entry.watchId.value(), entry); + } + + void removeEntry(Ids::WatchId const& watchId) override + { + entries.erase(watchId.value()); + removed.push_back(watchId.value()); + } + + void showEnded(SharedData::EndedWatches const& shown) override + { + ended.push_back(shown); + } + + void ask(SharedData::ServerChangedQuestion const& question) override + { + questions.push_back(question); + } + + void showOrphanedCount(std::size_t count) override + { + orphanedCounts.push_back(count); + } + + void showProgress(SharedData::TrackerProgress const& shown) override + { + progress.push_back(shown); + } + + std::map entries{}; + std::vector removed{}; + std::vector ended{}; + std::vector questions{}; + std::vector orphanedCounts{}; + std::vector progress{}; + }; + + using Opened = std::expected; + + boost::asio::io_context ioc_; + boost::asio::strand strand_{ioc_.get_executor()}; + std::filesystem::path root_; + std::unique_ptr instances_; + std::unique_ptr localWatches_; + SharedData::WatchOptions defaults_{ + .autoReupload = true, + .moveRemoteOnLocalMove = true, + .deleteRemoteOnLocalDelete = true, + .checkRemoteChanges = true, + }; + FakeRemote* remote_{nullptr}; + FakeView* view_{nullptr}; + std::unique_ptr tracker_; + + void SetUp() override + { + root_ = std::filesystem::canonical(temporaryRoot) / + ("tracker_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count())); + std::filesystem::create_directories(root_ / "downloads"); + std::filesystem::create_directories(root_ / "local" / "project" / "src"); + write(root_ / "local" / "notes.txt", "notes"); + write(root_ / "local" / "project" / "src" / "main.cpp", "int main() {}"); + instances_ = std::make_unique(strand_, root_ / "downloads"); + localWatches_ = std::make_unique(strand_, root_ / "downloads"); + auto remote = std::make_unique(); + auto view = std::make_unique(); + remote_ = remote.get(); + view_ = view.get(); + tracker_ = std::make_unique( + strand_, + *instances_, + *localWatches_, + std::move(remote), + std::move(view), + std::string{Persistence::defaultTempFileSuffix}, + [this]() + { + return defaults_; + } + ); + } + + void TearDown() override + { + tracker_.reset(); + localWatches_.reset(); + instances_.reset(); + if (!root_.empty()) + std::filesystem::remove_all(root_); + } + + static void write(std::filesystem::path const& path, std::string const& content) + { + std::ofstream{path, std::ios::binary | std::ios::trunc} << content; + } + + static Paths::RemotePath remote(std::string const& path) + { + return Paths::RemotePath::make(path).value(); + } + + static Paths::LocalPath local(std::filesystem::path const& path) + { + return Paths::LocalPath::make(path).value(); + } + + static FileTracking::ServerState stateOf(std::filesystem::path const& path) + { + const auto states = FileTracking::itemStatesOf(path); + return FileTracking::ServerState{.mtime = states.at(0).mtime, .size = states.at(0).size}; + } + + /** + * @brief The files an upload of @p source sends as the operation tells of them: the regular files at and + * below it, in the state they have now. + */ + static std::vector sentFilesOf(std::filesystem::path const& source) + { + std::vector sent; + for (auto const& state : FileTracking::itemStatesOf(source)) + { + if (const auto current = FileTracking::sourceStateOf(state.path)) + sent.push_back({.path = local(state.path), .state = *current}); + } + return sent; + } + + /** + * @brief Runs the handlers until @p condition holds, at most for @p timeout. + */ + bool + runUntil(std::function const& condition, std::chrono::milliseconds timeout = std::chrono::seconds{5}) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) + { + ioc_.restart(); + ioc_.run_for(std::chrono::milliseconds{20}); + if (condition()) + return true; + } + return false; + } + + void runFor(std::chrono::milliseconds duration) + { + runUntil( + []() + { + return false; + }, + duration + ); + } + + /** + * @brief Opens @p remotePath, the download brings @p content. + */ + Opened open(std::string const& remotePath, bool watch, std::string const& content = "server content") + { + std::optional opened; + tracker_->open( + remote(remotePath), + content.size(), + watch, + [&opened](Opened const& result) + { + opened = result; + } + ); + if (opened) + return *opened; + auto* download = remote_->of("download").back(); + write(download->destination, content); + remote_->server[remotePath] = stateOf(download->destination); + end(*download, SharedData::OperationCompletionReason::Completed); + EXPECT_TRUE(opened.has_value()); + return opened.value_or(std::unexpected{FileTracking::Tracker::OpenFailure{"not answered"}}); + } + + /** + * @brief Ends @p operation. Its callback stays, so nothing may depend on the queue dropping it. An upload that + * completes fails instead when the server no longer has what it expects, else it tells what the + * server has at its destination then. + */ + void end(FakeRemote::Queued& operation, SharedData::OperationCompletionReason reason) + { + operation.ended = true; + if (operation.onEnded) + operation.onEnded(reason); + if (!operation.onUploadEnded) + return; + if (reason != SharedData::OperationCompletionReason::Completed) + return operation.onUploadEnded({.reason = reason}); + if (!holdsExpected(operation)) + return operation.onUploadEnded( + {.reason = SharedData::OperationCompletionReason::Failed, .changedOnServer = true} + ); + operation.onUploadEnded( + {.reason = SharedData::OperationCompletionReason::Completed, + .destinationState = serverStateAt(operation.destination), + .sentFiles = sentFilesOf(operation.source)} + ); + } + + /** + * @brief Completes @p operation, an upload whose compare passed: the server has @p state at its destination. + */ + void completeWith(FakeRemote::Queued& operation, FileTracking::ServerState state) + { + remote_->server[operation.destination] = state; + operation.ended = true; + operation.onUploadEnded( + {.reason = SharedData::OperationCompletionReason::Completed, + .destinationState = state, + .sentFiles = sentFilesOf(operation.source)} + ); + } + + /** + * @brief Ends @p upload as @p ended tells, whatever the server has. + */ + static void endAs(FakeRemote::Queued& upload, FileTracking::UploadEnd const& ended) + { + upload.ended = true; + // The tracker queues what follows while it handles the end, which may move the queued operations. + const auto onUploadEnded = std::move(upload.onUploadEnded); + onUploadEnded(ended); + } + + std::optional serverStateAt(std::string const& path) const + { + const auto found = remote_->server.find(path); + if (found == remote_->server.end()) + return std::nullopt; + return found->second; + } + + /** + * @brief Whether the server still has what @p upload expects at its destination, as its compare sees it: a + * known file that is gone changed too. + */ + bool holdsExpected(FakeRemote::Queued const& upload) const + { + if (!upload.upload.expectedDestination) + return true; + return serverStateAt(upload.destination) == upload.upload.expectedDestination->file; + } + + /** + * @brief Ends every upload on its way as the server would, it has the local files then. + */ + void completeUploads() + { + for (std::size_t index = 0; index < remote_->queued.size(); ++index) + { + auto& operation = remote_->queued[index]; + if (operation.kind != "upload" || operation.ended) + continue; + // The compare comes before the server's copy is replaced. + if (!holdsExpected(operation)) + { + end(operation, SharedData::OperationCompletionReason::Completed); + continue; + } + const auto destination = operation.destination; + const auto source = operation.source; + serve(source, destination); + auto& served = remote_->queued[index]; + served.ended = true; + served.onUploadEnded( + {.reason = SharedData::OperationCompletionReason::Completed, + .destinationState = serverStateAt(destination), + .sentFiles = sentFilesOf(source)} + ); + } + } + + /** + * @brief The server gets @p source at @p destination as an upload brings it: files with their state, + * directories with their content and symbolic links as links. + */ + void serve(std::filesystem::path const& source, std::string const& destination) + { + // An upload skips what it cannot read (past MAX_PATH or being deleted on Windows). + std::error_code statusError; + const auto status = std::filesystem::symlink_status(source, statusError); + if (std::filesystem::is_symlink(status)) + remote_->server[destination] = FileTracking::ServerState{}; + else if (std::filesystem::is_regular_file(status)) + remote_->server[destination] = stateOf(source); + else if (std::filesystem::is_directory(status)) + { + remote_->server[destination] = FileTracking::ServerState{}; + std::error_code listError; + for (auto const& entry : std::filesystem::directory_iterator{source, listError}) + serve(entry.path(), destination + "/" + entry.path().filename().generic_string()); + } + } + + /** + * @brief Ends every delete, rename and discard on its way as the server would. + */ + void completeRemovesAndRenames() + { + for (std::size_t index = 0; index < remote_->queued.size(); ++index) + { + auto& operation = remote_->queued[index]; + if ((operation.kind != "remove" && operation.kind != "rename" && operation.kind != "discard") || + operation.ended) + continue; + // A discard removes the partial file of a stopped upload, which the fake server does not keep. + if (const auto found = remote_->server.find(operation.source); + operation.kind != "discard" && found != remote_->server.end()) + { + if (operation.kind == "rename") + renameOnServer(operation.source, operation.destination); + else + remote_->server.erase(operation.source); + } + end(remote_->queued[index], SharedData::OperationCompletionReason::Completed); + } + } + + /** + * @brief Ends what is on its way and what starts after it, until nothing is left. + */ + void completeEverything() + { + const auto onItsWay = [this]() + { + return std::ranges::any_of( + remote_->queued, + [](FakeRemote::Queued const& operation) + { + return operation.kind != "download" && !operation.ended; + } + ); + }; + for (int round = 0; round < 100; ++round) + { + completeUploads(); + completeRemovesAndRenames(); + runFor(std::chrono::milliseconds{100}); + if (!onItsWay()) + return; + } + ADD_FAILURE() << "operations kept coming"; + } + + /** + * @brief The server renames @p source to @p destination, a directory with its content. + */ + void renameOnServer(std::string const& source, std::string const& destination) + { + std::vector> moved; + for (auto entry = remote_->server.begin(); entry != remote_->server.end();) + { + if (entry->first != source && !entry->first.starts_with(source + "/")) + { + ++entry; + continue; + } + moved.emplace_back(destination + entry->first.substr(source.size()), entry->second); + entry = remote_->server.erase(entry); + } + for (auto& [path, state] : moved) + remote_->server.insert_or_assign(std::move(path), state); + } + + /** + * @brief Ends every upload the tracker stopped as the queue would, as cancelled. + */ + void endStoppedUploads() + { + // Taken first: what follows a stop may queue the next upload to the same destination. + std::vector stopped; + for (std::size_t index = 0; index < remote_->queued.size(); ++index) + { + auto const& operation = remote_->queued[index]; + const bool wasStopped = std::ranges::contains( + remote_->cancels | std::views::keys, operation.destination + ); + if (operation.kind == "upload" && !operation.ended && wasStopped) + stopped.push_back(index); + } + for (const auto index : stopped) + { + remote_->queued[index].ended = true; + // A copy: what follows may queue more and move the queued operations. + const auto ended = remote_->queued[index].onUploadEnded; + ended({.reason = SharedData::OperationCompletionReason::Canceled}); + } + } + + /** + * @brief Runs the handlers until the tracker stopped @p count uploads. + */ + bool runUntilStopped(std::size_t count = 1u) + { + return runUntil( + [this, count]() + { + return remote_->cancels.size() >= count; + } + ); + } + + /** + * @brief The upload to @p destination queued last, none when there is none. + */ + FakeRemote::Queued* lastUploadTo(std::string const& destination) + { + FakeRemote::Queued* found = nullptr; + for (auto* upload : remote_->of("upload")) + if (upload->destination == destination) + found = upload; + return found; + } + + std::size_t uploadsTo(std::string const& destination) + { + std::size_t count = 0; + for (auto const* upload : remote_->of("upload")) + count += upload->destination == destination ? 1u : 0u; + return count; + } + + SharedData::TrackedEntry const& onlyEntry() + { + EXPECT_EQ(view_->entries.size(), 1u); + return view_->entries.begin()->second; + } + + /** + * @brief The progress the view showed last, nothing on its way before it showed any. + */ + SharedData::TrackerProgress lastProgress() const + { + return view_->progress.empty() ? SharedData::TrackerProgress{} : view_->progress.back(); + } + + /** + * @brief Runs the handlers until the view shows @p expected. + */ + bool runUntilProgress(SharedData::TrackerProgress const& expected) + { + return runUntil( + [this, &expected]() + { + return lastProgress() == expected; + } + ); + } + }; + + TEST_F(TrackerTest, OpeningWithoutWatchingDownloadsIntoAnInstanceAndTracksNothing) + { + const auto opened = open("/srv/notes.txt", /*watch=*/false); + + ASSERT_TRUE(opened.has_value()); + EXPECT_EQ(opened->localPath.native().filename(), "notes.txt"); + EXPECT_EQ(opened->localPath.native().parent_path().parent_path(), root_ / "downloads"); + EXPECT_NE(instances_->findInstance(opened->instanceId.value()), nullptr); + EXPECT_TRUE(view_->entries.empty()); + } + + TEST_F(TrackerTest, OpenAndWatchShowsAnEntryOnceTheDownloadIsThere) + { + defaults_.checkRemoteChanges = false; + + const auto opened = open("/srv/notes.txt", /*watch=*/true); + + ASSERT_TRUE(opened.has_value()); + auto const& entry = onlyEntry(); + EXPECT_EQ(entry.name, "notes.txt"); + EXPECT_EQ(entry.remotePath.value(), "/srv/notes.txt"); + EXPECT_EQ( + entry.options, + (SharedData::WatchOptions{ + .autoReupload = true, + .moveRemoteOnLocalMove = false, + .deleteRemoteOnLocalDelete = false, + .checkRemoteChanges = false, + }) + ); + EXPECT_TRUE(entry.downloadedCopy); + EXPECT_FALSE(entry.uploading); + } + + TEST_F(TrackerTest, AnUploadedOriginalStartsWithTheDefaultOptions) + { + defaults_.checkRemoteChanges = false; + + ASSERT_TRUE( + tracker_->uploadAndWatch(local(root_ / "local" / "notes.txt"), remote("/srv/notes.txt")).has_value() + ); + + EXPECT_EQ(onlyEntry().options, defaults_); + EXPECT_FALSE(onlyEntry().downloadedCopy); + } + + TEST_F(TrackerTest, AnOpenRefusedForANameTheLocalSystemCannotHoldIsToldToTheCallerOnly) + { + const ErrorLogCapture errors; + const auto platform = std::exchange(Paths::defaultLocalPlatform(), Paths::LocalPlatform::Windows); + auto restore = Nui::ScopeExit{[platform]() noexcept { + Paths::defaultLocalPlatform() = platform; + }}; + + const auto opened = open("/srv/con", /*watch=*/true); + + ASSERT_FALSE(opened.has_value()); + EXPECT_TRUE(opened.error().has_value()); + EXPECT_TRUE(remote_->of("download").empty()); + EXPECT_EQ(errors.mentioning("/srv/con"), 0u); + } + + TEST_F(TrackerTest, AWatchOverlappingAnotherOnTheServerIsRefusedBeforeItDownloads) + { + std::optional first; + tracker_->open( + remote("/srv/project"), + 0, + true, + [&first](Opened const& opened) + { + first = opened; + } + ); + const auto below = open("/srv/project/main.cpp", /*watch=*/true); + const auto unwatched = open("/srv/project/main.cpp", /*watch=*/false); + + ASSERT_FALSE(below.has_value()); + ASSERT_TRUE(below.error().has_value()); + EXPECT_EQ(Utility::parseLocalizedMessage(*below.error()).key, "backend.fileTracking.remoteOverlaps"); + EXPECT_TRUE(unwatched.has_value()); + EXPECT_EQ(remote_->of("download").size(), 2u); + EXPECT_FALSE(first.has_value()); + } + + TEST_F(TrackerTest, AFailedDownloadEndsItsInstanceAndFreesTheRemotePath) + { + std::optional opened; + tracker_->open( + remote("/srv/notes.txt"), + 0, + true, + [&opened](Opened const& result) + { + opened = result; + } + ); + auto* download = remote_->of("download").back(); + const auto instanceDirectory = std::filesystem::path{download->destination}.parent_path(); + end(*download, SharedData::OperationCompletionReason::Failed); + + ASSERT_TRUE(opened.has_value()); + ASSERT_FALSE(opened->has_value()); + EXPECT_FALSE(opened->error().has_value()); + EXPECT_FALSE(std::filesystem::exists(instanceDirectory)); + EXPECT_TRUE(open("/srv/notes.txt", /*watch=*/true).has_value()); + } + + TEST_F(TrackerTest, ASavedChangeGoesBackToTheServer) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + + write(opened->localPath.native(), "changed locally"); + + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/notes.txt") == 1u; + } + )); + EXPECT_TRUE(view_->questions.empty()); + EXPECT_TRUE(onlyEntry().uploading); + const auto& expected = remote_->of("upload").back()->upload.expectedDestination; + ASSERT_TRUE(expected.has_value()); + EXPECT_EQ(expected->file, remote_->server.at("/srv/notes.txt")); + completeUploads(); + EXPECT_FALSE(onlyEntry().uploading); + EXPECT_EQ(remote_->server.at("/srv/notes.txt"), stateOf(opened->localPath.native())); + // The upload compares the server's copy itself, and its end tells what the server has now. + EXPECT_EQ(remote_->stats, 0u); + } + + TEST_F(TrackerTest, TheNextUploadExpectsWhatTheLastOneSaidTheServerHas) + { + const auto notes = root_ / "local" / "notes.txt"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(notes), remote("/srv/notes.txt")).has_value()); + ASSERT_EQ(remote_->of("upload").size(), 1u); + EXPECT_FALSE(remote_->of("upload").back()->upload.expectedDestination.has_value()); + // A server that keeps its own modification time. + const FileTracking::ServerState kept{.mtime = 42u, .size = 5u}; + completeWith(*remote_->of("upload").back(), kept); + + write(notes, "changed locally"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/notes.txt") == 2u; + } + )); + const auto& expected = remote_->of("upload").back()->upload.expectedDestination; + ASSERT_TRUE(expected.has_value()); + EXPECT_EQ(expected->file, kept); + completeUploads(); + runFor(std::chrono::milliseconds{200}); + EXPECT_TRUE(view_->questions.empty()); + EXPECT_EQ(remote_->server.at("/srv/notes.txt"), stateOf(notes)); + EXPECT_EQ(remote_->stats, 0u); + } + + TEST_F(TrackerTest, AServerCopyChangedMeanwhileIsAskedAboutOnceTheUploadFoundIt) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + remote_->server["/srv/notes.txt"].mtime += 100u; + const auto changedOnServer = remote_->server.at("/srv/notes.txt"); + + write(opened->localPath.native(), "changed locally"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/notes.txt") == 1u; + } + )); + EXPECT_TRUE(view_->questions.empty()); + completeUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return !view_->questions.empty(); + } + )); + EXPECT_EQ(view_->questions.back().remotePath.value(), "/srv/notes.txt"); + EXPECT_EQ(remote_->server.at("/srv/notes.txt"), changedOnServer); + EXPECT_TRUE(onlyEntry().uploading); + tracker_->answer(view_->questions.back().questionId, SharedData::ServerChangedAnswer::Keep); + runFor(std::chrono::milliseconds{200}); + EXPECT_EQ(uploadsTo("/srv/notes.txt"), 1u); + EXPECT_FALSE(onlyEntry().uploading); + EXPECT_EQ(remote_->server.at("/srv/notes.txt"), changedOnServer); + + write(opened->localPath.native(), "changed again"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/notes.txt") == 2u; + } + )); + completeUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return view_->questions.size() == 2u; + } + )); + tracker_->answer(view_->questions.back().questionId, SharedData::ServerChangedAnswer::Replace); + ASSERT_EQ(uploadsTo("/srv/notes.txt"), 3u); + EXPECT_FALSE(remote_->of("upload").back()->upload.expectedDestination.has_value()); + completeUploads(); + EXPECT_EQ(remote_->server.at("/srv/notes.txt"), stateOf(opened->localPath.native())); + EXPECT_FALSE(onlyEntry().uploading); + EXPECT_EQ(remote_->stats, 0u); + } + + TEST_F(TrackerTest, AFileTheServerLostIsAskedAboutBeforeItIsUploadedAgain) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + remote_->server.erase("/srv/notes.txt"); + + write(opened->localPath.native(), "changed locally"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/notes.txt") == 1u; + } + )); + completeUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return !view_->questions.empty(); + } + )); + EXPECT_FALSE(remote_->server.contains("/srv/notes.txt")); + tracker_->answer(view_->questions.back().questionId, SharedData::ServerChangedAnswer::Replace); + ASSERT_EQ(uploadsTo("/srv/notes.txt"), 2u); + completeUploads(); + EXPECT_EQ(remote_->server.at("/srv/notes.txt"), stateOf(opened->localPath.native())); + } + + TEST_F(TrackerTest, AFileThatAppearedOnTheServerBeforeItsFirstUploadIsAskedAbout) + { + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + // FSEvents also delivers the creation of src from just before the watch. Coming after new.cpp, it would + // upload what the server lacks below src without asking. + runFor(std::chrono::milliseconds{1000}); + completeUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return !onlyEntry().uploading; + } + )); + + write(root_ / "local" / "project" / "src" / "new.cpp", "new"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/src/new.cpp") == 1u; + } + )); + const auto& expected = remote_->of("upload").back()->upload.expectedDestination; + ASSERT_TRUE(expected.has_value()); + EXPECT_FALSE(expected->file.has_value()); + remote_->server["/srv/project/src/new.cpp"] = {.mtime = 7u, .size = 9u}; + completeUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return !view_->questions.empty(); + } + )); + EXPECT_EQ(view_->questions.back().remotePath.value(), "/srv/project/src/new.cpp"); + EXPECT_EQ(remote_->server.at("/srv/project/src/new.cpp"), (FileTracking::ServerState{.mtime = 7u, .size = 9u})); + } + + TEST_F(TrackerTest, TheProgressShowsAChangeOnItsWayUntilItsUploadEnded) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + + write(opened->localPath.native(), "changed locally"); + + ASSERT_TRUE(runUntilProgress({.pending = 1u, .done = 0u, .waitingForAnswer = 0u})); + completeUploads(); + ASSERT_TRUE(runUntilProgress({.pending = 0u, .done = 1u, .waitingForAnswer = 0u})); + const auto shown = view_->progress.size(); + runFor(std::chrono::milliseconds{500}); + EXPECT_EQ(view_->progress.size(), shown); + } + + TEST_F(TrackerTest, TheProgressShowsAChangeWaitingForAnAnswer) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + remote_->server["/srv/notes.txt"].mtime += 100u; + + write(opened->localPath.native(), "changed locally"); + + ASSERT_TRUE(runUntilProgress({.pending = 1u, .done = 0u, .waitingForAnswer = 0u})); + completeUploads(); + ASSERT_TRUE(runUntilProgress({.pending = 1u, .done = 0u, .waitingForAnswer = 1u})); + tracker_->answer(view_->questions.back().questionId, SharedData::ServerChangedAnswer::Keep); + ASSERT_TRUE(runUntilProgress({.pending = 0u, .done = 1u, .waitingForAnswer = 0u})); + } + + TEST_F(TrackerTest, TheProgressCountsADeleteOnTheServerUntilItEnded) + { + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return !view_->progress.empty() && lastProgress().pending == 0u; + } + )); + + std::filesystem::remove(root_ / "local" / "project" / "src" / "main.cpp"); + + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty() && lastProgress().pending > 0u; + } + )); + end(*remote_->of("remove").front(), SharedData::OperationCompletionReason::Completed); + completeUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return lastProgress().pending == 0u; + } + )); + EXPECT_GE(lastProgress().done, 1u); + } + + TEST_F(TrackerTest, AnAnswerToAnotherQuestionChangesNothing) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + remote_->server["/srv/notes.txt"].size += 1u; + + write(opened->localPath.native(), "changed locally"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/notes.txt") == 1u; + } + )); + completeUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return !view_->questions.empty(); + } + )); + tracker_->answer(Ids::generateQuestionId(), SharedData::ServerChangedAnswer::Replace); + + EXPECT_EQ(uploadsTo("/srv/notes.txt"), 1u); + EXPECT_TRUE(onlyEntry().uploading); + } + + TEST_F(TrackerTest, UploadAndWatchUploadsTheCanonicalItemFirst) + { + std::error_code linkError; + std::filesystem::create_symlink(root_ / "local" / "notes.txt", root_ / "local" / "link.txt", linkError); + if (linkError) + GTEST_SKIP() << "Cannot create a symbolic link: " << linkError.message(); + + const auto target = tracker_->checkUploadAndWatch(local(root_ / "local" / "link.txt"), remote("/srv")); + ASSERT_TRUE(target.has_value()) << target.error(); + EXPECT_EQ(target->item.native(), root_ / "local" / "notes.txt"); + EXPECT_EQ(target->destination.value(), "/srv/notes.txt"); + + ASSERT_TRUE(tracker_->uploadAndWatch(target->item, target->destination).has_value()); + + const auto uploads = remote_->of("upload"); + ASSERT_EQ(uploads.size(), 1u); + EXPECT_TRUE(uploads.front()->upload.refreshListing); + EXPECT_FALSE(uploads.front()->upload.createMissingDirectories); + EXPECT_EQ(remote_->stats, 0u); + EXPECT_EQ(onlyEntry().remotePath.value(), "/srv/notes.txt"); + EXPECT_TRUE(onlyEntry().uploading); + } + + TEST_F(TrackerTest, UploadAndWatchRefusesOverlapsLocallyAndOnTheServer) + { + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + + const auto sameItem = tracker_->checkUploadAndWatch(local(root_ / "local" / "project" / "src"), remote("/srv")); + const auto sameRemote = + tracker_->uploadAndWatch(local(root_ / "local" / "notes.txt"), remote("/srv/project/notes.txt")); + + ASSERT_FALSE(sameItem.has_value()); + EXPECT_EQ(Utility::parseLocalizedMessage(sameItem.error()).key, "backend.fileTracking.watchOverlaps"); + ASSERT_FALSE(sameRemote.has_value()); + EXPECT_EQ(Utility::parseLocalizedMessage(sameRemote.error()).key, "backend.fileTracking.remoteOverlaps"); + EXPECT_EQ(view_->entries.size(), 1u); + } + + TEST_F(TrackerTest, AFileDeletedInAWatchedDirectoryIsDeletedOnTheServer) + { + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeUploads(); + + std::filesystem::remove(root_ / "local" / "project" / "src" / "main.cpp"); + + ASSERT_TRUE(runUntil( + [this]() + { + // FSEvents may replay the creation of src after the removal renewed its change time, its upload goes + // first. + completeUploads(); + return !remote_->of("remove").empty(); + } + )); + EXPECT_EQ(remote_->of("remove").front()->source, "/srv/project/src/main.cpp"); + } + + TEST_F(TrackerTest, ASlowMoveAsideSaveKeepsTheServerFileUnderItsName) + { + const auto notes = root_ / "local" / "notes.txt"; + const auto backup = root_ / "local" / "notes.txt~"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(notes), remote("/srv/notes.txt")).has_value()); + completeUploads(); + + std::filesystem::rename(notes, backup); + ASSERT_TRUE(runUntil( + [this]() + { + return onlyEntry().name == "notes.txt~"; + } + )); + completeRemovesAndRenames(); + runFor(std::chrono::milliseconds{300}); + write(notes, "written anew"); + std::filesystem::remove(backup); + + // The upload waits in the server order until the rename back ended. + EXPECT_TRUE(runUntil( + [this]() + { + completeRemovesAndRenames(); + return uploadsTo("/srv/notes.txt") == 2u; + } + )); + runFor(std::chrono::milliseconds{500}); + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_TRUE(view_->questions.empty()); + EXPECT_EQ(onlyEntry().name, "notes.txt"); + EXPECT_EQ(onlyEntry().remotePath.value(), "/srv/notes.txt"); + const auto renames = remote_->of("rename"); + ASSERT_EQ(renames.size(), 2u); + EXPECT_EQ(renames[0]->source, "/srv/notes.txt"); + EXPECT_EQ(renames[0]->destination, "/srv/notes.txt~"); + EXPECT_EQ(renames[1]->source, "/srv/notes.txt~"); + EXPECT_EQ(renames[1]->destination, "/srv/notes.txt"); + } + +#ifdef __APPLE__ + TEST_F(TrackerTest, ChangesSpelledInAnotherNormalizationGoToTheFileTheServerHas) + { + const auto directory = root_ / "local" / "uni"; + const auto decomposed = directory / Paths::nativeFromUtf8("cafe\xCC\x81.txt"); + const auto composed = directory / Paths::nativeFromUtf8("caf\xC3\xA9.txt"); + std::filesystem::create_directories(directory); + write(decomposed, "v1"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(directory), remote("/srv/uni")).has_value()); + // FSEvents also delivers the creation of the directory from just before the watch, it uploads it again. + runFor(std::chrono::milliseconds{1000}); + completeUploads(); + remote_->server["/srv/uni/cafe\xCC\x81.txt"] = stateOf(decomposed); + const auto uploadsBefore = remote_->of("upload").size(); + const auto decomposedUploadsBefore = uploadsTo("/srv/uni/cafe\xCC\x81.txt"); + + // APFS finds the file under either spelling, efsw reports every name precomposed. + std::ofstream{composed, std::ios::binary | std::ios::app} << " edited"; + ASSERT_TRUE(runUntil( + [this, uploadsBefore]() + { + return remote_->of("upload").size() > uploadsBefore; + } + )); + runFor(std::chrono::milliseconds{500}); + EXPECT_EQ(uploadsTo("/srv/uni/cafe\xCC\x81.txt"), decomposedUploadsBefore + 1u); + EXPECT_EQ(uploadsTo("/srv/uni/caf\xC3\xA9.txt"), 0u); + completeUploads(); + std::filesystem::rename(decomposed, composed); + + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("rename").empty(); + } + )); + runFor(std::chrono::milliseconds{500}); + EXPECT_EQ(remote_->of("rename").front()->source, "/srv/uni/cafe\xCC\x81.txt"); + EXPECT_EQ(remote_->of("rename").front()->destination, "/srv/uni/caf\xC3\xA9.txt"); + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_EQ(uploadsTo("/srv/uni/caf\xC3\xA9.txt"), 0u); + } + + TEST_F(TrackerTest, ChangesSpelledInAnotherNormalizationFindTheServerFileInEveryScript) + { + const auto directory = root_ / "local" / "kana"; + // A kana with the voiced sound mark, decomposed like Finder writes it and precomposed like efsw reports it. + const auto decomposed = directory / Paths::nativeFromUtf8("\xE3\x81\x8B\xE3\x82\x99.txt"); + const auto composed = directory / Paths::nativeFromUtf8("\xE3\x81\x8C.txt"); + std::filesystem::create_directories(directory); + write(decomposed, "v1"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(directory), remote("/srv/kana")).has_value()); + runFor(std::chrono::milliseconds{1000}); + completeUploads(); + remote_->server["/srv/kana/\xE3\x81\x8B\xE3\x82\x99.txt"] = stateOf(decomposed); + const auto uploadsBefore = remote_->of("upload").size(); + const auto decomposedUploadsBefore = uploadsTo("/srv/kana/\xE3\x81\x8B\xE3\x82\x99.txt"); + + std::ofstream{composed, std::ios::binary | std::ios::app} << " edited"; + ASSERT_TRUE(runUntil( + [this, uploadsBefore]() + { + return remote_->of("upload").size() > uploadsBefore; + } + )); + runFor(std::chrono::milliseconds{500}); + + EXPECT_EQ(uploadsTo("/srv/kana/\xE3\x81\x8B\xE3\x82\x99.txt"), decomposedUploadsBefore + 1u); + EXPECT_EQ(uploadsTo("/srv/kana/\xE3\x81\x8C.txt"), 0u); + } + + TEST_F(TrackerTest, AFileInADecomposedDirectoryMirrorsItsEditsAndItsRename) + { + // Finder names in NFD, efsw reports every name precomposed. + const auto directory = root_ / "local" / Paths::nativeFromUtf8("Cafe\xCC\x81"); + const auto notes = directory / "notes.txt"; + std::filesystem::create_directories(directory); + write(notes, "v1"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(notes), remote("/srv/notes.txt")).has_value()); + // FSEvents also delivers the creation of the file from just before the watch. + runFor(std::chrono::milliseconds{1000}); + completeUploads(); + const auto uploadsBefore = uploadsTo("/srv/notes.txt"); + + write(notes, "v2, edited"); + ASSERT_TRUE(runUntil( + [this, uploadsBefore]() + { + return uploadsTo("/srv/notes.txt") > uploadsBefore; + } + )); + completeUploads(); + std::filesystem::rename(notes, directory / "renamed.txt"); + + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("rename").empty(); + } + )); + EXPECT_EQ(remote_->of("rename").front()->source, "/srv/notes.txt"); + EXPECT_EQ(remote_->of("rename").front()->destination, "/srv/renamed.txt"); + EXPECT_EQ(onlyEntry().name, "renamed.txt"); + EXPECT_TRUE(remote_->of("remove").empty()); + } + + TEST_F(TrackerTest, AFileInADecomposedDirectoryRemovedLocallyIsRemovedOnTheServer) + { + const auto directory = root_ / "local" / Paths::nativeFromUtf8("Cafe\xCC\x81"); + const auto notes = directory / "notes.txt"; + std::filesystem::create_directories(directory); + write(notes, "v1"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(notes), remote("/srv/notes.txt")).has_value()); + runFor(std::chrono::milliseconds{1000}); + completeUploads(); + const auto uploadsBefore = uploadsTo("/srv/notes.txt"); + + std::filesystem::remove(notes); + + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty(); + } + )); + EXPECT_EQ(remote_->of("remove").front()->source, "/srv/notes.txt"); + EXPECT_EQ(uploadsTo("/srv/notes.txt"), uploadsBefore); + } + + TEST_F(TrackerTest, ADecomposedDirectoryMirrorsTheChangesBelowIt) + { + const auto directory = root_ / "local" / Paths::nativeFromUtf8("Cafe\xCC\x81"); + std::filesystem::create_directories(directory); + write(directory / "notes.txt", "v1"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(directory), remote("/srv/cafe")).has_value()); + runFor(std::chrono::milliseconds{1000}); + completeUploads(); + remote_->server["/srv/cafe/notes.txt"] = stateOf(directory / "notes.txt"); + const auto uploadsBefore = uploadsTo("/srv/cafe/notes.txt"); + + write(directory / "notes.txt", "v2, edited"); + ASSERT_TRUE(runUntil( + [this, uploadsBefore]() + { + return uploadsTo("/srv/cafe/notes.txt") > uploadsBefore; + } + )); + completeUploads(); + const auto uploadsAfterTheEdit = uploadsTo("/srv/cafe/notes.txt"); + std::filesystem::remove(directory / "notes.txt"); + + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty(); + } + )); + EXPECT_EQ(remote_->of("remove").front()->source, "/srv/cafe/notes.txt"); + EXPECT_EQ(uploadsTo("/srv/cafe/notes.txt"), uploadsAfterTheEdit); + } +#endif + + TEST_F(TrackerTest, AnOptionTurnedOffIsNoLongerMirrored) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + + tracker_->setOption( + Ids::makeWatchId(onlyEntry().watchId.value()), SharedData::WatchOption::AutoReupload, false + ); + EXPECT_FALSE(onlyEntry().options.autoReupload); + write(opened->localPath.native(), "changed locally"); + runFor(std::chrono::milliseconds{1000}); + + EXPECT_EQ(uploadsTo("/srv/notes.txt"), 0u); + } + + TEST_F(TrackerTest, RemovingADownloadedCopyNeverDeletesTheServerFile) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + const auto watchId = onlyEntry().watchId; + + tracker_->setOption(watchId, SharedData::WatchOption::DeleteRemoteOnLocalDelete, true); + tracker_->setOption(watchId, SharedData::WatchOption::MoveRemoteOnLocalMove, true); + std::filesystem::remove(opened->localPath.native()); + runFor(std::chrono::milliseconds{1000}); + + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_TRUE(onlyEntry().downloadedCopy); + EXPECT_FALSE(onlyEntry().options.deleteRemoteOnLocalDelete); + EXPECT_FALSE(onlyEntry().options.moveRemoteOnLocalMove); + } + + TEST_F(TrackerTest, ADownloadedCopyWrittenAnewAfterItsRemovalGoesToTheServer) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + + std::filesystem::remove(opened->localPath.native()); + runFor(std::chrono::milliseconds{500}); + write(opened->localPath.native(), "written anew"); + + EXPECT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/notes.txt") == 1u; + } + )); + EXPECT_TRUE(remote_->of("remove").empty()); + runFor(FileTracking::copyReturnTime); + EXPECT_TRUE(view_->ended.empty()); + EXPECT_EQ(view_->entries.size(), 1u); + } + + TEST_F(TrackerTest, ADownloadedCopyMovedAsideAndWrittenAnewKeepsItsWatch) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + const auto aside = opened->localPath.native().parent_path() / "notes.backup"; + + std::filesystem::rename(opened->localPath.native(), aside); + runFor(std::chrono::milliseconds{500}); + write(opened->localPath.native(), "saved anew"); + std::filesystem::remove(aside); + + EXPECT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/notes.txt") == 1u; + } + )); + runFor(FileTracking::copyReturnTime); + EXPECT_TRUE(view_->ended.empty()); + EXPECT_EQ(view_->entries.size(), 1u); + EXPECT_TRUE(remote_->of("rename").empty()); + } + + TEST_F(TrackerTest, ADownloadedCopyDeletedForGoodEndsItsWatchAndShowsWhy) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + const auto watchId = onlyEntry().watchId; + + std::filesystem::remove(opened->localPath.native()); + runFor(std::chrono::milliseconds{1000}); + EXPECT_EQ(view_->entries.size(), 1u); + + ASSERT_TRUE(runUntil( + [this]() + { + return !view_->ended.empty(); + }, + FileTracking::copyReturnTime + std::chrono::seconds{2} + )); + EXPECT_EQ(view_->ended.front().reason, SharedData::WatchEndReason::CopyGone); + ASSERT_EQ(view_->ended.front().entries.size(), 1u); + EXPECT_EQ(view_->ended.front().entries.front().watchId, watchId); + EXPECT_EQ(view_->ended.front().entries.front().name, "notes.txt"); + EXPECT_TRUE(view_->entries.empty()); + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_EQ(instances_->findInstance(opened->instanceId.value()), nullptr); + // Nothing was left in the instance, it went with its directory. + EXPECT_FALSE(std::filesystem::exists(opened->localPath.native().parent_path())); + EXPECT_EQ(instances_->countOrphaned(), 0u); + EXPECT_TRUE(open("/srv/notes.txt", /*watch=*/true).has_value()); + } + + TEST_F(TrackerTest, ADownloadedCopyRenamedAwayEndsItsWatchAndStaysForTheCleanup) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + const auto renamed = opened->localPath.native().parent_path() / "renamed.txt"; + + std::filesystem::rename(opened->localPath.native(), renamed); + ASSERT_TRUE(runUntil( + [this]() + { + return !view_->ended.empty(); + }, + FileTracking::copyReturnTime + std::chrono::seconds{2} + )); + write(renamed, "edited after the rename"); + runFor(std::chrono::milliseconds{1000}); + + EXPECT_EQ(view_->ended.front().reason, SharedData::WatchEndReason::CopyGone); + EXPECT_TRUE(view_->entries.empty()); + EXPECT_EQ(uploadsTo("/srv/notes.txt"), 0u); + EXPECT_TRUE(remote_->of("rename").empty()); + EXPECT_TRUE(std::filesystem::exists(renamed)); + EXPECT_EQ(instances_->countOrphaned(), 1u); + } + + TEST_F(TrackerTest, StoppingADownloadedCopyEndsItsInstance) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + const auto watchId = onlyEntry().watchId; + + tracker_->stop(watchId); + + EXPECT_TRUE(view_->entries.empty()); + EXPECT_EQ(view_->removed, std::vector{watchId.value()}); + EXPECT_EQ(instances_->findInstance(opened->instanceId.value()), nullptr); + EXPECT_FALSE(view_->orphanedCounts.empty()); + EXPECT_TRUE(open("/srv/notes.txt", /*watch=*/true).has_value()); + } + + TEST_F(TrackerTest, StoppingAnUploadedOriginalLeavesItAndFreesIt) + { + ASSERT_TRUE( + tracker_->uploadAndWatch(local(root_ / "local" / "notes.txt"), remote("/srv/notes.txt")).has_value() + ); + + tracker_->stop(onlyEntry().watchId); + + EXPECT_TRUE(view_->entries.empty()); + EXPECT_TRUE(std::filesystem::exists(root_ / "local" / "notes.txt")); + EXPECT_TRUE(tracker_->checkUploadAndWatch(local(root_ / "local" / "notes.txt"), remote("/srv")).has_value()); + } + + TEST_F(TrackerTest, DiscardingADownloadEndsItsWatchAndItsInstance) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + + const auto discarded = tracker_->discard(opened->instanceId, opened->localPath); + + ASSERT_TRUE(discarded.has_value()) << discarded.error(); + EXPECT_EQ(*discarded, FileTracking::DiscardOutcome::InstanceDestroyed); + EXPECT_TRUE(view_->entries.empty()); + EXPECT_FALSE(std::filesystem::exists(opened->localPath.native())); + } + + TEST_F(TrackerTest, StoppingAllEndsEveryWatchWithoutTellingTheView) + { + ASSERT_TRUE( + tracker_->uploadAndWatch(local(root_ / "local" / "notes.txt"), remote("/srv/notes.txt")).has_value() + ); + completeUploads(); + + tracker_->stopAll(); + write(root_ / "local" / "notes.txt", "changed locally"); + runFor(std::chrono::milliseconds{1000}); + + EXPECT_EQ(uploadsTo("/srv/notes.txt"), 1u); + EXPECT_TRUE(view_->removed.empty()); + EXPECT_TRUE(tracker_->checkUploadAndWatch(local(root_ / "local" / "notes.txt"), remote("/srv")).has_value()); + } + + TEST_F(TrackerTest, StoppingAllReleasesTheCopiesSoTheyCountAsOrphaned) + { + const auto watched = open("/srv/watched.txt", /*watch=*/true); + const auto opened = open("/srv/opened.txt", /*watch=*/false); + ASSERT_TRUE(watched.has_value()); + ASSERT_TRUE(opened.has_value()); + ASSERT_EQ(instances_->countOrphaned(), 0u); + + tracker_->stopAll(); + + EXPECT_EQ(instances_->findInstance(watched->instanceId.value()), nullptr); + EXPECT_EQ(instances_->findInstance(opened->instanceId.value()), nullptr); + EXPECT_TRUE(std::filesystem::exists(watched->localPath.native())); + EXPECT_TRUE(std::filesystem::exists(opened->localPath.native())); + EXPECT_EQ(instances_->countOrphaned(), 2u); + EXPECT_EQ(instances_->removeOrphaned(), 2); + } + + TEST_F(TrackerTest, ALostConnectionEndsEveryWatchAndShowsWhy) + { + const auto copy = open("/srv/copy.txt", /*watch=*/true); + ASSERT_TRUE(copy.has_value()); + ASSERT_TRUE( + tracker_->uploadAndWatch(local(root_ / "local" / "notes.txt"), remote("/srv/notes.txt")).has_value() + ); + ASSERT_TRUE(runUntilProgress({.pending = 1})); + + tracker_->connectionLost(); + write(root_ / "local" / "notes.txt", "changed locally"); + write(copy->localPath.native(), "changed copy"); + runFor(std::chrono::milliseconds{1000}); + + EXPECT_TRUE(view_->entries.empty()); + EXPECT_EQ(view_->removed.size(), 2u); + ASSERT_EQ(view_->ended.size(), 1u); + EXPECT_EQ(view_->ended.front().reason, SharedData::WatchEndReason::ConnectionLost); + const auto names = view_->ended.front().entries | std::views::transform(&SharedData::TrackedEntry::name) | + std::ranges::to(); + EXPECT_EQ(names, (std::set{"copy.txt", "notes.txt"})); + EXPECT_EQ(lastProgress(), SharedData::TrackerProgress{}); + EXPECT_EQ(uploadsTo("/srv/notes.txt"), 1u); + EXPECT_EQ(uploadsTo("/srv/copy.txt"), 0u); + EXPECT_EQ(instances_->findInstance(copy->instanceId.value()), nullptr); + ASSERT_FALSE(view_->orphanedCounts.empty()); + EXPECT_EQ(view_->orphanedCounts.back(), 1u); + EXPECT_TRUE(tracker_->checkUploadAndWatch(local(root_ / "local" / "notes.txt"), remote("/srv")).has_value()); + } + + TEST_F(TrackerTest, ALostConnectionWithoutWatchesShowsNoEnd) + { + tracker_->connectionLost(); + + EXPECT_TRUE(view_->ended.empty()); + EXPECT_TRUE(view_->removed.empty()); + } + + TEST_F(TrackerTest, StoppingAllDropsADownloadOnItsWayWithItsInstance) + { + std::optional opened; + tracker_->open( + remote("/srv/notes.txt"), + 0, + /*watch=*/true, + [&opened](Opened const& result) + { + opened = result; + } + ); + ASSERT_EQ(remote_->of("download").size(), 1u); + + tracker_->stopAll(); + end(*remote_->of("download").front(), SharedData::OperationCompletionReason::Completed); + + ASSERT_TRUE(opened.has_value()); + EXPECT_FALSE(opened->has_value()); + EXPECT_TRUE(view_->entries.empty()); + EXPECT_TRUE(instances_->listInstanceIds().empty()); + EXPECT_EQ(instances_->countOrphaned(), 0u); + } + + TEST_F(TrackerTest, AFileRecreatedBeforeItsDeletionIsHandledIsOnlyUploaded) + { + const auto file = root_ / "local" / "project" / "src" / "main.cpp"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + + // git rewrites a file by unlinking it and creating it anew. + std::filesystem::remove(file); + write(file, "int main() { return 1; }"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/src/main.cpp") == 1u || !view_->questions.empty(); + } + )); + + EXPECT_TRUE(view_->questions.empty()); + EXPECT_TRUE(remote_->of("remove").empty()); + } + + TEST_F(TrackerTest, TheUploadAfterADeleteOfTheSamePathWaitsForItWithoutAQuestion) + { + const auto file = root_ / "local" / "project" / "src" / "main.cpp"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::remove(file); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty(); + } + )); + write(file, "int main() { return 1; }"); + runFor(std::chrono::milliseconds{1000}); + + EXPECT_TRUE(view_->questions.empty()); + EXPECT_EQ(uploadsTo("/srv/project/src/main.cpp"), 0u); + completeRemovesAndRenames(); + EXPECT_EQ(uploadsTo("/srv/project/src/main.cpp"), 1u); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, TheDeleteOfADirectoryWaitsForTheDeletesOfItsContent) + { + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::remove_all(root_ / "local" / "project" / "src"); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty(); + } + )); + runFor(std::chrono::milliseconds{500}); + + EXPECT_EQ(remote_->sourcesOf("remove"), std::vector{"/srv/project/src/main.cpp"}); + completeRemovesAndRenames(); + EXPECT_EQ( + remote_->sourcesOf("remove"), (std::vector{"/srv/project/src/main.cpp", "/srv/project/src"}) + ); + } + + TEST_F(TrackerTest, TheWatchedDirectoryRemovedWithItsContentLeavesNoServerTree) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + ASSERT_TRUE(remote_->server.contains("/srv/project/src/main.cpp")); + + std::filesystem::remove_all(project); + + // Windows also reports src modified, its delete waits until that change settled. + EXPECT_TRUE(runUntil( + [this]() + { + completeRemovesAndRenames(); + return !remote_->server.contains("/srv/project/src"); + } + )); + EXPECT_FALSE(remote_->server.contains("/srv/project/src/main.cpp")); + } + + TEST_F(TrackerTest, TheWatchedDirectoryMovedAwayKeepsItsServerTree) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + std::filesystem::create_directories(root_ / "trash"); + + std::filesystem::rename(project, root_ / "trash" / "project"); + runFor(std::chrono::milliseconds{1500}); + completeEverything(); + + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_TRUE(remote_->server.contains("/srv/project/src/main.cpp")); + } + + TEST_F(TrackerTest, TheWatchedDirectoryRemovedWithALargeTreeLeavesNoServerTree) + { + const auto project = root_ / "local" / "project"; + for (int directory = 0; directory < 8; ++directory) + { + const auto inner = project / ("d" + std::to_string(directory)); + std::filesystem::create_directories(inner); + for (int file = 0; file < 8; ++file) + write(inner / ("f" + std::to_string(file) + ".txt"), "content"); + } + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + ASSERT_TRUE(remote_->server.contains("/srv/project/d7/f7.txt")); + const auto serverTreeLeft = [this]() + { + return std::ranges::count_if( + remote_->server, + [](auto const& entry) + { + return entry.first.starts_with("/srv/project/"); + } + ); + }; + + std::filesystem::remove_all(project); + + EXPECT_TRUE(runUntil( + [&]() + { + completeRemovesAndRenames(); + return serverTreeLeft() == 0; + }, + std::chrono::seconds{10} + )) << serverTreeLeft() << " entries left"; + } + + TEST_F(TrackerTest, TheWatchedDirectoryRemovedWhileItsWatchIsBusyLeavesNoServerTree) + { + const auto project = root_ / "local" / "project"; + for (int directory = 0; directory < 4; ++directory) + { + const auto inner = project / ("d" + std::to_string(directory)); + std::filesystem::create_directories(inner); + for (int file = 0; file < 8; ++file) + write(inner / ("f" + std::to_string(file) + ".txt"), "content"); + } + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + ASSERT_TRUE(remote_->server.contains("/srv/project/d3/f7.txt")); + // Holds up efsw's thread at every event. Windows drops what changes meanwhile once the watched directory turns + // delete-pending, efsw reports that as missed. Busy, a sleep on Windows lasts a timer tick and leaves efsw's + // thread behind the removal of the item, whose watch ends before it hears of the lost changes. + const auto busyListener = FileTracking::DirectoryWatchService::shared()->subscribe( + project, + /*recursive=*/true, + [](FileTracking::DirectoryEvent const&) + { + const auto until = std::chrono::steady_clock::now() + std::chrono::milliseconds{1}; + while (std::chrono::steady_clock::now() < until) + { + } + } + ); + ASSERT_TRUE(busyListener.has_value()); + const auto serverTreeLeft = [this]() + { + return std::ranges::count_if( + remote_->server, + [](auto const& entry) + { + return entry.first.starts_with("/srv/project/"); + } + ); + }; + + std::filesystem::remove_all(project); + + EXPECT_TRUE(runUntil( + [&]() + { + completeRemovesAndRenames(); + return serverTreeLeft() == 0; + }, + std::chrono::seconds{10} + )) << serverTreeLeft() << " entries left"; + } + +#ifdef __linux__ + TEST_F(TrackerTest, AChangeTheWatchMissedIsUploadedOnceItsDirectoryIsLookedAtAnew) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + // inotify reports a write to the watches of the directory it went through only. + std::filesystem::create_directories(root_ / "elsewhere"); + std::filesystem::create_hard_link(project / "src" / "main.cpp", root_ / "elsewhere" / "main.cpp"); + write(root_ / "elsewhere" / "main.cpp", "int main() { return 2; }"); + runFor(std::chrono::milliseconds{1000}); + ASSERT_EQ(uploadsTo("/srv/project/src/main.cpp"), 0u); + + FileTracking::DirectoryWatchService::shared()->reportMissed(project / "src"); + + EXPECT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/src/main.cpp") == 1u; + } + )); + runFor(std::chrono::milliseconds{500}); + EXPECT_EQ(remote_->of("upload").size(), 2u); + EXPECT_TRUE(remote_->of("remove").empty()); + } + + TEST_F(TrackerTest, AChangeOfADownloadedCopyTheWatchMissedIsUploadedOnceItsDirectoryIsLookedAtAnew) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + const auto copy = opened->localPath.native(); + runFor(std::chrono::milliseconds{500}); + + FileTracking::DirectoryWatchService::shared()->reportMissed(copy.parent_path()); + runFor(std::chrono::milliseconds{500}); + EXPECT_EQ(uploadsTo("/srv/notes.txt"), 0u); + + std::filesystem::create_directories(root_ / "elsewhere"); + std::filesystem::create_hard_link(copy, root_ / "elsewhere" / "notes.txt"); + write(root_ / "elsewhere" / "notes.txt", "edited in the copy"); + runFor(std::chrono::milliseconds{1000}); + ASSERT_EQ(uploadsTo("/srv/notes.txt"), 0u); + FileTracking::DirectoryWatchService::shared()->reportMissed(copy.parent_path()); + + EXPECT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/notes.txt") == 1u; + } + )); + } +#endif + + TEST_F(TrackerTest, ADeletionTheWatchMissedGoesToTheServerOnceItsDirectoryIsLookedAtAnew) + { + const auto project = root_ / "local" / "project"; + write(project / "src" / "util.cpp", "util"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + // Out of the item the deletion reaches no watch, the server keeps the copy of a directory that left. + std::filesystem::rename(project / "src", root_ / "src"); + runFor(std::chrono::milliseconds{1000}); + std::filesystem::remove(root_ / "src" / "main.cpp"); + std::filesystem::rename(root_ / "src", project / "src"); + runFor(std::chrono::milliseconds{1000}); + completeEverything(); + ASSERT_TRUE(remote_->server.contains("/srv/project/src/main.cpp")); + + FileTracking::DirectoryWatchService::shared()->reportMissed(project); + + EXPECT_TRUE(runUntil( + [this]() + { + completeRemovesAndRenames(); + return !remote_->server.contains("/srv/project/src/main.cpp"); + } + )); + runFor(std::chrono::milliseconds{500}); + completeEverything(); + EXPECT_TRUE(remote_->server.contains("/srv/project/src/util.cpp")); + EXPECT_EQ(remote_->sourcesOf("remove"), std::vector{"/srv/project/src/main.cpp"}); + } + + TEST_F(TrackerTest, AWatchedDirectoryGoneWhenItsWatchMissedTheChangesIsDeletedWithItsTreeOnlyOnWindows) + { + const auto project = root_ / "local" / "project"; + write(project / "src" / "util.cpp", "util"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + // Like a watched directory deleted on Windows without POSIX semantics, which loses the last deletions. + std::filesystem::rename(project / "src", root_ / "src"); + runFor(std::chrono::milliseconds{1000}); + std::filesystem::remove_all(root_ / "src"); + ASSERT_TRUE(remote_->server.contains("/srv/project/src/main.cpp")); + + FileTracking::DirectoryWatchService::shared()->reportMissed(project / "src"); + +#ifdef _WIN32 + EXPECT_TRUE(runUntil( + [this]() + { + completeRemovesAndRenames(); + return !remote_->server.contains("/srv/project/src"); + } + )); + EXPECT_FALSE(remote_->server.contains("/srv/project/src/main.cpp")); + EXPECT_FALSE(remote_->server.contains("/srv/project/src/util.cpp")); +#else + // It may have gone to the trash. + runFor(std::chrono::milliseconds{1000}); + completeEverything(); + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_TRUE(remote_->server.contains("/srv/project/src/main.cpp")); + EXPECT_TRUE(remote_->server.contains("/srv/project/src/util.cpp")); +#endif + EXPECT_TRUE(remote_->server.contains("/srv/project")); + } + + TEST_F(TrackerTest, ADirectoryMovedWithinTheItemLeavesNoServerCopyAtItsOldPlace) + { + const auto project = root_ / "local" / "project"; + std::filesystem::create_directories(project / "src" / "x" / "y"); + std::filesystem::create_directories(project / "docs"); + write(project / "src" / "x" / "main.c", "main"); + write(project / "src" / "x" / "y" / "deep.txt", "deep"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::rename(project / "src" / "x", project / "docs" / "x"); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("rename").empty() || uploadsTo("/srv/project/docs/x") == 1u; + } + )); + runFor(std::chrono::milliseconds{500}); + completeEverything(); + + EXPECT_FALSE(remote_->server.contains("/srv/project/src/x/main.c")); + EXPECT_FALSE(remote_->server.contains("/srv/project/src/x")); + EXPECT_TRUE(remote_->server.contains("/srv/project/docs/x/y/deep.txt")); + EXPECT_TRUE(view_->questions.empty()); +#ifndef __APPLE__ + // The watch reports the move between directories as one, the server renames the directory. + ASSERT_EQ(remote_->of("rename").size(), 1u); + EXPECT_EQ(remote_->of("rename").front()->source, "/srv/project/src/x"); + EXPECT_EQ(remote_->of("rename").front()->destination, "/srv/project/docs/x"); + EXPECT_EQ(uploadsTo("/srv/project/docs/x"), 0u); + EXPECT_TRUE(remote_->of("remove").empty()); +#endif + } + + TEST_F(TrackerTest, AFileMovedToAnotherDirectoryOfTheItemIsRenamedOnTheServer) + { + const auto project = root_ / "local" / "project"; + std::filesystem::create_directories(project / "lib"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::rename(project / "src" / "main.cpp", project / "lib" / "main.cpp"); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("rename").empty() || !remote_->of("remove").empty(); + } + )); + runFor(std::chrono::milliseconds{500}); + completeEverything(); + + EXPECT_FALSE(remote_->server.contains("/srv/project/src/main.cpp")); + EXPECT_TRUE(remote_->server.contains("/srv/project/lib/main.cpp")); + EXPECT_TRUE(view_->questions.empty()); +#ifndef __APPLE__ + ASSERT_EQ(remote_->of("rename").size(), 1u); + EXPECT_EQ(remote_->of("rename").front()->source, "/srv/project/src/main.cpp"); + EXPECT_EQ(remote_->of("rename").front()->destination, "/srv/project/lib/main.cpp"); + EXPECT_EQ(uploadsTo("/srv/project/lib/main.cpp"), 0u); + EXPECT_TRUE(remote_->of("remove").empty()); +#endif + } + + TEST_F(TrackerTest, ADirectoryMovedOutAndInAgainLeavesNoServerCopyAtItsOldPlace) + { + const auto project = root_ / "local" / "project"; + std::filesystem::create_directories(project / "src" / "x"); + std::filesystem::create_directories(project / "docs"); + write(project / "src" / "x" / "main.c", "main"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + // Windows and macOS report a move between directories as a removal before the addition. + std::filesystem::rename(project / "src" / "x", root_ / "local" / "x"); + runFor(std::chrono::milliseconds{1000}); + EXPECT_TRUE(remote_->of("remove").empty()); + std::filesystem::rename(root_ / "local" / "x", project / "docs" / "x"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/docs/x") == 1u; + } + )); + completeEverything(); + + auto removed = remote_->sourcesOf("remove"); + std::ranges::sort(removed); + EXPECT_EQ(removed, (std::vector{"/srv/project/src/x", "/srv/project/src/x/main.c"})); + } + + TEST_F(TrackerTest, ADirectoryMovedOutOfTheItemKeepsItsServerCopy) + { + const auto project = root_ / "local" / "project"; + std::filesystem::create_directories(project / "src" / "x"); + write(project / "src" / "x" / "main.c", "main"); + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::rename(project / "src" / "x", root_ / "local" / "x"); + runFor(std::chrono::milliseconds{1000}); + completeEverything(); + + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_TRUE(remote_->server.contains("/srv/project/src/x/main.c")); + } + + TEST_F(TrackerTest, AChangeOfAPathGoneMeanwhileRaisesNoQuestionAndDeletesItOnTheServer) + { + const auto file = root_ / "local" / "project" / "src" / "main.cpp"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + remote_->server["/srv/project/src/main.cpp"].mtime += 100u; + + write(file, "changed"); + std::filesystem::remove(file); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty() || !view_->questions.empty(); + } + )); + runFor(std::chrono::milliseconds{500}); + + EXPECT_TRUE(view_->questions.empty()); + EXPECT_EQ(remote_->sourcesOf("remove"), std::vector{"/srv/project/src/main.cpp"}); + EXPECT_EQ(uploadsTo("/srv/project/src/main.cpp"), 0u); + } + + TEST_F(TrackerTest, AFileTurnedIntoADirectoryWhileItsUploadIsOnItsWayReplacesTheServerFile) + { + const auto kind = root_ / "local" / "project" / "kind"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + + write(kind, "file"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/kind") == 1u; + } + )); + // git checkout turns the file into a directory while its upload runs, the server got the file. + std::filesystem::remove(kind); + std::filesystem::create_directory(kind); + write(kind / "inside.txt", "inside"); + runFor(std::chrono::milliseconds{1000}); + completeWith(*remote_->of("upload").back(), FileTracking::ServerState{.mtime = 1, .size = 4}); + completeEverything(); + + std::vector atKind; + for (auto const& operation : remote_->queued) + if (operation.source == "/srv/project/kind" || operation.destination == "/srv/project/kind") + atKind.push_back(operation.kind); + EXPECT_EQ(atKind, (std::vector{"upload", "remove", "upload"})); + EXPECT_TRUE(view_->questions.empty()); + EXPECT_TRUE(remote_->server.contains("/srv/project/kind/inside.txt")); + } + + TEST_F(TrackerTest, ADirectoryDeletedWhileItsUploadRunsButThereAgainAfterwardsIsUploadedAgain) + { + const auto directory = root_ / "local" / "project" / "dir"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::create_directory(directory); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/dir") == 1u; + } + )); + std::filesystem::remove(directory); + runFor(std::chrono::milliseconds{500}); + // git brings it back, its addition is reported after the upload ended. The upload sent it before, empty. + std::filesystem::create_directory(directory); + write(directory / "a.txt", "a"); + auto* first = remote_->of("upload").back(); + remote_->server["/srv/project/dir"] = FileTracking::ServerState{}; + first->ended = true; + first->onUploadEnded({.reason = SharedData::OperationCompletionReason::Completed}); + completeEverything(); + + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_EQ(uploadsTo("/srv/project/dir"), 1u); + EXPECT_EQ(uploadsTo("/srv/project/dir/a.txt"), 1u); + EXPECT_TRUE(remote_->server.contains("/srv/project/dir/a.txt")); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, ADirectoryRemovedJustAfterItsUploadSentItLeavesNothingOnTheServer) + { + const auto project = root_ / "local" / "project"; + const auto directory = project / "moved-dir"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + // git checks out a branch with the directory, the next checkout removes it again while it uploads. + std::filesystem::create_directory(directory); + write(directory / "m0.txt", "m0"); + write(directory / "m1.txt", "m1"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/moved-dir") == 1u; + } + )); + auto* upload = remote_->of("upload").back(); + const auto sent = sentFilesOf(directory); + serve(directory, "/srv/project/moved-dir"); + // Gone before the upload ended, the removals are reported afterwards. + std::filesystem::remove_all(directory); + endAs(*upload, {.reason = SharedData::OperationCompletionReason::Completed, .sentFiles = sent}); + runFor(std::chrono::milliseconds{1000}); + completeEverything(); + + EXPECT_FALSE(remote_->server.contains("/srv/project/moved-dir/m0.txt")); + EXPECT_FALSE(remote_->server.contains("/srv/project/moved-dir/m1.txt")); + EXPECT_FALSE(remote_->server.contains("/srv/project/moved-dir")); + const auto removed = remote_->sourcesOf("remove"); + ASSERT_FALSE(removed.empty()); + EXPECT_EQ(removed.back(), "/srv/project/moved-dir"); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, ADirectoryRemovedWhileItUploadsLeavesNothingOfWhatTheUploadSent) + { + const auto project = root_ / "local" / "project"; + const auto directory = project / "moved-dir"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::create_directory(directory); + write(directory / "m0.txt", "m0"); + write(directory / "m1.txt", "m1"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/moved-dir") == 1u; + } + )); + auto* upload = remote_->of("upload").back(); + auto sent = sentFilesOf(directory); + // m0.txt went while it was read, the upload tells it without a state. + for (auto& file : sent) + if (file.path.filename()->view() == "m0.txt") + file.state.reset(); + serve(directory, "/srv/project/moved-dir"); + std::filesystem::remove_all(directory); + runFor(std::chrono::milliseconds{1000}); + endAs(*upload, {.reason = SharedData::OperationCompletionReason::Completed, .sentFiles = sent}); + completeEverything(); + + EXPECT_FALSE(remote_->server.contains("/srv/project/moved-dir/m0.txt")); + EXPECT_FALSE(remote_->server.contains("/srv/project/moved-dir/m1.txt")); + EXPECT_FALSE(remote_->server.contains("/srv/project/moved-dir")); + const auto removed = remote_->sourcesOf("remove"); + ASSERT_FALSE(removed.empty()); + EXPECT_EQ(removed.back(), "/srv/project/moved-dir"); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, AnEmptyDirectoryAnUploadLeftAfterItsFilesWentIsDeletedOnTheServer) + { + const auto project = root_ / "local" / "project"; + const auto directory = project / "moved-dir"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::create_directory(directory); + write(directory / "m0.txt", "m0"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/moved-dir") == 1u; + } + )); + auto* upload = remote_->of("upload").back(); + // The upload created the directory, its files were gone by the time it got to them. + remote_->server["/srv/project/moved-dir"] = FileTracking::ServerState{}; + std::filesystem::remove_all(directory); + endAs(*upload, {.reason = SharedData::OperationCompletionReason::Completed}); + runFor(std::chrono::milliseconds{1000}); + completeEverything(); + + EXPECT_FALSE(remote_->server.contains("/srv/project/moved-dir")); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, AFileDeletedWhileItsDirectoryUploadsIsDeletedOnTheServerIfTheUploadSentIt) + { + const auto project = root_ / "local" / "project"; + const auto staging = root_ / "staging"; + const auto directory = project / "moved-in"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + std::filesystem::create_directory(staging); + write(staging / "keep.txt", "keep"); + write(staging / "gone.txt", "gone"); + + // Moved in, its files are reported with it only. + std::filesystem::rename(staging, directory); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/moved-in") == 1u; + } + )); + auto* upload = remote_->of("upload").back(); + const auto sent = sentFilesOf(directory); + serve(directory, "/srv/project/moved-in"); + // An editor's temporary file, its removal is reported while the upload still runs. + std::filesystem::remove(directory / "gone.txt"); + runFor(std::chrono::milliseconds{1000}); + endAs(*upload, {.reason = SharedData::OperationCompletionReason::Completed, .sentFiles = sent}); + completeEverything(); + + EXPECT_EQ(remote_->sourcesOf("remove"), std::vector{"/srv/project/moved-in/gone.txt"}); + EXPECT_TRUE(remote_->server.contains("/srv/project/moved-in/keep.txt")); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, ALinkUploadedWithItsDirectoryIsDeletedOnTheServerToo) + { + const auto project = root_ / "local" / "project"; + std::error_code linkError; + std::filesystem::create_directory_symlink("src", project / "link-to-src", linkError); + if (linkError) + GTEST_SKIP() << "Cannot create a symbolic link: " << linkError.message(); + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::remove(project / "link-to-src"); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty(); + } + )); + + EXPECT_EQ(remote_->sourcesOf("remove"), std::vector{"/srv/project/link-to-src"}); + } + + TEST_F(TrackerTest, ALinkToADirectoryAddedLaterIsDeletedOnTheServerToo) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + std::error_code linkError; + std::filesystem::create_directory_symlink("src", project / "link-to-src", linkError); + if (linkError) + GTEST_SKIP() << "Cannot create a symbolic link: " << linkError.message(); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/link-to-src") == 1u; + } + )); + completeEverything(); + std::filesystem::remove(project / "link-to-src"); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty(); + } + )); + + EXPECT_EQ(remote_->sourcesOf("remove"), std::vector{"/srv/project/link-to-src"}); + } + + TEST_F(TrackerTest, ALinkRetargetedOverItselfRaisesNoQuestion) + { + const auto project = root_ / "local" / "project"; + std::error_code linkError; + std::filesystem::create_symlink("src/main.cpp", project / "in", linkError); + if (linkError) + GTEST_SKIP() << "Cannot create a symbolic link: " << linkError.message(); + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + // ln -sfn creates a link under a temporary name and renames it over the old one. + write(project / "src" / "other.cpp", "other"); + std::filesystem::create_symlink("src/other.cpp", project / "in.retarget", linkError); + std::filesystem::rename(project / "in.retarget", project / "in"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/in") == 1u || !view_->questions.empty(); + } + )); + + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, AnOpenedCopyIsNotUploadedBackUnchanged) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + + runFor(std::chrono::milliseconds{1000}); + + EXPECT_EQ(uploadsTo("/srv/notes.txt"), 0u); + } + + TEST_F(TrackerTest, UploadAndWatchUploadsWhatWasWrittenJustBeforeOnce) + { + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeUploads(); + + runFor(std::chrono::milliseconds{1000}); + + EXPECT_EQ(remote_->of("upload").size(), 1u); + } + + TEST_F(TrackerTest, AFileCreatedWithItsDirectoryGoesOnceWhenItIsReportedAfterTheDirectoryUpload) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + const auto uploadsBefore = remote_->of("upload").size(); + + // mkdir -p new/a/b && echo x > new/a/b/c.txt: the directory goes at once, the file once its change settled. + std::filesystem::create_directories(project / "new" / "a" / "b"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/new") == 1u; + } + )); + write(project / "new" / "a" / "b" / "c.txt", "x"); + // The directory's upload read the file. + completeUploads(); + runFor(std::chrono::milliseconds{1000}); + completeEverything(); + + EXPECT_EQ(remote_->of("upload").size(), uploadsBefore + 1u); + EXPECT_EQ(uploadsTo("/srv/project/new/a/b/c.txt"), 0u); + EXPECT_TRUE(remote_->server.contains("/srv/project/new/a/b/c.txt")); + } + + TEST_F(TrackerTest, ContentCreatedWithItsDirectoryGoesOnceWhenItIsReportedWhileTheDirectoryUploads) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + const auto uploadsBefore = remote_->of("upload").size(); + + std::filesystem::create_directory(project / "new"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/new") == 1u; + } + )); + std::filesystem::create_directory(project / "new" / "sub"); + write(project / "new" / "one.txt", "one"); + write(project / "new" / "sub" / "two.txt", "two"); + // The changes come while the directory's upload runs, which read them all. + runFor(std::chrono::milliseconds{1000}); + completeUploads(); + runFor(std::chrono::milliseconds{500}); + completeEverything(); + + EXPECT_EQ(remote_->of("upload").size(), uploadsBefore + 1u); + EXPECT_TRUE(remote_->server.contains("/srv/project/new/one.txt")); + EXPECT_TRUE(remote_->server.contains("/srv/project/new/sub/two.txt")); + } + + TEST_F(TrackerTest, AFileChangedAfterItsDirectoryUploadSentItGoesAgain) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::create_directory(project / "new"); + write(project / "new" / "one.txt", "first"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/new") == 1u; + } + )); + completeUploads(); + write(project / "new" / "one.txt", "second, after the upload read the first"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/new/one.txt") == 1u; + } + )); + completeEverything(); + + EXPECT_EQ(remote_->server.at("/srv/project/new/one.txt"), stateOf(project / "new" / "one.txt")); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, ASubdirectoryReportedWhileItsParentUploadsGoesWithIt) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + const auto uploadsBefore = remote_->of("upload").size(); + + std::filesystem::create_directory(project / "new"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/new") == 1u; + } + )); + runFor(std::chrono::milliseconds{200}); + std::filesystem::create_directory(project / "new" / "sub"); + runFor(std::chrono::milliseconds{500}); + completeUploads(); + runFor(std::chrono::milliseconds{500}); + completeEverything(); + + EXPECT_EQ(remote_->of("upload").size(), uploadsBefore + 1u); + EXPECT_TRUE(remote_->server.contains("/srv/project/new/sub")); + } + + TEST_F(TrackerTest, TheProgressCountsTheEntriesOfADirectoryMovedIn) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + ASSERT_TRUE(runUntilProgress({.pending = 0u, .done = 3u, .waitingForAnswer = 0u})); + const auto staging = root_ / "staging"; + std::filesystem::create_directories(staging / "sub"); + write(staging / "one.txt", "one"); + write(staging / "sub" / "two.txt", "two"); + + std::filesystem::rename(staging, project / "moved"); + + ASSERT_TRUE(runUntilProgress({.pending = 4u, .done = 0u, .waitingForAnswer = 0u})); + completeUploads(); + ASSERT_TRUE(runUntilProgress({.pending = 0u, .done = 4u, .waitingForAnswer = 0u})); + runFor(std::chrono::milliseconds{500}); + EXPECT_EQ(lastProgress(), (SharedData::TrackerProgress{.pending = 0u, .done = 4u, .waitingForAnswer = 0u})); + } + + TEST_F(TrackerTest, TheProgressCountsANewDirectoryAndTheLateReportsOfItsContentOnce) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::create_directory(project / "new"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/new") == 1u; + } + )); + write(project / "new" / "one.txt", "one"); + write(project / "new" / "two.txt", "two"); + write(project / "new" / "three.txt", "three"); + runFor(std::chrono::milliseconds{1000}); + completeUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return lastProgress().pending == 0u; + } + )); + runFor(std::chrono::milliseconds{500}); + completeEverything(); + + EXPECT_EQ(lastProgress(), (SharedData::TrackerProgress{.pending = 0u, .done = 4u, .waitingForAnswer = 0u})); + } + + TEST_F(TrackerTest, TheProgressCountsWhatADirectoryUploadLeftNotChangesThatCameToNothing) + { + const auto project = root_ / "local" / "project"; + const auto staging = root_ / "staging"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + ASSERT_TRUE(runUntilProgress({.pending = 0u, .done = 3u, .waitingForAnswer = 0u})); + std::filesystem::create_directory(staging); + write(staging / "one.txt", "one"); + + std::filesystem::rename(staging, project / "moved-in"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/moved-in") == 1u; + } + )); + // An editor's temporary files come and go while the directory uploads. + for (auto const* name : {"a.tmp", "b.tmp", "c.tmp"}) + { + write(project / "moved-in" / name, "temporary"); + std::filesystem::remove(project / "moved-in" / name); + } + runFor(std::chrono::milliseconds{1000}); + completeUploads(); + runFor(std::chrono::milliseconds{500}); + completeEverything(); + + EXPECT_TRUE(runUntilProgress({.pending = 0u, .done = 2u, .waitingForAnswer = 0u})); + EXPECT_TRUE(remote_->of("remove").empty()); + } + + TEST_F(TrackerTest, TheProgressShowsAtOnceThatNothingIsOnItsWayAnyMore) + { + const auto opened = open("/srv/notes.txt", /*watch=*/true); + ASSERT_TRUE(opened.has_value()); + write(opened->localPath.native(), "changed locally"); + ASSERT_TRUE(runUntilProgress({.pending = 1u, .done = 0u, .waitingForAnswer = 0u})); + + completeUploads(); + ioc_.restart(); + ioc_.run_for(std::chrono::milliseconds{50}); + + EXPECT_EQ(lastProgress(), (SharedData::TrackerProgress{.pending = 0u, .done = 1u, .waitingForAnswer = 0u})); + EXPECT_FALSE(onlyEntry().uploading); + } + +#ifdef _WIN32 + TEST_F(TrackerTest, EntriesPastMaxPathAreNeitherUploadedNorDeletedOnTheServer) + { + const auto project = root_ / "local" / "project"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(project), remote("/srv/project")).has_value()); + completeEverything(); + + // Without long path support std::filesystem does not find these, although they exist. + const std::wstring segment(48, L's'); + auto deep = project; + for (int level = 0; level < 6; ++level) + deep /= segment; + const auto extended = [](std::filesystem::path const& path) + { + return std::filesystem::path{LR"(\\?\)" + path.lexically_normal().make_preferred().native()}; + }; + auto removeTree = Nui::ScopeExit{[&]() noexcept { + std::error_code removeError; + std::filesystem::remove_all(extended(project / segment), removeError); + }}; + std::filesystem::create_directories(extended(deep)); + std::ofstream{extended(deep / "long-file.txt")} << "long"; + ASSERT_GE((deep / "long-file.txt").native().size(), std::size_t{MAX_PATH}); + for (int round = 0; round < 10; ++round) + { + completeUploads(); + runFor(std::chrono::milliseconds{200}); + } + + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_EQ( + std::ranges::count_if( + remote_->server, + [](auto const& entry) + { + return entry.first.ends_with("/long-file.txt"); + } + ), + 0 + ); + } +#else + TEST_F(TrackerTest, ADeletionThatCannotBeCheckedKeepsTheServerCopy) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root can look into any directory"; + const auto source = root_ / "local" / "project" / "src"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + + // Deleted, but its directory no longer lets anyone look: for the tracker it may as well still be there. + std::filesystem::remove(source / "main.cpp"); + std::filesystem::permissions(source, std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[&source]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(source, std::filesystem::perms::owner_all, errorCode); + }}; + runFor(std::chrono::milliseconds{1000}); + + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_TRUE(remote_->server.contains("/srv/project/src/main.cpp")); + } + + TEST_F(TrackerTest, AChangeThatCannotBeCheckedIsNotUploaded) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root can look into any directory"; + const auto source = root_ / "local" / "project" / "src"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + + write(source / "new.cpp", "new"); + std::filesystem::permissions(source, std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[&source]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(source, std::filesystem::perms::owner_all, errorCode); + }}; + runFor(std::chrono::milliseconds{1000}); + + EXPECT_EQ(uploadsTo("/srv/project/src/new.cpp"), 0u); + EXPECT_TRUE(remote_->of("remove").empty()); + } +#endif + + TEST_F(TrackerTest, ANewFileDeletedWhileItUploadsStopsTheUploadAndDeletesNothing) + { + const auto file = root_ / "local" / "project" / "big.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/big.bin") == 1u; + } + )); + + std::filesystem::remove(file); + + ASSERT_TRUE(runUntilStopped()); + EXPECT_EQ(remote_->cancels.front().first, "/srv/project/big.bin"); + EXPECT_EQ(remote_->cancels.front().second, FileTracking::PartialFile::Discarded); + endStoppedUploads(); + completeEverything(); + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_TRUE(remote_->of("discard").empty()); + EXPECT_FALSE(remote_->server.contains("/srv/project/big.bin")); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, AKnownFileDeletedWhileItsChangeUploadsStopsItThenDeletesTheServerCopy) + { + const auto file = root_ / "local" / "project" / "src" / "main.cpp"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "int main() { return 1; }"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/src/main.cpp") == 1u; + } + )); + + std::filesystem::remove(file); + + ASSERT_TRUE(runUntilStopped()); + endStoppedUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty(); + } + )); + EXPECT_EQ(remote_->of("remove").front()->source, "/srv/project/src/main.cpp"); + EXPECT_TRUE(remote_->of("discard").empty()); + } + + TEST_F(TrackerTest, AnUploadThatFinishedBeforeItsStopStillDeletesTheServerCopy) + { + const auto file = root_ / "local" / "project" / "big.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/big.bin") == 1u; + } + )); + std::filesystem::remove(file); + ASSERT_TRUE(runUntilStopped()); + + completeWith(*lastUploadTo("/srv/project/big.bin"), FileTracking::ServerState{.mtime = 1u, .size = 13u}); + + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("remove").empty(); + } + )); + EXPECT_EQ(remote_->of("remove").front()->source, "/srv/project/big.bin"); + } + + TEST_F(TrackerTest, AFileDeletedAndWrittenAgainWhileItUploadsGoesOnceMore) + { + const auto file = root_ / "local" / "project" / "big.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/big.bin") == 1u; + } + )); + std::filesystem::remove(file); + ASSERT_TRUE(runUntilStopped()); + + write(file, "written again"); + runFor(std::chrono::milliseconds{500}); + endStoppedUploads(); + + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/big.bin") == 2u; + } + )); + completeEverything(); + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_EQ(remote_->server.at("/srv/project/big.bin"), stateOf(file)); + } + + TEST_F(TrackerTest, ADeleteReportedAfterTheFileCameBackStopsNothing) + { + const auto file = root_ / "local" / "project" / "big.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/big.bin") == 1u; + } + )); + + // Both happen before the tracker hears of the deletion, as when Windows holds it back. + std::filesystem::remove(file); + write(file, "written again"); + runFor(std::chrono::milliseconds{1000}); + + EXPECT_TRUE(remote_->cancels.empty()); + completeEverything(); + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_EQ(remote_->server.at("/srv/project/big.bin"), stateOf(file)); + } + + TEST_F(TrackerTest, AFileMovedOverAKnownFileWhileItUploadsStopsIt) + { + const auto file = root_ / "local" / "project" / "src" / "new.cpp"; + const auto known = root_ / "local" / "project" / "src" / "main.cpp"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "int main() { return 2; }"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/src/new.cpp") == 1u; + } + )); + + std::filesystem::rename(file, known); + + ASSERT_TRUE(runUntilStopped()); + EXPECT_EQ(remote_->cancels.front().first, "/srv/project/src/new.cpp"); + EXPECT_EQ(remote_->cancels.front().second, FileTracking::PartialFile::Discarded); + endStoppedUploads(); + completeEverything(); + EXPECT_TRUE(remote_->of("rename").empty()); + EXPECT_EQ(remote_->server.at("/srv/project/src/main.cpp"), stateOf(known)); + EXPECT_FALSE(remote_->server.contains("/srv/project/src/new.cpp")); + } + + TEST_F(TrackerTest, ADirectoryDeletedWhileItUploadsIsNotStopped) + { + const auto directory = root_ / "local" / "project" / "dir"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + std::filesystem::create_directory(directory); + write(directory / "a.txt", "a"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/dir") == 1u; + } + )); + + std::filesystem::remove_all(directory); + runFor(std::chrono::milliseconds{1000}); + + for (auto const& destination : std::views::keys(remote_->cancels)) + ADD_FAILURE() << "stopped the upload to " << destination; + } + + TEST_F(TrackerTest, TheProgressCountsAStoppedUploadAsDone) + { + const auto file = root_ / "local" / "project" / "big.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + runFor(std::chrono::milliseconds{500}); + write(file, "large content"); + ASSERT_TRUE(runUntilProgress({.pending = 1u, .done = 0u, .waitingForAnswer = 0u})); + + std::filesystem::remove(file); + ASSERT_TRUE(runUntilStopped()); + endStoppedUploads(); + + EXPECT_TRUE(runUntilProgress({.pending = 0u, .done = 1u, .waitingForAnswer = 0u})); + } + + TEST_F(TrackerTest, StoppingAWatchLeavesItsRunningUploadAlone) + { + const auto file = root_ / "local" / "project" / "big.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/big.bin") == 1u; + } + )); + + tracker_->stop(onlyEntry().watchId); + runFor(std::chrono::milliseconds{200}); + + EXPECT_TRUE(remote_->cancels.empty()); + EXPECT_FALSE(lastUploadTo("/srv/project/big.bin")->ended); + } + + TEST_F(TrackerTest, AFileRenamedWhileItsFirstUploadRunsContinuesUnderTheNewName) + { + const auto file = root_ / "local" / "project" / "huge.bin"; + const auto renamed = root_ / "local" / "project" / "renamed.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/huge.bin") == 1u; + } + )); + + std::filesystem::rename(file, renamed); + + ASSERT_TRUE(runUntilStopped()); + EXPECT_EQ(remote_->cancels.front().first, "/srv/project/huge.bin"); + EXPECT_EQ(remote_->cancels.front().second, FileTracking::PartialFile::Kept); + endStoppedUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/renamed.bin") == 1u; + } + )); + EXPECT_EQ( + lastUploadTo("/srv/project/renamed.bin")->upload.continuesPartialOf, remote("/srv/project/huge.bin") + ); + completeEverything(); + EXPECT_TRUE(remote_->of("rename").empty()); + EXPECT_TRUE(remote_->of("remove").empty()); + EXPECT_TRUE(remote_->of("discard").empty()); + EXPECT_EQ(uploadsTo("/srv/project/huge.bin"), 1u); + EXPECT_FALSE(remote_->server.contains("/srv/project/huge.bin")); + EXPECT_EQ(remote_->server.at("/srv/project/renamed.bin"), stateOf(renamed)); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, AKnownFileRenamedWhileItsChangeUploadsRenamesTheServerCopyThenContinues) + { + const auto file = root_ / "local" / "project" / "src" / "main.cpp"; + const auto renamed = root_ / "local" / "project" / "src" / "start.cpp"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "int main() { return 3; }"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/src/main.cpp") == 1u; + } + )); + + std::filesystem::rename(file, renamed); + + ASSERT_TRUE(runUntilStopped()); + endStoppedUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("rename").empty(); + } + )); + EXPECT_EQ(remote_->of("rename").front()->source, "/srv/project/src/main.cpp"); + EXPECT_EQ(remote_->of("rename").front()->destination, "/srv/project/src/start.cpp"); + // The upload under the new name waits for the rename. + EXPECT_EQ(uploadsTo("/srv/project/src/start.cpp"), 0u); + completeRemovesAndRenames(); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/src/start.cpp") == 1u; + } + )); + EXPECT_EQ( + lastUploadTo("/srv/project/src/start.cpp")->upload.continuesPartialOf, remote("/srv/project/src/main.cpp") + ); + completeEverything(); + EXPECT_FALSE(remote_->server.contains("/srv/project/src/main.cpp")); + EXPECT_EQ(remote_->server.at("/srv/project/src/start.cpp"), stateOf(renamed)); + EXPECT_TRUE(view_->questions.empty()); + } + + TEST_F(TrackerTest, AFileRenamedTwiceWhileItUploadsContinuesUnderTheLastName) + { + const auto file = root_ / "local" / "project" / "a.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/a.bin") == 1u; + } + )); + + std::filesystem::rename(file, root_ / "local" / "project" / "b.bin"); + ASSERT_TRUE(runUntilStopped()); + std::filesystem::rename(root_ / "local" / "project" / "b.bin", root_ / "local" / "project" / "c.bin"); + ASSERT_TRUE(runUntilStopped(2u)); + endStoppedUploads(); + + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/c.bin") == 1u; + } + )); + EXPECT_EQ(lastUploadTo("/srv/project/c.bin")->upload.continuesPartialOf, remote("/srv/project/a.bin")); + completeEverything(); + EXPECT_EQ(uploadsTo("/srv/project/b.bin"), 0u); + EXPECT_TRUE(remote_->of("rename").empty()); + EXPECT_TRUE(remote_->of("discard").empty()); + } + + TEST_F(TrackerTest, AFileRenamedThenDeletedWhileItUploadsLeavesNoPartialFile) + { + const auto file = root_ / "local" / "project" / "a.bin"; + const auto renamed = root_ / "local" / "project" / "b.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/a.bin") == 1u; + } + )); + + std::filesystem::rename(file, renamed); + ASSERT_TRUE(runUntilStopped()); + std::filesystem::remove(renamed); + runFor(std::chrono::milliseconds{500}); + endStoppedUploads(); + + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("discard").empty(); + } + )); + EXPECT_EQ(remote_->of("discard").front()->source, "/srv/project/a.bin"); + EXPECT_EQ(uploadsTo("/srv/project/b.bin"), 0u); + EXPECT_TRUE(remote_->of("remove").empty()); + } + + TEST_F(TrackerTest, ARenameAfterTheUploadFinishedRenamesTheServerCopy) + { + const auto file = root_ / "local" / "project" / "a.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/a.bin") == 1u; + } + )); + std::filesystem::rename(file, root_ / "local" / "project" / "b.bin"); + ASSERT_TRUE(runUntilStopped()); + + completeWith(*lastUploadTo("/srv/project/a.bin"), FileTracking::ServerState{.mtime = 1u, .size = 13u}); + + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("rename").empty(); + } + )); + EXPECT_EQ(remote_->of("rename").front()->source, "/srv/project/a.bin"); + EXPECT_EQ(remote_->of("rename").front()->destination, "/srv/project/b.bin"); + completeEverything(); + EXPECT_EQ(uploadsTo("/srv/project/b.bin"), 0u); + EXPECT_TRUE(remote_->of("discard").empty()); + } + + TEST_F(TrackerTest, AWatchedFileRenamedWhileItUploadsContinuesUnderTheNewName) + { + const auto notes = root_ / "local" / "notes.txt"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(notes), remote("/srv/notes.txt")).has_value()); + ASSERT_EQ(uploadsTo("/srv/notes.txt"), 1u); + + std::filesystem::rename(notes, root_ / "local" / "memo.txt"); + + ASSERT_TRUE(runUntilStopped()); + EXPECT_EQ(remote_->cancels.front().first, "/srv/notes.txt"); + EXPECT_EQ(remote_->cancels.front().second, FileTracking::PartialFile::Kept); + endStoppedUploads(); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/memo.txt") == 1u; + } + )); + EXPECT_EQ(lastUploadTo("/srv/memo.txt")->upload.continuesPartialOf, remote("/srv/notes.txt")); + completeEverything(); + EXPECT_EQ(onlyEntry().remotePath.value(), "/srv/memo.txt"); + EXPECT_TRUE(remote_->of("rename").empty()); + EXPECT_FALSE(remote_->server.contains("/srv/notes.txt")); + } + + TEST_F(TrackerTest, AFileMovedOverAFileWhoseFirstUploadRunsReplacesIt) + { + const auto first = root_ / "local" / "project" / "a.bin"; + const auto second = root_ / "local" / "project" / "b.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(first, "first content"); + write(second, "second"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/a.bin") == 1u && uploadsTo("/srv/project/b.bin") == 1u; + } + )); + + std::filesystem::rename(first, second); + + ASSERT_TRUE(runUntilStopped()); + EXPECT_EQ(remote_->cancels.front().first, "/srv/project/a.bin"); + EXPECT_EQ(remote_->cancels.front().second, FileTracking::PartialFile::Discarded); + endStoppedUploads(); + completeEverything(); + EXPECT_TRUE(remote_->of("rename").empty()); + EXPECT_FALSE(remote_->server.contains("/srv/project/a.bin")); + EXPECT_EQ(remote_->server.at("/srv/project/b.bin"), stateOf(second)); + } + + TEST_F(TrackerTest, AFileMovedIntoANewDirectoryWhileItUploadsLeavesNoOldServerCopy) + { + const auto known = root_ / "local" / "project" / "src" / "main.cpp"; + const auto directory = root_ / "local" / "project" / "x"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + std::filesystem::create_directory(directory); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/x") == 1u; + } + )); + + std::filesystem::rename(known, directory / "main.cpp"); + runFor(std::chrono::milliseconds{1000}); + completeEverything(); + + EXPECT_TRUE(remote_->of("rename").empty()); + EXPECT_FALSE(remote_->server.contains("/srv/project/src/main.cpp")); + EXPECT_TRUE(remote_->server.contains("/srv/project/x/main.cpp")); + } + + TEST_F(TrackerTest, WithoutMirroredMovesARenamedFirstUploadStillUploadsTheNewName) + { + const auto file = root_ / "local" / "project" / "a.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + tracker_->setOption(onlyEntry().watchId, SharedData::WatchOption::MoveRemoteOnLocalMove, false); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/a.bin") == 1u; + } + )); + + std::filesystem::rename(file, root_ / "local" / "project" / "b.bin"); + + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/b.bin") == 1u; + } + )); + // A watch that reads the rename as a deletion and an addition stops the old upload, but never to go on under + // the new name. + EXPECT_TRUE(std::ranges::none_of( + remote_->cancels | std::views::values, + [](FileTracking::PartialFile partial) + { + return partial == FileTracking::PartialFile::Kept; + } + )); + EXPECT_FALSE(lastUploadTo("/srv/project/b.bin")->upload.continuesPartialOf.has_value()); + } + + TEST_F(TrackerTest, AFailedServerRenameUploadsTheNewName) + { + const auto file = root_ / "local" / "project" / "src" / "main.cpp"; + const auto renamed = root_ / "local" / "project" / "src" / "start.cpp"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + + std::filesystem::rename(file, renamed); + ASSERT_TRUE(runUntil( + [this]() + { + return !remote_->of("rename").empty(); + } + )); + // Something unknown to the tracker is in the way on the server. + end(*remote_->of("rename").front(), SharedData::OperationCompletionReason::Failed); + + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/src/start.cpp") == 1u; + } + )); + auto const* upload = lastUploadTo("/srv/project/src/start.cpp"); + ASSERT_TRUE(upload->upload.expectedDestination.has_value()); + EXPECT_FALSE(upload->upload.expectedDestination->file.has_value()); + } + + TEST_F(TrackerTest, AFileRenamedAndEditedWhileItUploadsGoesAnewUnderTheNewName) + { + const auto file = root_ / "local" / "project" / "a.bin"; + const auto renamed = root_ / "local" / "project" / "b.bin"; + ASSERT_TRUE(tracker_->uploadAndWatch(local(root_ / "local" / "project"), remote("/srv/project")).has_value()); + completeEverything(); + write(file, "large content"); + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/a.bin") == 1u; + } + )); + + std::filesystem::rename(file, renamed); + ASSERT_TRUE(runUntilStopped()); + write(renamed, "edited in the middle"); + runFor(std::chrono::milliseconds{1000}); + endStoppedUploads(); + + ASSERT_TRUE(runUntil( + [this]() + { + return uploadsTo("/srv/project/b.bin") == 1u; + } + )); + // Its old part must not be continued, the resume compares only the end of it. + EXPECT_FALSE(lastUploadTo("/srv/project/b.bin")->upload.continuesPartialOf.has_value()); + ASSERT_EQ(remote_->of("discard").size(), 1u); + EXPECT_EQ(remote_->of("discard").front()->source, "/srv/project/a.bin"); + completeEverything(); + EXPECT_EQ(remote_->server.at("/srv/project/b.bin"), stateOf(renamed)); + } +} diff --git a/backend/test/backend/test_tracking_state.hpp b/backend/test/backend/test_tracking_state.hpp new file mode 100644 index 00000000..1ead9a1d --- /dev/null +++ b/backend/test/backend/test_tracking_state.hpp @@ -0,0 +1,610 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include + +namespace Test::TrackingState +{ + using namespace FileTracking; + + namespace TrackingStateTestDetail + { + inline Paths::LocalPath local(std::string_view path) + { + return Paths::LocalPath::fromUtf8(path, Paths::LocalPlatform::Posix).value(); + } + + inline Paths::RemotePath remote(std::string_view path) + { + return Paths::RemotePath::make(path).value(); + } + } + + using namespace TrackingStateTestDetail; + + TEST(TrackingStateTests, UnchangedServerCopyIsNoChange) + { + EXPECT_FALSE(serverChanged(ServerState{.mtime = 5, .size = 10}, ServerState{.mtime = 5, .size = 10})); + } + + TEST(TrackingStateTests, OtherTimeOrSizeIsAChange) + { + EXPECT_TRUE(serverChanged(ServerState{.mtime = 5, .size = 10}, ServerState{.mtime = 6, .size = 10})); + EXPECT_TRUE(serverChanged(ServerState{.mtime = 5, .size = 10}, ServerState{.mtime = 5, .size = 11})); + } + + TEST(TrackingStateTests, VanishedServerCopyIsAChange) + { + EXPECT_TRUE(serverChanged(ServerState{.mtime = 5, .size = 10}, std::nullopt)); + } + + TEST(TrackingStateTests, UnseenServerFileIsAChangeAndAMissingOneIsNot) + { + EXPECT_TRUE(serverChanged(std::nullopt, ServerState{.mtime = 5, .size = 10})); + EXPECT_FALSE(serverChanged(std::nullopt, std::nullopt)); + } + + TEST(TrackingStateTests, ChangeMapsBelowTheRemoteItem) + { + const auto mapped = remotePathOfChange( + local("/tmp/instance/project/src/main.cpp"), local("/tmp/instance/project"), remote("/home/user/project") + ); + ASSERT_TRUE(mapped.has_value()); + EXPECT_EQ(mapped->value(), "/home/user/project/src/main.cpp"); + } + + TEST(TrackingStateTests, ChangeOfTheItemMapsToItsRemotePath) + { + const auto mapped = + remotePathOfChange(local("/tmp/instance/file.txt"), local("/tmp/instance/file.txt"), remote("/home/user/file.txt")); + ASSERT_TRUE(mapped.has_value()); + EXPECT_EQ(mapped->value(), "/home/user/file.txt"); + } + + TEST(TrackingStateTests, ChangeOutsideTheItemMapsToNothing) + { + const auto mapped = + remotePathOfChange(local("/tmp/instance/other.txt"), local("/tmp/instance/file.txt"), remote("/home/user/file.txt")); + ASSERT_FALSE(mapped.has_value()); + EXPECT_EQ(mapped.error().reason, Paths::PathRejection::EscapesRoot); + } + + TEST(TrackingStateTests, AnItemNamedOtherwiseLocallyStillMapsToItsRemotePath) + { + const auto mapped = + remotePathOfChange(local("/home/me/renamed/a.txt"), local("/home/me/renamed"), remote("/srv/original")); + ASSERT_TRUE(mapped.has_value()); + EXPECT_EQ(mapped->value(), "/srv/original/a.txt"); + } + + TEST(TrackingStateTests, MoveMeanings) + { + EXPECT_EQ(meaningOfMove(false, true), MoveMeaning::Rename); + EXPECT_EQ(meaningOfMove(true, true), MoveMeaning::ReplaceDestination); + EXPECT_EQ(meaningOfMove(true, false), MoveMeaning::ReplaceDestination); + EXPECT_EQ(meaningOfMove(false, false), MoveMeaning::ReplaceDestination); + } + + TEST(TrackingStateTests, KnownContentFindsFilesAndWhatLiesBelowDirectories) + { + KnownServerContent known; + known.rememberFile(local("/i/dir/a.txt"), ServerState{.mtime = 1, .size = 2}); + known.rememberDirectory(local("/i/empty")); + EXPECT_EQ(known.fileState(local("/i/dir/a.txt")), (ServerState{.mtime = 1, .size = 2})); + EXPECT_FALSE(known.fileState(local("/i/dir")).has_value()); + EXPECT_TRUE(known.has(local("/i/dir"))); + EXPECT_TRUE(known.hasBelow(local("/i/dir"))); + EXPECT_FALSE(known.hasBelow(local("/i/dir/a.txt"))); + EXPECT_TRUE(known.has(local("/i/empty"))); + EXPECT_FALSE(known.hasBelow(local("/i/empty"))); + EXPECT_FALSE(known.has(local("/i/di"))); + } + + TEST(TrackingStateTests, KnownContentForgetsATree) + { + KnownServerContent known; + known.rememberFile(local("/i/dir/a.txt"), ServerState{}); + known.rememberDirectory(local("/i/dir/sub")); + known.rememberFile(local("/i/other.txt"), ServerState{}); + known.forget(local("/i/dir")); + EXPECT_FALSE(known.has(local("/i/dir"))); + EXPECT_TRUE(known.has(local("/i/other.txt"))); + } + + TEST(TrackingStateTests, KnownContentMovesATree) + { + KnownServerContent known; + known.rememberFile(local("/i/dir/a.txt"), ServerState{.mtime = 3, .size = 4}); + known.rememberDirectory(local("/i/dir/sub")); + known.rememberFile(local("/i/dirt.txt"), ServerState{}); + known.move(local("/i/dir"), local("/i/renamed")); + EXPECT_FALSE(known.has(local("/i/dir"))); + EXPECT_EQ(known.fileState(local("/i/renamed/a.txt")), (ServerState{.mtime = 3, .size = 4})); + EXPECT_TRUE(known.has(local("/i/renamed/sub"))); + EXPECT_TRUE(known.has(local("/i/dirt.txt"))); + } + + TEST(TrackingStateTests, KnownContentTellsPathsBelowFromSiblingsThatSortBetween) + { + KnownServerContent known; + known.rememberFile(local("/i/dir.txt"), ServerState{}); + known.rememberFile(local("/i/dir-old/a.txt"), ServerState{}); + known.rememberDirectory(local("/i/dir")); + EXPECT_FALSE(known.hasBelow(local("/i/dir"))); + known.rememberFile(local("/i/dir/z.txt"), ServerState{}); + EXPECT_TRUE(known.hasBelow(local("/i/dir"))); + EXPECT_TRUE(known.hasBelow(local("/"))); + EXPECT_FALSE(known.hasBelow(local("/i/dir.txt"))); + + known.forget(local("/i/dir")); + EXPECT_FALSE(known.has(local("/i/dir"))); + EXPECT_TRUE(known.has(local("/i/dir.txt"))); + EXPECT_TRUE(known.has(local("/i/dir-old/a.txt"))); + } + + TEST(TrackingStateTests, KnownContentMovedOntoKnownPathsReplacesThem) + { + KnownServerContent known; + known.rememberFile(local("/i/a.txt"), ServerState{.mtime = 1, .size = 1}); + known.rememberFile(local("/i/b.txt"), ServerState{.mtime = 2, .size = 2}); + known.move(local("/i/a.txt"), local("/i/b.txt")); + EXPECT_FALSE(known.has(local("/i/a.txt"))); + EXPECT_EQ(known.fileState(local("/i/b.txt")), (ServerState{.mtime = 1, .size = 1})); + } + +#ifdef __APPLE__ + TEST(TrackingStateTests, KnownContentFindsTheSpellingItKnowsInAnotherNormalization) + { + KnownServerContent known; + known.rememberFile(local("/w/uni/cafe\xCC\x81.txt"), ServerState{.mtime = 1, .size = 1}); + known.rememberDirectory(local("/w/r\xC3\xA9sum\xC3\xA9s")); + known.rememberFile(local("/w/plain.txt"), ServerState{.mtime = 1, .size = 1}); + + EXPECT_EQ(known.knownSpellingOf(local("/w/uni/caf\xC3\xA9.txt")), local("/w/uni/cafe\xCC\x81.txt")); + EXPECT_EQ(known.knownSpellingOf(local("/w/uni/cafe\xCC\x81.txt")), local("/w/uni/cafe\xCC\x81.txt")); + EXPECT_EQ(known.knownSpellingOf(local("/w/re\xCC\x81sume\xCC\x81s")), local("/w/r\xC3\xA9sum\xC3\xA9s")); + EXPECT_EQ(known.knownSpellingOf(local("/w/plain.txt")), local("/w/plain.txt")); + EXPECT_FALSE(known.knownSpellingOf(local("/w/uni/CAF\xC3\x89.txt")).has_value()) << "case stays apart"; + EXPECT_FALSE(known.knownSpellingOf(local("/w/other.txt")).has_value()); + } + + TEST(TrackingStateTests, KnownContentFindsTheSpellingInEveryScript) + { + KnownServerContent known; + // Decomposed like Finder writes them: a kana with the voiced sound mark, a Hangul syllable as its jamo and a + // Vietnamese letter with two marks. + known.rememberFile(local("/w/\xE3\x81\x8B\xE3\x82\x99.txt"), ServerState{.mtime = 1, .size = 1}); + known.rememberFile(local("/w/\xE1\x84\x92\xE1\x85\xA1\xE1\x86\xAB.txt"), ServerState{.mtime = 1, .size = 1}); + known.rememberDirectory(local("/w/vie\xCC\xA3\xCC\x82t")); + known.rememberFile(local("/w/cafe\xCC\x81/notes.txt"), ServerState{.mtime = 1, .size = 1}); + + EXPECT_EQ(known.knownSpellingOf(local("/w/\xE3\x81\x8C.txt")), local("/w/\xE3\x81\x8B\xE3\x82\x99.txt")); + EXPECT_EQ( + known.knownSpellingOf(local("/w/\xED\x95\x9C.txt")), local("/w/\xE1\x84\x92\xE1\x85\xA1\xE1\x86\xAB.txt") + ); + EXPECT_EQ(known.knownSpellingOf(local("/w/vi\xE1\xBB\x87t")), local("/w/vie\xCC\xA3\xCC\x82t")); + EXPECT_EQ(known.knownSpellingOf(local("/w/vie\xCC\x82\xCC\xA3t")), local("/w/vie\xCC\xA3\xCC\x82t")) + << "the marks in another order"; + EXPECT_EQ(known.knownSpellingOf(local("/w/caf\xC3\xA9/notes.txt")), local("/w/cafe\xCC\x81/notes.txt")); + } + + TEST(TrackingStateTests, KnownContentFindsNoSpellingWhenSeveralMatch) + { + KnownServerContent known; + known.rememberFile(local("/w/r\xC3\xA9sum\xC3\xA9"), ServerState{.mtime = 1, .size = 1}); + known.rememberFile(local("/w/re\xCC\x81sume\xCC\x81"), ServerState{.mtime = 2, .size = 2}); + + EXPECT_FALSE(known.knownSpellingOf(local("/w/r\xC3\xA9sume\xCC\x81")).has_value()); + EXPECT_EQ(known.knownSpellingOf(local("/w/r\xC3\xA9sum\xC3\xA9")), local("/w/r\xC3\xA9sum\xC3\xA9")); + } +#else + TEST(TrackingStateTests, KnownContentKeepsSpellingsInAnotherNormalizationApart) + { + KnownServerContent known; + known.rememberFile(local("/w/uni/cafe\xCC\x81.txt"), ServerState{.mtime = 1, .size = 1}); + + EXPECT_EQ(known.knownSpellingOf(local("/w/uni/cafe\xCC\x81.txt")), local("/w/uni/cafe\xCC\x81.txt")); + EXPECT_FALSE(known.knownSpellingOf(local("/w/uni/caf\xC3\xA9.txt")).has_value()) << "ext4 and NTFS hold both"; + } +#endif + + TEST(TrackingStateTests, UploadsBelowAMovedDirectoryReadTheNewPaths) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/dir/a.txt")})); + ASSERT_TRUE(pending.start({.path = local("/i/other.txt")})); + EXPECT_FALSE(pending.movedAway(local("/i/dir/a.txt"))); + + pending.moveBelow(local("/i/dir"), local("/i/renamed"), /*serverCopiesFollow=*/true); + + EXPECT_TRUE(pending.movedAway(local("/i/dir/a.txt"))); + EXPECT_FALSE(pending.movedAway(local("/i/other.txt"))); + EXPECT_TRUE(pending.isPending(local("/i/renamed/a.txt"))); + const auto finished = pending.finish(local("/i/dir/a.txt")); + EXPECT_EQ(finished.movedTo, local("/i/renamed/a.txt")); + EXPECT_EQ(finished.serverCopyAt, local("/i/renamed/a.txt")); + } + + TEST(TrackingStateTests, ServerCopiesStayWhenTheServerDirectoryWasNotRenamed) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/dir/a.txt")})); + pending.moveBelow(local("/i/dir"), local("/i/renamed"), /*serverCopiesFollow=*/false); + const auto finished = pending.finish(local("/i/dir/a.txt")); + EXPECT_EQ(finished.movedTo, local("/i/renamed/a.txt")); + EXPECT_FALSE(finished.serverCopyAt.has_value()); + } + + TEST(TrackingStateTests, ChangesWaitingBelowADirectoryFollowAMoveBelowIt) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/item"), .isDirectory = true})); + ASSERT_FALSE(pending.start({.path = local("/i/item/sub/a.txt")})); + pending.moveBelow(local("/i/item/sub"), local("/i/item/moved"), /*serverCopiesFollow=*/true); + EXPECT_FALSE(pending.movedAway(local("/i/item"))); + EXPECT_EQ( + pending.finish(local("/i/item")).changedBelow, + (std::vector{{.path = local("/i/item/moved/a.txt")}}) + ); + } + + TEST(TrackingStateTests, SecondChangeOfAPendingUploadGoesAgainAfterIt) + { + PendingUploads pending; + EXPECT_TRUE(pending.start({.path = local("/i/a.txt")})); + EXPECT_FALSE(pending.start({.path = local("/i/a.txt")})); + EXPECT_TRUE(pending.isPending(local("/i/a.txt"))); + const auto finished = pending.finish(local("/i/a.txt")); + EXPECT_TRUE(finished.changedAgain); + EXPECT_FALSE(finished.removeRemote); + EXPECT_TRUE(pending.empty()); + } + + TEST(TrackingStateTests, ChangesBeforeTheUploadReadsTheFileNeedNothingMore) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); + ASSERT_FALSE(pending.start({.path = local("/i/a.txt")})); + pending.changesHandled(local("/i/a.txt")); + EXPECT_FALSE(pending.finish(local("/i/a.txt")).changedAgain); + + ASSERT_TRUE(pending.start({.path = local("/i/dir"), .isDirectory = true})); + ASSERT_FALSE(pending.start({.path = local("/i/dir/b.txt")})); + pending.changesHandled(local("/i/dir")); + ASSERT_FALSE(pending.start({.path = local("/i/dir/c.txt")})); + EXPECT_EQ(pending.finish(local("/i/dir")).changedBelow, (std::vector{{.path = local("/i/dir/c.txt")}})); + } + + TEST(TrackingStateTests, ChangesHandledKeepsAPendingRemoval) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); + ASSERT_TRUE(pending.removeRemoteWhenDone(local("/i/a.txt"))); + pending.changesHandled(local("/i/a.txt")); + EXPECT_TRUE(pending.finish(local("/i/a.txt")).removeRemote); + } + + TEST(TrackingStateTests, UploadWithoutFurtherChangeEndsQuietly) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); + const auto finished = pending.finish(local("/i/a.txt")); + EXPECT_FALSE(finished.changedAgain); + EXPECT_TRUE(finished.changedBelow.empty()); + } + + TEST(TrackingStateTests, ChangesBelowAPendingDirectoryWaitForIt) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/new"), .isDirectory = true})); + EXPECT_FALSE(pending.start({.path = local("/i/new/a.txt")})); + EXPECT_FALSE(pending.start({.path = local("/i/new/sub"), .isDirectory = true})); + EXPECT_FALSE(pending.start({.path = local("/i/new/a.txt")})); + EXPECT_FALSE(pending.isPending(local("/i/new/a.txt"))); + EXPECT_TRUE(pending.start({.path = local("/i/newer.txt")})); + + const auto finished = pending.finish(local("/i/new")); + EXPECT_EQ( + finished.changedBelow, + (std::vector{{.path = local("/i/new/a.txt")}, {.path = local("/i/new/sub"), .isDirectory = true}}) + ); + EXPECT_TRUE(pending.start({.path = local("/i/new/a.txt")})); + } + + TEST(TrackingStateTests, APendingUploadMovedAwayTellsWhereItWentWhenItEnds) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/dir"), .isDirectory = true})); + ASSERT_FALSE(pending.start({.path = local("/i/dir/a.txt")})); + EXPECT_FALSE(pending.moveWhenDone(local("/i/other"), local("/i/elsewhere"))); + ASSERT_TRUE(pending.moveWhenDone(local("/i/dir"), local("/i/renamed"))); + ASSERT_TRUE(pending.moveWhenDone(local("/i/renamed"), local("/i/final"))); + + EXPECT_FALSE(pending.isPending(local("/i/dir"))); + EXPECT_TRUE(pending.isPending(local("/i/final"))); + EXPECT_FALSE(pending.start({.path = local("/i/final/b.txt")})); + EXPECT_TRUE(pending.start({.path = local("/i/dir/c.txt")})); + + const auto finished = pending.finish(local("/i/dir")); + EXPECT_EQ(finished.movedTo, local("/i/final")); + EXPECT_FALSE(finished.changedAgain); + EXPECT_EQ( + finished.changedBelow, + (std::vector{{.path = local("/i/final/a.txt")}, {.path = local("/i/final/b.txt")}}) + ); + } + + TEST(TrackingStateTests, ChangesAtWhereAPendingUploadMovedGoAgainAfterIt) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); + ASSERT_TRUE(pending.moveWhenDone(local("/i/a.txt"), local("/i/b.txt"))); + EXPECT_FALSE(pending.start({.path = local("/i/b.txt")})); + EXPECT_TRUE(pending.finish(local("/i/a.txt")).changedAgain); + + ASSERT_TRUE(pending.start({.path = local("/i/c.txt")})); + ASSERT_TRUE(pending.moveWhenDone(local("/i/c.txt"), local("/i/d.txt"))); + EXPECT_TRUE(pending.removeRemoteWhenDone(local("/i/d.txt"))); + EXPECT_TRUE(pending.finish(local("/i/c.txt")).removeRemote); + } + + TEST(TrackingStateTests, FilesNextToAPendingFileStartTheirOwn) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); + EXPECT_TRUE(pending.start({.path = local("/i/a.txt.bak")})); + } + + TEST(TrackingStateTests, PendingUploadMovedAwayRemovesItsRemoteCopyInsteadOfGoingAgain) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/a.tmp")})); + ASSERT_FALSE(pending.start({.path = local("/i/a.tmp")})); + EXPECT_TRUE(pending.removeRemoteWhenDone(local("/i/a.tmp"))); + EXPECT_FALSE(pending.removeRemoteWhenDone(local("/i/b.tmp"))); + const auto finished = pending.finish(local("/i/a.tmp")); + EXPECT_TRUE(finished.removeRemote); + EXPECT_FALSE(finished.changedAgain); + } + + TEST(TrackingStateTests, ChangeAfterRemovalGoesAgainAndKeepsTheRemoteCopy) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); + ASSERT_TRUE(pending.removeRemoteWhenDone(local("/i/a.txt"))); + ASSERT_FALSE(pending.start({.path = local("/i/a.txt")})); + const auto finished = pending.finish(local("/i/a.txt")); + EXPECT_TRUE(finished.changedAgain); + EXPECT_FALSE(finished.removeRemote); + } + + TEST(TrackingStateTests, KeptDirectoriesAreForgottenAndMovedWithTheirTree) + { + KnownServerContent known; + known.rememberFile(local("/i/dir/sub/a.txt"), ServerState{}); + known.rememberDirectory(local("/i/dir/sub")); + known.keep(local("/i/dir/sub")); + known.keep(local("/i/dir")); + EXPECT_EQ(known.keptDirectories(), (std::vector{local("/i/dir"), local("/i/dir/sub")})); + + known.move(local("/i/dir"), local("/i/moved")); + EXPECT_EQ(known.keptDirectories(), (std::vector{local("/i/moved"), local("/i/moved/sub")})); + known.forget(local("/i/moved/sub")); + EXPECT_EQ(known.keptDirectories(), std::vector{local("/i/moved")}); + } + + TEST(TrackingStateTests, AKeptDirectoryIsEmptiedOnceNothingIsKnownBelowIt) + { + KnownServerContent known; + known.rememberDirectory(local("/i/new")); + known.rememberDirectory(local("/i/new/deep")); + known.rememberFile(local("/i/new/deep/n0.txt"), ServerState{}); + known.rememberFile(local("/i/new/deep/n1.txt"), ServerState{}); + known.keep(local("/i/new")); + known.keep(local("/i/new/deep")); + + known.forget(local("/i/new/deep/n0.txt")); + EXPECT_TRUE(known.emptiedKeptAbove(local("/i/new/deep/n0.txt")).empty()); + known.forget(local("/i/new/deep/n1.txt")); + EXPECT_EQ(known.emptiedKeptAbove(local("/i/new/deep/n1.txt")), std::vector{local("/i/new/deep")}); + known.forget(local("/i/new/deep")); + EXPECT_EQ(known.emptiedKeptAbove(local("/i/new/deep")), std::vector{local("/i/new")}); + } + + TEST(TrackingStateTests, ACopyHoldsAllBelowWhenEveryKnownEntryIsThereAlike) + { + KnownServerContent known; + for (auto const* base : {"/i/src/x", "/i/docs/x"}) + { + known.rememberDirectory(local(std::string{base} + "/y")); + known.rememberFile(local(std::string{base} + "/main.c"), ServerState{.mtime = 5, .size = 6}); + known.rememberFile(local(std::string{base} + "/y/deep.txt"), ServerState{.mtime = 7, .size = 8}); + } + known.rememberFile(local("/i/docs/x/added.txt"), ServerState{}); + known.rememberDirectory(local("/i/empty")); + + EXPECT_TRUE(known.holdsAllBelow(local("/i/docs/x"), local("/i/src/x"))); + EXPECT_FALSE(known.holdsAllBelow(local("/i/src/x"), local("/i/docs/x"))); + EXPECT_FALSE(known.holdsAllBelow(local("/i/docs/x"), local("/i/empty"))); + known.rememberFile(local("/i/docs/x/y/deep.txt"), ServerState{.mtime = 9, .size = 8}); + EXPECT_FALSE(known.holdsAllBelow(local("/i/docs/x"), local("/i/src/x"))); + } + + TEST(TrackingStateTests, ATreeIsRemovedDeepestFirst) + { + KnownServerContent known; + known.rememberDirectory(local("/i/x")); + known.rememberDirectory(local("/i/x/y")); + known.rememberFile(local("/i/x/y/deep.txt"), ServerState{}); + known.rememberFile(local("/i/x/main.c"), ServerState{}); + known.rememberFile(local("/i/x-other.txt"), ServerState{}); + + EXPECT_EQ( + known.deepestFirst(local("/i/x")), + (std::vector{local("/i/x/y/deep.txt"), local("/i/x/y"), local("/i/x/main.c"), local("/i/x")}) + ); + EXPECT_EQ(known.deepestFirst(local("/i/x-other.txt")), std::vector{local("/i/x-other.txt")}); + } + + TEST(TrackingStateTests, APendingUploadTellsTheKindItsPathHasAtTheLatestChange) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/kind"), .isDirectory = false})); + EXPECT_FALSE(pending.start({.path = local("/i/kind"), .isDirectory = true})); + EXPECT_FALSE(pending.start({.path = local("/i/kind/inside.txt")})); + const auto finished = pending.finish(local("/i/kind")); + EXPECT_TRUE(finished.changedAgain); + EXPECT_TRUE(finished.isDirectory); + EXPECT_EQ(finished.changedBelow, std::vector{{.path = local("/i/kind/inside.txt")}}); + } + + TEST(TrackingStateTests, AnOperationWaitsForTheEarlierOnesOnItsPathAboveAndBelow) + { + ServerOperationOrder order; + const auto [deleteFile, deleteFileStarts] = order.enter({remote("/r/dir/a.txt")}); + const auto [deleteSibling, deleteSiblingStarts] = order.enter({remote("/r/dir/b.txt")}); + const auto [deleteDirectory, deleteDirectoryStarts] = order.enter({remote("/r/dir")}); + const auto [upload, uploadStarts] = order.enter({remote("/r/dir/a.txt")}); + const auto [elsewhere, elsewhereStarts] = order.enter({remote("/r/other")}); + + EXPECT_TRUE(deleteFileStarts); + EXPECT_TRUE(deleteSiblingStarts); + EXPECT_FALSE(deleteDirectoryStarts); + EXPECT_FALSE(uploadStarts); + EXPECT_TRUE(elsewhereStarts); + + EXPECT_TRUE(order.leave(deleteFile).empty()); + EXPECT_EQ(order.leave(deleteSibling), std::vector{deleteDirectory}); + EXPECT_EQ(order.leave(deleteDirectory), std::vector{upload}); + EXPECT_TRUE(order.leave(upload).empty()); + EXPECT_TRUE(order.leave(elsewhere).empty()); + EXPECT_TRUE(order.empty()); + } + + TEST(TrackingStateTests, ARenameWaitsOnBothOfItsPaths) + { + ServerOperationOrder order; + const auto [upload, uploadStarts] = order.enter({remote("/r/b.txt")}); + const auto [rename, renameStarts] = order.enter({remote("/r/a.txt"), remote("/r/b.txt")}); + const auto [uploadOfSource, uploadOfSourceStarts] = order.enter({remote("/r/a.txt")}); + + EXPECT_TRUE(uploadStarts); + EXPECT_FALSE(renameStarts); + EXPECT_FALSE(uploadOfSourceStarts); + EXPECT_EQ(order.leave(upload), std::vector{rename}); + EXPECT_EQ(order.leave(rename), std::vector{uploadOfSource}); + } + + TEST(TrackingStateTests, LeavingAnUnknownTicketStartsNothing) + { + ServerOperationOrder order; + const auto [first, firstStarts] = order.enter({remote("/r/a.txt")}); + const auto [second, secondStarts] = order.enter({remote("/r/a.txt")}); + EXPECT_FALSE(secondStarts); + EXPECT_TRUE(order.leave(second + 1u).empty()); + EXPECT_EQ(order.leave(first), std::vector{second}); + } + + TEST(TrackingStateTests, KnownContentRemembersTheStateAKnownFileWasSentIn) + { + KnownServerContent known; + const SourceState sent{.size = 4, .modified = std::filesystem::file_time_type{std::chrono::seconds{7}}}; + known.rememberSent(local("/i/unknown.txt"), sent); + known.rememberFile(local("/i/a.txt"), ServerState{.mtime = 7, .size = 4}); + known.rememberSent(local("/i/a.txt"), sent); + + EXPECT_FALSE(known.sentFrom(local("/i/unknown.txt")).has_value()); + EXPECT_EQ(known.sentFrom(local("/i/a.txt")), sent); + known.rememberFile(local("/i/a.txt"), ServerState{.mtime = 8, .size = 4}); + EXPECT_FALSE(known.sentFrom(local("/i/a.txt")).has_value()); + } + + TEST(TrackingStateTests, KnownContentForgetsAndMovesWhatWasSentWithItsTree) + { + KnownServerContent known; + const SourceState sent{.size = 4, .modified = std::filesystem::file_time_type{std::chrono::seconds{7}}}; + known.rememberFile(local("/i/dir/a.txt"), ServerState{}); + known.rememberSent(local("/i/dir/a.txt"), sent); + known.rememberFile(local("/i/gone/b.txt"), ServerState{}); + known.rememberSent(local("/i/gone/b.txt"), sent); + + known.move(local("/i/dir"), local("/i/renamed")); + known.forget(local("/i/gone")); + + EXPECT_FALSE(known.sentFrom(local("/i/dir/a.txt")).has_value()); + EXPECT_EQ(known.sentFrom(local("/i/renamed/a.txt")), sent); + EXPECT_FALSE(known.sentFrom(local("/i/gone/b.txt")).has_value()); + } + + TEST(TrackingStateTests, AnUploadCoversItsPathAndAPendingDirectoryWhatLiesBelowIt) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/new"), .isDirectory = true})); + ASSERT_TRUE(pending.start({.path = local("/i/file.txt")})); + + EXPECT_TRUE(pending.covers(local("/i/new"))); + EXPECT_TRUE(pending.covers(local("/i/new/sub/a.txt"))); + EXPECT_TRUE(pending.covers(local("/i/file.txt"))); + EXPECT_FALSE(pending.covers(local("/i/file.txt/below"))); + EXPECT_FALSE(pending.covers(local("/i/newer.txt"))); + } + + TEST(TrackingStateTests, AnUploadCountsTheEntriesItCarriesAndLateChangesBelowItOnlyBeyondThem) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/new"), .isDirectory = true})); + EXPECT_FALSE(pending.start({.path = local("/i/new/a.txt")})); + EXPECT_EQ(pending.changeCount(), 2u); + + pending.changesHandled(local("/i/new")); + EXPECT_EQ(pending.changeCount(), 2u); + pending.carries(local("/i/new"), 4u); + EXPECT_EQ(pending.changeCount(), 4u); + EXPECT_FALSE(pending.start({.path = local("/i/new/a.txt")})); + EXPECT_FALSE(pending.start({.path = local("/i/new/b.txt")})); + EXPECT_EQ(pending.changeCount(), 4u); + EXPECT_FALSE(pending.start({.path = local("/i/new"), .isDirectory = true})); + EXPECT_EQ(pending.changeCount(), 4u); + + EXPECT_EQ(pending.finish(local("/i/new")).counted, 4u); + EXPECT_EQ(pending.changeCount(), 0u); + } + + TEST(TrackingStateTests, AFileThatChangedAgainCountsOnceMoreADirectoryDoesNot) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); + ASSERT_TRUE(pending.start({.path = local("/i/new"), .isDirectory = true})); + pending.carries(local("/i/new"), 3u); + ASSERT_EQ(pending.changeCount(), 4u); + + EXPECT_FALSE(pending.start({.path = local("/i/a.txt")})); + EXPECT_FALSE(pending.start({.path = local("/i/new"), .isDirectory = true})); + + EXPECT_EQ(pending.changeCount(), 5u); + } + + TEST(TrackingStateTests, AnUploadIsStoppedAsTheKindItWasSentAs) + { + PendingUploads pending; + ASSERT_TRUE(pending.start({.path = local("/i/dir"), .isDirectory = true})); + EXPECT_FALSE(pending.sentFileTo(local("/i/dir")).has_value()); + pending.sent(local("/i/dir"), remote("/r/dir"), /*isDirectory=*/true); + // A late report of the removed directory can no longer tell it was one. + EXPECT_FALSE(pending.start({.path = local("/i/dir"), .isDirectory = false})); + EXPECT_FALSE(pending.sentFileTo(local("/i/dir")).has_value()); + + ASSERT_TRUE(pending.start({.path = local("/i/file.txt")})); + EXPECT_FALSE(pending.sentFileTo(local("/i/file.txt")).has_value()); + pending.sent(local("/i/file.txt"), remote("/r/file.txt"), /*isDirectory=*/false); + EXPECT_EQ(pending.sentFileTo(local("/i/file.txt")), remote("/r/file.txt")); + EXPECT_EQ(pending.finish(local("/i/file.txt")).sentTo, remote("/r/file.txt")); + } +} diff --git a/backend/test/backend/test_upload_operation.hpp b/backend/test/backend/test_upload_operation.hpp index 25512cbd..5e7539a7 100644 --- a/backend/test/backend/test_upload_operation.hpp +++ b/backend/test/backend/test_upload_operation.hpp @@ -1,9 +1,21 @@ #pragma once #include +#include "error_log_capture.hpp" #include "real_server_tests.hpp" #include +#include + +#include +#include +#include +#include +#include + +#ifndef _WIN32 +# include +#endif extern std::filesystem::path temporaryRoot; @@ -637,6 +649,71 @@ namespace Test EXPECT_EQ(workResult.error().type, UploadOperation::ErrorType::FileNotFound); } + TEST_F(UploadOperationTests, AVanishedSourceThatMayVanishEndsCanceledWithoutATransfer) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/vanished.txt"), + .localPath = toLocal(temporaryRoot / "vanished.txt"), + .sourceMayVanish = true, + } + }; + + const auto workResult = operation.work(); + ASSERT_TRUE(workResult.has_value()) << workResult.error().toString(); + EXPECT_EQ(*workResult, UploadOperation::WorkStatus::Complete); + EXPECT_EQ(operation.state(), SharedData::OperationState::Canceled); + } + + TEST_F(UploadOperationTests, ASourceThatCannotBeCheckedFailsInsteadOfVanishing) + { +#ifdef _WIN32 + // Past MAX_PATH std::filesystem does not find a path that exists. + const std::wstring segment(60, L's'); + const auto directory = temporaryRoot / "unreachable" / segment / segment / segment / segment; + const auto extended = [](std::filesystem::path const& path) + { + return std::filesystem::path{LR"(\\?\)" + path.lexically_normal().make_preferred().native()}; + }; + std::filesystem::create_directories(extended(directory)); + std::ofstream{extended(directory / "source.txt")} << "content"; + auto restore = Nui::ScopeExit{[&]() noexcept { + std::error_code errorCode{}; + std::filesystem::remove_all(extended(temporaryRoot / "unreachable"), errorCode); + }}; +#else + if (::geteuid() == 0) + GTEST_SKIP() << "root can look into any directory"; + const auto directory = temporaryRoot / "unreachable"; + std::filesystem::create_directories(directory); + std::ofstream{directory / "source.txt"} << "content"; + std::filesystem::permissions(directory, std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[&directory]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(directory, std::filesystem::perms::owner_all, errorCode); + std::filesystem::remove_all(directory, errorCode); + }}; +#endif + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/source.txt"), + .localPath = toLocal(directory / "source.txt"), + .sourceMayVanish = true, + } + }; + + const auto workResult = operation.work(); + EXPECT_FALSE(workResult.has_value()) << "a source that may still exist must never complete as gone"; + } + TEST_F(UploadOperationTests, UploadLocalDirectoryFails) { CREATE_SERVER_AND_JOINER(sftpServer); @@ -1142,4 +1219,588 @@ namespace Test ASSERT_TRUE(closeResult.has_value()); } } + + namespace + { + /** + * @brief Records what an upload tells of the files it sent. + */ + struct SentRecorder + { + std::vector>> sent{}; + + UploadOperation::OnSent onSent() + { + return [this](Paths::LocalPath const& source, std::optional const& state) + { + sent.emplace_back(source.toUtf8(), state); + }; + } + }; + + void writeContent(std::filesystem::path const& path, std::string const& content) + { + std::ofstream{path, std::ios_base::binary | std::ios_base::trunc} << content; + } + } + + TEST_F(UploadOperationTests, AnUploadTellsTheStateOfTheFileItSent) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = temporaryRoot / "sent.txt"; + writeContent(source, std::string(30000, 's')); + SentRecorder recorder; + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/sent.txt"), + .localPath = toLocal(source), + .onSent = recorder.onSent(), + } + }; + + ASSERT_EQ(runToCompletion(operation), UploadOperation::WorkStatus::Complete); + + ASSERT_EQ(recorder.sent.size(), 1u); + EXPECT_EQ(recorder.sent.front().first, toLocal(source).toUtf8()); + EXPECT_EQ( + recorder.sent.front().second, + (UploadOperation::SentSource{.size = 30000u, .modified = std::filesystem::last_write_time(source)}) + ); + } + + TEST_F(UploadOperationTests, AFileChangedWhileItUploadsIsToldWithoutAState) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = temporaryRoot / "growing.txt"; + // Larger than the transfer buffer, so the progress of the first part comes before the rest is read. + writeContent(source, std::string(30000, 'g')); + SentRecorder recorder; + bool appended = false; + UploadOperation operation{ + sftp, + { + .progressCallback = + [&](auto, auto, auto current, auto) + { + if (appended || current == 0) + return; + appended = true; + std::ofstream{source, std::ios_base::binary | std::ios_base::app} << "more"; + }, + .remotePath = remote("/home/test/growing.txt"), + .localPath = toLocal(source), + .onSent = recorder.onSent(), + } + }; + + ASSERT_EQ(runToCompletion(operation), UploadOperation::WorkStatus::Complete); + + EXPECT_TRUE(appended); + ASSERT_EQ(recorder.sent.size(), 1u); + EXPECT_EQ(recorder.sent.front().first, toLocal(source).toUtf8()); + EXPECT_FALSE(recorder.sent.front().second.has_value()); + } + + TEST_F(UploadOperationTests, AContinuedUploadIsToldWithoutAState) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + { + auto openFut = sftp->openFile( + remote("/home/test/testfile.txt.filepart"), + SecureShell::SftpSession::OpenType::Write | SecureShell::SftpSession::OpenType::Create, + std::filesystem::perms::owner_read | std::filesystem::perms::owner_write + ); + ASSERT_EQ(openFut.wait_for(5s), std::future_status::ready); + auto opened = openFut.get(); + ASSERT_TRUE(opened.has_value()); + auto file = opened->lock(); + ASSERT_TRUE(file); + const auto partialContent = testFileContent.substr(0, testFileContent.size() / 2); + auto writeFut = file->write(partialContent); + ASSERT_EQ(writeFut.wait_for(5s), std::future_status::ready); + (void)writeFut.get(); + file->close(); + } + SentRecorder recorder; + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + .tryContinue = true, + .onSent = recorder.onSent(), + } + }; + + ASSERT_EQ(runToCompletion(operation), UploadOperation::WorkStatus::Complete); + + ASSERT_EQ(recorder.sent.size(), 1u); + EXPECT_FALSE(recorder.sent.front().second.has_value()); + } + + namespace + { + /** + * @brief Works @p operation until it sent some bytes, at most @p maxIterations steps. + * @return Whether it sent some and still has more to send. + */ + bool workUntilPartlySent(UploadOperation& operation, std::uint64_t const& sent, int maxIterations = 50) + { + for (int iteration = 0; iteration < maxIterations && sent == 0; ++iteration) + { + const auto worked = operation.work(); + if (!worked || *worked == UploadOperation::WorkStatus::Complete) + return false; + } + return sent > 0; + } + + /** + * @brief The size of what the server has at @p path, nothing when it has nothing there. + */ + std::optional remoteSizeOf(SecureShell::SftpSession& sftp, Paths::RemotePath const& path) + { + auto stat = sftp.lstat(path); + if (stat.wait_for(5s) != std::future_status::ready) + return std::nullopt; + const auto information = stat.get(); + if (!information) + return std::nullopt; + return information->size; + } + } + + TEST_F(UploadOperationTests, AStoppedUploadDiscardingItsPartialFileLeavesNone) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = temporaryRoot / "stopped.bin"; + writeContent(source, std::string(1024u * 1024u, 'd')); + std::uint64_t sent = 0; + UploadOperation operation{ + sftp, + { + .progressCallback = + [&sent](auto, auto, auto current, auto) + { + sent = static_cast(current); + }, + .remotePath = remote("/home/test/stopped.bin"), + .localPath = toLocal(source), + } + }; + ASSERT_TRUE(workUntilPartlySent(operation, sent)); + + operation.partialFileOnCancel(UploadOperation::PartialFile::Discarded); + ASSERT_TRUE(operation.cancel(true).has_value()); + + EXPECT_FALSE(remoteSizeOf(*sftp, remote("/home/test/stopped.bin.filepart")).has_value()); + EXPECT_FALSE(remoteSizeOf(*sftp, remote("/home/test/stopped.bin")).has_value()); + } + + TEST_F(UploadOperationTests, AStoppedUploadKeepingItsPartialFileLeavesWhatItSent) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = temporaryRoot / "kept.bin"; + writeContent(source, std::string(1024u * 1024u, 'k')); + std::uint64_t sent = 0; + UploadOperation operation{ + sftp, + { + .progressCallback = + [&sent](auto, auto, auto current, auto) + { + sent = static_cast(current); + }, + .remotePath = remote("/home/test/kept.bin"), + .localPath = toLocal(source), + } + }; + ASSERT_TRUE(workUntilPartlySent(operation, sent)); + + ASSERT_TRUE(operation.cancel(true).has_value()); + + const auto partial = remoteSizeOf(*sftp, remote("/home/test/kept.bin.filepart")); + ASSERT_TRUE(partial.has_value()); + EXPECT_GE(*partial, sent); + EXPECT_LT(*partial, 1024u * 1024u); + } + + TEST_F(UploadOperationTests, AStoppedUploadDiscardsNoPartialFileItDidNotOpen) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/testfile.txt.filepart"}, {"content", "another upload's part"}} + )); + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/testfile.txt"), + .localPath = toLocal(temporaryRoot / "testfile.txt"), + } + }; + + operation.partialFileOnCancel(UploadOperation::PartialFile::Discarded); + ASSERT_TRUE(operation.cancel(true).has_value()); + + EXPECT_TRUE(remoteSizeOf(*sftp, remote("/home/test/testfile.txt.filepart")).has_value()); + } + + namespace + { + /** + * @brief What the server has in the file @p path, nothing when it cannot be read. + */ + std::optional remoteContentOf(SecureShell::SftpSession& sftp, Paths::RemotePath const& path) + { + auto opening = sftp.openFile(path, SecureShell::SftpSession::OpenType::Read, std::filesystem::perms::none); + if (opening.wait_for(5s) != std::future_status::ready) + return std::nullopt; + auto opened = opening.get(); + if (!opened) + return std::nullopt; + const auto file = opened->lock(); + if (!file) + return std::nullopt; + std::string content; + std::string chunk(64u * 1024u, '\0'); + while (true) + { + auto reading = file->readSome(chunk.data(), chunk.size()); + if (reading.wait_for(5s) != std::future_status::ready) + return std::nullopt; + const auto read = reading.get(); + if (!read) + return std::nullopt; + if (*read == 0) + break; + content.append(chunk.data(), static_cast(*read)); + } + file->close(); + return content; + } + + /** + * @brief @p size bytes that differ along the way, so a part continued at the wrong place shows. + */ + std::string patternedContent(std::size_t size) + { + std::string content(size, '\0'); + for (std::size_t index = 0; index < size; ++index) + content[index] = static_cast('a' + index % 23u); + return content; + } + } + + TEST_F(UploadOperationTests, AnUploadContinuesThePartialFileOfItsOldName) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = temporaryRoot / "old.bin"; + const auto content = patternedContent(1024u * 1024u); + writeContent(source, content); + { + std::uint64_t sent = 0; + UploadOperation stopped{ + sftp, + { + .progressCallback = + [&sent](auto, auto, auto current, auto) + { + sent = static_cast(current); + }, + .remotePath = remote("/home/test/old.bin"), + .localPath = toLocal(source), + } + }; + ASSERT_TRUE(workUntilPartlySent(stopped, sent)); + ASSERT_TRUE(stopped.cancel(true).has_value()); + } + const auto kept = remoteSizeOf(*sftp, remote("/home/test/old.bin.filepart")); + ASSERT_TRUE(kept.has_value()); + const auto renamed = temporaryRoot / "new.bin"; + std::filesystem::rename(source, renamed); + + std::optional firstProgress; + UploadOperation continued{ + sftp, + { + .progressCallback = + [&firstProgress](auto, auto, auto current, auto) + { + if (!firstProgress) + firstProgress = static_cast(current); + }, + .remotePath = remote("/home/test/new.bin"), + .localPath = toLocal(renamed), + .adoptPartialOf = remote("/home/test/old.bin"), + } + }; + ASSERT_EQ(runToCompletion(continued), UploadOperation::WorkStatus::Complete); + + ASSERT_TRUE(firstProgress.has_value()); + EXPECT_GE(*firstProgress, *kept); + EXPECT_FALSE(remoteSizeOf(*sftp, remote("/home/test/old.bin.filepart")).has_value()); + EXPECT_FALSE(remoteSizeOf(*sftp, remote("/home/test/old.bin")).has_value()); + EXPECT_EQ(remoteContentOf(*sftp, remote("/home/test/new.bin")), content); + } + + TEST_F(UploadOperationTests, AStopWhileTheTransferReadsDiscardsItsPartialFileWithoutAnError) + { + const ErrorLogCapture errors; + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + // Each stop comes right after a work step, while the next turn of the transfer reads the source mostly. + for (int round = 0; round < 5; ++round) + { + const auto name = fmt::format("running{}.bin", round); + const auto source = temporaryRoot / name; + writeContent(source, std::string(8u * 1024u * 1024u, 'r')); + std::uint64_t sent = 0; + UploadOperation operation{ + sftp, + { + .progressCallback = + [&sent](auto, auto, auto current, auto) + { + sent = static_cast(current); + }, + .remotePath = remote(fmt::format("/home/test/{}", name)), + .localPath = toLocal(source), + .bigFileOptimized = true, + } + }; + ASSERT_TRUE(workUntilPartlySent(operation, sent)) << name; + + operation.partialFileOnCancel(UploadOperation::PartialFile::Discarded); + ASSERT_TRUE(operation.cancel(true).has_value()); + + EXPECT_EQ(operation.state(), SharedData::OperationState::Canceled) << name; + EXPECT_FALSE(remoteSizeOf(*sftp, remote(fmt::format("/home/test/{}.filepart", name))).has_value()) << name; + } + EXPECT_EQ(errors.mentioning("UploadOperation"), 0u); + } + + TEST_F(UploadOperationTests, AStopWhileTheTransferReadsKeepsItsPartialFileForTheUploadThatContinuesIt) + { + const ErrorLogCapture errors; + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = temporaryRoot / "old.bin"; + const auto content = patternedContent(1024u * 1024u); + writeContent(source, content); + { + std::uint64_t sent = 0; + UploadOperation stopped{ + sftp, + { + .progressCallback = + [&sent](auto, auto, auto current, auto) + { + sent = static_cast(current); + }, + .remotePath = remote("/home/test/old.bin"), + .localPath = toLocal(source), + .bigFileOptimized = true, + } + }; + ASSERT_TRUE(workUntilPartlySent(stopped, sent)); + ASSERT_TRUE(stopped.cancel(true).has_value()); + EXPECT_EQ(stopped.state(), SharedData::OperationState::Canceled); + } + const auto kept = remoteSizeOf(*sftp, remote("/home/test/old.bin.filepart")); + ASSERT_TRUE(kept.has_value()); + const auto renamed = temporaryRoot / "new.bin"; + std::filesystem::rename(source, renamed); + + UploadOperation continued{ + sftp, + { + .remotePath = remote("/home/test/new.bin"), + .localPath = toLocal(renamed), + .bigFileOptimized = true, + .adoptPartialOf = remote("/home/test/old.bin"), + } + }; + ASSERT_EQ(runToCompletion(continued, 1000), UploadOperation::WorkStatus::Complete); + + EXPECT_FALSE(remoteSizeOf(*sftp, remote("/home/test/old.bin.filepart")).has_value()); + EXPECT_EQ(remoteContentOf(*sftp, remote("/home/test/new.bin")), content); + EXPECT_EQ(errors.mentioning("UploadOperation"), 0u); + } + + TEST_F(UploadOperationTests, AStoppedUploadLogsItsSourceBeforeItsDestination) + { + const LogCapture infos{spdlog::level::info}; + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = toLocal(temporaryRoot / "testfile.txt"); + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/testfile.txt"), + .localPath = source, + } + }; + + ASSERT_TRUE(operation.cancel(true).has_value()); + + EXPECT_EQ( + infos.mentioning(fmt::format("Upload of '{}' to '/home/test/testfile.txt' canceled.", source.toUtf8())), 1u + ); + } + + TEST_F(UploadOperationTests, AnAdoptedPartialFileOfAnotherSourceStartsOver) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + ASSERT_TRUE(serverStartResult->command( + "createFile", {{"path", "/home/test/old.bin.filepart"}, {"content", "something else entirely"}} + )); + const auto source = temporaryRoot / "new.bin"; + const auto content = patternedContent(100000u); + writeContent(source, content); + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/new.bin"), + .localPath = toLocal(source), + .adoptPartialOf = remote("/home/test/old.bin"), + } + }; + + ASSERT_EQ(runToCompletion(operation), UploadOperation::WorkStatus::Complete); + + EXPECT_EQ(remoteContentOf(*sftp, remote("/home/test/new.bin")), content); + } + + TEST_F(UploadOperationTests, AnUploadWithoutTheOldPartialFileStartsFresh) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = temporaryRoot / "new.bin"; + const auto content = patternedContent(100000u); + writeContent(source, content); + UploadOperation operation{ + sftp, + { + .remotePath = remote("/home/test/new.bin"), + .localPath = toLocal(source), + .adoptPartialOf = remote("/home/test/old.bin"), + } + }; + + ASSERT_EQ(runToCompletion(operation), UploadOperation::WorkStatus::Complete); + + EXPECT_EQ(remoteContentOf(*sftp, remote("/home/test/new.bin")), content); + } + + TEST_F(UploadOperationTests, AnUploadKeepsTheModificationTimeOfASourceRenamedMeanwhile) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = temporaryRoot / "dated.bin"; + writeContent(source, patternedContent(1024u * 1024u)); + const auto modified = + std::chrono::file_clock::from_sys(std::chrono::sys_seconds{std::chrono::seconds{1600000000}}); + std::filesystem::last_write_time(source, modified); + std::uint64_t sent = 0; + UploadOperation operation{ + sftp, + { + .progressCallback = + [&sent](auto, auto, auto current, auto) + { + sent = static_cast(current); + }, + .remotePath = remote("/home/test/dated.bin"), + .localPath = toLocal(source), + } + }; + ASSERT_TRUE(workUntilPartlySent(operation, sent)); + + std::filesystem::rename(source, temporaryRoot / "dated-renamed.bin"); + ASSERT_EQ(runToCompletion(operation), UploadOperation::WorkStatus::Complete); + + auto stat = sftp->stat(remote("/home/test/dated.bin")); + ASSERT_EQ(stat.wait_for(5s), std::future_status::ready); + const auto information = stat.get(); + ASSERT_TRUE(information.has_value()); + EXPECT_EQ(information->mtime, 1600000000u); + } + + TEST_F(UploadOperationTests, AnUploadWhoseDestinationChangedSinceTheCompareFailsBeforeItWritesAnything) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + ASSERT_TRUE( + serverStartResult->command("createFile", {{"path", "/home/test/target.bin"}, {"content", "theirs"}}) + ); + const auto source = temporaryRoot / "target.bin"; + writeContent(source, patternedContent(1024u * 1024u)); + std::uint64_t sent = 0; + UploadOperation operation{ + sftp, + { + .progressCallback = + [&sent](auto, auto, auto current, auto) + { + sent = static_cast(current); + }, + .remotePath = remote("/home/test/target.bin"), + .localPath = toLocal(source), + .mayOverwrite = true, + // The compare saw nothing there. + .comparedDestination = SharedData::ComparedState{}, + } + }; + + const auto status = runToCompletion(operation); + + ASSERT_FALSE(status.has_value()); + EXPECT_EQ(status.error().type, SharedData::OperationErrorType::ChangedSinceCompare); + EXPECT_EQ(sent, 0u); + EXPECT_FALSE(remoteSizeOf(*sftp, remote("/home/test/target.bin.filepart")).has_value()); + EXPECT_EQ(remoteContentOf(*sftp, remote("/home/test/target.bin")), std::string{"theirs"}); + } + + TEST_F(UploadOperationTests, AnUploadWhoseDestinationChangesDuringTheTransferStillFailsAtTheFinalize) + { + CREATE_SERVER_AND_JOINER(sftpServer); + auto [_, sftp] = createSftpSession(serverStartResult->port); + const auto source = temporaryRoot / "target.bin"; + writeContent(source, patternedContent(1024u * 1024u)); + std::uint64_t sent = 0; + UploadOperation operation{ + sftp, + { + .progressCallback = + [&sent](auto, auto, auto current, auto) + { + sent = static_cast(current); + }, + .remotePath = remote("/home/test/target.bin"), + .localPath = toLocal(source), + .mayOverwrite = true, + .comparedDestination = SharedData::ComparedState{}, + } + }; + ASSERT_TRUE(workUntilPartlySent(operation, sent)); + + ASSERT_TRUE( + serverStartResult->command("createFile", {{"path", "/home/test/target.bin"}, {"content", "theirs"}}) + ); + const auto status = runToCompletion(operation); + + ASSERT_FALSE(status.has_value()); + EXPECT_EQ(status.error().type, SharedData::OperationErrorType::ChangedSinceCompare); + EXPECT_FALSE(remoteSizeOf(*sftp, remote("/home/test/target.bin.filepart")).has_value()); + EXPECT_EQ(remoteContentOf(*sftp, remote("/home/test/target.bin")), std::string{"theirs"}); + } } \ No newline at end of file diff --git a/dependencies/efsw b/dependencies/efsw index 87abe599..d71ed846 160000 --- a/dependencies/efsw +++ b/dependencies/efsw @@ -1 +1 @@ -Subproject commit 87abe599995d5646f5d83cf2e3a225bd73148b3a +Subproject commit d71ed846811cc67ee8827614341e2777d200231a diff --git a/frontend/include/frontend/file_explorer/local_side_model.hpp b/frontend/include/frontend/file_explorer/local_side_model.hpp index ca034259..31a2c2a4 100644 --- a/frontend/include/frontend/file_explorer/local_side_model.hpp +++ b/frontend/include/frontend/file_explorer/local_side_model.hpp @@ -24,6 +24,7 @@ #include class RemoteSideModel; +class FileTrackingPanel; class LocalSideModel : public SideModel @@ -62,8 +63,18 @@ class LocalSideModel * tar archive in the listed remote directory. */ void onTransferAsArchive(std::vector const& items); + + /** + * @brief Uploads @p item into the listed remote directory, or its subdirectory @p subDir, and watches it, so its + * changes go to the server too. A watch never overlaps another one, locally or on the server, such an item + * is refused before anything is uploaded. + */ + void onUploadAndWatch(NuiFileExplorer::Item const& item, std::optional const& subDir); std::vector - contextMenuItems(std::vector const& selectedItems) override; + contextMenuItems( + std::vector const& selectedItems, + std::vector const& otherSideSelectedItems + ) override; void onDropExternal( std::vector const& items, std::optional const& subDir, @@ -89,6 +100,7 @@ class LocalSideModel return openerCaps_; } void setRemoteModel(RemoteSideModel* model); + void setFileTracking(FileTrackingPanel* fileTracking); /** * @brief The directory the shown items were listed from, an error before the first listing. @@ -113,6 +125,18 @@ class LocalSideModel std::expected itemPathText(NuiFileExplorer::Item const& item) const override; + /** + * @brief Asks the user to upload @p items into the listed remote directory, or its subdirectory @p subDir, and to + * replace what exists there. @p onAccepted gets the uploads the user agreed to. + * @param sourceOf The local path that is uploaded for an item, the upload carries its name. + */ + void confirmUploads( + std::vector const& items, + std::optional const& subDir, + std::function(NuiFileExplorer::Item const&)> const& sourceOf, + std::function)> onAccepted + ); + /** * @brief Queues the uploads the user accepted, a single file as a transfer of its own. */ @@ -146,6 +170,7 @@ class LocalSideModel void probeOpenerCapabilities(); RemoteSideModel* remoteModel_{nullptr}; + FileTrackingPanel* fileTracking_{nullptr}; NuiFileExplorer::PathSuggestionCache pathSuggestionCache_; SharedData::OpenerCapabilities openerCaps_{}; }; \ No newline at end of file diff --git a/frontend/include/frontend/file_explorer/remote_side_model.hpp b/frontend/include/frontend/file_explorer/remote_side_model.hpp index 0bff1e52..83769b0f 100644 --- a/frontend/include/frontend/file_explorer/remote_side_model.hpp +++ b/frontend/include/frontend/file_explorer/remote_side_model.hpp @@ -66,7 +66,10 @@ class RemoteSideModel */ void onTransferAsArchive(std::vector const& items); std::vector - contextMenuItems(std::vector const& selectedItems) override; + contextMenuItems( + std::vector const& selectedItems, + std::vector const& otherSideSelectedItems + ) override; void onDropExternal( std::vector const& items, std::optional const& subDir, @@ -124,33 +127,10 @@ class RemoteSideModel enqueueDeletes(std::vector nonEmpties, std::vector filesAndEmptyDirs); /** - * @brief Downloads @p item into a file tracking instance and opens it, with @p watch also uploads it again when - * it changes. + * @brief Has the file tracker download @p item into a directory of its own and opens it, with @p watch the + * tracker also uploads it again when it changes. */ void onDownloadAndOpen(NuiFileExplorer::Item const& item, bool openWith, bool watch); - void onFileTrackingInstanceCreated( - NuiFileExplorer::Item const& item, - bool openWith, - bool synchronize, - Ids::InstanceId const& instanceId, - std::string const& instanceDirStr - ); - void onFileDownloadComplete( - Ids::InstanceId const& instanceId, - Paths::LocalPath const& instanceDirectory, - Paths::RemotePath const& remotePath, - Paths::LocalPath const& localPath, - bool openWith, - bool synchronize, - bool success - ); - void onFileWatchAdded( - Ids::InstanceId const& instanceId, - Paths::LocalPath const& instanceDirectory, - Paths::RemotePath const& remotePath, - Paths::LocalPath const& localPath, - bool openWith - ); /** * @brief What happens when the user decides not to open the download @p localPath of the instance * @p instanceId: the download is removed and its watch ends, see FileTrackingPanel::discard. diff --git a/frontend/include/frontend/session_components/file_tracking.hpp b/frontend/include/frontend/session_components/file_tracking.hpp index 97799274..d448e0fd 100644 --- a/frontend/include/frontend/session_components/file_tracking.hpp +++ b/frontend/include/frontend/session_components/file_tracking.hpp @@ -1,57 +1,75 @@ #pragma once -#include -#include #include -#include #include #include #include #include -#include - -class OperationQueue; +#include +#include +#include +#include +/** + * @brief Shows the file tracker of a session, which lives in the backend as "FileTracker::": its watches, + * its questions and the orphaned download directories. Forwards what the user asks for and decides nothing. + */ class FileTrackingPanel { public: - FileTrackingPanel(Persistence::StateHolder* stateHolder, FrontendEvents* events, ConfirmDialog* confirmDialog); + explicit FileTrackingPanel(ConfirmDialog* confirmDialog); ROAR_PIMPL_SPECIAL_FUNCTIONS(FileTrackingPanel); /** - * @brief Connect to a live session's operation queue. - * @param downloadTempSuffix The suffix the session's downloads write under until complete, see - * Persistence::effectiveTempFileSuffix. The watches ignore those files. + * @brief Shows the tracker of the session @p sessionId from now on. */ - void activate(OperationQueue* operationQueue, Ids::SessionId sessionId, std::string downloadTempSuffix); - - /** @brief The suffix of the session's downloads in progress, given to activate(). */ - std::string const& downloadTempSuffix() const; + void activate(Ids::SessionId sessionId); - /** @brief Disconnect from the session; clears all watches and entries. */ + /** + * @brief Shows nothing any more, the session has ended. + */ void deactivate(); /** - * @brief Register a new watched entry after a priority download has been initiated. - * - * @param instanceId The FileTracking instance ID returned by createInstance. - * @param instanceDirectory The local temporary directory for this instance. - * @param remotePath Absolute remote path that was downloaded. - * @param localPath Local path inside instanceDirectory where the file/dir lives. + * @brief Asks the tracker to download @p remotePath into a directory of its own, and to watch it there when + * @p watch is set. @p onOpened gets the copy, @p onRefused the translated reason when there is none. + * Neither is called when the download did not end well, its card in the queue tells why. + */ + void open( + Paths::RemotePath const& remotePath, + std::uint64_t sizeBytes, + bool watch, + std::function onOpened, + std::function onRefused + ); + + /** + * @brief Asks the tracker whether @p item could be uploaded into @p directory and watched now, before the user is + * asked anything about the upload. @p onWatchable gets the item canonical, as the watch will report it; + * @p onRefused gets the translated reason. + */ + void checkUploadAndWatch( + Paths::LocalPath const& item, + Paths::RemotePath const& directory, + std::function onWatchable, + std::function onRefused + ); + + /** + * @brief Asks the tracker to watch @p item, upload it to @p remotePath and mirror its changes from then on. + * @p onRefused gets the translated reason when it does not. */ - void startWatching( - Ids::InstanceId const& instanceId, - Paths::LocalPath const& instanceDirectory, + void uploadAndWatch( + Paths::LocalPath const& item, Paths::RemotePath const& remotePath, - Paths::LocalPath const& localPath + std::function onRefused ); /** - * @brief Removes @p localPath, a download of the instance @p instanceId the user decided not to open, and ends - * its watch like Stop does when it has one. Once nothing else is left in the instance, it ends and its - * directory is removed, so it leaves no orphan. + * @brief Asks the tracker to remove @p localPath, a download of the instance @p instanceId the user decided not to + * open, with its watch. */ void discard(Ids::InstanceId const& instanceId, Paths::LocalPath const& localPath); diff --git a/frontend/include/frontend/session_components/file_tracking_state.hpp b/frontend/include/frontend/session_components/file_tracking_state.hpp deleted file mode 100644 index 8bb7e00f..00000000 --- a/frontend/include/frontend/session_components/file_tracking_state.hpp +++ /dev/null @@ -1,382 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace FileTracking -{ - /** - * @brief Size and modification time (seconds) of a file on the server. - */ - struct ServerState - { - std::uint64_t mtime{0}; - std::uint64_t size{0}; - - friend bool operator==(ServerState const&, ServerState const&) = default; - }; - - /** - * @brief Whether the server's copy of a file changed since @p known was taken. A file the server has but nobody - * saw here, a vanished one and one with another size or time all changed. - * @param known What the server had when it was last seen, nothing when it had no file there. - * @param current What the server has now, nothing when it has no file there. - */ - inline bool serverChanged(std::optional const& known, std::optional const& current) - { - if (!known) - return current.has_value(); - return !current || *current != *known; - } - - /** - * @brief The remote path a change at @p changed maps to. The instance directory stands for the directory of - * @p remotePath, a change outside it or of the instance directory itself maps to nothing. - */ - inline std::expected remotePathOfChange( - Paths::LocalPath const& changed, - Paths::LocalPath const& instanceDirectory, - Paths::RemotePath const& remotePath - ) - { - const auto remoteDirectory = remotePath.parent(); - if (!remoteDirectory) - return std::unexpected( - Paths::PathError{.reason = Paths::PathRejection::EscapesRoot, .input = remotePath.value()} - ); - return changed.relativeTo(instanceDirectory) - .and_then( - [&changed, &remoteDirectory]( - Paths::RelativePath const& relative - ) -> std::expected - { - if (relative.empty()) - return std::unexpected( - Paths::PathError{.reason = Paths::PathRejection::Empty, .input = changed.toUtf8()} - ); - return *remoteDirectory / relative; - } - ); - } - - /** - * @brief What a move inside a tracked item does on the server. - */ - enum class MoveMeaning - { - /** @brief The server's copy of the source is renamed. */ - Rename, - /** - * @brief The destination's content changed and the source is gone: an editor saving through a temporary - * file, or a move over an existing file. A rename on the server would fail on the existing - * destination or find no source. - */ - ReplaceDestination, - }; - - /** - * @brief What a move means when the server has the destination (@p destinationOnServer) or the source - * (@p sourceOnServer), as far as file tracking knows. - */ - inline MoveMeaning meaningOfMove(bool destinationOnServer, bool sourceOnServer) - { - return destinationOnServer || !sourceOnServer ? MoveMeaning::ReplaceDestination : MoveMeaning::Rename; - } - - /** - * @brief What the server is known to have of one tracked item, by local path: what the download brought and - * what was uploaded since. - */ - class KnownServerContent - { - public: - void rememberFile(Paths::LocalPath const& path, ServerState state) - { - files_.insert_or_assign(path, state); - } - - void rememberDirectory(Paths::LocalPath const& path) - { - directories_.insert(path); - } - - /** - * @brief The known state of the file at @p path, nothing when no file is known there. - */ - std::optional fileState(Paths::LocalPath const& path) const - { - const auto found = files_.find(path); - if (found == files_.end()) - return std::nullopt; - return found->second; - } - - /** - * @brief Whether the server is known to have @p path or anything below it. - */ - bool has(Paths::LocalPath const& path) const - { - return files_.contains(path) || directories_.contains(path) || hasBelow(path); - } - - /** - * @brief Whether the server is known to have anything below @p path. - */ - bool hasBelow(Paths::LocalPath const& path) const - { - const auto isBelow = [&path](Paths::LocalPath const& candidate) - { - return candidate != path && candidate.isWithin(path); - }; - return std::ranges::any_of(std::views::keys(files_), isBelow) || std::ranges::any_of(directories_, isBelow); - } - - /** - * @brief Forgets @p path and everything below it. - */ - void forget(Paths::LocalPath const& path) - { - std::erase_if( - files_, - [&path](auto const& file) - { - return file.first.isWithin(path); - } - ); - std::erase_if( - directories_, - [&path](Paths::LocalPath const& directory) - { - return directory.isWithin(path); - } - ); - } - - /** - * @brief Moves what is known at @p from and below it to @p to, after the server renamed it. - */ - void move(Paths::LocalPath const& from, Paths::LocalPath const& to) - { - const auto moved = [&from, &to](Paths::LocalPath const& path) -> std::optional - { - return path.relativeTo(from) - .and_then( - [&to](Paths::RelativePath const& relative) - { - return to.join(relative); - } - ) - .transform( - [](Paths::LocalPath moved) - { - return std::optional{std::move(moved)}; - } - ) - .value_or(std::nullopt); - }; - std::map files; - for (auto const& [path, state] : files_) - files.insert_or_assign(moved(path).value_or(path), state); - std::set directories; - std::ranges::transform( - directories_, - std::inserter(directories, directories.end()), - [&moved](Paths::LocalPath const& path) - { - return moved(path).value_or(path); - } - ); - files_ = std::move(files); - directories_ = std::move(directories); - } - - private: - std::map files_; - std::set directories_; - }; - - /** - * @brief A changed path of a tracked item. - */ - struct LocalChange - { - Paths::LocalPath path; - bool isDirectory{false}; - - friend bool operator==(LocalChange const&, LocalChange const&) = default; - }; - - /** - * @brief The uploads of one tracked item that are on their way, from the change until the server's state after - * the upload is known. One path has at most one, and nothing below a directory being uploaded starts its - * own, so two uploads never write the same remote file at once. - */ - class PendingUploads - { - public: - /** - * @brief What became of the uploads of a path when its upload ended. - */ - struct Finished - { - /** @brief The path changed again meanwhile and has to go once more. */ - bool changedAgain{false}; - /** @brief The path was deleted or moved away meanwhile, its server copy has to go. */ - bool removeRemote{false}; - /** @brief Changes below the uploaded directory that arrived meanwhile, in arrival order. */ - std::vector changedBelow{}; - }; - - /** - * @brief Starts the upload of @p change unless one of its path or of a directory above it is on its way, the - * change then goes again once that one finished. - * @return Whether the upload starts. - */ - bool start(LocalChange const& change) - { - if (const auto found = pending_.find(change.path); found != pending_.end()) - { - found->second.changedAgain = true; - found->second.removeRemote = false; - return false; - } - const auto directoryAbove = std::ranges::find_if( - pending_, - [&change](auto const& pending) - { - return pending.second.isDirectory && change.path.isWithin(pending.first); - } - ); - if (directoryAbove != pending_.end()) - { - if (!std::ranges::contains(directoryAbove->second.changedBelow, change)) - directoryAbove->second.changedBelow.push_back(change); - return false; - } - pending_.emplace(change.path, Pending{.isDirectory = change.isDirectory}); - return true; - } - - /** - * @brief The content of @p path as it is now is being handled, by the upload reading it or by the answer to - * a question about it: the changes that arrived so far need nothing more. - */ - void changesHandled(Paths::LocalPath const& path) - { - if (const auto found = pending_.find(path); found != pending_.end()) - { - found->second.changedAgain = false; - found->second.changedBelow.clear(); - } - } - - /** - * @brief Whether an upload of @p path is on its way. - */ - bool isPending(Paths::LocalPath const& path) const - { - return pending_.contains(path); - } - - /** - * @brief The path of a pending upload was deleted or moved away: its server copy goes once the upload - * ended, and it does not go again. - * @return Whether an upload of @p path is on its way. - */ - bool removeRemoteWhenDone(Paths::LocalPath const& path) - { - const auto found = pending_.find(path); - if (found == pending_.end()) - return false; - found->second.removeRemote = true; - found->second.changedAgain = false; - return true; - } - - /** - * @brief Ends the upload of @p path. - */ - Finished finish(Paths::LocalPath const& path) - { - const auto found = pending_.find(path); - if (found == pending_.end()) - return {}; - auto finished = Finished{ - .changedAgain = found->second.changedAgain, - .removeRemote = found->second.removeRemote, - .changedBelow = std::move(found->second.changedBelow), - }; - pending_.erase(found); - return finished; - } - - bool empty() const - { - return pending_.empty(); - } - - private: - struct Pending - { - bool isDirectory{false}; - bool changedAgain{false}; - bool removeRemote{false}; - std::vector changedBelow{}; - }; - std::map pending_; - }; - - /** - * @brief The keys of a table's rows in the order they were added, so the row of a key can be found. - */ - template - class RowOrder - { - public: - /** - * @brief Appends the row of @p key. - */ - void add(KeyT key) - { - keys_.push_back(std::move(key)); - } - - /** - * @brief Forgets the row of @p key. - * @return The index the row had, nothing when @p key has no row. - */ - std::optional remove(KeyT const& key) - { - const auto found = std::ranges::find(keys_, key); - if (found == keys_.end()) - return std::nullopt; - const auto index = static_cast(std::distance(keys_.begin(), found)); - keys_.erase(found); - return index; - } - - void clear() - { - keys_.clear(); - } - - std::size_t size() const - { - return keys_.size(); - } - - private: - std::vector keys_{}; - }; -} diff --git a/frontend/include/frontend/session_components/row_order.hpp b/frontend/include/frontend/session_components/row_order.hpp new file mode 100644 index 00000000..e4bd7384 --- /dev/null +++ b/frontend/include/frontend/session_components/row_order.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace FileTracking +{ + /** + * @brief The keys of a table's rows in the order they were added, so the row of a key can be found. + */ + template + class RowOrder + { + public: + /** + * @brief Appends the row of @p key. + */ + void add(KeyT key) + { + keys_.push_back(std::move(key)); + } + + /** + * @brief Forgets the row of @p key. + * @return The index the row had, nothing when @p key has no row. + */ + std::optional remove(KeyT const& key) + { + const auto found = std::ranges::find(keys_, key); + if (found == keys_.end()) + return std::nullopt; + const auto index = static_cast(std::distance(keys_.begin(), found)); + keys_.erase(found); + return index; + } + + void clear() + { + keys_.clear(); + } + + std::size_t size() const + { + return keys_.size(); + } + + private: + std::vector keys_{}; + }; +} diff --git a/frontend/source/frontend/file_explorer/local_side_model.cpp b/frontend/source/frontend/file_explorer/local_side_model.cpp index a347f11f..ad5d0ce5 100644 --- a/frontend/source/frontend/file_explorer/local_side_model.cpp +++ b/frontend/source/frontend/file_explorer/local_side_model.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -486,7 +487,95 @@ void LocalSideModel::onTransfer( if (items.empty()) return; + Log::info("Upload items requested: {}", items.size()); + confirmUploads( + items, + subDir, + [this](NuiFileExplorer::Item const& item) + { + return localPathOf(item); + }, + lifetime_.guarded( + [this](std::vector accepted) + { + enqueueUploads(std::move(accepted)); + } + ) + ); +} + +void LocalSideModel::onUploadAndWatch(NuiFileExplorer::Item const& item, std::optional const& subDir) +{ + CHECK_COMPLETE(); + + if (fileTracking_ == nullptr || !remoteModel_) + { + Log::error("File tracking or the remote side is not available, cannot upload and watch."); + return; + } + const auto refusedTitle = language->get("localSideModel", "uploadAndWatchFailedTitle"); + const auto path = localPathOf(item); + if (!path) + return showPathRefused(refusedTitle, path.error()); + const auto targetDirectory = remoteModel_->currentRemotePath().and_then( + [&subDir](Paths::RemotePath const& directory) + { + return FileExplorer::targetDirectoryOf(directory, subDir); + } + ); + if (!targetDirectory) + return showPathRefused(refusedTitle, targetDirectory.error()); + Log::info("Upload and watch requested: {} into {}", path->toUtf8(), targetDirectory->value()); + + auto showRefused = lifetime_.guarded( + [this, refusedTitle](std::string const& reason) + { + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = refusedTitle, + .text = reason, + .buttons = ConfirmDialog::Button::Ok, + }); + } + ); + // The watch reports the item canonical, so it uploads under that name: a symbolic link under its target's. + fileTracking_->checkUploadAndWatch( + *path, + *targetDirectory, + lifetime_.guarded( + [this, item, subDir, showRefused](Paths::LocalPath const& canonicalItem) + { + confirmUploads( + {item}, + subDir, + [canonicalItem](NuiFileExplorer::Item const&) -> std::expected + { + return canonicalItem; + }, + lifetime_.guarded( + [this, showRefused](std::vector accepted) + { + if (accepted.empty() || fileTracking_ == nullptr) + return; + fileTracking_->uploadAndWatch( + accepted.front().source, accepted.front().destination, showRefused + ); + } + ) + ); + } + ), + showRefused + ); +} +void LocalSideModel::confirmUploads( + std::vector const& items, + std::optional const& subDir, + std::function(NuiFileExplorer::Item const&)> const& sourceOf, + std::function)> onAccepted +) +{ if (!remoteModel_) { Log::error("Cannot transfer items: remote model is not set"); @@ -499,8 +588,6 @@ void LocalSideModel::onTransfer( return; } - Log::info("Upload items requested: {}", items.size()); - const auto refuse = [this](Paths::PathError const& error) { showPathRefused(language->get("localSideModel", "transferFailedTitle"), error); @@ -513,14 +600,7 @@ void LocalSideModel::onTransfer( ); if (!destinationDirectory) return refuse(destinationDirectory.error()); - auto candidates = transferCandidates( - items, - [this](NuiFileExplorer::Item const& item) - { - return localPathOf(item); - }, - *destinationDirectory - ); + auto candidates = transferCandidates(items, sourceOf, *destinationDirectory); if (!candidates) return refuse(candidates.error()); @@ -534,12 +614,7 @@ void LocalSideModel::onTransfer( .overwriteHeaderText = language->get("localSideModel", "fileAlreadyExistsOverwriteHeader"), .overwriteText = language->get("localSideModel", "fileAlreadyExistsOverwrite"), }, - lifetime_.guarded( - [this](std::vector accepted) - { - enqueueUploads(std::move(accepted)); - } - ) + std::move(onAccepted) ); } @@ -802,11 +877,19 @@ void LocalSideModel::enqueueSingleUpload( } std::vector -LocalSideModel::contextMenuItems(std::vector const& selectedItems) +LocalSideModel::contextMenuItems( + std::vector const& selectedItems, + std::vector const& otherSideSelectedItems +) { namespace Snc = ScriptNuiComponents; const bool hasItems = !selectedItems.empty(); const bool singleItem = selectedItems.size() == 1; + // A single directory selected on the remote side receives a watched upload, any other selection is ignored. + const auto watchTarget = otherSideSelectedItems.size() == 1 && otherSideSelectedItems.front().isDirectoryLike() && + !otherSideSelectedItems.front().isParentLink && otherSideSelectedItems.front().nameIsValidUtf8 + ? std::optional{otherSideSelectedItems.front().name} + : std::nullopt; std::vector items; @@ -834,6 +917,37 @@ LocalSideModel::contextMenuItems(std::vector const& selec !hasItems || archiveTransferDialog_ == nullptr ) ); + items.push_back( + Snc::PopupMenu::item( + language->get("fileExplorer", "contextMenu", "uploadAndWatch"), + Ui5Icons::show(), + [this, selectedItems, watchTarget]() + { + onUploadAndWatch(selectedItems.front(), watchTarget); + }, + !singleItem || fileTracking_ == nullptr, + /*shortcut=*/{}, + fmt::format( + fmt::runtime(language->get( + "fileExplorer", "contextMenu", watchTarget ? "uploadAndWatchIntoSelectionHint" : "uploadAndWatchHint" + )), + remoteModel_->currentRemotePath() + .and_then( + [&watchTarget](Paths::RemotePath const& directory) + { + return FileExplorer::targetDirectoryOf(directory, watchTarget); + } + ) + .transform( + [](Paths::RemotePath const& directory) + { + return directory.value(); + } + ) + .value_or(std::string{}) + ) + ) + ); items.push_back(Snc::PopupMenu::separator()); } @@ -960,6 +1074,11 @@ void LocalSideModel::setRemoteModel(RemoteSideModel* model) remoteModel_ = model; } +void LocalSideModel::setFileTracking(FileTrackingPanel* fileTracking) +{ + fileTracking_ = fileTracking; +} + std::expected LocalSideModel::currentLocalPath() const { return Paths::LocalPath::fromUtf8(listedPath_.generic_string()); diff --git a/frontend/source/frontend/file_explorer/remote_side_model.cpp b/frontend/source/frontend/file_explorer/remote_side_model.cpp index cf7157fc..e64000fc 100644 --- a/frontend/source/frontend/file_explorer/remote_side_model.cpp +++ b/frontend/source/frontend/file_explorer/remote_side_model.cpp @@ -89,7 +89,10 @@ RemoteSideModel::RemoteSideModel( } std::vector -RemoteSideModel::contextMenuItems(std::vector const& selectedItems) +RemoteSideModel::contextMenuItems( + std::vector const& selectedItems, + std::vector const& /*otherSideSelectedItems*/ +) { namespace Snc = ScriptNuiComponents; const bool hasItems = !selectedItems.empty(); @@ -149,7 +152,9 @@ RemoteSideModel::contextMenuItems(std::vector const& sele { onDownloadAndOpen(selectedItems.front(), false, true); }, - !singleItem || fileTracking_ == nullptr || !canOpen + !singleItem || fileTracking_ == nullptr || !canOpen, + /*shortcut=*/{}, + language->get("fileExplorer", "contextMenu", "openAndWatchHint") ), Snc::PopupMenu::item( language->get("fileExplorer", "contextMenu", "openWithAndWatch"), @@ -158,7 +163,9 @@ RemoteSideModel::contextMenuItems(std::vector const& sele { onDownloadAndOpen(selectedItems.front(), true, true); }, - !singleItem || fileTracking_ == nullptr || !canOpen + !singleItem || fileTracking_ == nullptr || !canOpen, + /*shortcut=*/{}, + language->get("fileExplorer", "contextMenu", "openWithAndWatchHint") ), Snc::PopupMenu::separator(), Snc::PopupMenu::item( @@ -853,171 +860,34 @@ void RemoteSideModel::onDownloadAndOpen(NuiFileExplorer::Item const& item, bool Log::error("File tracking panel is not available, cannot download/open file."); return; } - - // The download goes into an instance directory of its own, which the watch observes. - Nui::RpcClient::callWithBackChannel( - "FileTracking::createInstance", - lifetime_.guarded( - [this, item, openWith, watch](Nui::val response) - { - if (!response.hasOwnProperty("success") || !response["success"].as()) - { - Log::error("FileTracking: createInstance failed"); - confirmDialog_->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Danger, - .headerText = language->get("fileTrackingPanel", "instanceCreationFailed"), - .text = language->get("fileTrackingPanel", "instanceCreationFailedDesc"), - .buttons = ConfirmDialog::Button::Ok, - }); - return; - } - onFileTrackingInstanceCreated( - item, - openWith, - watch, - Ids::makeInstanceId(response["instanceId"].as()), - response["instanceDir"].as() - ); - } - ) - ); -} - -void RemoteSideModel::onFileTrackingInstanceCreated( - NuiFileExplorer::Item const& item, - bool openWith, - bool synchronize, - Ids::InstanceId const& instanceId, - std::string const& instanceDirStr -) -{ - Log::info("FileTracking: createInstance succeeded, id={}, dir={}", instanceId.value(), instanceDirStr); - - const auto instanceDirectory = Paths::LocalPath::fromUtf8(instanceDirStr); + const auto failedTitle = language->get("remoteSideModel", watch ? "openAndWatchFailedTitle" : "downloadFailed"); const auto remotePath = remotePathOf(item); - const auto localPath = remotePath.and_then( - [&instanceDirectory](Paths::RemotePath const& path) - { - return Paths::LocalPathSegment::make(*path.filename()) - .and_then( - [&instanceDirectory](Paths::LocalPathSegment const& segment) - { - return instanceDirectory.transform( - [&segment](Paths::LocalPath const& directory) - { - return directory / segment; - } - ); - } - ); - } - ); - if (!localPath) - return showPathRefused(language->get("remoteSideModel", "downloadFailed"), localPath.error()); - - operationQueue_->enqueueDownload( + if (!remotePath) + return showPathRefused(failedTitle, remotePath.error()); + fileTracking_->open( *remotePath, - *localPath, item.size, + watch, lifetime_.guarded( - [this, - instanceId, - instanceDirectory = *instanceDirectory, - remotePath = *remotePath, - localPath = *localPath, - openWith, - synchronize](std::optional const& opId, std::string const& info) + [this, openWith](Ids::InstanceId const& instanceId, Paths::LocalPath const& localPath) { - if (!opId) - { - Log::error( - "FileTracking: failed to enqueue download for instance {}: {}", instanceId.value(), info - ); - return; - } - operationQueue_->addCompletionCallback( - *opId, - lifetime_.guarded( - [this, instanceId, instanceDirectory, remotePath, localPath, openWith, synchronize]( - bool success - ) - { - onFileDownloadComplete( - instanceId, instanceDirectory, remotePath, localPath, openWith, synchronize, success - ); - } - ) + Log::info("FileTracking: opening {} (openWith={})", localPath.toUtf8(), openWith); + FileExplorer::openLocalFile( + lifetime_, *confirmDialog_, localPath, openWith, discardWhenDeclined(instanceId, localPath) ); } ), - false, // allowOverwrite - false, // insertRefresh - false, // createMissingDirectories - SharedData::OperationMode::PriorityQueued - ); -} - -void RemoteSideModel::onFileDownloadComplete( - Ids::InstanceId const& instanceId, - Paths::LocalPath const& instanceDirectory, - Paths::RemotePath const& remotePath, - Paths::LocalPath const& localPath, - bool openWith, - bool synchronize, - bool success -) -{ - if (!success) - { - Log::error("FileTracking: download failed for {}", localPath.toUtf8()); - return; - } - Log::info("FileTracking: download completed for {}", localPath.toUtf8()); - - if (!synchronize) - { - Log::info("FileTracking: opening {} (openWith={}, no sync)", localPath.toUtf8(), openWith); - FileExplorer::openLocalFile( - lifetime_, *confirmDialog_, localPath, openWith, discardWhenDeclined(instanceId, localPath) - ); - return; - } - - Nui::RpcClient::callWithBackChannel( - "FileTracking::addWatch", lifetime_.guarded( - [this, instanceId, instanceDirectory, remotePath, localPath, openWith](Nui::val watchResponse) + [this, failedTitle](std::string const& reason) { - if (!watchResponse.hasOwnProperty("success") || !watchResponse["success"].as()) - { - Log::error("FileTracking: addWatch failed for instance {}", instanceId.value()); - return; - } - Log::info("FileTracking: addWatch succeeded for instance {}", instanceId.value()); - onFileWatchAdded(instanceId, instanceDirectory, remotePath, localPath, openWith); + confirmDialog_->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Danger, + .headerText = failedTitle, + .text = reason, + .buttons = ConfirmDialog::Button::Ok, + }); } - ), - instanceId.value(), - instanceDirectory.toUtf8(), - true, - fileTracking_ ? fileTracking_->downloadTempSuffix() : std::string{Persistence::defaultTempFileSuffix} - ); -} - -void RemoteSideModel::onFileWatchAdded( - Ids::InstanceId const& instanceId, - Paths::LocalPath const& instanceDirectory, - Paths::RemotePath const& remotePath, - Paths::LocalPath const& localPath, - bool openWith -) -{ - fileTracking_->startWatching(instanceId, instanceDirectory, remotePath, localPath); - - Log::info("FileTracking: opening {} (openWith={})", localPath.toUtf8(), openWith); - - FileExplorer::openLocalFile( - lifetime_, *confirmDialog_, localPath, openWith, discardWhenDeclined(instanceId, localPath) + ) ); } diff --git a/frontend/source/frontend/main_page.cpp b/frontend/source/frontend/main_page.cpp index 756aa666..3746c2c2 100644 --- a/frontend/source/frontend/main_page.cpp +++ b/frontend/source/frontend/main_page.cpp @@ -124,6 +124,15 @@ MainPage::MainPage(Persistence::StateHolder* stateHolder, FrontendEvents* events void MainPage::onSetupComplete() { Log::info("Setup is complete."); + // The watches of a frontend that loaded before this one (a reload) are not shown anywhere any more. + Nui::RpcClient::callWithBackChannel( + "FileTracker::stopAll", + [](Nui::val response) + { + if (!response.hasOwnProperty("success") || !response["success"].as()) + Log::error("FileTracker::stopAll failed"); + } + ); applyPanelTitles(); impl_->panelTitlesListener = language->listenToLanguageChange( [](std::string const&) diff --git a/frontend/source/frontend/session.cpp b/frontend/source/frontend/session.cpp index feb175cd..ab21e80a 100644 --- a/frontend/source/frontend/session.cpp +++ b/frontend/source/frontend/session.cpp @@ -162,7 +162,7 @@ struct Session::Implementation , operationQueue{this->stateHolder, this->events, this->persistenceName, this->engineOptions.queueOptions.value(), this->confirmDialog, &fileExplorerPanel.localSideModel(), fileExplorerPanel.remoteSideModel()} , frontendSessionManager{} , sessionOptions{params.stateHolder, params.events, this->persistenceName, this->sessionLayoutId, params.confirmDialog} - , fileTrackingPanel{params.stateHolder, params.events, params.confirmDialog} + , fileTrackingPanel{params.confirmDialog} , commandHistoryPanel{ params.commandStoreClient, params.events, @@ -725,14 +725,9 @@ void Session::openSftp(std::string const& username, bool forceOpen) localSideModel().engine(std::move(fileEngine)); if (remoteSideModel()) impl_->operationQueue.activate(remoteSideModel()->engine(), sshTerminalEngine->sshSessionId()); - impl_->fileTrackingPanel.activate( - &impl_->operationQueue, - sshTerminalEngine->sshSessionId(), - Persistence::effectiveTempFileSuffix( - opts.sftpOptions->downloadOptions ? opts.sftpOptions->downloadOptions->tempFileSuffix : std::nullopt - ) - ); + impl_->fileTrackingPanel.activate(sshTerminalEngine->sshSessionId()); localSideModel().operationQueue(&impl_->operationQueue); + localSideModel().setFileTracking(&impl_->fileTrackingPanel); if (remoteSideModel()) { remoteSideModel()->operationQueue(&impl_->operationQueue); diff --git a/frontend/source/frontend/session_components/file_tracking.cpp b/frontend/source/frontend/session_components/file_tracking.cpp index ee0f0dfb..a1fab749 100644 --- a/frontend/source/frontend/session_components/file_tracking.cpp +++ b/frontend/source/frontend/session_components/file_tracking.cpp @@ -1,100 +1,79 @@ #include -#include -#include - -#include -#include +#include +#include + +#include +#include +#include +#include +#include +#include #include #include #include #include #include -#include #include #include #include #include #include -#include #include #include +#include -#include -#include -#include +#include #include #include #include +#include #include +#include #include #include -using namespace std::string_literals; - struct FileTrackingPanel::Implementation { /** - * @brief One downloaded remote file or directory whose local copy is watched. + * @brief What the row of one watch shows, shared with its cells. */ - struct TrackedEntry + struct Row { - Ids::InstanceId instanceId; - Paths::LocalPath instanceDirectory; - Paths::RemotePath remotePath; - Paths::LocalPath localPath; - bool autoReupload{true}; - bool moveRemoteOnLocalMove{false}; - bool deleteRemote{false}; - /** @brief Ask before replacing a server copy that changed since it was opened or last uploaded. */ - bool checkRemoteChanges{true}; - /** @brief Whether uploads are on their way, shared with the status cell of the entry's row. */ + std::shared_ptr> name{std::make_shared>()}; + std::shared_ptr> remotePath{std::make_shared>()}; std::shared_ptr> uploading{std::make_shared>(false)}; - FileTracking::PendingUploads pendingUploads{}; - FileTracking::KnownServerContent serverContent{}; - }; - - /** - * @brief A changed path of a tracked entry on its way to the server. - */ - struct Upload - { - Ids::InstanceId instanceId; - FileTracking::LocalChange change; - Paths::RemotePath destination; - /** @brief Compare with the server first, when the entry asks for that. */ - bool checkServer{true}; - }; + std::shared_ptr> autoReupload{std::make_shared>(false)}; + std::shared_ptr> moveRemote{std::make_shared>(false)}; + std::shared_ptr> deleteRemote{std::make_shared>(false)}; + std::shared_ptr> checkRemote{std::make_shared>(false)}; + std::shared_ptr> downloadedCopy{std::make_shared>(false)}; - /** - * @brief How the upload of a change ended. - */ - enum class UploadOutcome - { - Uploaded, - NotUploaded, + void show(SharedData::TrackedEntry const& entry) + { + *name = entry.name; + *remotePath = entry.remotePath.value(); + *uploading = entry.uploading; + *autoReupload = entry.options.autoReupload; + *moveRemote = entry.options.moveRemoteOnLocalMove; + *deleteRemote = entry.options.deleteRemoteOnLocalDelete; + *checkRemote = entry.options.checkRemoteChanges; + *downloadedCopy = entry.downloadedCopy; + } }; - Persistence::StateHolder* stateHolder; - FrontendEvents* events; ConfirmDialog* confirmDialog; - OperationQueue* operationQueue{nullptr}; Ids::SessionId sessionId{}; - std::string downloadTempSuffix{Persistence::defaultTempFileSuffix}; - - std::map entries{}; - FileTracking::RowOrder rows{}; + std::map rows{}; + FileTracking::RowOrder rowOrder{}; Nui::Observed entryCount{std::size_t{0}}; - std::map fileChangeListeners{}; - /** @brief Server-changed questions waiting for the one on screen, so no question replaces another. */ - std::deque serverChangedQuestions{}; - bool serverChangedQuestionOpen{false}; - Nui::Observed orphanedCount{-1}; Nui::Observed cleanupRunning{false}; + Nui::Observed progress{}; + std::vector listeners{}; std::shared_ptr table{std::make_shared( ScriptNuiComponents::ResizableTable::HeaderRow{ @@ -144,24 +123,190 @@ struct FileTrackingPanel::Implementation Utility::LifetimeToken lifetime{}; - Implementation(Persistence::StateHolder* stateHolder, FrontendEvents* events, ConfirmDialog* confirmDialog) - : stateHolder{stateHolder} - , events{events} - , confirmDialog{confirmDialog} + explicit Implementation(ConfirmDialog* confirmDialog) + : confirmDialog{confirmDialog} {} /** - * @brief Shows what RPC replies and change events changed, they arrive outside of DOM event handlers. + * @brief Shows what RPC replies and tracker events changed, they arrive outside of DOM event handlers. */ static void syncView() { Nui::globalEventContext.executeActiveEventsImmediately(); } - TrackedEntry* findEntry(Ids::InstanceId const& instanceId) + /** + * @brief Yes replaces the server's version, No keeps it, no button means another dialog took the question's place. + */ + static SharedData::ServerChangedAnswer answerOf(std::optional const& button) + { + if (!button) + return SharedData::ServerChangedAnswer::Dismissed; + return *button == ConfirmDialog::Button::Yes ? SharedData::ServerChangedAnswer::Replace + : SharedData::ServerChangedAnswer::Keep; + } + + static bool succeeded(Nui::val const& response) + { + return response.hasOwnProperty("success") && response["success"].as(); + } + + /** + * @brief The translated error of a failed reply, nothing when it has none. + */ + static std::optional errorOf(Nui::val const& response) + { + if (!response.hasOwnProperty("error")) + return std::nullopt; + return language->translate(response["error"].as()); + } + + std::string rpcName(std::string_view function) const + { + return fmt::format("FileTracker::{}::{}", sessionId.value(), function); + } + + /** + * @brief Forwards @p function with @p arguments to the tracker, a refusal is only logged. + */ + template + void forward(std::string_view function, ArgumentsT&&... arguments) + { + Nui::RpcClient::callWithBackChannel( + rpcName(function), + [function = std::string{function}](Nui::val response) + { + if (!succeeded(response)) + Log::error("FileTracker::{} failed: {}", function, errorOf(response).value_or("")); + }, + std::forward(arguments)... + ); + } + + void listen() + { + listeners.push_back(Nui::RpcClient::autoRegisterFunction( + rpcName("onEntry"), + lifetime.guarded( + [this](SharedData::TrackedEntry const& entry) + { + showEntry(entry); + syncView(); + } + ) + )); + listeners.push_back(Nui::RpcClient::autoRegisterFunction( + rpcName("onEntryRemoved"), + lifetime.guarded( + [this](std::string const& watchId) + { + removeEntry(Ids::makeWatchId(watchId)); + syncView(); + } + ) + )); + listeners.push_back(Nui::RpcClient::autoRegisterFunction( + rpcName("onEntriesEnded"), + lifetime.guarded( + [](SharedData::EndedWatches const& ended) + { + notifyEnded(ended); + } + ) + )); + listeners.push_back(Nui::RpcClient::autoRegisterFunction( + rpcName("onQuestion"), + lifetime.guarded( + [this](SharedData::ServerChangedQuestion const& question) + { + ask(question); + } + ) + )); + listeners.push_back(Nui::RpcClient::autoRegisterFunction( + rpcName("onProgress"), + lifetime.guarded( + [this](SharedData::TrackerProgress const& shown) + { + progress = shown; + syncView(); + } + ) + )); + listeners.push_back(Nui::RpcClient::autoRegisterFunction( + rpcName("onOrphanedCount"), + lifetime.guarded( + [this](std::uint64_t count) + { + orphanedCount = static_cast(count); + syncView(); + } + ) + )); + } + + void showEntry(SharedData::TrackedEntry const& entry) { - const auto found = entries.find(instanceId); - return found == entries.end() ? nullptr : &found->second; + auto [row, inserted] = rows.try_emplace(entry.watchId); + row->second.show(entry); + if (!inserted) + return; + table->addRow(buildRow(entry.watchId, row->second)); + rowOrder.add(entry.watchId); + entryCount = rows.size(); + } + + void removeEntry(Ids::WatchId const& watchId) + { + rows.erase(watchId); + if (const auto row = rowOrder.remove(watchId)) + table->remove(*row); + entryCount = rows.size(); + } + + /** + * @brief Tells the user why watches ended that they did not stop, their rows are gone already. + */ + static void notifyEnded(SharedData::EndedWatches const& ended) + { + const auto text = [&ended]() + { + switch (ended.reason) + { + case SharedData::WatchEndReason::CopyGone: + return language->get("fileTracking", "watchEndedCopyGone"); + case SharedData::WatchEndReason::ConnectionLost: + break; + } + return language->get("fileTracking", "watchesEndedConnectionLost"); + }(); + const auto names = ended.entries | + std::views::transform( + [](SharedData::TrackedEntry const& entry) + { + return fmt::format("'{}'", entry.name); + } + ); + Notifications::warning(fmt::format(fmt::runtime(text), fmt::join(names, ", "))); + } + + void ask(SharedData::ServerChangedQuestion const& question) + { + confirmDialog->open({ + .styleVariant = ScriptNuiComponents::StyleVariant::Warning, + .headerText = language->get("fileTracking", "serverChangedHeader"), + .text = fmt::format( + fmt::runtime(language->get("fileTracking", "serverChangedText")), question.remotePath.value() + ), + .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, + .focusButton = ConfirmDialog::Button::No, + .onClose = lifetime.guarded( + [this, questionId = question.questionId](std::optional button) + { + forward("answer", questionId.value(), Utility::enumToString(answerOf(button))); + } + ), + }); } void refreshOrphanedCount() @@ -171,11 +316,8 @@ struct FileTrackingPanel::Implementation lifetime.guarded( [this](Nui::val response) { - if (!response.hasOwnProperty("success") || !response["success"].as()) - { - Log::error("FileTracking: countOrphaned RPC returned failure or no 'success' field"); - return; - } + if (!succeeded(response)) + return Log::error("FileTracking: countOrphaned failed: {}", errorOf(response).value_or("")); orphanedCount = response["count"].as(); syncView(); } @@ -192,21 +334,83 @@ struct FileTrackingPanel::Implementation [this](Nui::val response) { cleanupRunning = false; - const bool succeeded = response.hasOwnProperty("success") && response["success"].as(); - if (!succeeded) - Log::error("FileTracking: forceCleanup failed"); - else if (response.hasOwnProperty("removed")) + if (succeeded(response)) + { Log::info( "FileTracking: forceCleanup removed {} orphaned instance(s)", response["removed"].as() ); - if (succeeded) refreshOrphanedCount(); + } + else + Log::error("FileTracking: forceCleanup failed: {}", errorOf(response).value_or("")); syncView(); } ) ); } + /** + * @brief How far the tracker is with the changes on their way, apart from the operation queue. Stays mounted, so + * the bar moves smoothly, and is hidden while nothing is on its way. + */ + Nui::ElementRenderer renderProgress() + { + using namespace Nui::Elements; + using namespace Nui::Attributes; + using Nui::Elements::div; + using Nui::Elements::span; + + // clang-format off + return div{ + class_ = Nui::observe(progress).generate([this]() + { + auto const& shown = progress.value(); + const bool active = shown.pending > 0 || shown.waitingForAnswer > 0; + return std::string{active ? "file-tracking-progress" : "file-tracking-progress file-tracking-progress-idle"}; + }), + }( + div{class_ = "file-tracking-progress-track"}( + div{ + class_ = "file-tracking-progress-fill", + style = Nui::observe(progress).generate([this]() + { + auto const& shown = progress.value(); + const auto total = shown.pending + shown.done; + return fmt::format("width: {}%", total == 0 ? 0 : shown.done * 100 / total); + }), + }() + ), + span{class_ = "file-tracking-progress-text"}( + Nui::observe(progress).generate([this]() -> Nui::ElementRenderer + { + auto const& shown = progress.value(); + return Nui::Elements::text{fmt::format( + fmt::runtime(language->get("fileTracking", "progressSyncing")), + shown.done, + shown.pending + shown.done + )}(); + }) + ), + span{ + class_ = Nui::observe(progress).generate([this]() + { + return std::string{progress.value().waitingForAnswer > 0 + ? "file-tracking-progress-waiting" + : "file-tracking-progress-waiting file-tracking-progress-idle"}; + }), + }( + Nui::observe(progress).generate([this]() -> Nui::ElementRenderer + { + return Nui::Elements::text{fmt::format( + fmt::runtime(language->get("fileTracking", "progressWaitingForAnswer")), + progress.value().waitingForAnswer + )}(); + }) + ) + ); + // clang-format on + } + Nui::ElementRenderer renderHeader() { using namespace Nui::Elements; @@ -244,6 +448,7 @@ struct FileTrackingPanel::Implementation ) ) ), + renderProgress(), div{class_ = "file-tracking-header-actions"}( ScriptNuiComponents::button({ .text = Nui::observe(cleanupRunning).generate([](bool running) -> std::string @@ -269,23 +474,49 @@ struct FileTrackingPanel::Implementation } /** - * @brief A cell with a switch for the @p option of the entry @p instanceId. + * @brief A cell with a switch that shows @p shown and forwards its changes as @p option of @p watchId. + * @param downloadedCopy Whether the row is a temporary copy, which never moves or deletes the server's file: the + * switch of such an option is disabled then. Nothing for an option that applies to every row. */ - ScriptNuiComponents::ResizableTable::TableCell - switchCell(Ids::InstanceId const& instanceId, bool isChecked, bool TrackedEntry::* option) + ScriptNuiComponents::ResizableTable::TableCell switchCell( + Ids::WatchId const& watchId, + std::shared_ptr> shown, + SharedData::WatchOption option, + std::shared_ptr> downloadedCopy = nullptr + ) { using namespace ScriptNuiComponents; return ResizableTable::TableCell{ - [this, instanceId, isChecked, option](std::unique_ptr) - -> Nui::ElementRenderer + [this, watchId, shown = std::move(shown), option, downloadedCopy = std::move(downloadedCopy)]( + std::unique_ptr + ) -> Nui::ElementRenderer { + std::vector attributes{}; + if (downloadedCopy) + { + attributes.push_back(Nui::Attributes::disabled = Nui::observe(*downloadedCopy).generate( + [](bool isCopy) + { + return isCopy; + } + )); + attributes.push_back(Nui::Attributes::title = Nui::observe(*downloadedCopy).generate( + [](bool isCopy) + { + return isCopy ? std::string{language->get( + "fileTracking", "downloadedCopyOptionHint" + )} + : std::string{}; + } + )); + } return ScriptNuiComponents::switch_({ - .isChecked = isChecked, + .isChecked = shown, + .attributes = std::move(attributes), .onChange = lifetime.guarded( - [this, instanceId, option](bool checked, Nui::WebApi::MouseEvent const&) + [this, watchId, option](bool checked, Nui::WebApi::MouseEvent const&) { - if (auto* entry = findEntry(instanceId)) - entry->*option = checked; + forward("setOption", watchId.value(), Utility::enumToString(option), checked); } ), }); @@ -293,20 +524,41 @@ struct FileTrackingPanel::Implementation }; } - ScriptNuiComponents::ResizableTable::TableRow buildRowForEntry(TrackedEntry const& entry) + /** + * @brief A cell that shows @p text and follows its changes. + */ + static ScriptNuiComponents::ResizableTable::TableCell textCell(std::shared_ptr> text) { using namespace ScriptNuiComponents; - const auto& instanceId = entry.instanceId; + return ResizableTable::TableCell{ + [text = std::move(text)](std::unique_ptr) -> Nui::ElementRenderer + { + return Nui::Elements::div{ + Nui::Attributes::style = "display:contents", + }(Nui::observe(*text).generate( + [](std::string const& value) -> Nui::ElementRenderer + { + return Nui::Elements::text{value}(); + } + )); + }, + }; + } + ScriptNuiComponents::ResizableTable::TableRow buildRow(Ids::WatchId const& watchId, Row const& row) + { + using namespace ScriptNuiComponents; return ResizableTable::TableRow{ - entry.localPath.filename() ? std::string{entry.localPath.filename()->view()} : entry.localPath.toUtf8(), - entry.remotePath.value(), - switchCell(instanceId, entry.autoReupload, &TrackedEntry::autoReupload), - switchCell(instanceId, entry.moveRemoteOnLocalMove, &TrackedEntry::moveRemoteOnLocalMove), - switchCell(instanceId, entry.deleteRemote, &TrackedEntry::deleteRemote), - switchCell(instanceId, entry.checkRemoteChanges, &TrackedEntry::checkRemoteChanges), + textCell(row.name), + textCell(row.remotePath), + switchCell(watchId, row.autoReupload, SharedData::WatchOption::AutoReupload), + switchCell(watchId, row.moveRemote, SharedData::WatchOption::MoveRemoteOnLocalMove, row.downloadedCopy), + switchCell( + watchId, row.deleteRemote, SharedData::WatchOption::DeleteRemoteOnLocalDelete, row.downloadedCopy + ), + switchCell(watchId, row.checkRemote, SharedData::WatchOption::CheckRemoteChanges), ResizableTable::TableCell{ - [uploading = entry.uploading](std::unique_ptr) -> Nui::ElementRenderer + [uploading = row.uploading](std::unique_ptr) -> Nui::ElementRenderer { return Nui::Elements::div{ Nui::Attributes::class_ = "file-tracking-status", @@ -323,16 +575,16 @@ struct FileTrackingPanel::Implementation }, }, ResizableTable::TableCell{ - [this, instanceId](std::unique_ptr) -> Nui::ElementRenderer + [this, watchId](std::unique_ptr) -> Nui::ElementRenderer { return ScriptNuiComponents::button({ .text = std::string{language->get("fileTracking", "stopButton")}, .attributes = { Nui::Attributes::onClick = lifetime.guarded( - [this, instanceId](Nui::val) + [this, watchId](Nui::val) { - stopWatching(instanceId); + forward("stop", watchId.value()); } ), }, @@ -342,638 +594,116 @@ struct FileTrackingPanel::Implementation }, }; } - - void stopWatching(Ids::InstanceId const& instanceId) - { - Log::info("FileTracking: stopping watch for instance {}", instanceId.value()); - Nui::RpcClient::callWithBackChannel( - "FileTracking::destroyInstance", - lifetime.guarded( - [this, instanceId](Nui::val response) - { - if (!response.hasOwnProperty("success") || !response["success"].as()) - { - Log::error("FileTracking: destroyInstance failed for {}", instanceId.value()); - return; - } - forgetEntry(instanceId); - syncView(); - refreshOrphanedCount(); - } - ), - instanceId.value() - ); - } - - void forgetEntry(Ids::InstanceId const& instanceId) - { - fileChangeListeners.erase(instanceId); - entries.erase(instanceId); - if (const auto row = rows.remove(instanceId)) - table->remove(*row); - std::erase_if( - serverChangedQuestions, - [&instanceId](Upload const& upload) - { - return upload.instanceId == instanceId; - } - ); - entryCount = entries.size(); - } - - std::expected - remotePathOf(TrackedEntry const& entry, Paths::LocalPath const& local) const - { - return FileTracking::remotePathOfChange(local, entry.instanceDirectory, entry.remotePath); - } - - /** - * @brief Records the states of a FileTracking::fileStates reply, only those below @p below when given. - */ - static void rememberFileStates(TrackedEntry& entry, Nui::val const& files, std::optional const& below) - { - const auto count = files["length"].as(); - for (int index = 0; index < count; ++index) - { - const auto file = files[index]; - const auto path = Paths::LocalPath::fromUtf8(file["path"].as()); - if (!path || (below && !path->isWithin(*below))) - continue; - if (file.hasOwnProperty("directory")) - { - entry.serverContent.rememberDirectory(*path); - continue; - } - entry.serverContent.rememberFile( - *path, - FileTracking::ServerState{ - .mtime = static_cast(file["mtime"].as()), - .size = static_cast(file["size"].as()), - } - ); - } - } - - /** - * @brief Asks the backend for the files of @p instanceId: they carry the server's modification time after a - * download or an upload. @p onDone runs afterwards, also when the reply failed. - */ - void requestFileStates( - Ids::InstanceId const& instanceId, - std::optional below, - std::function onDone - ) - { - Nui::RpcClient::callWithBackChannel( - "FileTracking::fileStates", - lifetime.guarded( - [this, instanceId, below = std::move(below), onDone = std::move(onDone)](Nui::val reply) - { - if (auto* entry = findEntry(instanceId); entry && reply.hasOwnProperty("files")) - rememberFileStates(*entry, reply["files"], below); - else if (entry) - Log::warn("FileTracking: no file states for instance {}, re-uploads will ask.", instanceId.value()); - onDone(); - } - ), - instanceId.value() - ); - } - - void onFileChanged(Ids::InstanceId const& instanceId, Nui::val const& payload) - { - auto* entry = findEntry(instanceId); - if (!entry) - return; - - const auto action = payload["action"].as(); - const auto directory = payload["directory"].as(); - const auto filename = payload["filename"].as(); - Log::debug("FileTracking: file changed in instance {}: {} {}", instanceId.value(), action, filename); - - const auto localPathOfChange = [&directory](std::string const& name) - { - return Paths::LocalPath::fromUtf8(fmt::format("{}/{}", directory, name)); - }; - // Only the tracked file or directory maps to the remote side, anything else in the instance - // directory (Finder's .DS_Store for one) would land next to it on the server. - const auto isTracked = [entry](std::expected const& changed) - { - return changed && changed->isWithin(entry->localPath); - }; - const auto changedPath = localPathOfChange(filename); - if (!isTracked(changedPath)) - { - Log::debug("FileTracking: ignoring change outside the tracked item: {}", filename); - return; - } - const FileTracking::LocalChange change{ - .path = *changedPath, - .isDirectory = payload.hasOwnProperty("isDirectory") && payload["isDirectory"].as(), - }; - - if (action == "Deleted") - return onDeleted(*entry, change.path); - - // A move from outside the tracked item into it (an atomic save renaming a temporary over it) is a change. - const auto oldPath = localPathOfChange(payload["oldFilename"].as()); - if (action == "Moved" && isTracked(oldPath)) - return onMoved(*entry, *oldPath, change); - - onContentChanged(*entry, change); - } - - /** - * @param checkServer Compare with the server first, when the entry asks for that. Changes below a directory - * that was just uploaded skip it, the states recorded after that upload may already be newer. - */ - void onContentChanged(TrackedEntry& entry, FileTracking::LocalChange const& change, bool checkServer = true) - { - if (!entry.autoReupload || !operationQueue) - return; - const auto destination = remotePathOf(entry, change.path); - if (!destination) - { - Log::warn( - "FileTracking: not uploading the change in instance {}: {}", - entry.instanceId.value(), - destination.error().toString() - ); - return; - } - queueUpload(Upload{ - .instanceId = entry.instanceId, - .change = change, - .destination = *destination, - .checkServer = checkServer, - }); - } - - void onDeleted(TrackedEntry& entry, Paths::LocalPath const& path) - { - if (!entry.deleteRemote || !operationQueue) - return; - if (entry.pendingUploads.removeRemoteWhenDone(path)) - return; - if (!entry.serverContent.has(path)) - { - Log::debug("FileTracking: '{}' is not on the server, nothing to delete there.", path.toUtf8()); - return; - } - // Never recursive: a directory deleted entry by entry reports its content first, so only one moved away as a - // whole (to the trash) still has known content, and its server copy keeps it. - if (entry.serverContent.hasBelow(path)) - { - Log::info( - "FileTracking: not deleting the server's copy of directory '{}', it was removed with its content.", - path.toUtf8() - ); - return; - } - const auto remotePath = remotePathOf(entry, path); - if (!remotePath) - { - Log::error( - "FileTracking: not mirroring the deletion of '{}': {}", path.toUtf8(), remotePath.error().toString() - ); - return; - } - removeRemote(entry, path, *remotePath); - } - - void onMoved(TrackedEntry& entry, Paths::LocalPath const& oldPath, FileTracking::LocalChange const& change) - { - const auto oldRemotePath = remotePathOf(entry, oldPath); - const auto newRemotePath = remotePathOf(entry, change.path); - if (!oldRemotePath || !newRemotePath) - { - Log::warn( - "FileTracking: not mirroring the move in instance {}: {}", - entry.instanceId.value(), - (oldRemotePath ? newRemotePath.error() : oldRemotePath.error()).toString() - ); - return; - } - - const auto meaning = - FileTracking::meaningOfMove(entry.serverContent.has(change.path), entry.serverContent.has(oldPath)); - if (meaning == FileTracking::MoveMeaning::Rename) - { - if (!entry.moveRemoteOnLocalMove || !operationQueue) - return; - operationQueue->enqueueRename( - *oldRemotePath, - *newRemotePath, - [instanceId = entry.instanceId](std::optional const& operationId, std::string const& info) - { - if (!operationId) - Log::error( - "FileTracking: failed to enqueue remote rename for instance {}: {}", instanceId.value(), info - ); - }, - SharedData::OperationMode::PriorityQueued - ); - entry.serverContent.move(oldPath, change.path); - return; - } - - Log::info("FileTracking: '{}' replaced '{}', uploading it.", oldPath.toUtf8(), change.path.toUtf8()); - if (entry.moveRemoteOnLocalMove && operationQueue && !entry.pendingUploads.removeRemoteWhenDone(oldPath) && - entry.serverContent.has(oldPath) && !entry.serverContent.hasBelow(oldPath)) - removeRemote(entry, oldPath, *oldRemotePath); - onContentChanged(entry, change); - } - - void removeRemote(TrackedEntry& entry, Paths::LocalPath const& local, Paths::RemotePath const& remote) - { - if (!operationQueue) - return; - Log::info( - "FileTracking: enqueueing remote delete for instance {} due to local deletion: {}", - entry.instanceId.value(), - remote.value() - ); - operationQueue->enqueueDelete( - {remote}, - /*recursive=*/false, - [instanceId = entry.instanceId]( - std::optional> const& operationIds, std::string const& info - ) - { - if (!operationIds) - Log::error( - "FileTracking: failed to enqueue remote delete for instance {}: {}", instanceId.value(), info - ); - }, - SharedData::OperationMode::PriorityQueued - ); - entry.serverContent.forget(local); - } - - static void showUploading(TrackedEntry& entry) - { - *entry.uploading = !entry.pendingUploads.empty(); - } - - /** - * @brief Uploads a change unless the same path, or a directory above it, is already on its way; the change then - * goes once that upload ended. - */ - void queueUpload(Upload upload) - { - auto* entry = findEntry(upload.instanceId); - if (!entry || !operationQueue || !entry->pendingUploads.start(upload.change)) - return; - showUploading(*entry); - checkServerThenUpload(std::move(upload)); - } - - void checkServerThenUpload(Upload upload) - { - const auto* entry = findEntry(upload.instanceId); - const auto fileEngine = operationQueue ? operationQueue->fileEngine() : nullptr; - if (!entry || !entry->checkRemoteChanges || !upload.checkServer || !fileEngine) - return enqueueUpload(std::move(upload)); - - fileEngine->stat( - std::filesystem::path{upload.destination.value()}, - lifetime.guarded( - [this, upload]( - std::optional> const& result, std::string const& info - ) - { - const auto* entry = findEntry(upload.instanceId); - if (!entry) - return; - if (!result) - Log::warn("FileTracking: cannot check '{}' on the server: {}", upload.destination.value(), info); - // An unanswered check counts as changed. - const bool changed = !result || [&]() - { - auto const& [exists, stat] = *result; - if (upload.change.isDirectory) - return exists && !entry->serverContent.has(upload.change.path); - return FileTracking::serverChanged( - entry->serverContent.fileState(upload.change.path), - exists ? std::optional{FileTracking::ServerState{.mtime = stat.mtime, .size = stat.size}} - : std::nullopt - ); - }(); - if (changed) - askServerChanged(upload); - else - enqueueUpload(upload); - syncView(); - } - ) - ); - } - - void askServerChanged(Upload upload) - { - serverChangedQuestions.push_back(std::move(upload)); - if (!serverChangedQuestionOpen) - askNextServerChanged(); - } - - void askNextServerChanged() - { - while (!serverChangedQuestions.empty()) - { - auto upload = std::move(serverChangedQuestions.front()); - serverChangedQuestions.pop_front(); - if (!findEntry(upload.instanceId)) - continue; - - serverChangedQuestionOpen = true; - confirmDialog->open({ - .styleVariant = ScriptNuiComponents::StyleVariant::Warning, - .headerText = language->get("fileTracking", "serverChangedHeader"), - .text = fmt::format( - fmt::runtime(language->get("fileTracking", "serverChangedText")), upload.destination.value() - ), - .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, - .focusButton = ConfirmDialog::Button::No, - .onClose = lifetime.guarded( - [this, upload](std::optional button) - { - serverChangedQuestionOpen = false; - if (!button) - return abandonServerChangedQuestions(upload); - if (*button == ConfirmDialog::Button::Yes) - enqueueUpload(upload); - else - { - Log::info("FileTracking: kept the server's version of '{}'.", upload.destination.value()); - declineUpload(upload); - } - askNextServerChanged(); - } - ), - }); - return; - } - } - - /** - * @brief Another dialog replaced the question for @p upload: it and the waiting questions are asked again on the - * next save, nothing opens now over the dialog that replaced it. - */ - void abandonServerChangedQuestions(Upload const& upload) - { - Log::info("FileTracking: the question about '{}' was replaced by another dialog.", upload.destination.value()); - declineUpload(upload); - for (auto const& waiting : std::exchange(serverChangedQuestions, {})) - declineUpload(waiting); - } - - /** - * @brief The change is not uploaded, and the changes that arrived while it was asked about wait for the next - * save. - */ - void declineUpload(Upload const& upload) - { - if (auto* entry = findEntry(upload.instanceId)) - entry->pendingUploads.changesHandled(upload.change.path); - finishUpload(upload, UploadOutcome::NotUploaded); - } - - void enqueueUpload(Upload upload) - { - auto* entry = findEntry(upload.instanceId); - if (!entry || !operationQueue) - return finishUpload(upload, UploadOutcome::NotUploaded); - // The upload reads the file when it runs, the changes until now are in it. - entry->pendingUploads.changesHandled(upload.change.path); - // allowOverwrite: the tracked file replaces its remote original, checked above. - operationQueue->enqueueUpload( - upload.change.path, - upload.destination, - /*sizeBytes=*/0, - lifetime.guarded( - [this, upload](std::optional const& operationId, std::string const& info) - { - if (!operationId || !operationQueue) - { - // A temporary file an editor already renamed or removed ends up here. - Log::warn("FileTracking: did not upload '{}': {}", upload.change.path.toUtf8(), info); - finishUpload(upload, UploadOutcome::NotUploaded); - syncView(); - return; - } - operationQueue->addCompletionCallback( - *operationId, - lifetime.guarded( - [this, upload](bool success) - { - onUploadEnded(upload, success); - } - ) - ); - } - ), - /*allowOverwrite=*/true, - /*insertRefresh=*/false, - /*createMissingDirectories=*/false, - SharedData::OperationMode::PriorityQueued - ); - } - - /** - * @brief Once an upload ended, what the server has now is the state the next check compares with. The upload - * stays on its way until that state is known. - */ - void onUploadEnded(Upload const& upload, bool success) - { - if (!success) - return finishUpload(upload, UploadOutcome::NotUploaded); - - // Uploads carry the local modification time, so the local states are what the server has now. - if (upload.change.isDirectory) - return requestFileStates( - upload.instanceId, - upload.change.path, - [this, upload]() - { - finishUpload(upload, UploadOutcome::Uploaded); - syncView(); - } - ); - - const auto fileEngine = operationQueue ? operationQueue->fileEngine() : nullptr; - if (!fileEngine) - return finishUpload(upload, UploadOutcome::Uploaded); - fileEngine->stat( - std::filesystem::path{upload.destination.value()}, - lifetime.guarded( - [this, upload]( - std::optional> const& result, std::string const& - ) - { - if (auto* entry = findEntry(upload.instanceId); entry && result && result->first) - entry->serverContent.rememberFile( - upload.change.path, - FileTracking::ServerState{.mtime = result->second.mtime, .size = result->second.size} - ); - finishUpload(upload, UploadOutcome::Uploaded); - syncView(); - } - ) - ); - } - - /** - * @brief The upload of @p upload is no longer on its way. What changed meanwhile goes now: the path itself - * once more, the changes below an uploaded directory, or the removal of a path that went away. - */ - void finishUpload(Upload const& upload, UploadOutcome outcome) - { - auto* entry = findEntry(upload.instanceId); - if (!entry) - return; - auto finished = entry->pendingUploads.finish(upload.change.path); - showUploading(*entry); - if (finished.removeRemote) - { - if (outcome == UploadOutcome::Uploaded || entry->serverContent.has(upload.change.path)) - removeRemote(*entry, upload.change.path, upload.destination); - return; - } - if (finished.changedAgain) - queueUpload(upload); - for (auto const& change : finished.changedBelow) - onContentChanged(*entry, change, /*checkServer=*/false); - } }; -FileTrackingPanel::FileTrackingPanel( - Persistence::StateHolder* stateHolder, - FrontendEvents* events, - ConfirmDialog* confirmDialog -) - : impl_(std::make_unique(stateHolder, events, confirmDialog)) +FileTrackingPanel::FileTrackingPanel(ConfirmDialog* confirmDialog) + : impl_(std::make_unique(confirmDialog)) {} ROAR_PIMPL_SPECIAL_FUNCTIONS_IMPL(FileTrackingPanel); -void FileTrackingPanel::activate( - OperationQueue* operationQueue, - Ids::SessionId sessionId, - std::string downloadTempSuffix -) +void FileTrackingPanel::activate(Ids::SessionId sessionId) { - impl_->operationQueue = operationQueue; + deactivate(); impl_->sessionId = std::move(sessionId); - impl_->downloadTempSuffix = std::move(downloadTempSuffix); + impl_->listen(); impl_->refreshOrphanedCount(); } -std::string const& FileTrackingPanel::downloadTempSuffix() const -{ - return impl_->downloadTempSuffix; -} - void FileTrackingPanel::deactivate() { - impl_->fileChangeListeners.clear(); + impl_->listeners.clear(); impl_->table->clear(); + impl_->rowOrder.clear(); impl_->rows.clear(); - impl_->entries.clear(); impl_->entryCount = std::size_t{0}; - impl_->serverChangedQuestions.clear(); - impl_->operationQueue = nullptr; impl_->sessionId = {}; impl_->orphanedCount = -1; impl_->cleanupRunning = false; + impl_->progress = SharedData::TrackerProgress{}; } -void FileTrackingPanel::startWatching( - Ids::InstanceId const& instanceId, - Paths::LocalPath const& instanceDirectory, +void FileTrackingPanel::open( Paths::RemotePath const& remotePath, - Paths::LocalPath const& localPath + std::uint64_t sizeBytes, + bool watch, + std::function onOpened, + std::function onRefused ) { - Log::info( - "FileTracking: starting watch for instance {} (remote: {}, local: {})", - instanceId.value(), - remotePath.value(), - localPath.toUtf8() - ); - - impl_->fileChangeListeners.insert_or_assign( - instanceId, - Nui::RpcClient::autoRegisterFunction( - fmt::format("FileTracking::{}::onFileChanged", instanceId.value()), - impl_->lifetime.guarded( - [implementation = impl_.get(), instanceId](Nui::val payload) + Nui::RpcClient::callWithBackChannel( + impl_->rpcName("open"), + impl_->lifetime.guarded( + [remotePath, onOpened = std::move(onOpened), onRefused = std::move(onRefused)](Nui::val response) + { + if (!Implementation::succeeded(response)) { - implementation->onFileChanged(instanceId, payload); - Implementation::syncView(); + if (const auto error = Implementation::errorOf(response)) + return onRefused(*error); + return Log::warn("FileTracking: '{}' was not downloaded, it is not opened", remotePath.value()); } - ) - ) + const auto localPath = Paths::LocalPath::fromUtf8(response["localPath"].as()); + if (!localPath) + return onRefused(localPath.error().toString()); + onOpened(Ids::makeInstanceId(response["instanceId"].as()), *localPath); + } + ), + remotePath.value(), + static_cast(sizeBytes), + watch ); +} - auto const& options = impl_->stateHolder->stateCache().fileTrackingOptions; - const auto [entry, inserted] = impl_->entries.insert_or_assign( - instanceId, - Implementation::TrackedEntry{ - .instanceId = instanceId, - .instanceDirectory = instanceDirectory, - .remotePath = remotePath, - .localPath = localPath, - .autoReupload = options.autoReupload, - .moveRemoteOnLocalMove = options.moveRemoteOnLocalMove, - .deleteRemote = options.deleteRemoteOnLocalDelete, - .checkRemoteChanges = options.checkRemoteChanges.value_or(true), - } +void FileTrackingPanel::checkUploadAndWatch( + Paths::LocalPath const& item, + Paths::RemotePath const& directory, + std::function onWatchable, + std::function onRefused +) +{ + Nui::RpcClient::callWithBackChannel( + impl_->rpcName("checkUploadAndWatch"), + impl_->lifetime.guarded( + [onWatchable = std::move(onWatchable), onRefused = std::move(onRefused)](Nui::val response) + { + if (!Implementation::succeeded(response)) + return onRefused(Implementation::errorOf(response).value_or("")); + const auto canonicalItem = Paths::LocalPath::fromUtf8(response["item"].as()); + if (!canonicalItem) + return onRefused(canonicalItem.error().toString()); + onWatchable(*canonicalItem); + } + ), + item.toUtf8(), + directory.value() ); - if (inserted) - { - impl_->table->addRow(impl_->buildRowForEntry(entry->second)); - impl_->rows.add(instanceId); - } - impl_->entryCount = impl_->entries.size(); - Implementation::syncView(); - - // The downloaded files carry the server's modification time, so they tell what the server had. - impl_->requestFileStates(instanceId, std::nullopt, [] {}); } -void FileTrackingPanel::discard(Ids::InstanceId const& instanceId, Paths::LocalPath const& localPath) +void FileTrackingPanel::uploadAndWatch( + Paths::LocalPath const& item, + Paths::RemotePath const& remotePath, + std::function onRefused +) { - Log::info("FileTracking: discarding {} of instance {}", localPath.toUtf8(), instanceId.value()); - impl_->forgetEntry(instanceId); - Implementation::syncView(); Nui::RpcClient::callWithBackChannel( - "FileTracking::discardContent", + impl_->rpcName("uploadAndWatch"), impl_->lifetime.guarded( - [implementation = impl_.get(), instanceId, localPath](Nui::val response) + [onRefused = std::move(onRefused)](Nui::val response) { - if (!response.hasOwnProperty("success") || !response["success"].as()) - { - Log::error( - "FileTracking: discarding {} of instance {} failed: {}", - localPath.toUtf8(), - instanceId.value(), - response.hasOwnProperty("error") ? response["error"].as() : std::string{} - ); - return; - } - Log::info( - "FileTracking: discarded {}{}", - localPath.toUtf8(), - response["instanceDestroyed"].as() ? ", instance ended" : "" - ); - implementation->refreshOrphanedCount(); + if (!Implementation::succeeded(response)) + onRefused(Implementation::errorOf(response).value_or("")); } ), - instanceId.value(), - localPath.toUtf8() + item.toUtf8(), + remotePath.value() ); } +void FileTrackingPanel::discard(Ids::InstanceId const& instanceId, Paths::LocalPath const& localPath) +{ + impl_->forward("discard", instanceId.value(), localPath.toUtf8()); +} + Nui::ElementRenderer FileTrackingPanel::operator()() { using namespace Nui::Elements; diff --git a/frontend/test/frontend/main.cpp b/frontend/test/frontend/main.cpp index 98dc912c..a7117012 100644 --- a/frontend/test/frontend/main.cpp +++ b/frontend/test/frontend/main.cpp @@ -1,9 +1,9 @@ #include "test_drop_text.hpp" -#include "test_file_tracking_state.hpp" #include "test_item_names.hpp" #include "test_item_order.hpp" #include "test_numbered_saves.hpp" #include "test_overwrite_decisions.hpp" +#include "test_row_order.hpp" #include "test_selection_model.hpp" #include "test_side_paths.hpp" #include "test_transfer_target.hpp" diff --git a/frontend/test/frontend/test_file_tracking_state.hpp b/frontend/test/frontend/test_file_tracking_state.hpp deleted file mode 100644 index 116b048c..00000000 --- a/frontend/test/frontend/test_file_tracking_state.hpp +++ /dev/null @@ -1,242 +0,0 @@ -#pragma once - -#include - -#include - -#include -#include - -namespace FileTracking::Test -{ - namespace FileTrackingStateTestDetail - { - inline Paths::LocalPath local(std::string_view path) - { - return Paths::LocalPath::fromUtf8(path, Paths::LocalPlatform::Posix).value(); - } - - inline Paths::RemotePath remote(std::string_view path) - { - return Paths::RemotePath::make(path).value(); - } - } - - using namespace FileTrackingStateTestDetail; - - TEST(FileTrackingStateTests, UnchangedServerCopyIsNoChange) - { - EXPECT_FALSE(serverChanged(ServerState{.mtime = 5, .size = 10}, ServerState{.mtime = 5, .size = 10})); - } - - TEST(FileTrackingStateTests, OtherTimeOrSizeIsAChange) - { - EXPECT_TRUE(serverChanged(ServerState{.mtime = 5, .size = 10}, ServerState{.mtime = 6, .size = 10})); - EXPECT_TRUE(serverChanged(ServerState{.mtime = 5, .size = 10}, ServerState{.mtime = 5, .size = 11})); - } - - TEST(FileTrackingStateTests, VanishedServerCopyIsAChange) - { - EXPECT_TRUE(serverChanged(ServerState{.mtime = 5, .size = 10}, std::nullopt)); - } - - TEST(FileTrackingStateTests, UnseenServerFileIsAChangeAndAMissingOneIsNot) - { - EXPECT_TRUE(serverChanged(std::nullopt, ServerState{.mtime = 5, .size = 10})); - EXPECT_FALSE(serverChanged(std::nullopt, std::nullopt)); - } - - TEST(FileTrackingStateTests, ChangeMapsBelowTheRemoteDirectory) - { - const auto mapped = remotePathOfChange( - local("/tmp/instance/project/src/main.cpp"), local("/tmp/instance"), remote("/home/user/project") - ); - ASSERT_TRUE(mapped.has_value()); - EXPECT_EQ(mapped->value(), "/home/user/project/src/main.cpp"); - } - - TEST(FileTrackingStateTests, ChangeOfTheInstanceDirectoryMapsToNothing) - { - const auto mapped = - remotePathOfChange(local("/tmp/instance"), local("/tmp/instance"), remote("/home/user/file.txt")); - ASSERT_FALSE(mapped.has_value()); - EXPECT_EQ(mapped.error().reason, Paths::PathRejection::Empty); - } - - TEST(FileTrackingStateTests, ChangeOutsideTheInstanceMapsToNothing) - { - const auto mapped = - remotePathOfChange(local("/tmp/other/file.txt"), local("/tmp/instance"), remote("/home/user/file.txt")); - ASSERT_FALSE(mapped.has_value()); - EXPECT_EQ(mapped.error().reason, Paths::PathRejection::EscapesRoot); - } - - TEST(FileTrackingStateTests, TrackedRootHasNoRemoteDirectory) - { - EXPECT_FALSE(remotePathOfChange(local("/tmp/instance/x"), local("/tmp/instance"), Paths::RemotePath::root()) - .has_value()); - } - - TEST(FileTrackingStateTests, MoveMeanings) - { - EXPECT_EQ(meaningOfMove(false, true), MoveMeaning::Rename); - EXPECT_EQ(meaningOfMove(true, true), MoveMeaning::ReplaceDestination); - EXPECT_EQ(meaningOfMove(true, false), MoveMeaning::ReplaceDestination); - EXPECT_EQ(meaningOfMove(false, false), MoveMeaning::ReplaceDestination); - } - - TEST(FileTrackingStateTests, KnownContentFindsFilesAndWhatLiesBelowDirectories) - { - KnownServerContent known; - known.rememberFile(local("/i/dir/a.txt"), ServerState{.mtime = 1, .size = 2}); - known.rememberDirectory(local("/i/empty")); - EXPECT_EQ(known.fileState(local("/i/dir/a.txt")), (ServerState{.mtime = 1, .size = 2})); - EXPECT_FALSE(known.fileState(local("/i/dir")).has_value()); - EXPECT_TRUE(known.has(local("/i/dir"))); - EXPECT_TRUE(known.hasBelow(local("/i/dir"))); - EXPECT_FALSE(known.hasBelow(local("/i/dir/a.txt"))); - EXPECT_TRUE(known.has(local("/i/empty"))); - EXPECT_FALSE(known.hasBelow(local("/i/empty"))); - EXPECT_FALSE(known.has(local("/i/di"))); - } - - TEST(FileTrackingStateTests, KnownContentForgetsATree) - { - KnownServerContent known; - known.rememberFile(local("/i/dir/a.txt"), ServerState{}); - known.rememberDirectory(local("/i/dir/sub")); - known.rememberFile(local("/i/other.txt"), ServerState{}); - known.forget(local("/i/dir")); - EXPECT_FALSE(known.has(local("/i/dir"))); - EXPECT_TRUE(known.has(local("/i/other.txt"))); - } - - TEST(FileTrackingStateTests, KnownContentMovesATree) - { - KnownServerContent known; - known.rememberFile(local("/i/dir/a.txt"), ServerState{.mtime = 3, .size = 4}); - known.rememberDirectory(local("/i/dir/sub")); - known.rememberFile(local("/i/dirt.txt"), ServerState{}); - known.move(local("/i/dir"), local("/i/renamed")); - EXPECT_FALSE(known.has(local("/i/dir"))); - EXPECT_EQ(known.fileState(local("/i/renamed/a.txt")), (ServerState{.mtime = 3, .size = 4})); - EXPECT_TRUE(known.has(local("/i/renamed/sub"))); - EXPECT_TRUE(known.has(local("/i/dirt.txt"))); - } - - TEST(FileTrackingStateTests, SecondChangeOfAPendingUploadGoesAgainAfterIt) - { - PendingUploads pending; - EXPECT_TRUE(pending.start({.path = local("/i/a.txt")})); - EXPECT_FALSE(pending.start({.path = local("/i/a.txt")})); - EXPECT_TRUE(pending.isPending(local("/i/a.txt"))); - const auto finished = pending.finish(local("/i/a.txt")); - EXPECT_TRUE(finished.changedAgain); - EXPECT_FALSE(finished.removeRemote); - EXPECT_TRUE(pending.empty()); - } - - TEST(FileTrackingStateTests, ChangesBeforeTheUploadReadsTheFileNeedNothingMore) - { - PendingUploads pending; - ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); - ASSERT_FALSE(pending.start({.path = local("/i/a.txt")})); - pending.changesHandled(local("/i/a.txt")); - EXPECT_FALSE(pending.finish(local("/i/a.txt")).changedAgain); - - ASSERT_TRUE(pending.start({.path = local("/i/dir"), .isDirectory = true})); - ASSERT_FALSE(pending.start({.path = local("/i/dir/b.txt")})); - pending.changesHandled(local("/i/dir")); - ASSERT_FALSE(pending.start({.path = local("/i/dir/c.txt")})); - EXPECT_EQ(pending.finish(local("/i/dir")).changedBelow, (std::vector{{.path = local("/i/dir/c.txt")}})); - } - - TEST(FileTrackingStateTests, ChangesHandledKeepsAPendingRemoval) - { - PendingUploads pending; - ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); - ASSERT_TRUE(pending.removeRemoteWhenDone(local("/i/a.txt"))); - pending.changesHandled(local("/i/a.txt")); - EXPECT_TRUE(pending.finish(local("/i/a.txt")).removeRemote); - } - - TEST(FileTrackingStateTests, UploadWithoutFurtherChangeEndsQuietly) - { - PendingUploads pending; - ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); - const auto finished = pending.finish(local("/i/a.txt")); - EXPECT_FALSE(finished.changedAgain); - EXPECT_TRUE(finished.changedBelow.empty()); - } - - TEST(FileTrackingStateTests, ChangesBelowAPendingDirectoryWaitForIt) - { - PendingUploads pending; - ASSERT_TRUE(pending.start({.path = local("/i/new"), .isDirectory = true})); - EXPECT_FALSE(pending.start({.path = local("/i/new/a.txt")})); - EXPECT_FALSE(pending.start({.path = local("/i/new/sub"), .isDirectory = true})); - EXPECT_FALSE(pending.start({.path = local("/i/new/a.txt")})); - EXPECT_FALSE(pending.isPending(local("/i/new/a.txt"))); - EXPECT_TRUE(pending.start({.path = local("/i/newer.txt")})); - - const auto finished = pending.finish(local("/i/new")); - EXPECT_EQ( - finished.changedBelow, - (std::vector{{.path = local("/i/new/a.txt")}, {.path = local("/i/new/sub"), .isDirectory = true}}) - ); - EXPECT_TRUE(pending.start({.path = local("/i/new/a.txt")})); - } - - TEST(FileTrackingStateTests, FilesNextToAPendingFileStartTheirOwn) - { - PendingUploads pending; - ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); - EXPECT_TRUE(pending.start({.path = local("/i/a.txt.bak")})); - } - - TEST(FileTrackingStateTests, PendingUploadMovedAwayRemovesItsRemoteCopyInsteadOfGoingAgain) - { - PendingUploads pending; - ASSERT_TRUE(pending.start({.path = local("/i/a.tmp")})); - ASSERT_FALSE(pending.start({.path = local("/i/a.tmp")})); - EXPECT_TRUE(pending.removeRemoteWhenDone(local("/i/a.tmp"))); - EXPECT_FALSE(pending.removeRemoteWhenDone(local("/i/b.tmp"))); - const auto finished = pending.finish(local("/i/a.tmp")); - EXPECT_TRUE(finished.removeRemote); - EXPECT_FALSE(finished.changedAgain); - } - - TEST(FileTrackingStateTests, ChangeAfterRemovalGoesAgainAndKeepsTheRemoteCopy) - { - PendingUploads pending; - ASSERT_TRUE(pending.start({.path = local("/i/a.txt")})); - ASSERT_TRUE(pending.removeRemoteWhenDone(local("/i/a.txt"))); - ASSERT_FALSE(pending.start({.path = local("/i/a.txt")})); - const auto finished = pending.finish(local("/i/a.txt")); - EXPECT_TRUE(finished.changedAgain); - EXPECT_FALSE(finished.removeRemote); - } - - TEST(FileTrackingStateTests, RowOrderGivesTheIndexOfARowWhenItIsRemoved) - { - RowOrder rows; - rows.add("first"); - rows.add("second"); - rows.add("third"); - - EXPECT_EQ(rows.remove("second"), std::size_t{1}); - EXPECT_EQ(rows.remove("third"), std::size_t{1}); - EXPECT_EQ(rows.remove("first"), std::size_t{0}); - EXPECT_EQ(rows.size(), std::size_t{0}); - } - - TEST(FileTrackingStateTests, RowOrderKnowsNoRowOfAnUnknownOrRemovedKey) - { - RowOrder rows; - rows.add("only"); - - EXPECT_FALSE(rows.remove("other").has_value()); - ASSERT_TRUE(rows.remove("only").has_value()); - EXPECT_FALSE(rows.remove("only").has_value()); - } -} diff --git a/frontend/test/frontend/test_row_order.hpp b/frontend/test/frontend/test_row_order.hpp new file mode 100644 index 00000000..2175aa86 --- /dev/null +++ b/frontend/test/frontend/test_row_order.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include + +#include + +namespace FileTracking::Test +{ + TEST(RowOrderTests, GivesTheIndexOfARowWhenItIsRemoved) + { + RowOrder rows; + rows.add("first"); + rows.add("second"); + rows.add("third"); + + EXPECT_EQ(rows.remove("second"), std::size_t{1}); + EXPECT_EQ(rows.remove("third"), std::size_t{1}); + EXPECT_EQ(rows.remove("first"), std::size_t{0}); + EXPECT_EQ(rows.size(), std::size_t{0}); + } + + TEST(RowOrderTests, KnowsNoRowOfAnUnknownOrRemovedKey) + { + RowOrder rows; + rows.add("only"); + + EXPECT_FALSE(rows.remove("other").has_value()); + ASSERT_TRUE(rows.remove("only").has_value()); + EXPECT_FALSE(rows.remove("only").has_value()); + } +} diff --git a/ids/include/ids/ids.hpp b/ids/include/ids/ids.hpp index 5cc56b48..19875eab 100644 --- a/ids/include/ids/ids.hpp +++ b/ids/include/ids/ids.hpp @@ -9,4 +9,6 @@ DEFINE_ID_TYPE(UiSessionId) DEFINE_ID_TYPE(FileId) DEFINE_ID_TYPE(OperationId) DEFINE_ID_TYPE(InstanceId) -DEFINE_ID_TYPE(SyncSessionId) \ No newline at end of file +DEFINE_ID_TYPE(WatchId) +DEFINE_ID_TYPE(SyncSessionId) +DEFINE_ID_TYPE(QuestionId) \ No newline at end of file diff --git a/licenses/third_party.spdx.json b/licenses/third_party.spdx.json index 94d7b407..0b61b47e 100644 --- a/licenses/third_party.spdx.json +++ b/licenses/third_party.spdx.json @@ -232,8 +232,8 @@ "spdxId": "http://spdx.org/spdxdocs/nui-sftp/Package/efsw", "creationInfo": "_:creationInfo", "name": "efsw", - "software_packageVersion": "87abe599995d5646f5d83cf2e3a225bd73148b3a", - "software_downloadLocation": "https://github.com/SpartanJ/efsw/archive/87abe599995d5646f5d83cf2e3a225bd73148b3a.tar.gz", + "software_packageVersion": "d71ed846811cc67ee8827614341e2777d200231a", + "software_downloadLocation": "https://github.com/5cript/efsw/archive/d71ed846811cc67ee8827614341e2777d200231a.tar.gz", "software_homePage": "https://github.com/SpartanJ/efsw", "software_copyrightText": "Copyright (c) 2020 Martin Lucas Golini", "simplelicensing_declaredLicense": "MIT", diff --git a/nui-file-explorer/include/nui-file-explorer/side_model_interface.hpp b/nui-file-explorer/include/nui-file-explorer/side_model_interface.hpp index 08fbda69..a2a31ff1 100644 --- a/nui-file-explorer/include/nui-file-explorer/side_model_interface.hpp +++ b/nui-file-explorer/include/nui-file-explorer/side_model_interface.hpp @@ -173,9 +173,11 @@ namespace NuiFileExplorer * expose exactly the actions that make sense for its context. * * @param selectedItems The items currently selected (may be empty). + * @param otherSideSelectedItems The items selected on the other side (may be empty). * @return std::vector */ - virtual std::vector contextMenuItems(std::vector const& selectedItems) = 0; + virtual std::vector + contextMenuItems(std::vector const& selectedItems, std::vector const& otherSideSelectedItems) = 0; /** * @brief Optional interface: provides default system places (XDG / Windows known folders). diff --git a/nui-file-explorer/source/nui-file-explorer/side.cpp b/nui-file-explorer/source/nui-file-explorer/side.cpp index fead3b77..dc9ab55c 100644 --- a/nui-file-explorer/source/nui-file-explorer/side.cpp +++ b/nui-file-explorer/source/nui-file-explorer/side.cpp @@ -646,7 +646,8 @@ namespace NuiFileExplorer if (impl_->otherSide) impl_->otherSide->closeMenus(); - auto menuItems = impl_->model->contextMenuItems(clickItems); + const auto otherSelected = impl_->otherSide ? impl_->otherSide->selectedItems() : std::vector{}; + auto menuItems = impl_->model->contextMenuItems(clickItems, otherSelected); if (impl_->onSynchronize && impl_->otherSide) { @@ -658,7 +659,6 @@ namespace NuiFileExplorer return items.size() == 1 && items[0].isDirectoryLike() && items[0].nameIsValidUtf8; }; const bool thisSideOk = isSyncRoot(clickItems); - const auto otherSelected = impl_->otherSide->selectedItems(); const bool otherSideOk = isSyncRoot(otherSelected); menuItems.push_back(Snc::PopupMenu::separator()); diff --git a/shared_data/include/shared_data/file_operations/compared_state.hpp b/shared_data/include/shared_data/file_operations/compared_state.hpp index 1cc113c4..47c85b45 100644 --- a/shared_data/include/shared_data/file_operations/compared_state.hpp +++ b/shared_data/include/shared_data/file_operations/compared_state.hpp @@ -19,6 +19,11 @@ namespace SharedData std::uint64_t size{0}; std::uint64_t mtime{0}; std::optional linkTarget{}; + /** + * @brief Something seen present must still be there, for a file tracker that asks before recreating what the + * server lost. A sync leaves it off. + */ + bool mustStillExist{false}; bool operator==(ComparedState const&) const = default; }; @@ -30,9 +35,9 @@ namespace SharedData /** * @brief True when @p current is still what the compare saw in @p compared. A path seen absent must still be - * absent. One seen present may be gone now, nothing there can be lost. Otherwise the type must be the - * same, a symlink its literal, a regular file or a special one its size and modification time; a - * directory changes by its type only. + * absent. One seen present may be gone now, nothing there can be lost, unless @p compared says it must + * still exist. Otherwise the type must be the same, a symlink its literal, a regular file or a special one + * its size and modification time; a directory changes by its type only. */ bool unchangedSinceCompare(ComparedState const& compared, ComparedState const& current); } diff --git a/shared_data/include/shared_data/file_tracking/ended_watches.hpp b/shared_data/include/shared_data/file_tracking/ended_watches.hpp new file mode 100644 index 00000000..f1422cfe --- /dev/null +++ b/shared_data/include/shared_data/file_tracking/ended_watches.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + +#include + +namespace SharedData +{ + /** + * @brief Why watches ended that the user did not stop. CopyGone: the downloaded copy was deleted or renamed away + * and did not come back. ConnectionLost: the session lost its connection, either side may change until it + * is back. + */ + BOOST_DEFINE_ENUM_CLASS(WatchEndReason, CopyGone, ConnectionLost) + + /** + * @brief Watches of the session's tracker that ended for the same reason. + */ + struct EndedWatches + { + WatchEndReason reason; + /** @brief What the entries of the watches showed last. */ + std::vector entries; + }; + BOOST_DESCRIBE_STRUCT(EndedWatches, (), (reason, entries)) +} diff --git a/shared_data/include/shared_data/file_tracking/server_changed_question.hpp b/shared_data/include/shared_data/file_tracking/server_changed_question.hpp new file mode 100644 index 00000000..ad67f904 --- /dev/null +++ b/shared_data/include/shared_data/file_tracking/server_changed_question.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include + +namespace SharedData +{ + /** + * @brief The answer to a ServerChangedQuestion. Replace uploads the local version, Keep leaves the server's and + * asks again on the next save. Dismissed: another dialog took the place of the question, it and the + * waiting ones are asked again on the next save. + */ + BOOST_DEFINE_ENUM_CLASS(ServerChangedAnswer, Replace, Keep, Dismissed) + + /** + * @brief The tracker asks whether a local change may replace the server's copy at remotePath, which changed since + * it was opened or last uploaded. + */ + struct ServerChangedQuestion + { + Ids::QuestionId questionId; + Paths::RemotePath remotePath; + }; + BOOST_DESCRIBE_STRUCT(ServerChangedQuestion, (), (questionId, remotePath)) +} diff --git a/shared_data/include/shared_data/file_tracking/tracked_entry.hpp b/shared_data/include/shared_data/file_tracking/tracked_entry.hpp new file mode 100644 index 00000000..6ac19584 --- /dev/null +++ b/shared_data/include/shared_data/file_tracking/tracked_entry.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace SharedData +{ + /** + * @brief What the file tracking panel shows of one watch of the session's tracker. + */ + struct TrackedEntry + { + Ids::WatchId watchId; + /** @brief The name of the watched local file or directory. */ + std::string name; + /** @brief Where the watched item goes on the server. */ + Paths::RemotePath remotePath; + WatchOptions options; + /** @brief Uploads of the item are on their way. */ + bool uploading{false}; + /** + * @brief The item is a temporary copy of the server's file (Open and Watch): its moves and its removal never + * reach the server, the options for those stay off. + */ + bool downloadedCopy{false}; + }; + BOOST_DESCRIBE_STRUCT(TrackedEntry, (), (watchId, name, remotePath, options, uploading, downloadedCopy)) +} diff --git a/shared_data/include/shared_data/file_tracking/tracker_progress.hpp b/shared_data/include/shared_data/file_tracking/tracker_progress.hpp new file mode 100644 index 00000000..d7b8f2c8 --- /dev/null +++ b/shared_data/include/shared_data/file_tracking/tracker_progress.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +#include + +namespace SharedData +{ + /** + * @brief How far the tracker of a session is with mirroring the changes of its watches, apart from the operation + * queue: the panel shows it while something is on its way. + */ + struct TrackerProgress + { + /** @brief Changes on their way to the server: waiting for the check, an answer or the queue, or running. */ + std::uint64_t pending{0}; + /** @brief Changes that ended since nothing was on its way any more. */ + std::uint64_t done{0}; + /** @brief Changes among the pending ones that wait for the user's answer to whether the server's copy goes. */ + std::uint64_t waitingForAnswer{0}; + + bool operator==(TrackerProgress const&) const = default; + }; + BOOST_DESCRIBE_STRUCT(TrackerProgress, (), (pending, done, waitingForAnswer)) +} diff --git a/shared_data/include/shared_data/file_tracking/watch_options.hpp b/shared_data/include/shared_data/file_tracking/watch_options.hpp new file mode 100644 index 00000000..5e9ce42f --- /dev/null +++ b/shared_data/include/shared_data/file_tracking/watch_options.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +namespace SharedData +{ + BOOST_DEFINE_ENUM_CLASS( + WatchOption, + AutoReupload, + MoveRemoteOnLocalMove, + DeleteRemoteOnLocalDelete, + CheckRemoteChanges + ) + + /** + * @brief What one watch mirrors to the server, it starts with Persistence::FileTrackingOptions. + */ + struct WatchOptions + { + bool autoReupload{true}; + bool moveRemoteOnLocalMove{true}; + bool deleteRemoteOnLocalDelete{true}; + /** @brief Ask before replacing a server copy that changed since it was opened or last uploaded. */ + bool checkRemoteChanges{true}; + + bool& operator[](WatchOption option) + { + switch (option) + { + case WatchOption::AutoReupload: + return autoReupload; + case WatchOption::MoveRemoteOnLocalMove: + return moveRemoteOnLocalMove; + case WatchOption::DeleteRemoteOnLocalDelete: + return deleteRemoteOnLocalDelete; + case WatchOption::CheckRemoteChanges: + break; + } + return checkRemoteChanges; + } + + friend bool operator==(WatchOptions const&, WatchOptions const&) = default; + }; + BOOST_DESCRIBE_STRUCT( + WatchOptions, + (), + (autoReupload, moveRemoteOnLocalMove, deleteRemoteOnLocalDelete, checkRemoteChanges) + ) +} diff --git a/shared_data/source/shared_data/file_operations/compared_state.cpp b/shared_data/source/shared_data/file_operations/compared_state.cpp index 6ccce2c5..cf534d0c 100644 --- a/shared_data/source/shared_data/file_operations/compared_state.cpp +++ b/shared_data/source/shared_data/file_operations/compared_state.cpp @@ -18,7 +18,7 @@ namespace SharedData if (!compared.exists) return !current.exists; if (!current.exists) - return true; + return !compared.mustStillExist; if (compared.type != current.type) return false; switch (compared.type) diff --git a/shared_data/test/shared_data/test_compared_state.hpp b/shared_data/test/shared_data/test_compared_state.hpp index 82dded67..cb1e3d3a 100644 --- a/shared_data/test/shared_data/test_compared_state.hpp +++ b/shared_data/test/shared_data/test_compared_state.hpp @@ -46,6 +46,16 @@ namespace SharedData::Test EXPECT_TRUE(unchangedSinceCompare(file(3, 100), ComparedState{})); } + TEST(ComparedStateTests, SomethingThatMustStillExistChangedWhenItIsGone) + { + using namespace ComparedStateTestDetail; + auto compared = file(3, 100); + compared.mustStillExist = true; + EXPECT_FALSE(unchangedSinceCompare(compared, ComparedState{})); + EXPECT_TRUE(unchangedSinceCompare(compared, file(3, 100))); + EXPECT_FALSE(unchangedSinceCompare(compared, file(4, 100))); + } + TEST(ComparedStateTests, AnotherTypeChanged) { using namespace ComparedStateTestDetail; diff --git a/static/assets/languages/de_DE.yaml b/static/assets/languages/de_DE.yaml index b62342cf..c670cb3c 100644 --- a/static/assets/languages/de_DE.yaml +++ b/static/assets/languages/de_DE.yaml @@ -35,7 +35,9 @@ fileExplorer: openWith: "Öffnen mit" openInFileManager: "Im Dateimanager öffnen" openAndWatch: "Öffnen und Beobachten" + openAndWatchHint: "Lädt eine temporäre Kopie herunter und öffnet sie. Jede Änderung, die Sie an der Kopie speichern, wird wieder auf den Server hochgeladen, bis Sie die Beobachtung unter Dateiüberwachung beenden." openWithAndWatch: "Öffnen mit und Beobachten" + openWithAndWatchHint: "Lädt eine temporäre Kopie herunter und fragt, mit welcher Anwendung sie geöffnet wird. Jede Änderung, die Sie an der Kopie speichern, wird wieder auf den Server hochgeladen, bis Sie die Beobachtung unter Dateiüberwachung beenden." delete: "Löschen" rename: "Umbenennen" properties: "Eigenschaften" @@ -43,6 +45,9 @@ fileExplorer: downloadAsArchive: "Als Archiv herunterladen" removeFromFavorites: "Aus Favoriten entfernen" uploadAsArchive: "Als Archiv hochladen" + uploadAndWatch: "Hochladen und Beobachten" + uploadAndWatchHint: "Lädt es nach {} hoch und beobachtet es hier weiter. Jede Änderung, die Sie speichern, wird erneut hochgeladen, bis Sie die Beobachtung unter Dateiüberwachung beenden. Wählen Sie auf der Serverseite ein Verzeichnis aus, um stattdessen dorthin hochzuladen." + uploadAndWatchIntoSelectionHint: "Lädt es in das auf der Serverseite ausgewählte Verzeichnis {} hoch und beobachtet es hier weiter. Jede Änderung, die Sie speichern, wird erneut hochgeladen, bis Sie die Beobachtung unter Dateiüberwachung beenden." synchronize: "Synchronisieren..." synchronizeHint: "Wählen Sie auf jeder Seite genau ein Verzeichnis zum Synchronisieren aus." archiveDefaultName: "archiv" @@ -95,6 +100,7 @@ sideModel: failedToListDirectoryText: "Beim Auflisten des Inhalts des Verzeichnisses '{}' ist ein Fehler aufgetreten." failedToListDirectoryReasonText: "Beim Auflisten des Inhalts des Verzeichnisses '{}' ist ein Fehler aufgetreten: {}" remoteSideModel: + openAndWatchFailedTitle: "Öffnen und Beobachten nicht möglich" newDirectory: "Neues Verzeichnis" enterNewDirectoryName: "Geben Sie den Namen des neuen Verzeichnisses ein" createNewDirectory: "Neues Verzeichnis erstellen" @@ -162,6 +168,7 @@ localSideModel: deleteDirectoriesWithContent: "Dabei wird auch der gesamte Inhalt der ausgewählten Verzeichnisse gelöscht. Das lässt sich nicht rückgängig machen." remoteSideUnavailable: "Die Remote-Seite ist nicht verfügbar" transferFailedTitle: "Dateiübertragung fehlgeschlagen" + uploadAndWatchFailedTitle: "Hochladen und Beobachten nicht möglich" operationQueue: pause: "Pausieren" continue: "Fortsetzen" @@ -290,9 +297,6 @@ sessionFrontend: getHomeDirectoryFailed: "Home-Verzeichnis konnte nicht ermittelt werden" remoteSideModelUnavailable: "Das Modell der Remote-Seite ist nicht verfügbar, SFTP kann nicht geöffnet werden. Das ist ein Fehler und sollte nicht passieren." sftpInitializationFailed: "SFTP-Initialisierung fehlgeschlagen" -fileTrackingPanel: - instanceCreationFailed: "Datei-Tracking Instanzerstellung fehlgeschlagen" - instanceCreationFailedDesc: "Eine Datei-Tracking Instanz konnte im Backend nicht erstellt werden. Details finden Sie in den Logs." commandPanels: disconnectedTooltip: "Nicht verfügbar, solange die Verbindung getrennt ist" justNow: "gerade eben" @@ -505,6 +509,7 @@ fileTracking: columnMoveRemote: "Remote verschieben" columnDeleteRemote: "Remote löschen" columnCheckRemote: "Remote prüfen" + downloadedCopyOptionHint: "Eine temporäre Kopie verschiebt oder löscht die Datei auf dem Server nie." serverChangedHeader: "Datei auf dem Server geändert" serverChangedText: "Die Datei '{}' wurde auf dem Server geändert, nachdem Sie sie geöffnet oder zuletzt hochgeladen haben. Die Version auf dem Server durch die lokale ersetzen?\n\n'Nein' behält die Version auf dem Server. Beim nächsten Speichern werden Sie erneut gefragt." columnStatus: "Status" @@ -515,6 +520,10 @@ fileTracking: headerOrphaned: "Verwaist" headerCleanupButton: "Verwaiste bereinigen" headerCleanupRunning: "Bereinige…" + progressSyncing: "Synchronisiere {} von {}" + progressWaitingForAnswer: "{} warten auf Ihre Antwort" + watchEndedCopyGone: "Die Beobachtung von {} wurde beendet, die temporäre Kopie wurde gelöscht oder umbenannt. Die Datei auf dem Server ist unverändert, mit 'Öffnen und Beobachten' können Sie sie erneut bearbeiten." + watchesEndedConnectionLost: "Die Verbindung wurde unterbrochen, die Beobachtung von {} wurde beendet. Beobachten Sie sie erneut, sobald die Sitzung wieder verbunden ist." settings: version: "Version" settingsRestartRequiredHeader: "Neustart erforderlich" @@ -1202,6 +1211,15 @@ terminalPanel: terminalNoFreeName: "Terminal {}: Alle Namen bis '{}' existieren bereits." terminalWriteFailed: "Terminal {}: Schreiben von '{}' fehlgeschlagen." backend: + fileTracking: + watchUnreadable: "'{}' kann nicht beobachtet werden: {}" + watchRoot: "'{}' kann nicht beobachtet werden, es ist die Wurzel eines Dateisystems." + watchFailed: "'{}' kann nicht beobachtet werden, das Dateisystem lässt es nicht zu." + watchOverlaps: "'{}' kann nicht beobachtet werden, es überschneidet sich mit '{}', das bereits beobachtet wird. Beobachtungen können nicht verschachtelt werden." + watchOverlapsDownloads: "'{}' kann nicht beobachtet werden, es überschneidet sich mit dem Verzeichnis der temporären Downloads '{}', das von Öffnen und Beobachten beobachtet wird." + remoteOverlaps: "'{}' kann nicht beobachtet werden, es überschneidet sich mit '{}' auf dem Server, wohin bereits eine andere Beobachtung hochlädt. Beobachtungen können nicht verschachtelt werden." + instanceCreationFailed: "Für den Download konnte kein Verzeichnis angelegt werden. Details finden Sie in den Logs." + watchCopyFailed: "Der Download '{}' kann nicht beobachtet werden, das Dateisystem lässt es nicht zu." deleteGuard: noPath: "Es wurde nichts gelöscht. Ein Pfad fehlt." root: "Es wurde nichts gelöscht. '{}' ist das Wurzelverzeichnis." diff --git a/static/assets/languages/en_US.yaml b/static/assets/languages/en_US.yaml index 6fc87cf4..8e895307 100644 --- a/static/assets/languages/en_US.yaml +++ b/static/assets/languages/en_US.yaml @@ -35,7 +35,9 @@ fileExplorer: openWith: "Open With" openInFileManager: "Open in File Manager" openAndWatch: "Open and Watch" + openAndWatchHint: "Downloads a temporary copy and opens it. Every change you save to the copy is uploaded back to the server, until you stop the watch in File Tracking." openWithAndWatch: "Open With and Watch" + openWithAndWatchHint: "Downloads a temporary copy and asks which application opens it. Every change you save to the copy is uploaded back to the server, until you stop the watch in File Tracking." delete: "Delete" rename: "Rename" properties: "Properties" @@ -43,6 +45,9 @@ fileExplorer: downloadAsArchive: "Download as Archive" removeFromFavorites: "Remove from Favorites" uploadAsArchive: "Upload as Archive" + uploadAndWatch: "Upload and Watch" + uploadAndWatchHint: "Uploads it to {} and keeps watching it here. Every change you save is uploaded again, until you stop the watch in File Tracking. Select one directory on the remote side to upload into it instead." + uploadAndWatchIntoSelectionHint: "Uploads it into the directory selected on the remote side, {}, and keeps watching it here. Every change you save is uploaded again, until you stop the watch in File Tracking." synchronize: "Synchronize..." synchronizeHint: "Select exactly one directory on each side to synchronize." archiveDefaultName: "archive" @@ -193,6 +198,7 @@ sideModel: failedToListDirectoryText: "An error occurred while trying to list the contents of the directory '{}'." failedToListDirectoryReasonText: "An error occurred while trying to list the contents of the directory '{}': {}" remoteSideModel: + openAndWatchFailedTitle: "Cannot Open and Watch" newDirectory: "New Directory" enterNewDirectoryName: "Enter the name of the new directory" createNewDirectory: "Create New Directory" @@ -260,6 +266,7 @@ localSideModel: deleteDirectoriesWithContent: "This also deletes everything inside the selected directories. It cannot be undone." remoteSideUnavailable: "Remote side is not available" transferFailedTitle: "File Transfer Failed" + uploadAndWatchFailedTitle: "Cannot Upload and Watch" sessionFrontend: sessionCreationFailedHeader: "Session Creation Failed" sessionCreationFailedText: "Failed to create session instance:\n{}" @@ -290,9 +297,6 @@ sessionFrontend: getHomeDirectoryFailed: "Get Home Directory Failed" remoteSideModelUnavailable: "Remote side model is not available, cannot open SFTP. This is a bug and should not happen by design." sftpInitializationFailed: "SFTP Initialization Failed" -fileTrackingPanel: - instanceCreationFailed: "File Tracking Instance Creation Failed" - instanceCreationFailedDesc: "Failed to create a file tracking instance on the backend. Check the logs for details." commandPanels: disconnectedTooltip: "Not available while the connection is lost" justNow: "just now" @@ -505,6 +509,7 @@ fileTracking: columnMoveRemote: "Move Remote" columnDeleteRemote: "Delete Remote" columnCheckRemote: "Check Remote" + downloadedCopyOptionHint: "A temporary copy never moves or deletes the file on the server." serverChangedHeader: "File Changed on the Server" serverChangedText: "'{}' changed on the server after you opened or last uploaded it. Replace the server's version with your local one?\n\n'No' keeps the server's version. You will be asked again on the next save." columnStatus: "Status" @@ -515,6 +520,10 @@ fileTracking: headerOrphaned: "Orphaned" headerCleanupButton: "Clean Up Orphaned" headerCleanupRunning: "Cleaning…" + progressSyncing: "Syncing {} of {}" + progressWaitingForAnswer: "{} waiting for your answer" + watchEndedCopyGone: "The watch of {} ended, its temporary copy was deleted or renamed. The file on the server is unchanged, use Open and Watch to edit it again." + watchesEndedConnectionLost: "The connection was lost, the watches of {} ended. Watch them again once the session is connected." settings: version: "Version" settingsRestartRequiredHeader: "Restart Required" @@ -1204,6 +1213,15 @@ terminalPanel: terminalNoFreeName: "Terminal {}: every name up to '{}' already exists." terminalWriteFailed: "Terminal {}: writing '{}' failed." backend: + fileTracking: + watchUnreadable: "'{}' cannot be watched: {}" + watchRoot: "'{}' cannot be watched, it is the root of a file system." + watchFailed: "'{}' cannot be watched, the file system refused to watch it." + watchOverlaps: "'{}' cannot be watched, it overlaps '{}', which is watched already. Watches cannot be nested." + watchOverlapsDownloads: "'{}' cannot be watched, it overlaps the directory of temporary downloads '{}', which Open and Watch watches." + remoteOverlaps: "'{}' cannot be watched, it overlaps '{}' on the server, which another watch uploads to already. Watches cannot be nested." + instanceCreationFailed: "No directory for the download could be created. Check the logs for details." + watchCopyFailed: "The download '{}' cannot be watched, the file system refused to watch it." deleteGuard: noPath: "Nothing was deleted. A path is missing." root: "Nothing was deleted. '{}' is the root directory." diff --git a/static/styles/file_tracking_panel.css b/static/styles/file_tracking_panel.css index 34647777..61d0d57e 100644 --- a/static/styles/file_tracking_panel.css +++ b/static/styles/file_tracking_panel.css @@ -58,6 +58,63 @@ header.file-tracking-header { color: var(--warning-color, #f5a623); } +.file-tracking-progress { + display: inline-flex; + align-items: center; + gap: 8px; + min-width: 0; + font-size: 0.82em; +} + +.file-tracking-progress-idle { + display: none; +} + +.file-tracking-progress-track { + position: relative; + width: 140px; + height: 6px; + flex-shrink: 0; + border-radius: 3px; + overflow: hidden; + background: color-mix(in srgb, var(--theme-color, #4caf50) 20%, transparent); +} + +.file-tracking-progress-fill { + height: 100%; + background: var(--theme-color, #4caf50); + transition: width 0.2s ease-out; +} + +.file-tracking-progress-track::after { + content: ""; + position: absolute; + inset: 0; + width: 30%; + background: linear-gradient(to right, transparent, rgba(255, 255, 255, 0.35), transparent); + animation: file-tracking-progress-sweep 1.2s linear infinite; +} + +@keyframes file-tracking-progress-sweep { + from { + transform: translateX(-100%); + } + + to { + transform: translateX(340%); + } +} + +.file-tracking-progress-text { + white-space: nowrap; + font-variant-numeric: tabular-nums; +} + +.file-tracking-progress-waiting { + white-space: nowrap; + color: var(--warning-color, #f5a623); +} + .file-tracking-header-actions { display: flex; gap: 6px; diff --git a/utility/include/utility/checked_symlink_status.hpp b/utility/include/utility/checked_symlink_status.hpp new file mode 100644 index 00000000..0a42340f --- /dev/null +++ b/utility/include/utility/checked_symlink_status.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace Utility +{ + namespace Detail + { + /** + * @brief Reads the status of a path itself, like std::filesystem::symlink_status. + */ + using SymlinkStatusReader = + std::function; + + /** + * @brief checkedSymlinkStatus() with each look at @p path taken by @p readStatus. + */ + std::expected, std::error_code> + checkedSymlinkStatus(std::filesystem::path const& path, SymlinkStatusReader const& readStatus); + } + + /** + * @brief The status of @p path itself, a symbolic link not followed, where a failed check never reads as "gone". + * + * Only "no such file or directory" (ENOENT, ENOTDIR, ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND) means the path + * does not exist. On Windows std::filesystem also reports a path past MAX_PATH and an unreachable network path as + * not found, so that answer is checked again through the extended-length form (\\?\) of the path. A shorter path + * found that way gets one more look, it was written anew meanwhile (git deletes and writes files). A path that is + * being deleted (STATUS_DELETE_PENDING, until the last handle to it closes) or lies below one counts as gone. + * + * @return The status, nothing when the path does not exist, an error when that cannot be told: the path exists + * but cannot be reached as given (ERROR_FILENAME_EXCED_RANGE past MAX_PATH, ERROR_INVALID_NAME for a name + * only the extended-length form reaches), or the check failed. + */ + std::expected, std::error_code> + checkedSymlinkStatus(std::filesystem::path const& path); +} diff --git a/utility/include/utility/shared_input_file_stream.hpp b/utility/include/utility/shared_input_file_stream.hpp new file mode 100644 index 00000000..60a3239e --- /dev/null +++ b/utility/include/utility/shared_input_file_stream.hpp @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace Utility +{ + /** + * @brief Reads a file that others may rename, replace or delete while it is open: Windows refuses that for a file + * a std::filebuf holds, which opens it without FILE_SHARE_DELETE. + */ + class SharedInputFileBuffer : public std::streambuf + { + public: + SharedInputFileBuffer() = default; + ~SharedInputFileBuffer() override = default; + SharedInputFileBuffer(SharedInputFileBuffer const&) = delete; + SharedInputFileBuffer& operator=(SharedInputFileBuffer const&) = delete; + SharedInputFileBuffer(SharedInputFileBuffer&&) = delete; + SharedInputFileBuffer& operator=(SharedInputFileBuffer&&) = delete; + + /** + * @brief Opens @p path for reading in binary mode, closing the file open before. + * @return Whether the file is open. + */ + bool open(std::filesystem::path const& path); + bool isOpen() const; + void close(); + + protected: + int_type underflow() override; + std::streamsize xsgetn(char_type* destination, std::streamsize count) override; + pos_type seekoff(off_type offset, std::ios_base::seekdir direction, std::ios_base::openmode which) override; + pos_type seekpos(pos_type position, std::ios_base::openmode which) override; + + private: + struct FileCloser + { + void operator()(std::FILE* file) const; + }; + + private: + std::unique_ptr file_{}; + std::array buffer_{}; + }; + + /** + * @brief A std::ifstream in binary mode whose file others may rename, replace or delete while it is open, see + * SharedInputFileBuffer. Named like std::ifstream, so it can take its place. + */ + class SharedInputFileStream : public std::istream + { + public: + SharedInputFileStream(); + ~SharedInputFileStream() override = default; + SharedInputFileStream(SharedInputFileStream const&) = delete; + SharedInputFileStream& operator=(SharedInputFileStream const&) = delete; + SharedInputFileStream(SharedInputFileStream&&) = delete; + SharedInputFileStream& operator=(SharedInputFileStream&&) = delete; + + /** + * @brief Opens @p path, the failbit is set when that fails. + */ + void open(std::filesystem::path const& path); + bool is_open() const; + void close(); + + private: + SharedInputFileBuffer buffer_; + }; +} diff --git a/utility/source/utility/CMakeLists.txt b/utility/source/utility/CMakeLists.txt index 8544627e..69bf72e2 100644 --- a/utility/source/utility/CMakeLists.txt +++ b/utility/source/utility/CMakeLists.txt @@ -21,9 +21,11 @@ else() utility PRIVATE node/node.cpp + checked_symlink_status.cpp error_message.cpp permissions.cpp rename.cpp + shared_input_file_stream.cpp link_safe_remove.cpp no_follow_tree_walk.cpp download_mark.cpp diff --git a/utility/source/utility/checked_symlink_status.cpp b/utility/source/utility/checked_symlink_status.cpp new file mode 100644 index 00000000..2a506cd4 --- /dev/null +++ b/utility/source/utility/checked_symlink_status.cpp @@ -0,0 +1,141 @@ +#include + +#include + +#ifdef _WIN32 +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +#endif + +namespace Utility +{ + namespace + { +#ifdef _WIN32 + std::error_code systemError(DWORD code) + { + return std::error_code{static_cast(code), std::system_category()}; + } + + /** + * @brief @p path as an extended-length path (\\?\C:\x or \\?\UNC\server\share\x), which MAX_PATH does not + * limit. Nothing for a relative path and for one in the device namespace already. + */ + std::optional extendedLengthPathOf(std::filesystem::path const& path) + { + // The extended-length form skips the normalization of separators, "." and "..". + const std::wstring native = path.lexically_normal().make_preferred().native(); + if (native.starts_with(LR"(\\?\)") || native.starts_with(LR"(\\.\)")) + return std::nullopt; + if (native.starts_with(LR"(\\)")) + return LR"(\\?\UNC\)" + native.substr(2); + if (native.size() >= 3 && native[1] == L':' && native[2] == L'\\') + return LR"(\\?\)" + native; + return std::nullopt; + } + + using NtStatusReader = LONG(WINAPI*)(); + + /** + * @brief ntdll's RtlGetLastNtStatus, nothing when it is missing. Looked up before the call whose status it + * reads. + */ + NtStatusReader ntStatusReader() + { + static const auto reader = reinterpret_cast( + reinterpret_cast(::GetProcAddress(::GetModuleHandleW(L"ntdll.dll"), "RtlGetLastNtStatus")) + ); + return reader; + } + + /** + * @brief What Windows itself says about @p path, which @p readStatus did not find (@p failure unset) or could + * not check (@p failure). It tells apart what std::filesystem reports alike: a path past MAX_PATH from a + * missing one, and a file being deleted from one that denies access. A shorter path Windows finds was + * written anew meanwhile (git deletes and writes files) and gets one more look when @p lookAgain. + */ + std::expected, std::error_code> askWindows( + std::filesystem::path const& path, + std::error_code failure, + Detail::SymlinkStatusReader const& readStatus, + bool lookAgain + ); + + std::expected, std::error_code> + statusOf(std::filesystem::path const& path, Detail::SymlinkStatusReader const& readStatus, bool lookAgain) + { + std::error_code statusError{}; + const auto status = readStatus(path, statusError); + const bool notFound = status.type() == std::filesystem::file_type::not_found; + if (!notFound && !statusError) + return status; + return askWindows(path, notFound ? std::error_code{} : statusError, readStatus, lookAgain); + } + + std::expected, std::error_code> askWindows( + std::filesystem::path const& path, + std::error_code failure, + Detail::SymlinkStatusReader const& readStatus, + bool lookAgain + ) + { + const auto readNtStatus = ntStatusReader(); + const auto extended = extendedLengthPathOf(path); + const auto attributes = ::GetFileAttributesW(extended ? extended->c_str() : path.c_str()); + if (attributes != INVALID_FILE_ATTRIBUTES) + { + if (failure) + return std::unexpected(failure); + if (path.native().size() >= MAX_PATH) + return std::unexpected(systemError(ERROR_FILENAME_EXCED_RANGE)); + if (lookAgain) + return statusOf(path, readStatus, /*lookAgain=*/false); + return std::unexpected(systemError(ERROR_INVALID_NAME)); + } + const auto lastError = ::GetLastError(); + if (lastError == ERROR_FILE_NOT_FOUND || lastError == ERROR_PATH_NOT_FOUND) + return std::nullopt; + // Deleted without POSIX semantics (libc++'s remove), it and what is below it stay in the way as + // STATUS_DELETE_PENDING, which reads as access denied, until the last handle to it closes. + constexpr auto deletePending = static_cast(0xC0000056u); + if (lastError == ERROR_ACCESS_DENIED && readNtStatus && readNtStatus() == deletePending) + return std::nullopt; + return std::unexpected(systemError(lastError)); + } +#endif + } + + namespace Detail + { + std::expected, std::error_code> + checkedSymlinkStatus(std::filesystem::path const& path, SymlinkStatusReader const& readStatus) + { +#ifdef _WIN32 + return statusOf(path, readStatus, /*lookAgain=*/true); +#else + std::error_code statusError{}; + const auto status = readStatus(path, statusError); + const bool notFound = status.type() == std::filesystem::file_type::not_found; + if (!notFound && !statusError) + return status; + if (notFound) + return std::nullopt; + return std::unexpected(statusError); +#endif + } + } + + std::expected, std::error_code> + checkedSymlinkStatus(std::filesystem::path const& path) + { + return Detail::checkedSymlinkStatus( + path, + [](std::filesystem::path const& looked, std::error_code& error) + { + return std::filesystem::symlink_status(looked, error); + } + ); + } +} diff --git a/utility/source/utility/shared_input_file_stream.cpp b/utility/source/utility/shared_input_file_stream.cpp new file mode 100644 index 00000000..ff134561 --- /dev/null +++ b/utility/source/utility/shared_input_file_stream.cpp @@ -0,0 +1,225 @@ +#include + +#ifdef _WIN32 +# include +# include +# include +#else +# include +#endif + +#include +#include +#include +#include +#include +#include + +namespace Utility +{ + namespace + { +#ifdef _WIN32 + struct HandleCloser + { + void operator()(HANDLE handle) const + { + ::CloseHandle(handle); + } + }; + + /** + * @brief A C runtime file descriptor that is closed unless it was released. + */ + class OwnedDescriptor + { + public: + explicit OwnedDescriptor(int descriptor) + : descriptor_{descriptor} + {} + ~OwnedDescriptor() + { + if (descriptor_ != -1) + ::_close(descriptor_); + } + OwnedDescriptor(OwnedDescriptor const&) = delete; + OwnedDescriptor& operator=(OwnedDescriptor const&) = delete; + OwnedDescriptor(OwnedDescriptor&&) = delete; + OwnedDescriptor& operator=(OwnedDescriptor&&) = delete; + + int get() const + { + return descriptor_; + } + + int release() + { + return std::exchange(descriptor_, -1); + } + + private: + int descriptor_; + }; + + std::FILE* openShared(std::filesystem::path const& path) + { + auto handle = std::unique_ptr, HandleCloser>{::CreateFileW( + path.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + nullptr + )}; + if (handle.get() == INVALID_HANDLE_VALUE) + { + std::ignore = handle.release(); + return nullptr; + } + OwnedDescriptor descriptor{ + ::_open_osfhandle(reinterpret_cast(handle.get()), _O_RDONLY | _O_BINARY) + }; + if (descriptor.get() == -1) + return nullptr; + // The descriptor owns the handle now. + std::ignore = handle.release(); + auto* file = ::_fdopen(descriptor.get(), "rb"); + if (file != nullptr) + std::ignore = descriptor.release(); + return file; + } + + int seekFile(std::FILE* file, std::int64_t offset, int origin) + { + return ::_fseeki64(file, offset, origin); + } + + std::int64_t tellFile(std::FILE* file) + { + return ::_ftelli64(file); + } +#else + std::FILE* openShared(std::filesystem::path const& path) + { + return std::fopen(path.c_str(), "rb"); + } + + int seekFile(std::FILE* file, std::int64_t offset, int origin) + { + return ::fseeko(file, static_cast(offset), origin); + } + + std::int64_t tellFile(std::FILE* file) + { + return static_cast(::ftello(file)); + } +#endif + + int originOf(std::ios_base::seekdir direction) + { + if (direction == std::ios_base::beg) + return SEEK_SET; + if (direction == std::ios_base::cur) + return SEEK_CUR; + return SEEK_END; + } + } + + void SharedInputFileBuffer::FileCloser::operator()(std::FILE* file) const + { + std::fclose(file); + } + + bool SharedInputFileBuffer::open(std::filesystem::path const& path) + { + close(); + file_.reset(openShared(path)); + return isOpen(); + } + + bool SharedInputFileBuffer::isOpen() const + { + return file_ != nullptr; + } + + void SharedInputFileBuffer::close() + { + file_.reset(); + setg(buffer_.data(), buffer_.data(), buffer_.data()); + } + + SharedInputFileBuffer::int_type SharedInputFileBuffer::underflow() + { + if (gptr() < egptr()) + return traits_type::to_int_type(*gptr()); + if (!file_) + return traits_type::eof(); + const auto count = std::fread(buffer_.data(), 1, buffer_.size(), file_.get()); + if (count == 0) + return traits_type::eof(); + setg(buffer_.data(), buffer_.data(), buffer_.data() + count); + return traits_type::to_int_type(*gptr()); + } + + std::streamsize SharedInputFileBuffer::xsgetn(char_type* destination, std::streamsize count) + { + const auto buffered = std::min(egptr() - gptr(), count); + if (buffered > 0) + { + std::memcpy(destination, gptr(), static_cast(buffered)); + gbump(static_cast(buffered)); + } + if (buffered == count || !file_) + return buffered; + // The rest goes straight into the destination, the buffer would only copy it once more. + return buffered + + static_cast( + std::fread(destination + buffered, 1, static_cast(count - buffered), file_.get()) + ); + } + + SharedInputFileBuffer::pos_type + SharedInputFileBuffer::seekoff(off_type offset, std::ios_base::seekdir direction, std::ios_base::openmode which) + { + if (!file_ || (which & std::ios_base::in) == 0) + return pos_type(off_type(-1)); + // The file is ahead of the stream by what is buffered. + if (direction == std::ios_base::cur) + offset -= egptr() - gptr(); + setg(buffer_.data(), buffer_.data(), buffer_.data()); + if (seekFile(file_.get(), static_cast(offset), originOf(direction)) != 0) + return pos_type(off_type(-1)); + return pos_type(static_cast(tellFile(file_.get()))); + } + + SharedInputFileBuffer::pos_type SharedInputFileBuffer::seekpos(pos_type position, std::ios_base::openmode which) + { + return seekoff(off_type(position), std::ios_base::beg, which); + } + + SharedInputFileStream::SharedInputFileStream() + : std::istream{nullptr} + , buffer_{} + { + rdbuf(&buffer_); + } + + void SharedInputFileStream::open(std::filesystem::path const& path) + { + if (buffer_.open(path)) + clear(); + else + setstate(std::ios_base::failbit); + } + + bool SharedInputFileStream::is_open() const + { + return buffer_.isOpen(); + } + + void SharedInputFileStream::close() + { + buffer_.close(); + } +} diff --git a/utility/test/utility/main.cpp b/utility/test/utility/main.cpp index 9c6fcdcf..baadcef0 100644 --- a/utility/test/utility/main.cpp +++ b/utility/test/utility/main.cpp @@ -1,5 +1,6 @@ #include #include "test_case_convert.hpp" +#include "test_checked_symlink_status.hpp" #include "test_command_template.hpp" #include "test_directory_traversal.hpp" #include "test_download_mark.hpp" @@ -15,6 +16,7 @@ #include "test_rename.hpp" #include "test_resource_root.hpp" #include "test_link_safe_remove.hpp" +#include "test_shared_input_file_stream.hpp" #include "test_shell_integration.hpp" #include "test_typed_line_buffer.hpp" #include "test_user_directories.hpp" diff --git a/utility/test/utility/test_checked_symlink_status.hpp b/utility/test/utility/test_checked_symlink_status.hpp new file mode 100644 index 00000000..e5951ca7 --- /dev/null +++ b/utility/test/utility/test_checked_symlink_status.hpp @@ -0,0 +1,260 @@ +#pragma once + +#include +#include + +#include + +#include + +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +#else +# include +#endif + +extern std::filesystem::path temporaryRoot; + +namespace Utility::Test +{ + class CheckedSymlinkStatusTests : public ::testing::Test + { + protected: + TemporaryDirectory directory_{temporaryRoot, true}; + }; + + TEST_F(CheckedSymlinkStatusTests, AnExistingFileHasItsStatus) + { + std::ofstream{directory_.path() / "file.txt"} << "content"; + + const auto status = checkedSymlinkStatus(directory_.path() / "file.txt"); + + ASSERT_TRUE(status.has_value()) << status.error().message(); + ASSERT_TRUE(status->has_value()); + EXPECT_EQ((*status)->type(), std::filesystem::file_type::regular); + } + + TEST_F(CheckedSymlinkStatusTests, AMissingPathIsNothing) + { + const auto status = checkedSymlinkStatus(directory_.path() / "missing.txt"); + + ASSERT_TRUE(status.has_value()) << status.error().message(); + EXPECT_FALSE(status->has_value()); + } + + TEST_F(CheckedSymlinkStatusTests, APathBelowAFileIsNothing) + { + std::ofstream{directory_.path() / "file.txt"} << "content"; + + const auto status = checkedSymlinkStatus(directory_.path() / "file.txt" / "inside.txt"); + + ASSERT_TRUE(status.has_value()) << status.error().message(); + EXPECT_FALSE(status->has_value()); + } + + TEST_F(CheckedSymlinkStatusTests, ADanglingLinkIsALink) + { + std::error_code linkError; + std::filesystem::create_symlink(directory_.path() / "nowhere", directory_.path() / "link", linkError); + if (linkError) + GTEST_SKIP() << "Cannot create a symbolic link: " << linkError.message(); + + const auto status = checkedSymlinkStatus(directory_.path() / "link"); + + ASSERT_TRUE(status.has_value()) << status.error().message(); + ASSERT_TRUE(status->has_value()); + EXPECT_EQ((*status)->type(), std::filesystem::file_type::symlink); + } + +#ifdef _WIN32 + /** + * @brief A path past MAX_PATH is "not found" for std::filesystem without long path support, although it exists. + */ + class CheckedSymlinkStatusLongPathTests : public CheckedSymlinkStatusTests + { + protected: + void SetUp() override + { + const std::wstring segment(60, L's'); + longDirectory_ = directory_.path() / segment / segment / segment / segment / segment; + ASSERT_GE(longDirectory_.native().size(), std::size_t{MAX_PATH}); + std::filesystem::create_directories(extended(longDirectory_)); + } + + void TearDown() override + { + std::error_code removeError; + std::filesystem::remove_all(extended(directory_.path()), removeError); + } + + static std::filesystem::path extended(std::filesystem::path const& path) + { + return std::filesystem::path{LR"(\\?\)" + path.lexically_normal().make_preferred().native()}; + } + + std::filesystem::path longDirectory_{}; + }; + + TEST_F(CheckedSymlinkStatusLongPathTests, AnExistingPathPastMaxPathIsAnErrorNotGone) + { + std::ofstream{extended(longDirectory_ / "file.txt")} << "content"; + + const auto directory = checkedSymlinkStatus(longDirectory_); + const auto file = checkedSymlinkStatus(longDirectory_ / "file.txt"); + + ASSERT_FALSE(directory.has_value()); + EXPECT_EQ(directory.error().value(), ERROR_FILENAME_EXCED_RANGE); + ASSERT_FALSE(file.has_value()); + EXPECT_EQ(file.error().value(), ERROR_FILENAME_EXCED_RANGE); + } + + TEST_F(CheckedSymlinkStatusLongPathTests, AMissingPathPastMaxPathIsNothing) + { + const auto status = checkedSymlinkStatus(longDirectory_ / "missing.txt"); + + ASSERT_TRUE(status.has_value()) << status.error().message(); + EXPECT_FALSE(status->has_value()); + } + + TEST_F(CheckedSymlinkStatusTests, APathBeingDeletedAndWhatIsBelowItAreGone) + { + const auto doomed = directory_.path() / "doomed"; + std::filesystem::create_directory(doomed); + std::ofstream{doomed / "file.txt"} << "content"; + // An open handle, like a watch's, keeps what libc++ removes without POSIX semantics pending deletion. + const auto handle = std::unique_ptr{ + ::CreateFileW( + doomed.c_str(), + FILE_LIST_DIRECTORY, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + nullptr + ), + &::CloseHandle + }; + ASSERT_NE(handle.get(), INVALID_HANDLE_VALUE); + std::filesystem::remove(doomed / "file.txt"); + std::filesystem::remove(doomed); + std::error_code statusError; + std::filesystem::symlink_status(doomed, statusError); + if (statusError != std::errc::permission_denied) + GTEST_SKIP() << "the removal did not leave the directory pending deletion: " << statusError.message(); + + const auto status = checkedSymlinkStatus(doomed); + const auto below = checkedSymlinkStatus(doomed / "file.txt"); + + ASSERT_TRUE(status.has_value()) << status.error().message(); + EXPECT_FALSE(status->has_value()); + ASSERT_TRUE(below.has_value()) << below.error().message(); + EXPECT_FALSE(below->has_value()); + } + + TEST_F(CheckedSymlinkStatusTests, AFileWrittenAnewRightAfterALookMissedItIsThere) + { + const auto file = directory_.path() / "file.txt"; + std::ofstream{file} << "content"; + int looks = 0; + + // git deletes the file and writes it anew, the first look comes in between. + const auto status = Detail::checkedSymlinkStatus( + file, + [&looks](std::filesystem::path const& path, std::error_code& error) + { + if (looks++ == 0) + return std::filesystem::file_status{std::filesystem::file_type::not_found}; + return std::filesystem::symlink_status(path, error); + } + ); + + ASSERT_TRUE(status.has_value()) << status.error().message(); + ASSERT_TRUE(status->has_value()); + EXPECT_EQ((*status)->type(), std::filesystem::file_type::regular); + } + + TEST_F(CheckedSymlinkStatusTests, AFileGoneAgainOnTheSecondLookIsGone) + { + const auto file = directory_.path() / "file.txt"; + std::ofstream{file} << "content"; + int looks = 0; + + const auto status = Detail::checkedSymlinkStatus( + file, + [&looks, &file](std::filesystem::path const&, std::error_code&) + { + if (looks++ == 1) + std::filesystem::remove(file); + return std::filesystem::file_status{std::filesystem::file_type::not_found}; + } + ); + + ASSERT_TRUE(status.has_value()) << status.error().message(); + EXPECT_FALSE(status->has_value()); + } + + TEST_F(CheckedSymlinkStatusTests, ANameOnlyItsExtendedLengthPathReachesIsAnErrorNotGone) + { + // Win32 drops a trailing dot, so the plain path names another file. + const auto file = directory_.path() / "trailing."; + const auto extended = std::filesystem::path{LR"(\\?\)" + file.lexically_normal().make_preferred().native()}; + std::ofstream{extended} << "content"; + auto removeFile = Nui::ScopeExit{[&extended]() noexcept { + std::error_code removeError; + std::filesystem::remove(extended, removeError); + }}; + ASSERT_TRUE(std::filesystem::exists(extended)); + + const auto status = checkedSymlinkStatus(file); + + ASSERT_FALSE(status.has_value()); + EXPECT_EQ(status.error().value(), ERROR_INVALID_NAME); + } + + TEST_F(CheckedSymlinkStatusLongPathTests, AnExtendedLengthPathHasItsStatus) + { + const auto status = checkedSymlinkStatus(extended(longDirectory_)); + + ASSERT_TRUE(status.has_value()) << status.error().message(); + ASSERT_TRUE(status->has_value()); + EXPECT_EQ((*status)->type(), std::filesystem::file_type::directory); + } +#else + TEST_F(CheckedSymlinkStatusTests, APathThatCannotBeLookedAtIsAnErrorNotGone) + { + if (::geteuid() == 0) + GTEST_SKIP() << "root can look into any directory"; + const auto locked = directory_.path() / "locked"; + std::filesystem::create_directory(locked); + std::ofstream{locked / "inside.txt"} << "content"; + std::filesystem::permissions(locked, std::filesystem::perms::none); + auto restore = Nui::ScopeExit{[&locked]() noexcept { + std::error_code errorCode{}; + std::filesystem::permissions(locked, std::filesystem::perms::owner_all, errorCode); + }}; + + const auto inside = checkedSymlinkStatus(locked / "inside.txt"); + const auto missing = checkedSymlinkStatus(locked / "missing.txt"); + + ASSERT_FALSE(inside.has_value()); + EXPECT_EQ(inside.error(), std::errc::permission_denied); + EXPECT_FALSE(missing.has_value()) << "an unknown path must never count as gone"; + } + + TEST_F(CheckedSymlinkStatusTests, ANameTooLongToCheckIsAnErrorNotGone) + { + const auto status = checkedSymlinkStatus(directory_.path() / std::string(300, 'n')); + + ASSERT_FALSE(status.has_value()); + EXPECT_EQ(status.error(), std::errc::filename_too_long); + } +#endif +} diff --git a/utility/test/utility/test_shared_input_file_stream.hpp b/utility/test/utility/test_shared_input_file_stream.hpp new file mode 100644 index 00000000..037da60c --- /dev/null +++ b/utility/test/utility/test_shared_input_file_stream.hpp @@ -0,0 +1,114 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include +#include + +extern std::filesystem::path temporaryRoot; + +namespace Utility::Test +{ + class SharedInputFileStreamTests : public ::testing::Test + { + protected: + void SetUp() override + { + content_.reserve(200'000); + for (std::size_t index = 0; index < 200'000; ++index) + content_.push_back(static_cast('a' + index % 26)); + std::ofstream{path(), std::ios::binary} << content_; + } + + std::filesystem::path path() const + { + return directory_.path() / "file.bin"; + } + + TemporaryDirectory directory_{temporaryRoot, true}; + std::string content_{}; + }; + + TEST_F(SharedInputFileStreamTests, ReadsTheFileInChunks) + { + SharedInputFileStream stream; + stream.open(path()); + ASSERT_TRUE(stream.is_open()); + + std::string read; + std::array chunk{}; + while (stream.read(chunk.data(), chunk.size()) || stream.gcount() > 0) + read.append(chunk.data(), static_cast(stream.gcount())); + + EXPECT_EQ(read, content_); + EXPECT_TRUE(stream.eof()); + } + + TEST_F(SharedInputFileStreamTests, SeeksAndTellsLikeAFileStream) + { + SharedInputFileStream stream; + stream.open(path()); + + stream.seekg(0, std::ios::end); + EXPECT_EQ(static_cast(stream.tellg()), content_.size()); + stream.seekg(0, std::ios::beg); + std::array start{}; + ASSERT_TRUE(stream.read(start.data(), start.size())); + EXPECT_EQ(std::string(start.data(), start.size()), "abc"); + EXPECT_EQ(stream.tellg(), std::streampos{3}); + stream.seekg(100'000); + std::array middle{}; + ASSERT_TRUE(stream.read(middle.data(), middle.size())); + EXPECT_EQ(std::string(middle.data(), middle.size()), content_.substr(100'000, 2)); + EXPECT_TRUE(stream.good()); + } + + TEST_F(SharedInputFileStreamTests, ReadingPastTheEndFailsWithWhatThereWas) + { + SharedInputFileStream stream; + stream.open(path()); + stream.seekg(static_cast(content_.size() - 4)); + + std::array tail{}; + stream.read(tail.data(), tail.size()); + + EXPECT_EQ(stream.gcount(), 4); + EXPECT_FALSE(stream.good()); + EXPECT_EQ(std::string(tail.data(), 4), content_.substr(content_.size() - 4)); + } + + TEST_F(SharedInputFileStreamTests, TheFileCanBeRenamedAndDeletedWhileItIsOpen) + { + SharedInputFileStream stream; + stream.open(path()); + ASSERT_TRUE(stream.is_open()); + + std::error_code renameError; + std::filesystem::rename(path(), directory_.path() / "renamed.bin", renameError); + EXPECT_FALSE(renameError) << renameError.message(); + std::error_code removeError; + std::filesystem::remove(directory_.path() / "renamed.bin", removeError); + EXPECT_FALSE(removeError) << removeError.message(); + + std::array start{}; + ASSERT_TRUE(stream.read(start.data(), start.size())); + EXPECT_EQ(std::string(start.data(), start.size()), "abcde"); + stream.close(); + EXPECT_FALSE(stream.is_open()); + } + + TEST_F(SharedInputFileStreamTests, AMissingFileFailsToOpen) + { + SharedInputFileStream stream; + stream.open(directory_.path() / "missing.bin"); + + EXPECT_FALSE(stream.is_open()); + EXPECT_TRUE(stream.fail()); + } +} diff --git a/work_dependencies.json b/work_dependencies.json index 08670d2d..71588285 100644 --- a/work_dependencies.json +++ b/work_dependencies.json @@ -67,8 +67,8 @@ "name": "spdlog" }, { - "url": "https://github.com/SpartanJ/efsw.git", - "rev": "87abe599995d5646f5d83cf2e3a225bd73148b3a", + "url": "https://github.com/5cript/efsw.git", + "rev": "d71ed846811cc67ee8827614341e2777d200231a", "branch": "main", "name": "efsw" },