diff --git a/cpp/shared/io/PipelinedOps.cpp b/cpp/shared/io/PipelinedOps.cpp new file mode 100644 index 0000000..fe535c9 --- /dev/null +++ b/cpp/shared/io/PipelinedOps.cpp @@ -0,0 +1,171 @@ +#include "PipelinedOps.hpp" + +#include + +#include +#include +#include + +#include "../connection/SmbConnection.hpp" +#include "../util/SmbException.hpp" + +namespace react_native_smb { + +namespace { +constexpr auto kPollInterval = std::chrono::milliseconds(100); +constexpr auto kIdleTimeout = std::chrono::seconds(60); +// Long enough for cleanup (close, remove partial files) after a failure or cancel. +constexpr auto kDrainTimeout = std::chrono::seconds(5); +} // namespace + +PipelinedOps::PipelinedOps(smb2_context* ctx, SmbConnectionManager& manager, CancellationToken cancel, size_t maxInFlight, + std::string label) + : ctx_(ctx), manager_(manager), cancel_(std::move(cancel)), maxInFlight_(std::max(1, maxInFlight)), label_(std::move(label)) {} + +PipelinedOps::~PipelinedOps() { + // Pending callbacks point into this object: tear the transport down first. + if (ctx_ && !inFlight_.empty()) invalidate(); +} + +void PipelinedOps::submit(Issue issue, Done done, bool cleanup) { + auto op = std::make_unique(); + op->owner = this; + op->issue = std::move(issue); + op->done = std::move(done); + op->cleanup = cleanup; + queued_.push_back(std::move(op)); +} + +void PipelinedOps::fail(SmbErrorCode code, std::string message) { + if (!failure_ && !cancelled_) failure_ = Failure{code, std::move(message)}; + stop(); +} + +void PipelinedOps::failFromErrno(int result, const std::string& message) { + if (result == kNotSent) return; // only happens while already stopping + fail(static_cast(SmbErrorMapper::fromSmbFailure(result, 0, message)), message); +} + +void PipelinedOps::run() { + if (!ctx_) SmbException::raise(SmbErrorCode::NotConnected, label_ + ": SMB context is not connected"); + lastActivity_ = Clock::now(); + + for (;;) { + if (!stopping_ && cancel_.cancelled()) { + cancelled_ = true; + stop(); + } + launch(); + if (queued_.empty() && inFlight_.empty()) break; + + const auto now = Clock::now(); + if (stopping_ && drainDeadline_ && now > *drainDeadline_) { + invalidate(); + break; + } + if (now - lastActivity_ > kIdleTimeout) { + fail(SmbErrorCode::TimedOut, label_ + ": no reply from the server"); + invalidate(); + break; + } + + pollOnce(); + if (!ctx_) break; + } + + if (failure_) throw SmbException(failure_->code, failure_->message); +} + +void PipelinedOps::launch() { + while (ctx_ && !queued_.empty() && inFlight_.size() < maxInFlight_) { + std::unique_ptr op = std::move(queued_.front()); + queued_.pop_front(); + if (stopping_ && !op->cleanup) { + notify(*op, kNotSent, nullptr); + continue; + } + + Op* raw = op.get(); + inFlight_.push_back(std::move(op)); + const int result = raw->issue(ctx_, &PipelinedOps::onReply, raw); + if (result != 0) { + inFlight_.pop_back(); + failFromErrno(result, label_ + ": could not queue request: " + smb2_get_error(ctx_)); + } + } +} + +void PipelinedOps::onReply(smb2_context* /*smb2*/, int status, void* commandData, void* privateData) { + auto* raw = static_cast(privateData); + PipelinedOps& self = *raw->owner; + self.lastActivity_ = Clock::now(); + + auto it = std::find_if(self.inFlight_.begin(), self.inFlight_.end(), [raw](const std::unique_ptr& op) { return op.get() == raw; }); + if (it == self.inFlight_.end()) return; + std::unique_ptr op = std::move(*it); + self.inFlight_.erase(it); + self.notify(*op, status, commandData); +} + +void PipelinedOps::notify(Op& op, int status, void* commandData) { + try { + if (op.done) op.done(status, commandData); + } catch (const SmbException& e) { + fail(e.code(), e.what()); + } catch (const std::exception& e) { + fail(SmbErrorCode::Unknown, label_ + ": " + e.what()); + } catch (...) { + // Never unwind through libsmb2's C callback stack. + fail(SmbErrorCode::Unknown, label_ + ": unknown error"); + } +} + +void PipelinedOps::stop() { + if (stopping_) return; + stopping_ = true; + drainDeadline_ = Clock::now() + kDrainTimeout; +} + +void PipelinedOps::pollOnce() { + pollfd pfd{}; + pfd.fd = smb2_get_fd(ctx_); + pfd.events = static_cast(smb2_which_events(ctx_)); + if (pfd.fd < 0) { + fail(SmbErrorCode::NotConnected, label_ + ": connection lost"); + invalidate(); + return; + } + + const int result = ::poll(&pfd, 1, static_cast(kPollInterval.count())); + if (result < 0) { + if (errno == EINTR) return; + fail(static_cast(SmbErrorMapper::fromErrno(errno)), label_ + ": poll failed"); + invalidate(); + return; + } + if (result == 0) return; + + if (pfd.revents & (POLLERR | POLLHUP | POLLNVAL)) { + fail(SmbErrorCode::NotConnected, label_ + ": socket closed"); + invalidate(); + return; + } + if (smb2_service(ctx_, pfd.revents) < 0) { + const std::string message = label_ + ": " + smb2_get_error(ctx_); + fail(static_cast(SmbErrorMapper::fromSmbFailure(-1, smb2_get_nterror(ctx_), message, SmbErrorCode::NotConnected)), + message); + invalidate(); + } +} + +void PipelinedOps::invalidate() { + stop(); + if (!ctx_) return; + ctx_ = nullptr; + // Destroys the context; libsmb2 fires every pending callback with SHUTDOWN. + manager_.invalidateContext(); + inFlight_.clear(); + queued_.clear(); +} + +} // namespace react_native_smb diff --git a/cpp/shared/io/PipelinedOps.hpp b/cpp/shared/io/PipelinedOps.hpp new file mode 100644 index 0000000..9e1509e --- /dev/null +++ b/cpp/shared/io/PipelinedOps.hpp @@ -0,0 +1,97 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "../core/CancellationToken.hpp" +#include "../util/SmbErrorMapper.hpp" + +namespace react_native_smb { + +class SmbConnectionManager; + +// Runs async libsmb2 requests on an exclusively held context, up to +// `maxInFlight` at a time. Each submitted operation queues exactly one request; +// its `done` handler runs on the run() thread and may submit follow-up +// operations, which is how callers express ordering (e.g. rmdir after the +// folder's contents). Single-threaded: no locking. +// +// The first failure (or a cancel) stops new operations; requests already sent +// are drained. Operations submitted as `cleanup` still run while stopping, so +// partial results can be closed or removed. +class PipelinedOps { + public: + // Queues one request with the given callback; returns 0 when queued. + using Issue = std::function; + // Receives the request's status and reply (libsmb2's callback arguments), or + // kNotSent when the operation was dropped because the runner is stopping. + using Done = std::function; + static constexpr int kNotSent = INT_MIN; + + PipelinedOps(smb2_context* ctx, SmbConnectionManager& manager, CancellationToken cancel, size_t maxInFlight, std::string label); + ~PipelinedOps(); + PipelinedOps(const PipelinedOps&) = delete; + PipelinedOps& operator=(const PipelinedOps&) = delete; + + void submit(Issue issue, Done done, bool cleanup = false); + + // Records the first failure and stops new (non-cleanup) operations. + void fail(SmbErrorCode code, std::string message); + // Fails with the mapping of a high-level libsmb2 result (negative errno). + void failFromErrno(int result, const std::string& message); + bool stopping() const { return stopping_; } + + // Runs until nothing is queued or in flight. Returns normally when + // cancelled; throws SmbException on failure. + void run(); + + smb2_context* ctx() const { return ctx_; } + + private: + using Clock = std::chrono::steady_clock; + + struct Op { + PipelinedOps* owner = nullptr; + Issue issue; + Done done; + bool cleanup = false; + }; + struct Failure { + SmbErrorCode code; + std::string message; + }; + + static void onReply(smb2_context* smb2, int status, void* commandData, void* privateData); + void notify(Op& op, int status, void* commandData); + void launch(); + void stop(); + void pollOnce(); + void invalidate(); + + smb2_context* ctx_; + SmbConnectionManager& manager_; + CancellationToken cancel_; + size_t maxInFlight_; + std::string label_; + + std::deque> queued_; + std::list> inFlight_; + + bool stopping_{false}; + bool cancelled_{false}; + std::optional failure_; + std::optional drainDeadline_; + Clock::time_point lastActivity_; +}; + +} // namespace react_native_smb diff --git a/cpp/shared/io/SmbCopyTree.hpp b/cpp/shared/io/SmbCopyTree.hpp index 131593d..b6dd80f 100644 --- a/cpp/shared/io/SmbCopyTree.hpp +++ b/cpp/shared/io/SmbCopyTree.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -15,8 +16,10 @@ #include "../core/CancellationToken.hpp" #include "../core/SmbEnums.hpp" #include "../util/SmbException.hpp" +#include "PipelinedOps.hpp" #include "SmbPathUtil.hpp" #include "listing/DirectoryLister.hpp" +#include "transfer/SmbConcurrentFileCopy.hpp" #include "transfer/SmbFileCopyInternal.hpp" namespace react_native_smb { @@ -29,6 +32,26 @@ struct CopyTreeSourceInfo { }; namespace copy_tree_detail { +constexpr size_t kMaxFoldersInFlight = 16; +constexpr size_t kMaxFilesInFlight = 8; + +struct CopyJob { + std::string src; + std::string dst; + int64_t size; +}; + +// Every file of a listed tree with its destination path. +inline void collectFiles(const std::vector& items, const std::string& dstDir, std::vector& files) { + for (const auto& item : items) { + const std::string dest = path_util::buildUrl(dstDir, item.name); + if (item.isDirectory) + collectFiles(item.children, dest, files); + else + files.push_back({item.path, dest, item.size}); + } +} + inline int64_t totalFileSize(const std::vector& entries) { int64_t total = 0; for (const auto& entry : entries) total += entry.isDirectory ? totalFileSize(entry.children) : entry.size; @@ -113,12 +136,7 @@ inline void copyTreeImpl(SmbConnectionPool& pool, SmbOperatorKind kind, const st return; } - struct CopyJob { - std::string src; - std::string dst; - int64_t size; - }; - std::vector files; + std::vector files; const std::string normToRoot = normTo; sharedLease->submitSync([&](smb2_context* ctx) { if (cancel.cancelled()) { @@ -138,26 +156,30 @@ inline void copyTreeImpl(SmbConnectionPool& pool, SmbOperatorKind kind, const st } if (!recursive) return; - std::function&, const std::string&)> walk = [&](const std::vector& items, - const std::string& dstDir) { - for (const auto& item : items) { - if (cancel.cancelled()) return; - std::string destItem = path_util::buildUrl(dstDir, item.name); - if (item.isDirectory) { - std::string normDest = path_util::normalized(destItem); - const int r = smb2_mkdir(ctx, normDest.c_str()); - if (r < 0) { - SmbException::raiseFromSmb( - ctx, - r, "Failed to create destination directory: " + std::string(smb2_get_error(ctx))); - } - walk(item.children, destItem); - } else { - files.push_back({item.path, destItem, item.size}); + copy_tree_detail::collectFiles(source.entries, toPath, files); + + // Folders in parallel; each one's subfolders once it exists. + PipelinedOps ops(ctx, sharedLease->manager(), cancel, copy_tree_detail::kMaxFoldersInFlight, "Copy Failed"); + std::function&, const std::string&)> createFolders = + [&](const std::vector& items, const std::string& dstDir) { + for (const auto& item : items) { + if (!item.isDirectory) continue; + const std::string dest = path_util::buildUrl(dstDir, item.name); + const std::string norm = path_util::normalized(dest); + const std::vector* children = &item.children; + ops.submit([norm](smb2_context* c, smb2_command_cb cb, void* data) { return smb2_mkdir_async(c, norm.c_str(), cb, data); }, + [&, children, dest](int status, void*) { + if (status < 0) { + ops.failFromErrno(status, "Failed to create destination directory: " + + std::string(ops.ctx() ? smb2_get_error(ops.ctx()) : "connection lost")); + return; + } + createFolders(*children, dest); + }); } - } - }; - walk(source.entries, toPath); + }; + createFolders(source.entries, toPath); + ops.run(); }); if (cancel.cancelled()) return; @@ -168,12 +190,39 @@ inline void copyTreeImpl(SmbConnectionPool& pool, SmbOperatorKind kind, const st return; } - for (const auto& job : files) { - if (cancel.cancelled()) break; + const auto copyOne = [&](const copy_tree_detail::CopyJob& job) { sharedLease->submitSync([&](smb2_context* ctx) { smbCopyFileAsync(ctx, sharedLease->manager(), path_util::normalized(job.src), path_util::normalized(job.dst), totalBytesCopied, source.totalSize, onProgress, cancel); }); + }; + + // The first non-empty file goes alone: it finds out whether the server + // supports server-side copy (copying through the client if not). + auto probe = std::find_if(files.begin(), files.end(), [](const copy_tree_detail::CopyJob& job) { return job.size > 0; }); + if (probe == files.end()) probe = files.begin(); + copyOne(*probe); + files.erase(probe); + if (cancel.cancelled() || files.empty()) { + if (!cancel.cancelled() && onProgress) { + onProgress(static_cast(source.totalSize), static_cast(source.totalSize)); + } + return; + } + + if (sharedLease->manager().capabilities().serverSideCopyUnsupported.load()) { + for (const auto& job : files) { + if (cancel.cancelled()) break; + copyOne(job); + } + } else { + std::vector jobs; + jobs.reserve(files.size()); + for (const auto& job : files) jobs.push_back({path_util::normalized(job.src), path_util::normalized(job.dst)}); + sharedLease->submitSync([&](smb2_context* ctx) { + detail::copyFilesServerSide(ctx, sharedLease->manager(), jobs, copy_tree_detail::kMaxFilesInFlight, totalBytesCopied, + source.totalSize, onProgress, cancel); + }); } if (cancel.cancelled()) return; diff --git a/cpp/shared/io/transfer/SmbConcurrentFileCopy.cpp b/cpp/shared/io/transfer/SmbConcurrentFileCopy.cpp new file mode 100644 index 0000000..dfbfe2e --- /dev/null +++ b/cpp/shared/io/transfer/SmbConcurrentFileCopy.cpp @@ -0,0 +1,273 @@ +#include "SmbConcurrentFileCopy.hpp" + +#include + +#include +#include +#include + +#include +#include + +#include "../PipelinedOps.hpp" +#include "SmbCopyChunk.hpp" +#include "SmbTransferCore.hpp" + +namespace react_native_smb::detail { +namespace { + +namespace cc = copy_chunk; + +class ConcurrentCopy { + public: + ConcurrentCopy(PipelinedOps& ops, const std::vector& files, size_t maxFiles, std::shared_ptr totalCopied, + int64_t totalSize, std::function onProgress) + : ops_(ops), files_(files), maxFiles_(maxFiles), totalCopied_(std::move(totalCopied)), totalSize_(totalSize) { + progress_.progressHandler = std::move(onProgress); + } + + void start() { startMore(); } + + private: + struct Request { + bool busy = false; + uint64_t length = 0; + std::vector input; // libsmb2 references it until the reply arrives + }; + + struct Job { + const FileCopyJob* file = nullptr; + smb2fh* source = nullptr; + smb2fh* destination = nullptr; + bool destinationCreated = false; + smb2_stat_64 stat{}; + bool statReady = false; + uint8_t key[cc::kResumeKeySize]{}; + uint64_t size = 0; + uint64_t nextOffset = 0; + uint64_t copied = 0; + Request requests[cc::kRequestsInFlight]; + int pending = 0; // requests in flight for this file + bool aborted = false; // stop; close handles and remove the partial destination + bool closing = false; // handles are being closed (finished or cleaning up) + bool settled = false; + }; + + using Reply = std::function; + + // Sends one request for `job`; once its last request has replied, the job settles. + void send(Job& job, PipelinedOps::Issue issue, Reply onReply, bool cleanup = false) { + ++job.pending; + ops_.submit(std::move(issue), + [this, &job, onReply = std::move(onReply)](int status, void* data) { + --job.pending; + onReply(status, data); + if (job.pending == 0) settle(job); + }, + cleanup); + } + + // True when the job must stop: dropped request, a stopping runner, or an earlier abort. + bool stopped(Job& job, int status) { + if (status == PipelinedOps::kNotSent || ops_.stopping()) job.aborted = true; + return job.aborted; + } + + void abortErrno(Job& job, int status, const std::string& what) { + job.aborted = true; + ops_.failFromErrno(status, "Copy Failed: " + what + ": " + errorText()); + } + + void abortStatus(Job& job, uint32_t status, const std::string& what) { + job.aborted = true; + const std::string message = "Copy Failed: " + what + " for '" + job.file->from + "': " + cc::statusText(status); + ops_.fail(static_cast(SmbErrorMapper::fromSmbFailure(0, static_cast(status), message)), message); + } + + void abortWith(Job& job, SmbErrorCode code, const std::string& message) { + job.aborted = true; + ops_.fail(code, message); + } + + std::string errorText() const { return ops_.ctx() ? smb2_get_error(ops_.ctx()) : "connection lost"; } + + void startMore() { + while (!ops_.stopping() && active_ < maxFiles_ && next_ < files_.size()) { + jobs_.emplace_back(); + Job& job = jobs_.back(); + job.file = &files_[next_++]; + ++active_; + begin(job); + } + } + + void begin(Job& job) { + const std::string* from = &job.file->from; + const std::string* to = &job.file->to; + send(job, [from](smb2_context* ctx, smb2_command_cb cb, void* data) { return smb2_open_async(ctx, from->c_str(), O_RDONLY, cb, data); }, + [this, &job](int status, void* data) { + if (status == 0 && data) job.source = static_cast(data); + if (stopped(job, status)) return; + if (status < 0) return abortErrno(job, status, "Could not open source '" + job.file->from + "'"); + readSize(job); + }); + send(job, + [to](smb2_context* ctx, smb2_command_cb cb, void* data) { + return smb2_open_async(ctx, to->c_str(), O_WRONLY | O_CREAT | O_EXCL, cb, data); + }, + [this, &job](int status, void* data) { + if (status == 0 && data) { + job.destination = static_cast(data); + job.destinationCreated = true; + } + if (stopped(job, status)) return; + if (status < 0) return abortErrno(job, status, "Could not create destination '" + job.file->to + "'"); + startData(job); + }); + } + + void readSize(Job& job) { + send(job, [&job](smb2_context* ctx, smb2_command_cb cb, void* data) { return smb2_fstat_async(ctx, job.source, &job.stat, cb, data); }, + [this, &job](int status, void*) { + if (stopped(job, status)) return; + if (status < 0) return abortErrno(job, status, "Could not stat source '" + job.file->from + "'"); + job.statReady = true; + startData(job); + }); + } + + // Runs once the source is open and sized and the destination exists. + void startData(Job& job) { + if (!job.source || !job.statReady || !job.destination) return; + job.size = job.stat.smb2_size; + if (job.size == 0) return finish(job); + + send(job, + [&job](smb2_context* ctx, smb2_command_cb cb, void* data) { + return cc::queueFsctl(ctx, SMB2_FSCTL_SRV_REQUEST_RESUME_KEY, job.source, nullptr, cb, data) ? 0 : -ENOMEM; + }, + [this, &job](int status, void* data) { + const auto s = static_cast(status); + const bool keyReady = status != PipelinedOps::kNotSent && s == SMB2_STATUS_SUCCESS && ops_.ctx() && + cc::takeResumeKey(ops_.ctx(), data, job.key); + if (stopped(job, status)) return; + if (s != SMB2_STATUS_SUCCESS) return abortStatus(job, s, "resume key request failed"); + if (!keyReady) return abortWith(job, SmbErrorCode::Io, "Copy Failed: invalid resume key reply"); + fill(job); + }); + } + + void fill(Job& job) { + for (auto& request : job.requests) { + if (job.aborted || ops_.stopping() || job.nextOffset >= job.size) return; + if (request.busy) continue; + + const uint64_t length = cc::requestLength(job.size, job.nextOffset); + cc::encodeRequest(job.key, job.nextOffset, length, request.input); + request.busy = true; + request.length = length; + job.nextOffset += length; + + Request* req = &request; + send(job, + [&job, req](smb2_context* ctx, smb2_command_cb cb, void* data) { + return cc::queueFsctl(ctx, SMB2_FSCTL_SRV_COPYCHUNK_WRITE, job.destination, &req->input, cb, data) ? 0 : -ENOMEM; + }, + [this, &job, req](int status, void* data) { + req->busy = false; + const auto s = static_cast(status); + const int64_t written = + status != PipelinedOps::kNotSent && s == SMB2_STATUS_SUCCESS && ops_.ctx() ? cc::takeBytesWritten(ops_.ctx(), data) : -1; + if (stopped(job, status)) return; + if (s != SMB2_STATUS_SUCCESS) return abortStatus(job, s, "server-side copy failed"); + if (written != static_cast(req->length)) { + return abortWith(job, SmbErrorCode::Io, "Copy Failed: server copied " + std::to_string(written) + " of " + + std::to_string(req->length) + " bytes for '" + job.file->from + "'"); + } + job.copied += req->length; + if (totalCopied_) *totalCopied_ += static_cast(req->length); + progress_.emitProgress(totalCopied_ ? *totalCopied_ : 0, totalSize_); + if (job.copied == job.size) + finish(job); + else + fill(job); + }); + } + } + + void finish(Job& job) { + send(job, [&job](smb2_context* ctx, smb2_command_cb cb, void* data) { return smb2_fsync_async(ctx, job.destination, cb, data); }, + [this, &job](int status, void*) { + if (stopped(job, status)) return; + if (status < 0) return abortErrno(job, status, "smb2_fsync '" + job.file->to + "'"); + closeHandles(job, /*removeDestination=*/false); + }); + } + + // Closes whatever is open; on cleanup also removes the partial destination. + void closeHandles(Job& job, bool removeDestination) { + job.closing = true; + if (job.source) { + send(job, [&job](smb2_context* ctx, smb2_command_cb cb, void* data) { return smb2_close_async(ctx, job.source, cb, data); }, + [&job](int, void*) { job.source = nullptr; }, /*cleanup=*/true); + } + if (job.destination) { + send(job, + [&job](smb2_context* ctx, smb2_command_cb cb, void* data) { return smb2_close_async(ctx, job.destination, cb, data); }, + [this, &job, removeDestination](int status, void*) { + job.destination = nullptr; + if (removeDestination) return removePartial(job); + if (status < 0) abortErrno(job, status, "smb2_close '" + job.file->to + "'"); + }, + /*cleanup=*/true); + } else if (removeDestination) { + removePartial(job); + } + } + + void removePartial(Job& job) { + if (!job.destinationCreated) return; + const std::string* to = &job.file->to; + send(job, [to](smb2_context* ctx, smb2_command_cb cb, void* data) { return smb2_unlink_async(ctx, to->c_str(), cb, data); }, + [](int, void*) {}, /*cleanup=*/true); + } + + // Called once a job has no request in flight. + void settle(Job& job) { + if (job.settled) return; + if (job.aborted && !job.closing) { + closeHandles(job, /*removeDestination=*/true); + if (job.pending > 0) return; + } + if (!job.closing && !job.aborted) return; // still between steps (cannot happen while idle) + job.settled = true; + --active_; + startMore(); + } + + PipelinedOps& ops_; + const std::vector& files_; + size_t maxFiles_; + std::shared_ptr totalCopied_; + int64_t totalSize_; + TransferCore progress_; + + std::deque jobs_; // stable addresses + size_t next_ = 0; + size_t active_ = 0; +}; + +} // namespace + +void copyFilesServerSide(smb2_context* ctx, SmbConnectionManager& manager, const std::vector& files, size_t maxFiles, + const std::shared_ptr& totalCopied, int64_t totalSize, + const std::function& onProgress, const CancellationToken& cancel) { + if (files.empty()) return; + // Each file has at most its open/stat/close requests plus its copy requests in flight. + PipelinedOps ops(ctx, manager, cancel, maxFiles * (2 + cc::kRequestsInFlight), "Copy Failed"); + ConcurrentCopy copy(ops, files, maxFiles, totalCopied, totalSize, onProgress); + copy.start(); + ops.run(); +} + +} // namespace react_native_smb::detail diff --git a/cpp/shared/io/transfer/SmbConcurrentFileCopy.hpp b/cpp/shared/io/transfer/SmbConcurrentFileCopy.hpp new file mode 100644 index 0000000..3e8a59e --- /dev/null +++ b/cpp/shared/io/transfer/SmbConcurrentFileCopy.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../../core/CancellationToken.hpp" + +struct smb2_context; + +namespace react_native_smb { + +class SmbConnectionManager; + +namespace detail { + +struct FileCopyJob { + std::string from; // normalized + std::string to; // normalized +}; + +// Copies files on a held context with server-side copy, up to `maxFiles` at +// once; each file is open → resume key → COPYCHUNK_WRITE → fsync → close, as +// separate pipelined requests. Use only when the server is known to support +// server-side copy: a rejection fails the copy. On failure or cancel, files in +// progress are closed and their partial destinations removed; finished files +// stay. Returns normally when cancelled; throws SmbException on failure. +void copyFilesServerSide(smb2_context* ctx, SmbConnectionManager& manager, const std::vector& files, size_t maxFiles, + const std::shared_ptr& totalCopied, int64_t totalSize, + const std::function& onProgress, const CancellationToken& cancel); + +} // namespace detail +} // namespace react_native_smb diff --git a/cpp/shared/io/transfer/SmbCopyChunk.hpp b/cpp/shared/io/transfer/SmbCopyChunk.hpp new file mode 100644 index 0000000..a35e530 --- /dev/null +++ b/cpp/shared/io/transfer/SmbCopyChunk.hpp @@ -0,0 +1,103 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +// Wire format for server-side copy (FSCTL_SRV_REQUEST_RESUME_KEY and +// FSCTL_SRV_COPYCHUNK_WRITE), shared by single-file and concurrent copies. +namespace react_native_smb::detail::copy_chunk { + +// Within the protocol's default server limits (256 chunks, 1 MiB each, 16 MiB per request). +constexpr uint32_t kChunkLength = 1024 * 1024; +constexpr uint32_t kChunksPerRequest = 16; +constexpr uint64_t kRequestLength = static_cast(kChunkLength) * kChunksPerRequest; +constexpr int kRequestsInFlight = 4; +constexpr size_t kResumeKeySize = 24; + +inline bool isUnsupported(uint32_t status) { + return status == SMB2_STATUS_NOT_SUPPORTED || status == SMB2_STATUS_INVALID_DEVICE_REQUEST || + status == SMB2_STATUS_NOT_IMPLEMENTED || status == SMB2_STATUS_INVALID_PARAMETER; +} + +inline std::string statusText(uint32_t status) { + const char* text = nterror_to_str(status); + return text ? text : "unknown"; +} + +// Queues one FSCTL on `handle`. `input` must stay alive until the reply arrives. +inline smb2_pdu* queueFsctl(smb2_context* ctx, uint32_t ctlCode, smb2fh* handle, std::vector* input, smb2_command_cb cb, + void* cbData) { + smb2_ioctl_request request; + std::memset(&request, 0, sizeof(request)); + request.ctl_code = ctlCode; + std::memcpy(request.file_id, *smb2_get_file_id(handle), SMB2_FD_SIZE); + request.flags = SMB2_0_IOCTL_IS_FSCTL; + if (input) { + request.input_count = static_cast(input->size()); + request.input = input->data(); + } + smb2_pdu* pdu = smb2_cmd_ioctl_async(ctx, &request, cb, cbData); + if (pdu) smb2_queue_pdu(ctx, pdu); + return pdu; +} + +// Copies the resume key out of a RESUME_KEY reply and frees the reply data. +inline bool takeResumeKey(smb2_context* ctx, void* commandData, uint8_t (&key)[kResumeKeySize]) { + auto* reply = static_cast(commandData); + if (!reply || !reply->output) return false; + const bool ok = reply->output_count >= kResumeKeySize; + if (ok) std::memcpy(key, reply->output, kResumeKeySize); + smb2_free_data(ctx, reply->output); + return ok; +} + +// Bytes the server reports as written in a COPYCHUNK reply (frees the reply data); -1 if malformed. +inline int64_t takeBytesWritten(smb2_context* ctx, void* commandData) { + auto* reply = static_cast(commandData); + if (!reply || !reply->output) return -1; + int64_t written = -1; + if (reply->output_count >= 12) { + const auto* out = static_cast(reply->output) + 8; // TotalBytesWritten + written = static_cast(static_cast(out[0]) | (static_cast(out[1]) << 8) | + (static_cast(out[2]) << 16) | (static_cast(out[3]) << 24)); + } + smb2_free_data(ctx, reply->output); + return written; +} + +// Length of the next request starting at `offset`. +inline uint64_t requestLength(uint64_t fileSize, uint64_t offset) { return std::min(kRequestLength, fileSize - offset); } + +// SRV_COPYCHUNK_COPY for [offset, offset + length), same offsets on both sides. +inline void encodeRequest(const uint8_t (&key)[kResumeKeySize], uint64_t offset, uint64_t length, std::vector& out) { + const auto putU32 = [](uint8_t* p, uint32_t v) { + for (int i = 0; i < 4; ++i) p[i] = static_cast(v >> (8 * i)); + }; + const auto putU64 = [](uint8_t* p, uint64_t v) { + for (int i = 0; i < 8; ++i) p[i] = static_cast(v >> (8 * i)); + }; + constexpr size_t kHeaderSize = kResumeKeySize + 8; // key, chunk count, reserved + constexpr size_t kEntrySize = 24; // source offset, target offset, length, reserved + + const uint32_t chunks = static_cast((length + kChunkLength - 1) / kChunkLength); + out.assign(kHeaderSize + kEntrySize * chunks, 0); + std::memcpy(out.data(), key, kResumeKeySize); + putU32(out.data() + kResumeKeySize, chunks); + for (uint32_t i = 0; i < chunks; ++i) { + uint8_t* entry = out.data() + kHeaderSize + kEntrySize * i; + const uint64_t chunkOffset = offset + static_cast(i) * kChunkLength; + putU64(entry, chunkOffset); + putU64(entry + 8, chunkOffset); + putU32(entry + 16, static_cast(std::min(kChunkLength, offset + length - chunkOffset))); + } +} + +} // namespace react_native_smb::detail::copy_chunk diff --git a/cpp/shared/io/transfer/SmbServerSideCopy.cpp b/cpp/shared/io/transfer/SmbServerSideCopy.cpp index 60199e1..dbe48e3 100644 --- a/cpp/shared/io/transfer/SmbServerSideCopy.cpp +++ b/cpp/shared/io/transfer/SmbServerSideCopy.cpp @@ -1,52 +1,16 @@ #include "SmbServerSideCopy.hpp" -#include -#include #include #include -#include -#include -#include -#include - #include "../../connection/SmbConnection.hpp" #include "../../util/SmbLog.hpp" +#include "SmbCopyChunk.hpp" namespace react_native_smb::detail { namespace { -// Within the protocol's default server limits (256 chunks, 1 MiB each, 16 MiB per request). -constexpr uint32_t kChunkLength = 1024 * 1024; -constexpr uint32_t kChunksPerRequest = 16; -constexpr int kRequestsInFlight = 4; -constexpr size_t kResumeKeySize = 24; -constexpr size_t kChunkEntrySize = 24; -constexpr size_t kCopyHeaderSize = kResumeKeySize + 8; // key + chunk count + reserved -constexpr size_t kCopyResponseSize = 12; // chunks written, chunk bytes, total bytes - -bool isUnsupported(uint32_t status) { - return status == SMB2_STATUS_NOT_SUPPORTED || status == SMB2_STATUS_INVALID_DEVICE_REQUEST || - status == SMB2_STATUS_NOT_IMPLEMENTED || status == SMB2_STATUS_INVALID_PARAMETER; -} - -std::string statusText(uint32_t status) { - const char* text = nterror_to_str(status); - return text ? text : "unknown"; -} - -void putU32(uint8_t* out, uint32_t value) { - for (int i = 0; i < 4; ++i) out[i] = static_cast(value >> (8 * i)); -} - -void putU64(uint8_t* out, uint64_t value) { - for (int i = 0; i < 8; ++i) out[i] = static_cast(value >> (8 * i)); -} - -uint32_t getU32(const uint8_t* in) { - return static_cast(in[0]) | (static_cast(in[1]) << 8) | (static_cast(in[2]) << 16) | - (static_cast(in[3]) << 24); -} +namespace cc = copy_chunk; struct ServerCopy { struct Request { @@ -62,11 +26,10 @@ struct ServerCopy { std::shared_ptr totalCopied; int64_t totalSize; - uint8_t resumeKey[kResumeKeySize]{}; - bool keyReady = false; + uint8_t resumeKey[cc::kResumeKeySize]{}; bool unsupported = false; uint32_t rejectedStatus = 0; - Request requests[kRequestsInFlight]; + Request requests[cc::kRequestsInFlight]; ServerCopy(TransferCore& core, smb2fh* source, smb2fh* destination, std::shared_ptr totalCopied, int64_t totalSize) : core(core), source(source), destination(destination), totalCopied(std::move(totalCopied)), totalSize(totalSize) { @@ -75,32 +38,27 @@ struct ServerCopy { void fail(SmbErrorCode code, std::string message) { core.failure.fail(code, std::move(message)); } - void reject(uint32_t status) { - unsupported = true; - if (rejectedStatus == 0) rejectedStatus = status; + // A rejection only means "unsupported"; otherwise it fails the copy. + void reject(uint32_t status, const std::string& what) { + if (cc::isUnsupported(status)) { + unsupported = true; + if (rejectedStatus == 0) rejectedStatus = status; + return; + } + const std::string message = "Copy Failed: " + what + ": " + cc::statusText(status); + fail(static_cast(SmbErrorMapper::fromSmbFailure(0, static_cast(status), message)), message); } - bool queueIoctl(uint32_t ctlCode, smb2fh* handle, std::vector* input, smb2_command_cb cb, void* cbData) { - smb2_ioctl_request request; - std::memset(&request, 0, sizeof(request)); - request.ctl_code = ctlCode; - std::memcpy(request.file_id, *smb2_get_file_id(handle), SMB2_FD_SIZE); - request.flags = SMB2_0_IOCTL_IS_FSCTL; - if (input) { - request.input_count = static_cast(input->size()); - request.input = input->data(); - } - smb2_pdu* pdu = smb2_cmd_ioctl_async(core.ctx, &request, cb, cbData); - if (!pdu) { + bool queue(uint32_t ctlCode, smb2fh* handle, std::vector* input, smb2_command_cb cb, void* cbData) { + if (!cc::queueFsctl(core.ctx, ctlCode, handle, input, cb, cbData)) { fail(SmbErrorCode::Io, std::string("Copy Failed: could not queue server-side copy: ") + smb2_get_error(core.ctx)); return false; } - smb2_queue_pdu(core.ctx, pdu); ++core.inFlight; return true; } - void requestResumeKey() { queueIoctl(SMB2_FSCTL_SRV_REQUEST_RESUME_KEY, source, nullptr, onResumeKey, this); } + void requestResumeKey() { queue(SMB2_FSCTL_SRV_REQUEST_RESUME_KEY, source, nullptr, onResumeKey, this); } void fill() { for (auto& request : requests) { @@ -109,57 +67,29 @@ struct ServerCopy { fail(SmbErrorCode::Cancelled, "Copy cancelled"); return; } - if (!request.busy) submitCopy(request); - } - } - - void submitCopy(Request& request) { - const uint64_t offset = core.nextOffset; - const uint64_t length = std::min(static_cast(kChunkLength) * kChunksPerRequest, - static_cast(core.fileSize) - offset); - const uint32_t chunks = static_cast((length + kChunkLength - 1) / kChunkLength); - - request.input.assign(kCopyHeaderSize + kChunkEntrySize * chunks, 0); - uint8_t* out = request.input.data(); - std::memcpy(out, resumeKey, kResumeKeySize); - putU32(out + kResumeKeySize, chunks); - for (uint32_t i = 0; i < chunks; ++i) { - uint8_t* entry = out + kCopyHeaderSize + kChunkEntrySize * i; - const uint64_t chunkOffset = offset + static_cast(i) * kChunkLength; - putU64(entry, chunkOffset); // source offset - putU64(entry + 8, chunkOffset); // target offset - putU32(entry + 16, static_cast(std::min(kChunkLength, offset + length - chunkOffset))); + if (request.busy) continue; + + const uint64_t length = cc::requestLength(static_cast(core.fileSize), core.nextOffset); + cc::encodeRequest(resumeKey, core.nextOffset, length, request.input); + if (!queue(SMB2_FSCTL_SRV_COPYCHUNK_WRITE, destination, &request.input, onCopy, &request)) return; + request.busy = true; + request.length = length; + core.nextOffset += length; } - - if (!queueIoctl(SMB2_FSCTL_SRV_COPYCHUNK_WRITE, destination, &request.input, onCopy, &request)) return; - request.busy = true; - request.length = length; - core.nextOffset += length; } static void onResumeKey(smb2_context* smb2, int status, void* commandData, void* privateData) { auto& self = *static_cast(privateData); - auto* reply = static_cast(commandData); --self.core.inFlight; const auto s = static_cast(status); - - if (s == SMB2_STATUS_SUCCESS && reply && reply->output && reply->output_count >= kResumeKeySize) { - std::memcpy(self.resumeKey, reply->output, kResumeKeySize); - self.keyReady = true; - } - if (s == SMB2_STATUS_SUCCESS && reply && reply->output) smb2_free_data(smb2, reply->output); + const bool keyReady = s == SMB2_STATUS_SUCCESS && cc::takeResumeKey(smb2, commandData, self.resumeKey); if (self.core.failure.errored) return; if (s != SMB2_STATUS_SUCCESS) { - if (isUnsupported(s)) { - self.reject(s); - } else { - const std::string message = "Copy Failed: resume key request failed: " + statusText(s); - self.fail(static_cast(SmbErrorMapper::fromSmbFailure(0, status, message)), message); - } + self.reject(s, "resume key request failed"); return; } - if (!self.keyReady) { + if (!keyReady) { self.fail(SmbErrorCode::Io, "Copy Failed: invalid resume key reply"); return; } @@ -169,32 +99,17 @@ struct ServerCopy { static void onCopy(smb2_context* smb2, int status, void* commandData, void* privateData) { auto& request = *static_cast(privateData); auto& self = *request.owner; - auto* reply = static_cast(commandData); --self.core.inFlight; request.busy = false; const auto s = static_cast(status); - - uint32_t written = 0; - bool parsed = false; - if (s == SMB2_STATUS_SUCCESS && reply && reply->output) { - if (reply->output_count >= kCopyResponseSize) { - written = getU32(static_cast(reply->output) + 8); - parsed = true; - } - smb2_free_data(smb2, reply->output); - } + const int64_t written = s == SMB2_STATUS_SUCCESS ? cc::takeBytesWritten(smb2, commandData) : -1; if (self.core.failure.errored) return; if (s != SMB2_STATUS_SUCCESS) { - if (isUnsupported(s)) { - self.reject(s); - } else { - const std::string message = "Copy Failed: server-side copy failed: " + statusText(s); - self.fail(static_cast(SmbErrorMapper::fromSmbFailure(0, status, message)), message); - } + self.reject(s, "server-side copy failed"); return; } - if (!parsed || written != request.length) { + if (written != static_cast(request.length)) { self.fail(SmbErrorCode::Io, "Copy Failed: server copied " + std::to_string(written) + " of " + std::to_string(request.length) + " bytes"); return; @@ -228,12 +143,12 @@ ServerCopyResult serverSideCopy(TransferCore& core, SmbConnectionManager& manage if (core.failure.errored || !copy.unsupported) return ServerCopyResult::Copied; if (core.completedBytes > 0) { core.failure.fail(SmbErrorCode::Io, "Copy Failed: server-side copy stopped after " + std::to_string(core.completedBytes) + - " bytes: " + statusText(copy.rejectedStatus)); + " bytes: " + cc::statusText(copy.rejectedStatus)); return ServerCopyResult::Copied; } if (!capabilities.serverSideCopyUnsupported.exchange(true)) { - SMB_LOG_INFO("Copy: server-side copy not supported (%s); copying through the client", statusText(copy.rejectedStatus).c_str()); + SMB_LOG_INFO("Copy: server-side copy not supported (%s); copying through the client", cc::statusText(copy.rejectedStatus).c_str()); } core.nextOffset = 0; core.lastProgressBytes = 0; diff --git a/cpp/shared/operators/mutate/DeleteItemOperator.cpp b/cpp/shared/operators/mutate/DeleteItemOperator.cpp index c6c715e..d96d28c 100644 --- a/cpp/shared/operators/mutate/DeleteItemOperator.cpp +++ b/cpp/shared/operators/mutate/DeleteItemOperator.cpp @@ -3,9 +3,12 @@ #include #include +#include +#include #include #include "../../ReactNativeSmb.hpp" +#include "../../io/PipelinedOps.hpp" #include "../../io/SmbPathUtil.hpp" #include "../../io/listing/DirectoryLister.hpp" #include "../../util/SmbException.hpp" @@ -13,19 +16,70 @@ namespace react_native_smb { namespace { -// Flattens a listed tree; directories come after their descendants so they can -// be removed in order. -void collectItems(std::vector& nodes, std::vector& files, std::vector& dirs) { - for (auto& node : nodes) { - if (!node.isDirectory) { - files.push_back(std::move(node)); - continue; +constexpr size_t kMaxDeletesInFlight = 16; + +// Deletes a listed tree with pipelined requests: files and empty folders at +// once, every folder as soon as its contents are gone, the root folder last. +class TreeDeleter { + public: + explicit TreeDeleter(PipelinedOps& ops) : ops_(ops) {} + + void deleteFolder(const std::string& path, const std::vector& contents) { addFolder(path, contents, nullptr); } + + private: + struct Folder { + std::string path; // as given, for messages + Folder* parent; + size_t pending; // contents not yet deleted + }; + + void addFolder(const std::string& path, const std::vector& contents, Folder* parent) { + folders_.push_back(Folder{path, parent, contents.size()}); + Folder* folder = &folders_.back(); + for (const auto& item : contents) { + if (item.isDirectory) + addFolder(item.path, item.children, folder); + else + removeFile(item.path, folder); } - collectItems(node.children, files, dirs); - node.children.clear(); - dirs.push_back(std::move(node)); + if (folder->pending == 0) removeFolder(folder); } -} + + void removeFile(const std::string& path, Folder* parent) { + const std::string norm = path_util::normalized(path); + ops_.submit([norm](smb2_context* ctx, smb2_command_cb cb, void* data) { return smb2_unlink_async(ctx, norm.c_str(), cb, data); }, + [this, path, parent](int status, void*) { + if (status < 0) { + ops_.failFromErrno(status, "Recursive Delete Failed: Could not delete file '" + path + "'. Error: " + errorText()); + return; + } + release(parent); + }); + } + + void removeFolder(Folder* folder) { + const std::string norm = path_util::normalized(folder->path); + ops_.submit([norm](smb2_context* ctx, smb2_command_cb cb, void* data) { return smb2_rmdir_async(ctx, norm.c_str(), cb, data); }, + [this, folder](int status, void*) { + if (status < 0) { + const std::string what = folder->parent ? "Recursive Delete Failed: Could not delete sub-directory '" + : "Delete Failed: Could not delete directory '"; + ops_.failFromErrno(status, what + folder->path + "'. Error: " + errorText()); + return; + } + if (folder->parent) release(folder->parent); + }); + } + + void release(Folder* folder) { + if (folder->pending > 0 && --folder->pending == 0) removeFolder(folder); + } + + std::string errorText() const { return ops_.ctx() ? smb2_get_error(ops_.ctx()) : "connection lost"; } + + PipelinedOps& ops_; + std::deque folders_; // stable addresses +}; } // namespace void DeleteItemOperator::run() { @@ -57,32 +111,18 @@ void DeleteItemOperator::run() { return; } - std::vector tree; { listing::DirectoryListOptions options; options.maxDepth = -1; options.countChildren = false; - auto handle = requestContext(AcquireMode::Metadata); - tree = handle.submitSync([&](smb2_context* ctx) { return listing::listOnCtx(ctx, handle.manager(), path_, options, token); }); - } - if (isCancelled()) { - emitStatus(SmbTaskStatus::Cancelled); - return; - } - - std::vector fileItems; - std::vector dirItems; // children before their parent - collectItems(tree, fileItems, dirItems); - - if (!fileItems.empty() && !isCancelled()) { auto handle = requestContext(AcquireMode::Metadata); handle.submitSync([&](smb2_context* ctx) { - for (const auto& item : fileItems) { - if (isCancelled()) return; - std::string n = path_util::normalized(item.path); - const int r = smb2_unlink(ctx, n.c_str()); - if (r < 0) SmbException::raiseFromSmb(ctx, r, "Recursive Delete Failed: Could not delete file '" + item.path + "'. Error: " + smb2_get_error(ctx)); - } + const std::vector tree = listing::listOnCtx(ctx, handle.manager(), path_, options, token); + if (isCancelled()) return; + PipelinedOps ops(ctx, handle.manager(), token, kMaxDeletesInFlight, "Delete Failed"); + TreeDeleter deleter(ops); + deleter.deleteFolder(path_, tree); + ops.run(); }); } if (isCancelled()) { @@ -90,20 +130,6 @@ void DeleteItemOperator::run() { return; } - { - auto handle = requestContext(AcquireMode::Metadata); - handle.submitSync([&](smb2_context* ctx) { - for (const auto& item : dirItems) { - if (isCancelled()) return; - std::string n = path_util::normalized(item.path); - const int r = smb2_rmdir(ctx, n.c_str()); - if (r < 0) SmbException::raiseFromSmb(ctx, r, "Recursive Delete Failed: Could not delete sub-directory '" + item.path + "'. Error: " + smb2_get_error(ctx)); - } - const int rootR = smb2_rmdir(ctx, norm.c_str()); - if (rootR < 0) SmbException::raiseFromSmb(ctx, rootR, "Delete Failed: Could not delete directory '" + path_ + "'. Error: " + smb2_get_error(ctx)); - }); - } - emitStatus(SmbTaskStatus::Success); } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9c1e251..be8cbd1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -69,6 +69,18 @@ open source, create destination - A failure after data started moving fails the copy; the partial destination is deleted, as with the client copy. - Server-side copy only works within one share, which every pool slot uses. +## Pipelined tree operations + +Recursive delete and copy send one small request per file or folder. `PipelinedOps` keeps several of them in flight on one held slot; each completion may submit follow-up requests, which expresses the ordering. The first failure or a cancel stops new requests, drains the ones in flight, and still runs cleanup (closing handles, removing partial files). + +| Operation | In flight | Ordering | +|-----------|-----------|----------| +| `deleteItem` | 16 requests | files and empty folders at once; a folder once its contents are gone; the root folder last | +| Copy: folders | 16 requests | a folder once its parent exists | +| Copy: files | 8 files | each file runs open → stat → resume key → COPYCHUNK_WRITE → fsync → close as its own chain | + +A recursive copy sends its first non-empty file alone to find out whether the server supports server-side copy. If it does, the remaining files are copied 8 at a time; if not, they are copied through the client one after another. Files in progress when a copy fails or is cancelled are removed; finished files stay. + ## Listing pipeline `listDirectory` runs `DirectoryLister` on one slot — Interactive for `maxDepth: 0`, Metadata for nested listings — instead of libsmb2's `smb2_opendir` (which reads the whole directory before returning): @@ -86,7 +98,7 @@ The same engine backs every other directory walk through `listing::listOnCtx`, a | Caller | Listing | | ------ | ------- | -| `deleteItem` | whole tree once (`maxDepth: -1`, no counts), then files and directories deepest-first | +| `deleteItem` | whole tree once (`maxDepth: -1`, no counts), then pipelined deletes (see above) | | `copyItem`, `duplicateItem` | whole source tree once; total size and copy plan come from it | | `duplicateItem` naming | parent filtered by the server to `" copy*"`, no counts | | `getPathInfo` | none for files: `smb2_stat` and the SD query run together; folders read their own entries once for `childCount` |