From e584d7efdfe9188bf559c4bc50eab7ab9d0e8eef Mon Sep 17 00:00:00 2001 From: yfcyrus <79596426+yfcyrus@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:04:52 +0800 Subject: [PATCH 1/3] perf(listing): coalesce entry batches with notify-and-drain delivery Replace per-flush entry pushes with a payload-free signal per subscriber and a drainEntries pull, so a busy JS thread queues at most one pending delivery and receives every entry appended since its last drain in one batch. - track a cursor and scheduled flag per EntryStream subscriber instead of pushing each batch - add drainEntries to SmbTask and NativeSmbListTask - drain once right after subscribing so late subscribers still get the replay - keep SmbListTask.subscribeEntries API, server order, and delivery before terminal status --- cpp/shared/core/EntryStream.hpp | 59 ++++++++++++++++++++++++--------- cpp/shared/core/SmbTask.cpp | 3 ++ cpp/shared/core/SmbTask.hpp | 5 +-- src/core/SmbListTask.ts | 23 +++++++++---- src/types.ts | 8 +++-- 5 files changed, 73 insertions(+), 25 deletions(-) diff --git a/cpp/shared/core/EntryStream.hpp b/cpp/shared/core/EntryStream.hpp index 4011552..62f0307 100644 --- a/cpp/shared/core/EntryStream.hpp +++ b/cpp/shared/core/EntryStream.hpp @@ -14,33 +14,56 @@ namespace react_native_smb { // Append-only stream of listing entries owned by a task. It is also the single // storage of the final listing result. // -// Subscribers first receive everything appended so far as one batch, then each -// live batch. Listeners run under the lock so a replay can never be overtaken -// by a later batch; JS listeners only enqueue onto the JS thread, so this cannot -// re-enter. +// Delivery is notify-and-drain: appends wake each subscriber at most once until +// that subscriber drains. A busy JS thread therefore queues one pending signal +// per subscriber and receives every entry appended since its last drain as a +// single coalesced batch. JS drains once right after subscribe so late +// subscribers still get a replay of entries already stored. Signals run outside +// the lock and must not re-enter EntryStream synchronously. class EntryStream { public: using Batch = std::vector; - using Listener = std::function; + using Signal = std::function; void append(Batch batch) { if (batch.empty()) return; - std::lock_guard lock(mutex_); - for (auto& [id, listener] : listeners_) notify(listener, batch); - entries_.insert(entries_.end(), std::make_move_iterator(batch.begin()), std::make_move_iterator(batch.end())); + std::vector pending; + { + std::lock_guard lock(mutex_); + entries_.insert(entries_.end(), std::make_move_iterator(batch.begin()), std::make_move_iterator(batch.end())); + for (auto& [id, sub] : subscribers_) { + if (sub.scheduled) continue; + sub.scheduled = true; + if (sub.signal) pending.push_back(sub.signal); + } + } + for (const auto& signal : pending) notify(signal); } - std::string subscribe(Listener listener) { + std::string subscribe(Signal signal) { std::lock_guard lock(mutex_); - if (!entries_.empty()) notify(listener, entries_); std::string id = "ent_" + std::to_string(nextId_++); - listeners_.emplace(id, std::move(listener)); + subscribers_.emplace(id, Subscriber{std::move(signal), 0, false}); return id; } void unsubscribe(const std::string& id) { std::lock_guard lock(mutex_); - listeners_.erase(id); + subscribers_.erase(id); + } + + Batch drain(const std::string& id) { + std::lock_guard lock(mutex_); + auto it = subscribers_.find(id); + if (it == subscribers_.end()) return {}; + Subscriber& sub = it->second; + Batch out; + if (sub.cursor < entries_.size()) { + out.assign(entries_.begin() + static_cast(sub.cursor), entries_.end()); + sub.cursor = entries_.size(); + } + sub.scheduled = false; + return out; } template @@ -50,16 +73,22 @@ class EntryStream { } private: - static void notify(const Listener& listener, const Batch& batch) { + struct Subscriber { + Signal signal; + size_t cursor{0}; + bool scheduled{false}; + }; + + static void notify(const Signal& signal) { try { - if (listener) listener(batch); + if (signal) signal(); } catch (...) { } } mutable std::mutex mutex_; Batch entries_; - std::map listeners_; + std::map subscribers_; size_t nextId_{1}; }; diff --git a/cpp/shared/core/SmbTask.cpp b/cpp/shared/core/SmbTask.cpp index 0807844..63f2455 100644 --- a/cpp/shared/core/SmbTask.cpp +++ b/cpp/shared/core/SmbTask.cpp @@ -138,6 +138,8 @@ std::string SmbTask::subscribeEntries(const EntriesListener& listener) { void SmbTask::unsubscribeEntries(const std::string& id) { entries_->unsubscribe(id); } +std::vector SmbTask::drainEntries(const std::string& id) { return entries_->drain(id); } + void SmbTask::loadHybridMethods() { HybridObject::loadHybridMethods(); registerHybrids(this, [](Prototype& prototype) { @@ -148,6 +150,7 @@ void SmbTask::loadHybridMethods() { prototype.registerHybridMethod("unsubscribe", &SmbTask::unsubscribe); prototype.registerHybridMethod("subscribeEntries", &SmbTask::subscribeEntries); prototype.registerHybridMethod("unsubscribeEntries", &SmbTask::unsubscribeEntries); + prototype.registerHybridMethod("drainEntries", &SmbTask::drainEntries); prototype.registerRawHybridMethod("getResultValue", 0, &SmbTask::getResultValueRaw); }); } diff --git a/cpp/shared/core/SmbTask.hpp b/cpp/shared/core/SmbTask.hpp index 3d5c6bd..7ee7b87 100644 --- a/cpp/shared/core/SmbTask.hpp +++ b/cpp/shared/core/SmbTask.hpp @@ -25,7 +25,7 @@ using namespace margelo::nitro; class SmbTask : public HybridObject, public SmbTaskCore { public: - using EntriesListener = std::function&)>; + using EntriesListener = std::function; SmbTask(std::string id, std::unique_ptr seedOp, std::shared_ptr pool); ~SmbTask(); @@ -41,9 +41,10 @@ class SmbTask : public HybridObject, public SmbTaskCore { std::string subscribeMap(const std::function&)>& listener); void unsubscribe(const std::string& id) { SmbTaskCore::unsubscribe(id); } - // Incremental listing entries (replays entries emitted before subscribing). + // Incremental listing entries via notify-and-drain (coalesced while JS is busy). std::string subscribeEntries(const EntriesListener& listener); void unsubscribeEntries(const std::string& id); + std::vector drainEntries(const std::string& id); const std::shared_ptr& entryStream() const { return entries_; } std::future result() { return completionFuture(); } diff --git a/src/core/SmbListTask.ts b/src/core/SmbListTask.ts index 296ddda..c2eeb2b 100644 --- a/src/core/SmbListTask.ts +++ b/src/core/SmbListTask.ts @@ -5,6 +5,9 @@ import { SmbTask } from './SmbTask'; * Task returned by `listDirectory`. Top-level entries are delivered in batches * while the listing is still running; `result()` still resolves with the * complete list. + * + * Native flushes wake a payload-free signal; this wrapper drains coalesced + * entries so a busy JS thread receives fewer, larger batches. */ export class SmbListTask extends SmbTask { constructor( @@ -17,8 +20,9 @@ export class SmbListTask extends SmbTask { /** * Receive top-level entries as they become available, in server order. The * first call delivers every entry emitted so far, so subscribing late loses - * nothing. Each entry is final: directories carry `childCount`, and with a - * non-zero `maxDepth` their complete `children` subtree. + * nothing. While the JS thread is busy, native flushes coalesce into one + * batch per wake. Each entry is final: directories carry `childCount`, and + * with a non-zero `maxDepth` their complete `children` subtree. * * Entries received before a failure or cancellation are an incomplete listing. * @@ -30,15 +34,22 @@ export class SmbListTask extends SmbTask { */ subscribeEntries(listener: (batch: SmbFileInfo[]) => void): () => void { let active = true; - const subscriptionId = this.nativeList.subscribeEntries((batch) => { - if (active) listener(batch); - }); + let subscriptionId: string | null = null; + + const deliver = () => { + if (!active || subscriptionId == null) return; + const batch = this.nativeList.drainEntries(subscriptionId); + if (batch.length > 0) listener(batch); + }; + + subscriptionId = this.nativeList.subscribeEntries(deliver); + deliver(); return () => { if (!active) return; active = false; try { - this.nativeList.unsubscribeEntries(subscriptionId); + this.nativeList.unsubscribeEntries(subscriptionId!); } catch {} }; } diff --git a/src/types.ts b/src/types.ts index e875386..4efefe0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -173,7 +173,11 @@ export interface NativeSmbTask { } export interface NativeSmbListTask extends NativeSmbTask { - /** Replays entries emitted so far as one batch, then delivers each live batch. */ - subscribeEntries(listener: (entries: SmbFileInfo[]) => void): string; + /** + * Payload-free wake for new entries. Call {@link drainEntries} to pull every + * entry appended since the last drain (including a late-subscribe replay). + */ + subscribeEntries(listener: () => void): string; unsubscribeEntries(subscriptionId: string): void; + drainEntries(subscriptionId: string): SmbFileInfo[]; } From e2e452c224ae639f7c34b2de1f769d61fef25135 Mon Sep 17 00:00:00 2001 From: yfcyrus <79596426+yfcyrus@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:05:40 +0800 Subject: [PATCH 2/3] test(js): cover coalesced entry drains in SmbListTask Model the native cursor and deferred signals in FakeNativeListTask so coalescing, replay, and unsubscribe behavior stay covered without a device. - merge several appends behind one pending signal into a single batch - ignore signals that arrive after unsubscribe - drain the last entries before the terminal status settles --- tests/SmbListTask.test.ts | 106 ++++++++++++++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 9 deletions(-) diff --git a/tests/SmbListTask.test.ts b/tests/SmbListTask.test.ts index cae578a..17bf7a9 100644 --- a/tests/SmbListTask.test.ts +++ b/tests/SmbListTask.test.ts @@ -34,15 +34,23 @@ function entry(name: string): SmbFileInfo { }; } -/** Mirrors the native EntryStream: replays on subscribe, then delivers live batches. */ +type EntrySubscriber = { + signal: () => void; + cursor: number; + scheduled: boolean; +}; + +/** Mirrors native EntryStream notify-and-drain delivery. */ class FakeNativeListTask implements NativeSmbListTask { current: Record; readonly listeners = new Map) => void>(); - readonly entryListeners = new Map void>(); + readonly entrySubscribers = new Map(); readonly unsubscribedEntries: string[] = []; readonly emitted: SmbFileInfo[] = []; + readonly pendingSignals: Array<() => void> = []; resultReads = 0; private nextId = 1; + private deferSignals = false; constructor(current: Record) { this.current = current; @@ -69,21 +77,44 @@ class FakeNativeListTask implements NativeSmbListTask { return [...this.emitted]; } - subscribeEntries(listener: (entries: SmbFileInfo[]) => void): string { + subscribeEntries(listener: () => void): string { const id = `ent-${this.nextId++}`; - if (this.emitted.length > 0) listener([...this.emitted]); - this.entryListeners.set(id, listener); + this.entrySubscribers.set(id, { signal: listener, cursor: 0, scheduled: false }); return id; } unsubscribeEntries(subscriptionId: string): void { this.unsubscribedEntries.push(subscriptionId); - this.entryListeners.delete(subscriptionId); + this.entrySubscribers.delete(subscriptionId); + } + + drainEntries(subscriptionId: string): SmbFileInfo[] { + const sub = this.entrySubscribers.get(subscriptionId); + if (!sub) return []; + const out = this.emitted.slice(sub.cursor); + sub.cursor = this.emitted.length; + sub.scheduled = false; + return out; + } + + /** When true, signals queue until {@link flushSignals}; otherwise they run immediately. */ + setDeferSignals(defer: boolean): void { + this.deferSignals = defer; + } + + flushSignals(): void { + const queued = this.pendingSignals.splice(0); + for (const signal of queued) signal(); } emitEntries(batch: SmbFileInfo[]): void { - for (const listener of this.entryListeners.values()) listener(batch); this.emitted.push(...batch); + for (const sub of this.entrySubscribers.values()) { + if (sub.scheduled) continue; + sub.scheduled = true; + if (this.deferSignals) this.pendingSignals.push(sub.signal); + else sub.signal(); + } } emit(next: Record): void { @@ -104,7 +135,10 @@ describe('SmbListTask.subscribeEntries', () => { native.emitEntries([entry('a'), entry('b')]); native.emitEntries([entry('c')]); - expect(names(batches)).toEqual([['a', 'b'], ['c']]); + expect(names(batches)).toEqual([ + ['a', 'b'], + ['c'], + ]); }); it('replays earlier entries to late subscribers', () => { @@ -117,7 +151,10 @@ describe('SmbListTask.subscribeEntries', () => { task.subscribeEntries((batch) => batches.push(batch)); native.emitEntries([entry('c')]); - expect(names(batches)).toEqual([['a', 'b'], ['c']]); + expect(names(batches)).toEqual([ + ['a', 'b'], + ['c'], + ]); }); it('fans out to independent subscribers', () => { @@ -149,6 +186,57 @@ describe('SmbListTask.subscribeEntries', () => { expect(batches).toEqual([]); expect(native.unsubscribedEntries).toEqual(['ent-1']); }); + + it('merges several appends behind one pending signal into a single batch', () => { + const native = new FakeNativeListTask(snapshot(TaskStatus.Running)); + native.setDeferSignals(true); + const task = new SmbListTask('task-1', native); + const batches: SmbFileInfo[][] = []; + + task.subscribeEntries((batch) => batches.push(batch)); + native.emitEntries([entry('a')]); + native.emitEntries([entry('b')]); + native.emitEntries([entry('c')]); + + expect(batches).toEqual([]); + expect(native.pendingSignals).toHaveLength(1); + + native.flushSignals(); + + expect(names(batches)).toEqual([['a', 'b', 'c']]); + }); + + it('ignores signals that arrive after unsubscribe', () => { + const native = new FakeNativeListTask(snapshot(TaskStatus.Running)); + native.setDeferSignals(true); + const task = new SmbListTask('task-1', native); + const batches: SmbFileInfo[][] = []; + + const unsubscribe = task.subscribeEntries((batch) => batches.push(batch)); + native.emitEntries([entry('a')]); + unsubscribe(); + native.flushSignals(); + + expect(batches).toEqual([]); + }); + + it('drains the last entries before the terminal status settles', async () => { + const native = new FakeNativeListTask(snapshot(TaskStatus.Running)); + native.setDeferSignals(true); + const task = new SmbListTask('task-1', native); + const batches: SmbFileInfo[][] = []; + + task.subscribeEntries((batch) => batches.push(batch)); + native.emitEntries([entry('a')]); + native.emitEntries([entry('b')]); + + const done = task.wait(); + native.flushSignals(); + native.emit(snapshot(TaskStatus.Success)); + + await expect(done).resolves.toBeUndefined(); + expect(names(batches)).toEqual([['a', 'b']]); + }); }); describe('SmbListTask completion', () => { From 6e34b3a1f0d76b93a90cc3d5bc4c987880597dd7 Mon Sep 17 00:00:00 2001 From: yfcyrus <79596426+yfcyrus@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:06:32 +0800 Subject: [PATCH 3/3] docs(listing): document notify-and-drain entry delivery Describe how entry batches merge while the JS thread is busy and how batchSize and the 100ms delay now control native flushes rather than JS callbacks. - update subscribeEntries notes in API, task model, and architecture docs - clarify batch timing in the listDirectory JSDoc --- docs/API.md | 5 +++-- docs/ARCHITECTURE.md | 2 +- docs/TASK_MODEL.md | 4 ++-- src/core/SmbClient.ts | 7 ++++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/API.md b/docs/API.md index 8b327fd..ffbed42 100644 --- a/docs/API.md +++ b/docs/API.md @@ -87,7 +87,8 @@ try { ``` - Every streamed entry is final: directories already carry `childCount`, and `securityDescriptor` is set when requested. -- A batch is pushed once `batchSize` entries are ready or 100 ms after the previous batch; the first batch goes out as soon as anything is ready. A batch contains every entry ready at that moment, so it can be larger than `batchSize`. +- Native flushes when `batchSize` entries are ready or 100 ms after the previous flush (the first flush goes out as soon as anything is ready). Each flush takes every entry ready at that moment, so it can be larger than `batchSize`. +- Delivery to JS is notify-and-drain: a busy JS thread coalesces several native flushes into one `subscribeEntries` callback. - Subscribing late loses nothing: the first callback replays every entry emitted so far. - Entries arrive in server order. Sort in the UI if you need a stable order. - If the task fails or is cancelled, entries already received are an incomplete listing. @@ -126,7 +127,7 @@ See [Connection pool](./ARCHITECTURE.md). | Member | Description | | ------ | ----------- | -| `subscribeEntries(listener)` | Receive entry batches (replays earlier ones first); returns unsubscribe | +| `subscribeEntries(listener)` | Receive entry batches (replays earlier ones first; coalesces while JS is busy); returns unsubscribe | ### Errors diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index be8cbd1..33c4d15 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -92,7 +92,7 @@ A recursive copy sends its first non-empty file alone to find out whether the se └ per entry (opt-in): CREATE + QUERY_INFO + CLOSE security descriptor ``` -Lookups share one connection: up to 16 directory queries (expansions and counts) and, separately, up to 32 security-descriptor queries in flight. Lookups start in top-level order — everything under an earlier top-level entry before anything under a later one — and directory queries go on the wire before security descriptors. An entry is ready once its lookups finish — for an expanded directory, also all of its children, whose count becomes `childCount` without another query, so each directory is opened once. Ready top-level entries are emitted in server order through the task's `EntryStream` (`subscribeEntries`), which also stores the final result. +Lookups share one connection: up to 16 directory queries (expansions and counts) and, separately, up to 32 security-descriptor queries in flight. Lookups start in top-level order — everything under an earlier top-level entry before anything under a later one — and directory queries go on the wire before security descriptors. An entry is ready once its lookups finish — for an expanded directory, also all of its children, whose count becomes `childCount` without another query, so each directory is opened once. Ready top-level entries are appended in server order to the task's `EntryStream`, which also stores the final result. Each subscriber is woken with a payload-free signal at most once until it drains; `subscribeEntries` then pulls every entry since the last drain as one coalesced batch, so a busy JS thread does not queue one callback per native flush. The same engine backs every other directory walk through `listing::listOnCtx`, a non-streaming call on an already-held slot; libsmb2's `opendir` is no longer used: diff --git a/docs/TASK_MODEL.md b/docs/TASK_MODEL.md index d63b2c4..40f68c6 100644 --- a/docs/TASK_MODEL.md +++ b/docs/TASK_MODEL.md @@ -94,7 +94,7 @@ function DownloadBar({ task }) { ## Stream listing entries -`listDirectory` returns an `SmbListTask`. Besides status snapshots, a listing pushes batches of top-level entries through a separate channel — no `progress` is involved: +`listDirectory` returns an `SmbListTask`. Besides status snapshots, a listing delivers top-level entries through a separate channel — no `progress` is involved: ```typescript const task = smb.listDirectory('/Media'); @@ -103,7 +103,7 @@ await task.wait(); // completion without converting the full result again unsub(); ``` -`subscribeEntries` replays the entries emitted before the call, so it can be attached at any time. All batches are delivered before the task's terminal status. `result()` still resolves with the complete array, so existing callers are unaffected. +`subscribeEntries` replays the entries emitted before the call, so it can be attached at any time. Native flushes wake a payload-free signal; the JS wrapper drains every entry appended since the last wake, so a busy JS thread coalesces several flushes into one batch. All entry deliveries run before the task's terminal status. `result()` still resolves with the complete array, so existing callers are unaffected. ## Fire-and-forget diff --git a/src/core/SmbClient.ts b/src/core/SmbClient.ts index eb495c0..9dae541 100644 --- a/src/core/SmbClient.ts +++ b/src/core/SmbClient.ts @@ -79,9 +79,10 @@ export class SmbClient { /** * List a directory. Top-level entries also stream via - * {@link SmbListTask.subscribeEntries}: a batch is pushed once `batchSize` - * entries are ready or 100 ms have passed (the first batch goes out immediately). - * With a non-zero `maxDepth`, each streamed entry carries its complete subtree. + * {@link SmbListTask.subscribeEntries}. `batchSize` and the 100 ms delay control + * native flushes (the first flush goes out immediately); JS may receive several + * flushes as one coalesced batch while the thread is busy. With a non-zero + * `maxDepth`, each streamed entry carries its complete subtree. */ listDirectory(path: string, options: ListDirectoryOptions = {}): SmbListTask { const { maxDepth = 0, includeSecurityDescriptor = false, batchSize = DEFAULT_LIST_BATCH_SIZE } = options;