Skip to content

New note and Task stop re-reading the whole vault #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adibhanna merged 1 commit into from
Oct 23, 2026
Merged

New note and Task stop re-reading the whole vault #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 39 additions & 14 deletions src/bridge/native-fs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,14 @@ export class NativeFs {
readonly cloudRootUri: string | null
/** Native URI of the vault root, resolved at open. */
rootUri: string | null = null
/**
* Moves before and after every change this vault makes on disk, so a
* listing taken at one generation is known to predate nothing the app wrote
* since (MobileVault shares listings between overlapping tree walks on that
* promise). Writers outside the app are not counted; the foreground rescan
* is what finds those.
*/
writeGeneration = 0

constructor(vaultName: string, cloudRootUri: string | null = null) {
this.rootPath = `${VAULTS_DIR}/${vaultName}`
Expand All @@ -125,6 +133,15 @@ export class NativeFs {
return this.cloudRootUri !== null
}

private async changing<T>(change: () => Promise<T>): Promise<T> {
this.writeGeneration += 1
try {
return await change()
} finally {
this.writeGeneration += 1
}
}

private loc(relPath: string): FsLocation {
if (this.cloudRootUri) {
return {
Expand Down Expand Up @@ -232,6 +249,10 @@ export class NativeFs {

/** Verified native copy into a new staging path. */
async copyForSync(fromRel: string, toRel: string, byteLength: number): Promise<void> {
await this.changing(() => this.copyForSyncOnce(fromRel, toRel, byteLength))
}

private async copyForSyncOnce(fromRel: string, toRel: string, byteLength: number): Promise<void> {
if (fromRel === toRel || !Number.isSafeInteger(byteLength) || byteLength < 0) throw new Error('Invalid Cloud file copy.')
if (await this.statVerified(toRel) !== null) throw new Error('Cloud copy destination already exists.')
const source = await this.readForSync(fromRel, false)
Expand All @@ -245,6 +266,10 @@ export class NativeFs {

/** Stream a signed Cloud revision to a staging file; native verifies before resolving. */
async download(options: { url: string; headers: Record<string, string>; to: string; byteLength: number; sha256: string }): Promise<void> {
await this.changing(() => this.downloadOnce(options))
}

private async downloadOnce(options: { url: string; headers: Record<string, string>; to: string; byteLength: number; sha256: string }): Promise<void> {
if (await this.statVerified(options.to) !== null) throw new Error('Cloud download destination already exists.')
const parent = options.to.includes('/') ? options.to.slice(0, options.to.lastIndexOf('/')) : ''
if (parent) await this.mkdir(parent)
Expand All @@ -266,7 +291,7 @@ export class NativeFs {
const destination = this.fileUri(options.to)
if (!source || !destination) throw new Error('Vault root is not resolved.')
const { header, objectKey, aad } = options
return CloudFiles.decryptFile({ source, destination, header, objectKey, aad })
return await this.changing(() => CloudFiles.decryptFile({ source, destination, header, objectKey, aad }))
}

/**
Expand All @@ -280,7 +305,7 @@ export class NativeFs {
if (!destination) throw new Error('Vault root is not resolved.')
const discard = () => this.deleteFile(temporary).catch(() => {})
try {
const sealed = await CloudFiles.encryptFile({ source: sourceUri, destination, ...keys })
const sealed = await this.changing(() => CloudFiles.encryptFile({ source: sourceUri, destination, ...keys }))
return { ...sealed, uri: destination, discard }
} catch (error) {
await discard()
Expand All @@ -294,21 +319,21 @@ export class NativeFs {
* a note folder surfaces in listAssets and the sidebar as a stray file.
*/
async writeText(relPath: string, body: string): Promise<void> {
await Filesystem.writeFile({
await this.changing(() => Filesystem.writeFile({
...this.loc(relPath),
data: body,
encoding: Encoding.UTF8,
recursive: true
})
}))
}

/** Write binary data from a base64 string (Capacitor's native transport). */
async writeBase64(relPath: string, base64Data: string): Promise<void> {
await Filesystem.writeFile({
await this.changing(() => Filesystem.writeFile({
...this.loc(relPath),
data: base64Data,
recursive: true
})
}))
}

async statOrNull(relPath: string): Promise<StatResult | null> {
Expand Down Expand Up @@ -367,40 +392,40 @@ export class NativeFs {
}

async mkdir(relPath: string): Promise<void> {
await Filesystem.mkdir({ ...this.loc(relPath), recursive: true }).catch((err) => {
await this.changing(() => Filesystem.mkdir({ ...this.loc(relPath), recursive: true }).catch((err) => {
const msg = String((err as Error)?.message ?? err)
if (!/exist/i.test(msg)) throw err
})
}))
}

async rename(fromRel: string, toRel: string): Promise<void> {
const from = this.loc(fromRel)
const to = this.loc(toRel)
await Filesystem.rename({
await this.changing(() => Filesystem.rename({
from: from.path,
to: to.path,
directory: from.directory,
toDirectory: to.directory
})
}))
}

async copy(fromRel: string, toRel: string): Promise<void> {
const from = this.loc(fromRel)
const to = this.loc(toRel)
await Filesystem.copy({
await this.changing(() => Filesystem.copy({
from: from.path,
to: to.path,
directory: from.directory,
toDirectory: to.directory
})
}))
}

async deleteFile(relPath: string): Promise<void> {
await Filesystem.deleteFile(this.loc(relPath))
await this.changing(() => Filesystem.deleteFile(this.loc(relPath)))
}

async rmdir(relPath: string): Promise<void> {
await Filesystem.rmdir({ ...this.loc(relPath), recursive: true })
await this.changing(() => Filesystem.rmdir({ ...this.loc(relPath), recursive: true }))
}

async exists(relPath: string): Promise<boolean> {
Expand Down
171 changes: 136 additions & 35 deletions src/bridge/vault-fs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,19 @@ interface CachedMeta {
meta: NoteMeta
}

interface CachedTasks {
mtime: number
size: number
title: string
folder: NoteFolder
tasks: VaultTask[]
}

/** How long the listings of a finished walk stay shareable. Long enough for
* the resync a rescan emits to reuse the rescan's walk, short enough that no
* external change waits on it in a way a person could notice. */
const SHARED_LISTING_GRACE_MS = 1000

interface SearchCandidate {
path: string
title: string
Expand Down Expand Up @@ -176,6 +189,25 @@ export class MobileVault {
private metaCacheDirty = false
private metaCachePersistTimer: ReturnType<typeof setTimeout> | null = null
private settingsCache: VaultSettings | null = null
/**
* Directory listings the tree walks (notes, folders, assets) share. app-core
* asks for notes and folders together on every refresh, and a new note or a
* return to the app starts two refreshes at once, so the vault was listed
* four times over before a new note could open. On Android, where a picked
* folder answers each listing with a document-provider query, that was 12
* to 15 seconds on a budget phone (zennotesandroid#128); the walks here are
* the same code and multiplied the same way on a large vault.
* A listing is shared only with walks that overlap it or start
* within SHARED_LISTING_GRACE_MS of the last one ending (the resync a rescan
* emits), and never across a write this vault makes.
*/
private sharedListings: { generation: number; byDir: Map<string, Promise<FileInfo[]>> } | null = null
private walksInFlight = 0
private sharedListingsTimer: ReturnType<typeof setTimeout> | null = null
/** Each note's parsed tasks, reused while its listed mtime and size hold,
* the test the meta cache uses. Without it every task scan read every note
* in the vault, and adding one task runs a full scan (zennotesandroid#128). */
private taskCache = new Map<string, CachedTasks>()

constructor(name: string, cloudRootUri: string | null = null) {
this.name = name
Expand Down Expand Up @@ -418,6 +450,7 @@ export class MobileVault {

private invalidateMeta(relPath: string): void {
this.metaCache.delete(relPath)
this.taskCache.delete(relPath)
this.metaCacheDirty = true
}

Expand Down Expand Up @@ -495,20 +528,56 @@ export class MobileVault {
// Listing
// -------------------------------------------------------------------

async listNotes(): Promise<NoteMeta[]> {
const out: NoteMeta[] = []
const settings = await this.getVaultSettings()
const hidden = hiddenPrimaryRootNames(settings.systemFolderPaths)
for (const folder of FOLDERS) {
const topRel = await this.folderRootRel(folder)
if (folder !== 'inbox' || topRel !== '') {
const stat = await this.fs.statOrNull(topRel)
if (!stat || stat.type !== 'directory') continue
/** Run one tree walk with listings shared between overlapping walks. */
private async sharingListings<T>(walk: () => Promise<T>): Promise<T> {
this.walksInFlight += 1
if (this.sharedListingsTimer) {
clearTimeout(this.sharedListingsTimer)
this.sharedListingsTimer = null
}
try {
return await walk()
} finally {
this.walksInFlight -= 1
if (this.walksInFlight === 0) {
this.sharedListingsTimer = setTimeout(() => {
this.sharedListingsTimer = null
if (this.walksInFlight === 0) this.sharedListings = null
}, SHARED_LISTING_GRACE_MS)
}
await this.walkNotes(topRel, folder, topRel === '' || topRel === undefined, hidden, out)
}
this.scheduleMetaCachePersist()
return out
}

private walkReaddir(dirRel: string): Promise<FileInfo[]> {
const generation = this.fs.writeGeneration
if (this.sharedListings?.generation !== generation) {
this.sharedListings = { generation, byDir: new Map() }
}
const byDir = this.sharedListings.byDir
let listing = byDir.get(dirRel)
if (!listing) {
listing = this.fs.readdir(dirRel)
byDir.set(dirRel, listing)
}
return listing
}

async listNotes(): Promise<NoteMeta[]> {
return await this.sharingListings(async () => {
const out: NoteMeta[] = []
const settings = await this.getVaultSettings()
const hidden = hiddenPrimaryRootNames(settings.systemFolderPaths)
for (const folder of FOLDERS) {
const topRel = await this.folderRootRel(folder)
if (folder !== 'inbox' || topRel !== '') {
const stat = await this.fs.statOrNull(topRel)
if (!stat || stat.type !== 'directory') continue
}
await this.walkNotes(topRel, folder, topRel === '' || topRel === undefined, hidden, out)
}
this.scheduleMetaCachePersist()
return out
})
}

private async walkNotes(
Expand All @@ -518,7 +587,7 @@ export class MobileVault {
hidden: Set<string>,
out: NoteMeta[]
): Promise<void> {
const entries = await this.fs.readdir(dirRel)
const entries = await this.walkReaddir(dirRel)
for (const [index, entry] of entries.entries()) {
const childRel = dirRel ? `${dirRel}/${entry.name}` : entry.name
if (entry.type === 'directory') {
Expand All @@ -540,19 +609,21 @@ export class MobileVault {
}

async listFolders(): Promise<FolderEntry[]> {
const out: FolderEntry[] = []
const settings = await this.getVaultSettings()
const hidden = hiddenPrimaryRootNames(settings.systemFolderPaths)
for (const folder of FOLDERS) {
const topRel = await this.folderRootRel(folder)
const isPrimaryRoot = folder === 'inbox' && topRel === ''
if (!isPrimaryRoot) {
const stat = await this.fs.statOrNull(topRel)
if (!stat || stat.type !== 'directory') continue
return await this.sharingListings(async () => {
const out: FolderEntry[] = []
const settings = await this.getVaultSettings()
const hidden = hiddenPrimaryRootNames(settings.systemFolderPaths)
for (const folder of FOLDERS) {
const topRel = await this.folderRootRel(folder)
const isPrimaryRoot = folder === 'inbox' && topRel === ''
if (!isPrimaryRoot) {
const stat = await this.fs.statOrNull(topRel)
if (!stat || stat.type !== 'directory') continue
}
await this.walkFolders(topRel, folder, '', isPrimaryRoot, hidden, out)
}
await this.walkFolders(topRel, folder, '', isPrimaryRoot, hidden, out)
}
return out
return out
})
}

private async walkFolders(
Expand All @@ -563,7 +634,7 @@ export class MobileVault {
hidden: Set<string>,
out: FolderEntry[]
): Promise<void> {
const entries = await this.fs.readdir(dirRel)
const entries = await this.walkReaddir(dirRel)
for (const [index, entry] of entries.entries()) {
if (entry.type !== 'directory') continue
if (entry.name.startsWith('.')) continue
Expand All @@ -577,14 +648,16 @@ export class MobileVault {
}

async listAssets(): Promise<AssetMeta[]> {
const out: AssetMeta[] = []
await this.walkAssets('', true, out)
out.sort((a, b) => b.updatedAt - a.updatedAt || a.name.localeCompare(b.name))
return out
return await this.sharingListings(async () => {
const out: AssetMeta[] = []
await this.walkAssets('', true, out)
out.sort((a, b) => b.updatedAt - a.updatedAt || a.name.localeCompare(b.name))
return out
})
}

private async walkAssets(dirRel: string, isRoot: boolean, out: AssetMeta[]): Promise<void> {
const entries = await this.fs.readdir(dirRel)
const entries = await this.walkReaddir(dirRel)
for (const [index, entry] of entries.entries()) {
if (entry.name.startsWith('.')) continue
const childRel = dirRel ? `${dirRel}/${entry.name}` : entry.name
Expand Down Expand Up @@ -1252,15 +1325,43 @@ export class MobileVault {
const { excluded, included } = await this.tasksFolderLists()
const notes = await this.listNotes()
const out: VaultTask[] = []
const scanned = new Set<string>()
for (const note of notes) {
if (note.folder === 'trash') continue
if (isPathExcludedFromTasks(note.path, excluded, included)) continue
if (!isMarkdownPath(note.path)) continue
scanned.add(note.path)
const cached = this.taskCache.get(note.path)
if (
cached &&
cached.mtime === note.updatedAt &&
cached.size === note.size &&
cached.title === note.title &&
cached.folder === note.folder
) {
out.push(...cached.tasks)
continue
}
const body = await this.fs.readTextOrNull(note.path)
if (body === null) continue
out.push(
...this.parseAllTasks(body, { path: note.path, title: note.title, folder: note.folder })
)
if (body === null) {
this.taskCache.delete(note.path)
continue
}
const tasks = this.parseAllTasks(body, { path: note.path, title: note.title, folder: note.folder })
// A listing without a usable stamp cannot vouch for the next read.
if (Number.isFinite(note.updatedAt) && Number.isFinite(note.size)) {
this.taskCache.set(note.path, {
mtime: note.updatedAt,
size: note.size,
title: note.title,
folder: note.folder,
tasks
})
}
out.push(...tasks)
}
for (const path of this.taskCache.keys()) {
if (!scanned.has(path)) this.taskCache.delete(path)
}
return out
}
Expand Down
Loading
Loading