From 66a28f9b9a21084ccec6284fdb6218856f92ac82 Mon Sep 17 00:00:00 2001 From: plur9 Date: Fri, 23 Oct 2026 19:32:47 +0200 Subject: [PATCH 1/3] fix(update,snapshot): core modules, modules left detached, root not in registry - Modules shipped inside the root repo (gtd, research, outbox, ...) are core: update reports them as such instead of 'Not a git repository' and no longer counts them as failed; snapshot records them as source 'core' and restore/diff skip them. - snapshot restore puts modules and spaces ON their branch at the snapshot commit (switch -C, upstream set) instead of a detached HEAD. - update moves an already-detached module back to its default branch before pulling, and refuses without changing anything if commits would be left behind. - update makes sure the transport registry names the root repo (), creating a minimal repositories.yaml on a machine that has none and saying so plainly; passes --root to the transport. - snapshot create records the root repo's source; --with-registry carries the private registry files, written on restore only where absent. Co-Authored-By: Claude Opus 5.5 --- src/lib/gitbranch.ts | 94 ++++++++++ src/lib/module.ts | 83 ++++++++- src/lib/snapshot.ts | 112 ++++++++++-- src/lib/sync.ts | 77 +++++++- src/lib/upgrade.ts | 39 ++-- tests/second-machine.test.ts | 343 +++++++++++++++++++++++++++++++++++ 6 files changed, 702 insertions(+), 46 deletions(-) create mode 100644 src/lib/gitbranch.ts create mode 100644 tests/second-machine.test.ts diff --git a/src/lib/gitbranch.ts b/src/lib/gitbranch.ts new file mode 100644 index 0000000..c696b89 --- /dev/null +++ b/src/lib/gitbranch.ts @@ -0,0 +1,94 @@ +/** + * Keeping clones ON a branch. + * + * `snapshot restore` used to `git checkout `, which leaves a detached + * HEAD, and every later `git pull` then fails with "You are not currently on a + * branch". On a real second-machine install that was 27 of 37 modules failing + * the first `datacore update` after a successful restore. + * + * Two operations live here: put a fresh clone on its branch at an exact commit + * (restore), and move an already-detached clone back onto its default branch + * (update) -- the second only when no commit would be left behind. + */ +import { execFileSync } from './exec' + +function git(cwd: string, args: string[], timeout = 60000): { ok: boolean; out: string; err: string } { + try { + const out = execFileSync('git', args, { cwd, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'], timeout }) + return { ok: true, out: out.trim(), err: '' } + } catch (e) { + const x = e as { stdout?: string | Buffer; stderr?: string | Buffer; message?: string } + return { ok: false, out: String(x.stdout ?? '').trim(), err: String(x.stderr ?? x.message ?? '').trim() } + } +} + +/** The branch origin's HEAD points at, else main or master if origin has one. */ +export function defaultBranch(path: string): string | null { + const head = git(path, ['symbolic-ref', '--short', 'refs/remotes/origin/HEAD']) + if (head.ok && head.out.startsWith('origin/')) return head.out.slice('origin/'.length) + for (const b of ['main', 'master']) { + if (git(path, ['rev-parse', '--verify', '--quiet', `refs/remotes/origin/${b}`]).ok) return b + } + return null +} + +export function isDetached(path: string): boolean { + return !git(path, ['symbolic-ref', '-q', 'HEAD']).ok +} + +/** + * Put a FRESH clone on `branch` (or the default branch) at `commit`, tracking + * origin. Only for a clone made a moment ago: `switch -C` moves the local + * branch, which on a fresh clone holds nothing that is not also on origin. + */ +export function checkoutOnBranch(path: string, commit: string, branch?: string): { ok: boolean; warning?: string } { + const b = branch || defaultBranch(path) + if (!b) { + const r = git(path, ['checkout', '-q', commit]) + return r.ok + ? { ok: true, warning: `${path}: origin has no default branch, left at ${commit.slice(0, 7)} (detached)` } + : { ok: false, warning: r.err } + } + const sw = git(path, ['switch', '-q', '-C', b, commit]) + if (!sw.ok) return { ok: false, warning: sw.err } + if (git(path, ['rev-parse', '--verify', '--quiet', `refs/remotes/origin/${b}`]).ok) { + git(path, ['branch', '-q', `--set-upstream-to=origin/${b}`, b]) + } + return { ok: true } +} + +/** + * Move a detached clone back onto its default branch so `git pull` works. + * + * Never discards work: if HEAD holds commits that are on neither the local nor + * the remote default branch, nothing is changed and the error says how to keep + * them. A dirty working tree that would be overwritten makes `git switch` + * refuse, and that refusal is passed on unchanged. + */ +export function reattach(path: string): { ok: true; switched: boolean; branch?: string } | { ok: false; error: string } { + if (!isDetached(path)) return { ok: true, switched: false } + git(path, ['fetch', '-q', 'origin']) + const b = defaultBranch(path) + if (!b) return { ok: false, error: `${path} is on a detached HEAD and origin has no default branch to return to` } + + const remote = `refs/remotes/origin/${b}` + const local = `refs/heads/${b}` + const hasLocal = git(path, ['rev-parse', '--verify', '--quiet', local]).ok + const contained = git(path, ['merge-base', '--is-ancestor', 'HEAD', remote]).ok + || (hasLocal && git(path, ['merge-base', '--is-ancestor', 'HEAD', local]).ok) + if (!contained) { + const n = git(path, ['rev-list', '--count', 'HEAD', '--not', remote, ...(hasLocal ? [local] : [])]) + const count = n.ok ? n.out : 'some' + return { + ok: false, + error: `detached HEAD with ${count} commit(s) not on any branch (${b} does not contain them); nothing was changed. ` + + `Keep them with: git -C ${path} switch -c , then merge into ${b}`, + } + } + + const sw = hasLocal + ? git(path, ['switch', '-q', b]) + : git(path, ['switch', '-q', '-c', b, '--track', `origin/${b}`]) + if (!sw.ok) return { ok: false, error: `could not switch from detached HEAD to ${b}: ${sw.err.split('\n')[0]}` } + return { ok: true, switched: true, branch: b } +} diff --git a/src/lib/module.ts b/src/lib/module.ts index d22be3c..aa544ba 100644 --- a/src/lib/module.ts +++ b/src/lib/module.ts @@ -6,8 +6,9 @@ import { runModulePostInstall } from './init' */ import { existsSync, readdirSync, readFileSync, rmSync } from 'fs' -import { join, basename } from 'path' +import { join, basename, relative, isAbsolute } from 'path' import { execFileSync } from './exec' +import { reattach } from './gitbranch' import type { ModuleInfo } from '../types' import { dataDir } from './paths' @@ -350,33 +351,95 @@ export class ModuleDependencyError extends Error { } } +/** + * How a module is kept up to date. + * + * repo -- its own git clone (`.git` inside it): `git pull` updates it. + * core -- shipped inside the root Datacore repo (gtd, research, outbox, ...): + * no clone of its own, updated whenever the root repo is. + * local -- neither: a directory nobody tracks, so nothing can update it. + * + * Core modules used to be reported as "Not a git repository" and counted as + * failed updates -- 10 false failures on every update of a healthy install. + * The test is whether the root repo tracks files under the module's + * directory, which is the same rule the root .gitignore states. + */ +export function moduleKind(modulePath: string, root: string = DATA_DIR()): 'repo' | 'core' | 'local' { + if (existsSync(join(modulePath, '.git'))) return 'repo' + const rel = relative(root, modulePath) + if (!rel || rel.startsWith('..') || isAbsolute(rel)) return 'local' + try { + const out = execFileSync('git', ['ls-files', '--', rel], { + cwd: root, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 15000, + }) as unknown as string + return String(out).trim() ? 'core' : 'local' + } catch { + return 'local' + } +} + +export interface ModuleUpdateResult { + name: string + updated: boolean + error?: string + /** Shipped inside the root repo: updated with Datacore, not on its own. */ + core?: boolean + /** Was on a detached HEAD and was moved back onto this branch first. */ + reattached?: string +} + /** * Update all modules or a specific module. */ -export function updateModules(name?: string): Array<{ name: string; updated: boolean; error?: string }> { +export function updateModules(name?: string): ModuleUpdateResult[] { const modules = listModules() - const results: Array<{ name: string; updated: boolean; error?: string }> = [] + const results: ModuleUpdateResult[] = [] const toUpdate = name ? modules.filter((m) => m.name === name) : modules for (const mod of toUpdate) { try { - // Check if it's a git repo - const gitDir = join(mod.path, '.git') - if (existsSync(gitDir)) { - execFileSync('git', ['pull'], { cwd: mod.path, stdio: 'pipe', timeout: 60000 }) - results.push({ name: mod.name, updated: true }) - } else { + const kind = moduleKind(mod.path) + if (kind === 'core') { + results.push({ name: mod.name, updated: false, core: true }) + continue + } + if (kind === 'local') { results.push({ name: mod.name, updated: false, error: 'Not a git repository' }) + continue } + // A module restored by an older CLI sits on a detached HEAD, where + // `git pull` cannot work. Return it to its branch first -- or stop, + // untouched, if that would strand commits. + const back = reattach(mod.path) + if (!back.ok) { + results.push({ name: mod.name, updated: false, error: back.error }) + continue + } + execFileSync('git', ['pull'], { cwd: mod.path, stdio: 'pipe', timeout: 60000 }) + results.push({ name: mod.name, updated: true, ...(back.switched ? { reattached: back.branch } : {}) }) } catch (err) { - results.push({ name: mod.name, updated: false, error: (err as Error).message }) + // git's own words, not "Command failed: git pull" (the summary shows one line). + const stderr = String((err as { stderr?: Buffer | string }).stderr ?? '').trim() + results.push({ name: mod.name, updated: false, error: stderr || (err as Error).message }) } } return results } +/** Counts for the update summary. Core modules are neither updated nor failed. */ +export function tallyModuleResults(results: ModuleUpdateResult[]): { updated: number; current: number; core: number; failed: number } { + const t = { updated: 0, current: 0, core: 0, failed: 0 } + for (const r of results) { + if (r.core) t.core++ + else if (r.updated) t.updated++ + else if (r.error) t.failed++ + else t.current++ + } + return t +} + /** * Remove a module. */ diff --git a/src/lib/snapshot.ts b/src/lib/snapshot.ts index 717dd91..6e87a6b 100644 --- a/src/lib/snapshot.ts +++ b/src/lib/snapshot.ts @@ -5,12 +5,14 @@ * pip's requirements.txt or npm's package-lock.json. */ -import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs' +import { existsSync, readFileSync, writeFileSync, mkdirSync, chmodSync } from 'fs' import { execSync } from 'child_process' -import { join, basename } from 'path' +import { execFileSync } from './exec' +import { join } from 'path' import { parse, stringify } from 'yaml' import { listSpaces } from './space' -import { listModules } from './module' +import { listModules, moduleKind } from './module' +import { checkoutOnBranch } from './gitbranch' import { checkDependencies, checkDatacore } from './dependency' import { loadConfig } from '../config' import { dataDir } from './paths' @@ -25,7 +27,7 @@ const LOCK_FILE = () => join(dataDir(), 'datacore.lock.yaml') export interface ModuleLock { name: string - source: string // git remote URL or "local" + source: string // git remote URL, "core" (ships inside the root repo), or "local" version?: string // from module.yaml commit?: string // git commit hash for exact reproducibility branch?: string // git branch @@ -57,10 +59,25 @@ export interface Snapshot { spaces: SpaceLock[] dependencies: DependencyLock[] + /** Where the root Datacore repo itself comes from (a fork, usually). */ + root?: { source?: string; commit?: string; branch?: string } + // Configuration (base only, no local overrides for privacy) settings?: Record + + /** + * The installation's PRIVATE registry files (repositories, infrastructure, + * principals), verbatim. Only with `snapshot create --with-registry`: they + * are gitignored for a reason, so carrying them is an explicit choice. A + * second machine needs them; without repositories.yaml the sync transport + * refuses every repo that is not a space. + */ + registry?: Record } +/** Private registry files a second machine needs and git does not carry. */ +export const PRIVATE_REGISTRY_FILES = ['repositories.yaml', 'infrastructure.yaml', 'principals.yaml'] as const + /** * Get git info for a directory. */ @@ -94,12 +111,18 @@ function getGitInfo(path: string): { remote?: string; commit?: string; branch?: /** * Create a snapshot of the current Datacore installation. */ -export function createSnapshot(options: { includeSettings?: boolean } = {}): Snapshot { - const { includeSettings = false } = options +export function createSnapshot(options: { includeSettings?: boolean; includeRegistry?: boolean } = {}): Snapshot { + const { includeSettings = false, includeRegistry = false } = options // Gather module info with git details const modules: ModuleLock[] = [] for (const mod of listModules()) { + // A core module has no clone of its own: it arrives with the root repo, + // so recording it as "local" made restore warn it "cannot restore" it. + if (moduleKind(mod.path) === 'core') { + modules.push({ name: mod.name, source: 'core', version: mod.version }) + continue + } const gitInfo = getGitInfo(mod.path) modules.push({ name: mod.name, @@ -144,6 +167,20 @@ export function createSnapshot(options: { includeSettings?: boolean } = {}): Sna dependencies, } + const rootGit = getGitInfo(DATA_DIR()) + if (rootGit.remote || rootGit.commit) { + snapshot.root = { source: rootGit.remote, commit: rootGit.commit, branch: rootGit.branch } + } + + if (includeRegistry) { + const registry: Record = {} + for (const f of PRIVATE_REGISTRY_FILES) { + const p = join(DATA_DIR(), '.datacore', 'registry', f) + if (existsSync(p)) registry[f] = readFileSync(p, 'utf-8') + } + if (Object.keys(registry).length) snapshot.registry = registry + } + // Optionally include settings (base only, not local) if (includeSettings) { const settingsPath = join(DATA_DIR(), '.datacore', 'settings.yaml') @@ -168,7 +205,9 @@ export function saveSnapshot(snapshot: Snapshot, path?: string): string { const content = stringify(snapshot, { lineWidth: 0, // Don't wrap lines }) - writeFileSync(lockPath, content) + // Private registry data inside: readable by the owner only. + writeFileSync(lockPath, content, snapshot.registry ? { mode: 0o600 } : undefined) + if (snapshot.registry) { try { chmodSync(lockPath, 0o600) } catch { /* not POSIX */ } } return lockPath } @@ -225,6 +264,7 @@ export function diffSnapshot(snapshot: Snapshot): SnapshotDiff { diff.modules.added.push(name) } else { const expected = snapshotModules.get(name)! + if (expected.source === 'core' || mod.source === 'core') continue if (expected.commit && mod.commit && expected.commit !== mod.commit) { diff.modules.changed.push({ name, @@ -235,7 +275,10 @@ export function diffSnapshot(snapshot: Snapshot): SnapshotDiff { } } - for (const name of snapshotModules.keys()) { + for (const [name, mod] of snapshotModules) { + // A missing core module means the root repo is behind, not that a module + // needs installing. + if (mod.source === 'core') continue if (!currentModules.has(name)) { diff.modules.removed.push(name) } @@ -280,6 +323,8 @@ export interface RestoreResult { modulesInstalled: string[] modulesFailed: Array<{ name: string; error: string }> spacesCreated: string[] + /** Private registry files written (only those absent here). */ + registryWritten: string[] warnings: string[] } @@ -292,9 +337,27 @@ export function restoreFromSnapshot( modulesInstalled: [], modulesFailed: [], spacesCreated: [], + registryWritten: [], warnings: [], } + // Private registry files: write only what is missing, never overwrite. + if (snapshot.registry && !dryRun) { + const dir = join(DATA_DIR(), '.datacore', 'registry') + for (const f of PRIVATE_REGISTRY_FILES) { + const text = snapshot.registry[f] + const p = join(dir, f) + if (typeof text !== 'string' || existsSync(p)) continue + try { + mkdirSync(dir, { recursive: true }) + writeFileSync(p, text, { mode: 0o600 }) + result.registryWritten.push(f) + } catch (err) { + result.warnings.push(`Could not write registry file ${f}: ${(err as Error).message}`) + } + } + } + // Install missing modules if (modules) { const currentModules = new Set(listModules().map((m) => m.name)) @@ -304,6 +367,12 @@ export function restoreFromSnapshot( continue } + // Core modules come with the root repo; they are never cloned on their own. + if (mod.source === 'core') { + result.warnings.push(`Core module ${mod.name} is missing: it ships with Datacore itself; run: datacore update`) + continue + } + if (mod.source === 'local') { result.warnings.push(`Module ${mod.name} was local, cannot restore`) continue @@ -322,13 +391,18 @@ export function restoreFromSnapshot( const modulePath = join(modulesDir, mod.name) - // Clone at specific commit if available - execSync(`git clone ${mod.source} "${modulePath}"`, { stdio: 'pipe' }) + // execFile, not a shell string: the lock file comes from another + // machine, and a URL in it must not be able to run anything. + execFileSync('git', ['clone', '--quiet', mod.source, modulePath], { stdio: 'pipe', timeout: 300000 }) + // ON the branch at the snapshot commit -- never a detached HEAD, which + // made every later `git pull` fail ("not currently on a branch"). if (mod.commit) { - execSync(`git checkout ${mod.commit}`, { cwd: modulePath, stdio: 'pipe' }) + const on = checkoutOnBranch(modulePath, mod.commit, mod.branch) + if (!on.ok) throw new Error(on.warning || `could not check out ${mod.commit}`) + if (on.warning) result.warnings.push(on.warning) } else if (mod.branch) { - execSync(`git checkout ${mod.branch}`, { cwd: modulePath, stdio: 'pipe' }) + execFileSync('git', ['switch', '--quiet', mod.branch], { cwd: modulePath, stdio: 'pipe' }) } result.modulesInstalled.push(mod.name) @@ -347,6 +421,13 @@ export function restoreFromSnapshot( for (const space of snapshot.spaces) { if (currentSpaces.has(space.name)) { + // The collision a second machine hit: init had made a TEMPLATE space + // of the same name, so the real one was silently never restored. + if (space.source && !existsSync(join(DATA_DIR(), space.name, '.git'))) { + result.warnings.push( + `Space ${space.name} already exists here but is not a git clone, so the snapshot's copy was NOT restored. ` + + `If it is an unused template, move it aside (mv ${space.name} ${space.name}.template) and run restore again.`) + } continue } @@ -359,10 +440,13 @@ export function restoreFromSnapshot( if (space.source) { try { const spacePath = join(DATA_DIR(), space.name) - execSync(`git clone ${space.source} "${spacePath}"`, { stdio: 'pipe' }) + execFileSync('git', ['clone', '--quiet', space.source, spacePath], { stdio: 'pipe', timeout: 300000 }) + // Spaces too: on their default branch at the snapshot commit. if (space.commit) { - execSync(`git checkout ${space.commit}`, { cwd: spacePath, stdio: 'pipe' }) + const on = checkoutOnBranch(spacePath, space.commit) + if (!on.ok) result.warnings.push(`Space ${space.name} cloned but not moved to ${space.commit.slice(0, 7)}: ${on.warning}`) + else if (on.warning) result.warnings.push(on.warning) } result.spacesCreated.push(space.name) diff --git a/src/lib/sync.ts b/src/lib/sync.ts index 6b80424..7e2f38b 100644 --- a/src/lib/sync.ts +++ b/src/lib/sync.ts @@ -6,8 +6,9 @@ import { execSync } from 'child_process' import { execFileSync, spawn } from './exec' -import { existsSync } from 'fs' -import { join, basename } from 'path' +import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs' +import { join, basename, resolve } from 'path' +import { parseDocument } from 'yaml' import { listSpaces } from './space' import { dataDir, datacoreLib } from './paths' import { findPython } from './python' @@ -38,6 +39,70 @@ function findLedgerTransport(): { python: string; script: string } | null { * stderr to be guessed at. Surface it verbatim; that naming is the whole point * of routing through it. */ +/** + * The transport's arguments. `--root` is passed explicitly: the transport + * decides "this is the root repo" (registry key ``) by comparing paths, + * and left to its own default it guesses `~/Data`, which is wrong for any + * installation pointed elsewhere with DATACORE_ROOT. + */ +export function transportArgs(script: string, repoPath: string, root: string): string[] { + return [script, 'converge', '--space', repoPath, '--root', root] +} + +/** + * Make sure the transport's registry names the root repo. + * + * `repositories.yaml` is private and gitignored, and a snapshot does not carry + * it, so a second machine has none -- and the transport then refuses the root + * repo with "repository not in registry/repositories.yaml", although nothing + * is wrong. Spaces declare themselves by marker; the root has no marker, so + * it needs this one entry. Never rewrites an entry that is there, and never + * touches a file it cannot parse. + */ +export function ensureRootRegistered(root: string): 'created' | 'added' | 'present' | 'skipped' { + if (!existsSync(join(root, '.git'))) return 'skipped' + const dir = join(root, '.datacore', 'registry') + const file = join(dir, 'repositories.yaml') + const entry = { category: 'code', host: 'github', transport: 'git', repo: 'datacore' } + try { + if (!existsSync(file)) { + mkdirSync(dir, { recursive: true }) + writeFileSync(file, [ + '# Repository registry for the sync transport (private, gitignored).', + '# Created by `datacore update` because this machine had none. It names', + '# only the Datacore repo itself; spaces register by their own marker.', + '# Copy the full file from your other machine to add code repos.', + 'repositories:', + ' :', + ...Object.entries(entry).map(([k, v]) => ` ${k}: ${v}`), + '', + ].join('\n')) + return 'created' + } + const doc = parseDocument(readFileSync(file, 'utf-8')) + if (doc.errors.length) return 'skipped' + const repos = doc.get('repositories') as { has?: (k: string) => boolean } | undefined + if (repos && typeof repos.has === 'function' && repos.has('')) return 'present' + if (repos !== undefined && (typeof repos !== 'object' || repos === null || typeof repos.has !== 'function')) return 'skipped' + doc.setIn(['repositories', ''], doc.createNode(entry)) + writeFileSync(file, doc.toString()) + return 'added' + } catch { + return 'skipped' + } +} + +/** One plain sentence for the update output, or null when there is nothing to say. */ +export function registryNotice(r: ReturnType): string | null { + if (r === 'created') { + return 'This machine had no .datacore/registry/repositories.yaml (it is private, so git does not carry it). ' + + 'Created one that names only the Datacore repo. To bring over the full registry, copy it from your other machine, ' + + 'or there run `datacore snapshot create --settings --with-registry` and restore that snapshot here.' + } + if (r === 'added') return 'Added the Datacore repo itself () to .datacore/registry/repositories.yaml.' + return null +} + function convergeRepo( t: { python: string; script: string }, repoPath: string, @@ -45,7 +110,7 @@ function convergeRepo( try { const out = execFileSync( t.python, - [t.script, 'converge', '--space', repoPath], + transportArgs(t.script, repoPath, resolve(DATA_DIR())), { encoding: 'utf-8', timeout: 180000, stdio: ['pipe', 'pipe', 'pipe'] }, ) const parsed = JSON.parse(out) as { ok: boolean; reason?: string } @@ -183,10 +248,14 @@ export function getRepoStatus(path: string): SyncStatus | null { * is the safe failure. Silently restoring the removed behaviour would defeat * the fix on every machine that has not upgraded yet. */ -export function pullAll(options: { stream?: boolean } = {}): SyncResult[] { +export function pullAll(options: { stream?: boolean; onRegistry?: (notice: string) => void } = {}): SyncResult[] { const results: SyncResult[] = [] const repos = getGitRepos() const transport = findLedgerTransport() + if (transport) { + const notice = registryNotice(ensureRootRegistered(DATA_DIR())) + if (notice) options.onRegistry?.(notice) + } for (const repo of repos) { if (options.stream) { diff --git a/src/lib/upgrade.ts b/src/lib/upgrade.ts index 5df0363..4a8bcde 100644 --- a/src/lib/upgrade.ts +++ b/src/lib/upgrade.ts @@ -16,7 +16,7 @@ import { mergeUpstream, upgradeMcpPackages } from './selfupdate' import { richOutput } from './output' import { createInterface } from 'readline' import { detectPlatform, getInstallCommand, type Platform } from './platform' -import { updateModules, listModules } from './module' +import { updateModules, listModules, tallyModuleResults } from './module' import { dataDir } from './paths' import { createSnapshot, saveSnapshot } from './snapshot' import { pullAll } from './sync' @@ -184,7 +184,13 @@ function updateRepos( console.log(`${c.bold}Repositories${c.reset}`) } - const pullResults = pullAll({ stream: false }) + const pullResults = pullAll({ + stream: false, + onRegistry: (notice) => { + if (isTTY) console.log(` ${c.dim}${notice}${c.reset}`) + result.warnings.push(notice) + }, + }) // A forked install's `origin` is the user's own fork, which GitHub never // syncs, so the pull above brings in nothing from upstream. Merge upstream @@ -244,27 +250,24 @@ function updateModulesStep( } const results = updateModules() - - let updated = 0 - let upToDate = 0 - let failed = 0 + const t = tallyModuleResults(results) for (const r of results) { - if (r.updated) { - updated++ - result.updated.push(`Module: ${r.name}`) - } else if (r.error) { - failed++ - result.warnings.push(`Module ${r.name}: ${r.error}`) - } else { - upToDate++ - } + if (r.updated) result.updated.push(`Module: ${r.name}${r.reattached ? ` (back on ${r.reattached} from a detached HEAD)` : ''}`) + else if (r.error) result.warnings.push(`Module ${r.name}: ${r.error.split('\n')[0]}`) } if (isTTY) { - if (updated > 0) console.log(` ${c.green}✓${c.reset} ${updated} module(s) updated`) - if (upToDate > 0) console.log(` ${c.green}✓${c.reset} ${upToDate} module(s) already current`) - if (failed > 0) console.log(` ${c.yellow}⚠${c.reset} ${failed} module(s) failed to update`) + if (t.updated > 0) console.log(` ${c.green}✓${c.reset} ${t.updated} module(s) updated`) + if (t.current > 0) console.log(` ${c.green}✓${c.reset} ${t.current} module(s) already current`) + // Core modules ship inside the root repo and moved with it above. + if (t.core > 0) console.log(` ${c.green}✓${c.reset} ${t.core} core module(s) ${c.dim}(part of Datacore, updated with it)${c.reset}`) + if (t.failed > 0) { + console.log(` ${c.yellow}⚠${c.reset} ${t.failed} module(s) failed to update`) + for (const r of results.filter((x) => x.error)) { + console.log(` ${c.dim}${r.name}: ${r.error!.split('\n')[0]}${c.reset}`) + } + } console.log() } } diff --git a/tests/second-machine.test.ts b/tests/second-machine.test.ts new file mode 100644 index 0000000..d818f0d --- /dev/null +++ b/tests/second-machine.test.ts @@ -0,0 +1,343 @@ +/** + * Setting up Datacore on a second machine from a snapshot. + * + * Every case here comes from a real headless Ubuntu 24.04 install made from a + * snapshot of an existing installation (2026-10-23). The result worked, but + * `datacore update` afterwards reported "37 module(s) failed to update": + * + * - 10 modules that ship inside the root Datacore repo (gtd, research, + * outbox, ...) were reported as "Not a git repository" and counted as + * failures. They are core: tracked by the root repo, updated with it. + * - 27 modules failed `git pull` because `snapshot restore` had checked each + * one out at its exact commit, leaving it on a detached HEAD. + * - The root repo was reported as "not in registry" because a new machine + * has no repositories.yaml (it is gitignored, and the snapshot does not + * carry it). + * + * These tests use real git repositories, so they test what git does, not what + * we believe it does. + */ +import { describe, it, expect, beforeEach, afterEach } from 'bun:test' +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync } from 'fs' +import { execFileSync } from 'child_process' +import { tmpdir } from 'os' +import { join } from 'path' +import { parse } from 'yaml' + +const GIT_TIMEOUT = 60000 +const git = (cwd: string, ...args: string[]) => + execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'user.email=t@example.com', '-c', 'user.name=T', + '-c', 'init.defaultBranch=main', ...args], + { cwd, encoding: 'utf-8', stdio: 'pipe' }).trim() + +let dir = '' +let root = '' + +/** A root Datacore repo with one core module (tracked by the root). */ +function makeRoot(): string { + dir = mkdtempSync(join(tmpdir(), 'dc-second-machine-')) + root = join(dir, 'Data') + mkdirSync(join(root, '.datacore', 'modules', 'gtd'), { recursive: true }) + writeFileSync(join(root, '.datacore', 'modules', 'gtd', 'module.yaml'), 'name: gtd\nversion: 1.0.0\n') + // Own-repo modules are gitignored by the root, as in the real repo. + writeFileSync(join(root, '.gitignore'), '.datacore/modules/news/\n.datacore/modules/scratch/\n[1-9]-*/\n0-personal/\n') + git(dir, 'init', '-q', '-b', 'main', root) + git(root, 'add', '.') + git(root, 'commit', '-q', '-m', 'root') + process.env.DATACORE_ROOT = root + return root +} + +/** A bare upstream repo for a module, with two commits. Returns [url, firstCommit, secondCommit]. */ +function makeUpstream(name: string, manifest = true): [string, string, string] { + const work = join(dir, `${name}-work`) + git(dir, 'init', '-q', '-b', 'main', work) + if (manifest) writeFileSync(join(work, 'module.yaml'), `name: ${name}\nversion: 0.1.0\n`) + writeFileSync(join(work, 'README.md'), 'v1\n') + git(work, 'add', '.'); git(work, 'commit', '-q', '-m', 'v1') + const first = git(work, 'rev-parse', 'HEAD') + writeFileSync(join(work, 'README.md'), 'v2\n') + git(work, 'commit', '-q', '-am', 'v2') + const second = git(work, 'rev-parse', 'HEAD') + const bare = join(dir, `${name}.git`) + git(dir, 'clone', '-q', '--bare', work, bare) + return [bare, first, second] +} + +/** Push one more commit to a bare upstream. */ +function advance(name: string): string { + const work = join(dir, `${name}-work`) + writeFileSync(join(work, 'README.md'), 'v3\n') + git(work, 'commit', '-q', '-am', 'v3') + git(work, 'push', '-q', join(dir, `${name}.git`), 'main') + return git(work, 'rev-parse', 'HEAD') +} + +function cloneModule(url: string, name: string): string { + const path = join(root, '.datacore', 'modules', name) + git(dir, 'clone', '-q', url, path) + git(path, 'config', 'user.email', 't@example.com') + git(path, 'config', 'user.name', 'T') + git(path, 'config', 'core.hooksPath', '/dev/null') + return path +} + +beforeEach(() => { dir = ''; root = '' }) +afterEach(() => { + delete process.env.DATACORE_ROOT + if (dir) rmSync(dir, { recursive: true, force: true }) +}) + +// ─── 1. Core modules ───────────────────────────────────────────────────────── + +describe('core modules (shipped inside the root repo)', () => { + it('are recognised as core, not as "Not a git repository"', async () => { + makeRoot() + const { moduleKind } = await import('../src/lib/module') + expect(moduleKind(join(root, '.datacore', 'modules', 'gtd'))).toBe('core') + }, GIT_TIMEOUT) + + it('an untracked module with no repo of its own is still local', async () => { + makeRoot() + mkdirSync(join(root, '.datacore', 'modules', 'scratch')) + writeFileSync(join(root, '.datacore', 'modules', 'scratch', 'module.yaml'), 'name: scratch\n') + const { moduleKind } = await import('../src/lib/module') + expect(moduleKind(join(root, '.datacore', 'modules', 'scratch'))).toBe('local') + }, GIT_TIMEOUT) + + it('update reports them as core and does not count them as failed', async () => { + makeRoot() + const { updateModules, tallyModuleResults } = await import('../src/lib/module') + const results = updateModules() + const gtd = results.find((r) => r.name === 'gtd')! + expect(gtd.core).toBe(true) + expect(gtd.error).toBeUndefined() + const t = tallyModuleResults(results) + expect(t.failed).toBe(0) + expect(t.core).toBe(1) + }, GIT_TIMEOUT) + + it('snapshot create records them as core, and restore neither clones nor fails them', async () => { + makeRoot() + const { createSnapshot, restoreFromSnapshot } = await import('../src/lib/snapshot') + const snap = createSnapshot() + const gtd = snap.modules.find((m) => m.name === 'gtd')! + expect(gtd.source).toBe('core') + expect(gtd.commit).toBeUndefined() + + // A core module missing on the target is not a clone job and not a failure. + rmSync(join(root, '.datacore', 'modules', 'gtd'), { recursive: true, force: true }) + const r = restoreFromSnapshot(snap, { spaces: false }) + expect(r.modulesFailed).toEqual([]) + expect(r.modulesInstalled).toEqual([]) + expect(r.warnings.join('\n')).not.toContain('was local') + }, GIT_TIMEOUT) + + it('snapshot diff does not report core modules as changed or removed', async () => { + makeRoot() + const { createSnapshot, diffSnapshot } = await import('../src/lib/snapshot') + const snap = createSnapshot() + const d = diffSnapshot(snap) + expect(d.modules.changed).toEqual([]) + expect(d.modules.removed).toEqual([]) + }, GIT_TIMEOUT) + + it('snapshot create records where the root repo comes from', async () => { + makeRoot() + git(root, 'remote', 'add', 'origin', 'https://example.com/me/datacore.git') + const { createSnapshot } = await import('../src/lib/snapshot') + const snap = createSnapshot() + expect(snap.root?.source).toBe('https://example.com/me/datacore.git') + expect(snap.root?.branch).toBe('main') + }, GIT_TIMEOUT) +}) + +// ─── 3. Detached modules ───────────────────────────────────────────────────── + +describe('snapshot restore leaves modules on their branch', () => { + it('restores a module at the snapshot commit, on its branch, tracking origin', async () => { + makeRoot() + const [url, first] = makeUpstream('news') + const { restoreFromSnapshot } = await import('../src/lib/snapshot') + const r = restoreFromSnapshot({ + version: '1.0', created: '', cliVersion: '', platform: '', dependencies: [], spaces: [], + modules: [{ name: 'news', source: url, commit: first, branch: 'main' }], + }) + expect(r.modulesFailed).toEqual([]) + const path = join(root, '.datacore', 'modules', 'news') + expect(git(path, 'rev-parse', 'HEAD')).toBe(first) + expect(git(path, 'branch', '--show-current')).toBe('main') + expect(git(path, 'rev-parse', '--abbrev-ref', 'main@{upstream}')).toBe('origin/main') + }, GIT_TIMEOUT) + + it('a module restored with no branch recorded lands on the default branch', async () => { + makeRoot() + const [url, first] = makeUpstream('news') + const { restoreFromSnapshot } = await import('../src/lib/snapshot') + restoreFromSnapshot({ + version: '1.0', created: '', cliVersion: '', platform: '', dependencies: [], spaces: [], + modules: [{ name: 'news', source: url, commit: first }], + }) + const path = join(root, '.datacore', 'modules', 'news') + expect(git(path, 'branch', '--show-current')).toBe('main') + expect(git(path, 'rev-parse', 'HEAD')).toBe(first) + }, GIT_TIMEOUT) + + it('restores a space on its branch too', async () => { + makeRoot() + const [url, first] = makeUpstream('1-team', false) + const { restoreFromSnapshot } = await import('../src/lib/snapshot') + const r = restoreFromSnapshot({ + version: '1.0', created: '', cliVersion: '', platform: '', dependencies: [], modules: [], + spaces: [{ name: '1-team', number: 1, type: 'team', source: url, commit: first }], + }) + expect(r.spacesCreated).toEqual(['1-team']) + const path = join(root, '1-team') + expect(git(path, 'branch', '--show-current')).toBe('main') + expect(git(path, 'rev-parse', 'HEAD')).toBe(first) + }, GIT_TIMEOUT) + + it('the next update pulls a restored module forward', async () => { + makeRoot() + const [url, first] = makeUpstream('news') + const { restoreFromSnapshot } = await import('../src/lib/snapshot') + restoreFromSnapshot({ + version: '1.0', created: '', cliVersion: '', platform: '', dependencies: [], spaces: [], + modules: [{ name: 'news', source: url, commit: first, branch: 'main' }], + }) + const third = advance('news') + const { updateModules } = await import('../src/lib/module') + const res = updateModules().find((r) => r.name === 'news')! + expect(res.error).toBeUndefined() + expect(res.updated).toBe(true) + expect(git(join(root, '.datacore', 'modules', 'news'), 'rev-parse', 'HEAD')).toBe(third) + }, GIT_TIMEOUT) +}) + +describe('update repairs a module that is already detached', () => { + it('switches back to the default branch and pulls', async () => { + makeRoot() + const [url, first] = makeUpstream('news') + const path = cloneModule(url, 'news') + git(path, 'checkout', '-q', '--detach', first) // what restore used to do + const third = advance('news') + const { updateModules } = await import('../src/lib/module') + const res = updateModules().find((r) => r.name === 'news')! + expect(res.error).toBeUndefined() + expect(git(path, 'branch', '--show-current')).toBe('main') + expect(git(path, 'rev-parse', 'HEAD')).toBe(third) + }, GIT_TIMEOUT) + + it('refuses, changes nothing, and says how to keep them when local commits would be lost', async () => { + makeRoot() + const [url, first] = makeUpstream('news') + const path = cloneModule(url, 'news') + git(path, 'checkout', '-q', '--detach', first) + writeFileSync(join(path, 'mine.md'), 'my work\n') + git(path, 'add', '.'); git(path, 'commit', '-q', '-m', 'mine') + const head = git(path, 'rev-parse', 'HEAD') + const { updateModules } = await import('../src/lib/module') + const res = updateModules().find((r) => r.name === 'news')! + expect(res.updated).toBe(false) + expect(res.error).toContain('not on any branch') + expect(res.error).toContain('git -C') + expect(git(path, 'rev-parse', 'HEAD')).toBe(head) + expect(existsSync(join(path, 'mine.md'))).toBe(true) + }, GIT_TIMEOUT) +}) + +// ─── 2. Root repo registry ─────────────────────────────────────────────────── + +describe('root repo in the transport registry', () => { + it('a new machine with no registry gets one that names the root as ', async () => { + makeRoot() + const { ensureRootRegistered } = await import('../src/lib/sync') + expect(ensureRootRegistered(root)).toBe('created') + const reg = parse(readFileSync(join(root, '.datacore', 'registry', 'repositories.yaml'), 'utf-8')) + expect(reg.repositories[''].category).toBe('code') + }, GIT_TIMEOUT) + + it('leaves a registry that already lists untouched', async () => { + makeRoot() + mkdirSync(join(root, '.datacore', 'registry'), { recursive: true }) + const text = '# mine\nrepositories:\n :\n category: code\n repo: datacore\n' + writeFileSync(join(root, '.datacore', 'registry', 'repositories.yaml'), text) + const { ensureRootRegistered } = await import('../src/lib/sync') + expect(ensureRootRegistered(root)).toBe('present') + expect(readFileSync(join(root, '.datacore', 'registry', 'repositories.yaml'), 'utf-8')).toBe(text) + }, GIT_TIMEOUT) + + it('adds to a registry that lacks it, keeping the other entries and comments', async () => { + makeRoot() + mkdirSync(join(root, '.datacore', 'registry'), { recursive: true }) + writeFileSync(join(root, '.datacore', 'registry', 'repositories.yaml'), + '# keep me\nrepositories:\n 1-team:\n category: knowledge\n') + const { ensureRootRegistered } = await import('../src/lib/sync') + expect(ensureRootRegistered(root)).toBe('added') + const text = readFileSync(join(root, '.datacore', 'registry', 'repositories.yaml'), 'utf-8') + expect(text).toContain('# keep me') + const reg = parse(text) + expect(reg.repositories['1-team'].category).toBe('knowledge') + expect(reg.repositories[''].category).toBe('code') + }, GIT_TIMEOUT) + + it('does not touch a registry it cannot parse', async () => { + makeRoot() + mkdirSync(join(root, '.datacore', 'registry'), { recursive: true }) + writeFileSync(join(root, '.datacore', 'registry', 'repositories.yaml'), 'repositories: [oops\n') + const { ensureRootRegistered } = await import('../src/lib/sync') + expect(ensureRootRegistered(root)).toBe('skipped') + expect(readFileSync(join(root, '.datacore', 'registry', 'repositories.yaml'), 'utf-8')).toBe('repositories: [oops\n') + }, GIT_TIMEOUT) + + it('tells the transport which directory is the root', async () => { + makeRoot() + const { transportArgs } = await import('../src/lib/sync') + expect(transportArgs('/x/ledger_transport.py', root, root)).toEqual( + ['/x/ledger_transport.py', 'converge', '--space', root, '--root', root]) + }) + it('says plainly when this machine had no registry and what to do about it', async () => { + const { registryNotice } = await import('../src/lib/sync') + const n = registryNotice('created')! + expect(n).toContain('repositories.yaml') + expect(n).toContain('other machine') + expect(n).toContain('--with-registry') + expect(registryNotice('present')).toBeNull() + }) +}) + +describe('private registry files travel only when asked', () => { + it('snapshot create leaves them out by default and carries them with includeRegistry', async () => { + makeRoot() + mkdirSync(join(root, '.datacore', 'registry'), { recursive: true }) + writeFileSync(join(root, '.datacore', 'registry', 'repositories.yaml'), 'repositories: {}\n') + const { createSnapshot } = await import('../src/lib/snapshot') + expect(createSnapshot().registry).toBeUndefined() + expect(createSnapshot({ includeRegistry: true }).registry).toEqual({ 'repositories.yaml': 'repositories: {}\n' }) + }, GIT_TIMEOUT) + + it('restore writes a carried registry file only where none exists', async () => { + makeRoot() + mkdirSync(join(root, '.datacore', 'registry'), { recursive: true }) + writeFileSync(join(root, '.datacore', 'registry', 'principals.yaml'), 'mine\n') + const { restoreFromSnapshot } = await import('../src/lib/snapshot') + const r = restoreFromSnapshot({ + version: '1.0', created: '', cliVersion: '', platform: '', dependencies: [], spaces: [], modules: [], + registry: { 'repositories.yaml': 'repositories: {}\n', 'principals.yaml': 'theirs\n' }, + }) + expect(r.registryWritten).toEqual(['repositories.yaml']) + expect(readFileSync(join(root, '.datacore', 'registry', 'principals.yaml'), 'utf-8')).toBe('mine\n') + }, GIT_TIMEOUT) + + it('restore warns when a template space is in the way of the snapshot copy', async () => { + makeRoot() + mkdirSync(join(root, '0-personal', 'org'), { recursive: true }) + const { restoreFromSnapshot } = await import('../src/lib/snapshot') + const r = restoreFromSnapshot({ + version: '1.0', created: '', cliVersion: '', platform: '', dependencies: [], modules: [], + spaces: [{ name: '0-personal', number: 0, type: 'personal', source: 'https://example.com/p.git' }], + }) + expect(r.spacesCreated).toEqual([]) + expect(r.warnings.join('\n')).toContain('NOT restored') + }, GIT_TIMEOUT) +}) From f3201b4034285a9b53a4e212d758cc0ea7813804 Mon Sep 17 00:00:00 2001 From: plur9 Date: Fri, 23 Oct 2026 19:32:48 +0200 Subject: [PATCH 2/3] fix(init,doctor): check python venv first, keep module lock file, doctor on Linux - checkVenvSupport builds a throwaway venv before init relies on one and names the exact fix (sudo apt install python3.X-venv on Debian/Ubuntu); the venv failure message in update carries the same hint. - ensureModuleDeps installs with npm ci from the tracked lock file, puts the lock back byte for byte, and skips the install when nothing changed, so update no longer leaves the root repo dirty. - doctor skips the Claude Desktop check where there is no Claude Desktop (Linux, or app not installed), reads .mcp.json from the data dir, prints the ledger rows, and only says 'Missing recommended dependencies' when one is missing. Co-Authored-By: Claude Opus 5.5 --- src/lib/dependency.ts | 62 +++++++++++++++++++--------- src/lib/python-env.ts | 89 ++++++++++++++++++++++++++++++++++++++-- src/types/index.ts | 3 +- tests/python-env.test.ts | 47 ++++++++++++++++++++- 4 files changed, 175 insertions(+), 26 deletions(-) diff --git a/src/lib/dependency.ts b/src/lib/dependency.ts index f222572..0a7278c 100644 --- a/src/lib/dependency.ts +++ b/src/lib/dependency.ts @@ -12,6 +12,7 @@ import { detectPlatform, getInstallCommand, getPlatformInfo, type Platform } fro import { dataDir as resolveDataDir } from './paths' import { classifyVerifyFailure, findPython, PYTHON_MIN_VERSION } from './python' import { listSpaces } from './space' +import { harnessConfigs } from './harness' function commandExists(cmd: string): boolean { // Same lookup init uses, so doctor and the installer cannot disagree about @@ -249,32 +250,39 @@ export function checkCodePermissions(): { enableAll: boolean; mcpAllowed: boolea return { enableAll, mcpAllowed } } -export function checkMcpConfig(): { claudeDesktop: boolean; claudeCode: boolean } { - // Check Claude Desktop config - const home = homeDir() - const desktopPaths = [ - join(home, 'Library', 'Application Support', 'Claude', 'claude_desktop_config.json'), // macOS - join(home, '.config', 'claude', 'claude_desktop_config.json'), // Linux - ] - - let claudeDesktop = false - for (const p of desktopPaths) { - try { - if (existsSync(p)) { - const content = JSON.parse(readFileSync(p, 'utf-8')) - if (content?.mcpServers?.datacore) { - claudeDesktop = true - break +/** + * Is Datacore wired into Claude Desktop and Claude Code? + * + * `claudeDesktop` is null when the question does not apply: there is no + * Claude Desktop for Linux, and elsewhere the app may simply not be installed. + * Reporting "✗ Claude Desktop config" on a headless server sent people + * looking for a problem that does not exist. + */ +export function checkMcpConfig(opts: { home?: string; platform?: NodeJS.Platform; dataDir?: string } = {}): + { claudeDesktop: boolean | null; claudeCode: boolean } { + const home = opts.home ?? homeDir() + const platform = opts.platform ?? process.platform + const data = opts.dataDir ?? resolveDataDir() + + let claudeDesktop: boolean | null = null + if (platform !== 'linux') { + const desktop = harnessConfigs(home, platform, process.env).find((h) => h.name === 'claude-desktop')! + if (existsSync(desktop.detect)) { + claudeDesktop = false + try { + if (existsSync(desktop.path)) { + const content = JSON.parse(readFileSync(desktop.path, 'utf-8')) + if (content?.mcpServers?.datacore) claudeDesktop = true } + } catch { + // Ignore parse errors } - } catch { - // Ignore parse errors } } - // Check Claude Code .mcp.json + // Check Claude Code .mcp.json -- in the data dir, which is not always ~/Data. let claudeCode = false - const mcpJsonPath = join(home, 'Data', '.mcp.json') + const mcpJsonPath = join(data, '.mcp.json') try { if (existsSync(mcpJsonPath)) { const content = JSON.parse(readFileSync(mcpJsonPath, 'utf-8')) @@ -289,6 +297,20 @@ export function checkMcpConfig(): { claudeDesktop: boolean; claudeCode: boolean return { claudeDesktop, claudeCode } } +/** + * The closing line of `datacore doctor`. Every non-ready status used to print + * "Missing recommended dependencies" -- including a degraded ledger, with + * every dependency present. + */ +export function doctorVerdict(status: DoctorResult['status']): { level: 'ok' | 'warn' | 'error'; message: string } { + switch (status) { + case 'ready': return { level: 'ok', message: 'System ready for Datacore' } + case 'missing_required': return { level: 'error', message: 'Missing required dependencies' } + case 'missing_recommended': return { level: 'warn', message: 'Missing recommended dependencies' } + case 'ledger_degraded': return { level: 'warn', message: 'Ledger checks failed (see Ledger above)' } + } +} + export function checkDependencies(): DependencyCheck[] { const platform = detectPlatform() diff --git a/src/lib/python-env.ts b/src/lib/python-env.ts index 9eaff3c..96c5404 100644 --- a/src/lib/python-env.ts +++ b/src/lib/python-env.ts @@ -12,8 +12,9 @@ * tools instead of 45+. */ import { execFileSync } from './exec' -import { existsSync, rmSync } from 'fs' +import { existsSync, rmSync, readFileSync, writeFileSync, statSync, mkdtempSync } from 'fs' import { join } from 'path' +import { tmpdir } from 'os' export type Runner = (cmd: string, args: string[], opts?: { cwd?: string; timeout?: number }) => { ok: boolean; stderr?: string } @@ -73,7 +74,11 @@ export function ensureDatacoreVenv(dataDir: string, run: Runner = defaultRun, return { python: null, warnings: [`Could not create ${venvDir}: no Python >= 3.10 found. Install one (e.g. brew install python@3.12), then run: datacore update`] } } const made = run(base, ['-m', 'venv', '--system-site-packages', venvDir], { timeout: 120000 }) - if (!made.ok) return { python: null, warnings: [`Could not create ${venvDir} with ${base}: ${made.stderr ?? ''}`.trim()] } + if (!made.ok) { + const hint = /ensurepip|venv/i.test(made.stderr ?? '') + ? ` Fix: ${venvInstallHint(process.platform, versionOf(base), readOsRelease())}, then run: datacore update` : '' + return { python: null, warnings: [`Could not create ${venvDir} with ${base}: ${made.stderr ?? ''}`.trim() + hint] } + } } const req = join(dataDir, '.datacore', 'lib', 'requirements.txt') if (existsSync(req)) { @@ -85,15 +90,91 @@ export function ensureDatacoreVenv(dataDir: string, run: Runner = defaultRun, export function ensureModuleDeps(dataDir: string, run: Runner = defaultRun): string[] { const modulesDir = join(dataDir, '.datacore', 'modules') - if (!existsSync(join(modulesDir, 'package.json'))) return [] + const pkg = join(modulesDir, 'package.json') + if (!existsSync(pkg)) return [] + const lock = join(modulesDir, 'package-lock.json') + const hasLock = existsSync(lock) + + // Nothing changed since the last install: skip it (and its network round). + // npm writes node_modules/.package-lock.json on every install. + const hidden = join(modulesDir, 'node_modules', '.package-lock.json') + try { + if (existsSync(hidden)) { + const installed = statSync(hidden).mtimeMs + if (installed >= statSync(pkg).mtimeMs && (!hasLock || installed >= statSync(lock).mtimeMs)) return [] + } + } catch { /* fall through to an install */ } + + // The lock file is TRACKED by the root repo. `npm install` rewrote it on + // every update, leaving the root dirty, which then blocks the next merge + // from upstream. `npm ci` installs from the lock and never writes it; and + // whatever runs, the lock is put back byte for byte. + const before = hasLock ? readFileSync(lock) : null // --ignore-scripts: module tools import only @datacore-one/mcp/runtime (zod, // js-yaml, interpreter selection). The package's native sqlite build is never // loaded from here, and a host without a prebuilt binary or toolchain would // otherwise fail the whole install and lose every module tool. - const r = run('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: modulesDir, timeout: 300000 }) + const flags = ['--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund'] + let r = hasLock ? run('npm', ['ci', ...flags], { cwd: modulesDir, timeout: 300000 }) : { ok: false } + if (!r.ok) r = run('npm', ['install', ...flags], { cwd: modulesDir, timeout: 300000 }) + if (before) { + try { + if (!readFileSync(lock).equals(before)) writeFileSync(lock, before) + } catch { /* a lock we cannot restore is git's to show, not ours to hide */ } + } return r.ok ? [] : [`npm install in ${modulesDir} failed, so module tools will not load: ${r.stderr ?? ''}`.trim()] } +/** + * The command that makes `python3 -m venv` work on this machine. + * + * Debian and Ubuntu split venv's ensurepip into a separate package named after + * the exact Python version (python3.12-venv on Ubuntu 24.04); without it + * `python3 -m venv` fails and Datacore's MCP server cannot start. + */ +export function venvInstallHint(platform: NodeJS.Platform, version: [number, number] | null, osRelease = ''): string { + const v = version ? `${version[0]}.${version[1]}` : '3' + if (platform === 'darwin') return `brew install python@${version && version[1] >= 10 ? v : '3.12'}` + if (platform === 'win32') return 'winget install --id Python.Python.3.12 --exact' + if (/^(ID|ID_LIKE)=.*\b(debian|ubuntu)\b/m.test(osRelease)) { + return version && version[1] >= 10 ? `sudo apt install python${v}-venv` : 'sudo apt install python3 python3-venv' + } + return `install Python >= 3.10 with its venv module (your distribution's python${v === '3' ? '3' : v}-venv or python3 package)` +} + +function readOsRelease(): string { + try { return readFileSync('/etc/os-release', 'utf-8') } catch { return '' } +} + +/** + * Can this machine build Datacore's virtual environment? Checked by init + * BEFORE anything relies on it, by building a throwaway venv the same way + * ensureDatacoreVenv will (with pip: missing ensurepip is the usual failure, + * and `import venv` alone succeeds without it). + */ +export function checkVenvSupport(run: Runner = defaultRun, versionOf: VersionOf = defaultVersionOf, + platform: NodeJS.Platform = process.platform, osRelease: string = readOsRelease()): + { ok: boolean; python?: string; version?: [number, number]; error?: string; fix?: string } { + const base = BASE_CANDIDATES.find(bin => atLeast310(versionOf(bin))) + if (!base) { + return { ok: false, error: 'no Python >= 3.10 found', fix: venvInstallHint(platform, null, osRelease) } + } + const version = versionOf(base) ?? undefined + let probe = '' + try { + probe = mkdtempSync(join(tmpdir(), 'datacore-venv-check-')) + const made = run(base, ['-m', 'venv', join(probe, 'venv')], { timeout: 120000 }) + if (made.ok) return { ok: true, python: base, version } + return { + ok: false, python: base, version, + error: (made.stderr || 'python -m venv failed').split('\n').find(l => l.trim()) ?? 'python -m venv failed', + fix: venvInstallHint(platform, version ?? null, osRelease), + } + } finally { + if (probe) rmSync(probe, { recursive: true, force: true }) + } +} + /** * Whether `datacore init` succeeded. Errors count: a failed core Python install * was pushed to `errors` while success was computed from the other two lists, diff --git a/src/types/index.ts b/src/types/index.ts index 6bf4288..5d80bd3 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -73,7 +73,8 @@ export interface DoctorResult { datacoreExists: boolean dependencies: DependencyCheck[] status: 'ready' | 'missing_required' | 'missing_recommended' | 'ledger_degraded' - mcpConfig?: { claudeDesktop: boolean; claudeCode: boolean } + /** claudeDesktop is null where it does not apply (Linux, or the app is not installed). */ + mcpConfig?: { claudeDesktop: boolean | null; claudeCode: boolean } codePermissions?: { enableAll: boolean; mcpAllowed: boolean } ledger?: LedgerCheck[] /** The installation's own gaps (identity, principals, event logs, inbox, jobs), diff --git a/tests/python-env.test.ts b/tests/python-env.test.ts index a5c30f5..8fe42d8 100644 --- a/tests/python-env.test.ts +++ b/tests/python-env.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'bun:test' -import { mkdtempSync, mkdirSync, writeFileSync } from 'fs' +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, utimesSync } from 'fs' import { tmpdir } from 'os' import { join } from 'path' import { ensureDatacoreVenv, ensureModuleDeps, venvPython } from '../src/lib/python-env' @@ -98,3 +98,48 @@ describe('ensureModuleDeps native builds (review #5)', () => { expect(calls[0]).toContain('--ignore-scripts') }) }) + +/** + * `datacore update` rewrote the root repo's tracked + * .datacore/modules/package-lock.json on every run (seen on a fresh Ubuntu + * 24.04 install), leaving the root repo dirty, which then blocks the next + * merge from upstream. The lock file is input, not output. + */ +describe('ensureModuleDeps leaves the tracked lock file alone', () => { + const LOCK = '{"name":"modules","lockfileVersion":3,"packages":{}}\n' + function withLock(): string { + const d = fixture() + mkdirSync(join(d, '.datacore', 'modules'), { recursive: true }) + writeFileSync(join(d, '.datacore', 'modules', 'package.json'), '{"dependencies":{}}') + writeFileSync(join(d, '.datacore', 'modules', 'package-lock.json'), LOCK) + return d + } + it('installs from the lock with npm ci when there is one', () => { + const d = withLock() + const calls: string[][] = [] + expect(ensureModuleDeps(d, (cmd, args) => { calls.push([cmd, ...args]); return { ok: true } })).toEqual([]) + expect(calls[0]).toEqual(['npm', 'ci', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund']) + }) + it('leaves package-lock.json byte-identical even if npm rewrites it', () => { + const d = withLock() + const lock = join(d, '.datacore', 'modules', 'package-lock.json') + // Make ci fail so the fallback install runs, and have it rewrite the lock as npm does. + const run = (_cmd: string, args: string[]) => { + if (args[0] === 'ci') return { ok: false, stderr: 'lock out of sync' } + writeFileSync(lock, '{"rewritten":true}\n') + return { ok: true } + } + expect(ensureModuleDeps(d, run)).toEqual([]) + expect(readFileSync(lock, 'utf-8')).toBe(LOCK) + }) + it('does not reinstall when node_modules is already newer than package.json and the lock', () => { + const d = withLock() + const nm = join(d, '.datacore', 'modules', 'node_modules') + mkdirSync(nm, { recursive: true }) + const hidden = join(nm, '.package-lock.json') + writeFileSync(hidden, '{}') + const later = new Date(Date.now() + 60_000) + utimesSync(hidden, later, later) + expect(ensureModuleDeps(d, () => { throw new Error('should not run') })).toEqual([]) + }) +}) From 31b46b12d24501734134704f80cfcf4b740e7195 Mon Sep 17 00:00:00 2001 From: plur9 Date: Fri, 23 Oct 2026 19:32:49 +0200 Subject: [PATCH 3/3] feat(init): --from-snapshot for a second machine; space help lists join - datacore init --from-snapshot clones the root from the snapshot's source, restores its spaces and modules before anything else, installs the modules' dependencies, and never creates a template space the snapshot provides (a failed clone is an error, not a template). - Interactive init asks once whether Datacore already runs on another machine and explains snapshot create --settings there and --from-snapshot here. - space --help lists join and audit (both existed, neither was listed) and points to the snapshot route for a whole machine; init and snapshot help describe the new flags. - Fixes a pre-existing typecheck error in ensureRecallHook. Co-Authored-By: Claude Opus 5.5 --- src/help.ts | 33 ++++- src/index.ts | 60 ++++++-- src/lib/init.ts | 214 ++++++++++++++++++++++++++--- src/routing.ts | 3 +- tests/second-machine-setup.test.ts | 169 +++++++++++++++++++++++ 5 files changed, 446 insertions(+), 33 deletions(-) create mode 100644 tests/second-machine-setup.test.ts diff --git a/src/help.ts b/src/help.ts index 71921c4..d1ead50 100644 --- a/src/help.ts +++ b/src/help.ts @@ -107,6 +107,15 @@ Actions:`) console.log(` ${action.padEnd(18)} ${getActionDescription(resource as Resource, action)}`) } + if (resource === 'space') { + // `space join` clones ONE space. A whole second machine (every space and + // module) is the snapshot route, which nothing used to point to. + console.log(` +Setting up another machine with ALL your spaces and modules: + on the existing machine: datacore snapshot create --settings + on the new machine: datacore init --from-snapshot datacore.lock.yaml`) + } + console.log(` For detailed help on an action: datacore ${resource} --help @@ -173,6 +182,10 @@ Options: --skip-checks Skip dependency checks --print-questions Print the setup questions as JSON and exit --answers Answer the questions from a JSON file + --from-snapshot + Set up a second machine from a snapshot of an existing + one: clones its spaces and modules, and never creates + a template space the snapshot provides Environment: DATACORE_ROOT Installation directory (overrides ~/Data) @@ -194,6 +207,11 @@ Examples: # Custom location datacore init --path ~/my-data + # Second machine: on the first one run + # datacore snapshot create --settings (writes ~/Data/datacore.lock.yaml) + # copy that file here, then + datacore init --from-snapshot datacore.lock.yaml + # Agent asks, then answers on the user's behalf datacore init --print-questions > q.json datacore init --answers answers.json`) @@ -640,6 +658,9 @@ Arguments: Options: --settings Include base settings in snapshot + --with-registry Also include the PRIVATE registry files (repositories, + infrastructure, principals). They are gitignored, and a + second machine needs them; treat the file as private. --format json Output as JSON Examples: @@ -654,8 +675,16 @@ Usage: Description: Restores Datacore installation from a snapshot file: - - Installs missing modules (at exact commits) - - Creates missing spaces (if git sources available) + - Installs missing modules at their snapshot commits, ON their branch + (so the next datacore update can pull them) + - Clones missing spaces (if git sources available), also on their branch + - Skips core modules: they ship inside the Datacore repo itself + - Writes private registry files the snapshot carries (--with-registry), + only where this machine has none + + On a NEW machine use: datacore init --from-snapshot + (restore after a plain init would find init's template 0-personal and + skip the real one) Arguments: [path] Snapshot file (default: ~/Data/datacore.lock.yaml) diff --git a/src/index.ts b/src/index.ts index 42a529b..b701663 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,7 @@ import { showHelp, showResourceHelp, showActionHelp } from './help' import { detectFormat, output, success, info, warn, error as errorLog } from './format' import { CLIError } from './errors' import { readFileSync } from 'fs' -import { runDoctor } from './lib/dependency' +import { runDoctor, doctorVerdict } from './lib/dependency' import { loadConfig, getConfigValue, setConfigValue, getAllConfig } from './config' import { listSpaces, createSpace, initSpaceGit, detectForges, createSpaceRemote, joinSpace, auditSpace } from './lib/space' @@ -68,6 +68,19 @@ async function handleMeta( } } + const fromSnapshot = typeof flags['from-snapshot'] === 'string' ? flags['from-snapshot'] as string : undefined + if (flags['from-snapshot'] === true) { + errorLog('--from-snapshot needs a file: datacore init --from-snapshot datacore.lock.yaml') + process.exitCode = 1 + break + } + if (fromSnapshot && isInitialized() && !flags.force) { + // Already set up: restore adds what is missing without re-running setup. + info('Datacore is already set up here. To add the spaces and modules the snapshot has, run:') + info(` datacore snapshot restore ${fromSnapshot}`) + break + } + if (isInitialized() && !flags.force) { if (format === 'json') { output({ success: true, message: 'Datacore already initialized', hint: 'Use datacore update or --force to re-initialize' }, format) @@ -86,6 +99,7 @@ async function handleMeta( stream: format === 'human', verbose: flags.verbose === true, force: flags.force === true, + fromSnapshot, }) if (format === 'json') { @@ -217,7 +231,10 @@ async function handleMeta( } if (result.mcpConfig) { console.log('MCP Server:') - console.log(` ${result.mcpConfig.claudeDesktop ? '✓' : '✗'} Claude Desktop config`) + // null: no Claude Desktop on this machine (none exists for Linux) -- nothing to check. + if (result.mcpConfig.claudeDesktop !== null) { + console.log(` ${result.mcpConfig.claudeDesktop ? '✓' : '✗'} Claude Desktop config`) + } console.log(` ${result.mcpConfig.claudeCode ? '✓' : '✗'} Claude Code .mcp.json`) console.log() } @@ -239,13 +256,23 @@ async function handleMeta( console.log() } - if (result.status === 'ready') { - success('System ready for Datacore') - } else if (result.status === 'missing_required') { - errorLog('Missing required dependencies') + if (result.ledger?.length) { + console.log('Ledger:') + for (const l of result.ledger) { + console.log(` ${l.ok === true ? '✓' : l.ok === false ? '✗' : '?'} ${l.name}: ${l.detail}`) + if (l.ok !== true && l.fix) console.log(` Fix: ${l.fix}`) + } + console.log() + } + + const verdict = doctorVerdict(result.status) + if (verdict.level === 'ok') { + success(verdict.message) + } else if (verdict.level === 'error') { + errorLog(verdict.message) process.exitCode = 1 } else { - warn('Missing recommended dependencies') + warn(verdict.message) } } break @@ -616,8 +643,10 @@ async function handleResource( info('No modules to update') } else { for (const r of results) { - if (r.updated) { - success(`Updated: ${r.name}`) + if (r.core) { + info(`${r.name}: core module (updated with Datacore)`) + } else if (r.updated) { + success(`Updated: ${r.name}${r.reattached ? ` (was detached, back on ${r.reattached})` : ''}`) } else { warn(`${r.name}: ${r.error || 'not updated'}`) } @@ -875,10 +904,11 @@ async function handleResource( switch (action) { case 'create': { const includeSettings = flags.settings === true + const includeRegistry = flags['with-registry'] === true const outputPath = cmdArgs[0] info('Creating snapshot...') - const snapshot = createSnapshot({ includeSettings }) + const snapshot = createSnapshot({ includeSettings, includeRegistry }) const savedPath = saveSnapshot(snapshot, outputPath) if (format === 'json') { @@ -891,8 +921,12 @@ async function handleResource( if (includeSettings) { console.log(' Settings: included') } + if (snapshot.registry) { + console.log(` Private registry: ${Object.keys(snapshot.registry).join(', ')}`) + warn('This file now holds private registry data: copy it directly, do not commit or share it') + } console.log() - info('Share this file to replicate your setup') + info('On another machine: datacore init --from-snapshot ') } break } @@ -937,6 +971,10 @@ async function handleResource( errorLog(` ${name}: ${error}`) } } + if (result.registryWritten.length > 0) { + console.log('Registry files written:') + for (const name of result.registryWritten) success(` ${name}`) + } if (result.spacesCreated.length > 0) { console.log('Spaces created:') for (const name of result.spacesCreated) { diff --git a/src/lib/init.ts b/src/lib/init.ts index b1eca2f..9048245 100644 --- a/src/lib/init.ts +++ b/src/lib/init.ts @@ -27,7 +27,7 @@ import { AVAILABLE_MODULES, installModule, listModules, ModuleDependencyError } import { listSpaces } from './space' import { invokeAgent } from './agent' import { FALLBACK_NPM_PREFIX, resolveBinary, unresolvableMcpServers } from './upgrade' -import { createSnapshot, saveSnapshot } from './snapshot' +import { createSnapshot, saveSnapshot, loadSnapshot, restoreFromSnapshot, type Snapshot } from './snapshot' import { startOperation } from '../state' import { BANNER, INIT_COMPLETE, Spinner, sleep, section, completionSequence } from './animation' import { spawnBackground, type BackgroundJob } from './background' @@ -40,7 +40,7 @@ import { spawnBackground, type BackgroundJob } from './background' // success having created nothing — so every install bug had to be found by a // human on a clean laptop. Read at module load, which is correct for an env var // the caller sets before starting the process (see the init smoke test). -import { ensureDatacoreVenv, ensureModuleDeps, initSucceeded, pipInstallInto, venvPython } from './python-env' +import { checkVenvSupport, ensureDatacoreVenv, ensureModuleDeps, initSucceeded, pipInstallInto, venvPython } from './python-env' const DATA_DIR = process.env.DATACORE_ROOT || join(homeDir(), 'Data') const DATACORE_DIR = join(DATA_DIR, '.datacore') @@ -181,6 +181,15 @@ export interface InitOptions { * silently — every value was chosen by a human, just not at a prompt. */ answers?: InitAnswers + /** + * Set up a SECOND machine from `datacore snapshot create` on the first: + * clone the root from where the snapshot says, then its spaces and modules, + * and never create a template space the snapshot provides. Without this the + * only route was init (which writes a template 0-personal) followed by + * restore (which skips any space that exists), so the real personal space + * was silently not restored. + */ + fromSnapshot?: string } /** What the wizard would have asked. See `datacore init --print-questions`. */ @@ -508,7 +517,7 @@ function ensureRecallHook(isTTY: boolean | undefined, result: InitResult): void s.hooks = s.hooks ?? {} const groups = s.hooks.UserPromptSubmit ?? (s.hooks.UserPromptSubmit = []) if (groups.length === 0) groups.push({ hooks: [] }) - const group = groups[0] + const group = groups[0]! group.hooks = group.hooks ?? [] group.hooks.push({ type: 'command', @@ -1134,6 +1143,44 @@ async function configurePermissionPreference(result: InitResult): Promise } } +// ─── Second machine ─────────────────────────────────────────────────────────── + +/** What a user who already has Datacore elsewhere needs to know. */ +export const SECOND_MACHINE_HELP = [ + 'To bring your existing Datacore (all spaces and modules) to this machine:', + ' 1. On the machine you already use, run:', + ' datacore snapshot create --settings', + ' It writes ~/Data/datacore.lock.yaml (add --with-registry to carry the', + ' private registry files too; then keep the file private).', + ' 2. Copy that file to this machine (scp, a USB stick, anything).', + ' 3. Here, run:', + ' datacore init --from-snapshot datacore.lock.yaml', +] as const + +/** + * What step 5 does about the personal space. + * exists -- leave it alone + * missing-from-snapshot -- the snapshot provides it but it is not here + * (its clone failed): do NOT put a template in + * its place, or the real one is never restored + * create -- no snapshot provides one: create the template + */ +export function personalSpacePlan(dataDir: string, snapshot: Snapshot | null): 'exists' | 'missing-from-snapshot' | 'create' { + // Only a space with a git source can come from the snapshot; one without + // is skipped by restore, and a template is then the right thing. + const fromSnapshot = snapshot?.spaces.find((sp) => (sp.type === 'personal' || sp.number === 0) && sp.source) + if (fromSnapshot) return existsSync(join(dataDir, fromSnapshot.name)) ? 'exists' : 'missing-from-snapshot' + return existsSync(join(dataDir, '0-personal')) ? 'exists' : 'create' +} + +/** Clone the root from where the snapshot's root came from, before the defaults. */ +export function snapshotCloneUrls(snapshot: Snapshot | null, defaults: string[]): string[] { + const src = snapshot?.root?.source + if (!src) return defaults + const ssh = src.replace('https://github.com/', 'git@github.com:') + return [...new Set([src, ...(ssh !== src ? [ssh] : []), ...defaults])] +} + // ─── Public API ─────────────────────────────────────────────────────────────── /** @@ -1175,6 +1222,17 @@ export async function initDatacore(options: InitOptions = {}): Promise') + op.fail('stopped: second machine without a snapshot') + return result + } + snapshot = loadSnapshot(path) + if (!snapshot) { + result.errors.push(`Could not read snapshot ${path}. Nothing was changed.`) + op.fail('unreadable snapshot') + return result + } + } + } + // ═════════════════════════════════════════════════════════════════════ // STEP 1/10: ABOUT YOU // ═════════════════════════════════════════════════════════════════════ @@ -1416,6 +1502,33 @@ export async function initDatacore(options: InitOptions = {}): Promise (sp.type === 'personal' || sp.number === 0) && sp.source) + const personalPath = join(DATA_DIR, snapPersonal?.name ?? '0-personal') + if (plan === 'missing-from-snapshot') { + if (isTTY) console.log(` ${c.red}✗${c.reset} ${snapPersonal!.name} is in your snapshot but could not be cloned`) + result.errors.push( + `${snapPersonal!.name} is in your snapshot but could not be cloned (see the warnings). No template was created in its place. ` + + `Check access to ${snapPersonal!.source ?? 'its repository'}, then run: datacore snapshot restore ${options.fromSnapshot ?? ''}`) + } else if (plan === 'exists') { + if (isTTY) console.log(` ${c.green}✓${c.reset} Personal space ready (${basename(personalPath)}/)`) + if (!result.spacesCreated.includes(basename(personalPath))) result.spacesCreated.push(basename(personalPath)) } else { // Create personal space if (isTTY) console.log(` ${c.dim}Creating personal space...${c.reset}`) @@ -2126,7 +2285,10 @@ export async function initDatacore(options: InitOptions = {}): Promise = { - space: ['create', 'list'], + // join and audit have handlers; leaving them out here hid them from --help. + space: ['create', 'join', 'list', 'audit'], module: ['install', 'list', 'update', 'remove'], config: ['show', 'get', 'set'], nightshift: ['status', 'trigger', 'queue'], diff --git a/tests/second-machine-setup.test.ts b/tests/second-machine-setup.test.ts new file mode 100644 index 0000000..6a808b0 --- /dev/null +++ b/tests/second-machine-setup.test.ts @@ -0,0 +1,169 @@ +/** + * Second-machine setup, the non-git half: what init checks before it relies on + * Python, how it takes a snapshot instead of creating template spaces, what + * doctor says on a headless Linux server, and what `space --help` offers. + * + * From a real headless Ubuntu 24.04 install (2026-10-23): init went ahead + * without python3.12-venv so the MCP server could not start; init created a + * template 0-personal that then made `snapshot restore` skip the real one; + * doctor flagged a missing Claude Desktop config on a machine where Claude + * Desktop does not exist and ended "Missing recommended dependencies" with + * every dependency present; `space --help` promised a join it did not list. + */ +import { describe, it, expect, afterEach } from 'bun:test' +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { checkVenvSupport, venvInstallHint } from '../src/lib/python-env' +import { checkMcpConfig, doctorVerdict } from '../src/lib/dependency' +import { personalSpacePlan, snapshotCloneUrls, SECOND_MACHINE_HELP, initDatacore } from '../src/lib/init' +import { ACTIONS, parseArgs } from '../src/routing' +import { showResourceHelp } from '../src/help' +import type { Snapshot } from '../src/lib/snapshot' + +const dirs: string[] = [] +const tmp = (p: string) => { const d = mkdtempSync(join(tmpdir(), p)); dirs.push(d); return d } +afterEach(() => { while (dirs.length) rmSync(dirs.pop()!, { recursive: true, force: true }) }) + +const UBUNTU = 'NAME="Ubuntu"\nID=ubuntu\nID_LIKE=debian\nVERSION_ID="24.04"\n' + +// ─── 4. venv ──────────────────────────────────────────────────────────────── + +describe('init checks that Python can create a virtual environment', () => { + it('passes when `python -m venv` works', () => { + const r = checkVenvSupport((_c, _a) => ({ ok: true }), () => [3, 12], 'linux', UBUNTU) + expect(r.ok).toBe(true) + expect(r.python).toBe('python3.13') + }) + + it('names the exact apt package on Ubuntu when ensurepip is missing', () => { + const run = () => ({ ok: false, stderr: 'The virtual environment was not created successfully because ensurepip is not available.' }) + const r = checkVenvSupport(run, () => [3, 12], 'linux', UBUNTU) + expect(r.ok).toBe(false) + expect(r.fix).toBe('sudo apt install python3.12-venv') + }) + + it('actually builds a venv to test (ensurepip is what fails, not the venv module import)', () => { + const calls: string[][] = [] + checkVenvSupport((c, a) => { calls.push([c, ...a]); return { ok: true } }, () => [3, 12], 'linux', UBUNTU) + expect(calls[0]!.slice(0, 3)).toEqual(['python3.13', '-m', 'venv']) + expect(calls[0]).not.toContain('--without-pip') + }) + + it('install hints per platform', () => { + expect(venvInstallHint('linux', [3, 11], 'ID=debian\n')).toBe('sudo apt install python3.11-venv') + expect(venvInstallHint('darwin', [3, 12])).toContain('brew install python@3.12') + expect(venvInstallHint('linux', [3, 12], 'ID=fedora\n')).toContain('venv') + }) + + it('reports no Python >= 3.10 with an install command, not a crash', () => { + const r = checkVenvSupport(() => ({ ok: true }), () => [3, 8], 'linux', UBUNTU) + expect(r.ok).toBe(false) + expect(r.fix).toContain('apt') + }) +}) + +// ─── 5. init --from-snapshot ──────────────────────────────────────────────── + +const snap = (spaces: Snapshot['spaces'], root?: Snapshot['root']): Snapshot => ({ + version: '1.0', created: '', cliVersion: '', platform: '', modules: [], dependencies: [], spaces, root, +}) + +describe('init from a snapshot never creates a template space the snapshot provides', () => { + it('creates 0-personal from the template only without a snapshot that has one', () => { + const d = tmp('dc-init-') + expect(personalSpacePlan(d, null)).toBe('create') + expect(personalSpacePlan(d, snap([{ name: '1-team', number: 1, type: 'team', source: 'x' }]))).toBe('create') + }) + + it('a snapshot that provides 0-personal is never replaced by a template, even if its clone failed', () => { + const d = tmp('dc-init-') + const s = snap([{ name: '0-personal', number: 0, type: 'personal', source: 'https://example.com/p.git' }]) + expect(personalSpacePlan(d, s)).toBe('missing-from-snapshot') + mkdirSync(join(d, '0-personal')) + expect(personalSpacePlan(d, s)).toBe('exists') + }) + + it('clones the root from where the snapshot says it came from, first', () => { + const urls = snapshotCloneUrls(snap([], { source: 'https://example.com/me/datacore.git' }), + ['https://github.com/datacore-one/datacore.git']) + expect(urls[0]).toBe('https://example.com/me/datacore.git') + expect(urls).toContain('https://github.com/datacore-one/datacore.git') + }) + + it('refuses an unreadable snapshot before changing anything', async () => { + const r = await initDatacore({ fromSnapshot: '/nonexistent/datacore.lock.yaml', nonInteractive: true }) + expect(r.success).toBe(false) + expect(r.errors.join(' ')).toContain('/nonexistent/datacore.lock.yaml') + }) + + it('parses --from-snapshot ', () => { + const p = parseArgs(['init', '--from-snapshot', 'datacore.lock.yaml']) + expect(p).toMatchObject({ type: 'meta', command: 'init', flags: { 'from-snapshot': 'datacore.lock.yaml' } }) + }) + + it('tells a second-machine user both halves of the route', () => { + const text = SECOND_MACHINE_HELP.join('\n') + expect(text).toContain('datacore snapshot create --settings') + expect(text).toContain('datacore init --from-snapshot') + }) +}) + +// ─── doctor on a headless server ──────────────────────────────────────────── + +describe('doctor on a machine without Claude Desktop', () => { + it('does not check Claude Desktop on Linux', () => { + const home = tmp('dc-home-') + // Even a stray ~/.config/Claude must not turn the row back on: there is no Linux app. + mkdirSync(join(home, '.config', 'Claude'), { recursive: true }) + expect(checkMcpConfig({ home, platform: 'linux', dataDir: join(home, 'Data') }).claudeDesktop).toBeNull() + }) + + it('does not check it on a Mac without the app either', () => { + const home = tmp('dc-home-') + expect(checkMcpConfig({ home, platform: 'darwin', dataDir: join(home, 'Data') }).claudeDesktop).toBeNull() + }) + + it('still checks it where the app is installed', () => { + const home = tmp('dc-home-') + const dir = join(home, 'Library', 'Application Support', 'Claude') + mkdirSync(dir, { recursive: true }) + expect(checkMcpConfig({ home, platform: 'darwin', dataDir: join(home, 'Data') }).claudeDesktop).toBe(false) + writeFileSync(join(dir, 'claude_desktop_config.json'), '{"mcpServers":{"datacore":{}}}') + expect(checkMcpConfig({ home, platform: 'darwin', dataDir: join(home, 'Data') }).claudeDesktop).toBe(true) + }) + + it('reads .mcp.json from the data dir, not a hardcoded ~/Data', () => { + const home = tmp('dc-home-') + const data = join(home, 'elsewhere') + mkdirSync(data) + writeFileSync(join(data, '.mcp.json'), '{"mcpServers":{"datacore":{}}}') + expect(checkMcpConfig({ home, platform: 'linux', dataDir: data }).claudeCode).toBe(true) + }) + + it('says "missing recommended dependencies" only when one is missing', () => { + expect(doctorVerdict('ready').level).toBe('ok') + expect(doctorVerdict('missing_recommended').message).toContain('recommended') + expect(doctorVerdict('ledger_degraded').message).not.toContain('dependencies') + expect(doctorVerdict('missing_required').level).toBe('error') + }) +}) + +// ─── 6. space --help ──────────────────────────────────────────────────────── + +describe('space actions match the help text', () => { + it('lists join (it exists) and audit', () => { + expect(ACTIONS.space).toContain('join') + expect(ACTIONS.space).toContain('audit') + }) + it('space --help lists join and points a second machine to the snapshot route', () => { + const lines: string[] = [] + const orig = console.log + console.log = (...a: unknown[]) => { lines.push(a.join(' ')) } + try { showResourceHelp('space'); showResourceHelp('init') } finally { console.log = orig } + const text = lines.join('\n') + expect(text).toMatch(/\n join\s/) + expect(text).toContain('--from-snapshot') + expect(text).toContain('snapshot create --settings') + }) +})