Post-transcribe: local LLM (Gemma 4 E2B on llama.cpp, shared ggml)

.gitmodules

@@ -6,3 +6,8 @@
 	path = third_party/whisper.cpp
 	url = https://github.com/ggml-org/whisper.cpp
 	ignore = dirty
+[submodule "third_party/llama.cpp"]
+	path = third_party/llama.cpp
+	url = https://github.com/ggml-org/llama.cpp
+	ignore = dirty
+	shallow = true

.goreleaser.yml

@@ -2,11 +2,11 @@ version: 2
 
 before:
   hooks:
-    # Init the parakeet.cpp submodule and build both static archives before the
-    # arm64 cgo build links them (darwin/arm64 only; no-op elsewhere).
-    # whisper-lib depends on parakeet-lib, so this one target builds both.
+    # Init the submodules and build the static archives before the arm64 cgo
+    # build links them (darwin/arm64 only; no-op elsewhere). whisper-lib and
+    # llama-lib both depend on parakeet-lib, which builds the shared ggml.
     - git submodule update --init --recursive
-    - make whisper-lib
+    - make whisper-lib llama-lib
     - packaging/mkicns.sh packaging/appicon.png
 
 builds:

CLAUDE.md

@@ -106,7 +106,7 @@ Option+Space keydown → record audio → encode (mode-based) → API call → c
 - `vad.go` - voice activity detection using WebRTC VAD with debounced speech confirmation
 - `silence.go` - silence monitoring with warnings, repeat beeps, and auto-close (toggle mode)
 - `config/` - persistent settings (`config.json`) and vocabulary hints (`hints.txt`)
-- `posttranscribe/` - optional LLM rewrite of the transcript before paste (Groq chat completions); built-in profiles + user prompts from `config.json`'s `post_transcribe` block
+- `posttranscribe/` - optional LLM rewrite of the transcript before paste (Groq chat completions, or a local GGUF via `internal/llama`); profiles are full prompt templates in `config.json`'s `post_transcribe.profiles`, seeded once from the embedded `defaults.json`
 - `log.go` - diagnostic logging and panic capture to `diagnostics_log.txt`
 
 ## Design Philosophy

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build build-linux-amd64 build-linux-arm64 test test-integration benchmark bench-local bench-save clean bump-version release icns app parakeet-lib whisper-lib download-models manifest model-release
+.PHONY: build build-linux-amd64 build-linux-arm64 test test-integration benchmark bench-local bench-save clean bump-version release icns app parakeet-lib whisper-lib llama-lib download-models manifest model-release
 
 # --match 'v*' keeps model-release tags (models-vN) out of the app version.
 VERSION ?= $(shell git describe --tags --match 'v*' --always --dirty 2>/dev/null || echo "dev")
@@ -38,12 +38,19 @@ WHISPER_PATCHES := patches/whisper.cpp
 # no test can distinguish from a fast one. Bumping the submodule therefore has to
 # stop the build until a human re-validates and moves this pin.
 WHISPER_BASE  := f049fff95a089aa9969deb009cdd4892b3e74916
+# llama.cpp runs the local post-transcribe model and links the same ggml too
+# (-DLLAMA_USE_SYSTEM_GGML=ON). Its pin must be a llama.cpp commit whose
+# vendored ggml has the same version as GGML_BASE — for 0.26.0 that is
+# f05c8b278..4ed1c8507 (2026-10-05..15) — or libllama compiles against a ggml
+# API that is not the one it links. Move LLAMA_BASE together with GGML_BASE.
+LLAMA_DIR     := third_party/llama.cpp
+LLAMA_BASE    := f05c8b278091b716c002c331911a51d4927f0c74
 HOST          := $(shell go env GOOS)/$(shell go env GOARCH)
 ifeq ($(HOST),darwin/arm64)
 CGO_ENV := MACOSX_DEPLOYMENT_TARGET=$(MACOS_MIN) CGO_CFLAGS=-mmacosx-version-min=$(MACOS_MIN) CGO_LDFLAGS=-mmacosx-version-min=$(MACOS_MIN)
 endif
 
-build: whisper-lib download-models
+build: whisper-lib llama-lib download-models
 	$(CGO_ENV) go build -ldflags="-X main.version=$(VERSION)" -o zee
 
 # The dev model folder `localmodels download` writes to (cmd/localmodels keeps
@@ -84,7 +91,7 @@ parakeet-lib:
 	  git -C $(GGML_DIR) cat-file -e $(GGML_BASE)^{commit} 2>/dev/null || \
 	    git -C $(GGML_DIR) fetch --depth 1 origin $(GGML_BASE) || exit 1; \
 	  git -C $(GGML_DIR) checkout -f -q $(GGML_BASE) || exit 1; \
-	  rm -rf $(PARAKEET_DIR)/build-release $(WHISPER_DIR)/build-release; \
+	  rm -rf $(PARAKEET_DIR)/build-release $(WHISPER_DIR)/build-release $(LLAMA_DIR)/build-release; \
 	fi; \
 	pats=$$(ls $(CURDIR)/$(PARAKEET_PATCHES)/*.patch 2>/dev/null); \
 	if [ -n "$$pats" ]; then \
@@ -100,8 +107,8 @@ parakeet-lib:
 	if [ ! -d $(PARAKEET_DIR)/build-release ]; then \
 	  echo "==> configuring parakeet.cpp (one-time)"; \
 	  cmake -S $(PARAKEET_DIR) -B $(PARAKEET_DIR)/build-release \
-	    -DBUILD_SHARED_LIBS=OFF -DPARAKEET_SHARED=OFF -DPARAKEET_BUILD_CLI=OFF \
-	    -DPARAKEET_GGML_METAL=ON -DGGML_NATIVE=OFF \
+	    -DBUILD_SHARED_LIBS=OFF -DPARAKEET_SHARED=OFF -DPARAKEET_BUILD_CLI=OFF -DPARAKEET_BUILD_SERVER=OFF \
+	    -DPARAKEET_GGML_METAL=ON -DGGML_NATIVE=OFF -DGGML_OPENMP=OFF -DGGML_CCACHE=OFF \
 	    -DCMAKE_OSX_DEPLOYMENT_TARGET=$(MACOS_MIN) \
 	    -DCMAKE_INSTALL_PREFIX=$(GGML_PREFIX) \
 	    -DCMAKE_C_FLAGS="-mcpu=apple-m1" -DCMAKE_CXX_FLAGS="-mcpu=apple-m1"; \
@@ -159,13 +166,43 @@ whisper-lib: parakeet-lib
 	fi && \
 	cmake --build $(WHISPER_DIR)/build-release -j
 
+# Build libllama.a against the ggml parakeet installed, like whisper-lib: one
+# ggml in the process. Only the llama library is built (no tools, server or
+# network code).
+llama-lib: parakeet-lib
+	@if [ "$(HOST)" != "darwin/arm64" ]; then exit 0; fi; \
+	if [ ! -f $(LLAMA_DIR)/CMakeLists.txt ]; then \
+	  echo "==> initializing llama.cpp submodule (first checkout)"; \
+	  git submodule update --init $(LLAMA_DIR); \
+	fi; \
+	head=$$(git -C $(LLAMA_DIR) rev-parse HEAD); \
+	if [ "$$head" != "$(LLAMA_BASE)" ]; then \
+	  echo "ERROR: llama.cpp is at $$head, not $(LLAMA_BASE)."; \
+	  echo "       Its ggml must match GGML_BASE: move LLAMA_BASE only to a llama.cpp"; \
+	  echo "       commit whose ggml/ has the same version, then delete"; \
+	  echo "       $(LLAMA_DIR)/build-release and rebuild."; \
+	  exit 1; \
+	fi; \
+	if [ ! -d $(LLAMA_DIR)/build-release ]; then \
+	  echo "==> configuring llama.cpp (one-time)"; \
+	  cmake -S $(LLAMA_DIR) -B $(LLAMA_DIR)/build-release \
+	    -DLLAMA_USE_SYSTEM_GGML=ON -DCMAKE_PREFIX_PATH=$(GGML_PREFIX) \
+	    -DBUILD_SHARED_LIBS=OFF -DGGML_METAL=ON \
+	    -DLLAMA_BUILD_COMMON=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=OFF \
+	    -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_SERVER=OFF -DLLAMA_BUILD_APP=OFF \
+	    -DLLAMA_OPENSSL=OFF \
+	    -DCMAKE_OSX_DEPLOYMENT_TARGET=$(MACOS_MIN) \
+	    -DCMAKE_C_FLAGS="-mcpu=apple-m1" -DCMAKE_CXX_FLAGS="-mcpu=apple-m1"; \
+	fi && \
+	cmake --build $(LLAMA_DIR)/build-release --target llama -j
+
 build-linux-amd64:
 	GOOS=linux GOARCH=amd64 go build -ldflags="-X main.version=$(VERSION) -s -w" -o zee-linux-amd64
 
 build-linux-arm64:
 	GOOS=linux GOARCH=arm64 go build -ldflags="-X main.version=$(VERSION) -s -w" -o zee-linux-arm64
 
-test: whisper-lib
+test: whisper-lib llama-lib
 	$(CGO_ENV) go test -race -v ./...
 
 benchmark: build
@@ -217,7 +254,7 @@ bench-save: whisper-lib download-models
 		| grep -vE 'duplicate librar|Backend using device' >> "$(BENCH_FILE)"
 	@echo "appended a baseline block to $(BENCH_FILE)"
 
-test-integration: whisper-lib
+test-integration: whisper-lib llama-lib
 	@tmp=$$(mktemp -d) && \
 	$(CGO_ENV) go build -o "$$tmp/zee-test-bin" . && \
 	ZEE_TEST_BIN="$$tmp/zee-test-bin" $(CGO_ENV) go test -race -tags integration -v -timeout 600s -count=1 ./test/ ; \

config/config.go

@@ -33,15 +33,28 @@ type Settings struct {
 }
 
 // PostTranscribe is the post_transcribe block of config.json. Active picks a
-// profile by name — one of the user's Prompts, else a built-in; "" is off.
+// profile by name from Profiles; "" is off.
 type PostTranscribe struct {
-	Active  string            `json:"active"`
-	Model   string            `json:"model,omitempty"`   // "" = the package default
-	Prompts map[string]string `json:"prompts,omitempty"` // name → instruction
-	// Colors is each profile's overlay tag colour, "#RRGGBB". zee fills in a
-	// colour for a profile the first time it is used, so it stays the same from
-	// then on.
-	Colors map[string]string `json:"colors,omitempty"`
+	Active   string `json:"active"`
+	Provider string `json:"provider,omitempty"` // "groq" (the default) or "local"
+	Model    string `json:"model,omitempty"`    // "" = the provider's default
+	// Profiles has no omitempty: an empty map is the user deleting every
+	// profile, and must not read back as absent (which seeds the defaults).
+	Profiles map[string]Profile `json:"profiles"`
+	// Prompts and Colors are the earlier form (name → instruction, name →
+	// colour); posttranscribe.Normalize folds them into Profiles once.
+	Prompts map[string]string `json:"prompts,omitempty"`
+	Colors  map[string]string `json:"colors,omitempty"`
+}
+
+// Profile is one post-transcribe profile: the whole system prompt, a template
+// (see posttranscribe.Vars), plus its tray tooltip and overlay tag colour.
+type Profile struct {
+	Prompt      string `json:"prompt"`
+	Description string `json:"description,omitempty"`
+	// Color is "#RRGGBB". zee fills one in the first time the profile is used,
+	// so it stays the same from then on.
+	Color string `json:"color,omitempty"`
 }
 
 const settingsFile = "config.json"
@@ -55,10 +68,15 @@ const settingsFile = "config.json"
 const defaultTailWaitMs = 50
 
 var (
-	mu       sync.Mutex
-	current  Settings
-	dir      string
-	defaults = Settings{
+	mu      sync.Mutex
+	current Settings
+	// loadFailed is set while the live settings are the defaults standing in
+	// for a config.json that did not parse: Update does not save then, so no
+	// change made meanwhile writes them over the user's file. A good Load or
+	// Reload clears it.
+	loadFailed bool
+	dir        string
+	defaults   = Settings{
 		Language:   "en",
 		AutoPaste:  true,
 		TailWaitMs: defaultTailWaitMs,
@@ -156,8 +174,15 @@ func IsAppBundle() bool {
 	return ok
 }
 
+// Load reads config.json into the live settings. A missing file is the
+// defaults. A file that does not parse also leaves the defaults live, and is
+// returned as an error, and no save writes them over the user's file until
+// a Reload succeeds.
 func Load() error {
+	mu.Lock()
+	defer mu.Unlock()
 	current = defaults
+	loadFailed = false
 
 	data, err := os.ReadFile(settingsPath())
 	if err != nil {
@@ -173,8 +198,8 @@ func Load() error {
 	// (Auto-detect), which must persist.
 	s := defaults
 	if err := json.Unmarshal(data, &s); err != nil {
-		log.Warnf("settings: corrupt config.json, using defaults: %v", err)
-		return nil
+		loadFailed = true
+		return fmt.Errorf("invalid config.json, using defaults: %w", err)
 	}
 	current = s
 	return nil
@@ -206,15 +231,20 @@ func Reload() (Settings, error) {
 		return current, fmt.Errorf("invalid config.json: %w", err)
 	}
 	current = s
+	loadFailed = false
 	return s, nil
 }
 
 func Update(fn func(*Settings)) {
 	mu.Lock()
 	fn(&current)
-	s := current
+	s, failed := current, loadFailed
 	mu.Unlock()
 
+	if failed {
+		log.Warn("settings: config.json did not parse, not saving over it (fix it and pick Reload Config)")
+		return
+	}
 	save(s)
 }
 

config/config_test.go

@@ -118,8 +118,8 @@ func TestSettingsCorruptFile(t *testing.T) {
 
 	os.WriteFile(filepath.Join(d, "config.json"), []byte("not json{{{"), 0644)
 
-	if err := Load(); err != nil {
-		t.Fatalf("Load should not error on corrupt file: %v", err)
+	if err := Load(); err == nil {
+		t.Fatal("Load on corrupt file: want error, got nil")
 	}
 	s := Get()
 	if s.Language != "en" {
@@ -182,3 +182,53 @@ func TestReloadExternalEdits(t *testing.T) {
 		t.Fatalf("Get().Language = %q after failed reload, want fr (untouched)", got)
 	}
 }
+
+// TestEmptyProfilesPersist: a config with every profile deleted reads back
+// empty, not absent, so the defaults are not seeded again.
+func TestEmptyProfilesPersist(t *testing.T) {
+	SetDir(t.TempDir())
+	if err := Load(); err != nil {
+		t.Fatal(err)
+	}
+	Update(func(s *Settings) { s.PostTranscribe.Profiles = map[string]Profile{} })
+	if err := Load(); err != nil {
+		t.Fatal(err)
+	}
+	if p := Get().PostTranscribe.Profiles; p == nil || len(p) != 0 {
+		t.Errorf("profiles = %#v, want an empty map", p)
+	}
+}
+
+// TestCorruptFileNotSavedOver: after a Load that fails to parse, a setting
+// change must not write the defaults over the user's file; once the file is
+// fixed and reloaded, changes save again.
+func TestCorruptFileNotSavedOver(t *testing.T) {
+	d := t.TempDir()
+	SetDir(d)
+	path := filepath.Join(d, "config.json")
+	bad := []byte(`{"language": "fr",}`)
+	if err := os.WriteFile(path, bad, 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := Load(); err == nil {
+		t.Fatal("Load on corrupt file: want error, got nil")
+	}
+	Update(func(s *Settings) { s.Device = "Blue Yeti" })
+	if got, _ := os.ReadFile(path); string(got) != string(bad) {
+		t.Fatalf("corrupt config.json was saved over: %q", got)
+	}
+
+	if err := os.WriteFile(path, []byte(`{"language": "fr"}`), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Reload(); err != nil {
+		t.Fatalf("Reload: %v", err)
+	}
+	Update(func(s *Settings) { s.Device = "Blue Yeti" })
+	if err := Load(); err != nil {
+		t.Fatalf("Load after save: %v", err)
+	}
+	if s := Get(); s.Device != "Blue Yeti" || s.Language != "fr" {
+		t.Fatalf("after Reload, Update did not save: %+v", s)
+	}
+}

docs/reference.md

@@ -59,34 +59,60 @@ app):
 An optional LLM pass rewrites each transcript before it is pasted. It is off by
 default. Pick a profile from the tray (**Settings → Post-Transcribe**, **None**
 turns it off) or set it in `config.json` and **Reload Config**; either way it
-applies from the next recording, with no restart. It uses the `groq` key from
-`credentials.json`, with any provider for the transcription itself.
+applies from the next recording, with no restart. It works with any provider
+for the transcription itself.
 
 ```json
 "post_transcribe": {
-  "active": "friendly",
-  "model": "openai/gpt-oss-120b",
-  "prompts": {
-    "slack": "Adjust the tone to casual, for a Slack message, only where needed."
+  "active": "clean",
+  "provider": "local",
+  "profiles": {
+    "clean": {
+      "description": "Fix grammar and misheard words, drop filler; keep your tone.",
+      "prompt": "You clean up a speech-to-text transcript … {{if .Language}}\nThe transcript is in {{.Language}}.{{end}}\n\nInstruction: Change nothing more.{{if .Hints}}\n\nVocabulary the speaker uses: {{.Hints}} …{{end}}",
+      "color": "#7CFF4F"
+    }
   }
 }
 ```
 
 - `active` — the profile to run; `""` (the default) turns it off.
-- `prompts` — your own profiles, name → a one- or two-sentence instruction. A
-  name that matches a built-in one replaces it.
-- `model` — any Groq chat model; omit it for `openai/gpt-oss-120b`. The rewrite
-  has a 5 s budget, so pick a fast model: a slow or reasoning-heavy one times out
-  on longer dictations and the raw transcript is pasted instead.
-- `colors` — each profile's colour for the recording overlay, `"#RRGGBB"`. zee
-  gives a profile without one a random colour from its palette the first time
-  you use it, and saves it so the colour stays put; edit it to choose your own.
-
-Built-in profiles: `clean` (the cleanup alone, tone untouched), `friendly`
-(conversational tone), `professional` (work email tone). Every profile fixes
-grammar and misheard words (toward `hints.txt`), removes filler, repetition and
-false starts, keeps the sentence in the language you spoke it in (English words
-inside a Turkish sentence stay English), and never answers the dictated text.
+- `profiles` — every profile, by name. zee writes its defaults here the first
+  time (`clean`: the cleanup alone, tone untouched; `friendly`; `professional`),
+  and from then on this is the only copy: edit a prompt, add a profile, or
+  delete one, then **Reload Config**. Each profile has:
+  - `prompt` (required) — the whole system prompt the model gets, a template
+    with the variables below. The transcript itself is sent as the user turn.
+  - `description` — the tray tooltip.
+  - `color` — the profile's colour on the recording overlay, `"#RRGGBB"`. zee
+    gives a profile without one a random colour from its palette the first
+    time you use it, and saves it so it stays put.
+- `provider` — where the rewrite runs: `groq` (the default; it needs the `groq`
+  key) or `local`, on this Mac (Apple Silicon only, no key needed).
+- `model` — for `groq`, any Groq chat model; omit it for `openai/gpt-oss-120b`.
+  The Groq rewrite has a 5 s budget, so pick a fast model: a slow or
+  reasoning-heavy one times out on longer dictations and the raw transcript is
+  pasted instead. For `local`, omit it for `gemma-4-e2b`, the one local model
+  for now. Put `gemma-4-E2B-it-Q3_K_M.gguf` (2.5 GB) in the models folder; zee
+  loads it when a recording starts and keeps it in memory from then on. Its
+  time grows with the length of the dictation, and it has no 5 s budget.
+
+Prompt variables, in Go template syntax — `{{.Name}}`, and
+`{{if .Name}}…{{end}}` to leave a part out when it is empty:
+
+| Variable | Value |
+|---|---|
+| `{{.Language}}` | The transcript's language in English, e.g. `Turkish`: the one you set, or with Auto-detect the one the transcription detected (local Whisper, Groq, Deepgram and ElevenLabs report it; Mistral and OpenAI do not). Empty when unknown, and always empty for `groq`, whose models were measured without it. The local model needs it to not translate. |
+| `{{.Hints}}` | Your vocabulary from `hints.txt`, comma-separated; empty when there is none. |
+
+A prompt that uses another name, or does not parse, fails that rewrite: the
+raw transcript is pasted and the tray error names the variables there are.
+The default prompts keep each word in the language you spoke it in (English
+words inside a Turkish sentence stay English) and never answer the dictated
+text.
+
+An earlier config with `prompts` (name → instruction) and `colors` is
+converted once: each instruction becomes a profile with the default rules.
 
 While a profile is active, the recording overlay shows its name in the
 top-right corner, in its colour.
@@ -95,7 +121,7 @@ If the call fails or times out, the raw transcript is pasted and the tray shows
 the error. In streaming mode the text is pasted once at the end instead of
 while you speak. With `-debug-transcribe`, each recording folder keeps the raw
 `transcript.txt` beside the rewritten `post_transcript.txt`, and `info.json`
-records the profile, the model and any error. The rewrite's time is
+records the profile, the provider, the model and any error. The rewrite's time is
 `post_transcribe_ms` in the `felt_latency` line.
 
 Logs live in `~/Library/Logs/zee/`: `diagnostics_log.txt` (timing, errors;