Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,7 @@ Supported values:
| `llm.auth` | `subscription`, `api_key` |
| `llm.adapter` | `claude_cli`, `anthropic_api`, `openai_api`, `pi_rpc`, and `codex_cli` are usable for review. `codex_cli` requires `provider: openai` and `auth: subscription`, and is currently best-effort/beta because Codex does not yet expose an explicit all-tools-disabled flag. |
| `llm.model_map` keys | `small`, `medium`, `large` |
| `llm.max_effort` keys | `small`, `medium`, `large`; values `low`, `medium`, `high` |
| `llm.reviewer_model_tier` | `small`, `medium`, `large` |
| `review_policy.major_event` | `comment`, `request_changes` |
| `review_policy.resolve_threads` | `auto`, `never` |
Expand Down Expand Up @@ -659,6 +660,39 @@ Migration note: older releases treated reviewer `model_tier` as a direct map
lookup. Current releases treat it as a minimum acceptable tier, so profiles can
raise the reviewer baseline without editing shared agent catalogs.

### Capping Effort Per Tier

Agent catalogs declare an absolute `effort` (`low`, `medium`, `high`) that
becomes the provider's reasoning-effort setting. `llm.max_effort` caps that
value per tier so a deployment can bound spend on expensive models without
editing shared catalogs:

```yaml
llm:
model_map:
large: openai-codex/gpt-5.6-sol
medium: openai-codex/gpt-5.6-terra
max_effort:
large: medium
```

A tier absent from `max_effort` is uncapped. The cap is a ceiling only: an agent
declaring `low` under a `medium` ceiling still runs at `low`. Caps are keyed by
the tier resolved after the floor calculation above, and they apply to internal
stages (selection, synthesis, thread analysis) as well as reviewers, so capping
`medium` affects more than reviewer agents.

Four paths intentionally bypass the cap, because each is an explicit selection
of a concrete model or effort:

- `--reviewer-effort` and `--reviewer-model` on `cr review`
- agent `model_id`, which selects an exact model and has no tier to cap
- `cr benchmark run`, where `stages.reviewers.effort` is required so candidates
stay comparable

`cr init` preserves `max_effort` but cannot yet edit it; set it by hand in
`config.yml`.

Dry-run and no-post runs also record selected reviewer runtime resolution in
`agent-sources.json` for auditability. Each selected agent may include
`reviewer_runtime` with:
Expand Down
7 changes: 7 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,13 @@ This boundary exists so model catalog data, provider capabilities, token costs,
and profile-level tier floors can be added without touching individual review
stages. Runtime hard-coding bypasses user preference and is a bug.

The resolver also applies the profile's `llm.max_effort` ceiling, keyed by the
tier it resolved. Because the ceiling is tier-keyed, it does not apply to paths
that select a concrete model or effort directly: an explicit `ModelOverride`
returns before the clamp, and `agent.model_id` has no tier to key on. Callers
that override effort after the resolver returns, such as `--reviewer-effort`,
also win over the ceiling by construction.

Reviewer `agent.model_id` is an exact provider-specific model override. It must
still enter runtime execution through `stagemodel.ResolveStageModel` as a model
override rather than bypassing the resolver, but it intentionally bypasses the
Expand Down
28 changes: 28 additions & 0 deletions internal/cmd/initcmd/initcmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,7 @@ type initLLMRuntimeDraft struct {
CredentialStore string
CredentialRef string
ModelMap config.ModelMap
MaxEffort config.EffortMap
ReviewerModelTier config.ModelTier
}

Expand Down Expand Up @@ -3404,6 +3405,9 @@ func buildNonInteractiveInitPlan(cmd *cobra.Command, opts *root.Options, flags i
}
profile.LLM.ModelMap = modelMap
}
if previousProfile.LLM.MaxEffort != nil {
profile.LLM.MaxEffort = copyEffortMap(previousProfile.LLM.MaxEffort)
}
if !cmd.Flags().Changed("agent-source") {
profile.AgentSources = append([]string(nil), previousProfile.AgentSources...)
}
Expand Down Expand Up @@ -4541,6 +4545,7 @@ func initLLMRuntimeDraftFromConfig(llm config.LLMConfig) initLLMRuntimeDraft {
CredentialStore: initCredentialStoreDraftValue(llm.Credential.Store),
CredentialRef: strings.TrimSpace(llm.Credential.Name),
ModelMap: copyModelMap(llm.ModelMap),
MaxEffort: copyEffortMap(llm.MaxEffort),
ReviewerModelTier: llm.ReviewerModelTier,
}
if spec, ok := config.FindLLMRuntimeSpec(runtime.Provider, runtime.Auth, runtime.Adapter); ok &&
Expand All @@ -4559,6 +4564,7 @@ func (runtime initLLMRuntimeDraft) exportConfig() config.LLMConfig {
Auth: runtime.Auth,
Adapter: runtime.Adapter,
ModelMap: copyModelMap(runtime.ModelMap),
MaxEffort: copyEffortMap(runtime.MaxEffort),
ReviewerModelTier: runtime.ReviewerModelTier,
}
if runtime.Auth == config.LLMAuthAPIKey {
Expand All @@ -4577,13 +4583,23 @@ func (runtime initLLMRuntimeDraft) identityKey() string {
for _, tier := range modelKeys {
models = append(models, tier+"="+strings.TrimSpace(runtime.ModelMap[tier]))
}
effortKeys := make([]string, 0, len(runtime.MaxEffort))
for tier := range runtime.MaxEffort {
effortKeys = append(effortKeys, tier)
}
sort.Strings(effortKeys)
efforts := make([]string, 0, len(effortKeys))
for _, tier := range effortKeys {
efforts = append(efforts, tier+"="+strings.TrimSpace(runtime.MaxEffort[tier]))
}
return strings.Join([]string{
string(runtime.Provider),
string(runtime.Auth),
string(runtime.Adapter),
initCredentialStoreDraftValue(runtime.CredentialStore),
strings.TrimSpace(runtime.CredentialRef),
strings.Join(models, "\x1f"),
strings.Join(efforts, "\x1f"),
string(runtime.ReviewerModelTier),
}, "\x00")
}
Expand Down Expand Up @@ -4762,6 +4778,7 @@ func cloneInitLLMConfig(llm config.LLMConfig) config.LLMConfig {
cloned.ModelMap[tier] = model
}
}
cloned.MaxEffort = copyEffortMap(llm.MaxEffort)
return cloned
}

Expand Down Expand Up @@ -5814,6 +5831,17 @@ func initCredentialWritePlanSatisfiesEntry(entry initCredentialPlanEntry, target
return true
}

func copyEffortMap(effortMap config.EffortMap) config.EffortMap {
if len(effortMap) == 0 {
return nil
}
copied := make(config.EffortMap, len(effortMap))
for tier, ceiling := range effortMap {
copied[tier] = ceiling
}
return copied
}

func copyModelMap(modelMap config.ModelMap) config.ModelMap {
if len(modelMap) == 0 {
return nil
Expand Down
54 changes: 54 additions & 0 deletions internal/cmd/initcmd/initcmd_max_effort_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package initcmd

import (
"testing"

"github.com/open-cli-collective/codereview-cli/internal/config"
)

// A runtime round trip must preserve every LLMConfig field init does not edit.
// max_effort has no init editor, so a drop here silently discards a user's
// hand-written cost ceiling.
func TestLLMRuntimeDraftRoundTripPreservesMaxEffort(t *testing.T) {
original := config.LLMConfig{
Provider: config.LLMProviderOpenAI,
Auth: config.LLMAuthSubscription,
Adapter: config.LLMAdapterCodexCLI,
ModelMap: config.ModelMap{"large": "gpt-5.6-sol"},
MaxEffort: config.EffortMap{"large": "medium"},
}

got := initLLMRuntimeDraftFromConfig(original).exportConfig()

if len(got.MaxEffort) != 1 || got.MaxEffort["large"] != "medium" {
t.Fatalf("max_effort after round trip = %#v, want large=medium", got.MaxEffort)
}
if len(got.ModelMap) != 1 || got.ModelMap["large"] != "gpt-5.6-sol" {
t.Fatalf("model_map after round trip = %#v", got.ModelMap)
}
}

func TestLLMRuntimeIdentityKeyDistinguishesMaxEffort(t *testing.T) {
base := initLLMRuntimeDraft{
Provider: config.LLMProviderOpenAI,
Auth: config.LLMAuthSubscription,
Adapter: config.LLMAdapterCodexCLI,
ModelMap: config.ModelMap{"large": "gpt-5.6-sol"},
}
capped := base
capped.MaxEffort = config.EffortMap{"large": "medium"}

if base.identityKey() == capped.identityKey() {
t.Fatalf("identityKey collides for runtimes differing only by max_effort")
}
}

func TestCloneInitLLMConfigDeepCopiesMaxEffort(t *testing.T) {
original := config.LLMConfig{MaxEffort: config.EffortMap{"large": "medium"}}
cloned := cloneInitLLMConfig(original)
cloned.MaxEffort["large"] = "high"

if original.MaxEffort["large"] != "medium" {
t.Fatalf("clone aliased max_effort: original = %#v", original.MaxEffort)
}
}
40 changes: 40 additions & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"github.com/open-cli-collective/cli-common/credstore"
"github.com/open-cli-collective/cli-common/statedir"
"gopkg.in/yaml.v3"

"github.com/open-cli-collective/codereview-cli/internal/modelprefs"
)

const (
Expand Down Expand Up @@ -356,12 +358,17 @@ type LLMConfig struct {
Adapter LLMAdapter `yaml:"adapter" json:"adapter"`
Credential CredentialLocation `yaml:"credential,omitempty" json:"credential,omitempty"`
ModelMap ModelMap `yaml:"model_map,omitempty" json:"model_map,omitempty"`
MaxEffort EffortMap `yaml:"max_effort,omitempty" json:"max_effort,omitempty"`
ReviewerModelTier ModelTier `yaml:"reviewer_model_tier,omitempty" json:"reviewer_model_tier,omitempty"`
}

// ModelMap maps portable model tiers to provider-specific model identifiers.
type ModelMap map[string]string

// EffortMap caps reasoning effort per model tier. A tier absent from the map is
// uncapped, so the agent-declared or stage-default effort applies unchanged.
type EffortMap map[string]string

// ModelTier is a provider-neutral model slot.
type ModelTier string

Expand Down Expand Up @@ -688,6 +695,27 @@ func ResolveModelTier(llm LLMConfig, tier ModelTier) (ModelMapResolution, bool)
return resolved, ok
}

// ResolveMaxEffort returns the configured effort ceiling for one portable tier.
// It reports false when the tier is uncapped, which leaves the requested effort
// unchanged.
func ResolveMaxEffort(llm LLMConfig, tier ModelTier) (modelprefs.Effort, bool) {
tier = ModelTier(strings.TrimSpace(string(tier)))
if !tier.Valid() {
return "", false
}
for configured, ceiling := range llm.MaxEffort {
if ModelTier(strings.TrimSpace(configured)) != tier {
continue
}
effort := modelprefs.Effort(strings.TrimSpace(ceiling))
if !effort.Valid() {
return "", false
}
return effort, true
}
return "", false
}

// ReviewMajorEvent identifies how major findings affect the review event.
type ReviewMajorEvent string

Expand Down Expand Up @@ -1415,6 +1443,18 @@ func validateLLMConfig(field string, llm LLMConfig) error {
return invalid("%s.model_map.%s is required", field, tier)
}
}
for tier, ceiling := range llm.MaxEffort {
modelTier := ModelTier(tier)
if !modelTier.Valid() {
return invalid("%s.max_effort tier %q is invalid", field, tier)
}
if strings.TrimSpace(ceiling) == "" {
return invalid("%s.max_effort.%s is required", field, tier)
}
if !modelprefs.Effort(strings.TrimSpace(ceiling)).Valid() {
return invalid("%s.max_effort.%s %q is invalid; must be one of low, medium, high", field, tier, ceiling)
}
}
if llm.ReviewerModelTier != "" && !llm.ReviewerModelTier.Valid() {
return invalid("%s.reviewer_model_tier %q is invalid; must be one of small, medium, large", field, llm.ReviewerModelTier)
}
Expand Down
61 changes: 61 additions & 0 deletions internal/config/config_max_effort_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package config

import (
"errors"
"strings"
"testing"

"github.com/open-cli-collective/codereview-cli/internal/modelprefs"
)

func TestValidateAcceptsMaxEffortCeiling(t *testing.T) {
cfg := validFile()
runtime := cfg.LLMRuntimes["home-llm"]
runtime.MaxEffort = EffortMap{"large": "medium"}
cfg.LLMRuntimes["home-llm"] = runtime
if err := Validate(cfg); err != nil {
t.Fatalf("Validate error = %v, want nil", err)
}
}

func TestValidateRejectsUnknownMaxEffortTier(t *testing.T) {
cfg := validFile()
runtime := cfg.LLMRuntimes["home-llm"]
runtime.MaxEffort = EffortMap{"enormous": "medium"}
cfg.LLMRuntimes["home-llm"] = runtime
err := Validate(cfg)
if !errors.Is(err, ErrInvalid) {
t.Fatalf("Validate error = %v, want ErrInvalid", err)
}
if !strings.Contains(err.Error(), "max_effort") {
t.Fatalf("Validate error = %v, want max_effort mention", err)
}
}

func TestValidateRejectsUnknownMaxEffortValue(t *testing.T) {
cfg := validFile()
runtime := cfg.LLMRuntimes["home-llm"]
runtime.MaxEffort = EffortMap{"large": "xhigh"}
cfg.LLMRuntimes["home-llm"] = runtime
err := Validate(cfg)
if !errors.Is(err, ErrInvalid) {
t.Fatalf("Validate error = %v, want ErrInvalid", err)
}
if !strings.Contains(err.Error(), "low, medium, high") {
t.Fatalf("Validate error = %v, want valid-value mention", err)
}
}

func TestResolveMaxEffortReportsUncappedTiers(t *testing.T) {
llm := LLMConfig{MaxEffort: EffortMap{"large": "medium"}}
got, ok := ResolveMaxEffort(llm, ModelTierLarge)
if !ok || got != modelprefs.EffortMedium {
t.Fatalf("ResolveMaxEffort(large) = %q, %v; want medium, true", got, ok)
}
if _, ok := ResolveMaxEffort(llm, ModelTierMedium); ok {
t.Fatalf("ResolveMaxEffort(medium) reported a ceiling, want uncapped")
}
if _, ok := ResolveMaxEffort(llm, ModelTier("bogus")); ok {
t.Fatalf("ResolveMaxEffort(bogus) reported a ceiling, want uncapped")
}
}
30 changes: 30 additions & 0 deletions internal/modelprefs/modelprefs.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,33 @@ func (e Effort) Valid() bool {
return false
}
}

// Rank orders effort values from cheapest to most expensive. Unknown values
// rank 0 so they never win a comparison against a valid effort.
func (e Effort) Rank() int {
switch e {
case EffortLow:
return 1
case EffortMedium:
return 2
case EffortHigh:
return 3
default:
return 0
}
}

// MinEffort returns the cheaper of left and right. Invalid values are ignored
// so a missing ceiling leaves the requested effort untouched.
func MinEffort(left, right Effort) Effort {
if !left.Valid() {
return right
}
if !right.Valid() {
return left
}
if left.Rank() <= right.Rank() {
return left
}
return right
}
Loading
Loading