diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ffc7e4..baf6fec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,32 @@ ## [Unreleased] +## [0.3.2a2] — 2026-11-09 (beta) + +Adds narrower recomputation, two lookahead options and web UI fields for every long-context switch, and fixes a crash found by training on the real corpus. Every new switch is off by default. `reports/long_context_20261109/real_corpus_ab.json` holds the measurements below. + +### Added +- `recompute_scope="mlp"`: with `activation_checkpointing`, recompute only the MoE/FFN of each block (`"block"` recomputes the whole block, as before). At 32k on the real corpus it saves 1.4 GB (5%) and loses 11% speed, against 6.9 GB (25%) and 19% for whole blocks, so it suits runs that need only a little more memory. +- `mla_upproj_recompute`: keep only the MLA latent and rebuild K and V per attention block. Exact, but it saved no memory (24.47 -> 24.29 GiB at 64k) and costs 6.7% speed, so it stays off. +- `lookahead_input_detach` (the lookahead router reads a gradient-free hidden state) and `temporal_grad_span` (cut the temporal-locality gradient chain every n positions). Both are exact in the forward pass. On the en-zh data they were neutral (validation CE within 1e-3), so they stay off. +- Web UI: every long-context field (`attention_block_size`, `ce_chunk_size`, `moe_chunk_tokens`, `recompute_scope`, `recompute_policy`, `mla_upproj_recompute`, `document_masking`, `reset_router_at_documents`, `reset_lookahead_at_documents`, `lookahead_input_detach`, `temporal_grad_span`) with labels and help in all four languages. + +### Fixed +- `reset_router_at_documents` and `document_masking` raised "Attempting to eval an array without a primitive" inside the compiled LongRun step, because the document ids were built with NumPy from a traced array. `chronos.expert_clusters.segment_ids_mx` builds them in MLX, and a regression test runs the option through `mx.compile`. +- `reset_lookahead_at_documents="false"` (the string the web UI sends) was read as true. Strings are now parsed; the empty string follows the router switch. + +### Measured on the real en-zh corpus (production 50M config, random init, M5 Max, bf16) +`long` = `attention_block_size=1024`, `ce_chunk_size=1024`, `moe_chunk_tokens=8192`. Validation CE differs by at most 3e-3 between arms. + +| Context | Default: tokens/s, peak | long: tokens/s, peak | +|---|---|---| +| 8k | 8.9k, 40.6 GiB | 19.5k, 7.5 GiB | +| 16k | 1.9k, 141.5 GiB (over this machine's 128 GB) | 15.2k, 13.8 GiB | +| 32k | not run | 8.8k, 27.0 GiB | +| 64k | not run | 3.7k, 54.6 GiB | + +Larger attention blocks speed up one MLA layer (0.62 s -> 0.53 s at 64k with blocks of 8192) but slow the whole model, because the sliding-window layers get slower (9.9 s -> 16.6 s per step); 1024 stays the recommendation. + ## [0.3.2a1] — 2026-11-09 (beta) This release lets the MLX trainer run long contexts on one Mac. Every new switch is off by default, so existing configs, checkpoints and recipes behave as before. diff --git a/chronos/__init__.py b/chronos/__init__.py index 914085a..61ba8e2 100644 --- a/chronos/__init__.py +++ b/chronos/__init__.py @@ -1,5 +1,5 @@ # Project Chronos: On-device low-latency lookahead dual-layer MoE inference -__version__ = "0.3.2a1" +__version__ = "0.3.2a2" # Register with HuggingFace AutoXXX factories so # AutoModelForCausalLM.from_pretrained("path/to/chronos") works out of the box. diff --git a/chronos/config_schema.py b/chronos/config_schema.py index a6fd961..d635a78 100644 --- a/chronos/config_schema.py +++ b/chronos/config_schema.py @@ -46,7 +46,7 @@ "recommended_resident_experts", "offload_working_set_experts", "storage_format", "cluster_manifest_path", "training_paging", "optimizer_paging", "optimizer_page_dir", "page_size_mb", "activation_checkpointing", "ce_chunk_size", "attention_block_size", - "reset_router_at_documents", "document_masking", "recompute_policy", "moe_chunk_tokens", + "reset_router_at_documents", "document_masking", "recompute_policy", "recompute_scope", "lookahead_input_detach", "temporal_grad_span", "moe_chunk_tokens", "mla_upproj_recompute", "reset_lookahead_at_documents", "rope_train_scaling", "modalities", "auto_detect_modalities", "image_size", "image_patch_size", "max_image_tokens", "audio_sample_rate", "max_audio_seconds", "max_audio_tokens", diff --git a/chronos/expert_clusters.py b/chronos/expert_clusters.py index ffea145..7c116de 100644 --- a/chronos/expert_clusters.py +++ b/chronos/expert_clusters.py @@ -226,6 +226,17 @@ def segment_ids(input_ids, chunk: int = 0, boundary_id: int = SEGMENT_BOUNDARY_I return np.cumsum(new, axis=1) +def segment_ids_mx(input_ids, boundary_id: int = SEGMENT_BOUNDARY_ID): + """``segment_ids`` (chunk = 0) for an MLX array, traceable under ``mx.compile``. + + Same rule: a separator token closes the document it ends, so the id of position t is the + number of separators strictly before t. Returns an int32 ``[B, S]`` array.""" + import mlx.core as mx + is_bnd = (input_ids == int(boundary_id)).astype(mx.int32) + before = mx.cumsum(is_bnd, axis=1) - is_bnd + return mx.stop_gradient(before) + + def segment_index(segments): """``(inverse [B*S] int32, n)``: dense batch-wide segment number of each position.""" import numpy as np diff --git a/chronos/mlx/attention.py b/chronos/mlx/attention.py index 0bc12a2..6d734b1 100644 --- a/chronos/mlx/attention.py +++ b/chronos/mlx/attention.py @@ -21,7 +21,7 @@ from chronos.mlx.checkpointing import Dropout as TapeDropout # replayed under recomputation from chronos.mlx.kv_quant import kv_read, kv_store -from chronos.mlx.blockwise_attention import blockwise_attention, use_blockwise +from chronos.mlx.blockwise_attention import blockwise_attention, blockwise_mla_attention, use_blockwise def _rope_freqs( @@ -308,16 +308,22 @@ def __call__( if self._use_absorbed(S, T_full): out = self._absorbed(q_nope, q_rope, c_kv_full, k_rope, scale, allowed) else: - k_nope = self.k_nope_proj(c_kv_full).reshape(B, T_full, self.n_kv_heads, self.nope_dim) - v = self.v_proj(c_kv_full).reshape(B, T_full, self.n_kv_heads, self.head_dim) - q = mx.concatenate([q_nope, q_rope], axis=-1).transpose(0, 2, 1, 3) # [B, nH, S, d] - k = mx.concatenate([k_nope, k_rope], axis=-1).transpose(0, 2, 1, 3) # [B, nKV, T, d] blk = use_blockwise(self._block_cfg, S, T_full, self.training) if mask is None else 0 - if blk: - out = blockwise_attention(q, k, v.transpose(0, 2, 1, 3), scale=scale, window=None, block=blk, - segments=getattr(self, "_segments", None), past=T_full - S) + seg = getattr(self, "_segments", None) + q = mx.concatenate([q_nope, q_rope], axis=-1).transpose(0, 2, 1, 3) # [B, nH, S, d] + if blk and self.training and getattr(self._block_cfg, "mla_upproj_recompute", False): + # Keep c_kv only: K and V are rebuilt per block (forward and backward). + out = blockwise_mla_attention(q, c_kv_full, k_rope, self.k_nope_proj.weight, self.v_proj.weight, + scale=scale, block=blk, segments=seg, past=T_full - S) else: - out = self._attend(q, k, v.transpose(0, 2, 1, 3), scale, allowed) + k_nope = self.k_nope_proj(c_kv_full).reshape(B, T_full, self.n_kv_heads, self.nope_dim) + v = self.v_proj(c_kv_full).reshape(B, T_full, self.n_kv_heads, self.head_dim) + k = mx.concatenate([k_nope, k_rope], axis=-1).transpose(0, 2, 1, 3) # [B, nKV, T, d] + if blk: + out = blockwise_attention(q, k, v.transpose(0, 2, 1, 3), scale=scale, window=None, block=blk, + segments=seg, past=T_full - S) + else: + out = self._attend(q, k, v.transpose(0, 2, 1, 3), scale, allowed) out = out.transpose(0, 2, 1, 3) out = self.resid_dropout(self.o_proj(out.reshape(B, S, -1))) if return_shared_kv: diff --git a/chronos/mlx/blockwise_attention.py b/chronos/mlx/blockwise_attention.py index 5b2b4f2..d064fef 100644 --- a/chronos/mlx/blockwise_attention.py +++ b/chronos/mlx/blockwise_attention.py @@ -80,6 +80,51 @@ def blockwise_attention(q, k, v, *, scale, window=None, block=1024, segments=Non return outs[0] if len(outs) == 1 else mx.concatenate(outs, axis=2) +def _mla_block(qb, cb, rb, w_k, w_v, mask, scale): + """One query block of MLA attention, K and V rebuilt from the latent slice. + + ``cb [B, t, L]`` latent, ``rb [B, t, nKV, r]`` rotated rope keys, + ``w_k [nKV*nope, L]`` and ``w_v [nKV*d, L]`` the up-projections.""" + B, t, _ = cb.shape + n_kv, r = rb.shape[2], rb.shape[3] + k_nope = (cb @ w_k.T).reshape(B, t, n_kv, -1) + v = (cb @ w_v.T).reshape(B, t, n_kv, -1) + k = mx.concatenate([k_nope, rb], axis=-1).transpose(0, 2, 1, 3) + return mx.fast.scaled_dot_product_attention(qb, k, v.transpose(0, 2, 1, 3), scale=scale, mask=mask) + + +_mla_block_ckpt = mx.checkpoint(_mla_block) + + +def blockwise_mla_attention(q, c_kv, k_rope, w_k, w_v, *, scale, block=1024, segments=None, past=0): + """Full-causal MLA attention that never keeps the expanded K and V. + + ``q [B, nH, S, d]``; ``c_kv [B, T, L]``; ``k_rope [B, T, nKV, r]`` (already + rotated); ``w_k``/``w_v`` the ``k_nope_proj`` / ``v_proj`` weights. Every + block runs under ``mx.checkpoint`` with the latent slice as its input, so + the forward pass keeps ``c_kv`` (L wide) instead of K and V (nKV*d wide + each), and the backward pass rebuilds them one block at a time. The same + keys and softmax set as :func:`blockwise_attention`, so the output is + equal up to summation order. Returns ``[B, nH, S, d]``.""" + S, T = q.shape[2], c_kv.shape[1] + block = max(1, int(block)) + seg = None if segments is None else mx.stop_gradient(segments) + outs, prev = [], None + for start in range(0, S, block): + end = min(S, start + block) + qa0, qa1 = past + start, past + end + k1 = min(T, qa1) + seg_q = None if seg is None else seg[:, qa0:qa1] + seg_k = None if seg is None else seg[:, :k1] + mask = _mask_for(qa0, qa1, 0, k1, None, seg_q, seg_k) + qb = q[:, :, start:end] + if prev is not None: + qb = mx.depends(qb, prev) + prev = _mla_block_ckpt(qb, c_kv[:, :k1], k_rope[:, :k1], w_k, w_v, mask, scale) + outs.append(prev) + return outs[0] if len(outs) == 1 else mx.concatenate(outs, axis=2) + + def use_blockwise(config, S, T, training): """Rows per attention block from ``config.attention_block_size`` (0 = off). diff --git a/chronos/mlx/model.py b/chronos/mlx/model.py index 1c03fbb..9abb443 100644 --- a/chronos/mlx/model.py +++ b/chronos/mlx/model.py @@ -96,6 +96,7 @@ def __init__(self, layer_id: int, config): self.self_attn = SlidingWindowAttentionMLX(config) self.self_attn.window_size = None # dense causal GQA (a 2**63 window overflowed int32) + object.__setattr__(self, "_cfg", config) if getattr(config, 'use_moe', True): self.mlp = ChronosMLXMOE(config) else: @@ -121,10 +122,16 @@ def __call__( h, new_cache = out x = x + h mlp_in = self.post_norm(x) + # recompute_scope="mlp": only the FFN/MoE is a recomputed region. + sub = (self.training and cache is None and getattr(self._cfg, "recompute_scope", "block") == "mlp" + and bool(getattr(self._cfg, "activation_checkpointing", False))) if isinstance(self.mlp, ChronosMLXMOE): - x = x + self.mlp(mlp_in, available_expert_mask=available_expert_mask) + if sub: + x = x + call_block(self.mlp, mlp_in, available_expert_mask=available_expert_mask) + else: + x = x + self.mlp(mlp_in, available_expert_mask=available_expert_mask) else: - x = x + self.mlp(mlp_in) + x = x + call_block(self.mlp, mlp_in) if sub else x + self.mlp(mlp_in) if return_shared_kv: return x, new_cache, shared return x, new_cache @@ -232,7 +239,8 @@ def _forward(self, input_ids, cache, available_expert_masks, logits_dtype, depth def run(i, hidden, slot=None, **kw): avail = available_expert_masks[i] if available_expert_masks is not None else None c = cache[i if slot is None else slot] if "kv_override" not in kw else None - if recompute and c is None and _recompute_block(cfg, i, self.layers[i]): + if (recompute and c is None and getattr(cfg, "recompute_scope", "block") == "block" + and _recompute_block(cfg, i, self.layers[i])): return call_block(self.layers[i], hidden, mask=mask, cache=None, available_expert_mask=avail, **kw) return self.layers[i](hidden, mask=mask, cache=c, available_expert_mask=avail, **kw) @@ -257,7 +265,8 @@ def run(i, hidden, slot=None, **kw): for i in range(cfg.prelude_layers): x, new_caches[i] = run(i, x) # Lookahead observes the post-prelude state (Torch parity). - lookahead_probs = self.lookahead_router(x) + lookahead_probs = self.lookahead_router( + mx.stop_gradient(x) if getattr(cfg, "lookahead_input_detach", False) else x) # MLX always returns a cache, so per_iteration (one slot per # iteration) never exits early; shared_first/first_only may. allow_exit = mode != "per_iteration" @@ -321,7 +330,8 @@ def run(i, hidden, slot=None, **kw): x, candidate = run(i, x) new_caches.append(candidate) if i == 0: - lookahead_probs = self.lookahead_router(x) + lookahead_probs = self.lookahead_router( + mx.stop_gradient(x) if getattr(cfg, "lookahead_input_detach", False) else x) x = self.norm(x) if return_hidden: diff --git a/chronos/mlx/training/trainer.py b/chronos/mlx/training/trainer.py index 0fb76bd..582b49e 100644 --- a/chronos/mlx/training/trainer.py +++ b/chronos/mlx/training/trainer.py @@ -162,10 +162,26 @@ def _group_mask(token_mask: mx.array | None, groups: int) -> mx.array | None: return mx.repeat(token_mask, groups, axis=0) +_TEMPORAL_SPAN: list = [0] # temporal_grad_span of the loss being built (0 = full chain) + + +def _span_detach(prev: mx.array) -> mx.array: + """The older side of every adjacent pair, stop-gradient at span boundaries. + + With ``temporal_grad_span`` = n the sequence is cut in blocks of n positions: inside a + block the older token keeps its gradient; the pair that straddles a block edge detaches + it, so no gradient chain runs across more than n steps.""" + n = int(_TEMPORAL_SPAN[0] or 0) + if n <= 0 or prev.shape[1] <= 1: + return prev + edge = (mx.arange(prev.shape[1]) % n) == (n - 1) + return mx.where(edge[None, :, None], mx.stop_gradient(prev), prev) + + def _temporal_locality_loss(router_mean: mx.array, token_mask: mx.array | None = None) -> mx.array: if router_mean.shape[1] <= 1: return mx.zeros(()) - diff = router_mean[:, 1:, :] - router_mean[:, :-1, :] + diff = router_mean[:, 1:, :] - _span_detach(router_mean[:, :-1, :]) return _masked_mean((diff * diff).sum(axis=-1), _pair_mask(token_mask)) @@ -571,18 +587,20 @@ def _chronos_regularized_loss(model, base_loss, lookahead_probs, config, router_ reset_look = getattr(config, "reset_lookahead_at_documents", None) reset_look = reset_router if reset_look is None else bool(reset_look) if input_ids is not None and (reset_router or reset_look): - from chronos.expert_clusters import segment_ids - docs = mx.array(segment_ids(input_ids)) + from chronos.expert_clusters import segment_ids_mx + docs = segment_ids_mx(input_ids) segments = docs if reset_router else None look_segments = docs if reset_look else None _PAIR_SEGMENTS[0] = segments _LOOKAHEAD_SEGMENTS[0] = look_segments + _TEMPORAL_SPAN[0] = int(getattr(config, "temporal_grad_span", 0) or 0) try: return _regularized_terms(model, base_loss, lookahead_probs, config, router_ref, metrics, draft_kl, token_mask, input_ids) finally: _PAIR_SEGMENTS[0] = None _LOOKAHEAD_SEGMENTS[0] = None + _TEMPORAL_SPAN[0] = 0 def _regularized_terms(model, base_loss, lookahead_probs, config, router_ref, metrics, draft_kl, @@ -648,10 +666,10 @@ def lm_ce(model, input_ids, labels, *, aligned: bool = False, need_logits: bool from chronos.mlx.chunked_ce import ce_chunk_for, chunked_lm_loss, chunked_shifted_lm_loss chunk = 0 if need_logits else ce_chunk_for(model.config, int(input_ids.shape[0]) * int(input_ids.shape[1])) if getattr(model.config, "document_masking", False) and "segments" not in forward_kw: - from chronos.expert_clusters import segment_ids + from chronos.expert_clusters import segment_ids_mx # Varlen packing: queries attend only inside their own document; the # separator token closes the document it ends (segment_ids rule). - forward_kw["segments"] = mx.array(segment_ids(input_ids)) + forward_kw["segments"] = segment_ids_mx(input_ids) if chunk: hidden, lookahead, _cache = model(input_ids, return_hidden=True, **forward_kw) loss_fn = chunked_lm_loss if aligned else chunked_shifted_lm_loss diff --git a/chronos/model/config.py b/chronos/model/config.py index d7f88f9..dda133f 100644 --- a/chronos/model/config.py +++ b/chronos/model/config.py @@ -111,6 +111,15 @@ def __init__(self, **kwargs): self.recompute_policy: str = str(kwargs.pop("recompute_policy", "all") or "all").lower() if self.recompute_policy not in ("all", "mla", "swa", "alternate"): raise ValueError(f"unknown recompute_policy {self.recompute_policy!r}") + # What activation_checkpointing recomputes: "block" (a whole block) or "mlp" (only the + # MoE/FFN of each block: most saved activations, none of the quadratic attention). + # MLX only; recompute_policy applies to scope=block. + self.recompute_scope: str = str(kwargs.pop("recompute_scope", "block") or "block").lower() + if self.recompute_scope not in ("block", "mlp"): + raise ValueError(f"unknown recompute_scope {self.recompute_scope!r}") + # MLA training with attention_block_size: keep only the latent c_kv and rebuild K and V + # per attention block, in both passes (no [B, T, nKV, d] K/V tensors). Exact. + self.mla_upproj_recompute: bool = bool(kwargs.pop("mla_upproj_recompute", False)) # MoE sparse dispatch in chunks of this many tokens, each recomputed in the # backward pass, so [N*K, I] expert activations never exist for a whole row # (0 = one dispatch, the historical path). @@ -119,7 +128,17 @@ def __init__(self, **kwargs): # reset_router_at_documents; True/False decouples it from the temporal and # router-locality terms. _lrd = kwargs.pop("reset_lookahead_at_documents", None) + if isinstance(_lrd, str): # web UI tri-state: "" follows the router switch + _lrd = None if _lrd.strip() == "" else _lrd.strip().lower() in ("1", "true", "yes", "on") self.reset_lookahead_at_documents = None if _lrd is None else bool(_lrd) + # Two-stage lookahead decoupling. True: the LookaheadRouter reads a stop-gradient copy of + # the hidden state, so its losses train only the router and add no gradient path + # through the backbone (fine-tune the router on a frozen trunk). + self.lookahead_input_detach: bool = bool(kwargs.pop("lookahead_input_detach", False)) + # > 0: the temporal / locality terms keep gradient only for the newer token of a pair + # every this-many positions (blocks of that length, the older side is detached), so + # no gradient chain crosses a whole row. 0 = full chain (historical). + self.temporal_grad_span: int = int(kwargs.pop("temporal_grad_span", 0) or 0) # Long-context RoPE for training: {"type": "yarn"|"linear"|"ntk", "factor": f, # "original_max_position_embeddings": n, ...}; applied after the base init. _rts = kwargs.pop("rope_train_scaling", None) diff --git a/chronos/webui/i18n/en.json b/chronos/webui/i18n/en.json index 1ae7888..b922937 100644 --- a/chronos/webui/i18n/en.json +++ b/chronos/webui/i18n/en.json @@ -19,8 +19,11 @@ "choice.absorbed": "Absorbed", "choice.adaptive": "Adaptive", "choice.aggressive": "Aggressive", + "choice.all": "All blocks", + "choice.alternate": "Every second block", "choice.auto": "Auto", "choice.balanced": "Balanced", + "choice.block": "Whole block", "choice.cluster": "By cluster", "choice.cluster_locality_mode.segment": "Documents (segment)", "choice.cluster_locality_mode.window": "Fixed windows (window)", @@ -30,9 +33,12 @@ "choice.experts": "Active experts", "choice.extreme": "Extreme", "choice.fallback_diagnostic": "Never load, diagnostic (fallback_diagnostic)", + "choice.false": "Off", "choice.fixed": "Fixed", "choice.fixed_then_adaptive": "Fixed, then adaptive", "choice.fullload": "Full memory only", + "choice.mla": "Global (MLA) blocks", + "choice.mlp": "MoE/FFN only", "choice.none": "None", "choice.offload": "Lazy experts only", "choice.on_demand": "Load, may skip late experts (on_demand)", @@ -46,7 +52,9 @@ "choice.scheduled": "Scheduled", "choice.segment": "By document", "choice.shared": "Shared", + "choice.swa": "Sliding-window blocks", "choice.sync_on_demand": "Load and wait (sync_on_demand)", + "choice.true": "On", "choice.tune.search.lr": "Learning rate", "choice.window": "Recent-use window", "config.active": "Active per token", @@ -136,10 +144,14 @@ "field.adaptive_halting.label": "Adaptive halting (ACT)", "field.alpha.help": "Distillation mix: α × teacher KL + (1 − α) × CE.", "field.alpha.label": "α (distill)", + "field.attention_block_size.help": "Run long attention in query blocks of this many tokens, each reading only the keys it can see. 0 = one call. Exact; 1024 was fastest across the global and sliding-window layers.", + "field.attention_block_size.label": "Attention block size", "field.batch_size.help": "Sequences per micro-batch. Limited by memory.", "field.batch_size.label": "Batch size", "field.beta.help": "Strength of the KL tie to the reference policy in preference training.", "field.beta.label": "β (DPO/GRPO)", + "field.ce_chunk_size.help": "Compute the language-model loss in row chunks from the last hidden state, so the [batch, length, vocabulary] logits never exist. 0 = full logits. Exact.", + "field.ce_chunk_size.label": "Loss chunk size", "field.cluster_locality_clusters.help": "Number of expert clusters for the locality term. 0 chooses it from the expert count.", "field.cluster_locality_clusters.label": "Clusters", "field.cluster_locality_mode.help": "What one stream is for the cluster constraint. window: fixed windows of the length above. segment: each document of the batch (split at the data's own separators, long ones cut to the window), so similar documents learn to share a cluster without labels or task tags.", @@ -156,6 +168,8 @@ "field.depth_schedule.label": "Depth schedule", "field.depth_schedule_warmup_fraction.help": "Fraction of training that runs at fixed full depth before the schedule becomes adaptive.", "field.depth_schedule_warmup_fraction.label": "Schedule warm-up", + "field.document_masking.help": "Packed rows: attention stays inside each document, cut at the separator token.", + "field.document_masking.label": "Document masking", "field.draft_exit_depth.help": "Recurrent depth whose exit is trained as the speculative draft.", "field.draft_exit_depth.label": "Draft depth (training)", "field.draft_exit_kl_coef.help": "Extra KL between the draft-depth exit and the full-depth output, for self-speculative decoding. 0 disables it.", @@ -262,6 +276,8 @@ "field.learning_rate.label": "Learning rate", "field.log_interval.help": "Write a metrics row every N steps.", "field.log_interval.label": "Log interval", + "field.lookahead_input_detach.help": "The lookahead router reads a gradient-free copy of the hidden state, so its losses train only the router. Measured neutral on the en-zh data.", + "field.lookahead_input_detach.label": "Detach lookahead input", "field.lookahead_steps.help": "How many future tokens the lookahead router predicts experts for, so they can be prefetched before they are needed.", "field.lookahead_steps.label": "Lookahead steps", "field.max_gen_len.help": "Tokens generated per GRPO rollout.", @@ -282,8 +298,12 @@ "field.mla_cache_rope.label": "Cache RoPE keys", "field.mla_decode.help": "expanded: reference path. absorbed: folds projections into the query (less compute per step). auto mixes paths and showed a small logit gap, so it stays opt-in.", "field.mla_decode.label": "MLA decode path", + "field.mla_upproj_recompute.help": "Keep only the latent and rebuild K and V per attention block. Exact, but measured no memory gain at 64k, so it stays off.", + "field.mla_upproj_recompute.label": "MLA up-projection recompute", "field.modalities.help": "Extra input encoders, comma-separated (image, audio). Leave empty for text-only models; datasets are auto-detected.", "field.modalities.label": "Modalities", + "field.moe_chunk_tokens.help": "Dispatch tokens to the experts in chunks of this size, each recomputed in the backward pass. 0 = one dispatch. Exact.", + "field.moe_chunk_tokens.label": "MoE chunk tokens", "field.moe_intermediate_size.help": "Hidden width of each expert's feed-forward network. 0 derives it from the hidden size.", "field.moe_intermediate_size.label": "Expert FFN width", "field.num_attention_heads.help": "Query heads. Hidden size must divide evenly by this number.", @@ -370,6 +390,10 @@ "field.probe.verify_wait_s.label": "Verify wait (s)", "field.ram_budget_mb.help": "RAM the lazy expert cache may use, in MB. Empty uses the RAM ratio instead.", "field.ram_budget_mb.label": "RAM budget", + "field.recompute_policy.help": "Which blocks are recomputed when checkpointing is on: all, only global (MLA) blocks, only sliding-window blocks, or every second block.", + "field.recompute_policy.label": "Recompute blocks", + "field.recompute_scope.help": "block: recompute whole blocks (most memory saved). mlp: recompute only each block's MoE/FFN, which keeps the attention results and costs less time.", + "field.recompute_scope.label": "Recompute scope", "field.recurrent_attention_mode.help": "How attention runs inside the loop. shared_first: every iteration attends, K/V from iteration 0 (exact cache, KV does not grow with depth). per_iteration: one KV per iteration (D× KV). first_only: v2 behaviour.", "field.recurrent_attention_mode.label": "Recurrent attention", "field.recurrent_depth.help": "Iterations of each recurrent group during training. Virtual layers = prelude + groups × depth + coda.", @@ -386,6 +410,10 @@ "field.recurrent_stability_coef.label": "Stability loss", "field.recurrent_strategy.help": "Sampling strategy for the recurrent depth bucket during training.", "field.recurrent_strategy.label": "Depth strategy", + "field.reset_lookahead_at_documents.help": "Empty follows the router switch above; true or false sets the lookahead terms on their own.", + "field.reset_lookahead_at_documents.label": "Reset lookahead at documents", + "field.reset_router_at_documents.help": "Packed rows: the temporal and locality losses skip token pairs that straddle a document boundary.", + "field.reset_router_at_documents.label": "Reset router at documents", "field.reward_spec.help": "toy for the built-in reward, or lm:/path/to/reward-model[,device=cuda].", "field.reward_spec.label": "Reward (GRPO)", "field.rope_dim.help": "Dimensions per head that carry rotary position information in MLA layers.", @@ -434,6 +462,8 @@ "field.storage_format.label": "Expert storage", "field.temperature.help": "Sampling temperature for GRPO rollouts.", "field.temperature.label": "Rollout temperature", + "field.temporal_grad_span.help": "Cut the temporal-locality gradient chain every this many positions. 0 = full chain. Measured neutral on the en-zh data.", + "field.temporal_grad_span.label": "Temporal gradient span", "field.tie_word_embeddings.help": "Share the input embedding with the output head. Saves vocab × hidden parameters with little quality cost at small sizes.", "field.tie_word_embeddings.label": "Tie embeddings", "field.tok.fields.help": "JSON fields to read. Empty auto-detects text, conversations and pairs.", diff --git a/chronos/webui/i18n/ja.json b/chronos/webui/i18n/ja.json index b8d04de..ccb4e97 100644 --- a/chronos/webui/i18n/ja.json +++ b/chronos/webui/i18n/ja.json @@ -19,8 +19,11 @@ "choice.absorbed": "吸収", "choice.adaptive": "適応", "choice.aggressive": "積極的", + "choice.all": "すべてのブロック", + "choice.alternate": "1つおきのブロック", "choice.auto": "自動", "choice.balanced": "バランス", + "choice.block": "ブロック全体", "choice.cluster": "クラスタ単位", "choice.cluster_locality_mode.segment": "文書単位(segment)", "choice.cluster_locality_mode.window": "固定ウィンドウ(window)", @@ -30,9 +33,12 @@ "choice.experts": "アクティブエキスパート数", "choice.extreme": "最大限", "choice.fallback_diagnostic": "読み込まない・診断用(fallback_diagnostic)", + "choice.false": "オフ", "choice.fixed": "固定", "choice.fixed_then_adaptive": "固定の後に適応", "choice.fullload": "全メモリのみ", + "choice.mla": "グローバル(MLA)ブロック", + "choice.mlp": "MoE/FFN のみ", "choice.none": "なし", "choice.offload": "遅延エキスパートのみ", "choice.on_demand": "読み込み、遅延したエキスパートは省略(on_demand)", @@ -46,7 +52,9 @@ "choice.scheduled": "スケジュール", "choice.segment": "文書単位", "choice.shared": "共有", + "choice.swa": "スライディングウィンドウブロック", "choice.sync_on_demand": "読み込んで待機(sync_on_demand)", + "choice.true": "オン", "choice.tune.search.lr": "学習率", "choice.window": "最近使用ウィンドウ", "config.active": "トークンあたりのアクティブパラメータ", @@ -136,10 +144,14 @@ "field.adaptive_halting.label": "適応的停止(ACT)", "field.alpha.help": "蒸留の混合比:α × 教師 KL + (1 − α) × CE。", "field.alpha.label": "α(蒸留)", + "field.attention_block_size.help": "長いアテンションをこのトークン数のクエリブロックで計算し、各ブロックは見えるキーだけを読みます。0 で分割なし。結果は厳密で、1024 がグローバル層とウィンドウ層の合計で最速でした。", + "field.attention_block_size.label": "アテンションのブロックサイズ", "field.batch_size.help": "マイクロバッチあたりの系列数。メモリで制限されます。", "field.batch_size.label": "バッチサイズ", "field.beta.help": "選好学習で参照方策への KL の結びつきの強さ。", "field.beta.label": "β(DPO/GRPO)", + "field.ce_chunk_size.help": "最後の隠れ状態から行チャンクごとに言語モデルの損失を計算し、[バッチ, 長さ, 語彙] の logits を作りません。0 で全 logits。結果は厳密です。", + "field.ce_chunk_size.label": "損失のチャンクサイズ", "field.cluster_locality_clusters.help": "局所性項で使うエキスパートクラスタの数。0 ならエキスパート数から決めます。", "field.cluster_locality_clusters.label": "クラスタ数", "field.cluster_locality_mode.help": "クラスタ制約で一続きの入力とみなす単位。window: 上の長さの固定ウィンドウ。segment: バッチ内の各文書(データ自身の区切りで分割し、長い文書はウィンドウ長で再分割)。ラベルやタスクタグなしで、似た文書が同じクラスタを使うよう自発的に学習します。", @@ -156,6 +168,8 @@ "field.depth_schedule.label": "深さスケジュール", "field.depth_schedule_warmup_fraction.help": "スケジュールが適応に切り替わる前に、固定の最大深さで学習する割合。", "field.depth_schedule_warmup_fraction.label": "スケジュールのウォームアップ", + "field.document_masking.help": "パックした行で、アテンションを区切りトークンで分けた各ドキュメント内に限定します。", + "field.document_masking.label": "ドキュメントマスク", "field.draft_exit_depth.help": "投機的ドラフトとして出口を学習させる再帰深さ。", "field.draft_exit_depth.label": "ドラフト深さ(学習)", "field.draft_exit_kl_coef.help": "自己投機的デコード用に、ドラフト深さの出口と最大深さの出力の間に追加する KL。0 で無効。", @@ -262,6 +276,8 @@ "field.learning_rate.label": "学習率", "field.log_interval.help": "N ステップごとにメトリクスを 1 行書き出します。", "field.log_interval.label": "ログ間隔", + "field.lookahead_input_detach.help": "ルックアヘッドルーターは勾配を持たない隠れ状態のコピーを読み、その損失はルーターだけを学習します。中英データでは影響は測定されませんでした。", + "field.lookahead_input_detach.label": "ルックアヘッド入力の勾配遮断", "field.lookahead_steps.help": "先読みルーターが何トークン先までエキスパートを予測するか。必要になる前にプリフェッチできます。", "field.lookahead_steps.label": "先読みステップ", "field.max_gen_len.help": "GRPO のロールアウトごとに生成するトークン数。", @@ -282,8 +298,12 @@ "field.mla_cache_rope.label": "RoPE キーをキャッシュ", "field.mla_decode.help": "expanded:参照経路。absorbed:射影をクエリに畳み込みます(1 ステップあたりの計算が少ない)。auto は経路を混在させ、わずかなロジット差が出たため、明示的に選んだ場合のみ使います。", "field.mla_decode.label": "MLA デコード経路", + "field.mla_upproj_recompute.help": "潜在ベクトルだけを保持し、アテンションブロックごとに K と V を再構築します。厳密ですが 64k ではメモリ削減が測定されなかったため既定はオフです。", + "field.mla_upproj_recompute.label": "MLA 上位射影の再計算", "field.modalities.help": "追加の入力エンコーダ(image、audio をカンマ区切り)。テキストのみのモデルでは空欄にします。データセットは自動判定されます。", "field.modalities.label": "モダリティ", + "field.moe_chunk_tokens.help": "このサイズのチャンクでトークンをエキスパートに振り分け、各チャンクは逆伝播で再計算します。0 で一括。結果は厳密です。", + "field.moe_chunk_tokens.label": "MoE チャンクのトークン数", "field.moe_intermediate_size.help": "各エキスパートのフィードフォワードネットワークの隠れ幅。0 なら隠れ層サイズから求めます。", "field.moe_intermediate_size.label": "エキスパート FFN 幅", "field.num_attention_heads.help": "クエリヘッドの数。隠れ層サイズはこの数で割り切れる必要があります。", @@ -370,6 +390,10 @@ "field.probe.verify_wait_s.label": "検証待機(秒)", "field.ram_budget_mb.help": "遅延エキスパートキャッシュが使える RAM(MB)。空欄なら RAM 比率を使います。", "field.ram_budget_mb.label": "RAM 予算", + "field.recompute_policy.help": "チェックポイント有効時に再計算するブロック: すべて、グローバル(MLA)のみ、スライディングウィンドウのみ、または1つおき。", + "field.recompute_policy.label": "再計算するブロック", + "field.recompute_scope.help": "block: ブロック全体を再計算(メモリ削減が最大)。mlp: 各ブロックの MoE/FFN のみ再計算し、アテンション結果を保持して時間を抑えます。", + "field.recompute_scope.label": "再計算の範囲", "field.recurrent_attention_mode.help": "ループ内でのアテンションの実行方法。shared_first:毎反復でアテンションし、K/V は反復 0 のもの(キャッシュは厳密で、KV は深さで増えない)。per_iteration:反復ごとに KV を持つ(KV が D 倍)。first_only:v2 の挙動。", "field.recurrent_attention_mode.label": "再帰内アテンション", "field.recurrent_depth.help": "学習中に各再帰グループを反復する回数。仮想層数 = プレリュード + グループ数 × 深さ + コーダ。", @@ -386,6 +410,10 @@ "field.recurrent_stability_coef.label": "安定化損失", "field.recurrent_strategy.help": "学習時に再帰深さのバケットをサンプリングする戦略。", "field.recurrent_strategy.label": "深さの戦略", + "field.reset_lookahead_at_documents.help": "空欄は上のルーター設定に従い、true / false でルックアヘッド項だけを個別に設定します。", + "field.reset_lookahead_at_documents.label": "文書境界でルックアヘッドをリセット", + "field.reset_router_at_documents.help": "パックした行で、時間的局所性と局所性の損失が文書境界をまたぐトークン対を除外します。", + "field.reset_router_at_documents.label": "文書境界でルーターをリセット", "field.reward_spec.help": "組み込み報酬なら toy、または lm:/path/to/reward-model[,device=cuda]。", "field.reward_spec.label": "報酬(GRPO)", "field.rope_dim.help": "MLA 層で回転位置情報を持つヘッドあたりの次元数。", @@ -434,6 +462,8 @@ "field.storage_format.label": "エキスパート保存形式", "field.temperature.help": "GRPO ロールアウトのサンプリング温度。", "field.temperature.label": "ロールアウト温度", + "field.temporal_grad_span.help": "この位置数ごとに時間的局所性損失の勾配の連鎖を切ります。0 で連鎖を切りません。中英データでは影響は測定されませんでした。", + "field.temporal_grad_span.label": "時間勾配のスパン", "field.tie_word_embeddings.help": "入力埋め込みを出力ヘッドと共有します。語彙×隠れ層サイズ分のパラメータを節約でき、小規模モデルでは品質低下はわずかです。", "field.tie_word_embeddings.label": "埋め込みの共有", "field.tok.fields.help": "読み込む JSON フィールド。空欄なら text、conversations、pairs を自動検出します。", diff --git a/chronos/webui/i18n/zh-Hans.json b/chronos/webui/i18n/zh-Hans.json index 4dd0950..f69a411 100644 --- a/chronos/webui/i18n/zh-Hans.json +++ b/chronos/webui/i18n/zh-Hans.json @@ -19,8 +19,11 @@ "choice.absorbed": "吸收", "choice.adaptive": "自适应", "choice.aggressive": "激进", + "choice.all": "全部块", + "choice.alternate": "隔块", "choice.auto": "自动", "choice.balanced": "均衡", + "choice.block": "整个块", "choice.cluster": "按簇", "choice.cluster_locality_mode.segment": "按文档(segment)", "choice.cluster_locality_mode.window": "固定窗口(window)", @@ -30,9 +33,12 @@ "choice.experts": "激活专家数", "choice.extreme": "极限", "choice.fallback_diagnostic": "不加载,仅诊断(fallback_diagnostic)", + "choice.false": "关", "choice.fixed": "固定", "choice.fixed_then_adaptive": "先固定,后自适应", "choice.fullload": "仅全内存", + "choice.mla": "全局(MLA)块", + "choice.mlp": "仅 MoE/FFN", "choice.none": "无", "choice.offload": "仅懒加载专家", "choice.on_demand": "按需加载,可跳过迟到专家(on_demand)", @@ -46,7 +52,9 @@ "choice.scheduled": "按调度", "choice.segment": "按文档", "choice.shared": "共享", + "choice.swa": "滑动窗口块", "choice.sync_on_demand": "同步按需加载(sync_on_demand)", + "choice.true": "开", "choice.tune.search.lr": "学习率", "choice.window": "最近使用窗口", "config.active": "每 token 激活参数", @@ -136,10 +144,14 @@ "field.adaptive_halting.label": "自适应停止(ACT)", "field.alpha.help": "蒸馏混合比例:α × 教师 KL + (1 − α) × CE。", "field.alpha.label": "α(蒸馏)", + "field.attention_block_size.help": "把长注意力按这么多个 token 的查询块计算,每块只读取可见的键。0 表示不分块。结果精确;1024 在全局层和滑动窗口层上整体最快。", + "field.attention_block_size.label": "注意力分块大小", "field.batch_size.help": "每个微批的序列数,受内存限制。", "field.batch_size.label": "批大小", "field.beta.help": "偏好训练中与参考策略 KL 约束的强度。", "field.beta.label": "β(DPO/GRPO)", + "field.ce_chunk_size.help": "按行分块、从最后的隐状态计算语言模型损失,不再生成 [批, 长度, 词表] 的 logits。0 表示完整 logits。结果精确。", + "field.ce_chunk_size.label": "损失分块大小", "field.cluster_locality_clusters.help": "局部性约束使用的专家簇数。0 表示按专家数自动选择。", "field.cluster_locality_clusters.label": "簇数", "field.cluster_locality_mode.help": "簇约束以什么作为一段连续输入。window:按上面的长度切固定窗口。segment:批次中的每篇文档(按数据自身的分隔符切分,过长的按窗口再切),相似文档会自发学到同一个簇,不依赖标签或任务标记。", @@ -156,6 +168,8 @@ "field.depth_schedule.label": "深度调度", "field.depth_schedule_warmup_fraction.help": "训练中先以固定全深度运行的比例,之后才切换为自适应。", "field.depth_schedule_warmup_fraction.label": "调度预热比例", + "field.document_masking.help": "打包行:注意力只在每篇文档内部,以分隔 token 为界。", + "field.document_masking.label": "文档掩码", "field.draft_exit_depth.help": "作为推测草稿来训练的循环深度。", "field.draft_exit_depth.label": "草稿深度(训练)", "field.draft_exit_kl_coef.help": "草稿深度出口与全深度输出之间的额外 KL,用于自推测解码。0 表示关闭。", @@ -262,6 +276,8 @@ "field.learning_rate.label": "学习率", "field.log_interval.help": "每 N 步写一行指标。", "field.log_interval.label": "日志间隔", + "field.lookahead_input_detach.help": "前瞻路由读取不带梯度的隐状态副本,其损失只训练路由本身。在中英数据上实测影响不大。", + "field.lookahead_input_detach.label": "前瞻输入断梯度", "field.lookahead_steps.help": "前瞻路由为未来多少个 token 预测专家,以便在用到之前预取。", "field.lookahead_steps.label": "前瞻步数", "field.max_gen_len.help": "每次 GRPO 采样生成的 token 数。", @@ -282,8 +298,12 @@ "field.mla_cache_rope.label": "缓存 RoPE 键", "field.mla_decode.help": "展开:参考路径。吸收:把投影折叠进查询(每步计算更少)。自动会混用两条路径,实测有微小 logit 偏差,因此需手动开启。", "field.mla_decode.label": "MLA 解码路径", + "field.mla_upproj_recompute.help": "只保留潜向量,按注意力块重建 K 和 V。结果精确,但在 64k 实测没有省内存,所以默认关闭。", + "field.mla_upproj_recompute.label": "MLA 上投影重算", "field.modalities.help": "额外的输入编码器,用逗号分隔(image、audio)。纯文本模型留空,数据集会自动检测。", "field.modalities.label": "模态", + "field.moe_chunk_tokens.help": "按这个大小分块把 token 分发给专家,每块在反向传播时重算。0 表示一次性分发。结果精确。", + "field.moe_chunk_tokens.label": "MoE 分块 token 数", "field.moe_intermediate_size.help": "每个专家前馈网络的隐藏宽度。0 表示按隐藏维度推导。", "field.moe_intermediate_size.label": "专家 FFN 宽度", "field.num_attention_heads.help": "查询头数。隐藏维度必须能被它整除。", @@ -370,6 +390,10 @@ "field.probe.verify_wait_s.label": "校验等待(秒)", "field.ram_budget_mb.help": "懒加载专家缓存可用的内存,单位 MB。留空则改用内存比例。", "field.ram_budget_mb.label": "内存预算", + "field.recompute_policy.help": "开启检查点时重算哪些块:全部、仅全局(MLA)块、仅滑动窗口块,或隔块重算。", + "field.recompute_policy.label": "重算的块", + "field.recompute_scope.help": "block:重算整个块(最省内存)。mlp:只重算每块的 MoE/FFN,保留注意力结果,耗时更少。", + "field.recompute_scope.label": "重算范围", "field.recurrent_attention_mode.help": "循环内注意力的运行方式。shared_first:每轮都做注意力,K/V 取第 0 轮(缓存精确,KV 不随深度增长)。per_iteration:每轮一份 KV(D 倍 KV)。first_only:v2 行为。", "field.recurrent_attention_mode.label": "循环注意力", "field.recurrent_depth.help": "训练时每个循环组的迭代次数。虚拟层数 = 前置 + 组数 × 深度 + 收尾。", @@ -386,6 +410,10 @@ "field.recurrent_stability_coef.label": "稳定性损失", "field.recurrent_strategy.help": "训练时循环深度分桶的采样策略。", "field.recurrent_strategy.label": "深度策略", + "field.reset_lookahead_at_documents.help": "留空则跟随上面的路由开关;true 或 false 单独设置前瞻项。", + "field.reset_lookahead_at_documents.label": "文档边界重置前瞻", + "field.reset_router_at_documents.help": "打包行:时间局部性与局部性损失跳过跨文档边界的 token 对。", + "field.reset_router_at_documents.label": "文档边界重置路由", "field.reward_spec.help": "toy 使用内置奖励,或 lm:/path/to/reward-model[,device=cuda]。", "field.reward_spec.label": "奖励(GRPO)", "field.rope_dim.help": "MLA 层每个头中携带旋转位置信息的维度数。", @@ -434,6 +462,8 @@ "field.storage_format.label": "专家存储格式", "field.temperature.help": "GRPO 采样的温度。", "field.temperature.label": "采样温度", + "field.temporal_grad_span.help": "每隔这么多个位置截断时间局部性损失的梯度链。0 表示完整链。在中英数据上实测影响不大。", + "field.temporal_grad_span.label": "时间梯度跨度", "field.tie_word_embeddings.help": "输入嵌入与输出头共用权重。可省下 词表 × 隐藏维度 个参数,小模型几乎不损失质量。", "field.tie_word_embeddings.label": "共享嵌入", "field.tok.fields.help": "要读取的 JSON 字段。留空自动识别 text、conversations 和 pairs。", diff --git a/chronos/webui/i18n/zh-Hant.json b/chronos/webui/i18n/zh-Hant.json index 49d0faf..c4ac22a 100644 --- a/chronos/webui/i18n/zh-Hant.json +++ b/chronos/webui/i18n/zh-Hant.json @@ -19,8 +19,11 @@ "choice.absorbed": "吸收", "choice.adaptive": "自適應", "choice.aggressive": "積極", + "choice.all": "全部區塊", + "choice.alternate": "隔塊", "choice.auto": "自動", "choice.balanced": "平衡", + "choice.block": "整個區塊", "choice.cluster": "按叢集", "choice.cluster_locality_mode.segment": "按文件(segment)", "choice.cluster_locality_mode.window": "固定窗口(window)", @@ -30,9 +33,12 @@ "choice.experts": "啟用專家數", "choice.extreme": "極限", "choice.fallback_diagnostic": "不載入,僅診斷(fallback_diagnostic)", + "choice.false": "關", "choice.fixed": "固定", "choice.fixed_then_adaptive": "先固定,後自適應", "choice.fullload": "僅全記憶體", + "choice.mla": "全域(MLA)區塊", + "choice.mlp": "僅 MoE/FFN", "choice.none": "無", "choice.offload": "僅延遲載入專家", "choice.on_demand": "按需載入,可略過遲到專家(on_demand)", @@ -46,7 +52,9 @@ "choice.scheduled": "依排程", "choice.segment": "按文件", "choice.shared": "共用", + "choice.swa": "滑動視窗區塊", "choice.sync_on_demand": "同步按需載入(sync_on_demand)", + "choice.true": "開", "choice.tune.search.lr": "學習率", "choice.window": "最近使用視窗", "config.active": "每 token 啟用參數", @@ -136,10 +144,14 @@ "field.adaptive_halting.label": "自適應停止(ACT)", "field.alpha.help": "蒸餾混合比例:α × 教師 KL + (1 − α) × CE。", "field.alpha.label": "α(蒸餾)", + "field.attention_block_size.help": "把長注意力按這麼多個 token 的查詢塊計算,每塊只讀取可見的鍵。0 表示不分塊。結果精確;1024 在全域層與滑動視窗層上整體最快。", + "field.attention_block_size.label": "注意力分塊大小", "field.batch_size.help": "每個微批次的序列數,受記憶體限制。", "field.batch_size.label": "批次大小", "field.beta.help": "偏好訓練中與參考策略 KL 約束的強度。", "field.beta.label": "β(DPO/GRPO)", + "field.ce_chunk_size.help": "按列分塊、從最後的隱狀態計算語言模型損失,不再產生 [批, 長度, 詞表] 的 logits。0 表示完整 logits。結果精確。", + "field.ce_chunk_size.label": "損失分塊大小", "field.cluster_locality_clusters.help": "區域性約束使用的專家叢集數。0 表示按專家數自動選擇。", "field.cluster_locality_clusters.label": "叢集數", "field.cluster_locality_mode.help": "簇約束以什麼作為一段連續輸入。window:按上面的長度切固定窗口。segment:批次中的每篇文件(按資料自身的分隔符切分,過長的按窗口再切),相似文件會自發學到同一個簇,不依賴標籤或任務標記。", @@ -156,6 +168,8 @@ "field.depth_schedule.label": "深度排程", "field.depth_schedule_warmup_fraction.help": "訓練中先以固定全深度執行的比例,之後才切換為自適應。", "field.depth_schedule_warmup_fraction.label": "排程暖機比例", + "field.document_masking.help": "打包列:注意力只在每篇文件內部,以分隔 token 為界。", + "field.document_masking.label": "文件遮罩", "field.draft_exit_depth.help": "作為推測草稿來訓練的循環深度。", "field.draft_exit_depth.label": "草稿深度(訓練)", "field.draft_exit_kl_coef.help": "草稿深度出口與全深度輸出之間的額外 KL,用於自推測解碼。0 表示關閉。", @@ -262,6 +276,8 @@ "field.learning_rate.label": "學習率", "field.log_interval.help": "每 N 步寫一行指標。", "field.log_interval.label": "日誌間隔", + "field.lookahead_input_detach.help": "前瞻路由讀取不帶梯度的隱狀態副本,其損失只訓練路由本身。在中英資料上實測影響不大。", + "field.lookahead_input_detach.label": "前瞻輸入斷梯度", "field.lookahead_steps.help": "前瞻路由為未來多少個 token 預測專家,以便在用到之前預先擷取。", "field.lookahead_steps.label": "前瞻步數", "field.max_gen_len.help": "每次 GRPO 取樣生成的 token 數。", @@ -282,8 +298,12 @@ "field.mla_cache_rope.label": "快取 RoPE 鍵", "field.mla_decode.help": "展開:參考路徑。吸收:把投影折入查詢(每步運算更少)。自動會混用兩條路徑,實測有微小 logit 偏差,因此需手動開啟。", "field.mla_decode.label": "MLA 解碼路徑", + "field.mla_upproj_recompute.help": "只保留潛向量,按注意力塊重建 K 與 V。結果精確,但在 64k 實測沒有省記憶體,所以預設關閉。", + "field.mla_upproj_recompute.label": "MLA 上投影重算", "field.modalities.help": "額外的輸入編碼器,以逗號分隔(image、audio)。純文字模型留空,資料集會自動偵測。", "field.modalities.label": "模態", + "field.moe_chunk_tokens.help": "按這個大小分塊把 token 分發給專家,每塊在反向傳播時重算。0 表示一次性分發。結果精確。", + "field.moe_chunk_tokens.label": "MoE 分塊 token 數", "field.moe_intermediate_size.help": "每個專家前饋網路的隱藏寬度。0 表示按隱藏維度推導。", "field.moe_intermediate_size.label": "專家 FFN 寬度", "field.num_attention_heads.help": "查詢頭數。隱藏維度必須能被它整除。", @@ -370,6 +390,10 @@ "field.probe.verify_wait_s.label": "驗證等待(秒)", "field.ram_budget_mb.help": "延遲載入專家快取可用的記憶體,單位 MB。留空則改用記憶體比例。", "field.ram_budget_mb.label": "記憶體預算", + "field.recompute_policy.help": "開啟檢查點時重算哪些區塊:全部、僅全域(MLA)區塊、僅滑動視窗區塊,或隔塊重算。", + "field.recompute_policy.label": "重算的區塊", + "field.recompute_scope.help": "block:重算整個區塊(最省記憶體)。mlp:只重算每個區塊的 MoE/FFN,保留注意力結果,耗時較少。", + "field.recompute_scope.label": "重算範圍", "field.recurrent_attention_mode.help": "循環內注意力的執行方式。shared_first:每輪都做注意力,K/V 取第 0 輪(快取精確,KV 不隨深度成長)。per_iteration:每輪一份 KV(D 倍 KV)。first_only:v2 行為。", "field.recurrent_attention_mode.label": "循環注意力", "field.recurrent_depth.help": "訓練時每個循環組的迭代次數。虛擬層數 = 前置 + 組數 × 深度 + 收尾。", @@ -386,6 +410,10 @@ "field.recurrent_stability_coef.label": "穩定性損失", "field.recurrent_strategy.help": "訓練時循環深度分桶的取樣策略。", "field.recurrent_strategy.label": "深度策略", + "field.reset_lookahead_at_documents.help": "留空則跟隨上面的路由開關;true 或 false 單獨設定前瞻項。", + "field.reset_lookahead_at_documents.label": "文件邊界重置前瞻", + "field.reset_router_at_documents.help": "打包列:時間局部性與局部性損失跳過跨文件邊界的 token 對。", + "field.reset_router_at_documents.label": "文件邊界重置路由", "field.reward_spec.help": "toy 使用內建獎勵,或 lm:/path/to/reward-model[,device=cuda]。", "field.reward_spec.label": "獎勵(GRPO)", "field.rope_dim.help": "MLA 層每個頭中攜帶旋轉位置資訊的維度數。", @@ -434,6 +462,8 @@ "field.storage_format.label": "專家儲存格式", "field.temperature.help": "GRPO 取樣的溫度。", "field.temperature.label": "取樣溫度", + "field.temporal_grad_span.help": "每隔這麼多個位置截斷時間局部性損失的梯度鏈。0 表示完整鏈。在中英資料上實測影響不大。", + "field.temporal_grad_span.label": "時間梯度跨度", "field.tie_word_embeddings.help": "輸入嵌入與輸出頭共用權重。可省下 詞彙表 × 隱藏維度 個參數,小模型幾乎不損失品質。", "field.tie_word_embeddings.label": "共用嵌入", "field.tok.fields.help": "要讀取的 JSON 欄位。留空自動辨識 text、conversations 與 pairs。", diff --git a/chronos/webui/schema.py b/chronos/webui/schema.py index 2aae9e2..8039113 100644 --- a/chronos/webui/schema.py +++ b/chronos/webui/schema.py @@ -150,6 +150,21 @@ def _f(key, group, type_, default=None, **kw) -> Field: _f("page_size_mb", "memory", "int", 64, min=1, max=4096, step=1, advanced=True), _f("optimizer_page_dir", "memory", "path", "./optimizer_pages", advanced=True), _f("activation_checkpointing", "memory", "bool", False), + _f("recompute_scope", "memory", "choice", "block", choices=["block", "mlp"], recommended="block", source="code", + advanced=True), + _f("recompute_policy", "memory", "choice", "all", choices=["all", "mla", "swa", "alternate"], advanced=True), + _f("attention_block_size", "memory", "int", 0, min=0, max=65536, step=256, recommended=1024, source="measured", + ref="reports/long_context_20261109"), + _f("ce_chunk_size", "memory", "int", 0, min=0, max=65536, step=256, recommended=1024, source="measured", + ref="reports/long_context_20261109"), + _f("moe_chunk_tokens", "memory", "int", 0, min=0, max=1048576, step=1024, recommended=8192, source="measured", + ref="reports/long_context_20261109"), + _f("mla_upproj_recompute", "memory", "bool", False, advanced=True), + _f("document_masking", "objectives", "bool", False), + _f("reset_router_at_documents", "objectives", "bool", False), + _f("reset_lookahead_at_documents", "objectives", "choice", "", choices=["", "true", "false"], advanced=True), + _f("lookahead_input_detach", "objectives", "bool", False, advanced=True), + _f("temporal_grad_span", "objectives", "int", 0, min=0, max=1048576, step=256, advanced=True), # ---- optimizer --------------------------------------------------------- _f("learning_rate", "optimizer", "float", 5e-4, min=1e-7, max=1.0, step=1e-5, recommended="planner", source="planner"), diff --git a/pyproject.toml b/pyproject.toml index a9c4a09..b07567a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "Project_Chronos" -version = "0.3.2a1" +version = "0.3.2a2" description = "Storage-aware recurrent-depth MoE: lookahead expert prefetch, SSD-lazy experts, hybrid MLA/SWA attention, self-speculative decoding" readme = "README.md" license = "Apache-2.0" diff --git a/reports/long_context_20261109/real_corpus_ab.json b/reports/long_context_20261109/real_corpus_ab.json new file mode 100644 index 0000000..4c8f552 --- /dev/null +++ b/reports/long_context_20261109/real_corpus_ab.json @@ -0,0 +1,150 @@ +{ + "date": "2026-11-09", + "machine": "Apple M5 Max, 128 GB, MLX 0.32.4, bfloat16", + "method": "real en-zh shards (Translation Training Data/shards_v1), production 50M model_config, random init, same seed and data order per arm, 3-6 optimizer steps, valid CE on 6 fixed validation windows. long = attention_block_size 1024 + ce_chunk_size 1024 + moe_chunk_tokens 8192. Few steps: the CE column shows equal quality between arms, not final model quality.", + "real_corpus": [ + { + "arm": "base", + "seq": 8192, + "tok_per_s": 8884, + "peak_gb": 40.64, + "valid_ce": 8.4561 + }, + { + "arm": "long", + "seq": 8192, + "tok_per_s": 19466, + "peak_gb": 7.51, + "valid_ce": 8.4585 + }, + { + "arm": "base", + "seq": 16384, + "tok_per_s": 1877, + "peak_gb": 141.5, + "valid_ce": 8.7118 + }, + { + "arm": "long", + "seq": 16384, + "tok_per_s": 15226, + "peak_gb": 13.81, + "valid_ce": 8.7121 + }, + { + "arm": "long", + "seq": 32768, + "tok_per_s": 8792, + "peak_gb": 27.01, + "valid_ce": 8.8754 + }, + { + "arm": "long+recompute_block", + "seq": 32768, + "tok_per_s": 7158, + "peak_gb": 20.16, + "valid_ce": 8.8757 + }, + { + "arm": "long+recompute_mlp", + "seq": 32768, + "tok_per_s": 7794, + "peak_gb": 25.64, + "valid_ce": 8.8756 + }, + { + "arm": "long", + "seq": 65536, + "tok_per_s": 3690, + "peak_gb": 54.64, + "valid_ce": 9.0575 + }, + { + "arm": "long+recompute_block", + "seq": 65536, + "tok_per_s": 3512, + "peak_gb": 46.85, + "valid_ce": 9.0575 + } + ], + "lookahead_decoupling_real_corpus": [ + { + "arm": "reset_router_at_documents", + "seq": 16384, + "tok_per_s": 17231, + "valid_ce": 8.3937 + }, + { + "arm": "+temporal_grad_span=1024", + "seq": 16384, + "tok_per_s": 17267, + "valid_ce": 8.3946 + }, + { + "arm": "+lookahead_input_detach", + "seq": 16384, + "tok_per_s": 17018, + "valid_ce": 8.3938 + } + ], + "synthetic_batch1_h512": { + "block_size_full_model_64k": { + "1024": [ + 9.909, + 24.47 + ], + "2048": [ + 10.979, + 27.83 + ], + "4096": [ + 12.799, + 35.93 + ], + "8192": [ + 16.602, + 56.71 + ] + }, + "mla_layer_64k_block_sweep_s_and_MiB": { + "512": [ + 0.713, + 3108 + ], + "1024": [ + 0.62, + 2511 + ], + "2048": [ + 0.565, + 2197 + ], + "4096": [ + 0.535, + 1987 + ], + "8192": [ + 0.527, + 1865 + ] + }, + "mla_upproj_recompute_full_model_64k": { + "off": [ + 9.598, + 24.47 + ], + "on": [ + 10.242, + 24.29 + ] + } + }, + "conclusions": [ + "long vs base: 2.2x faster at 8k, 8.1x at 16k, with 5.4x and 10.2x less peak memory; base at 16k used 141.5 GB of the 128 GB machine.", + "valid CE differs by at most 3e-3 between arms: the options do not change what the model learns.", + "recompute_scope=mlp at 32k: saves 1.4 GB (5%) and loses 11% speed; whole-block recompute saves 6.9 GB (25%) and loses 19% speed. The memory saved per speed lost is better for whole-block, so mlp is only worth it when a little more memory is enough.", + "mla_upproj_recompute: no memory benefit (24.47 -> 24.29 GiB), 6.7% slower; ships off.", + "larger attention blocks help one MLA layer but hurt the full model (sliding-window layers); 1024 stays.", + "temporal_grad_span and lookahead_input_detach: neutral on this data (valid CE within 1e-3); ship off." + ] +} \ No newline at end of file diff --git a/tests/test_long_context.py b/tests/test_long_context.py index d6a611e..16fdbb8 100644 --- a/tests/test_long_context.py +++ b/tests/test_long_context.py @@ -363,3 +363,107 @@ def test_torch_recompute_policy_selects_expected_layers(): for policy, want in expect.items(): model.config.recompute_policy = policy assert [model._recompute_block(layer) for layer in model.layers] == want + + +# ---- 0.3.2a2: narrow recompute scope and MLA up-projection recompute ---- + +@pytest.mark.parametrize("miss", [0.0, 0.4]) +def test_mlp_scope_recompute_keeps_loss_and_gradients(miss): + from chronos.mlx.training.trainer import _ce_stage_loss + ref_cfg = _config(fallback_mask_prob=miss) + sub_cfg = _config(fallback_mask_prob=miss, activation_checkpointing=True, recompute_scope="mlp") + ids, labels = _batch(ref_cfg) + mx.random.seed(5) + a = _grads(_model(ref_cfg), _ce_stage_loss, ids, labels) + mx.random.seed(5) + b = _grads(_model(sub_cfg), _ce_stage_loss, ids, labels) + _grad_equal(a, b, tol=5e-4) + + +def test_recompute_scope_is_validated_and_registered(): + from chronos.config_schema import MODEL_FIELDS + assert _config().recompute_scope == "block" + with pytest.raises(ValueError): + _config(recompute_scope="attention") + assert {"recompute_scope", "mla_upproj_recompute"} <= set(MODEL_FIELDS) + + +@pytest.mark.parametrize("segments", [False, True]) +def test_mla_upproj_recompute_equals_expanded_kv(segments): + from chronos.mlx.training.trainer import _ce_stage_loss + kw = dict(attention_block_size=8, rope_layout="per_head", document_masking=segments) + ref_cfg, up_cfg = _config(**kw), _config(mla_upproj_recompute=True, **kw) + ids, labels = _batch(ref_cfg) + if segments: + ids = mx.where(mx.arange(ids.shape[1])[None, :] % 13 == 12, 0, ids) + _grad_equal(_grads(_model(ref_cfg), _ce_stage_loss, ids, labels), + _grads(_model(up_cfg), _ce_stage_loss, ids, labels)) + + +def _flat_grads(cfg): + from chronos.mlx.training.trainer import _ce_stage_loss + ids, labels = _batch(cfg) + return _grads(_model(cfg), _ce_stage_loss, ids, labels)[1] + + +def test_lookahead_input_detach_keeps_loss_and_cuts_trunk_gradient(): + kw = dict(lambda_lookahead=5.0, lambda_lookahead_topk=0.0, lambda_lookahead_union=0.0, + lambda_temporal=0.0, lambda_router_locality=0.0, lambda_balance=0.0, ponder_loss_coef=0.0) + off = _flat_grads(_config(lookahead_input_detach=False, **kw)) + on = _flat_grads(_config(lookahead_input_detach=True, **kw)) + router = [n for n in off if 'lookahead_router' in n] + assert router + for n in router: # the router keeps its full gradient + assert _rel(off[n], on[n]) < 1e-5, n + # the trunk loses the router's gradient path: some trunk tensor changes beyond rounding + trunk = [n for n in off if 'lookahead_router' not in n] + assert max(_rel(off[n], on[n]) for n in trunk) > 1e-4 + + +def test_temporal_grad_span_changes_only_gradients(): + import numpy as np + from chronos.mlx.training import trainer as T + mx.random.seed(3) + probs = mx.softmax(mx.random.normal((2, 40, 6)), axis=-1) + loss = lambda p: T._temporal_locality_loss(p) # noqa: E731 + T._TEMPORAL_SPAN[0] = 0 + full_v, full_g = mx.value_and_grad(loss)(probs) + T._TEMPORAL_SPAN[0] = 8 + try: + span_v, span_g = mx.value_and_grad(loss)(probs) + finally: + T._TEMPORAL_SPAN[0] = 0 + mx.eval(full_v, full_g, span_v, span_g) + assert float(full_v.item()) == pytest.approx(float(span_v.item()), rel=1e-6) # the value is identical + diff = np.abs(np.array(full_g) - np.array(span_g)) + assert diff.max() > 1e-6 # the gradient is cut + assert np.allclose(np.array(span_g)[:, 1:7], np.array(full_g)[:, 1:7], atol=1e-6) # inside a block it is intact + + +def test_new_lookahead_fields_are_registered(): + from chronos.config_schema import MODEL_FIELDS + assert {"lookahead_input_detach", "temporal_grad_span"} <= set(MODEL_FIELDS) + assert _config().lookahead_input_detach is False and _config().temporal_grad_span == 0 + + +def test_segment_ids_mx_matches_numpy_and_traces_under_compile(): + import numpy as np + from chronos.expert_clusters import segment_ids, segment_ids_mx + ids = np.random.default_rng(0).integers(0, 5, size=(3, 41)) + assert np.array_equal(np.array(segment_ids_mx(mx.array(ids))), segment_ids(ids)) + compiled = mx.compile(segment_ids_mx) + assert np.array_equal(np.array(compiled(mx.array(ids))), segment_ids(ids)) + + +def test_document_resets_work_in_the_compiled_step(): + from chronos.mlx.training.compiled import make_grad_fn + from chronos.mlx.training.trainer import _ce_stage_loss + cfg = _config(reset_router_at_documents=True, document_masking=True, attention_block_size=8) + ids, labels = _batch(cfg) + ids = mx.where(mx.arange(ids.shape[1])[None, :] % 11 == 10, 0, ids) + eager = _grads(_model(cfg), _ce_stage_loss, ids, labels) + model = _model(cfg) + step = make_grad_fn(model, _ce_stage_loss, mode="on", log=lambda m: None) + loss, grads = step(model, ids, labels) + mx.eval(loss, grads) + assert abs(loss.item() - eager[0].item()) < 1e-4