From bef5a15e35d5915aee5fe66c22c8cce7654ea184 Mon Sep 17 00:00:00 2001 From: Fona <62028332+FonaTech@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:04:59 +0800 Subject: [PATCH] 0.3.2a0: native kernel, compiled MLX step, MLX activation recomputation - chronos/model/kernel.py replaces the minimind kernel: no clone, no sys.path changes, offline import; logits bit-identical on the existing checkpoints; one shared FP32 RoPE table (3.50 -> 0.50 MiB at 50M). - Dense GQA mode works on Torch and MLX (both crashed before); BF16 release checkpoints keep their dtype; trainer logs reach the Web UI. - decode_visible keeps / tags that a custom tokenizer registers as special tokens (serving, workbench, GRPO rewards). - MLX: compiled micro step by default (1.08x SFT, 1.18x pretrain measured on the 50M en->zh config) and activation recomputation with replayed draws and per-dtype packed region outputs (mlx 0.32.4 drops gradients of repeated region outputs; peak 5.62 -> 3.50 GB SFT). - RoPE tables built single-threaded (1-ulp prefix mismatch at 18 threads). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 38 + README.md | 90 +- README_zh.md | 88 +- THIRD_PARTY_NOTICES.md | 47 +- benchmark_compare.py | 33 +- chronos/__init__.py | 6 +- chronos/app.py | 3 +- chronos/assets/__init__.py | 0 chronos/assets/tokenizers/legacy_6400/LICENSE | 201 + chronos/assets/tokenizers/legacy_6400/NOTICE | 15 + .../tokenizers/legacy_6400/tokenizer.json | 31191 ++++++++++++++++ .../legacy_6400/tokenizer_config.json | 335 + chronos/data/flexible_dataset.py | 10 +- chronos/deps.py | 94 +- chronos/eval/benchmark.py | 5 +- chronos/eval/io_profiler.py | 5 +- chronos/io/cluster_layout.py | 5 +- chronos/io/expert_store.py | 5 +- chronos/mlx/__init__.py | 3 +- chronos/mlx/attention.py | 25 +- chronos/mlx/checkpointing.py | 246 + chronos/mlx/inference.py | 4 +- chronos/mlx/model.py | 12 +- chronos/mlx/moe.py | 50 +- chronos/mlx/recurrent.py | 3 + chronos/mlx/training/compiled.py | 242 + chronos/mlx/training/longrun.py | 9 +- chronos/mlx/training/trainer.py | 18 +- chronos/model/checkpoint.py | 3 +- chronos/model/config.py | 11 +- chronos/model/hf_io.py | 35 +- chronos/model/hybrid_attention.py | 37 +- chronos/model/kernel.py | 317 + chronos/model/model_chronos.py | 37 +- chronos/model/moe_chronos.py | 11 +- chronos/model/precision_ops.py | 12 +- chronos/model/temporal_loss.py | 2 +- chronos/runtime/inference_engine.py | 3 - chronos/serving/adapter.py | 4 +- chronos/serving/runtime.py | 4 +- chronos/tokenizer/visible.py | 72 + chronos/trainer/chronos_trainer.py | 21 +- chronos/trainer/distill_trainer.py | 7 +- chronos/trainer/dpo_trainer.py | 3 +- chronos/trainer/grpo_trainer.py | 3 +- chronos/trainer/loss_mixin.py | 2 +- chronos/trainer/optim_utils.py | 2 +- chronos/trainer/orpo_trainer.py | 3 +- chronos/trainer/reward.py | 14 +- chronos/trainer/sft_trainer.py | 14 +- chronos/trainer/stage_utils.py | 6 +- chronos/tuning/chronos_auto_tuner.py | 1 - chronos/verify.py | 1 - chronos/webui/i18n/en.json | 6 +- chronos/webui/i18n/ja.json | 6 +- chronos/webui/i18n/zh-Hans.json | 6 +- chronos/webui/i18n/zh-Hant.json | 6 +- chronos/webui/server.py | 2 +- chronos/workbench/data.py | 1 - chronos/workbench/inference.py | 6 +- chronos/workbench/training.py | 34 +- demos/translation/README.md | 10 +- demos/translation/README_zh.md | 8 +- .../chronos_translate/bundle/builder.py | 28 +- .../chronos_translate/bundle/docs.py | 20 +- demos/translation/pyproject.toml | 8 +- .../scripts/make_docmix_packages.py | 12 +- demos/translation/tests/test_bundle.py | 3 +- diagnose_checkpoint.py | 2 +- pyproject.toml | 3 +- tests/test_interface_regressions.py | 4 +- tests/test_lazy_experts.py | 4 +- tests/test_longrun.py | 58 + tests/test_mlx_checkpointing.py | 225 + tests/test_native_kernel.py | 236 + tests/test_planner_ui.py | 2 +- tests/test_smoke.py | 53 +- tests/test_smoke_cuda.py | 1 - tests/test_tool_tags_special_tokens.py | 102 + tools/_minimind_baseline.py | 43 + tools/benchmark_chronos_v2.py | 1 - tools/compare_minimind_chronos.py | 8 +- tools/compare_minimind_chronos_v2.py | 6 +- tools/compare_minimind_chronos_v3.py | 3 +- train_chronos.py | 3 +- train_chronos_distill.py | 3 +- train_chronos_dpo.py | 1 - train_chronos_grpo.py | 1 - train_chronos_orpo.py | 1 - train_chronos_sft.py | 1 - ui/estimator.py | 6 +- ui/i18n.py | 8 +- ui/presets.py | 46 +- ui/tabs/autotune_tab.py | 1 - ui/tabs/benchmark_tab.py | 15 +- ui/tabs/config_tab.py | 10 +- ui/tabs/data_tab.py | 1 - ui/tabs/inference_tab.py | 4 +- ui/tabs/train_tab.py | 1 - 99 files changed, 33931 insertions(+), 486 deletions(-) create mode 100644 chronos/assets/__init__.py create mode 100644 chronos/assets/tokenizers/legacy_6400/LICENSE create mode 100644 chronos/assets/tokenizers/legacy_6400/NOTICE create mode 100644 chronos/assets/tokenizers/legacy_6400/tokenizer.json create mode 100644 chronos/assets/tokenizers/legacy_6400/tokenizer_config.json create mode 100644 chronos/mlx/checkpointing.py create mode 100644 chronos/mlx/training/compiled.py create mode 100644 chronos/model/kernel.py create mode 100644 chronos/tokenizer/visible.py create mode 100644 tests/test_mlx_checkpointing.py create mode 100644 tests/test_native_kernel.py create mode 100644 tests/test_tool_tags_special_tokens.py create mode 100644 tools/_minimind_baseline.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 237adc1..f4602e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,44 @@ ## [Unreleased] +## [0.3.2a0] — 2026-11-08 (beta) + +This release replaces the minimind kernel with a native one, compiles the MLX training step, adds MLX activation recomputation, and keeps tool-call tags that a custom tokenizer registers as special tokens. README "Native kernel (0.3.2)" has the migration reasons and the measured comparison. + +### Changed +- `chronos/model/kernel.py` replaces the minimind kernel (config base, RoPE/YaRN, gated FFN, dense attention, trainer utilities). + - `import chronos` no longer clones anything or changes `sys.path`, and works offline. + - Config field names and defaults, RoPE tables and FFN parameter names are unchanged. Logits are bit-identical on `out/chronos_256_moe.pth` and the 50M en→zh checkpoint (full, padded and prefill+decode paths). + - All layers share one FP32 RoPE table per `(dim, base, scaling, device)`: 3.50 → 0.50 MiB at the 50M shape, 216 → 24 MiB at the 768H base shape. + - Resume state is written to `chronos__moe_resume.pth`; the release checkpoint keeps its dtype. +- `chronos.deps` is a shim. `get_tokenizer_path()` returns the bundled legacy 6400-token tokenizer (`CHRONOS_DEFAULT_TOKENIZER` overrides it); `ensure_minimind()` is a no-op with a deprecation warning. +- `tools/compare_minimind_chronos*.py` use minimind only as an external baseline, from `--minimind-dir` or `CHRONOS_MINIMIND_DIR`. Model bundles no longer ship `third_party/minimind`. +- `benchmark_compare.py` compares Chronos with a dense GQA baseline built from the same kernel (result key `baseline`, formerly `minimind`). +- The preset "MiniMind-MoE (default)" is now "Chronos-v2 legacy (768H, 6400 vocab)", and `MINIMIND_MOE_DEFAULTS` is now `BASE_MOE_DEFAULTS`. The old names remain as aliases. +- MLX training micro steps are compiled by default (`chronos longrun`, and `run_mlx_stage` pretrain and SFT). `CHRONOS_MLX_COMPILE=auto|on|off` or a recipe's `extra.mlx_compile` switches it. + +### Added +- `chronos.mlx.training.compiled`: one `mx.compile` graph per signature for loss and gradients. + - The recurrent depth is sampled once per step outside the trace. Depth, the ACT switch, shapes and plain config values select the graph. + - Weights and the RNG state are compile inputs and outputs, so dropout and the cache-miss simulation consume the RNG stream as in eager mode. + - Router probabilities, loss components and recurrent diagnostics are returned from the graph and written back to the modules. + - Multimodal batches, lazy experts and signatures that fail to trace run eagerly; the reason is logged once. + - Measured on real en→zh shards with the 50M config (M5 Max, bf16, shared GPU): 1.08× (SFT) and 1.18× (pretrain) per step, loss curves within 2e-3 of eager. +- `chronos.mlx.checkpointing`: MLX activation recomputation for `activation_checkpointing=true`. + - Each block call is one recomputed region. Dropout masks, the cache-miss draws and routing indices are recorded in the forward pass and replayed during recomputation. + - Region outputs are packed per dtype. mlx 0.32.4 keeps the gradient of only one copy when a checkpointed function returns the same graph node twice, and a full-length cache slice is such a copy. + - Measured peak memory: 5.62 → 3.50 GB (SFT) and 6.07 → 3.62 GB (pretrain), at 17% and 21% more time per step than the compiled step alone. It stays off by default at 50M; the planner's memory-constrained tiers enable it. +- `chronos.tokenizer.visible.decode_visible`: hides control tokens and keeps ``, ``, `` and `` as text, for HF tokenizers and `ChronosTokenizer`. Serving, the workbench and GRPO rewards use it. +- Tests: `tests/test_native_kernel.py`, `tests/test_tool_tags_special_tokens.py`, `tests/test_mlx_checkpointing.py`, and per-micro-step exactness of the compiled and recomputed steps inside the LongRun loop (`tests/test_longrun.py`). + +### Fixed +- Tool calls were lost when a custom HF tokenizer registered `` as a special token: decoding with `skip_special_tokens=True` removed the tags, so the parser saw bare JSON. +- Dense mode (`use_hybrid_attention=False`) crashed on Torch in every recurrent mode, and on MLX with an int32 overflow (a 2**63 window). It now runs dense causal GQA; prefill+decode matches the full forward within 1e-6. +- The Torch pretrain save rewrote the BF16 release checkpoint as FP16. +- Trainer log lines did not reach the Web UI. +- RoPE tables: with many torch threads, a cached table's prefix differed by one ulp from a shorter table (measured with 18 threads on torch 2.10). The tables are now built single-threaded. +- `tools/compare_minimind_chronos*.py` used `chronos.deps` without importing it. + ## [0.3.1a3] — 2026-11-08 (beta) This release fixes the router masks and the lazy-expert decode. It also adds a document-level cluster objective, MLX multimodal training in all six stages, tool-call SFT records and tool-call parsing in `chronos serve`, a kernel-level MLX cache limit and progress-based ETAs. diff --git a/README.md b/README.md index 1726d3d..e1ade87 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,72 @@ Web UI: the Tokenizer tab (extract → train → evaluate) and the Serve tab (st --- +## Native kernel (0.3.2): minimind removed + +Up to 0.3.1a3, Chronos took its config base, RoPE tables, gated FFN, dense attention and trainer utilities from [minimind](https://github.com/jingyaogong/minimind). It loaded them at import time: `chronos.deps` git-cloned the `master` branch into `~/.cache/chronos/minimind-master/` and put that folder first on `sys.path`. 0.3.2 replaces those pieces with `chronos/model/kernel.py`. Nothing is downloaded, and `sys.path` is not changed. + +**Why it was migrated** + +- **Reproducibility.** The clone followed an unpinned `master`, so two installs could run different kernel code. +- **Offline installs.** On a machine without the cache, `import chronos` ran `git clone` and failed without network access. +- **Namespace collisions.** minimind's top-level `model`, `trainer` and `dataset` packages went first on `sys.path`. They shadowed any user module with the same name. +- **Dense mode was broken.** `use_hybrid_attention=False` used minimind's `Attention`, which crashed in every recurrent mode (`The size of tensor a (8) must match the size of tensor b (1024)`). It also could not use the shared KV cache. +- **BF16 checkpoints were overwritten as FP16.** The Torch pretrain save called minimind's `lm_checkpoint`. That function rewrote the release file `chronos__moe.pth` as FP16 right after Chronos had saved it in BF16. +- **Trainer logs were lost.** The Web UI captured trainer output by patching `trainer_utils.Logger`. The trainers import `Logger` by name at load time, so the patch never reached them. + +**What the native kernel does** + +- `ChronosBaseConfig` keeps the old field names and defaults. RoPE/YaRN tables and FFN parameter names are unchanged too, so existing checkpoints and sidecars load as before. +- One FP32 RoPE table per `(dim, base, scaling, device)` is shared by all layers and grows on demand. Before, every attention layer kept its own copy. +- `GlobalAttention` is the dense GQA path. It is the sliding-window kernel without a window, so all three recurrent modes, int8 KV, padding masks and speculative rollback work on it. +- Logging uses a sink registry (`add_log_sink`). Resume state goes to `chronos__moe_resume.pth`, and the release checkpoint keeps its dtype. +- The 6400-token tokenizer that old checkpoints use ships as data only (`chronos/assets/tokenizers/legacy_6400`, Apache-2.0, sha256 recorded in its `NOTICE`). + +**Old vs new, measured 2026-11-08** + +Setup: Apple M5 Max, PyTorch 2.10 on CPU, FP32, 4 threads, median of 5–7 runs, each kernel in its own process, runs alternated twice. The 50M en→zh SFT run was training on the GPU at the same time, so absolute timings carry noise. Old = 0.3.1a3 + minimind `24896cd`. + +| Check | 0.3.1a3 + minimind | 0.3.2 native | +|---|---|---| +| Logits, `out/chronos_256_moe.pth` and the 50M en→zh checkpoint (full, padded and prefill+decode paths) | reference | bit-identical (max abs diff 0) | +| Random-init `state_dict` and config values (same seed) | reference | identical | +| `import chronos` with no cache and no network | fails (`git clone` error) | works | +| minimind on `sys.path`, top-level `model` module imported | yes | no | +| RoPE table memory, 50M shape (1024 positions) | 3.50 MiB | 0.50 MiB | +| RoPE table memory, 768H base shape (32768 positions) | 216 MiB | 24 MiB | +| Dense GQA mode (`use_hybrid_attention=False`) | crashes | works; prefill+decode vs full forward ≤ 1e-6 | +| Release checkpoint dtype after BF16 Torch pretrain | FP16 | BF16 | +| Trainer lines captured by the Web UI sink | 0 of 2 | 2 of 2 | +| Train step, 50M shape, batch 4×128 | 0.318–0.326 s | 0.317–0.321 s | +| Prefill 128 tokens, 50M shape | 40.0–40.7 ms | 41.4–42.8 ms | +| Decode tokens/s, 50M shape | 146–157 | 146–151 | + +A repeat with 40 prefill runs per process gave medians of 64–86 ms (old) and 74–86 ms (new) as the training load changed; neither kernel was consistently faster. Compute speed is unchanged within noise: the kernel keeps the same math. The gains are in memory, robustness and the bugs listed above. The parity check runs in CI as `tests/test_native_kernel.py`, which covers config defaults, RoPE bit-identity, dense mode, imports, logging, resume and the bundled tokenizer. + +`chronos.deps` remains as a shim: `get_tokenizer_path()` returns the bundled tokenizer (`CHRONOS_DEFAULT_TOKENIZER` overrides it), and `ensure_minimind()` is a no-op with a deprecation warning. The comparison tools in `tools/compare_minimind_chronos*.py` still use minimind as an external baseline, from a checkout you provide (`--minimind-dir` or `CHRONOS_MINIMIND_DIR`). `benchmark_compare.py` now compares Chronos with a dense GQA baseline built from the same kernel. + +### MLX training: compiled step and activation recomputation (0.3.2) + +Up to 0.3.1a3 the MLX trainers ran every micro step eagerly and kept every activation for the backward pass. 0.3.2 adds both missing pieces to `chronos longrun` and to the Stage 1–2 MLX trainer (pretrain, SFT): + +- **Compiled micro step** (`chronos/mlx/training/compiled.py`, on by default). Loss and gradients run as one `mx.compile` graph per signature. The executed recurrent depth, the ACT switch, input shapes and plain config values are part of the signature, so a step never reuses a stale graph. Weights and the RNG state are compile inputs and outputs, so dropout and the cache-miss simulation consume the RNG stream exactly as before. Multimodal batches, lazy (paged) experts and any signature that fails to trace run eagerly, and the reason is logged once. `CHRONOS_MLX_COMPILE=auto|on|off` or the recipe's `extra.mlx_compile` switches it. +- **Activation recomputation** (`chronos/mlx/checkpointing.py`, `activation_checkpointing=true`; the planner's memory-constrained tiers turn it on). Each block call becomes one recomputed region. A plain `mx.checkpoint` is wrong here in two ways, both measured on mlx 0.32.4. Dropout masks and the cache-miss draws would be drawn again during recomputation; they are recorded in the forward pass and replayed. A region that returns the same graph node twice keeps the gradient of only one copy. A block's KV cache and its shared view are such a pair whenever the SWA window covers the sequence (a full-length slice is a new Python object over the same node), and the gradients were 37–40% off. Region outputs are therefore packed into one array per dtype. + +**Measured 2026-11-08.** Setup: Apple M5 Max, mlx 0.32.4, bf16, the 50M en→zh recipe's model config from random init, real en→zh shards, the recipe's step sizes (SFT 8 micro-batches per step, pretrain 16). Median of the second half of 16 (SFT) or 12 (pretrain) steps. The 72-hour run was training on the same GPU, so absolute timings carry noise. + +| Micro step | SFT s/step | SFT peak memory | Pretrain s/step | Pretrain peak memory | +|---|---|---|---|---| +| Eager (0.3.1a3) | 2.915 | 5.62 GB | 7.154 | 6.07 GB | +| Compiled (new default) | 2.700 (1.08×) | 5.86 GB | 6.057 (1.18×) | 6.28 GB | +| Recomputation + compiled | 3.148 | 3.50 GB | 7.357 | 3.62 GB | +| Recomputation, eager | 3.663 | 3.42 GB | — | — | + +The loss curves of all variants follow the eager one: within 2e-3 over 16 SFT steps and 2e-4 over 12 pretrain steps. The first compiled step also traces the graph. Recomputation cuts peak memory by 38–40%. With compilation on, a step then takes 17% (SFT) or 21% (pretrain) longer than the compiled step without recomputation, and 8% or 3% longer than the old eager step. At 50M parameters memory is not the limit, so recomputation stays off by default; it is meant for larger models or longer sequences on the same machine. + +Exactness is tested per micro step on the same weights, batch and RNG state inside the real LongRun loop (`tests/test_longrun.py`, `tests/test_mlx_checkpointing.py`). On CPU FP32 the recomputed step's gradients match the eager ones within 4e-7 relative, the compiled step's within 1.7e-5, and with both on within 1e-4. That is FP32 reassociation: scaling the weights by (1 + 1e-7·N(0,1)) moves the eager gradient by 1.8e-4. On the M5 GPU, FP32 matrix products round like TF32. There, the eager gradient itself differs from the CPU FP32 one by 1.5e-2, and the recomputed one differs from the eager one by 1.4e-4. + +--- + ## Three-tier storage (M1: cluster-aware safetensors) ```mermaid @@ -294,7 +360,7 @@ Memory, media and tools on MLX: - Training: the chat template renders `tools`, assistant `tool_calls` and `tool` results. `chronos.data.chat_records.encode_chat` gives each token a loss flag (assistant text and calls are learned; everything else is context), and `write_sft_records` writes `.lmask` shards that `SFTRecords` reads. - Serving: when a request carries `tools`, `chronos serve` parses the model's `` output into Ollama `message.tool_calls` or OpenAI `tool_calls`, in streaming and non-streaming responses. -The full six-stage comparison harness lives in `tools/compare_minimind_chronos_v3.py`. +The full six-stage comparison harness lives in `tools/compare_minimind_chronos_v3.py` (needs a minimind checkout via `--minimind-dir`). --- @@ -468,10 +534,7 @@ pip install -e demos/translation # en↔zh demo extension, from a check pip install vllm ``` -> **minimind dependency**: Project Chronos uses [minimind](https://github.com/jingyaogong/minimind) as its MoE kernel. -> If it is not found locally, Chronos clones it automatically into `~/.cache/chronos/minimind-master/` on first import (this needs `git` and network access). -> Set `CHRONOS_MINIMIND_DIR=/path/to/minimind` to use a local copy instead; model bundles ship one under `third_party/minimind` and their launcher sets the variable. -> minimind is licensed under **Apache-2.0**. See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for attribution details. +> **No external kernel**: since 0.3.2 the kernel is self-contained (`chronos/model/kernel.py`). Installing and importing Chronos needs no `git` and no network. See [Native kernel (0.3.2)](#native-kernel-032-minimind-removed) and [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). **Requirements**: Python 3.10+, PyTorch 2.6+ @@ -573,7 +636,7 @@ Bundles belong to the translation extension (`demos/translation`). Build the two ```bash chronos-translate bundle build --run-dir runs/enzh_50m_v1 --which best --pack \ - --wheel dist/project_chronos-0.3.1a3-py3-none-any.whl --wheel dist/chronos_translate-0.3.1a3-py3-none-any.whl + --wheel dist/project_chronos-0.3.2a0-py3-none-any.whl --wheel dist/chronos_translate-0.3.2a0-py3-none-any.whl ``` | Bundle path | Content | @@ -581,8 +644,7 @@ chronos-translate bundle build --run-dir runs/enzh_50m_v1 --which best --pack \ | `model/` | weights + `.config.json` sidecar, tokenizer (`model/tokenizer/`) | | `eval/` | FLORES+ dev/devtest (CC BY-SA 4.0) for `bundle eval` / `probe` | | `data/sft_valid/` | the run's SFT validation shards | -| `third_party/minimind/` | vendored minimind (Apache-2.0), so no git clone is needed | -| `wheels/`, `bin/setup.sh`, `bin/chronos` | offline install of the kernel and extension wheels into `.venv`; launcher that sets `CHRONOS_BUNDLE` / `CHRONOS_MINIMIND_DIR` and sends `bundle …` to `chronos-translate` | +| `wheels/`, `bin/setup.sh`, `bin/chronos` | offline install of the kernel and extension wheels into `.venv`; launcher that sets `CHRONOS_BUNDLE` and sends `bundle …` to `chronos-translate` | | `recipes/source_run_spec.json`, `recommended.json` | the recipe that produced the weights; fine-tune tiers `smoke` / `quick` / `standard` / `full` | | `bundle.json` | provenance (step, tokens, valid CE), measured quality probe, SHA-256 of every file | @@ -675,8 +737,10 @@ python benchmark_training_backends.py --backends cpu mps mlx --dtypes auto bfloa ### End-to-end comparison (minimind vs Chronos) +minimind is an external baseline here; pass your own checkout (Chronos does not download it). + ```bash -python tools/compare_minimind_chronos_v3.py \ +python tools/compare_minimind_chronos_v3.py --minimind-dir /path/to/minimind \ --pretrain_steps 150 --align_steps 30 --distill_steps 30 \ --simulated_ssd_ms 30 --device cpu \ --output results/compare_results_v3.json @@ -729,9 +793,10 @@ tuner.start( ```text Project_Chronos/ ├── chronos/ -│ ├── deps.py # Auto-download minimind if missing +│ ├── deps.py # Shim: default tokenizer path (no downloads) │ ├── __init__.py # AutoConfig / AutoModelForCausalLM registration │ ├── model/ +│ │ ├── kernel.py # Native base: ChronosBaseConfig, shared RoPE/YaRN tables, FFN, GlobalAttention, logging, resume state │ │ ├── config.py # ChronosConfig (v3 fields, v2 migration) │ │ ├── recurrent.py # Loop/group embeddings, contractive step, tensor-only ACT │ │ ├── precision_ops.py # RMSNorm/SDPA/RoPE with one compute dtype @@ -756,7 +821,8 @@ Project_Chronos/ │ │ ├── attention.py / moe.py / model.py / expert_store.py / inference.py │ │ ├── moe_kernels.py # Sorted gather_mm dispatch (mlx >= 0.32.4) │ │ ├── speculative.py # MLX self-speculative decoding (Torch parity) -│ │ └── training/ # longrun.py, telemetry.py, recipes/ +│ │ ├── checkpointing.py # Activation recomputation with replayed draws +│ │ └── training/ # longrun.py, compiled.py (mx.compile step), telemetry.py, recipes/ │ ├── runtime/ │ │ ├── speculative.py # Recurrent-depth self-speculative decoding │ │ ├── cache_manager.py # prefetch_for_next_step / ensure_resident @@ -943,7 +1009,7 @@ mindmap ## Third-party attribution -Project Chronos builds on **jingyaogong**'s [minimind](https://github.com/jingyaogong/minimind), licensed under **Apache-2.0**. Full attribution lives in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). +Project Chronos started from **jingyaogong**'s [minimind](https://github.com/jingyaogong/minimind) (Apache-2.0). Since 0.3.2 no minimind code is used or downloaded. The native kernel keeps minimind's config field names and RoPE layout for checkpoint compatibility, and the legacy 6400-token tokenizer ships as data files with their licence. Full attribution lives in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). --- diff --git a/README_zh.md b/README_zh.md index b1a9a11..297b8b6 100644 --- a/README_zh.md +++ b/README_zh.md @@ -148,6 +148,72 @@ Web UI 新增 Tokenizer Tab（抽取 → 训练 → 评估）和 Serve Tab（启 --- +## 原生内核（0.3.2）：移除 minimind + +0.3.1a3 及之前，Chronos 的 config 基类、RoPE 表、门控 FFN、稠密注意力和训练工具函数都来自 [minimind](https://github.com/jingyaogong/minimind)，并在导入时加载：`chronos.deps` 把 `master` 分支 git clone 到 `~/.cache/chronos/minimind-master/`，再把这个目录插到 `sys.path` 最前面。0.3.2 用 `chronos/model/kernel.py` 替换了这些部分，不再下载任何东西，也不再改动 `sys.path`。 + +**迁移原因** + +- **可复现性。** clone 跟随未固定版本的 `master`，两次安装可能运行不同的内核代码。 +- **离线安装。** 机器上没有缓存时，`import chronos` 会执行 `git clone`，没有网络就失败。 +- **命名空间冲突。** minimind 的顶层包 `model`、`trainer`、`dataset` 排在 `sys.path` 最前面，会遮蔽用户同名的模块。 +- **稠密模式不可用。** `use_hybrid_attention=False` 走 minimind 的 `Attention`，在三种循环模式下都会崩溃（`The size of tensor a (8) must match the size of tensor b (1024)`），也无法使用共享 KV cache。 +- **BF16 checkpoint 被改写成 FP16。** Torch 预训练保存时调用 minimind 的 `lm_checkpoint`，它在 Chronos 刚以 BF16 保存发布文件 `chronos__moe.pth` 之后，又把它改写成 FP16。 +- **训练日志丢失。** Web UI 通过替换 `trainer_utils.Logger` 截获训练输出，但各 trainer 在加载时按名字导入了 `Logger`，替换到不了它们。 + +**原生内核做了什么** + +- `ChronosBaseConfig` 保留原有字段名和默认值；RoPE/YaRN 表和 FFN 参数名也不变，已有 checkpoint 和 sidecar 照常加载。 +- 每个 `(dim, base, scaling, device)` 只有一张 FP32 RoPE 表，所有层共享，按需增长。以前每个注意力层各存一份。 +- `GlobalAttention` 是稠密 GQA 路径，即不带窗口的滑动窗口内核，因此三种循环模式、int8 KV、padding mask 和推测解码回滚都能用。 +- 日志改用 sink 注册表（`add_log_sink`）。续跑状态写入 `chronos__moe_resume.pth`，发布 checkpoint 保持原 dtype。 +- 旧 checkpoint 使用的 6400 词表分词器只作为数据随包发布（`chronos/assets/tokenizers/legacy_6400`，Apache-2.0，sha256 记录在其 `NOTICE` 中）。 + +**新旧对比（2026-11-08 实测）** + +条件：Apple M5 Max，PyTorch 2.10 CPU，FP32，4 线程，取 5–7 次的中位数；每个内核在独立进程中运行，交替运行两轮。测量时 50M 英译中 SFT 正在 GPU 上训练，绝对耗时有噪声。旧版 = 0.3.1a3 + minimind `24896cd`。 + +| 检查项 | 0.3.1a3 + minimind | 0.3.2 原生 | +|---|---|---| +| Logits：`out/chronos_256_moe.pth` 与 50M 英译中 checkpoint（完整、padding、prefill+decode 三条路径） | 参考 | 逐位一致（最大绝对差 0） | +| 随机初始化的 `state_dict` 和 config 取值（同一种子） | 参考 | 一致 | +| 无缓存、无网络时 `import chronos` | 失败（`git clone` 报错） | 正常 | +| minimind 在 `sys.path` 上，导入了顶层 `model` 模块 | 是 | 否 | +| RoPE 表内存，50M 形状（1024 个位置） | 3.50 MiB | 0.50 MiB | +| RoPE 表内存，768H 基础形状（32768 个位置） | 216 MiB | 24 MiB | +| 稠密 GQA 模式（`use_hybrid_attention=False`） | 崩溃 | 可用；prefill+decode 与 full forward 差 ≤ 1e-6 | +| BF16 Torch 预训练后发布 checkpoint 的 dtype | FP16 | BF16 | +| Web UI sink 截获的训练日志行 | 2 行中 0 行 | 2 行中 2 行 | +| 训练一步，50M 形状，batch 4×128 | 0.318–0.326 s | 0.317–0.321 s | +| Prefill 128 token，50M 形状 | 40.0–40.7 ms | 41.4–42.8 ms | +| Decode tokens/s，50M 形状 | 146–157 | 146–151 | + +每个进程跑 40 次 prefill 的复测中，随训练负载变化，中位数为 64–86 ms（旧）和 74–86 ms（新），两者没有稳定的快慢之分。内核数学不变，计算速度在噪声范围内相同；收益在内存、健壮性和上面列出的 bug。CI 中的 `tests/test_native_kernel.py` 执行这套一致性检查，覆盖 config 默认值、RoPE 逐位一致、稠密模式、导入、日志、续跑和随包分词器。 + +`chronos.deps` 保留为兼容层：`get_tokenizer_path()` 返回随包分词器（可用 `CHRONOS_DEFAULT_TOKENIZER` 覆盖），`ensure_minimind()` 不做任何事并给出弃用警告。`tools/compare_minimind_chronos*.py` 仍把 minimind 当作外部基线，需自行提供 checkout（`--minimind-dir` 或 `CHRONOS_MINIMIND_DIR`）。`benchmark_compare.py` 改为对比 Chronos 与用同一内核搭建的稠密 GQA 基线。 + +### MLX 训练：编译的训练步与激活重计算（0.3.2） + +0.3.1a3 及之前，MLX 训练器的每个 micro step 都以 eager 方式执行，并为反向传播保留全部激活。0.3.2 为 `chronos longrun` 和 Stage 1–2 的 MLX 训练器（预训练、SFT）补上了这两项： + +- **编译的 micro step**（`chronos/mlx/training/compiled.py`，默认开启）。损失和梯度按签名各编译成一张 `mx.compile` 图。实际执行的循环深度、ACT 开关、输入形状和普通 config 取值都属于签名，所以训练步不会复用过期的图。权重和 RNG 状态同时是编译的输入和输出，dropout 与缓存未命中模拟消耗 RNG 流的方式与之前完全相同。多模态 batch、懒加载（分页）专家，以及任何 trace 失败的签名都以 eager 方式运行，并记录一次原因。用 `CHRONOS_MLX_COMPILE=auto|on|off` 或 recipe 的 `extra.mlx_compile` 切换。 +- **激活重计算**（`chronos/mlx/checkpointing.py`，`activation_checkpointing=true`；规划器的内存受限档位会自动开启）。每次 block 调用成为一个重计算区域。在这里直接用 `mx.checkpoint` 会出错，两处问题都在 mlx 0.32.4 上实测确认。第一，重计算时 dropout mask 和缓存未命中抽样会被重新抽取；现在前向时记录，重计算时回放。第二，区域两次返回同一个图节点时，只有一份能拿到梯度。只要 SWA 窗口覆盖整个序列，block 的 KV cache 和它的共享视图就是这样一对（全长切片是同一节点上的新 Python 对象），梯度偏差达 37–40%。因此区域输出按 dtype 打包成每种 dtype 一个数组。 + +**2026-11-08 实测。** 条件：Apple M5 Max，mlx 0.32.4，bf16，50M 英译中 recipe 的模型配置，随机初始化，真实英译中分片，recipe 的步长（SFT 每步 8 个 micro-batch，预训练 16 个）。取 16 步（SFT）或 12 步（预训练）后半段的中位数。72 小时训练任务同时在同一块 GPU 上运行，绝对耗时有噪声。 + +| Micro step | SFT s/步 | SFT 峰值内存 | 预训练 s/步 | 预训练峰值内存 | +|---|---|---|---|---| +| Eager（0.3.1a3） | 2.915 | 5.62 GB | 7.154 | 6.07 GB | +| 编译（新默认） | 2.700（1.08×） | 5.86 GB | 6.057（1.18×） | 6.28 GB | +| 重计算 + 编译 | 3.148 | 3.50 GB | 7.357 | 3.62 GB | +| 重计算，eager | 3.663 | 3.42 GB | — | — | + +各变体的 loss 曲线都与 eager 一致：16 步 SFT 内相差不超过 2e-3，12 步预训练内不超过 2e-4。编译的第一步还要 trace 计算图。重计算把峰值内存降低 38–40%。开启编译时，每步比不做重计算的编译步慢 17%（SFT）或 21%（预训练），比原来的 eager 步慢 8% 或 3%。50M 参数时内存不是瓶颈，所以重计算默认关闭；它面向同一台机器上更大的模型或更长的序列。 + +正确性在真实 LongRun 循环里按 micro step 检验：同一组权重、batch 和 RNG 状态下对比（`tests/test_longrun.py`、`tests/test_mlx_checkpointing.py`）。CPU FP32 上，重计算步的梯度与 eager 的相对差不超过 4e-7，编译步不超过 1.7e-5，两者同时开启不超过 1e-4。这属于 FP32 求和顺序的差异：把权重乘以 (1 + 1e-7·N(0,1))，eager 梯度本身就变化 1.8e-4。M5 GPU 上 FP32 矩阵乘按 TF32 精度舍入：eager 梯度与 CPU FP32 的差为 1.5e-2，重计算与 eager 的差为 1.4e-4。 + +--- + ## 三级存储架构（M1：cluster-aware safetensors） ```mermaid @@ -469,10 +535,7 @@ pip install -e demos/translation # 英↔中 demo 扩展 chronos-transl pip install vllm ``` -> **minimind 依赖**：Project Chronos 使用 [minimind](https://github.com/jingyaogong/minimind) 作为 MoE 内核。 -> 若本地未找到，首次 import 时自动克隆至 `~/.cache/chronos/minimind-master/`（需要 `git` 和网络）。 -> 设置 `CHRONOS_MINIMIND_DIR=/path/to/minimind` 可改用本地副本；模型包自带 `third_party/minimind`，启动脚本会设置这个变量。 -> minimind 采用 **Apache-2.0** 授权，完整归属见 [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)。 +> **无外部内核**：自 0.3.2 起内核自包含（`chronos/model/kernel.py`），安装和导入 Chronos 不需要 `git`，也不需要网络。见[原生内核（0.3.2）](#原生内核032移除-minimind)和 [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)。 **环境要求**：Python 3.10+，PyTorch 2.6+ @@ -577,7 +640,7 @@ OLLAMA_HOST=127.0.0.1:11435 ollama run <模型名> "你好" ```bash chronos-translate bundle build --run-dir runs/enzh_50m_v1 --which best --pack \ - --wheel dist/project_chronos-0.3.1a3-py3-none-any.whl --wheel dist/chronos_translate-0.3.1a3-py3-none-any.whl + --wheel dist/project_chronos-0.3.2a0-py3-none-any.whl --wheel dist/chronos_translate-0.3.2a0-py3-none-any.whl ``` | 包内路径 | 内容 | @@ -585,8 +648,7 @@ chronos-translate bundle build --run-dir runs/enzh_50m_v1 --which best --pack \ | `model/` | 权重 + `.config.json` 配置文件，分词器（`model/tokenizer/`） | | `eval/` | FLORES+ dev/devtest（CC BY-SA 4.0），供 `bundle eval` / `probe` 使用 | | `data/sft_valid/` | 该次运行的 SFT 验证分片 | -| `third_party/minimind/` | 内置 minimind（Apache-2.0），无需 git clone | -| `wheels/`、`bin/setup.sh`、`bin/chronos` | 把内核和扩展两个 wheel 离线安装到 `.venv`；启动脚本设置 `CHRONOS_BUNDLE` / `CHRONOS_MINIMIND_DIR`，并把 `bundle …` 转给 `chronos-translate` | +| `wheels/`、`bin/setup.sh`、`bin/chronos` | 把内核和扩展两个 wheel 离线安装到 `.venv`；启动脚本设置 `CHRONOS_BUNDLE`，并把 `bundle …` 转给 `chronos-translate` | | `recipes/source_run_spec.json`、`recommended.json` | 产生这份权重的配方；微调档位 `smoke` / `quick` / `standard` / `full` | | `bundle.json` | 来源信息（步数、token 数、valid CE）、实测质量探针、每个文件的 SHA-256 | @@ -675,8 +737,10 @@ python benchmark_training_backends.py --backends cpu mps mlx --dtypes auto bfloa ### 端到端对比（minimind vs Chronos） +这里 minimind 是外部基线，需自行提供 checkout（Chronos 不会下载它）。 + ```bash -python tools/compare_minimind_chronos_v3.py \ +python tools/compare_minimind_chronos_v3.py --minimind-dir /path/to/minimind \ --pretrain_steps 150 --align_steps 30 --distill_steps 30 \ --simulated_ssd_ms 30 --device cpu \ --output results/compare_results_v3.json @@ -738,9 +802,10 @@ tuner.start( ``` Project_Chronos/ ├── chronos/ -│ ├── deps.py # 自动下载 minimind（若本地未找到） +│ ├── deps.py # 兼容层：默认分词器路径（不下载任何东西） │ ├── __init__.py # 注册 AutoConfig / AutoModelForCausalLM │ ├── model/ +│ │ ├── kernel.py # 原生基础：ChronosBaseConfig、共享 RoPE/YaRN 表、FFN、GlobalAttention、日志、续跑状态 │ │ ├── config.py # ChronosConfig（lookahead/temporal/anchor/storage_format、v3 字段与 v2 迁移） │ │ ├── recurrent.py # 循环/组嵌入、收缩步长、纯张量 ACT │ │ ├── precision_ops.py # 单一计算精度的 RMSNorm/SDPA/RoPE @@ -765,7 +830,8 @@ Project_Chronos/ │ │ ├── attention.py / moe.py / model.py / expert_store.py / inference.py │ │ ├── moe_kernels.py # 排序 gather_mm 调度（mlx >= 0.32.4） │ │ ├── speculative.py # MLX 自推测解码（与 Torch 对齐） -│ │ └── training/ # longrun.py、telemetry.py、recipes/ +│ │ ├── checkpointing.py # 激活重计算（随机抽样回放） +│ │ └── training/ # longrun.py、compiled.py（mx.compile 训练步）、telemetry.py、recipes/ │ ├── runtime/ │ │ ├── speculative.py # 循环深度自推测解码 │ │ ├── cache_manager.py # prefetch_for_next_step / ensure_resident（M3） @@ -952,7 +1018,7 @@ mindmap ## 第三方归属 -Project Chronos 基于 **jingyaogong** 的 [minimind](https://github.com/jingyaogong/minimind)（Apache-2.0）构建。完整归属见 [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)。 +Project Chronos 起步于 **jingyaogong** 的 [minimind](https://github.com/jingyaogong/minimind)（Apache-2.0）。自 0.3.2 起不再使用或下载任何 minimind 代码。原生内核保留 minimind 的 config 字段名和 RoPE 布局，以兼容已有 checkpoint；旧版 6400 词表分词器作为数据文件随包发布，附带其许可证。完整归属见 [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)。 --- diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 918e351..355898b 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -13,36 +13,26 @@ below or linked. **License**: Apache License, Version 2.0 **Copyright**: Copyright jingyaogong and MiniMind Contributors -**Files derived from MiniMind**: +**Status since 0.3.2: no runtime dependency, no bundled code.** Releases up to +0.3.1a3 imported a minimind checkout at runtime (`chronos.deps` cloned +`master` into `~/.cache/chronos/minimind-master/` and put it on `sys.path`). +0.3.2 removes that dependency. The pieces Chronos used are re-implemented in +`chronos/model/kernel.py` (`ChronosBaseConfig`, RoPE/YaRN tables, gated FFN, +logging and resume utilities). Nothing is downloaded. -| Chronos file | Derived from MiniMind | -|---|---| -| `chronos/model/config.py` | `model/model_minimind.py` — `MiniMindConfig` | -| `chronos/model/model_chronos.py` | `model/model_minimind.py` — `MiniMindModel`, `MiniMindForCausalLM`, `MiniMindBlock` | -| `chronos/model/moe_chronos.py` | `model/model_minimind.py` — `MOEFeedForward`, `FeedForward` | -| `chronos/model/hybrid_attention.py` | `model/model_minimind.py` — `Attention`, `apply_rotary_pos_emb`, `repeat_kv`, `RMSNorm` | -| `chronos/model/lookahead_router.py` | Architecture inspired by MiniMind's routing design | -| `chronos/trainer/chronos_trainer.py` | `trainer/train_pretrain.py` — pretraining loop structure | -| `chronos/trainer/sft_trainer.py` | `trainer/train_full_sft.py` — SFT loop + chat-template loss masking | -| `chronos/trainer/dpo_trainer.py` | `trainer/train_dpo.py` — log-σ(β·logits) DPO formulation, `_logits_to_log_probs` helper | -| `chronos/trainer/grpo_trainer.py` | `trainer/train_grpo.py` — GRPO group-relative PG + KL formulation | -| `chronos/trainer/distill_trainer.py` | `trainer/train_distillation.py` — temperature-scaled KL distillation | -| `chronos/trainer/reward.py` | `trainer/trainer_utils.py` — `LMForRewardModel` ABI (`get_score`) wrapper | -| `chronos/eval/io_profiler.py` | `eval_llm.py` — evaluation patterns | - -**Nature of modifications**: +**What still derives from MiniMind**: -- `ChronosConfig` extends `MiniMindConfig` with new fields: `lookahead_steps`, hybrid attention dims, λ loss coefficients (balance / temporal / lookahead / router-anchor), VRAM budget, pinned RAM fraction, `storage_format`, `cluster_manifest_path`. -- `ChronosForCausalLM` replaces `MiniMindForCausalLM` with hybrid attention, LookaheadRouter injection, dual-layer routing hooks, an `available_expert_masks` forward kwarg, and `_tied_weights_keys` / `all_tied_weights_keys` shims for HF safetensors round-trips. -- `ChronosMOEFeedForward` extends `MOEFeedForward` with always-resident shared experts and compile-safe soft gating (`avail[i] * expert_out + (1−avail[i]) * shared_out`). -- `MLAAttention` and `SlidingWindowAttention` are new implementations sharing RMSNorm and RoPE utilities from MiniMind's `Attention`. -- The full Stage 1–6 trainer family (Pretrain / SFT / DPO / ORPO / GRPO / Distill) wraps the corresponding minimind loops where they exist (Pretrain / SFT / DPO / GRPO / Distill) and adds **ORPO from scratch** (Hong et al. 2024 — minimind has no ORPO trainer). Every wrapper injects `chronos_loss_term` (lookahead + temporal + balance) and an optional `router_kl_anchor` term. -- `ChronosGRPOTrainer` is **self-contained**: it ships its own rollout (vanilla `model.generate` style sampling) and accepts a pluggable `reward_fn`. It does not require minimind's sglang-based `rollout_engine`. +| Chronos file | What it keeps from MiniMind | +|---|---| +| `chronos/model/kernel.py` | Field names and defaults of `MiniMindConfig`, the rotate-half RoPE/YaRN layout and numerics, and the `gate_proj/up_proj/down_proj` FFN naming. They are kept so that checkpoints trained before 0.3.2 load bit-identically. The code is a new implementation. | +| `chronos/assets/tokenizers/legacy_6400/` | `tokenizer.json` and `tokenizer_config.json`, copied unchanged (data files only) from commit `24896cd2c4e9328a0911cf69ff61ace17a459b71`, with its `LICENSE` and a `NOTICE` that records the sha256 of each file. It is the default tokenizer only for old checkpoints that never recorded one. | +| `chronos/trainer/sft_trainer.py`, `dpo_trainer.py`, `grpo_trainer.py`, `distill_trainer.py` | The loop structure and loss formulations first followed minimind's `train_full_sft.py`, `train_dpo.py`, `train_grpo.py` and `train_distillation.py` (chat-template loss masking, log-σ(β·Δ) DPO, group-relative policy gradient with KL, temperature-scaled KD). These trainers are self-contained Chronos code. ORPO was written from scratch (Hong et al. 2024). | +| `chronos/trainer/reward.py` | The `get_score(tokenizer, messages)` reward-model ABI wrapper. | +| `tools/compare_minimind_chronos*.py` | Optional comparison harnesses that run minimind as an external baseline. They need a checkout the user provides through `--minimind-dir` or `CHRONOS_MINIMIND_DIR`. | -**Automatic download**: When `minimind-master` is not found locally, Project Chronos -automatically clones it from `https://github.com/jingyaogong/minimind` into -`~/.cache/chronos/minimind-master/`. This download is governed by the Apache-2.0 -license of the MiniMind project. +Chronos-specific parts (MLA/SWA hybrid attention, recurrent-depth controller, +LookaheadRouter, shared-expert MoE, lazy expert paging, speculative decoding, +MLX backend, tokenizer builder) are original work. **Apache-2.0 License Text**: @@ -60,7 +50,8 @@ See the License for the specific language governing permissions and limitations under the License. ``` -Full license: https://github.com/jingyaogong/minimind/blob/master/LICENSE +Full license: https://github.com/jingyaogong/minimind/blob/master/LICENSE (also shipped as +`chronos/assets/tokenizers/legacy_6400/LICENSE`) --- diff --git a/benchmark_compare.py b/benchmark_compare.py index 5335327..37bfb15 100644 --- a/benchmark_compare.py +++ b/benchmark_compare.py @@ -1,5 +1,5 @@ """ -benchmark_compare.py — Background benchmark: minimind (pure RAM) vs Chronos (SSD+DRAM) +benchmark_compare.py — Background benchmark: dense baseline (pure RAM) vs Chronos (SSD+DRAM) Runs in background, writes results to benchmark_results.json Usage: python benchmark_compare.py @@ -11,7 +11,6 @@ if _pkg_root not in sys.path: sys.path.insert(0, _pkg_root) -import chronos.deps # auto-bootstrap minimind on sys.path import torch import psutil @@ -68,18 +67,25 @@ def measure_tokens_per_sec(model, vocab_size, device, n_tokens=50, seq_len=32): elapsed = time.monotonic() - t0 return n_tokens / max(elapsed, 1e-6) -# ── Benchmark 1: minimind pure RAM ──────────────────────────────── +# ── Benchmark 1: dense baseline (pure RAM) ───────────────────────── +# A conventional transformer built from the same native kernel: full causal +# GQA (no MLA/SWA), one pass per layer (no recurrence), all experts resident. +# Before 0.3.2 this slot ran minimind; tools/compare_minimind_chronos*.py +# still compare against an external minimind checkout on request. -def bench_minimind(): - log('=== Benchmark 1: MiniMind (pure RAM) ===') - from model.model_minimind import MiniMindConfig, MiniMindForCausalLM +def bench_baseline(): + log('=== Benchmark 1: dense baseline (pure RAM) ===') + from chronos.model.config import ChronosConfig + from chronos.model.model_chronos import ChronosForCausalLM - cfg = MiniMindConfig( + cfg = ChronosConfig( hidden_size=512, num_hidden_layers=8, use_moe=True, - num_experts=4, num_experts_per_tok=1, + num_experts=4, num_experts_per_tok=1, num_shared_experts=0, + use_hybrid_attention=False, recurrent_enabled=False, + lambda_lookahead=0.0, lambda_temporal=0.0, ) ram_before = ram_used_gb() - model = MiniMindForCausalLM(cfg) + model = ChronosForCausalLM(cfg) model.eval() ram_after = ram_used_gb() ram_model_gb = ram_after - ram_before @@ -93,7 +99,7 @@ def bench_minimind(): train_losses = [] for step in range(5): x = torch.randint(0, cfg.vocab_size, (2, 64)) - out = model(x, labels=x) + out, _lookahead = model(x, labels=x) loss = out.loss + out.aux_loss loss.backward() optimizer.step() @@ -111,13 +117,14 @@ def bench_minimind(): log(f' TPS={tps:.1f} RAM_train={ram_train:.3f}GB RAM_infer={ram_infer:.3f}GB') result = { - 'name': 'MiniMind (pure RAM)', + 'name': 'Dense baseline (GQA, pure RAM)', 'params_m': round(params_m, 1), 'ram_model_gb': round(ram_model_gb, 3), 'ram_train_gb': round(ram_train, 3), 'ram_infer_gb': round(ram_infer, 3), 'tokens_per_sec': round(tps, 1), 'train_losses': [round(l, 4) for l in train_losses], + 'kv_cache_type': 'GQA (full K/V)', } del model gc.collect() @@ -241,9 +248,9 @@ def bench_chronos(): results = {} try: - results['minimind'] = bench_minimind() + results['baseline'] = bench_baseline() except Exception as e: - log(f'minimind bench failed: {e}') + log(f'baseline bench failed: {e}') import traceback; log(traceback.format_exc()) try: diff --git a/chronos/__init__.py b/chronos/__init__.py index a5591e4..22af209 100644 --- a/chronos/__init__.py +++ b/chronos/__init__.py @@ -1,9 +1,5 @@ # Project Chronos: On-device low-latency lookahead dual-layer MoE inference -__version__ = "0.3.1a3" - -# Bootstrap minimind dependency before any submodule imports -from chronos.deps import ensure_minimind as _ensure_minimind -_ensure_minimind() +__version__ = "0.3.2a0" # Register with HuggingFace AutoXXX factories so # AutoModelForCausalLM.from_pretrained("path/to/chronos") works out of the box. diff --git a/chronos/app.py b/chronos/app.py index 0206203..db6d458 100644 --- a/chronos/app.py +++ b/chronos/app.py @@ -36,7 +36,6 @@ from ui.gradio_compat import gr from ui.event_contract import UIEventContract, event_contract_app_kwargs -import chronos.deps # auto-bootstrap minimind from ui.i18n import ( LANGUAGE_CHOICES, DEFAULT_LANGUAGE, t, set_current_lang, build_language_update, get_registered_components, @@ -78,7 +77,7 @@ def _hero_html() -> str: "Apache-2.0" "Python 3.10+" "PyTorch 2.1+" - "MoE kernel: minimind" + "Kernel: Chronos native" "" "" ) diff --git a/chronos/assets/__init__.py b/chronos/assets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chronos/assets/tokenizers/legacy_6400/LICENSE b/chronos/assets/tokenizers/legacy_6400/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/chronos/assets/tokenizers/legacy_6400/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/chronos/assets/tokenizers/legacy_6400/NOTICE b/chronos/assets/tokenizers/legacy_6400/NOTICE new file mode 100644 index 0000000..bd46831 --- /dev/null +++ b/chronos/assets/tokenizers/legacy_6400/NOTICE @@ -0,0 +1,15 @@ +Legacy 6400-token tokenizer (data files only) + +tokenizer.json and tokenizer_config.json are copied unchanged from +minimind (https://github.com/jingyaogong/minimind), commit +24896cd2c4e9328a0911cf69ff61ace17a459b71. Copyright jingyaogong. +Licensed under the Apache License, Version 2.0 (see LICENSE in this folder). + +sha256 + tokenizer.json 71f32c68cf63a15355a8fc171b7594b3d41870fe0ddb54fc6aefa55f73a4a668 + tokenizer_config.json d7cd6a60c9f191c4f9ffec69b2eb289ad2f960366724f328b681ae1e46e4c110 + +Chronos does not use any minimind code. These files are kept so that +checkpoints that never recorded a tokenizer (vocab_size 6400, trained +before 0.3.2) still decode. New models should train their own +tokenizer with `chronos tokenizer` (Chronos Tokenizer Builder). diff --git a/chronos/assets/tokenizers/legacy_6400/tokenizer.json b/chronos/assets/tokenizers/legacy_6400/tokenizer.json new file mode 100644 index 0000000..e6ca86c --- /dev/null +++ b/chronos/assets/tokenizers/legacy_6400/tokenizer.json @@ -0,0 +1,31191 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4, + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 5, + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 6, + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 7, + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 8, + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 9, + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 10, + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 11, + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 12, + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 13, + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 14, + "content": "<|audio_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 15, + "content": "<|audio_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 16, + "content": "<|audio_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 17, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 18, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 19, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 20, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 21, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 22, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 23, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 24, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 25, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 26, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 27, + "content": "<|buffer1|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 28, + "content": "<|buffer2|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 29, + "content": "<|buffer3|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 30, + "content": "<|buffer4|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 31, + "content": "<|buffer5|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 32, + "content": "<|buffer6|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 33, + "content": "<|buffer7|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 34, + "content": "<|buffer8|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 35, + "content": "<|buffer9|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }, + "post_processor": null, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": { + "<|endoftext|>": 0, + "<|im_start|>": 1, + "<|im_end|>": 2, + "<|object_ref_start|>": 3, + "<|object_ref_end|>": 4, + "<|box_start|>": 5, + "<|box_end|>": 6, + "<|quad_start|>": 7, + "<|quad_end|>": 8, + "<|vision_start|>": 9, + "<|vision_end|>": 10, + "<|vision_pad|>": 11, + "<|image_pad|>": 12, + "<|video_pad|>": 13, + "<|audio_start|>": 14, + "<|audio_end|>": 15, + "<|audio_pad|>": 16, + "": 17, + "": 18, + "": 19, + "": 20, + "": 21, + "": 22, + "": 23, + "": 24, + "": 25, + "": 26, + "<|buffer1|>": 27, + "<|buffer2|>": 28, + "<|buffer3|>": 29, + "<|buffer4|>": 30, + "<|buffer5|>": 31, + "<|buffer6|>": 32, + "<|buffer7|>": 33, + "<|buffer8|>": 34, + "<|buffer9|>": 35, + "!": 36, + "\"": 37, + "#": 38, + "$": 39, + "%": 40, + "&": 41, + "'": 42, + "(": 43, + ")": 44, + "*": 45, + "+": 46, + ",": 47, + "-": 48, + ".": 49, + "/": 50, + "0": 51, + "1": 52, + "2": 53, + "3": 54, + "4": 55, + "5": 56, + "6": 57, + "7": 58, + "8": 59, + "9": 60, + ":": 61, + ";": 62, + "<": 63, + "=": 64, + ">": 65, + "?": 66, + "@": 67, + "A": 68, + "B": 69, + "C": 70, + "D": 71, + "E": 72, + "F": 73, + "G": 74, + "H": 75, + "I": 76, + "J": 77, + "K": 78, + "L": 79, + "M": 80, + "N": 81, + "O": 82, + "P": 83, + "Q": 84, + "R": 85, + "S": 86, + "T": 87, + "U": 88, + "V": 89, + "W": 90, + "X": 91, + "Y": 92, + "Z": 93, + "[": 94, + "\\": 95, + "]": 96, + "^": 97, + "_": 98, + "`": 99, + "a": 100, + "b": 101, + "c": 102, + "d": 103, + "e": 104, + "f": 105, + "g": 106, + "h": 107, + "i": 108, + "j": 109, + "k": 110, + "l": 111, + "m": 112, + "n": 113, + "o": 114, + "p": 115, + "q": 116, + "r": 117, + "s": 118, + "t": 119, + "u": 120, + "v": 121, + "w": 122, + "x": 123, + "y": 124, + "z": 125, + "{": 126, + "|": 127, + "}": 128, + "~": 129, + "¡": 130, + "¢": 131, + "£": 132, + "¤": 133, + "¥": 134, + "¦": 135, + "§": 136, + "¨": 137, + "©": 138, + "ª": 139, + "«": 140, + "¬": 141, + "®": 142, + "¯": 143, + "°": 144, + "±": 145, + "²": 146, + "³": 147, + "´": 148, + "µ": 149, + "¶": 150, + "·": 151, + "¸": 152, + "¹": 153, + "º": 154, + "»": 155, + "¼": 156, + "½": 157, + "¾": 158, + "¿": 159, + "À": 160, + "Á": 161, + "Â": 162, + "Ã": 163, + "Ä": 164, + "Å": 165, + "Æ": 166, + "Ç": 167, + "È": 168, + "É": 169, + "Ê": 170, + "Ë": 171, + "Ì": 172, + "Í": 173, + "Î": 174, + "Ï": 175, + "Ð": 176, + "Ñ": 177, + "Ò": 178, + "Ó": 179, + "Ô": 180, + "Õ": 181, + "Ö": 182, + "×": 183, + "Ø": 184, + "Ù": 185, + "Ú": 186, + "Û": 187, + "Ü": 188, + "Ý": 189, + "Þ": 190, + "ß": 191, + "à": 192, + "á": 193, + "â": 194, + "ã": 195, + "ä": 196, + "å": 197, + "æ": 198, + "ç": 199, + "è": 200, + "é": 201, + "ê": 202, + "ë": 203, + "ì": 204, + "í": 205, + "î": 206, + "ï": 207, + "ð": 208, + "ñ": 209, + "ò": 210, + "ó": 211, + "ô": 212, + "õ": 213, + "ö": 214, + "÷": 215, + "ø": 216, + "ù": 217, + "ú": 218, + "û": 219, + "ü": 220, + "ý": 221, + "þ": 222, + "ÿ": 223, + "Ā": 224, + "ā": 225, + "Ă": 226, + "ă": 227, + "Ą": 228, + "ą": 229, + "Ć": 230, + "ć": 231, + "Ĉ": 232, + "ĉ": 233, + "Ċ": 234, + "ċ": 235, + "Č": 236, + "č": 237, + "Ď": 238, + "ď": 239, + "Đ": 240, + "đ": 241, + "Ē": 242, + "ē": 243, + "Ĕ": 244, + "ĕ": 245, + "Ė": 246, + "ė": 247, + "Ę": 248, + "ę": 249, + "Ě": 250, + "ě": 251, + "Ĝ": 252, + "ĝ": 253, + "Ğ": 254, + "ğ": 255, + "Ġ": 256, + "ġ": 257, + "Ģ": 258, + "ģ": 259, + "Ĥ": 260, + "ĥ": 261, + "Ħ": 262, + "ħ": 263, + "Ĩ": 264, + "ĩ": 265, + "Ī": 266, + "ī": 267, + "Ĭ": 268, + "ĭ": 269, + "Į": 270, + "į": 271, + "İ": 272, + "ı": 273, + "IJ": 274, + "ij": 275, + "Ĵ": 276, + "ĵ": 277, + "Ķ": 278, + "ķ": 279, + "ĸ": 280, + "Ĺ": 281, + "ĺ": 282, + "Ļ": 283, + "ļ": 284, + "Ľ": 285, + "ľ": 286, + "Ŀ": 287, + "ŀ": 288, + "Ł": 289, + "ł": 290, + "Ń": 291, + "ï¼": 292, + "ä¸": 293, + "ï¼Į": 294, + "çļ": 295, + "çļĦ": 296, + "Ġt": 297, + "ãĢ": 298, + "Ġa": 299, + "he": 300, + "in": 301, + "ãĢĤ": 302, + "äº": 303, + "ä»": 304, + "åı": 305, + "re": 306, + "ä½": 307, + "æľ": 308, + "Ġthe": 309, + "on": 310, + "er": 311, + "**": 312, + "è¿": 313, + "at": 314, + "åħ": 315, + "Ġs": 316, + "Ġc": 317, + "åĪ": 318, + "Ġo": 319, + "å®": 320, + "çĶ": 321, + "æĪ": 322, + "åĬ": 323, + "is": 324, + "an": 325, + "ou": 326, + "en": 327, + "åľ": 328, + "es": 329, + "å¤": 330, + "åĴ": 331, + "ĠĠ": 332, + "æĺ": 333, + "åĴĮ": 334, + "åIJ": 335, + "or": 336, + "ï¼ļ": 337, + "it": 338, + "Ġw": 339, + "æķ": 340, + "Ġf": 341, + "ç»": 342, + "è¯": 343, + "Ġp": 344, + "ãĢģ": 345, + "Ġan": 346, + "æĸ": 347, + "è®": 348, + "ä¸Ģ": 349, + "ing": 350, + "Ġm": 351, + "al": 352, + "Ġin": 353, + "Ġof": 354, + "ĥ½": 355, + "éĢ": 356, + "æĺ¯": 357, + "以": 358, + "Ġ**": 359, + "åı¯": 360, + "le": 361, + "åŃ": 362, + "Ġb": 363, + "Ġto": 364, + "ä¹": 365, + "个": 366, + "åº": 367, + "çĶ¨": 368, + "åľ¨": 369, + "ar": 370, + "ion": 371, + "èĥ½": 372, + "Ġd": 373, + "**ï¼ļ": 374, + "ic": 375, + "Ġand": 376, + "å°": 377, + "æĹ": 378, + "ä¼": 379, + "éĩ": 380, + "å¸": 381, + "ed": 382, + "åĮ": 383, + "è§": 384, + "Ġy": 385, + "åĽ": 386, + "è¡": 387, + "ä¸Ń": 388, + "å¯": 389, + "as": 390, + "æķ°": 391, + "åĩ": 392, + "ro": 393, + "è¿Ļ": 394, + "Ġis": 395, + "¦Ĥ": 396, + "æĿ": 397, + "æľī": 398, + "Ġn": 399, + "å¼": 400, + "Ġyou": 401, + "æĢ": 402, + "ä¿": 403, + "å¦Ĥ": 404, + "æŀ": 405, + "Ġl": 406, + "åĨ": 407, + "çŃ": 408, + "çİ": 409, + "åı¯ä»¥": 410, + "éĹ": 411, + "ç§": 412, + "Ġe": 413, + "è¦": 414, + "çĽ": 415, + "çī": 416, + "ä¸į": 417, + "为": 418, + "çļĦæ": 419, + "è¦ģ": 420, + "çIJ": 421, + "Ġre": 422, + "äºĨ": 423, + "å¹": 424, + "ent": 425, + "ct": 426, + "å·": 427, + "æı": 428, + "Ġh": 429, + "èĢ": 430, + "é¢": 431, + "st": 432, + "æ³": 433, + "æī": 434, + "ĊĠĠ": 435, + "æĬ": 436, + "ä¾": 437, + "æł": 438, + "et": 439, + "æį": 440, + "ä½ł": 441, + "ĠI": 442, + "æŶ": 443, + "om": 444, + "Ġth": 445, + "el": 446, + "人": 447, + "åį": 448, + "``": 449, + "Ġ=": 450, + "ow": 451, + "Ġ\\": 452, + "å½": 453, + "æıIJ": 454, + "çIJĨ": 455, + "äºİ": 456, + "um": 457, + "å¾": 458, + "æĪĸ": 459, + "id": 460, + "对": 461, + "âĢ": 462, + "æĪij": 463, + "çŃī": 464, + "æİ": 465, + "ä¸Ģ个": 466, + "im": 467, + "éĢļ": 468, + "éĿ": 469, + "æĢ§": 470, + "è´": 471, + "è¿ĩ": 472, + "ce": 473, + "è¡Į": 474, + "ç®": 475, + "使": 476, + "æĮ": 477, + "ve": 478, + "æĶ": 479, + "èĩ": 480, + "æĭ": 481, + "Ġ1": 482, + "Ġu": 483, + "æĸ¹": 484, + "il": 485, + "çĶŁ": 486, + "æį®": 487, + "æĪIJ": 488, + "ation": 489, + "åѦ": 490, + "æĽ": 491, + "Ġg": 492, + "æŃ": 493, + "åĵ": 494, + "th": 495, + "ers": 496, + "am": 497, + "ä½ľ": 498, + "ig": 499, + "ol": 500, + "ut": 501, + "å¿": 502, + "Ġfor": 503, + "ä¸ĭ": 504, + "æĦ": 505, + "éľ": 506, + "if": 507, + "ä¼ļ": 508, + "æĿ¥": 509, + "å¥": 510, + "åĮĸ": 511, + "åĪ°": 512, + "åĪĨ": 513, + "è¾": 514, + "å®ļ": 515, + "ä¿Ŀ": 516, + "æĽ´": 517, + "äºĽ": 518, + "ç¡": 519, + "æµ": 520, + "éĻ": 521, + "Ġor": 522, + "Ġas": 523, + "éľĢ": 524, + "å±": 525, + "们": 526, + "ĠT": 527, + "ç¨": 528, + "ĠA": 529, + "ä½ĵ": 530, + "åĬ¨": 531, + "åıij": 532, + "使çĶ¨": 533, + "解": 534, + "ĠC": 535, + "Ġ\"": 536, + "qu": 537, + "è¿Ľ": 538, + "æŀľ": 539, + "Ġpro": 540, + "Ġcan": 541, + "ly": 542, + "ur": 543, + "éĺ": 544, + "åºĶ": 545, + "Ġ2": 546, + "åij": 547, + "Ġ-": 548, + "ul": 549, + "ch": 550, + "åŁ": 551, + "ä¸İ": 552, + "åľ°": 553, + "ï¼ī": 554, + "ï¼Ī": 555, + "æ³ķ": 556, + "ate": 557, + "èµ": 558, + "ra": 559, + "åIJĮ": 560, + "é«": 561, + "ä¸Ĭ": 562, + "Ġit": 563, + "é«ĺ": 564, + "大": 565, + "ç½": 566, + "ç¡®": 567, + "ith": 568, + "æķ°æį®": 569, + "ist": 570, + "åIJĪ": 571, + "ab": 572, + "se": 573, + "æĸĩ": 574, + "è½": 575, + "æ¯": 576, + "度": 577, + "åı¯èĥ½": 578, + "èĩª": 579, + "Ġst": 580, + "å®ŀ": 581, + "Ġhe": 582, + "ge": 583, + "éĩı": 584, + "计": 585, + "ç¨ĭ": 586, + "çº": 587, + "åİ": 588, + "æ°": 589, + "å¤ļ": 590, + "Ġthat": 591, + "æĥ": 592, + "ot": 593, + "åĬĽ": 594, + "åĬł": 595, + "çİ°": 596, + "åĩº": 597, + "ç¤": 598, + "Ġwith": 599, + "Ġbe": 600, + "ç³": 601, + "ä¸ļ": 602, + "Ġv": 603, + "ç§į": 604, + "éĿ¢": 605, + "常": 606, + "éĹ´": 607, + "åĬ¡": 608, + "设": 609, + "æĬĢ": 610, + "est": 611, + "å»": 612, + "ä¾Ľ": 613, + "Ġcon": 614, + "éĹ®": 615, + "åĬ©": 616, + "æľ¯": 617, + "æ´": 618, + "Ġon": 619, + "æ±": 620, + "Ġmo": 621, + "æĻ": 622, + "å°Ĩ": 623, + "```": 624, + "å¢": 625, + "表": 626, + "éģ": 627, + "'s": 628, + "åħ³": 629, + "éľĢè¦ģ": 630, + "ĠS": 631, + "é¢ĺ": 632, + "å·¥": 633, + "##": 634, + "建": 635, + "åħ¶": 636, + "éĩį": 637, + "æľº": 638, + "ir": 639, + "æĸ°": 640, + "çī©": 641, + "éĢļè¿ĩ": 642, + "åħ¨": 643, + "æıIJä¾Ľ": 644, + "ä¿¡": 645, + "çĦ": 646, + "ç³»": 647, + "æĮģ": 648, + "çł": 649, + "å¼ı": 650, + "un": 651, + "ver": 652, + "pe": 653, + "Ġwh": 654, + "ay": 655, + "ç´": 656, + "æģ": 657, + "好": 658, + "åĢ": 659, + "ç«": 660, + "å¦Ĥæŀľ": 661, + "èī": 662, + "åŃIJ": 663, + "æ¸": 664, + "Ġ(": 665, + "ĠH": 666, + "çĦ¶": 667, + "iv": 668, + "åģ": 669, + "00": 670, + "åħ·": 671, + "ç±": 672, + "Ł¥": 673, + "ld": 674, + "çĤ": 675, + "ib": 676, + "è°": 677, + "ä»ĸ": 678, + "é¡": 679, + "ç®Ĺ": 680, + "ãģ": 681, + "åĪ¶": 682, + "âĢľ": 683, + "âĢĿ": 684, + "ä¹ł": 685, + "åŀ": 686, + "em": 687, + "ide": 688, + "æľ¬": 689, + "æĪ·": 690, + "ĠP": 691, + "è¶": 692, + "ke": 693, + "nt": 694, + "æľĢ": 695, + "åĮħ": 696, + "å®ĥ": 697, + "çī¹": 698, + "Ġcom": 699, + "å°ı": 700, + "åIJİ": 701, + "åŃĹ": 702, + "帮": 703, + "èĢĮ": 704, + "æķĪ": 705, + "èĢħ": 706, + "Ġ3": 707, + "igh": 708, + "é£": 709, + "åĵģ": 710, + "代": 711, + "件": 712, + "de": 713, + "umb": 714, + "yth": 715, + "éļ": 716, + "åħ¥": 717, + "Ĥ¨": 718, + "Ġex": 719, + "ä¹Ł": 720, + "æ²": 721, + "lo": 722, + "缮": 723, + "åŀĭ": 724, + "ĊĠĠĠ": 725, + "家": 726, + "å¼Ģ": 727, + "and": 728, + "¨¡": 729, + "Ġne": 730, + "åĽł": 731, + "Ġare": 732, + "æĦı": 733, + "æ¬": 734, + "çļĦç": 735, + "Ġha": 736, + "pp": 737, + "åī": 738, + "帮åĬ©": 739, + "ain": 740, + "te": 741, + "æĬĢæľ¯": 742, + "å®ī": 743, + "ython": 744, + "ure": 745, + "éĹ®é¢ĺ": 746, + "å±ķ": 747, + "ĠĠĠĠ": 748, + "çĤ¹": 749, + "The": 750, + "éĶ": 751, + "ä¾ĭ": 752, + "çĿ": 753, + "Ġ+": 754, + "æģ¯": 755, + "æľį": 756, + "Ġr": 757, + "éĢī": 758, + "模": 759, + "ä½Ĩ": 760, + "è¯Ń": 761, + "Ġse": 762, + "åĽ½": 763, + "åĨħ": 764, + "Ġme": 765, + "åĪĽ": 766, + "绣": 767, + "åŃ¦ä¹ł": 768, + "Ġby": 769, + "éķ": 770, + "her": 771, + "èĤ": 772, + "æĪij们": 773, + "ight": 774, + "ä¹ĭ": 775, + "ï¼Ł": 776, + "æĦŁ": 777, + "ç±»": 778, + "æ±Ĥ": 779, + "æ´»": 780, + "ess": 781, + "è¿ĻäºĽ": 782, + "éª": 783, + "éĽ": 784, + "ad": 785, + "åħ¬": 786, + "Ġde": 787, + "for": 788, + "Ġal": 789, + "产": 790, + "ment": 791, + "ç¼": 792, + "ç½ij": 793, + "ĠThe": 794, + "å¿ĥ": 795, + "Ġwe": 796, + "并": 797, + "ç¬": 798, + "æīĢ": 799, + "缸": 800, + "Ġ\\(": 801, + "çķ": 802, + "åĻ": 803, + "åĿ": 804, + "æľŁ": 805, + "int": 806, + "æ·": 807, + "Ġ`": 808, + "ri": 809, + "nd": 810, + "ant": 811, + "ase": 812, + "ç»ı": 813, + "çݯ": 814, + "è¿Ľè¡Į": 815, + "ç»ĵ": 816, + "èĬ": 817, + "å¹³": 818, + "çĶµ": 819, + "ç©": 820, + "Ġnumb": 821, + "éĢĤ": 822, + "lp": 823, + "oo": 824, + "ction": 825, + "交": 826, + "åĻ¨": 827, + "åıĺ": 828, + "æłĩ": 829, + "主": 830, + "æº": 831, + "us": 832, + "ple": 833, + "age": 834, + "å¤Ħ": 835, + "çĿĢ": 836, + "æľįåĬ¡": 837, + "éĥ": 838, + "ä¿¡æģ¯": 839, + "xt": 840, + "Ġprov": 841, + "æİ¥": 842, + "Ġhelp": 843, + "èµĦ": 844, + "åĨ³": 845, + "Ġus": 846, + "å½±": 847, + "Ġle": 848, + "æĤ¨": 849, + "以ä¸ĭ": 850, + "æĺİ": 851, + "éĩĮ": 852, + "çľ": 853, + "è·": 854, + "Ġsu": 855, + "æĥħ": 856, + "éĥ½": 857, + "åľº": 858, + "ç¡®ä¿Ŀ": 859, + "ä»İ": 860, + "éķ¿": 861, + "Wh": 862, + "çĶ¨æĪ·": 863, + "éĥ¨": 864, + "rom": 865, + "ort": 866, + "åıĬ": 867, + "èģ": 868, + "ĠM": 869, + "ity": 870, + "å¤Ł": 871, + "åĪĹ": 872, + "å¾Ĺ": 873, + "Ġhave": 874, + "pt": 875, + "form": 876, + "åŁº": 877, + "å®īåħ¨": 878, + "åĢ¼": 879, + "è°ĥ": 880, + "æĭ¬": 881, + "èĻ": 882, + "ific": 883, + "ive": 884, + "社": 885, + "çłģ": 886, + "ter": 887, + "ä½ķ": 888, + "res": 889, + "Ġch": 890, + "éĢīæĭ": 891, + "ck": 892, + "Ġqu": 893, + "éĢīæĭ©": 894, + "计ç®Ĺ": 895, + "é¦": 896, + "æºIJ": 897, + "åĮħæĭ¬": 898, + "å½ĵ": 899, + "Ġ5": 900, + "end": 901, + "管": 902, + "çİĩ": 903, + "è¿Ļ个": 904, + "æıIJé«ĺ": 905, + "ame": 906, + "ç³»ç»Ł": 907, + "åºĶçĶ¨": 908, + "è¿ĺ": 909, + "æķ´": 910, + "æŶéĹ´": 911, + "è®®": 912, + "art": 913, + "Ġk": 914, + "ould": 915, + "ind": 916, + "ä½į": 917, + "ä»»": 918, + "æ¯ı": 919, + "è´¨": 920, + "åĵį": 921, + "ue": 922, + "ä¸ĢäºĽ": 923, + "å¤į": 924, + "ies": 925, + "ial": 926, + "Ġpo": 927, + "äºĭ": 928, + "å¸Ĥ": 929, + "èī²": 930, + "åŃĺ": 931, + "Ġ4": 932, + "åĮº": 933, + "èĥ½å¤Ł": 934, + "示": 935, + "å¹´": 936, + "è¢": 937, + "åīį": 938, + "åIJį": 939, + "ç´ł": 940, + "ç»Ħ": 941, + "ang": 942, + "æ¡": 943, + "ence": 944, + "ĠF": 945, + "Ġspe": 946, + "Ġwor": 947, + "ich": 948, + "æ¶": 949, + "éħ": 950, + "å¢ŀ": 951, + "æ°´": 952, + "æŃ¥": 953, + "å¢ĥ": 954, + "out": 955, + "æŀĦ": 956, + "ĠE": 957, + "å¤ĦçIJĨ": 958, + "ill": 959, + "请": 960, + "Ġfrom": 961, + "å°ij": 962, + "éĩįè¦ģ": 963, + "ere": 964, + "项": 965, + "Ġco": 966, + "æ¯Ķ": 967, + "æŃ£": 968, + "ä¸įåIJĮ": 969, + "åĩ½": 970, + "åĪ©": 971, + "éĤ": 972, + "被": 973, + "åĩĨ": 974, + "Ġab": 975, + "æĶ¯": 976, + "ä»ħ": 977, + "ĠB": 978, + "æł¹": 979, + "è¯Ĩ": 980, + "ä¹Ī": 981, + "导": 982, + "åħĥ": 983, + "åĩ½æķ°": 984, + "ä¼ĺ": 985, + "\\)": 986, + "ions": 987, + "Ġ$": 988, + "æĹł": 989, + "注": 990, + "éĿŀ": 991, + "pl": 992, + "天": 993, + "æŁ¥": 994, + "æĬ¤": 995, + "æµģ": 996, + "æµĭ": 997, + "è§Ħ": 998, + "his": 999, + "çĸ": 1000, + "æĺ¯ä¸Ģ个": 1001, + "ĠAl": 1002, + "Ġthis": 1003, + "ĠW": 1004, + "ult": 1005, + "åºı": 1006, + "线": 1007, + "è¯ķ": 1008, + "åıijå±ķ": 1009, + "å½±åĵį": 1010, + "###": 1011, + "ç¾": 1012, + "ç½®": 1013, + "åŁŁ": 1014, + "åıĸ": 1015, + "ine": 1016, + "Ġprovide": 1017, + "é£İ": 1018, + "å½¢": 1019, + "Ġspec": 1020, + "ä¼ł": 1021, + "å¤ĩ": 1022, + "强": 1023, + "ike": 1024, + "act": 1025, + "åħ·ä½ĵ": 1026, + "Ġnot": 1027, + "iz": 1028, + "容": 1029, + "low": 1030, + "Ġass": 1031, + "é¢Ĩ": 1032, + "éªĮ": 1033, + "Ġma": 1034, + "åĩı": 1035, + "Ġmore": 1036, + "åķ": 1037, + "管çIJĨ": 1038, + "Ġyour": 1039, + "æĢģ": 1040, + "Ġdo": 1041, + "ä¸ŃçļĦ": 1042, + "æ»": 1043, + "ĠG": 1044, + "Ġwhich": 1045, + "æŀIJ": 1046, + "Ġte": 1047, + "Ġnumber": 1048, + "able": 1049, + "è¿°": 1050, + "aba": 1051, + "è¾ĵ": 1052, + "æĹ¥": 1053, + "æł¹æį®": 1054, + "åĽŀ": 1055, + "çŃĸ": 1056, + "Ġen": 1057, + "ĠD": 1058, + "éĵ": 1059, + "Ġpr": 1060, + "è¨": 1061, + "çݯå¢ĥ": 1062, + "ibaba": 1063, + "æĸ¹æ³ķ": 1064, + "åĪ«": 1065, + "åįķ": 1066, + "per": 1067, + "rou": 1068, + "Ł³": 1069, + "Ġneed": 1070, + "è¿IJ": 1071, + "ring": 1072, + "æł¼": 1073, + "put": 1074, + "ä¸įä»ħ": 1075, + "ust": 1076, + "æ£": 1077, + "ite": 1078, + "ance": 1079, + "æĻº": 1080, + "éĽĨ": 1081, + "åıĹ": 1082, + "æѤ": 1083, + "our": 1084, + "å¤ĸ": 1085, + "代çłģ": 1086, + "ç§ij": 1087, + "åIJij": 1088, + "èĬĤ": 1089, + "éĢļ常": 1090, + "设计": 1091, + "å§": 1092, + "are": 1093, + "æ·±": 1094, + "ated": 1095, + "ical": 1096, + "æŁ": 1097, + "Ġlike": 1098, + "oud": 1099, + "é¢Ħ": 1100, + "éĢŁ": 1101, + "èĢĥ": 1102, + "now": 1103, + "è¨Ģ": 1104, + "åĪĨæŀIJ": 1105, + "ans": 1106, + "åĽ¾": 1107, + "ĊĠĠĠĠĠĠ": 1108, + "个人": 1109, + "cl": 1110, + "Ġwas": 1111, + "åĬŁ": 1112, + "av": 1113, + "ĠHow": 1114, + "çŁ¥": 1115, + "ç»Ļ": 1116, + "æĮĩ": 1117, + "åijĺ": 1118, + "æ®": 1119, + "åıĤ": 1120, + "ç»Ń": 1121, + "Ġso": 1122, + "loud": 1123, + "ä¾ĭå¦Ĥ": 1124, + "æĥ³": 1125, + "ice": 1126, + "åIJĮæŶ": 1127, + "Ġsh": 1128, + "æł·": 1129, + "cc": 1130, + "ud": 1131, + "论": 1132, + "åıį": 1133, + "çģ": 1134, + "模åŀĭ": 1135, + "ä¹ī": 1136, + "è¾¾": 1137, + "äºij": 1138, + "缴": 1139, + "çĥ": 1140, + "ĠAlibaba": 1141, + "èĥ": 1142, + "çIJĥ": 1143, + "Ġtr": 1144, + "å·±": 1145, + "What": 1146, + "æķĻ": 1147, + "unction": 1148, + "Ġany": 1149, + "åĨµ": 1150, + "æīĭ": 1151, + "Ġassist": 1152, + "èĩªå·±": 1153, + "ast": 1154, + "\\[": 1155, + "ary": 1156, + "åĽ¢": 1157, + "å·´": 1158, + "å·¥ä½ľ": 1159, + "符": 1160, + "建议": 1161, + "è±": 1162, + "ag": 1163, + "éĻħ": 1164, + "Ġint": 1165, + "èį": 1166, + "次": 1167, + "æĶ¹": 1168, + "ãĤ": 1169, + "æī¾": 1170, + "ç¥": 1171, + "og": 1172, + "åħ±": 1173, + "ime": 1174, + "ap": 1175, + "formation": 1176, + "Ġ0": 1177, + "Ġres": 1178, + "åģ¥": 1179, + "ä»·": 1180, + "两": 1181, + "æĻ¯": 1182, + "è¿ĩç¨ĭ": 1183, + "åİŁ": 1184, + "å¦Ĥä½ķ": 1185, + "è§Ĥ": 1186, + "ther": 1187, + "æĸ¹å¼ı": 1188, + "èIJ": 1189, + "Ġ20": 1190, + "ç«ĭ": 1191, + "转": 1192, + "Ġ'": 1193, + "å¼ķ": 1194, + "æ°Ķ": 1195, + "Ġabout": 1196, + "Ġspecific": 1197, + "ĠL": 1198, + "å®ŀçİ°": 1199, + "Ġad": 1200, + "çIJĨ解": 1201, + "让": 1202, + "æĺ¯ä¸Ģ": 1203, + "æľª": 1204, + "çĶ¨äºİ": 1205, + "ç¼ĸ": 1206, + "lease": 1207, + "Ġim": 1208, + "é¢ĨåŁŁ": 1209, + "æݨ": 1210, + "éĺ¿": 1211, + "Ġknow": 1212, + "è§Ĩ": 1213, + "ä½łçļĦ": 1214, + "解åĨ³": 1215, + "绾": 1216, + "Ġinformation": 1217, + "10": 1218, + "Ġpl": 1219, + "åĩıå°ij": 1220, + "åħī": 1221, + "å®Į": 1222, + "text": 1223, + "åħĪ": 1224, + "Ġar": 1225, + "åĨĻ": 1226, + "éļı": 1227, + "æĶ¶": 1228, + "ç½ij绾": 1229, + "æĻºèĥ½": 1230, + "ĠN": 1231, + "éĺŁ": 1232, + "客": 1233, + "èµ·": 1234, + "Ġfunction": 1235, + "ĠJ": 1236, + "king": 1237, + "éĤ£": 1238, + "ore": 1239, + "çĶ±": 1240, + "Ġif": 1241, + "ations": 1242, + "ĠR": 1243, + "康": 1244, + "åĪĻ": 1245, + "ĠCh": 1246, + "éľĢæ±Ĥ": 1247, + "Ġat": 1248, + "åħį": 1249, + "ener": 1250, + "æĶ¾": 1251, + "æĶ¯æĮģ": 1252, + "Ġup": 1253, + "ç²": 1254, + "æľĽ": 1255, + "ä¸ĵ": 1256, + "说": 1257, + "ä¿ĿæĮģ": 1258, + "ount": 1259, + "çĬ": 1260, + "åİĨ": 1261, + "urn": 1262, + "Ġwhe": 1263, + "ĠThis": 1264, + "éĩij": 1265, + "all": 1266, + "éģĵ": 1267, + "ach": 1268, + "so": 1269, + "Ġfe": 1270, + "éŁ³": 1271, + "ell": 1272, + "空": 1273, + "çłĶ": 1274, + "ks": 1275, + "åķĨ": 1276, + "åį³": 1277, + "è§Ĵ": 1278, + "ç¾İ": 1279, + "åģ¥åº·": 1280, + "社ä¼ļ": 1281, + "è¯Ńè¨Ģ": 1282, + "tain": 1283, + "å£": 1284, + "ange": 1285, + "èº": 1286, + "ata": 1287, + "èĦ": 1288, + "ign": 1289, + "Ġhas": 1290, + "è®°": 1291, + "ect": 1292, + "ä¹IJ": 1293, + "èİ": 1294, + "çĹ": 1295, + "身": 1296, + "Ġret": 1297, + "çĶŁæ´»": 1298, + "ile": 1299, + "æݧ": 1300, + "ous": 1301, + "ç»´": 1302, + "éĴ": 1303, + "å§ĭ": 1304, + "éĺ¿éĩĮ": 1305, + "产åĵģ": 1306, + "éĿŀ常": 1307, + "è£": 1308, + "erm": 1309, + "åĸ": 1310, + "ç®Ģ": 1311, + "os": 1312, + "ĠIf": 1313, + "å¸Ĥåľº": 1314, + "Ġj": 1315, + "å¿«": 1316, + "段": 1317, + "od": 1318, + "Ġser": 1319, + "pr": 1320, + "äºĨ解": 1321, + "Ġ6": 1322, + "åįĩ": 1323, + "å°±": 1324, + "èµĦæºIJ": 1325, + "èĩ´": 1326, + "缮æłĩ": 1327, + "åı°": 1328, + "rac": 1329, + "象": 1330, + "主è¦ģ": 1331, + "åIJ¦": 1332, + "æĸĻ": 1333, + "èĥ½åĬĽ": 1334, + "æĸŃ": 1335, + "çľĭ": 1336, + "à¸": 1337, + "Ġcre": 1338, + "AI": 1339, + "du": 1340, + "æ£Ģ": 1341, + "Ġits": 1342, + "ĠIt": 1343, + "以åıĬ": 1344, + "è¯ģ": 1345, + "ä»ĸ们": 1346, + "ĠCloud": 1347, + "ail": 1348, + "python": 1349, + "对äºİ": 1350, + "éĻIJ": 1351, + "ub": 1352, + "ä¼ģ": 1353, + "20": 1354, + "åı¸": 1355, + "游": 1356, + "æĵ": 1357, + "é£Ł": 1358, + "满": 1359, + "æĸĩåĮĸ": 1360, + "()": 1361, + "Ġbut": 1362, + "ç»Ĩ": 1363, + "ï¼ģ": 1364, + "å¸Į": 1365, + "Ġmy": 1366, + "ip": 1367, + "ä¼ģä¸ļ": 1368, + "èģĶ": 1369, + "ĠO": 1370, + "é¡¹çĽ®": 1371, + "æķ°åŃĹ": 1372, + "çķĮ": 1373, + "éģ¿": 1374, + "åĩł": 1375, + "}{": 1376, + "ong": 1377, + "ä¿ĿæĬ¤": 1378, + "æĪĺ": 1379, + "è¯Ħ": 1380, + "ä»Ģ": 1381, + "ree": 1382, + "æĢ»": 1383, + "积": 1384, + "该": 1385, + "éħį": 1386, + "ĊĠĠĠĠĠĠĠ": 1387, + "ass": 1388, + "ever": 1389, + "Ġx": 1390, + "ces": 1391, + "æĽ´åĬł": 1392, + "Ġquest": 1393, + "éĢł": 1394, + "Ġgener": 1395, + "æĿ¡": 1396, + "frac": 1397, + "Ġreturn": 1398, + "èĮ": 1399, + "ie": 1400, + "Ġ\\)": 1401, + "çĬ¶": 1402, + "ä»Ģä¹Ī": 1403, + "æ¡Ī": 1404, + "ä¸ŃåĽ½": 1405, + "Ġun": 1406, + "åIJ«": 1407, + "é¦ĸ": 1408, + "广": 1409, + "æĺĵ": 1410, + "ä¹ĭéĹ´": 1411, + "åĽ¢éĺŁ": 1412, + "èĪ": 1413, + "Ġhow": 1414, + "vel": 1415, + "Ġsc": 1416, + "Ġfind": 1417, + "Ġuse": 1418, + "é©": 1419, + "没": 1420, + "Ġ10": 1421, + "'m": 1422, + "å¯Ĩ": 1423, + "çĶŁæĪIJ": 1424, + "èĻij": 1425, + "port": 1426, + "oc": 1427, + "'re": 1428, + "åħ¬åı¸": 1429, + "åĬŁèĥ½": 1430, + "æķħ": 1431, + "Ġincl": 1432, + "ard": 1433, + "ates": 1434, + "èĤ²": 1435, + "åħĥç´ł": 1436, + "éĻ©": 1437, + "Ġval": 1438, + "åĮ»": 1439, + "æķĪçİĩ": 1440, + "ä¸ī": 1441, + "èĩªçĦ¶": 1442, + "com": 1443, + "æĢ§åĴĮ": 1444, + "'t": 1445, + "æį¢": 1446, + "æ¸ħ": 1447, + "Ġapp": 1448, + "ç´¢": 1449, + "éģ¿åħį": 1450, + "æĿĤ": 1451, + "Ġ[": 1452, + "Ġdes": 1453, + "æĿĥ": 1454, + "å®ĥ们": 1455, + "足": 1456, + "åĨħ容": 1457, + "èĭ": 1458, + "Ġlist": 1459, + "åIJĦ": 1460, + "æĺ¯åIJ¦": 1461, + "æĸ¹éĿ¢": 1462, + "Ġequ": 1463, + "没æľī": 1464, + "ĠChin": 1465, + "çĭ": 1466, + "æİĴ": 1467, + "åĪĴ": 1468, + "çĻ": 1469, + "æīĢæľī": 1470, + "认": 1471, + "éĶ®": 1472, + "äºĴ": 1473, + "è¯į": 1474, + "ire": 1475, + "åij³": 1476, + "æĸ½": 1477, + "è®Ń": 1478, + "æľīæķĪ": 1479, + "é¥": 1480, + "Ġfol": 1481, + "ari": 1482, + "带": 1483, + "æ´»åĬ¨": 1484, + "è´¹": 1485, + "In": 1486, + "æŀģ": 1487, + "ä½İ": 1488, + "ace": 1489, + "ely": 1490, + "Ġcode": 1491, + "åħ¶ä»ĸ": 1492, + "irst": 1493, + "Ġwill": 1494, + "ä¼ĺåĮĸ": 1495, + "ç»ĥ": 1496, + "å¼Ģåıij": 1497, + "车": 1498, + "享": 1499, + "Ġmost": 1500, + "Ġsom": 1501, + "Ġfollow": 1502, + "Ġdet": 1503, + "çĪ": 1504, + "å¤įæĿĤ": 1505, + "op": 1506, + "åºĵ": 1507, + "Ġform": 1508, + "书": 1509, + "æĢĿ": 1510, + "究": 1511, + "iven": 1512, + "å¸ĮæľĽ": 1513, + "åĪĹ表": 1514, + "Ġcomm": 1515, + "è¿Ļç§į": 1516, + "åĿĹ": 1517, + "gh": 1518, + "è½»": 1519, + "åŃĹ符": 1520, + "çķ¥": 1521, + "ep": 1522, + "Ġacc": 1523, + "ĠHowever": 1524, + "ally": 1525, + "è¾ĥ": 1526, + "åij½": 1527, + "åĽłä¸º": 1528, + "趣": 1529, + "ç¦": 1530, + "èĢĥèĻij": 1531, + "è¯Ŀ": 1532, + "ord": 1533, + "Ġmod": 1534, + "ĠU": 1535, + "Ġcomp": 1536, + "ãĥ": 1537, + "Ġ7": 1538, + "èİ·": 1539, + "åĥ": 1540, + "å¢ŀåĬł": 1541, + "įä½ľ": 1542, + "ĠPython": 1543, + "è·¯": 1544, + "ac": 1545, + "åijĬ": 1546, + "åıª": 1547, + "以ä¸ĭæĺ¯": 1548, + "ä½łåı¯ä»¥": 1549, + "çłĶ究": 1550, + "è°ĥæķ´": 1551, + "Ġro": 1552, + "Ġdata": 1553, + "çĸĹ": 1554, + "读": 1555, + "å¾ĭ": 1556, + "ry": 1557, + "è§ģ": 1558, + "print": 1559, + "è£ħ": 1560, + "ample": 1561, + "\\]": 1562, + "Ġsent": 1563, + "Ġmight": 1564, + "ä¿ĥ": 1565, + "çīĩ": 1566, + "ħ§": 1567, + "Ġ\\]": 1568, + "æĮģç»Ń": 1569, + ".\"": 1570, + "åı¯èĥ½ä¼ļ": 1571, + "åı£": 1572, + "å¸ĥ": 1573, + "å¾Ī": 1574, + "åı·": 1575, + "éĵ¾": 1576, + "ello": 1577, + "çī¹å®ļ": 1578, + "æĸĩ件": 1579, + "--": 1580, + "èĩ³": 1581, + "ãĢĬ": 1582, + "æĪı": 1583, + "ãĢĭ": 1584, + "urre": 1585, + "åĥı": 1586, + "ems": 1587, + "åħ´": 1588, + "ä¸ĸ": 1589, + "è´¨éĩı": 1590, + "æ¼": 1591, + "ress": 1592, + "éĻ¤": 1593, + "æĥħåĨµ": 1594, + "Ġ{": 1595, + "Ġ8": 1596, + "çĦ¶åIJİ": 1597, + "å¦Ĥæŀľä½ł": 1598, + "åı¥": 1599, + "Python": 1600, + "ph": 1601, + "Hello": 1602, + "æŁIJ": 1603, + "å¹³åı°": 1604, + "æĶ¿": 1605, + "ertain": 1606, + "å½ķ": 1607, + "imes": 1608, + "Ġpart": 1609, + "ound": 1610, + "wen": 1611, + "Ġnumbers": 1612, + "Ġper": 1613, + "åĪĽå»º": 1614, + "sw": 1615, + "åıĺåĮĸ": 1616, + "fere": 1617, + "å·¥åħ·": 1618, + "Ġnew": 1619, + "éĺ²": 1620, + "Ġwhat": 1621, + "angu": 1622, + "Ġone": 1623, + "æķĻèĤ²": 1624, + "Ġserv": 1625, + "Ġname": 1626, + "ff": 1627, + "Ġ#": 1628, + "ç®Ĺæ³ķ": 1629, + "误": 1630, + "åħħ": 1631, + "ition": 1632, + "con": 1633, + "one": 1634, + "Ġtheir": 1635, + "å®ŀéĻħ": 1636, + "ack": 1637, + "ft": 1638, + "é»": 1639, + "ram": 1640, + "åĪ©çĶ¨": 1641, + "æīĵ": 1642, + "æĵįä½ľ": 1643, + "Ġsol": 1644, + "æ¶Ī": 1645, + "称": 1646, + "):": 1647, + "æĪĸèĢħ": 1648, + "Ġother": 1649, + "Ġall": 1650, + "Ġusing": 1651, + "ript": 1652, + "Qwen": 1653, + "çļĦæĺ¯": 1654, + "ear": 1655, + "ç»ĵæŀľ": 1656, + "ä¿ĥè¿Ľ": 1657, + "Ġcl": 1658, + "æľªæĿ¥": 1659, + "ak": 1660, + "æݧåĪ¶": 1661, + "ory": 1662, + "çŁ": 1663, + "Ġlangu": 1664, + "Ġsy": 1665, + "åĽłæѤ": 1666, + "éĢĤåIJĪ": 1667, + "vers": 1668, + "Ġinto": 1669, + "Ġinte": 1670, + "To": 1671, + "Ġman": 1672, + "温": 1673, + "好çļĦ": 1674, + "æľīåĬ©": 1675, + "23": 1676, + "ç»ĵæŀĦ": 1677, + "æĿIJ": 1678, + "èIJ¥": 1679, + "Ġvari": 1680, + "æľīåĬ©äºİ": 1681, + "Ġem": 1682, + "Ġcol": 1683, + "Ġcont": 1684, + "ng": 1685, + "ç«Ļ": 1686, + "æĸ°çļĦ": 1687, + "æıı": 1688, + "ish": 1689, + "Ġoff": 1690, + "询": 1691, + "åİ»": 1692, + "缸åħ³": 1693, + "æĺ¾": 1694, + "Ġout": 1695, + "ased": 1696, + "èŀ": 1697, + "rite": 1698, + "æµİ": 1699, + "æĺŁ": 1700, + "Ġbet": 1701, + "æķħäºĭ": 1702, + "åı²": 1703, + "Ġansw": 1704, + "为äºĨ": 1705, + "èĩªå·±çļĦ": 1706, + "Ġworld": 1707, + "设å¤ĩ": 1708, + "Ġwould": 1709, + "Ġfirst": 1710, + "æĬķ": 1711, + "第": 1712, + "Ġev": 1713, + "åģļ": 1714, + "æ£ĢæŁ¥": 1715, + "ç»ıæµİ": 1716, + "Ġty": 1717, + "ä¸ĵä¸ļ": 1718, + "åĶ": 1719, + "帮åĬ©ä½ł": 1720, + "ĠÐ": 1721, + "软": 1722, + "缴æİ¥": 1723, + "èij": 1724, + "å¼Ģå§ĭ": 1725, + "ia": 1726, + "Ġinclud": 1727, + "æľĪ": 1728, + "ç²¾": 1729, + "åĽ´": 1730, + "ments": 1731, + "ature": 1732, + "éĩĬ": 1733, + "Ġterm": 1734, + "åĤ¨": 1735, + "urrent": 1736, + "ru": 1737, + "__": 1738, + "ç±»åŀĭ": 1739, + "ä»»åĬ¡": 1740, + "Ġ*": 1741, + "åĵª": 1742, + "Ġcal": 1743, + "Ġalso": 1744, + "éĶĻ": 1745, + "Ġam": 1746, + "we": 1747, + "ä½ľä¸º": 1748, + "Ġanswer": 1749, + "念": 1750, + "游æĪı": 1751, + "æľºåĻ¨": 1752, + "day": 1753, + "Ġlet": 1754, + "Ġused": 1755, + "çĹħ": 1756, + "åIJ¸": 1757, + "两个": 1758, + "æī¾åĪ°": 1759, + "级": 1760, + "Ġsum": 1761, + "åħ³ç³»": 1762, + "ne": 1763, + "注æĦı": 1764, + "Ġplease": 1765, + "Ġeach": 1766, + "çļĦæĥ": 1767, + "导èĩ´": 1768, + "é¢ij": 1769, + "ç¥ŀ": 1770, + "å¤ļ个": 1771, + "Ġsp": 1772, + "便": 1773, + "èŀį": 1774, + "Ġexp": 1775, + "è¾ĵåħ¥": 1776, + "åŁİ": 1777, + "è¿ŀ": 1778, + "æıIJåįĩ": 1779, + "ten": 1780, + "åİĭ": 1781, + "eth": 1782, + "åħ³éĶ®": 1783, + "éĥ¨åĪĨ": 1784, + "çŃĸçķ¥": 1785, + "Ġref": 1786, + "cul": 1787, + "å±Ĥ": 1788, + "æ¢": 1789, + "Ġsentence": 1790, + "çĮ": 1791, + "许": 1792, + "éĩĩ": 1793, + "骤": 1794, + "ning": 1795, + "ä½ĵéªĮ": 1796, + "è¿ľ": 1797, + "ify": 1798, + "ertainly": 1799, + "表达": 1800, + "æĽ´å¤ļ": 1801, + "lic": 1802, + "ä¼°": 1803, + "å·²": 1804, + "åĪĽæĸ°": 1805, + "Ġche": 1806, + "ating": 1807, + "éĶĢ": 1808, + "èĩªåĬ¨": 1809, + "èīº": 1810, + "èı": 1811, + "\",": 1812, + "串": 1813, + "ç¢": 1814, + "ä¿®": 1815, + "æĤ": 1816, + "Ġ|": 1817, + "Ġwant": 1818, + "Ġcurrent": 1819, + "Ġexample": 1820, + "ç¨ĭåºı": 1821, + "欢": 1822, + "Ġlar": 1823, + "nder": 1824, + "æĺ¯ä¸Ģç§į": 1825, + "çĽij": 1826, + "份": 1827, + "Ġwhere": 1828, + "é£İéĻ©": 1829, + "Ġprovid": 1830, + "ath": 1831, + "å¼Ĥ": 1832, + "å¯Į": 1833, + "çļĦæĬ": 1834, + "çĥŃ": 1835, + "æĢ§èĥ½": 1836, + "red": 1837, + "çĭ¬": 1838, + "åĩĨç¡®": 1839, + "æ¯ı个": 1840, + "äºĮ": 1841, + "æ°ij": 1842, + "ss": 1843, + "Ġquestions": 1844, + "æij": 1845, + "Ġdif": 1846, + "Ġbo": 1847, + "Ġsuch": 1848, + "æµ·": 1849, + "éĻį": 1850, + "Ġtext": 1851, + "åIJĪä½ľ": 1852, + "Ġloo": 1853, + "Ġtw": 1854, + "失": 1855, + "客æĪ·": 1856, + "è®Ńç»ĥ": 1857, + "éŨ": 1858, + "ijæĪĺ": 1859, + "ices": 1860, + "Ġ9": 1861, + "ä¸įåIJĮçļĦ": 1862, + "åĮħåIJ«": 1863, + "ject": 1864, + "æĴ": 1865, + "åĬ¿": 1866, + "å°Ŀ": 1867, + "ä»»ä½ķ": 1868, + "Ġmodel": 1869, + "ents": 1870, + "æİ¢": 1871, + "Ġchar": 1872, + "é¦ĸåħĪ": 1873, + "åİĨåı²": 1874, + "ä¸Ķ": 1875, + "人类": 1876, + "ĠHere": 1877, + "æĪIJæľ¬": 1878, + "å¾Ģ": 1879, + "date": 1880, + "ds": 1881, + "æIJ": 1882, + "Ġprint": 1883, + "èĬ±": 1884, + "ual": 1885, + "è´Ń": 1886, + "å¾®": 1887, + "ĠY": 1888, + "Ġtas": 1889, + "Ġtrans": 1890, + "ĠAI": 1891, + "è¿Ļæł·": 1892, + "å®ļæľŁ": 1893, + "æ¯Ķå¦Ĥ": 1894, + "olog": 1895, + "æĸĩæľ¬": 1896, + "设置": 1897, + "å·´å·´": 1898, + "éĺ¿éĩĮå·´å·´": 1899, + "au": 1900, + "Ġfollowing": 1901, + "è¾¹": 1902, + "Ġlanguage": 1903, + "Ġhere": 1904, + "Ġele": 1905, + "æııè¿°": 1906, + "les": 1907, + "éļ¾": 1908, + "èģĮ": 1909, + "ava": 1910, + "软件": 1911, + "о": 1912, + "约": 1913, + "Certainly": 1914, + "Ġcap": 1915, + "è¡Į为": 1916, + "Ġrange": 1917, + "¿Ģ": 1918, + "ative": 1919, + "ues": 1920, + ").": 1921, + "Ġtech": 1922, + "æ³Ľ": 1923, + "Ġfeel": 1924, + "ĊĠ": 1925, + "ĠSt": 1926, + "Can": 1927, + "000": 1928, + "bs": 1929, + "ful": 1930, + "It": 1931, + "å°½": 1932, + "åŃĺåĤ¨": 1933, + "æ²»": 1934, + "åIJĹ": 1935, + "æĬ¥": 1936, + "建ç«ĭ": 1937, + "åıĤä¸İ": 1938, + "å¿«éĢŁ": 1939, + "åĩ»": 1940, + "çª": 1941, + "è¯Ħä¼°": 1942, + "Ġunder": 1943, + "å°Ŀè¯ķ": 1944, + "ors": 1945, + "Ġtra": 1946, + "Ġ/": 1947, + "çļĦæĥħ": 1948, + "éļıçĿĢ": 1949, + "ä¸ĸçķĮ": 1950, + "Ġrec": 1951, + "æĮī": 1952, + "Ġtime": 1953, + "头": 1954, + "Ġstring": 1955, + "åħ³äºİ": 1956, + "åħ·ä½ĵçļĦ": 1957, + "è¾ĵåĩº": 1958, + "åŁºæľ¬": 1959, + "åħ·æľī": 1960, + "åĪĿ": 1961, + "çħ§": 1962, + "åĬ±": 1963, + "ä¸Ģç§į": 1964, + "åŃĹ符串": 1965, + "cript": 1966, + "èĮĥ": 1967, + "ä½łå¥½": 1968, + "Ġsim": 1969, + "以ä¸ĭæĺ¯ä¸ĢäºĽ": 1970, + "æŃ¥éª¤": 1971, + "æ¤": 1972, + "Ġ<": 1973, + "Ġvalue": 1974, + "Ġthere": 1975, + "ery": 1976, + "款": 1977, + "æ¿Ģ": 1978, + "eng": 1979, + "æĭŁ": 1980, + "Ġdis": 1981, + "ray": 1982, + "Ġcontext": 1983, + "åıĭ": 1984, + "å¾ģ": 1985, + "æĹħ": 1986, + "æµĭè¯ķ": 1987, + "Ġdiffere": 1988, + "ll": 1989, + "åħ´è¶£": 1990, + "ç§ijåѦ": 1991, + "åħ¨çIJĥ": 1992, + "Ġinter": 1993, + "èĥĮ": 1994, + "æĬĢæľ¯çļĦ": 1995, + "å®ĮæĪIJ": 1996, + "æĸ¹æ¡Ī": 1997, + "ital": 1998, + "ä¸įæĸŃ": 1999, + "ark": 2000, + "Ġph": 2001, + "çĽĬ": 2002, + "led": 2003, + "Ġi": 2004, + "æļ": 2005, + "ç¿": 2006, + "å·®": 2007, + "éļIJ": 2008, + "ber": 2009, + "ä¹ĭéĹ´çļĦ": 2010, + "ĠQwen": 2011, + "ays": 2012, + "ĠK": 2013, + "å¿ħ": 2014, + "éĤ£ä¹Ī": 2015, + "åij¨": 2016, + "å¹¿æ³Ľ": 2017, + "roup": 2018, + "erson": 2019, + "ics": 2020, + "çŁ¥è¯Ĩ": 2021, + "ç§ijæĬĢ": 2022, + "ese": 2023, + "æıIJä¾ĽäºĨ": 2024, + "å¸Ī": 2025, + "times": 2026, + "è´Ł": 2027, + "åħ»": 2028, + "Ġnum": 2029, + "ict": 2030, + "è¡Įä¸ļ": 2031, + "Ġtechn": 2032, + "éĶĻ误": 2033, + "çļĦæķ°æį®": 2034, + "è¿Ķ": 2035, + "èī¯": 2036, + "',": 2037, + "Ġcheck": 2038, + "),": 2039, + "ails": 2040, + "æĿ¡ä»¶": 2041, + "ä»·åĢ¼": 2042, + "çĪ±": 2043, + "è¿ij": 2044, + "Ġfree": 2045, + "gram": 2046, + "åħĭ": 2047, + "ç±³": 2048, + "Ġhis": 2049, + "åŃĺåľ¨": 2050, + "hen": 2051, + "ç®Ģåįķ": 2052, + "访": 2053, + "声": 2054, + "ular": 2055, + "gest": 2056, + "ĠFor": 2057, + "åı¤": 2058, + "ç§ģ": 2059, + "åįı": 2060, + "åŁºäºİ": 2061, + "æĽ´å¥½": 2062, + "Ġadd": 2063, + "As": 2064, + "è´£": 2065, + "æķ°ç»Ħ": 2066, + "Ġrel": 2067, + "å®ļä¹ī": 2068, + "çīĮ": 2069, + "她": 2070, + "è¯ĨåĪ«": 2071, + "人工": 2072, + "åŁºç¡": 2073, + "åŁºç¡Ģ": 2074, + "表示": 2075, + "çĶ»": 2076, + "离": 2077, + "Ġcomput": 2078, + "row": 2079, + "åºĶ该": 2080, + "ä»İèĢĮ": 2081, + "è¶Ĭ": 2082, + "è¿ĶåĽŀ": 2083, + "èijĹ": 2084, + "æĿIJæĸĻ": 2085, + "æŀĹ": 2086, + "Ġresult": 2087, + "æĺ¯åľ¨": 2088, + "ç¼ĸç¨ĭ": 2089, + "(\"": 2090, + "andom": 2091, + "def": 2092, + "解éĩĬ": 2093, + "åģĩ": 2094, + "Ġì": 2095, + "é¼": 2096, + "ï¼īï¼Į": 2097, + "éģĩ": 2098, + "ient": 2099, + "cial": 2100, + "ens": 2101, + "Ġtwo": 2102, + "寻": 2103, + "Ġsome": 2104, + "Ġthem": 2105, + "QL": 2106, + "ern": 2107, + "ä¸ĬçļĦ": 2108, + "Ġwho": 2109, + "Ġthey": 2110, + "ä½Ĩæĺ¯": 2111, + "å¢ŀ强": 2112, + "åı¯èĥ½éľĢè¦ģ": 2113, + "ä¼Ĺ": 2114, + "velop": 2115, + "åŁİå¸Ĥ": 2116, + "æĤ¨çļĦ": 2117, + "е": 2118, + "der": 2119, + "åĭ": 2120, + "åħ±åIJĮ": 2121, + "Ġtoday": 2122, + "å¡": 2123, + "Ġcharact": 2124, + "fore": 2125, + "åľºæĻ¯": 2126, + "åĽłç´ł": 2127, + "Ġseems": 2128, + "ble": 2129, + "移": 2130, + "羣": 2131, + "ather": 2132, + "rough": 2133, + "丰": 2134, + "numb": 2135, + "))": 2136, + "ater": 2137, + "cess": 2138, + "Ġset": 2139, + "çģµ": 2140, + "Ġover": 2141, + "ize": 2142, + "åŁ¹": 2143, + "è¦ģæ±Ĥ": 2144, + "åĿĩ": 2145, + "Ġdetails": 2146, + "æĬķèµĦ": 2147, + "èħ": 2148, + "ç»ĩ": 2149, + "ĠIn": 2150, + "ility": 2151, + "èĢģ": 2152, + "ween": 2153, + "å¤ļç§į": 2154, + "Ġequation": 2155, + "ible": 2156, + "rit": 2157, + "åı¯ä»¥éĢļè¿ĩ": 2158, + "Ġpop": 2159, + "Ġincluding": 2160, + "æĽ´å¥½åľ°": 2161, + "èīºæľ¯": 2162, + "its": 2163, + "Ġknown": 2164, + "Ġpre": 2165, + "ood": 2166, + "ç½ijç«Ļ": 2167, + "Ġprogram": 2168, + "çŃĶ": 2169, + "ities": 2170, + "转æį¢": 2171, + "Ġbetween": 2172, + "Ġtasks": 2173, + "éĺ¿éĩĮäºij": 2174, + "è¶ħ": 2175, + "ĠChina": 2176, + "示ä¾ĭ": 2177, + "çĻ½": 2178, + "太": 2179, + "ï¼īãĢĤ": 2180, + "ç¡®å®ļ": 2181, + "Ġ19": 2182, + "产çĶŁ": 2183, + "æŃ¢": 2184, + "Ġsign": 2185, + "ŁéĢļ": 2186, + "Ġyear": 2187, + "Ġpe": 2188, + "访éĹ®": 2189, + "ook": 2190, + "æī§": 2191, + "Ġdevelop": 2192, + "ï¼ī**ï¼ļ": 2193, + "Ġthan": 2194, + "duct": 2195, + "èĭ±": 2196, + "Ġthese": 2197, + "Ġperson": 2198, + "循": 2199, + "äºī": 2200, + "Ġcalcul": 2201, + "é¢Ħæµĭ": 2202, + "rib": 2203, + "æĮijæĪĺ": 2204, + "åIJ¬": 2205, + "Ġboth": 2206, + "Ġdon": 2207, + "12": 2208, + "tal": 2209, + "åıijçİ°": 2210, + "ys": 2211, + "**:": 2212, + "åĮºåĿĹ": 2213, + "çļĦå½±åĵį": 2214, + "å¨": 2215, + "Ġ12": 2216, + "çīĪ": 2217, + "stand": 2218, + "ç»Ī": 2219, + "åĸľ": 2220, + "list": 2221, + "ä¹°": 2222, + "åĨį": 2223, + "This": 2224, + "人工æĻºèĥ½": 2225, + "åĮ»çĸĹ": 2226, + "ä»ĺ": 2227, + "å±ŀ": 2228, + "æ¼Ķ": 2229, + "çļĦ人": 2230, + "çļĦæł": 2231, + "计åĪĴ": 2232, + "pon": 2233, + "è¿ĩç¨ĭä¸Ń": 2234, + "丰å¯Į": 2235, + "模å¼ı": 2236, + "åIJĦç§į": 2237, + "Ġwhen": 2238, + "积æŀģ": 2239, + "åĩĨå¤ĩ": 2240, + "ä¸Ĭè¿°": 2241, + "è¿IJè¡Į": 2242, + "æĪIJ为": 2243, + "è¯ī": 2244, + "è§ī": 2245, + "Ġtotal": 2246, + "èįIJ": 2247, + "æ¦Ĥ": 2248, + "é¸": 2249, + "Ġcons": 2250, + "éĺ³": 2251, + "ä½ľçĶ¨": 2252, + "åŃ©": 2253, + "ä¸įæĺ¯": 2254, + "é«ĺæķĪ": 2255, + "Ġtyp": 2256, + "ower": 2257, + "æ·»": 2258, + "讨": 2259, + "а": 2260, + "稳": 2261, + "åĨ³çŃĸ": 2262, + "to": 2263, + "éĿĻ": 2264, + "Ġra": 2265, + "åı¯ä»¥ä½¿çĶ¨": 2266, + "Ġdiv": 2267, + "带æĿ¥": 2268, + "è¾ij": 2269, + "åºŃ": 2270, + "è®°å½ķ": 2271, + "stem": 2272, + "Ġgre": 2273, + "çļĦæī": 2274, + "è¯Ĺ": 2275, + "Ġlast": 2276, + "Ġ100": 2277, + "åѦçĶŁ": 2278, + "端": 2279, + "Ġinteg": 2280, + "Ġgiven": 2281, + "Ġsub": 2282, + "ĠChinese": 2283, + "è¥": 2284, + "Ġadv": 2285, + "社交": 2286, + "令": 2287, + "Ġcould": 2288, + "Who": 2289, + "è¿IJåĬ¨": 2290, + "èıľ": 2291, + "atest": 2292, + "åıĤæķ°": 2293, + "pro": 2294, + "Ġvers": 2295, + "å®Ŀ": 2296, + "çļĦéĩįè¦ģ": 2297, + "èĻ½": 2298, + "ĠV": 2299, + "çĶŁçī©": 2300, + "Ġdifferent": 2301, + "å±±": 2302, + "ä½įç½®": 2303, + "西": 2304, + "ise": 2305, + "大çļĦ": 2306, + "é¦Ī": 2307, + "Ġgenerate": 2308, + "Ġthrough": 2309, + "Ġdoes": 2310, + "è´¢": 2311, + "ªæĸ½": 2312, + "è½½": 2313, + "19": 2314, + "umn": 2315, + "15": 2316, + "éĺħ": 2317, + "Ġind": 2318, + "详": 2319, + "Ġcount": 2320, + "ÑĤ": 2321, + "Sure": 2322, + "åĩłä¸ª": 2323, + "åĵģçīĮ": 2324, + "æ³ķå¾ĭ": 2325, + "æī§è¡Į": 2326, + "åħ¸": 2327, + "èĻ½çĦ¶": 2328, + "çģ«": 2329, + "èĥĮæĻ¯": 2330, + "éŁ³ä¹IJ": 2331, + "ä¸ļåĬ¡": 2332, + "æĸ¯": 2333, + "So": 2334, + "åħ¶ä¸Ń": 2335, + "éĢŁåº¦": 2336, + "ä¼¼": 2337, + "æݨèįIJ": 2338, + "Ġmult": 2339, + "åĽ½éĻħ": 2340, + "çĬ¶æĢģ": 2341, + "åĪ¶ä½ľ": 2342, + "Ġsqu": 2343, + "Ġtechnolog": 2344, + "æŀ¶": 2345, + "åıįé¦Ī": 2346, + "Ġstart": 2347, + "绿": 2348, + "è¡¡": 2349, + "Ġget": 2350, + "è§Ĵèī²": 2351, + "åĽ½å®¶": 2352, + "get": 2353, + "xim": 2354, + "Ġdesign": 2355, + "co": 2356, + "åį°": 2357, + "If": 2358, + "满足": 2359, + "ä¸Ģä¸ĭ": 2360, + "家åºŃ": 2361, + "ç»Ħç»ĩ": 2362, + "交æµģ": 2363, + "riend": 2364, + "çļĦæĦ": 2365, + "Ġmat": 2366, + "èİ·åıĸ": 2367, + "ĠCould": 2368, + "追": 2369, + "å¼ł": 2370, + "ob": 2371, + "ä¼łç»Ł": 2372, + "æłij": 2373, + "é¼ĵ": 2374, + "Ġunderstand": 2375, + "Ġcor": 2376, + "å¿ĥçIJĨ": 2377, + "æݪæĸ½": 2378, + "Ġgo": 2379, + "Ġhum": 2380, + "èį¯": 2381, + "交æĺĵ": 2382, + "round": 2383, + "Ġimport": 2384, + "çļĦä¸Ģ": 2385, + "èĻļ": 2386, + "稳å®ļ": 2387, + "ä»ĭ": 2388, + "è¿Ļä¸Ģ": 2389, + "åĩºçİ°": 2390, + "и": 2391, + "ational": 2392, + "nce": 2393, + "é¢Ŀ": 2394, + "cre": 2395, + "Ġno": 2396, + "Ġart": 2397, + "åĪĩ": 2398, + "åª": 2399, + "Ġel": 2400, + "常è§ģ": 2401, + "çļĦæĮ": 2402, + "éĩıåŃIJ": 2403, + "Ġke": 2404, + "åĽĽ": 2405, + "å²": 2406, + "å½Ĵ": 2407, + "社åĮº": 2408, + "Ġmain": 2409, + "è¿ŀæİ¥": 2410, + "åľ¨çº¿": 2411, + "饮": 2412, + "å½ĵçĦ¶": 2413, + "åĶ®": 2414, + "ç»´æĬ¤": 2415, + "çļĦå®īåħ¨": 2416, + "çŁŃ": 2417, + "è¯ij": 2418, + "Ġlatest": 2419, + "æĥħæĦŁ": 2420, + "ç«ŀ": 2421, + "ian": 2422, + "Ġthen": 2423, + "æ¶ī": 2424, + "æ¤į": 2425, + "éĩıçļĦ": 2426, + "Ġ2023": 2427, + "Ġrem": 2428, + "wor": 2429, + "å±Ģ": 2430, + "num": 2431, + "红": 2432, + "Ġbeen": 2433, + "ake": 2434, + "åħ³æ³¨": 2435, + "æĪIJåijĺ": 2436, + "ick": 2437, + "()`": 2438, + "详ç»Ĩ": 2439, + "ident": 2440, + "æłĩåĩĨ": 2441, + "Ġsignific": 2442, + "è¿ĻéĩĮ": 2443, + "sq": 2444, + "èĮĥåĽ´": 2445, + "ength": 2446, + "iness": 2447, + "æķĪæŀľ": 2448, + "å¹²": 2449, + "çİ©": 2450, + "Ġcomple": 2451, + "æ´ģ": 2452, + "æĬĢèĥ½": 2453, + "Ġoften": 2454, + "ages": 2455, + "çļĦæĶ": 2456, + "éļIJç§ģ": 2457, + "rest": 2458, + "Ġrefer": 2459, + "èĽ": 2460, + "æ³¢": 2461, + "éĥ½æĺ¯": 2462, + ");": 2463, + "åŃ©åŃIJ": 2464, + "ç´§": 2465, + "æħ": 2466, + "éĢı": 2467, + "å°Ķ": 2468, + "åĸĦ": 2469, + "hes": 2470, + "æĿ¾": 2471, + "ä¾Ŀ": 2472, + "åĢĻ": 2473, + "Ġservices": 2474, + "çĶµåŃIJ": 2475, + "æŃ£ç¡®": 2476, + "å·²ç»ı": 2477, + "æīį": 2478, + "çļĦæİ": 2479, + "æľºåĪ¶": 2480, + "çī¹åĪ«": 2481, + "åĪĽéĢł": 2482, + "使å¾Ĺ": 2483, + "åªĴ": 2484, + "11": 2485, + "Ġdep": 2486, + "Is": 2487, + "ä¸ĢæŃ¥": 2488, + "åĵªäºĽ": 2489, + "èģĶç³»": 2490, + "é¼ĵåĬ±": 2491, + "å¥ĩ": 2492, + "éĢĤåºĶ": 2493, + "ï¼Ľ": 2494, + "çļĦæĬĢ": 2495, + "个æĢ§": 2496, + "ç¿»": 2497, + "Ġjust": 2498, + "Ġbus": 2499, + "Ġreg": 2500, + "å½¢æĪIJ": 2501, + "ĠĠĠĠĠ": 2502, + "here": 2503, + "çªģ": 2504, + "éģĩåĪ°": 2505, + "âĢĿãĢĤ": 2506, + "Write": 2507, + "çļĦéĹ®é¢ĺ": 2508, + "åĪ»": 2509, + "lect": 2510, + "ä¹ĭä¸Ģ": 2511, + "å¾Ħ": 2512, + "人们": 2513, + "Ġfact": 2514, + "æ·»åĬł": 2515, + "åijĺå·¥": 2516, + "交éĢļ": 2517, + "ose": 2518, + "é¾": 2519, + "ove": 2520, + "ax": 2521, + "Ġvarious": 2522, + "Ġbased": 2523, + "åĬłåħ¥": 2524, + "cept": 2525, + "æĪIJåĬŁ": 2526, + "ĠGroup": 2527, + "åĮºåĿĹéĵ¾": 2528, + "50": 2529, + "åģı": 2530, + "çļĦä¿¡æģ¯": 2531, + "Ġoper": 2532, + "name": 2533, + "åIJ¸å¼ķ": 2534, + "æ¹": 2535, + "åį¡": 2536, + "ract": 2537, + "æݢ索": 2538, + "æĽ´æĸ°": 2539, + "Ġlear": 2540, + "空éĹ´": 2541, + "inal": 2542, + "æ¶īåıĬ": 2543, + "æ½": 2544, + "ural": 2545, + "ars": 2546, + "Ġmany": 2547, + "blem": 2548, + "Ġsk": 2549, + "è¿ĺèĥ½": 2550, + "ä¸ĭéĿ¢": 2551, + "æľīæīĢ": 2552, + "Ġcreate": 2553, + "æĺİç¡®": 2554, + "ince": 2555, + "æŀĦ建": 2556, + "ings": 2557, + "æĹłæ³ķ": 2558, + "éĢIJ": 2559, + "æĦıåij³": 2560, + "æĪ¿": 2561, + "å¾ĹåĪ°": 2562, + "人çļĦ": 2563, + "åªĴä½ĵ": 2564, + "âĢĿï¼Į": 2565, + "Ġsupp": 2566, + "è¿Ļæĺ¯": 2567, + "æĴŃ": 2568, + "old": 2569, + "åıĹåĪ°": 2570, + "çı": 2571, + "çĦ¶èĢĮ": 2572, + "pect": 2573, + "Ġsquare": 2574, + "计ç®Ĺæľº": 2575, + "ude": 2576, + "æѤå¤ĸ": 2577, + "è¿İ": 2578, + "ically": 2579, + "æİĴåºı": 2580, + "æľĢåIJİ": 2581, + "Ġprocess": 2582, + "ause": 2583, + "éĩijèŀį": 2584, + "害": 2585, + "Ġloc": 2586, + "åĪĨ享": 2587, + "æ²¹": 2588, + "Ġsm": 2589, + "çļĦæ°": 2590, + "课": 2591, + "ä»·æł¼": 2592, + "页": 2593, + "dition": 2594, + "åĪ¤": 2595, + "æŁIJäºĽ": 2596, + "Ġbl": 2597, + "æķ°æį®åºĵ": 2598, + ",\"": 2599, + "·": 2600, + "çī¹å¾ģ": 2601, + "éļľ": 2602, + "åģĩ设": 2603, + "åįĹ": 2604, + "Given": 2605, + "Ġexpress": 2606, + "åijĬè¯ī": 2607, + "Ġfil": 2608, + "åĽ¾åĥı": 2609, + "ci": 2610, + "Ġquestion": 2611, + "个æĢ§åĮĸ": 2612, + "ç¯": 2613, + "ced": 2614, + "ĠĊ": 2615, + "é¤": 2616, + "Ġpos": 2617, + "elf": 2618, + "å°¤": 2619, + "å¦ĤæŀľæĤ¨": 2620, + "æĪijåı¯ä»¥": 2621, + "å®ļçļĦ": 2622, + "èĦij": 2623, + "Ġlead": 2624, + "æĦıåij³çĿĢ": 2625, + "Ġproblem": 2626, + "å±ħ": 2627, + "ix": 2628, + "æŃĮ": 2629, + "Ġtable": 2630, + "è´¦": 2631, + "群": 2632, + "ited": 2633, + "åľ°åĮº": 2634, + "çĶŁäº§": 2635, + "çļĦåıijå±ķ": 2636, + "æĶ¶éĽĨ": 2637, + "èµĽ": 2638, + "åıĪ": 2639, + "审": 2640, + "ä¸Ģ次": 2641, + "Ġcommon": 2642, + "ople": 2643, + "å·§": 2644, + "ution": 2645, + "Ġappro": 2646, + "æĶ¿çŃĸ": 2647, + "é¤IJ": 2648, + "çݰ代": 2649, + "è§ĦåĪĴ": 2650, + "åŃ£": 2651, + "ä½ı": 2652, + "Ġtransl": 2653, + "Ġdef": 2654, + "Ġpar": 2655, + "å¾·": 2656, + "çݯä¿Ŀ": 2657, + "主é¢ĺ": 2658, + "Ġbusiness": 2659, + "表çİ°": 2660, + "ãģ®": 2661, + "èĤ¡": 2662, + "深度": 2663, + "Ġmake": 2664, + "æŁĵ": 2665, + "éĺħ读": 2666, + "ĠUn": 2667, + "éģµ": 2668, + "èIJ½": 2669, + "ĠEng": 2670, + "讲": 2671, + "å¾ħ": 2672, + "Ġever": 2673, + "Ġact": 2674, + "çļĦæ¸": 2675, + "解åĨ³æĸ¹æ¡Ī": 2676, + "Ġrandom": 2677, + "çļĦåºĶçĶ¨": 2678, + "èģĶç½ij": 2679, + "iqu": 2680, + "æ¶Īè´¹": 2681, + "irect": 2682, + "æľºåĻ¨åŃ¦ä¹ł": 2683, + "100": 2684, + "lish": 2685, + "Ġinv": 2686, + "åIJĪéĢĤ": 2687, + "rt": 2688, + "mer": 2689, + "åĪĽå»ºä¸Ģ个": 2690, + "éĻįä½İ": 2691, + "é²": 2692, + "å®ĺ": 2693, + "ĠPlease": 2694, + "string": 2695, + "èİ·å¾Ĺ": 2696, + "æľºä¼ļ": 2697, + "èĤī": 2698, + "室": 2699, + "æĭ¥": 2700, + "许å¤ļ": 2701, + "..": 2702, + "æľĭ": 2703, + "æĿ¥è¯´": 2704, + "ec": 2705, + "Ġonly": 2706, + "ç»ĵåIJĪ": 2707, + "é¦Ļ": 2708, + "ä¸Ńå¿ĥ": 2709, + "åıijçĶŁ": 2710, + "$.": 2711, + "åIJĪçIJĨ": 2712, + "åİĭåĬĽ": 2713, + "åħħ满": 2714, + "çĨ": 2715, + "verage": 2716, + "ï¼īãĢģ": 2717, + "Ġproduct": 2718, + "val": 2719, + "ost": 2720, + "icial": 2721, + "å¦": 2722, + "çļĦæŃ": 2723, + "Ġcompan": 2724, + "è§Ĩé¢ij": 2725, + "ts": 2726, + "ula": 2727, + "mat": 2728, + "iel": 2729, + "Ġweather": 2730, + "æ¯į": 2731, + "å¤ľ": 2732, + "Ġfriend": 2733, + "Ġrequ": 2734, + "ä½ľåĵģ": 2735, + "度åĴĮ": 2736, + "æł¸": 2737, + "ily": 2738, + "numbers": 2739, + "imal": 2740, + "åıĺéĩı": 2741, + "Ġask": 2742, + "Ġwide": 2743, + "æݨåĬ¨": 2744, + "Ġinst": 2745, + "ĠWor": 2746, + "è´Ńä¹°": 2747, + "н": 2748, + "åIJ¯": 2749, + "åįİ": 2750, + "Ġcloud": 2751, + "æĹ¢": 2752, + "èĥ½æºIJ": 2753, + "ized": 2754, + "refore": 2755, + "Ġpopul": 2756, + "è¿ĽæŃ¥": 2757, + "uring": 2758, + "è¿ĺæĺ¯": 2759, + "Al": 2760, + "á»": 2761, + "eters": 2762, + "å°Ħ": 2763, + "up": 2764, + "åĪ¶å®ļ": 2765, + "åĦ": 2766, + "ething": 2767, + "ute": 2768, + "åľ¨è¿Ļ个": 2769, + "å¸ģ": 2770, + "èĽĭ": 2771, + "Ġrespon": 2772, + "Ġway": 2773, + "ç³ĸ": 2774, + "app": 2775, + "çĶµå½±": 2776, + "æľĭåıĭ": 2777, + "Ġeven": 2778, + "æĢ§çļĦ": 2779, + "æ²ŁéĢļ": 2780, + "çļĦæĥħåĨµ": 2781, + "Ġfound": 2782, + "éĢĴ": 2783, + "Ġdec": 2784, + "çļĦæ¯": 2785, + "hip": 2786, + "åĽŃ": 2787, + "Ġeff": 2788, + "è¡Ģ": 2789, + "ç«ŀäºī": 2790, + "own": 2791, + "'d": 2792, + "Ñģ": 2793, + "Ġsignificant": 2794, + "Ġlight": 2795, + "èĻļæĭŁ": 2796, + "Ġmean": 2797, + "Ġsomething": 2798, + "ash": 2799, + "åĪĨç±»": 2800, + "严": 2801, + "继": 2802, + "Ġnam": 2803, + "欢è¿İ": 2804, + "æĹłè®º": 2805, + "æĿ¿": 2806, + "Ġreal": 2807, + "Ġarea": 2808, + "åķ¡": 2809, + "çĶ±äºİ": 2810, + "å¿Ĺ": 2811, + "ç¼ĵ": 2812, + "ife": 2813, + "Ġdig": 2814, + "å°Ĭ": 2815, + "é£ŀ": 2816, + "士": 2817, + "Ġ\\[": 2818, + "身ä½ĵ": 2819, + "éªĮè¯ģ": 2820, + "æĺł": 2821, + "éĥ½æľī": 2822, + "Ġstand": 2823, + "ER": 2824, + "éķ¿æľŁ": 2825, + "quence": 2826, + "举": 2827, + "ving": 2828, + "arr": 2829, + "Ġhapp": 2830, + "æĶ¯ä»ĺ": 2831, + "Ġaccur": 2832, + "Ġher": 2833, + "对象": 2834, + "æī©": 2835, + "æŁ¥è¯¢": 2836, + "æĥ³è¦ģ": 2837, + "loy": 2838, + "奶": 2839, + "Ġaccess": 2840, + "Ġsystem": 2841, + "Ġofficial": 2842, + "顺": 2843, + "Ġhigh": 2844, + "Ġcapital": 2845, + "票": 2846, + "åĴĸ": 2847, + "å½ĵåīį": 2848, + "éĢģ": 2849, + "è·µ": 2850, + "import": 2851, + "rect": 2852, + "Ġpartic": 2853, + "Ġdesigned": 2854, + "æķ´æķ°": 2855, + "综": 2856, + "ĠCan": 2857, + "ç«ł": 2858, + "EC": 2859, + "'ll": 2860, + "å±ķ示": 2861, + "循çݯ": 2862, + "cy": 2863, + "log": 2864, + "Ġplay": 2865, + "Ġsolve": 2866, + "æ¡£": 2867, + "åĴĸåķ¡": 2868, + "æĸ¹åIJij": 2869, + "çļĦåľ°": 2870, + "å®ŀè·µ": 2871, + "转æį¢ä¸º": 2872, + "ild": 2873, + "çķĻ": 2874, + "å®Ī": 2875, + "Ġ15": 2876, + "èģĮä¸ļ": 2877, + "å°¤åħ¶": 2878, + "æĭī": 2879, + "åıĮ": 2880, + "äºĨä¸Ģ个": 2881, + "Ġclass": 2882, + "Ġformula": 2883, + "äºĶ": 2884, + "çļĦæŶéĹ´": 2885, + "æĶ¹åıĺ": 2886, + "Ġstate": 2887, + "ç»į": 2888, + "çIJĨ论": 2889, + "Ġpoint": 2890, + "Ġent": 2891, + "St": 2892, + "Ġday": 2893, + "æįŁ": 2894, + "缺": 2895, + "ately": 2896, + "æī¿": 2897, + "èµ·æĿ¥": 2898, + "Ġtry": 2899, + "Ġsupport": 2900, + "Ġrele": 2901, + "ä¼ļè®®": 2902, + "è¶ĭ": 2903, + "ä¹Łæĺ¯": 2904, + "ources": 2905, + "ending": 2906, + "çļĦæµ": 2907, + "Ġversion": 2908, + "Ġline": 2909, + "py": 2910, + "é£İæł¼": 2911, + "ting": 2912, + "¨è¯¢": 2913, + "åĴ¨è¯¢": 2914, + "ĠHe": 2915, + "临": 2916, + "åł": 2917, + "çļĦæ³": 2918, + "ä»ĭç»į": 2919, + "éĩįè¦ģçļĦ": 2920, + "30": 2921, + "оÐ": 2922, + "大å°ı": 2923, + "Ġtop": 2924, + "è´§": 2925, + "马": 2926, + "Ġbest": 2927, + "ç»Ļåĩº": 2928, + "åĨ·": 2929, + "è¿Ľä¸ĢæŃ¥": 2930, + "è§ĦåĪĻ": 2931, + "ov": 2932, + "æ¦Ĥ念": 2933, + "æĪIJéķ¿": 2934, + "èĹ": 2935, + "Ġperform": 2936, + "Ġcreated": 2937, + "ç¼ĸåĨĻ": 2938, + "Ġide": 2939, + "çŁ³": 2940, + "ľç´¢": 2941, + "èĩªæĪij": 2942, + "Ġwork": 2943, + "light": 2944, + "cle": 2945, + "âĢĻ": 2946, + "Ġinterest": 2947, + "彩": 2948, + "arch": 2949, + "å®īè£ħ": 2950, + "tern": 2951, + "åĪĽä½ľ": 2952, + "25": 2953, + "Ġlong": 2954, + "çī¹çĤ¹": 2955, + "Ġrep": 2956, + "ience": 2957, + "Ġinput": 2958, + "éĺ¶": 2959, + "责任": 2960, + "Ġbook": 2961, + "Ġdirect": 2962, + "ĊĠĠĠĠĠĠĠĠĠĠĠ": 2963, + "å½¢å¼ı": 2964, + "åºĹ": 2965, + "ublic": 2966, + "ìĿ": 2967, + "ĠWorld": 2968, + "åĨ²": 2969, + "çĭ¬çī¹": 2970, + "æķ°æį®çļĦ": 2971, + "æīĢ以": 2972, + "Ġshould": 2973, + "寻æī¾": 2974, + "Ġpeople": 2975, + "ĠEnglish": 2976, + "æķ°åѦ": 2977, + "ruct": 2978, + "ox": 2979, + "åĨħéĥ¨": 2980, + "è¶ĭåĬ¿": 2981, + "Ġcomputing": 2982, + "çī¹å®ļçļĦ": 2983, + "çļ®": 2984, + "éĨ": 2985, + "åĨ³å®ļ": 2986, + "æľįåĬ¡åĻ¨": 2987, + "åĽ°": 2988, + "ä¹ĺ": 2989, + "åIJį称": 2990, + "ator": 2991, + "硬": 2992, + "å¿ħè¦ģ": 2993, + "Ġlarge": 2994, + "Ġmin": 2995, + "Ġassistance": 2996, + "äºĭ件": 2997, + "çļĦæ²": 2998, + "åľ°çIJĥ": 2999, + "çĽĸ": 3000, + "åıĬæŶ": 3001, + "Ġwebs": 3002, + "gy": 3003, + "èī¯å¥½çļĦ": 3004, + "ming": 3005, + "é": 3006, + "ç¿»è¯ij": 3007, + "16": 3008, + "éĻ¢": 3009, + "ink": 3010, + "æķ°éĩı": 3011, + "Ġupdate": 3012, + "æĺ¯ä»Ģä¹Ī": 3013, + "åĦ¿": 3014, + "èĬĤçĤ¹": 3015, + "åĨľ": 3016, + "Ġquery": 3017, + "çī©çIJĨ": 3018, + "女": 3019, + "æķ£": 3020, + "æĿİ": 3021, + "Ġtechnology": 3022, + "Ġarray": 3023, + "åľŁ": 3024, + "Ġstr": 3025, + "Ġvalues": 3026, + "Script": 3027, + "ÑĢ": 3028, + "hed": 3029, + "å¯Ł": 3030, + "ä¸įä¼ļ": 3031, + "ows": 3032, + "çĽĺ": 3033, + "éĵ¶": 3034, + "åķĨåĵģ": 3035, + "Ġlargest": 3036, + "eb": 3037, + "é¢ľ": 3038, + "åı¶": 3039, + "str": 3040, + "ctions": 3041, + "æ²»çĸĹ": 3042, + "raph": 3043, + "Ġbu": 3044, + "åĸľæ¬¢": 3045, + "æ°´å¹³": 3046, + "æ½ľ": 3047, + "Ġword": 3048, + "触": 3049, + "çĸ¾": 3050, + "éĴŁ": 3051, + "Ġwhile": 3052, + "Ġexper": 3053, + "æľºæŀĦ": 3054, + "çļĦæĻ": 3055, + "åĽº": 3056, + "Ġcharacter": 3057, + "ides": 3058, + "Ġac": 3059, + "èģļ": 3060, + "Java": 3061, + "Ġstep": 3062, + "illion": 3063, + "认为": 3064, + "éĿ¢å¯¹": 3065, + "ert": 3066, + "Ġdist": 3067, + "ew": 3068, + "ric": 3069, + "ons": 3070, + "梦": 3071, + "æĤ£": 3072, + "å®Įåħ¨": 3073, + "èĮ¶": 3074, + "cd": 3075, + "åij½ä»¤": 3076, + "éħįç½®": 3077, + "fter": 3078, + "è·Ŀ": 3079, + "Ġpol": 3080, + "Ġlearning": 3081, + "æĺ¯æĮĩ": 3082, + "Ġcalculate": 3083, + "âĢĶ": 3084, + "课ç¨ĭ": 3085, + "Ġ}": 3086, + "顾": 3087, + "Ġcorrect": 3088, + "ä¸Ģèµ·": 3089, + "Ġla": 3090, + "ĠçļĦ": 3091, + "ä¿®æĶ¹": 3092, + "word": 3093, + "Ġread": 3094, + "14": 3095, + "äºĴåĬ¨": 3096, + "Ġcent": 3097, + "^{": 3098, + "ustom": 3099, + "arth": 3100, + "Ġwell": 3101, + "æĺ¥": 3102, + "ä¾ĭåŃIJ": 3103, + "åĮĹ": 3104, + "Ġelse": 3105, + "åijĬè¯īæĪij": 3106, + "éĩĩçĶ¨": 3107, + "ا": 3108, + "对è¯Ŀ": 3109, + "åı¯ä»¥åľ¨": 3110, + "çļĦæĸ¹æ³ķ": 3111, + "åı¯ä»¥å¸®åĬ©": 3112, + "ough": 3113, + "çŃ¾": 3114, + "itive": 3115, + "æ¸IJ": 3116, + "é»ij": 3117, + "èĢħçļĦ": 3118, + "çĮ®": 3119, + "SS": 3120, + "åıįåºĶ": 3121, + "æĺ¾ç¤º": 3122, + "讨论": 3123, + "Ġsec": 3124, + "Ġlooking": 3125, + "çļĦå·¥ä½ľ": 3126, + "ĠAd": 3127, + "\\(": 3128, + "ah": 3129, + "åĬłå¯Ĩ": 3130, + "Ġhuman": 3131, + "æ¡Ĩ": 3132, + "éĤ®": 3133, + "æĪijå°Ĩ": 3134, + "ä½ĵçİ°": 3135, + "Ġeng": 3136, + "éĶĢåĶ®": 3137, + "line": 3138, + "\\).": 3139, + "çĶŁæĢģ": 3140, + "åIJĮæĹ¶ä¹Ł": 3141, + "éĴĪ": 3142, + "åºľ": 3143, + "äºĨä¸Ģ": 3144, + "åŁ¹è®Ń": 3145, + "æĸĩæ¡£": 3146, + "伤": 3147, + "éģį": 3148, + "\":": 3149, + "Ġsun": 3150, + "iss": 3151, + "æĭ¥æľī": 3152, + "åºĶ对": 3153, + "åı¯æĮģç»Ń": 3154, + "çļĦæľĢ": 3155, + "Ġposs": 3156, + "çļĦæĬĢæľ¯": 3157, + "çĶŁåij½": 3158, + "ik": 3159, + "åĬŀ": 3160, + "Ġrun": 3161, + "æµģç¨ĭ": 3162, + "Ġpopulation": 3163, + "ases": 3164, + "æĹ¥å¸¸": 3165, + "并ä¸į": 3166, + "Ġpersonal": 3167, + "Ġaccurate": 3168, + "ploy": 3169, + "Ġbre": 3170, + "Ġgu": 3171, + "éħĴ": 3172, + "丽": 3173, + "èĪª": 3174, + "Ġsimple": 3175, + "éļıæľº": 3176, + "æIJľç´¢": 3177, + "åı¦": 3178, + "åį³ä½¿": 3179, + "æĹ©": 3180, + "SQL": 3181, + "gle": 3182, + "åı¥åŃIJ": 3183, + "comm": 3184, + "éĩĩåıĸ": 3185, + "ethod": 3186, + "éķ¿åº¦": 3187, + "è¿ĻæľīåĬ©äºİ": 3188, + "PI": 3189, + "Ġchange": 3190, + "Ġshe": 3191, + "ãģĻ": 3192, + "ren": 3193, + "继ç»Ń": 3194, + "æĦŁåĪ°": 3195, + "æł¡": 3196, + "Ġ16": 3197, + "çł´": 3198, + "设å®ļ": 3199, + "wn": 3200, + "hether": 3201, + "Ġmess": 3202, + "Ġ30": 3203, + "è´¢åĬ¡": 3204, + "Ġ==": 3205, + "é¢ľèī²": 3206, + "æŨ": 3207, + "è¾¾åĪ°": 3208, + "alth": 3209, + "çĸ¾çĹħ": 3210, + "ï¼Į`": 3211, + "vert": 3212, + "Ġë": 3213, + "åĬª": 3214, + "代表": 3215, + "itions": 3216, + "Ġtype": 3217, + "éĢıæĺİ": 3218, + "ome": 3219, + "ĠWhether": 3220, + "æĦ¿": 3221, + "æĢ¥": 3222, + "\")": 3223, + "æĥħåĨµä¸ĭ": 3224, + "ef": 3225, + "æ¸ħæĻ": 3226, + "ĠWhat": 3227, + "平衡": 3228, + "Ġmulti": 3229, + "lob": 3230, + "åĪĨå¸ĥ": 3231, + "é²ľ": 3232, + "Ġaverage": 3233, + "iron": 3234, + "Ġcolumn": 3235, + "æİ¥åıĹ": 3236, + "SE": 3237, + "Ġest": 3238, + "æĹħè¡Į": 3239, + "Ġdesc": 3240, + "Ġpass": 3241, + "éĻIJåĪ¶": 3242, + "Ġz": 3243, + "æĺ¾èijĹ": 3244, + "æ·±åħ¥": 3245, + "å°Ĭéĩį": 3246, + "èĴ": 3247, + "åģļåĩº": 3248, + "gan": 3249, + "Ġhappy": 3250, + "åĬªåĬĽ": 3251, + "移åĬ¨": 3252, + "Ġlength": 3253, + "ability": 3254, + "以ä¸ĭåĩłä¸ª": 3255, + "å¸ĮæľĽè¿ĻäºĽ": 3256, + "ument": 3257, + "ĊĠĠĠĠ": 3258, + "ç¨İ": 3259, + "ä½Ĩä¸į": 3260, + "éĢ»": 3261, + "ä½łåı¯ä»¥ä½¿çĶ¨": 3262, + "Ġ>": 3263, + "ined": 3264, + "绪": 3265, + "è°ĥçĶ¨": 3266, + "ied": 3267, + "缩": 3268, + "大éĩı": 3269, + "Ġpoem": 3270, + "ures": 3271, + "ique": 3272, + "ven": 3273, + "追æ±Ĥ": 3274, + "akes": 3275, + "æĹ¨åľ¨": 3276, + "Ġpower": 3277, + "éĢ»è¾ij": 3278, + "resent": 3279, + "åı¯èĥ½çļĦ": 3280, + "è¿Ļåı¯èĥ½": 3281, + "Ġsequence": 3282, + "çļĦçĶŁæ´»": 3283, + "ĠZ": 3284, + "¢å¤į": 3285, + "Ġgood": 3286, + "ex": 3287, + "åºıåĪĹ": 3288, + "rans": 3289, + "ç³»åĪĹ": 3290, + "ä½³": 3291, + "äºļ": 3292, + "als": 3293, + "You": 3294, + "ä¾ĽåºĶ": 3295, + "åĬ¨çī©": 3296, + "Ġob": 3297, + "Ġhist": 3298, + "é»Ħ": 3299, + "Ġconvers": 3300, + "æĪij们åı¯ä»¥": 3301, + "驶": 3302, + "åī§": 3303, + "ug": 3304, + "符åIJĪ": 3305, + "Ġshort": 3306, + "Ġag": 3307, + "举": 3308, + "my": 3309, + "ware": 3310, + "Ġkey": 3311, + "å¤ļæł·": 3312, + "çģµæ´»": 3313, + "Ġwrite": 3314, + "Ġpur": 3315, + "22": 3316, + "æīĵåį°": 3317, + "ç¡®å®ŀ": 3318, + "Ġhand": 3319, + "çݩ家": 3320, + "éģĹ": 3321, + "à¹": 3322, + "çļĦæ·": 3323, + "ç§ĺ": 3324, + "els": 3325, + "Ġprim": 3326, + "Ġlife": 3327, + "çļĦç¼": 3328, + "çĽijæݧ": 3329, + "社交åªĴä½ĵ": 3330, + "Ġ25": 3331, + "åħ¨éĿ¢": 3332, + "èĵ": 3333, + "Ġgrow": 3334, + "å®ŀéªĮ": 3335, + "aut": 3336, + "æķı": 3337, + "Ġexpl": 3338, + "çĵ": 3339, + "åºķ": 3340, + "çĤ¹åĩ»": 3341, + "çİ°å®ŀ": 3342, + "ç®ĢåįķçļĦ": 3343, + "ç»ıéªĮ": 3344, + "è·¨": 3345, + "èĹı": 3346, + "éĩįå¤į": 3347, + "æĶ¿åºľ": 3348, + "Ġsoft": 3349, + "æijĦ": 3350, + "éŃ": 3351, + "çľ¼": 3352, + "é£Łçī©": 3353, + "Ġwords": 3354, + "伦": 3355, + "ĠSQL": 3356, + "åIJİçļĦ": 3357, + "Ġyears": 3358, + "亮": 3359, + "æĥ¯": 3360, + "æ¤įçī©": 3361, + "ä¼ij": 3362, + "æ¸ħæĻ°": 3363, + "Ġaround": 3364, + "称为": 3365, + "ä¸ĩ": 3366, + "éļıæŶ": 3367, + "å¿Ĩ": 3368, + "åĬ¨æĢģ": 3369, + "'ve": 3370, + "ĊĊĠĠĠ": 3371, + "Ġapplic": 3372, + "以ä¸Ĭ": 3373, + "Ġwebsite": 3374, + "æīĭæľº": 3375, + "Yes": 3376, + "viron": 3377, + "Ġmethod": 3378, + "Ġcompany": 3379, + "RO": 3380, + "åİŁåĽł": 3381, + "ä¹Łåı¯ä»¥": 3382, + "sqrt": 3383, + "è§Ħ模": 3384, + "ps": 3385, + "Ġwrit": 3386, + "äºij计ç®Ĺ": 3387, + "Ġoutput": 3388, + "Ġdat": 3389, + "Ġgive": 3390, + "ets": 3391, + "Ġimpro": 3392, + "\\),": 3393, + "å¼ĢæĶ¾": 3394, + "æ³ķè§Ħ": 3395, + "æĻ®": 3396, + "æĮīçħ§": 3397, + "18": 3398, + "éĹ»": 3399, + "æģ¢å¤į": 3400, + "Ġtem": 3401, + "è°¢": 3402, + "æľĢç»Ī": 3403, + "Ġ//": 3404, + "Here": 3405, + "æĶ¹è¿Ľ": 3406, + "Ġinf": 3407, + "cdot": 3408, + "æĹ¥æľŁ": 3409, + "ĠJava": 3410, + "å®ĩ": 3411, + "For": 3412, + "Ġjo": 3413, + "æĦıè¯Ĩ": 3414, + "以确ä¿Ŀ": 3415, + "ien": 3416, + "æĽ´å¤ļçļĦ": 3417, + "æļĸ": 3418, + "ream": 3419, + "++": 3420, + "AR": 3421, + "21": 3422, + "éĻIJäºİ": 3423, + "æĬĢå·§": 3424, + "åĨ°": 3425, + "å¹³åĿĩ": 3426, + "çļĦè¿ĩç¨ĭ": 3427, + "ä¸įè¦ģ": 3428, + "Ġreferring": 3429, + "ç¦ı": 3430, + "æĻļ": 3431, + "å°ıæĺİ": 3432, + "太éĺ³": 3433, + "ä¸įåı¯": 3434, + "åĮ»çĶŁ": 3435, + "Ġsmall": 3436, + "æ¯Ĵ": 3437, + "ç©¿": 3438, + "Ġmeans": 3439, + "ML": 3440, + "ze": 3441, + "享åıĹ": 3442, + "çīĽ": 3443, + "çļĦæ¦Ĥ": 3444, + "ines": 3445, + "Ġ使çĶ¨": 3446, + "How": 3447, + "人åijĺ": 3448, + "åĿļ": 3449, + "ĠTo": 3450, + "èĦļ": 3451, + "Ġglob": 3452, + "æ£Ģæµĭ": 3453, + "èµĸ": 3454, + "ode": 3455, + "强è°ĥ": 3456, + "模åŀĭçļĦ": 3457, + "('": 3458, + "vie": 3459, + "ç£": 3460, + "Ġbr": 3461, + "Ġsumm": 3462, + "Ġchat": 3463, + "é±": 3464, + "ä¹łæĥ¯": 3465, + "ellig": 3466, + "亲": 3467, + "Ġparticular": 3468, + "çĹĩ": 3469, + "Ġins": 3470, + "æķ´ä¸ª": 3471, + "çļĦåŃ¦ä¹ł": 3472, + "çļĦæĦı": 3473, + "Ġmark": 3474, + "è¿Ļä¸įä»ħ": 3475, + "æĬ¥åijĬ": 3476, + "ploye": 3477, + "éħ¸": 3478, + "çļĦéľĢæ±Ĥ": 3479, + "etwor": 3480, + "驾": 3481, + "Ch": 3482, + "çļĦæı": 3483, + "Ġåľ¨": 3484, + "è¸": 3485, + "ety": 3486, + "绿èī²": 3487, + "Ġexpression": 3488, + "read": 3489, + "Ġcharacters": 3490, + "èµ°": 3491, + "Ġstory": 3492, + "åĵįåºĶ": 3493, + "è¡ĮåĬ¨": 3494, + ")^": 3495, + "ĠPro": 3496, + "Ġ18": 3497, + "Ġbetter": 3498, + "åįģ": 3499, + "Ġdevelopment": 3500, + "以ä¸ĭæĺ¯ä¸Ģ个": 3501, + "æĿŁ": 3502, + "èIJ¥éĶĢ": 3503, + "æľīæķĪåľ°": 3504, + "é±¼": 3505, + "äºĴèģĶç½ij": 3506, + "æĭħ": 3507, + "深度åŃ¦ä¹ł": 3508, + "Ġprovided": 3509, + "éĺ²æŃ¢": 3510, + "Ġ11": 3511, + "æĸĩç«ł": 3512, + "çĬ¶åĨµ": 3513, + "å¢ŀéķ¿": 3514, + "Ġorder": 3515, + "ç»ĦæĪIJ": 3516, + "éĺµ": 3517, + "Ġmed": 3518, + "鼨": 3519, + "éĢī项": 3520, + "Ġenviron": 3521, + "oh": 3522, + "建设": 3523, + "Ġadvice": 3524, + "Ġend": 3525, + "íķ": 3526, + "Ġfam": 3527, + "Ġfin": 3528, + "Ġusers": 3529, + "ç»ĻæĪij": 3530, + "verse": 3531, + "Ġelements": 3532, + "ĠEx": 3533, + "----": 3534, + "éĿł": 3535, + "请æ±Ĥ": 3536, + "rase": 3537, + "æ±½": 3538, + "æĢ»ä¹ĭ": 3539, + "å®ŀæŶ": 3540, + "ives": 3541, + "åħģ": 3542, + "æıIJåĩº": 3543, + "çŃij": 3544, + "Ġconst": 3545, + "ius": 3546, + "å¯Ĩçłģ": 3547, + "æ´Ĺ": 3548, + "Ġconf": 3549, + "Ġshare": 3550, + "Therefore": 3551, + "24": 3552, + "森": 3553, + "commend": 3554, + "pi": 3555, + "ä¾Ŀèµĸ": 3556, + "çĩ": 3557, + "缸åIJĮ": 3558, + "pen": 3559, + "å®Ļ": 3560, + "çīĪæľ¬": 3561, + "Ġarr": 3562, + "åħ¬åħ±": 3563, + "è§Ī": 3564, + "åĬĽéĩı": 3565, + "Ġident": 3566, + "çĨŁ": 3567, + "液": 3568, + "åij¼": 3569, + "æł¼å¼ı": 3570, + "inu": 3571, + "çļĦåħ·ä½ĵ": 3572, + "åIJĪéĢĤçļĦ": 3573, + "çİĭ": 3574, + "大æķ°æį®": 3575, + "Ġvis": 3576, + "åľĨ": 3577, + "æĢĿèĢĥ": 3578, + "æĽ²": 3579, + "ç§ĭ": 3580, + "Ġtypically": 3581, + "åħģ许": 3582, + "çĮ«": 3583, + "Ġcomplex": 3584, + "æħ§": 3585, + "Ġmaking": 3586, + "ional": 3587, + "ward": 3588, + "Ġsecon": 3589, + "Ġrecommend": 3590, + "å®ĩå®Ļ": 3591, + "//": 3592, + "æŃ£åľ¨": 3593, + "gin": 3594, + "åĪĨéħį": 3595, + "è·Ŀ离": 3596, + "ä¸įèĥ½": 3597, + "Ġcommun": 3598, + "åĿĢ": 3599, + "å¾Ģå¾Ģ": 3600, + ":**": 3601, + "ä¼´": 3602, + "ries": 3603, + "io": 3604, + "éĽĨåĽ¢": 3605, + "ä¸Ńæĸĩ": 3606, + "ember": 3607, + "Ġorig": 3608, + "å¼Ĥ常": 3609, + "ways": 3610, + "åŁ¹åħ»": 3611, + "Ġinclude": 3612, + "èľ": 3613, + "Ġsug": 3614, + "åĭĩ": 3615, + "Ġtrying": 3616, + "Ġmessage": 3617, + "ror": 3618, + "ä½Ļ": 3619, + "Ġsym": 3620, + "æ··": 3621, + "åģľ": 3622, + "çī©è´¨": 3623, + "Ġmach": 3624, + "ergy": 3625, + "ç²ī": 3626, + "Ġdid": 3627, + "çļĦæķħäºĭ": 3628, + "Ġevery": 3629, + "åĽ¢éĺŁæĪIJåijĺ": 3630, + "ãģĹ": 3631, + "å±ķçİ°": 3632, + "éĢIJæ¸IJ": 3633, + "Ġsing": 3634, + "ious": 3635, + "çļĦç¬": 3636, + "Ġallow": 3637, + "Ġtell": 3638, + "ĠUnited": 3639, + "礼": 3640, + "Ġsame": 3641, + "ç®±": 3642, + "ipp": 3643, + "建çŃij": 3644, + "èĢIJ": 3645, + "çļĦåħ³éĶ®": 3646, + "æĦıä¹ī": 3647, + "çľĭåĪ°": 3648, + "èĩªçĦ¶è¯Ńè¨Ģ": 3649, + "Ġcustom": 3650, + "Ġprime": 3651, + "atform": 3652, + "ential": 3653, + "å¦Ĥæŀľä½łæľī": 3654, + "Ġservice": 3655, + "urity": 3656, + "ilities": 3657, + "Ġsee": 3658, + "èŃ": 3659, + "天æ°Ķ": 3660, + "éĥ½èĥ½": 3661, + "Ġbec": 3662, + "é¥®é£Ł": 3663, + "æ¶Īè´¹èĢħ": 3664, + "erv": 3665, + "Ġfeature": 3666, + "ä½łèĥ½": 3667, + "Ġupd": 3668, + "çļĦ主è¦ģ": 3669, + "å®ŀæĸ½": 3670, + "ä»į": 3671, + "Ġmay": 3672, + "çŁ¥éģĵ": 3673, + "解åĨ³éĹ®é¢ĺ": 3674, + "ale": 3675, + "Ġterms": 3676, + "å®ĺæĸ¹": 3677, + "éĢĶ": 3678, + "çľł": 3679, + "æĹı": 3680, + "éľ²": 3681, + "å¥ĸ": 3682, + "Ġcontain": 3683, + "çĶ¨çļĦ": 3684, + "åĪĨéĴŁ": 3685, + "ä¸Ģå®ļ": 3686, + "Ġfl": 3687, + "温度": 3688, + "ai": 3689, + "Ġright": 3690, + "¹ģ": 3691, + "æĶ»": 3692, + "ãĤĴ": 3693, + "åĪĽæĦı": 3694, + "è¿Ļå°Ĩ": 3695, + "åı¯ä»¥æıIJä¾Ľ": 3696, + "æľºåĻ¨äºº": 3697, + "Ġwid": 3698, + "å¦Ĥä¸ĭ": 3699, + "Ġstud": 3700, + "ĠSh": 3701, + "åĪ¤æĸŃ": 3702, + "æīįèĥ½": 3703, + "鼶": 3704, + "驾驶": 3705, + "Ġcat": 3706, + "ç¥ŀç»ı": 3707, + "ä»Ĭ": 3708, + "air": 3709, + "çļĦæ·±": 3710, + "åķĨä¸ļ": 3711, + "\".": 3712, + "$,": 3713, + "æĪijä¼ļ": 3714, + "ç±į": 3715, + "Ġsky": 3716, + "以便": 3717, + "è¡¥": 3718, + "èª": 3719, + "çIJĨ解åĴĮ": 3720, + "çĻ»": 3721, + "å¤įæĿĤçļĦ": 3722, + "æī©å±ķ": 3723, + "Ġav": 3724, + "æĪijçļĦ": 3725, + "å½ĵçĦ¶åı¯ä»¥": 3726, + "å·®å¼Ĥ": 3727, + "Ġstor": 3728, + "æ±ĩ": 3729, + "áº": 3730, + "è¾ħ": 3731, + "åĽ°éļ¾": 3732, + "Ġfile": 3733, + "è¶³å¤Ł": 3734, + "Ġcle": 3735, + "ivid": 3736, + "å°±æĺ¯": 3737, + "yn": 3738, + "ç»ĥä¹ł": 3739, + "åĨ³äºİ": 3740, + "åıĸåĨ³äºİ": 3741, + "æĦŁåħ´è¶£": 3742, + "åįĬ": 3743, + "æĽ¿": 3744, + "ä¿¡åı·": 3745, + "ç»ıåİĨ": 3746, + "isp": 3747, + "èī¾": 3748, + "13": 3749, + "åĮĸçļĦ": 3750, + "Ġcar": 3751, + "Of": 3752, + "æ°§": 3753, + "å·¥ç¨ĭ": 3754, + "ä¸įè¿ĩ": 3755, + "Ġcost": 3756, + "count": 3757, + "æħ¢": 3758, + "Ġcour": 3759, + "åĮºåŁŁ": 3760, + "çĽ¸å¯¹": 3761, + "Ġbusinesses": 3762, + "éĤ®ä»¶": 3763, + "AT": 3764, + "°": 3765, + "Ġroot": 3766, + "rix": 3767, + "Ġuser": 3768, + "é¾Ħ": 3769, + "atic": 3770, + "Ġ50": 3771, + "ç³»ç»ŁçļĦ": 3772, + "æŁ¥çľĭ": 3773, + "ä¸ī个": 3774, + "Ġanal": 3775, + "说æĺİ": 3776, + "åĵĪ": 3777, + "èĢĹ": 3778, + "çļĦ模": 3779, + "ithm": 3780, + "99": 3781, + "Ġens": 3782, + "Ġatt": 3783, + "Ġelement": 3784, + "Ġthree": 3785, + "çļĦç¾": 3786, + "Ġcult": 3787, + "æŁIJ个": 3788, + "Ġproviding": 3789, + "æ¯Ķè¾ĥ": 3790, + "Ġintern": 3791, + "ä¼łè¾ĵ": 3792, + "常è§ģçļĦ": 3793, + "ively": 3794, + "Ġ\\(\\": 3795, + "è§Ĩè§ī": 3796, + "utions": 3797, + "æĢģ度": 3798, + "Ġproject": 3799, + "ull": 3800, + "èĩªçĶ±": 3801, + "纸": 3802, + "éĺ¶æ®µ": 3803, + "åıįæĺł": 3804, + "èįī": 3805, + "Ġmu": 3806, + "sh": 3807, + "纪": 3808, + "Th": 3809, + "Ġsuggest": 3810, + "itle": 3811, + "ç½²": 3812, + "éĢļè¿ĩä¸Ĭè¿°": 3813, + "åħ±äº«": 3814, + "æĿ¥æºIJ": 3815, + "ä¼ĺåĬ¿": 3816, + "åİŁçIJĨ": 3817, + "åŃ¦ä¹łåĴĮ": 3818, + "å¤ı": 3819, + "ãģ¾": 3820, + "elligence": 3821, + "çĭ¬çī¹çļĦ": 3822, + "ëĭ": 3823, + "åĢ¾": 3824, + "ivers": 3825, + "åĨħåŃĺ": 3826, + "容æĺĵ": 3827, + "Ġ14": 3828, + "Ġenergy": 3829, + "å¨ģ": 3830, + "Ġprom": 3831, + "Ġidea": 3832, + "å®īåħ¨æĢ§": 3833, + "ructure": 3834, + "Ġsides": 3835, + "çģ¯": 3836, + "],": 3837, + "result": 3838, + "Ġmil": 3839, + "æĮĩçļĦæĺ¯": 3840, + "åĮħæĭ¬ä½Ĩä¸į": 3841, + "åĮħæĭ¬ä½Ĩä¸įéĻIJäºİ": 3842, + "ves": 3843, + "å®ŀä¾ĭ": 3844, + "Ġtrue": 3845, + "ä»ĸçļĦ": 3846, + "ä½ĵçļĦ": 3847, + "Ġfiel": 3848, + "ense": 3849, + "ote": 3850, + "`ï¼Į": 3851, + "èĨ": 3852, + "Ġwere": 3853, + "åĽŀçŃĶ": 3854, + "ç¬Ķ": 3855, + "ont": 3856, + "ĠTh": 3857, + "ç»ĦåIJĪ": 3858, + "ht": 3859, + "饰": 3860, + "other": 3861, + "ÙĦ": 3862, + "æĸĩåѦ": 3863, + "ximately": 3864, + "è´¡": 3865, + "æ¯ı个人": 3866, + "Ġimportant": 3867, + "Ġmillion": 3868, + "ç»ĺ": 3869, + "è·¯å¾Ħ": 3870, + "oogle": 3871, + "Ġprogramming": 3872, + "60": 3873, + "sum": 3874, + "ä¸Ģ段": 3875, + "æľīä¸Ģ个": 3876, + "åħ¬å¼ı": 3877, + "æĹłè®ºæĺ¯": 3878, + "订": 3879, + "æĬĹ": 3880, + "è§Ĵ度": 3881, + "Ġenvironment": 3882, + "ID": 3883, + "Ģæľī": 3884, + "åĨĮ": 3885, + "Ġremain": 3886, + "Ġplatform": 3887, + "son": 3888, + "æľĢ大": 3889, + "Ġopt": 3890, + "çļĦåŁºæľ¬": 3891, + "èĩ³åħ³": 3892, + "èį¯çī©": 3893, + "arm": 3894, + "æıĴ": 3895, + "ibr": 3896, + "è´¡çĮ®": 3897, + "Ġafter": 3898, + "ohn": 3899, + "æ±ł": 3900, + "Ġback": 3901, + "åįļ": 3902, + "Ġseries": 3903, + "time": 3904, + "ĠSp": 3905, + "ering": 3906, + "Ġnetwor": 3907, + "uss": 3908, + "Ġdescrib": 3909, + "é«ĺçļĦ": 3910, + "ĠYou": 3911, + "alse": 3912, + "ä»ĸ们çļĦ": 3913, + "åĮ¹": 3914, + "ty": 3915, + "æ¥": 3916, + "è¯ļ": 3917, + "æĿij": 3918, + "æľ¨": 3919, + "çļĦæ°´": 3920, + "æķĪçİĩåĴĮ": 3921, + "延": 3922, + "aves": 3923, + "perature": 3924, + "æĪijæĥ³": 3925, + "ything": 3926, + "ä¹İ": 3927, + "æĶ¹åĸĦ": 3928, + "éĵ¶è¡Į": 3929, + "ï¼ļâĢľ": 3930, + "磩": 3931, + "åıĺå¾Ĺ": 3932, + "宽": 3933, + "åı¯ä»¥å¸®åĬ©ä½ł": 3934, + "éģįåİĨ": 3935, + "èĢĮä¸įæĺ¯": 3936, + "Ġsn": 3937, + "ank": 3938, + "æ´ŀ": 3939, + "è¿Ļæĺ¯ä¸Ģ个": 3940, + "¡çľł": 3941, + "Ġè¾ĵåĩº": 3942, + "____": 3943, + "京": 3944, + "Do": 3945, + "Ġsur": 3946, + "Ġalways": 3947, + "è±Ĩ": 3948, + "Ġwater": 3949, + "stit": 3950, + "éĿĴ": 3951, + "Ġlim": 3952, + "Ġhig": 3953, + "`,": 3954, + "ma": 3955, + "Ġpossible": 3956, + "Ġcir": 3957, + "Ġstatement": 3958, + "Ġmachine": 3959, + "Ġinteger": 3960, + "èĵĿ": 3961, + "ĠAs": 3962, + "Ġhour": 3963, + "丰å¯ĮçļĦ": 3964, + "Ġinterested": 3965, + "±ä¹IJ": 3966, + "泡": 3967, + "åįĥ": 3968, + "综åIJĪ": 3969, + "ãĢĤâĢĿ": 3970, + "åįķè¯į": 3971, + "Ġcity": 3972, + "let": 3973, + "åĨĽ": 3974, + "Ġinde": 3975, + "Ġdeterm": 3976, + "寻æ±Ĥ": 3977, + "è¿Ľåħ¥": 3978, + "Ġbeaut": 3979, + "Ġair": 3980, + "л": 3981, + "Ġdi": 3982, + "Ġhealth": 3983, + "é¡»": 3984, + "ness": 3985, + "碳": 3986, + "å°ģ": 3987, + "çļĦæ´": 3988, + "`.": 3989, + "æĢİ": 3990, + "ä½įäºİ": 3991, + "汽车": 3992, + "çݯå¢ĥä¸Ń": 3993, + "åĬĽåĴĮ": 3994, + "å½ĵåľ°": 3995, + "Ġobject": 3996, + "atural": 3997, + "å®Įæķ´": 3998, + "ĠFr": 3999, + "çĶ³": 4000, + "Ġvary": 4001, + "Ġcontent": 4002, + "Ġbit": 4003, + "Ġour": 4004, + "çļĦçĥ": 4005, + "Ġunique": 4006, + "Ġwithout": 4007, + "åĴĮè°": 4008, + "è¯ŃéŁ³": 4009, + "åĪł": 4010, + "伦çIJĨ": 4011, + "ource": 4012, + "åĿı": 4013, + "ave": 4014, + "çļĦæĸ¹å¼ı": 4015, + "Ġbeing": 4016, + "Ġcase": 4017, + "æ¯ıä¸Ģ": 4018, + "Ġsal": 4019, + "èĤ¤": 4020, + "Ġbel": 4021, + "èĪĴ": 4022, + "Ġsecurity": 4023, + "ç¹ģ": 4024, + "ually": 4025, + "éĩįæĸ°": 4026, + "çŃīé¢ĨåŁŁ": 4027, + "é¢ĺ缮": 4028, + "çļĦæ¯ı": 4029, + "çļĦ使çĶ¨": 4030, + "éĿ©": 4031, + "ĠSe": 4032, + "ãģ«": 4033, + "Ġscript": 4034, + "éĤ£äºĽ": 4035, + "Ġmon": 4036, + "æ®Ĭ": 4037, + "身份": 4038, + "Ġap": 4039, + "ĠRe": 4040, + "':": 4041, + "çĶµèĦij": 4042, + "è§Ĥå¯Ł": 4043, + "è¿ĻæĦıåij³çĿĢ": 4044, + "æ¡Ĩæŀ¶": 4045, + "çļĦç¼ĸ": 4046, + "Ġincludes": 4047, + "åı³": 4048, + "Ġlog": 4049, + "à¤": 4050, + "çļĦç¾İ": 4051, + "Ġer": 4052, + "Ġimp": 4053, + "Ġ13": 4054, + "çľģ": 4055, + "ner": 4056, + "书ç±į": 4057, + "çī¹æ®Ĭ": 4058, + "åıĤåĬł": 4059, + "Ġdate": 4060, + "åıijå¸ĥ": 4061, + "çļĦæīĢæľī": 4062, + "è¯Ĭ": 4063, + "Ġpublic": 4064, + "éĥ¨ç½²": 4065, + "ross": 4066, + "ãģĦ": 4067, + "åĪłéĻ¤": 4068, + "Ġ%": 4069, + "ization": 4070, + "ä¼łæĴŃ": 4071, + "Ġmaxim": 4072, + "reate": 4073, + "box": 4074, + "çij": 4075, + "Ġdown": 4076, + "èѦ": 4077, + "Ġcourse": 4078, + "sole": 4079, + "ĠåĴĮ": 4080, + "座": 4081, + "以ä¸ĭåĩł": 4082, + "ĠMy": 4083, + "ĠQ": 4084, + "ï¼īï¼ļ": 4085, + "éĢļä¿¡": 4086, + "Re": 4087, + "è½®": 4088, + "ä¸Ĭä¸ĭ": 4089, + "ç¥Ŀ": 4090, + "::": 4091, + "gor": 4092, + "èĸ": 4093, + "å¹´é¾Ħ": 4094, + "Ġinvol": 4095, + "Ġtemperature": 4096, + "ä¿¡ä»»": 4097, + "æıIJéĹ®": 4098, + "Ġprof": 4099, + "ier": 4100, + "è®°å¿Ĩ": 4101, + "森æŀĹ": 4102, + "ä¸įä»ħèĥ½å¤Ł": 4103, + "æĤ£èĢħ": 4104, + "ger": 4105, + "ç»Ļå®ļ": 4106, + "çļĦå°ı": 4107, + "è¯Ńåı¥": 4108, + "ä½łéľĢè¦ģ": 4109, + "éģµå®Ī": 4110, + "Ġrole": 4111, + "åŃĹåħ¸": 4112, + "Ġreleased": 4113, + "Ex": 4114, + "bin": 4115, + "ECT": 4116, + "oy": 4117, + "æ¿Ģåıij": 4118, + "çĶļ": 4119, + "Ġconvert": 4120, + "Ġcommand": 4121, + "åĮ¹éħį": 4122, + "ployees": 4123, + "çļĦçŁ¥": 4124, + "éº": 4125, + "é¦Ĩ": 4126, + "å¡ij": 4127, + "Ġpositive": 4128, + "ON": 4129, + "Ġconcept": 4130, + "ĠEarth": 4131, + "Ġfriendly": 4132, + "åı¯éĿł": 4133, + "å°ıæŶ": 4134, + "åºĶçĶ¨ç¨ĭåºı": 4135, + "è¿Ļ段": 4136, + "Ġindivid": 4137, + "ws": 4138, + "Ġnamed": 4139, + "Ġplan": 4140, + "Ġsort": 4141, + "Ġdoc": 4142, + "è§ĤçĤ¹": 4143, + "ãģ§": 4144, + "Ġiss": 4145, + "ä¸ĭçļĦ": 4146, + "设æĸ½": 4147, + "åĴĮè°IJ": 4148, + "空æ°Ķ": 4149, + "æij©": 4150, + "åĬ³": 4151, + "请æıIJä¾Ľ": 4152, + "Ġsources": 4153, + "ĠCom": 4154, + "å¤įæĿĤ度": 4155, + "apt": 4156, + "Ġsecond": 4157, + "æĶ»åĩ»": 4158, + "')": 4159, + "Ġown": 4160, + "Ġmeaning": 4161, + "éĢĢ": 4162, + "ä½Ľ": 4163, + "è¡£": 4164, + "cho": 4165, + "æľīä»Ģä¹Ī": 4166, + "æ²³": 4167, + "缸åºĶ": 4168, + "èĶ": 4169, + "çļĦåĢ¼": 4170, + "Ġeffect": 4171, + "LECT": 4172, + "set": 4173, + "ogn": 4174, + "HT": 4175, + "Ġ```": 4176, + "åIJĥ": 4177, + "æĥħ绪": 4178, + "éķĩ": 4179, + "ocus": 4180, + "éĢĤå½ĵ": 4181, + "å¿ħé¡»": 4182, + "çĶļèĩ³": 4183, + "åĽ¾çīĩ": 4184, + "çļĦæĮĩ": 4185, + "çļĦåŁºç¡Ģ": 4186, + "Ġphys": 4187, + "ç»Ŀ": 4188, + "ality": 4189, + "æīĵå¼Ģ": 4190, + "Ġtre": 4191, + "Ġbecause": 4192, + "irt": 4193, + "çļĦæĥħåĨµä¸ĭ": 4194, + "Ġpat": 4195, + "Ġ200": 4196, + "åĮĸåѦ": 4197, + "åģı好": 4198, + "éĹŃ": 4199, + "åľ°åĿĢ": 4200, + "iter": 4201, + "aging": 4202, + "å¦Ĥä½ķåľ¨": 4203, + "Ġtake": 4204, + "åIJį为": 4205, + "åį´": 4206, + "åĪĨåĪ«": 4207, + "çļĦçľ": 4208, + "Ġplace": 4209, + "æľĢè¿ij": 4210, + "Ġ$\\": 4211, + "á": 4212, + "number": 4213, + "Ġmaximum": 4214, + "ç¯ĩ": 4215, + "rue": 4216, + "ĠInt": 4217, + "Ġ24": 4218, + "èĥŀ": 4219, + "case": 4220, + "Ġdeep": 4221, + "ROM": 4222, + "sc": 4223, + "Ġincre": 4224, + "ĠWe": 4225, + "éĥ½ä¼ļ": 4226, + "ç²¾ç¥ŀ": 4227, + "å©": 4228, + "åįıä½ľ": 4229, + "åı¯ä»¥æĺ¯": 4230, + "çŃīå¤ļ个": 4231, + "çĽijæµĭ": 4232, + "åĴĮæľįåĬ¡": 4233, + "Ġdue": 4234, + "Ġear": 4235, + "è¿IJèIJ¥": 4236, + "of": 4237, + "污": 4238, + "é«ĺ度": 4239, + "æī¹": 4240, + "çĸ«": 4241, + "Ġindividual": 4242, + "èĩªçĦ¶è¯Ńè¨Ģå¤ĦçIJĨ": 4243, + "ele": 4244, + "sive": 4245, + "ä¹Łèĥ½": 4246, + "æĻºæħ§": 4247, + "sort": 4248, + "af": 4249, + "åħ¬åı¸çļĦ": 4250, + "çĶ¨æĿ¥": 4251, + "çī¹æĢ§": 4252, + "èļ": 4253, + "Ġexact": 4254, + "åŃĹæ¯į": 4255, + "place": 4256, + "å¾Īå¤ļ": 4257, + "æĽ´åĩĨç¡®": 4258, + "æľīæīĢ帮åĬ©": 4259, + "ä¹ĭåīį": 4260, + "Ġtakes": 4261, + "ral": 4262, + "å®ĥ们çļĦ": 4263, + "çİ°åľ¨": 4264, + "å¹ķ": 4265, + "Ġduring": 4266, + "ç¼ĸè¾ij": 4267, + "gre": 4268, + "仿": 4269, + "æľīä¸Ģ": 4270, + "ters": 4271, + "Ġsince": 4272, + "ention": 4273, + "æĬ½": 4274, + "页éĿ¢": 4275, + "æ½ľåľ¨": 4276, + "è¯Ńæ³ķ": 4277, + "Ġfocus": 4278, + "æĢĿç»´": 4279, + "Ġlocated": 4280, + "!\"": 4281, + "纳": 4282, + "缸åħ³çļĦ": 4283, + "顶": 4284, + "Ġhelpful": 4285, + "Ġfactor": 4286, + "angle": 4287, + "缴åĪ°": 4288, + "ĠStates": 4289, + "湿": 4290, + "æį·": 4291, + "éĵģ": 4292, + "è´Łè´£": 4293, + "iction": 4294, + "Ġnatural": 4295, + "Ġorgan": 4296, + "娱ä¹IJ": 4297, + "ches": 4298, + "Ġleft": 4299, + "Ġblue": 4300, + "ok": 4301, + "é»ĺ": 4302, + "Ġmade": 4303, + "Ġprovides": 4304, + "pose": 4305, + "çļĦçģ": 4306, + "éķľ": 4307, + "åĿIJ": 4308, + "æĿ¥å®ŀçİ°": 4309, + "indows": 4310, + "abilities": 4311, + "请注æĦı": 4312, + "Ġteam": 4313, + "éĢłæĪIJ": 4314, + "çļĦèĥ½åĬĽ": 4315, + "éĨĴ": 4316, + "çĶ¨æĪ·çļĦ": 4317, + "æĹĭ": 4318, + "race": 4319, + "Ġphrase": 4320, + "17": 4321, + "对æĸ¹": 4322, + "Ġage": 4323, + "ĠAnd": 4324, + "çļĦé£İéĻ©": 4325, + "Ġaut": 4326, + "åį«": 4327, + "ä¸Ģèĩ´": 4328, + "ç»Ĩèĥŀ": 4329, + "æĬĢæľ¯çļĦåıijå±ķ": 4330, + "çĶŁæĪIJä¸Ģ个": 4331, + "Ġdivis": 4332, + "å°¤åħ¶æĺ¯": 4333, + "çŃīæĸ¹éĿ¢": 4334, + "Ġasking": 4335, + "缮çļĦ": 4336, + "第äºĮ": 4337, + "èĪĴéĢĤ": 4338, + "æĬĺ": 4339, + "Ġrecent": 4340, + "åįıè®®": 4341, + "è¾Ĩ": 4342, + "å·¨": 4343, + "åīĤ": 4344, + "the": 4345, + "æķ°çļĦ": 4346, + "Ġheart": 4347, + "value": 4348, + "éĿ¢ä¸´": 4349, + "å®ŀéĻħä¸Ĭ": 4350, + "æĿ¯": 4351, + "dd": 4352, + "ĠX": 4353, + "è´µ": 4354, + "è¯ģæĺİ": 4355, + "å±ı": 4356, + "new": 4357, + "ody": 4358, + "éģµå¾ª": 4359, + "Ġglobal": 4360, + "Ġdepending": 4361, + "åĩĢ": 4362, + "å·¦": 4363, + "ĠØ": 4364, + "æĽ´åħ·ä½ĵçļĦ": 4365, + "帮åĬ©æĤ¨": 4366, + "缸äºĴ": 4367, + "Ġreturns": 4368, + "å¸ĮæľĽè¿ĻäºĽå»ºè®®": 4369, + "Ġimprove": 4370, + "Ġmuch": 4371, + "ä»ĸ人": 4372, + "å¼¹": 4373, + "45": 4374, + "åįķä½į": 4375, + "çĦ¦": 4376, + "åĪĹ表ä¸Ń": 4377, + "`ãĢĤ": 4378, + "conom": 4379, + "èĥ½éĩı": 4380, + "Ġcalled": 4381, + "Ġrate": 4382, + "é¢Ħç®Ĺ": 4383, + "èĩ³åħ³éĩįè¦ģ": 4384, + "acc": 4385, + "ç»ĵæĿŁ": 4386, + "åı¯ä»¥èĢĥèĻij": 4387, + "ç»ĨèĬĤ": 4388, + "Ġve": 4389, + "å¯¹ä½ł": 4390, + "ĠThese": 4391, + "è¿ĩ度": 4392, + "ified": 4393, + "è¯Ħä»·": 4394, + "æķ°æį®éĽĨ": 4395, + "é«ĺè´¨éĩı": 4396, + "èŀįåIJĪ": 4397, + "åİŁåĪĻ": 4398, + "Ġapproximately": 4399, + "Ġrepresent": 4400, + "ificial": 4401, + "ason": 4402, + "æĸ¹ç¨ĭ": 4403, + "Ġsocial": 4404, + "Ġfig": 4405, + "SELECT": 4406, + "####": 4407, + "人æ°ij": 4408, + "ãĢij": 4409, + "hest": 4410, + "ãĢIJ": 4411, + "Ġdigital": 4412, + "以ä¸ĭåĩłä¸ªæĸ¹éĿ¢": 4413, + "Ġclo": 4414, + "å°Ĩåħ¶": 4415, + "çļĦåĨħ容": 4416, + "æı¡": 4417, + "random": 4418, + "å·ŀ": 4419, + "è´Ńçī©": 4420, + "hou": 4421, + "éĥ¨éŨ": 4422, + "Ġsnipp": 4423, + "tract": 4424, + "çݰ象": 4425, + "Ġneeds": 4426, + "æķ°åĪĹ": 4427, + "Ġgame": 4428, + "Ġtri": 4429, + "è´´": 4430, + "çĭ¬ç«ĭ": 4431, + "å¿ħè¦ģçļĦ": 4432, + "èĪŀ": 4433, + "æ¸ħæ´ģ": 4434, + "象å¾ģ": 4435, + "è½»æĿ¾": 4436, + "鸡": 4437, + "åĨ²çªģ": 4438, + "Ġgeneral": 4439, + "Ġblo": 4440, + "çļĦæĦŁ": 4441, + "åĨħçļĦ": 4442, + "Ñĥ": 4443, + "ins": 4444, + "çļĦçŁ¥è¯Ĩ": 4445, + "模åĿĹ": 4446, + "éĻ¤äºĨ": 4447, + "è¦Ĩ": 4448, + "yle": 4449, + "Ġintelligence": 4450, + "åĨĻä½ľ": 4451, + "Ġed": 4452, + "Ġrad": 4453, + "cing": 4454, + "Please": 4455, + "ved": 4456, + "èĥ½å¸®åĬ©ä½ł": 4457, + "é¢Ŀå¤ĸ": 4458, + "Ġmarket": 4459, + "Ġnight": 4460, + "input": 4461, + "æķ°æį®åĪĨæŀIJ": 4462, + "æ´²": 4463, + "me": 4464, + "wh": 4465, + "Ġeas": 4466, + "å¤ļå°ij": 4467, + "ãģ¯": 4468, + "ock": 4469, + "Create": 4470, + "Ġloop": 4471, + "When": 4472, + "çħ": 4473, + "gs": 4474, + "åı¯ä»¥æł¹æį®": 4475, + "Ġrecogn": 4476, + "æĸ°éĹ»": 4477, + "Ġ\\),": 4478, + "ä¼ijæģ¯": 4479, + "ĠMa": 4480, + "ä¸Ĭä¸ĭæĸĩ": 4481, + "Ġmem": 4482, + "åĬłå¼º": 4483, + "æ±Ł": 4484, + "éĺ¿éĩĮå·´å·´éĽĨåĽ¢": 4485, + "Ġsingle": 4486, + "æ´ĭ": 4487, + "PU": 4488, + "imate": 4489, + "çļĦæ²ŁéĢļ": 4490, + "åľ¨æŁIJäºĽ": 4491, + "浪": 4492, + "ä¸Ŀ": 4493, + "表éĿ¢": 4494, + "èī¯å¥½": 4495, + "æŀ¶æŀĦ": 4496, + "éĺ»": 4497, + "ãģª": 4498, + "åı¯æĮģç»Ńåıijå±ķ": 4499, + "Ġenh": 4500, + "sql": 4501, + "çŃīäºİ": 4502, + "Ġmultiple": 4503, + "plic": 4504, + "æĸĩåŃĹ": 4505, + "ä¹Łä¼ļ": 4506, + "éĩİ": 4507, + "ountain": 4508, + "200": 4509, + "ling": 4510, + "åĪ¶éĢł": 4511, + "âĢĿï¼Ī": 4512, + "Ġdisc": 4513, + "API": 4514, + "Ġ17": 4515, + "èĻ«": 4516, + "éļĶ": 4517, + "plify": 4518, + "Ġclar": 4519, + "åĪ°äºĨ": 4520, + "æ¹ĸ": 4521, + "Ġder": 4522, + "Ġrequest": 4523, + "æŶåĢĻ": 4524, + "æľªæĿ¥çļĦ": 4525, + "ç½Ĺ": 4526, + "åħ¬å¹³": 4527, + "Ġchang": 4528, + "æĮĩ导": 4529, + "åı«": 4530, + "Ġless": 4531, + "æľīçĿĢ": 4532, + "çļĦæł¸": 4533, + "pper": 4534, + "ony": 4535, + "éĢĤçĶ¨äºİ": 4536, + "ĠÃ": 4537, + "å®ļä½į": 4538, + "æľī趣": 4539, + "çļĦæŃ£": 4540, + "JavaScript": 4541, + "åĬ¨åĬĽ": 4542, + "und": 4543, + "ðŁ": 4544, + "å¼ĢåıijèĢħ": 4545, + "ç¾İåĽ½": 4546, + "èĪ¬": 4547, + "...": 4548, + "åĪĨ为": 4549, + "çļĦæĮijæĪĺ": 4550, + "Ġamount": 4551, + "Ġfilm": 4552, + "å°½éĩı": 4553, + "çļĦæ³ķ": 4554, + "Ġprimary": 4555, + "Ġpopular": 4556, + "ume": 4557, + "self": 4558, + "80": 4559, + "æ¯ĶèµĽ": 4560, + "åIJįåŃĹ": 4561, + "å¡«": 4562, + "èĩªåĬ¨åĮĸ": 4563, + "Ġes": 4564, + "磩éĺµ": 4565, + "rehen": 4566, + "łéģĵ": 4567, + "为ä»Ģä¹Ī": 4568, + "çıŃ": 4569, + "Ġcapabilities": 4570, + "Ġsnippet": 4571, + "è·Ł": 4572, + "" + ], + [ + "in", + "ed" + ], + [ + "ç»", + "ª" + ], + [ + "è°ĥ", + "çĶ¨" + ], + [ + "i", + "ed" + ], + [ + "ç¼", + "©" + ], + [ + "大", + "éĩı" + ], + [ + "Ġpo", + "em" + ], + [ + "ure", + "s" + ], + [ + "iqu", + "e" + ], + [ + "v", + "en" + ], + [ + "追", + "æ±Ĥ" + ], + [ + "ak", + "es" + ], + [ + "æŨ", + "åľ¨" + ], + [ + "Ġp", + "ower" + ], + [ + "éĢ»", + "è¾ij" + ], + [ + "res", + "ent" + ], + [ + "åı¯èĥ½", + "çļĦ" + ], + [ + "è¿Ļ", + "åı¯èĥ½" + ], + [ + "Ġse", + "quence" + ], + [ + "çļĦ", + "çĶŁæ´»" + ], + [ + "Ġ", + "Z" + ], + [ + "¢", + "å¤į" + ], + [ + "Ġg", + "ood" + ], + [ + "e", + "x" + ], + [ + "åºı", + "åĪĹ" + ], + [ + "r", + "ans" + ], + [ + "ç³»", + "åĪĹ" + ], + [ + "ä½", + "³" + ], + [ + "äº", + "ļ" + ], + [ + "al", + "s" + ], + [ + "Y", + "ou" + ], + [ + "ä¾Ľ", + "åºĶ" + ], + [ + "åĬ¨", + "çī©" + ], + [ + "Ġo", + "b" + ], + [ + "Ġh", + "ist" + ], + [ + "é»", + "Ħ" + ], + [ + "Ġcon", + "vers" + ], + [ + "æĪij们", + "åı¯ä»¥" + ], + [ + "é©", + "¶" + ], + [ + "åī", + "§" + ], + [ + "u", + "g" + ], + [ + "符", + "åIJĪ" + ], + [ + "Ġsh", + "ort" + ], + [ + "Ġa", + "g" + ], + [ + "ä¸", + "¾" + ], + [ + "m", + "y" + ], + [ + "w", + "are" + ], + [ + "Ġke", + "y" + ], + [ + "å¤ļ", + "æł·" + ], + [ + "çģµ", + "æ´»" + ], + [ + "Ġw", + "rite" + ], + [ + "Ġp", + "ur" + ], + [ + "2", + "2" + ], + [ + "æīĵ", + "åį°" + ], + [ + "ç¡®", + "å®ŀ" + ], + [ + "Ġh", + "and" + ], + [ + "çİ©", + "家" + ], + [ + "éģ", + "Ĺ" + ], + [ + "à", + "¹" + ], + [ + "çļĦæ", + "·" + ], + [ + "ç§", + "ĺ" + ], + [ + "el", + "s" + ], + [ + "Ġpr", + "im" + ], + [ + "Ġl", + "ife" + ], + [ + "çļĦç", + "¼" + ], + [ + "çĽij", + "æݧ" + ], + [ + "社交", + "åªĴä½ĵ" + ], + [ + "Ġ2", + "5" + ], + [ + "åħ¨", + "éĿ¢" + ], + [ + "è", + "ĵ" + ], + [ + "Ġg", + "row" + ], + [ + "å®ŀ", + "éªĮ" + ], + [ + "a", + "ut" + ], + [ + "æķ", + "ı" + ], + [ + "Ġex", + "pl" + ], + [ + "ç", + "ĵ" + ], + [ + "åº", + "ķ" + ], + [ + "çĤ¹", + "åĩ»" + ], + [ + "çİ°", + "å®ŀ" + ], + [ + "ç®Ģåįķ", + "çļĦ" + ], + [ + "ç»ı", + "éªĮ" + ], + [ + "è·", + "¨" + ], + [ + "èĹ", + "ı" + ], + [ + "éĩį", + "å¤į" + ], + [ + "æĶ¿", + "åºľ" + ], + [ + "Ġso", + "ft" + ], + [ + "æij", + "Ħ" + ], + [ + "é", + "Ń" + ], + [ + "çľ", + "¼" + ], + [ + "é£Ł", + "çī©" + ], + [ + "Ġwor", + "ds" + ], + [ + "ä¼", + "¦" + ], + [ + "ĠS", + "QL" + ], + [ + "åIJİ", + "çļĦ" + ], + [ + "Ġyear", + "s" + ], + [ + "äº", + "®" + ], + [ + "æĥ", + "¯" + ], + [ + "æ¤į", + "çī©" + ], + [ + "ä¼", + "ij" + ], + [ + "æ¸ħæĻ", + "°" + ], + [ + "Ġa", + "round" + ], + [ + "称", + "为" + ], + [ + "ä¸", + "ĩ" + ], + [ + "éļı", + "æŶ" + ], + [ + "å¿", + "Ĩ" + ], + [ + "åĬ¨", + "æĢģ" + ], + [ + "'", + "ve" + ], + [ + "Ċ", + "ĊĠĠĠ" + ], + [ + "Ġapp", + "lic" + ], + [ + "以", + "ä¸Ĭ" + ], + [ + "Ġwebs", + "ite" + ], + [ + "æīĭ", + "æľº" + ], + [ + "Y", + "es" + ], + [ + "v", + "iron" + ], + [ + "Ġm", + "ethod" + ], + [ + "Ġcompan", + "y" + ], + [ + "R", + "O" + ], + [ + "åİŁ", + "åĽł" + ], + [ + "ä¹Ł", + "åı¯ä»¥" + ], + [ + "sq", + "rt" + ], + [ + "è§Ħ", + "模" + ], + [ + "p", + "s" + ], + [ + "Ġw", + "rit" + ], + [ + "äºij", + "计ç®Ĺ" + ], + [ + "Ġout", + "put" + ], + [ + "Ġd", + "at" + ], + [ + "Ġg", + "ive" + ], + [ + "et", + "s" + ], + [ + "Ġim", + "pro" + ], + [ + "\\)", + "," + ], + [ + "å¼Ģ", + "æĶ¾" + ], + [ + "æ³ķ", + "è§Ħ" + ], + [ + "æĻ", + "®" + ], + [ + "æĮī", + "çħ§" + ], + [ + "1", + "8" + ], + [ + "éĹ", + "»" + ], + [ + "æģ", + "¢å¤į" + ], + [ + "Ġt", + "em" + ], + [ + "è°", + "¢" + ], + [ + "æľĢ", + "ç»Ī" + ], + [ + "Ġ/", + "/" + ], + [ + "H", + "ere" + ], + [ + "æĶ¹", + "è¿Ľ" + ], + [ + "Ġin", + "f" + ], + [ + "cd", + "ot" + ], + [ + "æĹ¥", + "æľŁ" + ], + [ + "ĠJ", + "ava" + ], + [ + "å®", + "ĩ" + ], + [ + "F", + "or" + ], + [ + "Ġj", + "o" + ], + [ + "æĦı", + "è¯Ĩ" + ], + [ + "以", + "ç¡®ä¿Ŀ" + ], + [ + "i", + "en" + ], + [ + "æĽ´å¤ļ", + "çļĦ" + ], + [ + "æļ", + "ĸ" + ], + [ + "re", + "am" + ], + [ + "+", + "+" + ], + [ + "A", + "R" + ], + [ + "2", + "1" + ], + [ + "éĻIJ", + "äºİ" + ], + [ + "æĬĢ", + "å·§" + ], + [ + "åĨ", + "°" + ], + [ + "å¹³", + "åĿĩ" + ], + [ + "çļĦ", + "è¿ĩç¨ĭ" + ], + [ + "ä¸į", + "è¦ģ" + ], + [ + "Ġrefer", + "ring" + ], + [ + "ç¦", + "ı" + ], + [ + "æĻ", + "ļ" + ], + [ + "å°ı", + "æĺİ" + ], + [ + "太", + "éĺ³" + ], + [ + "ä¸į", + "åı¯" + ], + [ + "åĮ»", + "çĶŁ" + ], + [ + "Ġsm", + "all" + ], + [ + "æ¯", + "Ĵ" + ], + [ + "ç©", + "¿" + ], + [ + "Ġme", + "ans" + ], + [ + "M", + "L" + ], + [ + "z", + "e" + ], + [ + "享", + "åıĹ" + ], + [ + "çī", + "Ľ" + ], + [ + "çļĦæ", + "¦Ĥ" + ], + [ + "in", + "es" + ], + [ + "Ġ", + "使çĶ¨" + ], + [ + "H", + "ow" + ], + [ + "人", + "åijĺ" + ], + [ + "åĿ", + "ļ" + ], + [ + "ĠT", + "o" + ], + [ + "èĦ", + "ļ" + ], + [ + "Ġg", + "lob" + ], + [ + "æ£Ģ", + "æµĭ" + ], + [ + "èµ", + "ĸ" + ], + [ + "o", + "de" + ], + [ + "强", + "è°ĥ" + ], + [ + "模åŀĭ", + "çļĦ" + ], + [ + "(", + "'" + ], + [ + "v", + "ie" + ], + [ + "ç", + "£" + ], + [ + "Ġb", + "r" + ], + [ + "Ġsum", + "m" + ], + [ + "Ġch", + "at" + ], + [ + "é", + "±" + ], + [ + "ä¹ł", + "æĥ¯" + ], + [ + "ell", + "ig" + ], + [ + "äº", + "²" + ], + [ + "Ġpartic", + "ular" + ], + [ + "çĹ", + "ĩ" + ], + [ + "Ġin", + "s" + ], + [ + "æķ´", + "个" + ], + [ + "çļĦ", + "åŃ¦ä¹ł" + ], + [ + "çļĦæĦ", + "ı" + ], + [ + "Ġm", + "ark" + ], + [ + "è¿Ļ", + "ä¸įä»ħ" + ], + [ + "æĬ¥", + "åijĬ" + ], + [ + "ploy", + "e" + ], + [ + "éħ", + "¸" + ], + [ + "çļĦ", + "éľĢæ±Ĥ" + ], + [ + "et", + "wor" + ], + [ + "é©", + "¾" + ], + [ + "C", + "h" + ], + [ + "çļĦæ", + "ı" + ], + [ + "Ġ", + "åľ¨" + ], + [ + "è", + "¸" + ], + [ + "et", + "y" + ], + [ + "绿", + "èī²" + ], + [ + "Ġexpress", + "ion" + ], + [ + "re", + "ad" + ], + [ + "Ġcharact", + "ers" + ], + [ + "èµ", + "°" + ], + [ + "Ġst", + "ory" + ], + [ + "åĵį", + "åºĶ" + ], + [ + "è¡Į", + "åĬ¨" + ], + [ + ")", + "^" + ], + [ + "ĠP", + "ro" + ], + [ + "Ġ1", + "8" + ], + [ + "Ġbet", + "ter" + ], + [ + "åį", + "ģ" + ], + [ + "Ġdevelop", + "ment" + ], + [ + "以ä¸ĭ", + "æĺ¯ä¸Ģ个" + ], + [ + "æĿ", + "Ł" + ], + [ + "èIJ¥", + "éĶĢ" + ], + [ + "æľīæķĪ", + "åľ°" + ], + [ + "é±", + "¼" + ], + [ + "äºĴ", + "èģĶç½ij" + ], + [ + "æĭ", + "ħ" + ], + [ + "深度", + "åŃ¦ä¹ł" + ], + [ + "Ġprovid", + "ed" + ], + [ + "éĺ²", + "æŃ¢" + ], + [ + "Ġ1", + "1" + ], + [ + "æĸĩ", + "ç«ł" + ], + [ + "çĬ¶", + "åĨµ" + ], + [ + "å¢ŀ", + "éķ¿" + ], + [ + "Ġor", + "der" + ], + [ + "ç»Ħ", + "æĪIJ" + ], + [ + "éĺ", + "µ" + ], + [ + "Ġm", + "ed" + ], + [ + "éĽ", + "¨" + ], + [ + "éĢī", + "项" + ], + [ + "Ġen", + "viron" + ], + [ + "o", + "h" + ], + [ + "建", + "设" + ], + [ + "Ġadv", + "ice" + ], + [ + "Ġ", + "end" + ], + [ + "í", + "ķ" + ], + [ + "Ġf", + "am" + ], + [ + "Ġf", + "in" + ], + [ + "Ġus", + "ers" + ], + [ + "ç»Ļ", + "æĪij" + ], + [ + "vers", + "e" + ], + [ + "Ġele", + "ments" + ], + [ + "ĠE", + "x" + ], + [ + "--", + "--" + ], + [ + "éĿ", + "ł" + ], + [ + "请", + "æ±Ĥ" + ], + [ + "r", + "ase" + ], + [ + "æ±", + "½" + ], + [ + "æĢ»", + "ä¹ĭ" + ], + [ + "å®ŀ", + "æŶ" + ], + [ + "iv", + "es" + ], + [ + "åħ", + "ģ" + ], + [ + "æıIJ", + "åĩº" + ], + [ + "çŃ", + "ij" + ], + [ + "Ġcon", + "st" + ], + [ + "i", + "us" + ], + [ + "å¯Ĩ", + "çłģ" + ], + [ + "æ´", + "Ĺ" + ], + [ + "Ġcon", + "f" + ], + [ + "Ġsh", + "are" + ], + [ + "The", + "refore" + ], + [ + "2", + "4" + ], + [ + "æ£", + "®" + ], + [ + "comm", + "end" + ], + [ + "p", + "i" + ], + [ + "ä¾Ŀ", + "èµĸ" + ], + [ + "ç", + "ĩ" + ], + [ + "缸", + "åIJĮ" + ], + [ + "p", + "en" + ], + [ + "å®", + "Ļ" + ], + [ + "çīĪ", + "æľ¬" + ], + [ + "Ġar", + "r" + ], + [ + "åħ¬", + "åħ±" + ], + [ + "è§", + "Ī" + ], + [ + "åĬĽ", + "éĩı" + ], + [ + "Ġ", + "ident" + ], + [ + "çĨ", + "Ł" + ], + [ + "æ¶", + "²" + ], + [ + "åij", + "¼" + ], + [ + "æł¼", + "å¼ı" + ], + [ + "in", + "u" + ], + [ + "çļĦ", + "åħ·ä½ĵ" + ], + [ + "åIJĪéĢĤ", + "çļĦ" + ], + [ + "çİ", + "ĭ" + ], + [ + "大", + "æķ°æį®" + ], + [ + "Ġv", + "is" + ], + [ + "åľ", + "Ĩ" + ], + [ + "æĢĿ", + "èĢĥ" + ], + [ + "æĽ", + "²" + ], + [ + "ç§", + "ĭ" + ], + [ + "Ġtyp", + "ically" + ], + [ + "åħģ", + "许" + ], + [ + "çĮ", + "«" + ], + [ + "Ġcomple", + "x" + ], + [ + "æ", + "ħ§" + ], + [ + "Ġma", + "king" + ], + [ + "ion", + "al" + ], + [ + "w", + "ard" + ], + [ + "Ġse", + "con" + ], + [ + "Ġre", + "commend" + ], + [ + "å®ĩ", + "å®Ļ" + ], + [ + "/", + "/" + ], + [ + "æŃ£", + "åľ¨" + ], + [ + "g", + "in" + ], + [ + "åĪĨ", + "éħį" + ], + [ + "è·Ŀ", + "离" + ], + [ + "ä¸į", + "èĥ½" + ], + [ + "Ġcomm", + "un" + ], + [ + "åĿ", + "Ģ" + ], + [ + "å¾Ģ", + "å¾Ģ" + ], + [ + ":", + "**" + ], + [ + "ä¼", + "´" + ], + [ + "ri", + "es" + ], + [ + "i", + "o" + ], + [ + "éĽĨ", + "åĽ¢" + ], + [ + "ä¸Ń", + "æĸĩ" + ], + [ + "em", + "ber" + ], + [ + "Ġor", + "ig" + ], + [ + "å¼Ĥ", + "常" + ], + [ + "w", + "ays" + ], + [ + "åŁ¹", + "åħ»" + ], + [ + "Ġincl", + "ude" + ], + [ + "è", + "ľ" + ], + [ + "Ġsu", + "g" + ], + [ + "åĭ", + "ĩ" + ], + [ + "Ġtry", + "ing" + ], + [ + "Ġmess", + "age" + ], + [ + "r", + "or" + ], + [ + "ä½", + "Ļ" + ], + [ + "Ġsy", + "m" + ], + [ + "æ·", + "·" + ], + [ + "åģ", + "ľ" + ], + [ + "çī©", + "è´¨" + ], + [ + "Ġma", + "ch" + ], + [ + "er", + "gy" + ], + [ + "ç²", + "ī" + ], + [ + "Ġd", + "id" + ], + [ + "çļĦ", + "æķħäºĭ" + ], + [ + "Ġever", + "y" + ], + [ + "åĽ¢éĺŁ", + "æĪIJåijĺ" + ], + [ + "ãģ", + "Ĺ" + ], + [ + "å±ķ", + "çİ°" + ], + [ + "éĢIJ", + "æ¸IJ" + ], + [ + "Ġs", + "ing" + ], + [ + "i", + "ous" + ], + [ + "çļĦç", + "¬" + ], + [ + "Ġal", + "low" + ], + [ + "Ġt", + "ell" + ], + [ + "ĠUn", + "ited" + ], + [ + "ç¤", + "¼" + ], + [ + "Ġs", + "ame" + ], + [ + "ç®", + "±" + ], + [ + "i", + "pp" + ], + [ + "建", + "çŃij" + ], + [ + "èĢ", + "IJ" + ], + [ + "çļĦ", + "åħ³éĶ®" + ], + [ + "æĦı", + "ä¹ī" + ], + [ + "çľĭ", + "åĪ°" + ], + [ + "èĩªçĦ¶", + "è¯Ńè¨Ģ" + ], + [ + "Ġc", + "ustom" + ], + [ + "Ġpr", + "ime" + ], + [ + "at", + "form" + ], + [ + "ent", + "ial" + ], + [ + "å¦Ĥæŀľä½ł", + "æľī" + ], + [ + "Ġserv", + "ice" + ], + [ + "ur", + "ity" + ], + [ + "il", + "ities" + ], + [ + "Ġse", + "e" + ], + [ + "è", + "Ń" + ], + [ + "天", + "æ°Ķ" + ], + [ + "éĥ½", + "èĥ½" + ], + [ + "Ġbe", + "c" + ], + [ + "饮", + "é£Ł" + ], + [ + "æ¶Īè´¹", + "èĢħ" + ], + [ + "er", + "v" + ], + [ + "Ġfe", + "ature" + ], + [ + "ä½ł", + "èĥ½" + ], + [ + "Ġup", + "d" + ], + [ + "çļĦ", + "主è¦ģ" + ], + [ + "å®ŀ", + "æĸ½" + ], + [ + "ä»", + "į" + ], + [ + "Ġm", + "ay" + ], + [ + "çŁ¥", + "éģĵ" + ], + [ + "解åĨ³", + "éĹ®é¢ĺ" + ], + [ + "al", + "e" + ], + [ + "Ġterm", + "s" + ], + [ + "å®ĺ", + "æĸ¹" + ], + [ + "éĢ", + "Ķ" + ], + [ + "çľ", + "ł" + ], + [ + "æĹ", + "ı" + ], + [ + "éľ", + "²" + ], + [ + "å¥", + "ĸ" + ], + [ + "Ġcon", + "tain" + ], + [ + "çĶ¨", + "çļĦ" + ], + [ + "åĪĨ", + "éĴŁ" + ], + [ + "ä¸Ģ", + "å®ļ" + ], + [ + "Ġf", + "l" + ], + [ + "温", + "度" + ], + [ + "a", + "i" + ], + [ + "Ġr", + "ight" + ], + [ + "¹", + "ģ" + ], + [ + "æĶ", + "»" + ], + [ + "ãĤ", + "Ĵ" + ], + [ + "åĪĽ", + "æĦı" + ], + [ + "è¿Ļ", + "å°Ĩ" + ], + [ + "åı¯ä»¥", + "æıIJä¾Ľ" + ], + [ + "æľºåĻ¨", + "人" + ], + [ + "Ġw", + "id" + ], + [ + "å¦Ĥ", + "ä¸ĭ" + ], + [ + "Ġst", + "ud" + ], + [ + "ĠS", + "h" + ], + [ + "åĪ¤", + "æĸŃ" + ], + [ + "æīį", + "èĥ½" + ], + [ + "éĽ", + "¶" + ], + [ + "驾", + "驶" + ], + [ + "Ġc", + "at" + ], + [ + "ç¥ŀ", + "ç»ı" + ], + [ + "ä»", + "Ĭ" + ], + [ + "a", + "ir" + ], + [ + "çļĦæ·", + "±" + ], + [ + "åķĨ", + "ä¸ļ" + ], + [ + "\"", + "." + ], + [ + "$", + "," + ], + [ + "æĪij", + "ä¼ļ" + ], + [ + "ç±", + "į" + ], + [ + "Ġsk", + "y" + ], + [ + "以", + "便" + ], + [ + "è¡", + "¥" + ], + [ + "è", + "ª" + ], + [ + "çIJĨ解", + "åĴĮ" + ], + [ + "çĻ", + "»" + ], + [ + "å¤įæĿĤ", + "çļĦ" + ], + [ + "æī©", + "å±ķ" + ], + [ + "Ġa", + "v" + ], + [ + "æĪij", + "çļĦ" + ], + [ + "å½ĵçĦ¶", + "åı¯ä»¥" + ], + [ + "å·®", + "å¼Ĥ" + ], + [ + "Ġst", + "or" + ], + [ + "æ±", + "ĩ" + ], + [ + "á", + "º" + ], + [ + "è¾", + "ħ" + ], + [ + "åĽ°", + "éļ¾" + ], + [ + "Ġf", + "ile" + ], + [ + "足", + "å¤Ł" + ], + [ + "Ġc", + "le" + ], + [ + "iv", + "id" + ], + [ + "å°±", + "æĺ¯" + ], + [ + "y", + "n" + ], + [ + "ç»ĥ", + "ä¹ł" + ], + [ + "åĨ³", + "äºİ" + ], + [ + "åıĸ", + "åĨ³äºİ" + ], + [ + "æĦŁ", + "åħ´è¶£" + ], + [ + "åį", + "Ĭ" + ], + [ + "æĽ", + "¿" + ], + [ + "ä¿¡", + "åı·" + ], + [ + "ç»ı", + "åİĨ" + ], + [ + "is", + "p" + ], + [ + "èī", + "¾" + ], + [ + "1", + "3" + ], + [ + "åĮĸ", + "çļĦ" + ], + [ + "Ġc", + "ar" + ], + [ + "O", + "f" + ], + [ + "æ°", + "§" + ], + [ + "å·¥", + "ç¨ĭ" + ], + [ + "ä¸į", + "è¿ĩ" + ], + [ + "Ġco", + "st" + ], + [ + "c", + "ount" + ], + [ + "æħ", + "¢" + ], + [ + "Ġc", + "our" + ], + [ + "åĮº", + "åŁŁ" + ], + [ + "缸", + "对" + ], + [ + "Ġbusiness", + "es" + ], + [ + "éĤ®", + "件" + ], + [ + "A", + "T" + ], + [ + "Â", + "°" + ], + [ + "Ġro", + "ot" + ], + [ + "ri", + "x" + ], + [ + "Ġus", + "er" + ], + [ + "é¾", + "Ħ" + ], + [ + "at", + "ic" + ], + [ + "Ġ5", + "0" + ], + [ + "ç³»ç»Ł", + "çļĦ" + ], + [ + "æŁ¥", + "çľĭ" + ], + [ + "ä¸ī", + "个" + ], + [ + "Ġan", + "al" + ], + [ + "说", + "æĺİ" + ], + [ + "åĵ", + "Ī" + ], + [ + "èĢ", + "Ĺ" + ], + [ + "çļĦæ", + "¨¡" + ], + [ + "ith", + "m" + ], + [ + "9", + "9" + ], + [ + "Ġen", + "s" + ], + [ + "Ġat", + "t" + ], + [ + "Ġele", + "ment" + ], + [ + "Ġth", + "ree" + ], + [ + "çļĦç", + "¾" + ], + [ + "Ġc", + "ult" + ], + [ + "æŁIJ", + "个" + ], + [ + "Ġprovid", + "ing" + ], + [ + "æ¯Ķ", + "è¾ĥ" + ], + [ + "Ġinter", + "n" + ], + [ + "ä¼ł", + "è¾ĵ" + ], + [ + "常è§ģ", + "çļĦ" + ], + [ + "iv", + "ely" + ], + [ + "Ġ\\(", + "\\" + ], + [ + "è§Ĩ", + "è§ī" + ], + [ + "ut", + "ions" + ], + [ + "æĢģ", + "度" + ], + [ + "Ġpro", + "ject" + ], + [ + "ul", + "l" + ], + [ + "èĩª", + "çĶ±" + ], + [ + "çº", + "¸" + ], + [ + "éĺ¶", + "段" + ], + [ + "åıį", + "æĺł" + ], + [ + "èį", + "ī" + ], + [ + "Ġm", + "u" + ], + [ + "s", + "h" + ], + [ + "çº", + "ª" + ], + [ + "T", + "h" + ], + [ + "Ġsug", + "gest" + ], + [ + "it", + "le" + ], + [ + "ç½", + "²" + ], + [ + "éĢļè¿ĩ", + "ä¸Ĭè¿°" + ], + [ + "åħ±", + "享" + ], + [ + "æĿ¥", + "æºIJ" + ], + [ + "ä¼ĺ", + "åĬ¿" + ], + [ + "åİŁ", + "çIJĨ" + ], + [ + "åŃ¦ä¹ł", + "åĴĮ" + ], + [ + "å¤", + "ı" + ], + [ + "ãģ", + "¾" + ], + [ + "ellig", + "ence" + ], + [ + "çĭ¬çī¹", + "çļĦ" + ], + [ + "ë", + "ĭ" + ], + [ + "åĢ", + "¾" + ], + [ + "iv", + "ers" + ], + [ + "åĨħ", + "åŃĺ" + ], + [ + "容", + "æĺĵ" + ], + [ + "Ġ1", + "4" + ], + [ + "Ġen", + "ergy" + ], + [ + "å¨", + "ģ" + ], + [ + "Ġpro", + "m" + ], + [ + "Ġide", + "a" + ], + [ + "å®īåħ¨", + "æĢ§" + ], + [ + "ruct", + "ure" + ], + [ + "Ġs", + "ides" + ], + [ + "çģ", + "¯" + ], + [ + "]", + "," + ], + [ + "res", + "ult" + ], + [ + "Ġm", + "il" + ], + [ + "æĮĩ", + "çļĦæĺ¯" + ], + [ + "åĮħæĭ¬", + "ä½Ĩä¸į" + ], + [ + "åĮħæĭ¬ä½Ĩä¸į", + "éĻIJäºİ" + ], + [ + "v", + "es" + ], + [ + "å®ŀ", + "ä¾ĭ" + ], + [ + "Ġtr", + "ue" + ], + [ + "ä»ĸ", + "çļĦ" + ], + [ + "ä½ĵ", + "çļĦ" + ], + [ + "Ġf", + "iel" + ], + [ + "en", + "se" + ], + [ + "ot", + "e" + ], + [ + "`", + "ï¼Į" + ], + [ + "è", + "Ĩ" + ], + [ + "Ġwe", + "re" + ], + [ + "åĽŀ", + "çŃĶ" + ], + [ + "ç¬", + "Ķ" + ], + [ + "on", + "t" + ], + [ + "ĠT", + "h" + ], + [ + "ç»Ħ", + "åIJĪ" + ], + [ + "h", + "t" + ], + [ + "é¥", + "°" + ], + [ + "ot", + "her" + ], + [ + "Ù", + "Ħ" + ], + [ + "æĸĩ", + "åѦ" + ], + [ + "xim", + "ately" + ], + [ + "è´", + "¡" + ], + [ + "æ¯ı", + "个人" + ], + [ + "Ġimport", + "ant" + ], + [ + "Ġm", + "illion" + ], + [ + "ç»", + "ĺ" + ], + [ + "è·¯", + "å¾Ħ" + ], + [ + "oo", + "gle" + ], + [ + "Ġprogram", + "ming" + ], + [ + "6", + "0" + ], + [ + "s", + "um" + ], + [ + "ä¸Ģ", + "段" + ], + [ + "æľī", + "ä¸Ģ个" + ], + [ + "åħ¬", + "å¼ı" + ], + [ + "æĹłè®º", + "æĺ¯" + ], + [ + "è®", + "¢" + ], + [ + "æĬ", + "Ĺ" + ], + [ + "è§Ĵ", + "度" + ], + [ + "Ġenviron", + "ment" + ], + [ + "I", + "D" + ], + [ + "Ģ", + "æľī" + ], + [ + "åĨ", + "Į" + ], + [ + "Ġrem", + "ain" + ], + [ + "Ġpl", + "atform" + ], + [ + "s", + "on" + ], + [ + "æľĢ", + "大" + ], + [ + "Ġo", + "pt" + ], + [ + "çļĦ", + "åŁºæľ¬" + ], + [ + "èĩ³", + "åħ³" + ], + [ + "èį¯", + "çī©" + ], + [ + "ar", + "m" + ], + [ + "æı", + "Ĵ" + ], + [ + "ib", + "r" + ], + [ + "è´¡", + "çĮ®" + ], + [ + "Ġa", + "fter" + ], + [ + "oh", + "n" + ], + [ + "æ±", + "ł" + ], + [ + "Ġb", + "ack" + ], + [ + "åį", + "ļ" + ], + [ + "Ġser", + "ies" + ], + [ + "t", + "ime" + ], + [ + "ĠS", + "p" + ], + [ + "er", + "ing" + ], + [ + "Ġn", + "etwor" + ], + [ + "us", + "s" + ], + [ + "Ġdesc", + "rib" + ], + [ + "é«ĺ", + "çļĦ" + ], + [ + "ĠY", + "ou" + ], + [ + "al", + "se" + ], + [ + "ä»ĸ们", + "çļĦ" + ], + [ + "åĮ", + "¹" + ], + [ + "t", + "y" + ], + [ + "æ", + "¥" + ], + [ + "è¯", + "ļ" + ], + [ + "æĿ", + "ij" + ], + [ + "æľ", + "¨" + ], + [ + "çļĦæ°", + "´" + ], + [ + "æķĪçİĩ", + "åĴĮ" + ], + [ + "å»", + "¶" + ], + [ + "av", + "es" + ], + [ + "per", + "ature" + ], + [ + "æĪij", + "æĥ³" + ], + [ + "yth", + "ing" + ], + [ + "ä¹", + "İ" + ], + [ + "æĶ¹", + "åĸĦ" + ], + [ + "éĵ¶", + "è¡Į" + ], + [ + "ï¼ļ", + "âĢľ" + ], + [ + "çŁ", + "©" + ], + [ + "åıĺ", + "å¾Ĺ" + ], + [ + "å®", + "½" + ], + [ + "åı¯ä»¥", + "帮åĬ©ä½ł" + ], + [ + "éģį", + "åİĨ" + ], + [ + "èĢĮ", + "ä¸įæĺ¯" + ], + [ + "Ġs", + "n" + ], + [ + "an", + "k" + ], + [ + "æ´", + "ŀ" + ], + [ + "è¿Ļ", + "æĺ¯ä¸Ģ个" + ], + [ + "¡", + "çľł" + ], + [ + "Ġ", + "è¾ĵåĩº" + ], + [ + "__", + "__" + ], + [ + "äº", + "¬" + ], + [ + "D", + "o" + ], + [ + "Ġs", + "ur" + ], + [ + "Ġal", + "ways" + ], + [ + "è±", + "Ĩ" + ], + [ + "Ġw", + "ater" + ], + [ + "st", + "it" + ], + [ + "éĿ", + "Ĵ" + ], + [ + "Ġl", + "im" + ], + [ + "Ġh", + "ig" + ], + [ + "`", + "," + ], + [ + "m", + "a" + ], + [ + "Ġposs", + "ible" + ], + [ + "Ġc", + "ir" + ], + [ + "Ġstate", + "ment" + ], + [ + "Ġmach", + "ine" + ], + [ + "Ġinteg", + "er" + ], + [ + "èĵ", + "Ŀ" + ], + [ + "ĠA", + "s" + ], + [ + "Ġh", + "our" + ], + [ + "丰å¯Į", + "çļĦ" + ], + [ + "Ġinterest", + "ed" + ], + [ + "±", + "ä¹IJ" + ], + [ + "æ³", + "¡" + ], + [ + "åį", + "ĥ" + ], + [ + "综", + "åIJĪ" + ], + [ + "ãĢĤ", + "âĢĿ" + ], + [ + "åįķ", + "è¯į" + ], + [ + "Ġc", + "ity" + ], + [ + "le", + "t" + ], + [ + "åĨ", + "Ľ" + ], + [ + "Ġin", + "de" + ], + [ + "Ġdet", + "erm" + ], + [ + "寻", + "æ±Ĥ" + ], + [ + "è¿Ľ", + "åħ¥" + ], + [ + "Ġbe", + "aut" + ], + [ + "Ġa", + "ir" + ], + [ + "Ð", + "»" + ], + [ + "Ġd", + "i" + ], + [ + "Ġhe", + "alth" + ], + [ + "é¡", + "»" + ], + [ + "n", + "ess" + ], + [ + "ç¢", + "³" + ], + [ + "å°", + "ģ" + ], + [ + "çļĦæ", + "´" + ], + [ + "`", + "." + ], + [ + "æĢ", + "İ" + ], + [ + "ä½į", + "äºİ" + ], + [ + "æ±½", + "车" + ], + [ + "çݯå¢ĥ", + "ä¸Ń" + ], + [ + "åĬĽ", + "åĴĮ" + ], + [ + "å½ĵ", + "åľ°" + ], + [ + "Ġob", + "ject" + ], + [ + "at", + "ural" + ], + [ + "å®Į", + "æķ´" + ], + [ + "ĠF", + "r" + ], + [ + "çĶ", + "³" + ], + [ + "Ġv", + "ary" + ], + [ + "Ġcont", + "ent" + ], + [ + "Ġb", + "it" + ], + [ + "Ġo", + "ur" + ], + [ + "çļĦç", + "ĥ" + ], + [ + "Ġun", + "ique" + ], + [ + "Ġwith", + "out" + ], + [ + "åĴĮ", + "è°" + ], + [ + "è¯Ń", + "éŁ³" + ], + [ + "åĪ", + "ł" + ], + [ + "伦", + "çIJĨ" + ], + [ + "our", + "ce" + ], + [ + "åĿ", + "ı" + ], + [ + "a", + "ve" + ], + [ + "çļĦ", + "æĸ¹å¼ı" + ], + [ + "Ġbe", + "ing" + ], + [ + "Ġc", + "ase" + ], + [ + "æ¯ı", + "ä¸Ģ" + ], + [ + "Ġs", + "al" + ], + [ + "èĤ", + "¤" + ], + [ + "Ġb", + "el" + ], + [ + "èĪ", + "Ĵ" + ], + [ + "Ġsec", + "urity" + ], + [ + "ç", + "¹ģ" + ], + [ + "u", + "ally" + ], + [ + "éĩį", + "æĸ°" + ], + [ + "çŃī", + "é¢ĨåŁŁ" + ], + [ + "é¢ĺ", + "缮" + ], + [ + "çļĦæ¯", + "ı" + ], + [ + "çļĦ", + "使çĶ¨" + ], + [ + "éĿ", + "©" + ], + [ + "ĠS", + "e" + ], + [ + "ãģ", + "«" + ], + [ + "Ġsc", + "ript" + ], + [ + "éĤ£", + "äºĽ" + ], + [ + "Ġm", + "on" + ], + [ + "æ®", + "Ĭ" + ], + [ + "身", + "份" + ], + [ + "Ġa", + "p" + ], + [ + "ĠR", + "e" + ], + [ + "'", + ":" + ], + [ + "çĶµ", + "èĦij" + ], + [ + "è§Ĥ", + "å¯Ł" + ], + [ + "è¿Ļ", + "æĦıåij³çĿĢ" + ], + [ + "æ¡Ĩ", + "æŀ¶" + ], + [ + "çļĦç¼", + "ĸ" + ], + [ + "Ġinclud", + "es" + ], + [ + "åı", + "³" + ], + [ + "Ġl", + "og" + ], + [ + "à", + "¤" + ], + [ + "çļĦç¾", + "İ" + ], + [ + "Ġ", + "er" + ], + [ + "Ġim", + "p" + ], + [ + "Ġ1", + "3" + ], + [ + "çľ", + "ģ" + ], + [ + "n", + "er" + ], + [ + "书", + "ç±į" + ], + [ + "çī¹", + "æ®Ĭ" + ], + [ + "åıĤ", + "åĬł" + ], + [ + "Ġd", + "ate" + ], + [ + "åıij", + "å¸ĥ" + ], + [ + "çļĦæī", + "Ģæľī" + ], + [ + "è¯", + "Ĭ" + ], + [ + "Ġp", + "ublic" + ], + [ + "éĥ¨", + "ç½²" + ], + [ + "ro", + "ss" + ], + [ + "ãģ", + "Ħ" + ], + [ + "åĪł", + "éĻ¤" + ], + [ + "Ġ", + "%" + ], + [ + "iz", + "ation" + ], + [ + "ä¼ł", + "æĴŃ" + ], + [ + "Ġma", + "xim" + ], + [ + "re", + "ate" + ], + [ + "b", + "ox" + ], + [ + "ç", + "ij" + ], + [ + "Ġd", + "own" + ], + [ + "èŃ", + "¦" + ], + [ + "Ġcour", + "se" + ], + [ + "so", + "le" + ], + [ + "Ġ", + "åĴĮ" + ], + [ + "åº", + "§" + ], + [ + "以ä¸ĭ", + "åĩł" + ], + [ + "ĠM", + "y" + ], + [ + "Ġ", + "Q" + ], + [ + "ï¼ī", + "ï¼ļ" + ], + [ + "éĢļ", + "ä¿¡" + ], + [ + "R", + "e" + ], + [ + "è½", + "®" + ], + [ + "ä¸Ĭ", + "ä¸ĭ" + ], + [ + "ç¥", + "Ŀ" + ], + [ + ":", + ":" + ], + [ + "g", + "or" + ], + [ + "è", + "ĸ" + ], + [ + "å¹´", + "é¾Ħ" + ], + [ + "Ġinv", + "ol" + ], + [ + "Ġtem", + "perature" + ], + [ + "ä¿¡", + "ä»»" + ], + [ + "æıIJ", + "éĹ®" + ], + [ + "Ġpro", + "f" + ], + [ + "i", + "er" + ], + [ + "è®°", + "å¿Ĩ" + ], + [ + "森", + "æŀĹ" + ], + [ + "ä¸įä»ħ", + "èĥ½å¤Ł" + ], + [ + "æĤ£", + "èĢħ" + ], + [ + "g", + "er" + ], + [ + "ç»Ļ", + "å®ļ" + ], + [ + "çļĦ", + "å°ı" + ], + [ + "è¯Ń", + "åı¥" + ], + [ + "ä½ł", + "éľĢè¦ģ" + ], + [ + "éģµ", + "å®Ī" + ], + [ + "Ġro", + "le" + ], + [ + "åŃĹ", + "åħ¸" + ], + [ + "Ġrele", + "ased" + ], + [ + "E", + "x" + ], + [ + "b", + "in" + ], + [ + "EC", + "T" + ], + [ + "o", + "y" + ], + [ + "æ¿Ģ", + "åıij" + ], + [ + "çĶ", + "ļ" + ], + [ + "Ġcon", + "vert" + ], + [ + "Ġcomm", + "and" + ], + [ + "åĮ¹", + "éħį" + ], + [ + "ploye", + "es" + ], + [ + "çļĦç", + "Ł¥" + ], + [ + "é", + "º" + ], + [ + "é¦", + "Ĩ" + ], + [ + "å¡", + "ij" + ], + [ + "Ġpos", + "itive" + ], + [ + "O", + "N" + ], + [ + "Ġcon", + "cept" + ], + [ + "ĠE", + "arth" + ], + [ + "Ġfriend", + "ly" + ], + [ + "åı¯", + "éĿł" + ], + [ + "å°ı", + "æŶ" + ], + [ + "åºĶçĶ¨", + "ç¨ĭåºı" + ], + [ + "è¿Ļ", + "段" + ], + [ + "Ġind", + "ivid" + ], + [ + "w", + "s" + ], + [ + "Ġnam", + "ed" + ], + [ + "Ġpl", + "an" + ], + [ + "Ġs", + "ort" + ], + [ + "Ġdo", + "c" + ], + [ + "è§Ĥ", + "çĤ¹" + ], + [ + "ãģ", + "§" + ], + [ + "Ġis", + "s" + ], + [ + "ä¸ĭ", + "çļĦ" + ], + [ + "设", + "æĸ½" + ], + [ + "åĴĮè°", + "IJ" + ], + [ + "空", + "æ°Ķ" + ], + [ + "æij", + "©" + ], + [ + "åĬ", + "³" + ], + [ + "请", + "æıIJä¾Ľ" + ], + [ + "Ġs", + "ources" + ], + [ + "ĠC", + "om" + ], + [ + "å¤įæĿĤ", + "度" + ], + [ + "a", + "pt" + ], + [ + "Ġsecon", + "d" + ], + [ + "æĶ»", + "åĩ»" + ], + [ + "'", + ")" + ], + [ + "Ġo", + "wn" + ], + [ + "Ġmean", + "ing" + ], + [ + "éĢ", + "Ģ" + ], + [ + "ä½", + "Ľ" + ], + [ + "è¡", + "£" + ], + [ + "ch", + "o" + ], + [ + "æľī", + "ä»Ģä¹Ī" + ], + [ + "æ²", + "³" + ], + [ + "缸", + "åºĶ" + ], + [ + "è", + "Ķ" + ], + [ + "çļĦ", + "åĢ¼" + ], + [ + "Ġeff", + "ect" + ], + [ + "L", + "ECT" + ], + [ + "s", + "et" + ], + [ + "og", + "n" + ], + [ + "H", + "T" + ], + [ + "Ġ", + "```" + ], + [ + "åIJ", + "ĥ" + ], + [ + "æĥħ", + "绪" + ], + [ + "éķ", + "ĩ" + ], + [ + "oc", + "us" + ], + [ + "éĢĤ", + "å½ĵ" + ], + [ + "å¿ħ", + "é¡»" + ], + [ + "çĶļ", + "èĩ³" + ], + [ + "åĽ¾", + "çīĩ" + ], + [ + "çļĦæĮ", + "ĩ" + ], + [ + "çļĦ", + "åŁºç¡Ģ" + ], + [ + "Ġph", + "ys" + ], + [ + "ç»", + "Ŀ" + ], + [ + "al", + "ity" + ], + [ + "æīĵ", + "å¼Ģ" + ], + [ + "Ġt", + "re" + ], + [ + "Ġbec", + "ause" + ], + [ + "ir", + "t" + ], + [ + "çļĦæĥħåĨµ", + "ä¸ĭ" + ], + [ + "Ġp", + "at" + ], + [ + "Ġ2", + "00" + ], + [ + "åĮĸ", + "åѦ" + ], + [ + "åģı", + "好" + ], + [ + "éĹ", + "Ń" + ], + [ + "åľ°", + "åĿĢ" + ], + [ + "it", + "er" + ], + [ + "ag", + "ing" + ], + [ + "å¦Ĥä½ķ", + "åľ¨" + ], + [ + "Ġt", + "ake" + ], + [ + "åIJį", + "为" + ], + [ + "åį", + "´" + ], + [ + "åĪĨ", + "åĪ«" + ], + [ + "çļĦç", + "ľ" + ], + [ + "Ġpl", + "ace" + ], + [ + "æľĢ", + "è¿ij" + ], + [ + "Ġ$", + "\\" + ], + [ + "Ã", + "¡" + ], + [ + "numb", + "er" + ], + [ + "Ġmaxim", + "um" + ], + [ + "ç¯", + "ĩ" + ], + [ + "r", + "ue" + ], + [ + "ĠI", + "nt" + ], + [ + "Ġ2", + "4" + ], + [ + "èĥ", + "ŀ" + ], + [ + "c", + "ase" + ], + [ + "Ġde", + "ep" + ], + [ + "RO", + "M" + ], + [ + "s", + "c" + ], + [ + "Ġin", + "cre" + ], + [ + "ĠW", + "e" + ], + [ + "éĥ½", + "ä¼ļ" + ], + [ + "ç²¾", + "ç¥ŀ" + ], + [ + "å", + "©" + ], + [ + "åįı", + "ä½ľ" + ], + [ + "åı¯ä»¥", + "æĺ¯" + ], + [ + "çŃī", + "å¤ļ个" + ], + [ + "çĽij", + "æµĭ" + ], + [ + "åĴĮ", + "æľįåĬ¡" + ], + [ + "Ġd", + "ue" + ], + [ + "Ġe", + "ar" + ], + [ + "è¿IJ", + "èIJ¥" + ], + [ + "o", + "f" + ], + [ + "æ±", + "¡" + ], + [ + "é«ĺ", + "度" + ], + [ + "æī", + "¹" + ], + [ + "çĸ", + "«" + ], + [ + "Ġindivid", + "ual" + ], + [ + "èĩªçĦ¶è¯Ńè¨Ģ", + "å¤ĦçIJĨ" + ], + [ + "e", + "le" + ], + [ + "s", + "ive" + ], + [ + "ä¹Ł", + "èĥ½" + ], + [ + "æĻº", + "æħ§" + ], + [ + "s", + "ort" + ], + [ + "a", + "f" + ], + [ + "åħ¬åı¸", + "çļĦ" + ], + [ + "çĶ¨", + "æĿ¥" + ], + [ + "çī¹", + "æĢ§" + ], + [ + "è", + "ļ" + ], + [ + "Ġex", + "act" + ], + [ + "åŃĹ", + "æ¯į" + ], + [ + "pl", + "ace" + ], + [ + "å¾Ī", + "å¤ļ" + ], + [ + "æĽ´", + "åĩĨç¡®" + ], + [ + "æľīæīĢ", + "帮åĬ©" + ], + [ + "ä¹ĭ", + "åīį" + ], + [ + "Ġt", + "akes" + ], + [ + "r", + "al" + ], + [ + "å®ĥ们", + "çļĦ" + ], + [ + "çİ°", + "åľ¨" + ], + [ + "å¹", + "ķ" + ], + [ + "Ġd", + "uring" + ], + [ + "ç¼ĸ", + "è¾ij" + ], + [ + "g", + "re" + ], + [ + "ä»", + "¿" + ], + [ + "æľī", + "ä¸Ģ" + ], + [ + "t", + "ers" + ], + [ + "Ġs", + "ince" + ], + [ + "ent", + "ion" + ], + [ + "æĬ", + "½" + ], + [ + "页", + "éĿ¢" + ], + [ + "æ½ľ", + "åľ¨" + ], + [ + "è¯Ń", + "æ³ķ" + ], + [ + "Ġf", + "ocus" + ], + [ + "æĢĿ", + "ç»´" + ], + [ + "Ġloc", + "ated" + ], + [ + "!", + "\"" + ], + [ + "çº", + "³" + ], + [ + "缸åħ³", + "çļĦ" + ], + [ + "é¡", + "¶" + ], + [ + "Ġhelp", + "ful" + ], + [ + "Ġfact", + "or" + ], + [ + "ang", + "le" + ], + [ + "缴", + "åĪ°" + ], + [ + "ĠSt", + "ates" + ], + [ + "æ¹", + "¿" + ], + [ + "æį", + "·" + ], + [ + "éĵ", + "ģ" + ], + [ + "è´Ł", + "è´£" + ], + [ + "ict", + "ion" + ], + [ + "Ġn", + "atural" + ], + [ + "Ġor", + "gan" + ], + [ + "å¨", + "±ä¹IJ" + ], + [ + "c", + "hes" + ], + [ + "Ġle", + "ft" + ], + [ + "Ġbl", + "ue" + ], + [ + "o", + "k" + ], + [ + "é»", + "ĺ" + ], + [ + "Ġma", + "de" + ], + [ + "Ġprovid", + "es" + ], + [ + "p", + "ose" + ], + [ + "çļĦç", + "ģ" + ], + [ + "éķ", + "ľ" + ], + [ + "åĿ", + "IJ" + ], + [ + "æĿ¥", + "å®ŀçİ°" + ], + [ + "ind", + "ows" + ], + [ + "ab", + "ilities" + ], + [ + "请", + "注æĦı" + ], + [ + "Ġte", + "am" + ], + [ + "éĢł", + "æĪIJ" + ], + [ + "çļĦ", + "èĥ½åĬĽ" + ], + [ + "éĨ", + "Ĵ" + ], + [ + "çĶ¨æĪ·", + "çļĦ" + ], + [ + "æĹ", + "ĭ" + ], + [ + "ra", + "ce" + ], + [ + "Ġph", + "rase" + ], + [ + "1", + "7" + ], + [ + "对", + "æĸ¹" + ], + [ + "Ġa", + "ge" + ], + [ + "ĠA", + "nd" + ], + [ + "çļĦ", + "é£İéĻ©" + ], + [ + "Ġa", + "ut" + ], + [ + "åį", + "«" + ], + [ + "ä¸Ģ", + "èĩ´" + ], + [ + "ç»Ĩ", + "èĥŀ" + ], + [ + "æĬĢæľ¯çļĦ", + "åıijå±ķ" + ], + [ + "çĶŁæĪIJ", + "ä¸Ģ个" + ], + [ + "Ġdiv", + "is" + ], + [ + "å°¤åħ¶", + "æĺ¯" + ], + [ + "çŃī", + "æĸ¹éĿ¢" + ], + [ + "Ġas", + "king" + ], + [ + "缮", + "çļĦ" + ], + [ + "第", + "äºĮ" + ], + [ + "èĪĴ", + "éĢĤ" + ], + [ + "æĬ", + "ĺ" + ], + [ + "Ġrec", + "ent" + ], + [ + "åįı", + "è®®" + ], + [ + "è¾", + "Ĩ" + ], + [ + "å·", + "¨" + ], + [ + "åī", + "Ĥ" + ], + [ + "t", + "he" + ], + [ + "æķ°", + "çļĦ" + ], + [ + "Ġhe", + "art" + ], + [ + "val", + "ue" + ], + [ + "éĿ¢", + "临" + ], + [ + "å®ŀéĻħ", + "ä¸Ĭ" + ], + [ + "æĿ", + "¯" + ], + [ + "d", + "d" + ], + [ + "Ġ", + "X" + ], + [ + "è´", + "µ" + ], + [ + "è¯ģ", + "æĺİ" + ], + [ + "å±", + "ı" + ], + [ + "ne", + "w" + ], + [ + "od", + "y" + ], + [ + "éģµ", + "循" + ], + [ + "Ġglob", + "al" + ], + [ + "Ġdep", + "ending" + ], + [ + "åĩ", + "Ģ" + ], + [ + "å·", + "¦" + ], + [ + "Ġ", + "Ø" + ], + [ + "æĽ´", + "åħ·ä½ĵçļĦ" + ], + [ + "帮åĬ©", + "æĤ¨" + ], + [ + "缸", + "äºĴ" + ], + [ + "Ġreturn", + "s" + ], + [ + "å¸ĮæľĽè¿ĻäºĽ", + "建议" + ], + [ + "Ġimpro", + "ve" + ], + [ + "Ġmu", + "ch" + ], + [ + "ä»ĸ", + "人" + ], + [ + "å¼", + "¹" + ], + [ + "4", + "5" + ], + [ + "åįķ", + "ä½į" + ], + [ + "çĦ", + "¦" + ], + [ + "åĪĹ表", + "ä¸Ń" + ], + [ + "`", + "ãĢĤ" + ], + [ + "con", + "om" + ], + [ + "èĥ½", + "éĩı" + ], + [ + "Ġcal", + "led" + ], + [ + "Ġr", + "ate" + ], + [ + "é¢Ħ", + "ç®Ĺ" + ], + [ + "èĩ³åħ³", + "éĩįè¦ģ" + ], + [ + "a", + "cc" + ], + [ + "ç»ĵ", + "æĿŁ" + ], + [ + "åı¯ä»¥", + "èĢĥèĻij" + ], + [ + "ç»Ĩ", + "èĬĤ" + ], + [ + "Ġ", + "ve" + ], + [ + "对", + "ä½ł" + ], + [ + "ĠThe", + "se" + ], + [ + "è¿ĩ", + "度" + ], + [ + "if", + "ied" + ], + [ + "è¯Ħ", + "ä»·" + ], + [ + "æķ°æį®", + "éĽĨ" + ], + [ + "é«ĺ", + "è´¨éĩı" + ], + [ + "èŀį", + "åIJĪ" + ], + [ + "åİŁ", + "åĪĻ" + ], + [ + "Ġappro", + "ximately" + ], + [ + "Ġrep", + "resent" + ], + [ + "ific", + "ial" + ], + [ + "as", + "on" + ], + [ + "æĸ¹", + "ç¨ĭ" + ], + [ + "Ġso", + "cial" + ], + [ + "Ġf", + "ig" + ], + [ + "SE", + "LECT" + ], + [ + "##", + "##" + ], + [ + "人", + "æ°ij" + ], + [ + "ãĢ", + "ij" + ], + [ + "he", + "st" + ], + [ + "ãĢ", + "IJ" + ], + [ + "Ġdig", + "ital" + ], + [ + "以ä¸ĭåĩłä¸ª", + "æĸ¹éĿ¢" + ], + [ + "Ġc", + "lo" + ], + [ + "å°Ĩ", + "åħ¶" + ], + [ + "çļĦ", + "åĨħ容" + ], + [ + "æı", + "¡" + ], + [ + "r", + "andom" + ], + [ + "å·", + "ŀ" + ], + [ + "è´Ń", + "çī©" + ], + [ + "h", + "ou" + ], + [ + "éĥ¨", + "éŨ" + ], + [ + "Ġsn", + "ipp" + ], + [ + "t", + "ract" + ], + [ + "çİ°", + "象" + ], + [ + "Ġneed", + "s" + ], + [ + "æķ°", + "åĪĹ" + ], + [ + "Ġg", + "ame" + ], + [ + "Ġt", + "ri" + ], + [ + "è´", + "´" + ], + [ + "çĭ¬", + "ç«ĭ" + ], + [ + "å¿ħè¦ģ", + "çļĦ" + ], + [ + "èĪ", + "ŀ" + ], + [ + "æ¸ħ", + "æ´ģ" + ], + [ + "象", + "å¾ģ" + ], + [ + "è½»", + "æĿ¾" + ], + [ + "é¸", + "¡" + ], + [ + "åĨ²", + "çªģ" + ], + [ + "Ġgener", + "al" + ], + [ + "Ġb", + "lo" + ], + [ + "çļĦæĦ", + "Ł" + ], + [ + "åĨħ", + "çļĦ" + ], + [ + "Ñ", + "ĥ" + ], + [ + "in", + "s" + ], + [ + "çļĦçŁ¥", + "è¯Ĩ" + ], + [ + "模", + "åĿĹ" + ], + [ + "éĻ¤", + "äºĨ" + ], + [ + "è¦", + "Ĩ" + ], + [ + "y", + "le" + ], + [ + "Ġint", + "elligence" + ], + [ + "åĨĻ", + "ä½ľ" + ], + [ + "Ġ", + "ed" + ], + [ + "Ġra", + "d" + ], + [ + "c", + "ing" + ], + [ + "P", + "lease" + ], + [ + "v", + "ed" + ], + [ + "èĥ½", + "帮åĬ©ä½ł" + ], + [ + "é¢Ŀ", + "å¤ĸ" + ], + [ + "Ġmark", + "et" + ], + [ + "Ġn", + "ight" + ], + [ + "in", + "put" + ], + [ + "æķ°æį®", + "åĪĨæŀIJ" + ], + [ + "æ´", + "²" + ], + [ + "m", + "e" + ], + [ + "w", + "h" + ], + [ + "Ġe", + "as" + ], + [ + "å¤ļ", + "å°ij" + ], + [ + "ãģ", + "¯" + ], + [ + "o", + "ck" + ], + [ + "C", + "reate" + ], + [ + "Ġloo", + "p" + ], + [ + "W", + "hen" + ], + [ + "ç", + "ħ" + ], + [ + "g", + "s" + ], + [ + "åı¯ä»¥", + "æł¹æį®" + ], + [ + "Ġrec", + "ogn" + ], + [ + "æĸ°", + "éĹ»" + ], + [ + "Ġ\\)", + "," + ], + [ + "ä¼ij", + "æģ¯" + ], + [ + "ĠM", + "a" + ], + [ + "ä¸Ĭä¸ĭ", + "æĸĩ" + ], + [ + "Ġm", + "em" + ], + [ + "åĬł", + "强" + ], + [ + "æ±", + "Ł" + ], + [ + "éĺ¿éĩĮå·´å·´", + "éĽĨåĽ¢" + ], + [ + "Ġsing", + "le" + ], + [ + "æ´", + "ĭ" + ], + [ + "P", + "U" + ], + [ + "im", + "ate" + ], + [ + "çļĦæ²", + "ŁéĢļ" + ], + [ + "åľ¨", + "æŁIJäºĽ" + ], + [ + "æµ", + "ª" + ], + [ + "ä¸", + "Ŀ" + ], + [ + "表", + "éĿ¢" + ], + [ + "èī¯", + "好" + ], + [ + "æŀ¶", + "æŀĦ" + ], + [ + "éĺ", + "»" + ], + [ + "ãģ", + "ª" + ], + [ + "åı¯æĮģç»Ń", + "åıijå±ķ" + ], + [ + "Ġen", + "h" + ], + [ + "sq", + "l" + ], + [ + "çŃī", + "äºİ" + ], + [ + "Ġmulti", + "ple" + ], + [ + "pl", + "ic" + ], + [ + "æĸĩ", + "åŃĹ" + ], + [ + "ä¹Ł", + "ä¼ļ" + ], + [ + "éĩ", + "İ" + ], + [ + "ount", + "ain" + ], + [ + "2", + "00" + ], + [ + "l", + "ing" + ], + [ + "åĪ¶", + "éĢł" + ], + [ + "âĢĿ", + "ï¼Ī" + ], + [ + "Ġdis", + "c" + ], + [ + "A", + "PI" + ], + [ + "Ġ1", + "7" + ], + [ + "èĻ", + "«" + ], + [ + "éļ", + "Ķ" + ], + [ + "pl", + "ify" + ], + [ + "Ġcl", + "ar" + ], + [ + "åĪ°", + "äºĨ" + ], + [ + "æ¹", + "ĸ" + ], + [ + "Ġd", + "er" + ], + [ + "Ġrequ", + "est" + ], + [ + "æŶ", + "åĢĻ" + ], + [ + "æľªæĿ¥", + "çļĦ" + ], + [ + "ç½", + "Ĺ" + ], + [ + "åħ¬", + "å¹³" + ], + [ + "Ġch", + "ang" + ], + [ + "æĮĩ", + "导" + ], + [ + "åı", + "«" + ], + [ + "Ġl", + "ess" + ], + [ + "æľī", + "çĿĢ" + ], + [ + "çļĦæł", + "¸" + ], + [ + "pp", + "er" + ], + [ + "on", + "y" + ], + [ + "éĢĤ", + "çĶ¨äºİ" + ], + [ + "Ġ", + "Ã" + ], + [ + "å®ļ", + "ä½į" + ], + [ + "æľī", + "趣" + ], + [ + "çļĦæŃ", + "£" + ], + [ + "Java", + "Script" + ], + [ + "åĬ¨", + "åĬĽ" + ], + [ + "un", + "d" + ], + [ + "ð", + "Ł" + ], + [ + "å¼Ģåıij", + "èĢħ" + ], + [ + "ç¾İ", + "åĽ½" + ], + [ + "èĪ", + "¬" + ], + [ + "..", + "." + ], + [ + "åĪĨ", + "为" + ], + [ + "çļĦæĮ", + "ijæĪĺ" + ], + [ + "Ġam", + "ount" + ], + [ + "Ġfil", + "m" + ], + [ + "å°½", + "éĩı" + ], + [ + "çļĦæ³", + "ķ" + ], + [ + "Ġprim", + "ary" + ], + [ + "Ġpop", + "ular" + ], + [ + "um", + "e" + ], + [ + "s", + "elf" + ], + [ + "8", + "0" + ], + [ + "æ¯Ķ", + "èµĽ" + ], + [ + "åIJį", + "åŃĹ" + ], + [ + "å¡", + "«" + ], + [ + "èĩªåĬ¨", + "åĮĸ" + ], + [ + "Ġ", + "es" + ], + [ + "磩", + "éĺµ" + ], + [ + "re", + "hen" + ], + [ + "ł", + "éģĵ" + ], + [ + "为", + "ä»Ģä¹Ī" + ], + [ + "çı", + "Ń" + ], + [ + "Ġcap", + "abilities" + ], + [ + "Ġsnipp", + "et" + ], + [ + "è·", + "Ł" + ], + [ + "<", + "/" + ], + [ + "Ð", + "º" + ], + [ + "è¸", + "ª" + ], + [ + "en", + "ces" + ], + [ + "Ġwid", + "ely" + ], + [ + "Ġd", + "el" + ], + [ + "Ġcon", + "c" + ], + [ + "n", + "ov" + ], + [ + "åİ", + "ļ" + ], + [ + "è¿Ļ个", + "éĹ®é¢ĺ" + ], + [ + "éĢļ常", + "ä¼ļ" + ], + [ + "ä¿Ŀ", + "åŃĺ" + ], + [ + "at", + "s" + ], + [ + "è¿ľ", + "ç¨ĭ" + ], + [ + "t", + "ed" + ], + [ + "è´¦", + "æĪ·" + ], + [ + "åħ¬", + "ä¼Ĺ" + ], + [ + "O", + "S" + ], + [ + "id", + "er" + ], + [ + "Ġg", + "roup" + ], + [ + "e", + "xt" + ], + [ + "Ġn", + "ature" + ], + [ + "e", + "k" + ], + [ + "表达", + "å¼ı" + ], + [ + "Ġunderstand", + "ing" + ], + [ + "ile", + "d" + ], + [ + "Ġeff", + "ic" + ], + [ + "çĶ³", + "请" + ], + [ + "è¯Ĭ", + "æĸŃ" + ], + [ + "模", + "æĭŁ" + ], + [ + "ip", + "ly" + ], + [ + "çĶ¨æĪ·", + "ä½ĵéªĮ" + ], + [ + "ol", + "s" + ], + [ + "éĽĨ", + "æĪIJ" + ], + [ + "éģĵ", + "å¾·" + ], + [ + "é¾", + "Ļ" + ], + [ + "com", + "e" + ], + [ + "4", + "0" + ], + [ + "Ġhist", + "ory" + ], + [ + "æİ", + "Ī" + ], + [ + "污", + "æŁĵ" + ], + [ + "é", + "Ł" + ], + [ + "æıIJé«ĺ", + "äºĨ" + ], + [ + "Ġrel", + "ated" + ], + [ + "å¥", + "Ĺ" + ], + [ + "åı¯èĥ½", + "æĺ¯" + ], + [ + "æ¯ı", + "天" + ], + [ + "an", + "cial" + ], + [ + "åģ", + "¶" + ], + [ + "Ġsc", + "ient" + ], + [ + "éĿŀ常", + "éĩįè¦ģ" + ], + [ + "ch", + "ie" + ], + [ + "åı¯ä»¥", + "çĶ¨äºİ" + ], + [ + "Ġorig", + "inal" + ], + [ + "æľĢ", + "å°ı" + ], + [ + "èĤ", + "Į" + ], + [ + "åij¼", + "åIJ¸" + ], + [ + "Ġno", + "vel" + ], + [ + "Ġupd", + "ates" + ], + [ + "çļĦæĥħ", + "æĦŁ" + ], + [ + "ä½ĵ", + "ç³»" + ], + [ + "èĭ±", + "è¯Ń" + ], + [ + "Al", + "ibaba" + ], + [ + "éĢĴ", + "å½Ĵ" + ], + [ + "çļĦ", + "ä¸Ģ个" + ], + [ + "ist", + "s" + ], + [ + "Ġs", + "ize" + ], + [ + "åľ¨", + "å®ŀéĻħ" + ], + [ + "d", + "ata" + ], + [ + "æĪIJ", + "为äºĨ" + ], + [ + "Ġf", + "ur" + ], + [ + "æ»", + "¤" + ], + [ + "Ġrequ", + "ire" + ], + [ + "æķı", + "æĦŁ" + ], + [ + "æİ", + "Į" + ], + [ + "ãĤ", + "ĭ" + ], + [ + "Ġt", + "est" + ], + [ + "åıij", + "éĢģ" + ], + [ + "Ġap", + "pe" + ], + [ + "Ġk", + "il" + ], + [ + "ç¨ĭ", + "度" + ], + [ + "Ġfe", + "w" + ], + [ + "æ°Ķ", + "åĢĻ" + ], + [ + "ç¡®", + "认" + ], + [ + "æĪIJ", + "æŀľ" + ], + [ + "Ġne", + "xt" + ], + [ + "ä¸ĵ", + "注" + ], + [ + "Ġvari", + "able" + ], + [ + "Ġbe", + "gin" + ], + [ + "box", + "ed" + ], + [ + "Ġs", + "il" + ], + [ + "åѦ", + "æł¡" + ], + [ + "Ġfeature", + "s" + ], + [ + "æĿ¥", + "èĩª" + ], + [ + "çļĦæĻ", + "º" + ], + [ + "å®ī", + "æİĴ" + ], + [ + "å®", + "ģ" + ], + [ + "èµĦ", + "éĩij" + ], + [ + "p", + "ar" + ], + [ + "èī²", + "彩" + ], + [ + "让", + "æĪij们" + ], + [ + "am", + "es" + ], + [ + "èIJ¥", + "åħ»" + ], + [ + "Ġf", + "ore" + ], + [ + "Ġe", + "conom" + ], + [ + "Ġg", + "ent" + ], + [ + "é©", + "±" + ], + [ + "å±ŀ", + "æĢ§" + ], + [ + "ç®Ģ", + "æ´ģ" + ], + [ + "Ġcomp", + "rehen" + ], + [ + "ç«ĭ", + "åį³" + ], + [ + "Ġ\\)", + "." + ], + [ + "Ġequ", + "al" + ], + [ + "çļĦ", + "代çłģ" + ], + [ + "Ġbeaut", + "y" + ], + [ + "大", + "è§Ħ模" + ], + [ + "åIJij", + "éĩı" + ], + [ + "ı", + "è§Ī" + ], + [ + "æľ", + "«" + ], + [ + "re", + "qu" + ], + [ + "åĪ°", + "çļĦ" + ], + [ + "åĬł", + "éĢŁ" + ], + [ + "_", + "{" + ], + [ + "Ù", + "Ĭ" + ], + [ + "Ġd", + "ays" + ], + [ + "è´", + "¥" + ], + [ + "Ġstand", + "ard" + ], + [ + "ĠA", + "n" + ], + [ + "è´¹", + "çĶ¨" + ], + [ + "èijĹ", + "åIJį" + ], + [ + "æ±", + "ī" + ], + [ + "m", + "in" + ], + [ + "çļĦç", + "²" + ], + [ + "Ġlangu", + "ages" + ], + [ + "irt", + "ual" + ], + [ + "Ġp", + "res" + ], + [ + "Ġv", + "ast" + ], + [ + "ĥ", + "åľ" + ], + [ + "pl", + "es" + ], + [ + "çļĦæ´", + "»" + ], + [ + "åĵ", + "²" + ], + [ + "çļĦæĶ", + "¯" + ], + [ + "硬", + "件" + ], + [ + "é¢Ĩ", + "导" + ], + [ + "èĤ¡", + "票" + ], + [ + "åŀ", + "ĥåľ" + ], + [ + "åŀĥåľ", + "¾" + ], + [ + "åı¯ä»¥", + "æıIJé«ĺ" + ], + [ + "éĹ®é¢ĺ", + "æĪĸ" + ], + [ + "Ġre", + "se" + ], + [ + "æ²", + "ī" + ], + [ + "è·", + "³" + ], + [ + "Ġ", + "ê" + ], + [ + "æİ¥", + "æĶ¶" + ], + [ + "ut", + "ure" + ], + [ + "on", + "d" + ], + [ + "Ġha", + "d" + ], + [ + "满", + "æĦı" + ], + [ + "Ġav", + "ail" + ], + [ + "A", + "B" + ], + [ + "Ġpro", + "per" + ], + [ + "I", + "P" + ], + [ + "æľī", + "çļĦ" + ], + [ + "u", + "th" + ], + [ + "æĶ¶", + "åħ¥" + ], + [ + "Ġt", + "imes" + ], + [ + "è§Ħ", + "å®ļ" + ], + [ + "ç´§", + "æĢ¥" + ], + [ + "Ġloo", + "k" + ], + [ + "绣", + "计" + ], + [ + "Ġm", + "et" + ], + [ + "å²", + "ģ" + ], + [ + "Ġcon", + "d" + ], + [ + "ãĥ", + "¼" + ], + [ + "Ġassist", + "ant" + ], + [ + "dition", + "ally" + ], + [ + "Ġfiel", + "d" + ], + [ + "交", + "äºĴ" + ], + [ + "åĪĿ", + "å§ĭ" + ], + [ + "èĽĭ", + "çĻ½" + ], + [ + "å", + "³" + ], + [ + "Ġs", + "ay" + ], + [ + "çĸ", + "ij" + ], + [ + "æĶ¾", + "æĿ¾" + ], + [ + "æ¶", + "¦" + ], + [ + "Ġsoft", + "ware" + ], + [ + "æľīæķĪ", + "çļĦ" + ], + [ + "æ", + "©" + ], + [ + "çĶŁ", + "éķ¿" + ], + [ + "é¢ij", + "çİĩ" + ], + [ + "T", + "rans" + ], + [ + "注", + "åĨĮ" + ], + [ + "èĭ±", + "æĸĩ" + ], + [ + "æİ¢", + "讨" + ], + [ + "顺", + "åºı" + ], + [ + "Ġl", + "and" + ], + [ + "Ġnot", + "e" + ], + [ + "p", + "os" + ], + [ + "\"", + "?" + ], + [ + "æŁ¥", + "æī¾" + ], + [ + "人", + "çī©" + ], + [ + "æŶ", + "æľŁ" + ], + [ + "in", + "ing" + ], + [ + "iz", + "es" + ], + [ + "åĢ", + "Ĵ" + ], + [ + "Ġgener", + "ating" + ], + [ + "Ġsome", + "one" + ], + [ + "Ġfur", + "ther" + ], + [ + "ãģ", + "Į" + ], + [ + "主", + "ä¹ī" + ], + [ + "åIJ«", + "ä¹ī" + ], + [ + "个人", + "çļĦ" + ], + [ + "åľ¨", + "ä¸Ģ个" + ], + [ + "ro", + "id" + ], + [ + "Ġn", + "ow" + ], + [ + "åį", + "·" + ], + [ + "车", + "è¾Ĩ" + ], + [ + "t", + "a" + ], + [ + "åĴĮ", + "管çIJĨ" + ], + [ + "åĢ", + "ĭ" + ], + [ + "çĹ", + "Ľ" + ], + [ + "ä»į", + "çĦ¶" + ], + [ + "è¦Ĩ", + "çĽĸ" + ], + [ + "Ġn", + "p" + ], + [ + "Ġo", + "ld" + ], + [ + "ãģ", + "¦" + ], + [ + "å½¢", + "象" + ], + [ + "èĢĮ", + "æĺ¯" + ], + [ + "Ġform", + "at" + ], + [ + "Ġfin", + "ancial" + ], + [ + "åı¥", + "è¯Ŀ" + ], + [ + "çĻ", + "¾" + ], + [ + "H", + "e" + ], + [ + "l", + "u" + ], + [ + "åĨ", + "Ĵ" + ], + [ + "æ·±", + "åĪ»" + ], + [ + "并", + "ä¸Ķ" + ], + [ + "av", + "as" + ], + [ + "åIJ¯", + "åĬ¨" + ], + [ + "å¹", + "»" + ], + [ + "Ġdat", + "ab" + ], + [ + "æľĢ", + "ä½³" + ], + [ + "Ġnam", + "es" + ], + [ + "ç¬", + "ij" + ], + [ + "âĢĿ", + "ãĢģ" + ], + [ + "Ġcomput", + "er" + ], + [ + "Ġab", + "ility" + ], + [ + "Ġparticular", + "ly" + ], + [ + "åıĸ", + "å¾Ĺ" + ], + [ + "çļĦç¬", + "¬" + ], + [ + "avas", + "cript" + ], + [ + "o", + "ice" + ], + [ + "Ġdoc", + "ument" + ], + [ + "ĠL", + "et" + ], + [ + "Ġapplic", + "ations" + ], + [ + "缮", + "åīį" + ], + [ + "Ġcle", + "ar" + ], + [ + "isp", + "ers" + ], + [ + "Ã", + "³" + ], + [ + "ç«", + "¥" + ], + [ + "è¿Ļæł·", + "çļĦ" + ], + [ + "帮", + "æĪij" + ], + [ + "çľĭ", + "èµ·æĿ¥" + ], + [ + "å¹", + "¸" + ], + [ + "å°ı", + "说" + ], + [ + "pl", + "ay" + ], + [ + "d", + "ict" + ], + [ + "is", + "hed" + ], + [ + "Ġp", + "ract" + ], + [ + "çķĮ", + "éĿ¢" + ], + [ + "Ġa", + "chie" + ], + [ + "Ġ", + "å°Ĩ" + ], + [ + "çłĶ", + "åıij" + ], + [ + "æ¼", + "«" + ], + [ + "åı¤", + "代" + ], + [ + "çļĦç¼ĸ", + "ç¨ĭ" + ], + [ + "y", + "stem" + ], + [ + "Ġwith", + "in" + ], + [ + "ç£", + "ģ" + ], + [ + "Ġm", + "ar" + ], + [ + "r", + "ight" + ], + [ + "ä¿®", + "å¤į" + ], + [ + "led", + "ge" + ], + [ + "åį", + "ł" + ], + [ + "E", + "O" + ], + [ + "æĪij们", + "çļĦ" + ], + [ + "çī©", + "æµģ" + ], + [ + "ãģ¾", + "ãģĻ" + ], + [ + "Ġcount", + "ry" + ], + [ + "Ġse", + "ver" + ], + [ + "å±ŀ", + "äºİ" + ], + [ + "N", + "A" + ], + [ + "ĠN", + "ew" + ], + [ + "åıĬ", + "åħ¶" + ], + [ + "Ġpoint", + "s" + ], + [ + "ç´", + "¯" + ], + [ + "å°½", + "åĬĽ" + ], + [ + "æĢİ", + "ä¹Ī" + ], + [ + "d", + "ing" + ], + [ + "çļĦæł¸", + "å¿ĥ" + ], + [ + "大", + "éĩıçļĦ" + ], + [ + "Ġev", + "ent" + ], + [ + "an", + "e" + ], + [ + "Ġth", + "ose" + ], + [ + "éĢīæĭ©", + "åIJĪéĢĤçļĦ" + ], + [ + "Ġinteg", + "ers" + ], + [ + "Ġ", + "æĺ¯" + ], + [ + "ab", + "les" + ], + [ + "çŃĶ", + "æ¡Ī" + ], + [ + "l", + "and" + ], + [ + "m", + "l" + ], + [ + "ãģ", + "¨" + ], + [ + "Ġder", + "iv" + ], + [ + "âĢĿãĢģ", + "âĢľ" + ], + [ + "he", + "l" + ], + [ + "Ġ", + "â" + ], + [ + "åħ", + "°" + ], + [ + "æ··", + "åIJĪ" + ], + [ + "æĢĿ", + "æĥ³" + ], + [ + "å¸ĥ", + "å±Ģ" + ], + [ + "çī", + "Ļ" + ], + [ + "æģ", + "¶" + ], + [ + "å¼ķ", + "èµ·" + ], + [ + "Ġ", + "å¦Ĥæŀľ" + ], + [ + "å°", + "¾" + ], + [ + "è¿Ļ个", + "åĩ½æķ°" + ], + [ + "è§Ĥ", + "ä¼Ĺ" + ], + [ + "åŁº", + "åĽł" + ], + [ + "ER", + "E" + ], + [ + "L", + "et" + ], + [ + "çļĦ", + "æķ°åŃĹ" + ], + [ + "è¿", + "Ń" + ], + [ + "çª", + "Ĺ" + ], + [ + "lo", + "ad" + ], + [ + "Ġdeterm", + "ine" + ], + [ + "om", + "s" + ], + [ + "åѦ", + "ä¼ļ" + ], + [ + "èĦ", + "Ĥ" + ], + [ + "if", + "t" + ], + [ + "éĿ¢", + "积" + ], + [ + "家", + "人" + ], + [ + "äº", + "Ī" + ], + [ + "Ġp", + "ot" + ], + [ + "ä½ł", + "对" + ], + [ + "iv", + "ing" + ], + [ + "ç±»åŀĭ", + "çļĦ" + ], + [ + "严", + "éĩį" + ], + [ + "Ġsever", + "al" + ], + [ + "çĬ", + "¯" + ], + [ + "ç±»", + "ä¼¼" + ], + [ + "us", + "h" + ], + [ + "第", + "ä¸Ģ" + ], + [ + "è¾ħ", + "åĬ©" + ], + [ + "ä¾ĽåºĶ", + "éĵ¾" + ], + [ + "åĴĮ", + "对" + ], + [ + "æ»", + "ij" + ], + [ + "Ġequ", + "ations" + ], + [ + "ä¸ĵä¸ļ", + "çļĦ" + ], + [ + "çļĦ", + "åıĺåĮĸ" + ], + [ + "Ġsol", + "utions" + ], + [ + "Ġb", + "ase" + ], + [ + "Ġcommun", + "ic" + ], + [ + "æį", + "ķ" + ], + [ + "åѦ", + "èĢħ" + ], + [ + "Ġl", + "ove" + ], + [ + "çļĦæł", + "ĩ" + ], + [ + "Ġhig", + "hest" + ], + [ + "j", + "avascript" + ], + [ + "Ġrec", + "ord" + ], + [ + "Ġm", + "ath" + ], + [ + "an", + "ced" + ], + [ + "Ġpur", + "pose" + ], + [ + "çļĦ", + "æĢ»" + ], + [ + "é£Ł", + "åĵģ" + ], + [ + "AI", + "æĬĢæľ¯" + ], + [ + "H", + "ERE" + ], + [ + "he", + "mat" + ], + [ + "åĢ", + "º" + ], + [ + "Ã", + "Ń" + ], + [ + "Ġac", + "ross" + ], + [ + "ä¸ī", + "è§Ĵ" + ], + [ + "æ¶īåıĬ", + "åĪ°" + ], + [ + "Ġwor", + "king" + ], + [ + "is", + "m" + ], + [ + "stit", + "ute" + ], + [ + "éĴĪ", + "对" + ], + [ + "Ġin", + "it" + ], + [ + "a", + "iled" + ], + [ + "n", + "ot" + ], + [ + "çĽ", + "IJ" + ], + [ + "æĺĵ", + "äºİ" + ], + [ + "Ġlead", + "ing" + ], + [ + "Ġv", + "ery" + ], + [ + "Ġon", + "line" + ], + [ + "o", + "ot" + ], + [ + "ç´¢", + "å¼ķ" + ], + [ + "è´§", + "å¸ģ" + ], + [ + "çĥ", + "¤" + ], + [ + "é¸", + "Ł" + ], + [ + "ĠP", + "r" + ], + [ + "ut", + "es" + ], + [ + "çļĦ", + "åħĥç´ł" + ], + [ + "en", + "e" + ], + [ + "è¶ħ", + "è¿ĩ" + ], + [ + "严", + "æł¼" + ], + [ + "æĿĥ", + "éĻIJ" + ], + [ + "fere", + "nce" + ], + [ + "Ġsym", + "b" + ], + [ + "st", + "ems" + ], + [ + "åľ¨", + "äºİ" + ], + [ + "'", + "]" + ], + [ + "Ġem", + "b" + ], + [ + "a", + "ces" + ], + [ + "r", + "and" + ], + [ + "Ġcol", + "or" + ], + [ + "çļĦçĥ", + "Ń" + ], + [ + "N", + "ow" + ], + [ + "d", + "iv" + ], + [ + "é¢", + "Ĺ" + ], + [ + "çļĦåľ°", + "æĸ¹" + ], + [ + "ä¿Ŀ", + "éĻ©" + ], + [ + "çī©", + "åĵģ" + ], + [ + "æłĩ", + "çŃ¾" + ], + [ + "Ġrespon", + "se" + ], + [ + "Ġm", + "eters" + ], + [ + "Python", + "ä¸Ń" + ], + [ + "è¯", + "º" + ], + [ + "å¤ĸ", + "éĥ¨" + ], + [ + "Ġin", + "nov" + ], + [ + "è¿Ľ", + "åĪ¶" + ], + [ + "èĭ", + "ı" + ], + [ + "oo", + "ks" + ], + [ + "ç¼", + "ĺ" + ], + [ + "çļ®", + "èĤ¤" + ], + [ + "让", + "人" + ], + [ + "Ġsol", + "ution" + ], + [ + "et", + "er" + ], + [ + "çĻ»", + "å½ķ" + ], + [ + "Ġ", + "Wh" + ], + [ + "客", + "æľį" + ], + [ + "åİŁ", + "å§ĭ" + ], + [ + "åĴĮ", + "åºĶçĶ¨" + ], + [ + "èĢIJ", + "å¿ĥ" + ], + [ + "b", + "y" + ], + [ + "çļĦ", + "缮æłĩ" + ], + [ + "ä¿Ŀ", + "è¯ģ" + ], + [ + "ä½ł", + "æĥ³è¦ģ" + ], + [ + "容", + "åĻ¨" + ], + [ + "p", + "re" + ], + [ + "çĹħ", + "æ¯Ĵ" + ], + [ + "Ġimp", + "act" + ], + [ + "æıIJ", + "åıĸ" + ], + [ + "æĥ", + "ł" + ], + [ + "--", + "-" + ], + [ + "ers", + "hip" + ], + [ + "åĴĮ", + "社ä¼ļ" + ], + [ + "åħħ", + "åĪĨ" + ], + [ + "æķ°", + "åĢ¼" + ], + [ + "Ġcond", + "itions" + ], + [ + "id", + "d" + ], + [ + "åºĶçĶ¨", + "äºİ" + ], + [ + "è£ħ", + "饰" + ], + [ + "é¢ĨåŁŁ", + "çļĦ" + ], + [ + "Ġrespon", + "s" + ], + [ + "çļĦæ³ķ", + "å¾ĭ" + ], + [ + "æĬ", + "Ĭ" + ], + [ + "åĵ²", + "åѦ" + ], + [ + "强", + "大çļĦ" + ], + [ + "Ġse", + "lect" + ], + [ + "c", + "os" + ], + [ + "çī¹", + "èī²" + ], + [ + "ĠThe", + "y" + ], + [ + "线", + "æĢ§" + ], + [ + "Ġs", + "ide" + ], + [ + "æµ", + "ĵ" + ], + [ + "æķ´", + "ä½ĵ" + ], + [ + "W", + "eb" + ], + [ + "Ġsp", + "ace" + ], + [ + "Ġart", + "ic" + ], + [ + "éĵ¾", + "æİ¥" + ], + [ + "Ġjo", + "ke" + ], + [ + "w", + "ay" + ], + [ + "çļĦ", + "æľįåĬ¡" + ], + [ + "Ġmodel", + "s" + ], + [ + "å±ħ", + "æ°ij" + ], + [ + "ï¼Į", + "âĢľ" + ], + [ + "æĹħ", + "游" + ], + [ + "åĴĮ", + "åĪĨæŀIJ" + ], + [ + "æµ", + "ıè§Ī" + ], + [ + "l", + "ess" + ], + [ + "Ġp", + "ain" + ], + [ + "çļĦæ¦Ĥ", + "念" + ], + [ + "çī¹åĪ«", + "æĺ¯" + ], + [ + "Ġperform", + "ance" + ], + [ + "id", + "i" + ], + [ + "=", + "\"" + ], + [ + "r", + "on" + ], + [ + "Ġm", + "ind" + ], + [ + "Ġvari", + "ety" + ], + [ + "Ù", + "ħ" + ], + [ + "ĠB", + "e" + ], + [ + "Ġst", + "ore" + ], + [ + "æŃ£", + "常" + ], + [ + "Ġpro", + "du" + ], + [ + "广", + "åijĬ" + ], + [ + "çĶ", + "²" + ], + [ + "æľĢ", + "æĸ°çļĦ" + ], + [ + "顾", + "客" + ], + [ + "æĺ¯", + "ä¸ŃåĽ½" + ], + [ + "Ġrel", + "ations" + ], + [ + "Ġm", + "ountain" + ], + [ + "Ġfact", + "ors" + ], + [ + "è´", + "·" + ], + [ + "æ·±", + "è¿ľ" + ], + [ + "å°", + "¼" + ], + [ + "Ġgent", + "le" + ], + [ + "Ġmil", + "es" + ], + [ + "Ġbe", + "fore" + ], + [ + "çĥ", + "Ī" + ], + [ + "ar", + "ray" + ], + [ + "æķ", + "ij" + ], + [ + "Ġv", + "ol" + ], + [ + "Ġens", + "ure" + ], + [ + "æ¬", + "§" + ], + [ + "æľīæīĢ", + "ä¸įåIJĮ" + ], + [ + "Ġknow", + "ledge" + ], + [ + "ç¢", + "į" + ], + [ + "åıįæĺł", + "äºĨ" + ], + [ + "è°", + "Ī" + ], + [ + "Ġo", + "pen" + ], + [ + "åĨľ", + "ä¸ļ" + ], + [ + "è¿ŀ", + "ç»Ń" + ], + [ + "éĶ", + "ģ" + ], + [ + "ĠAd", + "ditionally" + ], + [ + "ç»ı", + "åħ¸" + ], + [ + "Ġb", + "ro" + ], + [ + "æĮ", + "¥" + ], + [ + "ä¸ĭ", + "æĿ¥" + ], + [ + "ä¹Ł", + "åı¯èĥ½" + ], + [ + "ä¸ĭ", + "éĻį" + ], + [ + "éĽĨ", + "åIJĪ" + ], + [ + "ag", + "raph" + ], + [ + "Ġexp", + "lo" + ], + [ + "mer", + "ic" + ], + [ + "Ġl", + "en" + ], + [ + "éĽ", + "ª" + ], + [ + "Ñ", + "ĭ" + ], + [ + "Ġst", + "ructure" + ], + [ + "Ġman", + "age" + ], + [ + "æŀĦ", + "æĪIJ" + ], + [ + "ä»·åĢ¼", + "è§Ĥ" + ], + [ + "T", + "P" + ], + [ + "ä¾", + "§" + ], + [ + "设计", + "ä¸Ģ个" + ], + [ + "åĬ©", + "æīĭ" + ], + [ + "Ġre", + "st" + ], + [ + "ä½ł", + "åı¯èĥ½éľĢè¦ģ" + ], + [ + "Ġde", + "v" + ], + [ + "åĮ", + "Ģ" + ], + [ + "P", + "ro" + ], + [ + "çŃī", + "æĸ¹å¼ı" + ], + [ + "çĩ", + "ĥ" + ], + [ + "Ġg", + "old" + ], + [ + "åıĤ", + "èĢĥ" + ], + [ + "o", + "ber" + ], + [ + "Ġ\\", + "," + ], + [ + "Ġdevelop", + "ed" + ], + [ + "è¯Ĺ", + "æŃĮ" + ], + [ + "c", + "ent" + ], + [ + "è¿", + "¹" + ], + [ + "大", + "åѦ" + ], + [ + "ĠD", + "e" + ], + [ + "cc", + "ess" + ], + [ + "Ġ", + "&" + ], + [ + "Ġm", + "us" + ], + [ + "æ°", + "¸" + ], + [ + "ct", + "or" + ], + [ + "æ¸", + "łéģĵ" + ], + [ + "éļIJç§ģ", + "ä¿ĿæĬ¤" + ], + [ + "acc", + "i" + ], + [ + "æĢ§", + "è´¨" + ], + [ + "æŃ£ç¡®", + "çļĦ" + ], + [ + "åį", + "±" + ], + [ + "åIJĪ", + "åIJĮ" + ], + [ + "æłĩ", + "é¢ĺ" + ], + [ + "b", + "e" + ], + [ + "è¿", + "ħ" + ], + [ + "åį", + "Ī" + ], + [ + "iv", + "ity" + ], + [ + "ib", + "on" + ], + [ + "æ³", + "Ħ" + ], + [ + "转", + "åĮĸ" + ], + [ + "éĺ¿éĩĮå·´å·´", + "äºij" + ], + [ + "çĶµ", + "æ±ł" + ], + [ + "get", + "her" + ], + [ + "çļĦ", + "建议" + ], + [ + "Ġto", + "gether" + ], + [ + "a", + "de" + ], + [ + "Ġl", + "ibr" + ], + [ + "çļĦæı", + "IJ" + ], + [ + "Ġhour", + "s" + ], + [ + "ç»", + "©" + ], + [ + "大", + "åľ°" + ], + [ + "Ġle", + "vel" + ], + [ + "ibon", + "acci" + ], + [ + "éĺ³", + "åħī" + ], + [ + "Ġavail", + "able" + ], + [ + "Ġus", + "es" + ], + [ + "Ġanal", + "y" + ], + [ + "çļĦæĬ", + "ķ" + ], + [ + "åij¨", + "æľŁ" + ], + [ + "èĦļ", + "æľ¬" + ], + [ + "Ġident", + "ify" + ], + [ + "ä¹", + "ħ" + ], + [ + "Ñ", + "ı" + ], + [ + "æľ", + "Ŀ" + ], + [ + "ĠC", + "on" + ], + [ + "is", + "ion" + ], + [ + "å®ļ", + "åĪ¶" + ], + [ + "éľĢè¦ģ", + "注æĦı" + ], + [ + "åħĥç´ł", + "çļĦ" + ], + [ + "Ġart", + "ificial" + ], + [ + "Ġma", + "x" + ], + [ + "ern", + "et" + ], + [ + "ãģ", + "Ł" + ], + [ + "çļĦç", + "©" + ], + [ + "å¹³", + "æĸ¹" + ], + [ + "æŁ", + "Ķ" + ], + [ + "I", + "N" + ], + [ + "羣", + "å®ŀ" + ], + [ + "æ²", + "¿" + ], + [ + "æĪIJ", + "å°±" + ], + [ + "ä¼ĺ", + "åħĪ" + ], + [ + "Ġres", + "ources" + ], + [ + "Ġb", + "as" + ], + [ + "å°", + "º" + ], + [ + "æIJ", + "Ń" + ], + [ + "åĮºåĿĹéĵ¾", + "æĬĢæľ¯" + ], + [ + "æī", + "®" + ], + [ + "Ġmon", + "th" + ], + [ + "å¥", + "ı" + ], + [ + "第", + "ä¸ī" + ], + [ + "Ġref", + "lect" + ], + [ + "çļĦä¸Ģ", + "éĥ¨åĪĨ" + ], + [ + "çļĦ", + "åģ¥åº·" + ], + [ + "è®°", + "ä½ı" + ], + [ + "梦", + "æĥ³" + ], + [ + "é¢Ħ", + "éĺ²" + ], + [ + "æķ", + "¸" + ], + [ + "Ġch", + "all" + ], + [ + "Ġcont", + "rib" + ], + [ + "S", + "ON" + ], + [ + "大", + "åŀĭ" + ], + [ + "缸", + "ä¼¼" + ], + [ + "Ġs", + "ense" + ], + [ + "an", + "ces" + ], + [ + "æ¼", + "ı" + ], + [ + "ç¥ŀç»ı", + "ç½ij绾" + ], + [ + "Ġb", + "ir" + ], + [ + "æľ¬", + "身" + ], + [ + "æ¶", + "µ" + ], + [ + "Ġgre", + "at" + ], + [ + "}", + "\\)" + ], + [ + "Ġgrow", + "th" + ], + [ + "Ġbe", + "h" + ], + [ + "Ġpar", + "agraph" + ], + [ + "è¿Ń", + "代" + ], + [ + "åĬĽ", + "çļĦ" + ], + [ + "å¹´", + "è½»" + ], + [ + "Ġc", + "all" + ], + [ + "Ġstep", + "s" + ], + [ + "3", + "3" + ], + [ + "ar", + "get" + ], + [ + "éĵ", + "º" + ], + [ + "åı¦", + "ä¸Ģ个" + ], + [ + "Ġexpl", + "ain" + ], + [ + ")", + "`" + ], + [ + "Ġan", + "n" + ], + [ + "ot", + "al" + ], + [ + "æīĢ", + "éľĢ" + ], + [ + "åį", + "ĸ" + ], + [ + "æ²", + "Ļ" + ], + [ + "Ġcir", + "cle" + ], + [ + "Ġclar", + "ify" + ], + [ + "m", + "an" + ], + [ + "i", + "ver" + ], + [ + "r", + "ame" + ], + [ + "èĢĮ", + "ä¸Ķ" + ], + [ + "èµĦ", + "产" + ], + [ + "ä¹", + "¡" + ], + [ + "Ġv", + "er" + ], + [ + "Ġ6", + "0" + ], + [ + "A", + "nd" + ], + [ + "ĠS", + "o" + ], + [ + "Ġ4", + "0" + ], + [ + "声", + "éŁ³" + ], + [ + "åĴĮ", + "æĬĢæľ¯" + ], + [ + "al", + "cul" + ], + [ + "æĭ", + "į" + ], + [ + "æ¯ı", + "次" + ], + [ + "æľī", + "åħ³" + ], + [ + "iv", + "al" + ], + [ + "çļĦç", + "Ł" + ], + [ + "èĩ³", + "å°ij" + ], + [ + "Ġre", + "du" + ], + [ + "æŶ", + "çļĦ" + ], + [ + "å¹´", + "çļĦ" + ], + [ + "Ġind", + "ust" + ], + [ + "Ġexper", + "ience" + ], + [ + "åĪ", + "¸" + ], + [ + "两", + "ç§į" + ], + [ + "æł¸", + "å¿ĥ" + ], + [ + "gor", + "ithm" + ], + [ + "æİĮ", + "æı¡" + ], + [ + "çģ", + "¾" + ], + [ + "大", + "å¤ļ" + ], + [ + "å·¥", + "ä¸ļ" + ], + [ + "ac", + "y" + ], + [ + "ãĢĭ", + "ï¼Ī" + ], + [ + "inu", + "x" + ], + [ + "9", + "0" + ], + [ + "è¿Ľ", + "å±ķ" + ], + [ + "çļĦæ¸", + "©" + ], + [ + "åħħ满", + "äºĨ" + ], + [ + "éŃ", + "Ķ" + ], + [ + "Ð", + "¼" + ], + [ + "Ġsu", + "bs" + ], + [ + "Python", + "çļĦ" + ], + [ + "åĪĨ", + "åŃIJ" + ], + [ + "S", + "T" + ], + [ + "æľī", + "äºĽ" + ], + [ + "温", + "æļĸ" + ], + [ + "Ġs", + "it" + ], + [ + "Ġdet", + "ailed" + ], + [ + "å¿", + "ĺ" + ], + [ + "Ġhe", + "ad" + ], + [ + "éĻ", + "Ħ" + ], + [ + "å®ĥ", + "çļĦ" + ], + [ + "æĸ¹", + "便" + ], + [ + "ĠJava", + "Script" + ], + [ + "Ġw", + "on" + ], + [ + "让", + "ä½ł" + ], + [ + "Ġtransl", + "ate" + ], + [ + "æĺ¯åIJ¦", + "æľī" + ], + [ + "Ġv", + "irtual" + ], + [ + "C", + "om" + ], + [ + "以", + "åħ¶" + ], + [ + "as", + "ure" + ], + [ + "ut", + "put" + ], + [ + "Ã", + "ł" + ], + [ + "Ġfollow", + "s" + ], + [ + "Ľ", + "åĽ´" + ], + [ + "ä¿ĥè¿Ľ", + "äºĨ" + ], + [ + "ä½ĵçİ°", + "äºĨ" + ], + [ + "æŶ", + "代" + ], + [ + "Ġmat", + "hemat" + ], + [ + "Ġf", + "uture" + ], + [ + "æĿİ", + "æĺİ" + ], + [ + "L", + "P" + ], + [ + "è¿", + "Ŀ" + ], + [ + "æĸ°", + "é²ľ" + ], + [ + "注", + "éĩį" + ], + [ + "Ġpat", + "tern" + ], + [ + "éļ¾", + "以" + ], + [ + "éĢīæĭ©", + "ä¸Ģ个" + ], + [ + "ç¼ĸ", + "çłģ" + ], + [ + "f", + "ace" + ], + [ + "o", + "id" + ], + [ + "ur", + "y" + ], + [ + "éĩį", + "è§Ĩ" + ], + [ + "çļĦ模", + "åŀĭ" + ], + [ + "è¿IJ", + "ç®Ĺ" + ], + [ + "c", + "er" + ], + [ + "i", + "od" + ], + [ + "Ġc", + "r" + ], + [ + "(", + "-" + ], + [ + "Ġnew", + "s" + ], + [ + "ĠFr", + "ance" + ], + [ + "åº", + "Ĭ" + ], + [ + "ä¸ĵ", + "家" + ], + [ + "Ġrese", + "arch" + ], + [ + "Ġgre", + "ater" + ], + [ + "a", + "pe" + ], + [ + "åıª", + "æľī" + ], + [ + "åĵĪ", + "å¸Į" + ], + [ + "èģ", + "Ĭ" + ], + [ + "é«ĺ", + "级" + ], + [ + "èĶ", + "¬" + ], + [ + "å¢", + "Ļ" + ], + [ + "Ġspec", + "ial" + ], + [ + "Ġdirect", + "ly" + ], + [ + "å®īåħ¨", + "çļĦ" + ], + [ + "ç²", + "Ĵ" + ], + [ + "å¤ĩ", + "份" + ], + [ + "åĪ", + "º" + ], + [ + "Ġpr", + "ice" + ], + [ + "çĭ", + "Ĺ" + ], + [ + "Ġt", + "itle" + ], + [ + "æľī", + "åĵªäºĽ" + ], + [ + "Ġrelations", + "hip" + ], + [ + "Ñ", + "Į" + ], + [ + "ä¾", + "µ" + ], + [ + "çŃī", + "å¾ħ" + ], + [ + "ord", + "ing" + ], + [ + "åľ°", + "çĤ¹" + ], + [ + "Ġme", + "et" + ], + [ + "ä¼ł", + "éĢĴ" + ], + [ + "çļĦ", + "æĢģ度" + ], + [ + "æķħ", + "éļľ" + ], + [ + "Ġre", + "verse" + ], + [ + "et", + "ic" + ], + [ + "交", + "æį¢" + ], + [ + "Ġm", + "ust" + ], + [ + "é«ĺ", + "åħ´" + ], + [ + "ound", + "s" + ], + [ + "é¢Ħ", + "æľŁ" + ], + [ + "Ġfe", + "et" + ], + [ + "åĪĿ", + "åѦèĢħ" + ], + [ + "ä¼", + "Ļ" + ], + [ + "æıIJä¾Ľ", + "ä¸Ģ个" + ], + [ + "åİ", + "ħ" + ], + [ + "çļĦæµ", + "ģ" + ], + [ + "t", + "le" + ], + [ + "ç¼ĸç¨ĭ", + "è¯Ńè¨Ģ" + ], + [ + "åŃ£", + "èĬĤ" + ], + [ + "éĩıåŃIJ", + "计ç®Ĺ" + ], + [ + "u", + "ro" + ], + [ + "Ġse", + "a" + ], + [ + "æ¯Ķ", + "çī¹" + ], + [ + "æľī", + "éĻIJ" + ], + [ + "Ġar", + "g" + ], + [ + "F", + "ROM" + ], + [ + "Ġh", + "ome" + ], + [ + "Ġdisc", + "uss" + ], + [ + "åħ·", + "å¤ĩ" + ], + [ + "æļ", + "Ĺ" + ], + [ + "Ġpre", + "v" + ], + [ + "æī®", + "æ¼Ķ" + ], + [ + "åľ°", + "æĸ¹" + ], + [ + "åĮĹ", + "京" + ], + [ + "å®ģ", + "éĿĻ" + ], + [ + "è§Ħ", + "èĮĥ" + ], + [ + "社ä¼ļ", + "çļĦ" + ], + [ + "缮", + "å½ķ" + ], + [ + "åѦ", + "æľ¯" + ], + [ + "Ġm", + "ix" + ], + [ + "åľ¨", + "å¤ĦçIJĨ" + ], + [ + "ic", + "ro" + ], + [ + "ke", + "y" + ], + [ + "Ġwe", + "b" + ], + [ + "æĵįä½ľ", + "ç³»ç»Ł" + ], + [ + "¹", + "æŀľ" + ], + [ + "çī©", + "èģĶç½ij" + ], + [ + "æľįåĬ¡", + "çļĦ" + ], + [ + "æıIJ", + "åīį" + ], + [ + "çĤ", + "İ" + ], + [ + "åIJĮ", + "æł·" + ], + [ + "Ġem", + "ployees" + ], + [ + "Ġc", + "ut" + ], + [ + "åIJĪ", + "è§Ħ" + ], + [ + "çļĦ", + "éĢīæĭ©" + ], + [ + "å¥", + "¥" + ], + [ + "c", + "ol" + ], + [ + "ç§", + "Ģ" + ], + [ + "è¿Ľ", + "ç¨ĭ" + ], + [ + "}", + "}" + ], + [ + "æķĻ", + "åѦ" + ], + [ + "éĶ", + "ħ" + ], + [ + "u", + "c" + ], + [ + "ä¿¡", + "çĶ¨" + ], + [ + "ä¿¡æģ¯", + "çļĦ" + ], + [ + "çļĦæİ", + "¨" + ], + [ + "åĢ", + "į" + ], + [ + "éļľ", + "ç¢į" + ], + [ + "or", + "n" + ], + [ + "ĠI", + "nd" + ], + [ + "æľª", + "çŁ¥" + ], + [ + "çķ", + "ħ" + ], + [ + "è½»", + "è½»" + ], + [ + "ä¿Ŀ", + "éļľ" + ], + [ + "ĠA", + "pp" + ], + [ + "ä¸Ģ", + "ç³»åĪĹ" + ], + [ + "åĨ", + "¬" + ], + [ + "æĪij们", + "é¦ĸåħĪ" + ], + [ + "大å¤ļ", + "æķ°" + ], + [ + "çļĦ", + "产åĵģ" + ], + [ + "å½¢", + "çĬ¶" + ], + [ + "Ġallow", + "s" + ], + [ + "ol", + "ution" + ], + [ + "Ġs", + "ource" + ], + [ + "Ġo", + "cc" + ], + [ + "åIJĪ", + "æ³ķ" + ], + [ + "åľ¨", + "Pythonä¸Ń" + ], + [ + "大", + "约" + ], + [ + "Ġan", + "other" + ], + [ + "éĢIJ", + "æŃ¥" + ], + [ + "使çĶ¨", + "äºĨ" + ], + [ + "åĿĩ", + "åĮĢ" + ], + [ + "çļĦ", + "å¿ĥ" + ], + [ + "Ġw", + "ind" + ], + [ + "æĸ", + "Ĺ" + ], + [ + "Ġl", + "ower" + ], + [ + "解", + "æŀIJ" + ], + [ + "çĿ", + "£" + ], + [ + "å°", + "ļ" + ], + [ + "æģ", + "Ĵ" + ], + [ + "çĿ", + "¡çľł" + ], + [ + "ĠA", + "meric" + ], + [ + "è·", + "ij" + ], + [ + "âĢĶ", + "âĢĶ" + ], + [ + "ant", + "s" + ], + [ + "çīĽ", + "奶" + ], + [ + "æ°§", + "åĮĸ" + ], + [ + "m", + "od" + ], + [ + "å½¢", + "çļĦ" + ], + [ + "çļĦç©", + "º" + ], + [ + "Ð", + "²" + ], + [ + "al", + "es" + ], + [ + "an", + "a" + ], + [ + "æıIJä¾Ľ", + "æĽ´" + ], + [ + "Ġs", + "or" + ], + [ + "èĥ", + "ģ" + ], + [ + "i", + "et" + ], + [ + "ç§", + "Ĵ" + ], + [ + "ch", + "ange" + ], + [ + "Ġcount", + "ries" + ], + [ + "è¿ħ", + "éĢŁ" + ], + [ + "èĶ¬", + "èıľ" + ], + [ + "Ġth", + "ough" + ], + [ + "ĠC", + "EO" + ], + [ + "ide", + "red" + ], + [ + "èĭ", + "¹æŀľ" + ], + [ + "Ġmult", + "iply" + ], + [ + "Ġwrit", + "ing" + ], + [ + "j", + "or" + ], + [ + "çĥ", + "§" + ], + [ + "Ġtas", + "k" + ], + [ + "s", + "er" + ], + [ + "Ġdec", + "imal" + ], + [ + "int", + "er" + ], + [ + "æ³ķå¾ĭ", + "æ³ķè§Ħ" + ], + [ + "Ġphys", + "ical" + ], + [ + "Ġrad", + "ius" + ], + [ + "è·", + "ĥ" + ], + [ + "å°½", + "管" + ], + [ + "羣", + "æŃ£" + ], + [ + "æĢ»", + "ç»ĵ" + ], + [ + "å¯¹ä½ł", + "æľīæīĢ帮åĬ©" + ], + [ + "çļĦç²", + "¾" + ], + [ + "è§Ĩ", + "为" + ], + [ + "éĹ®é¢ĺ", + "çļĦ" + ], + [ + "Ġcons", + "idered" + ], + [ + "å¦ĤæŀľæĤ¨", + "æľī" + ], + [ + "Ġtop", + "ics" + ], + [ + "æĸĩ", + "æĺİ" + ], + [ + "Ġg", + "ra" + ], + [ + "è°", + "±" + ], + [ + "èĥ", + "ľ" + ], + [ + "åŁºæľ¬", + "çļĦ" + ], + [ + "dition", + "al" + ], + [ + "åı¯ä»¥", + "å°Ĩ" + ], + [ + "Ġre", + "d" + ], + [ + "ä¸ĭéĿ¢", + "æĺ¯ä¸Ģ个" + ], + [ + "æĹ¥å¸¸", + "çĶŁæ´»" + ], + [ + "å¦Ĥæŀľ", + "æľī" + ], + [ + "è¿ĺ", + "æľī" + ], + [ + "Ġpr", + "iv" + ], + [ + "å¨ģ", + "èĥģ" + ], + [ + "æĦŁ", + "åıĹ" + ], + [ + "åľ¨", + "ä¸ŃåĽ½" + ], + [ + "Ġadd", + "ition" + ], + [ + ")", + "\\)" + ], + [ + "ä¸ĸ", + "纪" + ], + [ + "ĠO", + "ct" + ], + [ + "讲", + "è¿°" + ], + [ + "Ġh", + "y" + ], + [ + "t", + "on" + ], + [ + "u", + "le" + ], + [ + "L", + "e" + ], + [ + "ĠA", + "re" + ], + [ + "Ġres", + "pect" + ], + [ + "ç«ŀäºī", + "åĬĽ" + ], + [ + "æī", + "ĺ" + ], + [ + "äºĨ解", + "ä¸Ģä¸ĭ" + ], + [ + "读", + "åıĸ" + ], + [ + "以", + "æıIJé«ĺ" + ], + [ + "å°¤åħ¶", + "æĺ¯åľ¨" + ], + [ + "éĿ¢", + "çļĦ" + ], + [ + "读", + "èĢħ" + ], + [ + "ä¸ĵ注", + "äºİ" + ], + [ + "Ġtyp", + "es" + ], + [ + "2", + "9" + ], + [ + "A", + "re" + ], + [ + "使", + "åħ¶" + ], + [ + "am", + "b" + ], + [ + "Ġst", + "ill" + ], + [ + "å±ı", + "å¹ķ" + ], + [ + "çĤ", + "¼" + ], + [ + "æĽ´åĩĨç¡®", + "åľ°" + ], + [ + "Ġpot", + "ential" + ], + [ + "ç¡®ä¿Ŀ", + "æīĢæľī" + ], + [ + "W", + "e" + ], + [ + "åĮº", + "åĪ«" + ], + [ + "顺", + "åĪ©" + ], + [ + "åħ", + "Ń" + ], + [ + "线", + "ç¨ĭ" + ], + [ + "g", + "en" + ], + [ + "0", + "1" + ], + [ + "Ġhe", + "ight" + ], + [ + "Ġmo", + "on" + ], + [ + "vers", + "ed" + ], + [ + "èı", + "Į" + ], + [ + "ا", + "ÙĦ" + ], + [ + "]", + ")" + ], + [ + "vie", + "w" + ], + [ + "Ġit", + "er" + ], + [ + "Ġanswer", + "ing" + ], + [ + "l", + "ished" + ], + [ + "z", + "hou" + ], + [ + "æľŁ", + "æľĽ" + ], + [ + "l", + "ate" + ], + [ + "éĩįè¦ģ", + "çļĦæĺ¯" + ], + [ + "ç®Ģ", + "åĮĸ" + ], + [ + "k", + "es" + ], + [ + "åı¯ä»¥", + "æľīæķĪ" + ], + [ + "ä¸Ĭ", + "æīĢ" + ], + [ + "ĠThe", + "refore" + ], + [ + "L", + "E" + ], + [ + "çļĦæ¯", + "Ķ" + ], + [ + "èĩªåĬ¨", + "驾驶" + ], + [ + "Ġdatab", + "ase" + ], + [ + "ar", + "a" + ], + [ + "Ġoff", + "er" + ], + [ + "ist", + "ics" + ], + [ + "åı¯èĥ½ä¼ļ", + "导èĩ´" + ], + [ + "好", + "å¥ĩ" + ], + [ + "èĦ", + "±" + ], + [ + "çļĦç", + "Ī" + ], + [ + "èĤ", + "¥" + ], + [ + "Ġcontain", + "s" + ], + [ + "åºĶçĶ¨", + "ä¸Ń" + ], + [ + "Ġun", + "its" + ], + [ + "åĴĮ", + "建议" + ], + [ + "çļĦ", + "ç»ĵæŀľ" + ], + [ + "ĠJ", + "ack" + ], + [ + "æī", + "°" + ], + [ + "使çĶ¨", + "çļĦ" + ], + [ + "æĮĩ", + "æłĩ" + ], + [ + "åĨį", + "次" + ], + [ + "Ġs", + "ong" + ], + [ + "ro", + "p" + ], + [ + "Ġrel", + "i" + ], + [ + "ren", + "ch" + ], + [ + "Ġ", + "ess" + ], + [ + "æݨ", + "广" + ], + [ + "åĴĮ", + "æĸĩåĮĸ" + ], + [ + "çĤ", + "Ĵ" + ], + [ + "å¡ij", + "æĸĻ" + ], + [ + "A", + "N" + ], + [ + "m", + "ax" + ], + [ + "çļĦæ", + "¶" + ], + [ + "度", + "çļĦ" + ], + [ + "å®ĥ", + "ä¸įä»ħ" + ], + [ + "Ġnum", + "er" + ], + [ + "E", + "ach" + ], + [ + "Ġs", + "elf" + ], + [ + "éķ¿", + "æŶéĹ´" + ], + [ + "ä»Ĭ", + "天" + ], + [ + "çĪ", + "¶" + ], + [ + "çļĦ", + "帮åĬ©" + ], + [ + "Ġst", + "at" + ], + [ + "and", + "as" + ], + [ + "Ġsy", + "stems" + ], + [ + "Ġadd", + "ress" + ], + [ + "Ġan", + "ything" + ], + [ + "n", + "ces" + ], + [ + "å¹", + "ħ" + ], + [ + "æī¾", + "åĩº" + ], + [ + "Ġnum", + "py" + ], + [ + "Ġf", + "low" + ], + [ + "ç§", + "Ł" + ], + [ + "åĶ", + "¯" + ], + [ + "éĹ®é¢ĺæĪĸ", + "éľĢè¦ģ" + ], + [ + "W", + "HERE" + ], + [ + "åĦ¿", + "ç«¥" + ], + [ + "Ġqu", + "ick" + ], + [ + "è¯į", + "æ±ĩ" + ], + [ + "6", + "4" + ], + [ + "am", + "ent" + ], + [ + "ĠA", + "r" + ], + [ + "ç»Ļ", + "å®ļçļĦ" + ], + [ + "ne", + "y" + ], + [ + "ä¸ĬæīĢ", + "è¿°" + ], + [ + "å¼", + "±" + ], + [ + "å¤ļæł·", + "æĢ§" + ], + [ + "为", + "çĶ¨æĪ·" + ], + [ + "Ġu", + "nt" + ], + [ + "Ġdescrib", + "e" + ], + [ + "çļĦ", + "åħī" + ], + [ + "åĬł", + "è½½" + ], + [ + "a", + "res" + ], + [ + "çļĦ", + "éķ¿åº¦" + ], + [ + "åĪĨ", + "解" + ], + [ + "èİ", + "ī" + ], + [ + "æĽ´å¥½åľ°", + "çIJĨ解" + ], + [ + "a", + "w" + ], + [ + "id", + "ed" + ], + [ + "è¿ĩ", + "滤" + ], + [ + "20", + "23" + ], + [ + "éĿ¢", + "è¯ķ" + ], + [ + "仿", + "ä½Ľ" + ], + [ + "综", + "ä¸ĬæīĢè¿°" + ], + [ + "认", + "è¯Ĩ" + ], + [ + "产çĶŁ", + "äºĨ" + ], + [ + "åĽº", + "å®ļ" + ], + [ + "çļĦæ´»", + "åĬ¨" + ], + [ + "C", + "D" + ], + [ + "Ġp", + "ie" + ], + [ + "æķ°æį®", + "å®īåħ¨" + ], + [ + "æĦŁ", + "åıĹåĪ°" + ], + [ + "èµĦ", + "æĸĻ" + ], + [ + "an", + "ation" + ], + [ + "Ġ+", + "=" + ], + [ + "iz", + "ing" + ], + [ + "Ġlet", + "ter" + ], + [ + "e", + "ad" + ], + [ + "èª", + "ī" + ], + [ + "b", + "ash" + ], + [ + "Ġf", + "our" + ], + [ + "Ġal", + "gorithm" + ], + [ + "U", + "n" + ], + [ + "ä¸Ģ", + "份" + ], + [ + "æĢ", + "Ģ" + ], + [ + "St", + "ring" + ], + [ + "or", + "k" + ], + [ + "ar", + "ily" + ], + [ + "é", + "¹" + ], + [ + "æĪij们", + "éľĢè¦ģ" + ], + [ + "æ¢", + "¯" + ], + [ + "åĬŀ", + "åħ¬" + ], + [ + "or", + "m" + ], + [ + "ä½ł", + "们" + ], + [ + "æ¯", + "Ľ" + ], + [ + "b", + "an" + ], + [ + "w", + "w" + ], + [ + "Ġsor", + "ry" + ], + [ + "æĿ¥", + "è¶Ĭ" + ], + [ + "ad", + "r" + ], + [ + "Ġa", + "im" + ], + [ + "h", + "ang" + ], + [ + "h", + "ai" + ], + [ + "帮åĬ©", + "çĶ¨æĪ·" + ], + [ + "åij¨", + "åĽ´" + ], + [ + "è¶Ĭ", + "æĿ¥è¶Ĭ" + ], + [ + "ç»", + "ķ" + ], + [ + "éĢļ常", + "éľĢè¦ģ" + ], + [ + "Ċ", + "ĊĠĠ" + ], + [ + "åħĪ", + "è¿Ľ" + ], + [ + "ĠOct", + "ober" + ], + [ + "g", + "o" + ], + [ + "ro", + "du" + ], + [ + "éĴ", + "±" + ], + [ + "Ù", + "Ĩ" + ], + [ + "请", + "éĹ®" + ], + [ + "Ġex", + "t" + ], + [ + "Ġim", + "age" + ], + [ + "æ¶Ī", + "èĢĹ" + ], + [ + "çļĦæŃ", + "¥" + ], + [ + "Ġeng", + "aging" + ], + [ + "J", + "ohn" + ], + [ + "ç»ı", + "è¿ĩ" + ], + [ + "æIJ", + "º" + ], + [ + "å¾Ī", + "é«ĺåħ´" + ], + [ + "A", + "r" + ], + [ + "åı¯ä»¥", + "å°Ŀè¯ķ" + ], + [ + "çļĦç", + "Ĭ" + ], + [ + "Ġcolumn", + "s" + ], + [ + "çݯå¢ĥ", + "ä¿ĿæĬ¤" + ], + [ + "éĴ", + "¥" + ], + [ + "ä¸Ģ", + "ä½į" + ], + [ + "éĢļè¿ĩ", + "è¿ĻäºĽ" + ], + [ + "Ġsent", + "ences" + ], + [ + "å±Ĥ", + "次" + ], + [ + "an", + "ish" + ], + [ + "失", + "è´¥" + ], + [ + "ou", + "n" + ], + [ + "ï¼ļ", + "**" + ], + [ + "Ġwe", + "ek" + ], + [ + "in", + "ary" + ], + [ + "Ġcre", + "ative" + ], + [ + "åĩº", + "äºĨ" + ], + [ + "ver", + "n" + ], + [ + "转", + "åŀĭ" + ], + [ + "7", + "5" + ], + [ + "æįŁ", + "失" + ], + [ + "åĴĮ", + "使çĶ¨" + ], + [ + "Ġwrit", + "ten" + ], + [ + "Ġd", + "em" + ], + [ + "认", + "è¯ģ" + ], + [ + "ĠÐ", + "²" + ], + [ + "A", + "P" + ], + [ + "åĨħ", + "ç½®" + ], + [ + "Ġwor", + "ks" + ], + [ + "ç¾İ", + "好" + ], + [ + "çļĦ", + "çĶ¨æĪ·" + ], + [ + "å¾®", + "ä¿¡" + ], + [ + "å®", + "£" + ], + [ + "Ġiss", + "ues" + ], + [ + ":", + "\"," + ], + [ + "Ġal", + "ong" + ], + [ + "av", + "or" + ], + [ + "a", + "king" + ], + [ + "Ġprint", + "s" + ], + [ + "Ġstart", + "ed" + ], + [ + "Ġkil", + "om" + ], + [ + "æĪIJ", + "åĪĨ" + ], + [ + "被", + "称为" + ], + [ + "驱", + "åĬ¨" + ], + [ + "R", + "L" + ], + [ + "ä¸į", + "足" + ], + [ + "Ð", + "´" + ], + [ + "è", + "²" + ], + [ + "çļĦ", + "åı¯" + ], + [ + "Ġf", + "lo" + ], + [ + "æİĴ", + "æĶ¾" + ], + [ + "Ġbre", + "ak" + ], + [ + "A", + "n" + ], + [ + "Ġc", + "ru" + ], + [ + "ä¸ĭ", + "è½½" + ], + [ + "äºĭ", + "项" + ], + [ + "è¿", + "·" + ], + [ + "æŃ£", + "å¼ı" + ], + [ + "éĴ", + "®" + ], + [ + "Ġd", + "en" + ], + [ + "åĩı", + "è½»" + ], + [ + "å®", + "«" + ], + [ + "M", + "y" + ], + [ + "S", + "C" + ], + [ + "alcul", + "ate" + ], + [ + "ro", + "l" + ], + [ + "æıIJä¾Ľ", + "æĽ´åĬł" + ], + [ + "Ġmed", + "ia" + ], + [ + "ang", + "zhou" + ], + [ + "æĽ¿", + "æį¢" + ], + [ + "f", + "unction" + ], + [ + "er", + "o" + ], + [ + "è½", + "´" + ], + [ + "éª", + "¨" + ], + [ + "am", + "p" + ], + [ + "è°", + "·" + ], + [ + "ë", + "Ĭ" + ], + [ + "Ġem", + "ot" + ], + [ + "Ġnetwor", + "k" + ], + [ + "Ġb", + "ooks" + ], + [ + "åı¯ä»¥", + "åĩıå°ij" + ], + [ + "éĩĬ", + "æĶ¾" + ], + [ + "f", + "rom" + ], + [ + "ĠP", + "ar" + ], + [ + "èĴ", + "¸" + ], + [ + "Ġsumm", + "ary" + ], + [ + "now", + "n" + ], + [ + "åĴĮ", + "ç»´æĬ¤" + ], + [ + "æĹł", + "éĻIJ" + ], + [ + "åıĮ", + "æĸ¹" + ], + [ + "ä¸Ģ", + "èĪ¬" + ], + [ + "æĹ", + "¦" + ], + [ + "ç³»ç»Ł", + "ä¸Ń" + ], + [ + "Ġf", + "ull" + ], + [ + "Ġm", + "iss" + ], + [ + "Ġlet", + "ters" + ], + [ + "çĪ±", + "好" + ], + [ + "u", + "ation" + ], + [ + "çļĦæ", + "ĭ" + ], + [ + "åī", + "ª" + ], + [ + "åĽ¾", + "å½¢" + ], + [ + "æĪĺ", + "çķ¥" + ], + [ + "Ġachie", + "ve" + ], + [ + "C", + "l" + ], + [ + "åĴĮ", + "ä¼ĺåĮĸ" + ], + [ + "è¿ĩ", + "åİ»" + ], + [ + "éĢĤ", + "éĩı" + ], + [ + "ä¸įä»ħ", + "ä»ħ" + ], + [ + "du", + "le" + ], + [ + "以ä¸ĭæĺ¯ä¸ĢäºĽ", + "建议" + ], + [ + "åı", + "ī" + ], + [ + "é", + "½" + ], + [ + "Ġ**", + "ãĢĬ" + ], + [ + "çļĦæī", + "ĭ" + ], + [ + "幸", + "ç¦ı" + ], + [ + "çĶ", + "ľ" + ], + [ + "æĹ¥", + "å¿Ĺ" + ], + [ + "al", + "k" + ], + [ + "Ġle", + "g" + ], + [ + "ä¾ĭåŃIJ", + "ä¸Ń" + ], + [ + "Ġen", + "c" + ], + [ + "ç§ijåѦ", + "家" + ], + [ + "欢è¿İ", + "éļıæŶ" + ], + [ + "çīĪ", + "æĿĥ" + ], + [ + "ĊĠĠĠĠ", + "ĊĠĠĠ" + ], + [ + "S", + "p" + ], + [ + "Ġinit", + "ial" + ], + [ + "Ġs", + "ure" + ], + [ + "Ġf", + "ac" + ], + [ + "åĩ", + "Ŀ" + ], + [ + "å¿", + "½" + ], + [ + "Ġtransl", + "ation" + ], + [ + "Ġfam", + "ily" + ], + [ + "便", + "æį·" + ], + [ + "Ġconvers", + "ations" + ], + [ + "æļ", + "´" + ], + [ + "ĠG", + "re" + ], + [ + "r", + "ation" + ], + [ + "¹", + "é¥" + ], + [ + "è¿Ļ", + "两个" + ], + [ + "ä¼¼", + "ä¹İ" + ], + [ + "æµ", + "®" + ], + [ + "¹é¥", + "ª" + ], + [ + "åĽŀ", + "æĶ¶" + ], + [ + "N", + "LP" + ], + [ + "æĭ", + "¼" + ], + [ + "Ġd", + "r" + ], + [ + "Ġexper", + "ien" + ], + [ + "æĵ", + "İ" + ], + [ + "m", + "ed" + ], + [ + "Ġprof", + "ess" + ], + [ + "F", + "irst" + ], + [ + "Ġinf", + "lu" + ], + [ + "å¯", + "¸" + ], + [ + "ä¸į", + "ç¡®å®ļ" + ], + [ + "I", + "nt" + ], + [ + "ä¼ł", + "æĦŁ" + ], + [ + "Ġfunction", + "s" + ], + [ + "å£", + "¤" + ], + [ + "åĽ¢éĺŁ", + "åIJĪä½ľ" + ], + [ + "以ä¸ĭåĩł", + "ç§į" + ], + [ + "l", + "ike" + ], + [ + "åĬ¨", + "ä½ľ" + ], + [ + "é£ŀ", + "è¡Į" + ], + [ + "Ġd", + "ream" + ], + [ + "èĢ", + "³" + ], + [ + "å¼ķ", + "æĵİ" + ], + [ + "Ġp", + "a" + ], + [ + "å¤ļ", + "æĸ¹éĿ¢" + ], + [ + "产", + "ä¸ļ" + ], + [ + "W", + "here" + ], + [ + "Ġa", + "uth" + ], + [ + "åįĩ", + "级" + ], + [ + "æ´", + "¾" + ], + [ + "Ġgre", + "en" + ], + [ + "ç¾İ", + "åij³" + ], + [ + "os", + "p" + ], + [ + "æŶ", + "åħī" + ], + [ + "end", + "s" + ], + [ + "Ġcult", + "ural" + ], + [ + "Ġkilom", + "eters" + ], + [ + "ãĤ", + "Į" + ], + [ + "æ¥", + "¼" + ], + [ + "e", + "ch" + ], + [ + "æ", + "¾" + ], + [ + "èĩª", + "身" + ], + [ + "åŃIJ", + "çļĦ" + ], + [ + "åī", + "©" + ], + [ + "ff", + "ic" + ], + [ + "7", + "0" + ], + [ + "Ġf", + "ra" + ], + [ + "Ġqu", + "e" + ], + [ + "产åĵģ", + "çļĦ" + ], + [ + "ëĭ", + "¤" + ], + [ + "Ġst", + "re" + ], + [ + ")", + "ï¼Į" + ], + [ + "S", + "ub" + ], + [ + "çĽij", + "çĿ£" + ], + [ + "it", + "ing" + ], + [ + "ä½ĵ", + "èĤ²" + ], + [ + "è¿Ļ段", + "代çłģ" + ], + [ + "è´·", + "款" + ], + [ + "家", + "å±ħ" + ], + [ + "Ġsh", + "ow" + ], + [ + "AT", + "E" + ], + [ + "iction", + "ary" + ], + [ + "çĶ", + "·" + ], + [ + "im", + "plify" + ], + [ + "æµģ", + "éĩı" + ], + [ + "ç¾İ", + "丽" + ], + [ + "Ġsc", + "ience" + ], + [ + "Ġoper", + "ations" + ], + [ + "群", + "ä½ĵ" + ], + [ + "Ġmin", + "utes" + ], + [ + "i", + "um" + ], + [ + "æŶéĹ´", + "åĴĮ" + ], + [ + "Ġmain", + "tain" + ], + [ + "erv", + "ice" + ], + [ + "ä¼Ļ", + "ä¼´" + ], + [ + "2", + "8" + ], + [ + "ä½ľ", + "èĢħ" + ], + [ + "Ġmo", + "vie" + ], + [ + "{", + "\\" + ], + [ + "Ġdiffere", + "nce" + ], + [ + "è´Ł", + "éĿ¢" + ], + [ + "Ġ", + "éĢīæĭ©" + ], + [ + "le", + "ft" + ], + [ + "éĽ", + "ħ" + ], + [ + "çĶµ", + "è¯Ŀ" + ], + [ + "Ġcons", + "ider" + ], + [ + "æİ¥", + "触" + ], + [ + "c", + "el" + ], + [ + "Wh", + "y" + ], + [ + "ĠF", + "rench" + ], + [ + "Ġpre", + "c" + ], + [ + "Ġlead", + "ership" + ], + [ + "为", + "æĤ¨" + ], + [ + "åĪĹ", + "åĩº" + ], + [ + "R", + "E" + ], + [ + "ç½ij", + "页" + ], + [ + "W", + "indows" + ], + [ + "ä¿Ŀ", + "çķĻ" + ], + [ + "æĮī", + "éĴ®" + ], + [ + "çļĦæ¦Ĥ", + "çİĩ" + ], + [ + "åı£", + "åij³" + ], + [ + "Ġmat", + "rix" + ], + [ + "How", + "ever" + ], + [ + "ĠÐ", + "¿" + ], + [ + "èĮĥåĽ´", + "åĨħ" + ], + [ + "Ġhist", + "or" + ], + [ + "çĹĩ", + "çĬ¶" + ], + [ + "æŃ", + "»" + ], + [ + "çī©", + "ä½ĵ" + ], + [ + "æīĭ", + "段" + ], + [ + "ä¹IJ", + "æĦı" + ], + [ + "ä¼łæĦŁ", + "åĻ¨" + ], + [ + "æķ", + "¢" + ], + [ + "Ġma", + "jor" + ], + [ + "Ġloc", + "al" + ], + [ + "å¥", + "ij" + ], + [ + "Ġpro", + "te" + ], + [ + "æľ¬", + "书" + ], + [ + "ç»ı", + "常" + ], + [ + "åľŁ", + "壤" + ], + [ + "Ġsymb", + "ol" + ], + [ + "æľŁ", + "å¾ħ" + ], + [ + "ä¹IJ", + "趣" + ], + [ + "Ġcont", + "in" + ], + [ + "Ġit", + "ems" + ], + [ + "çı", + "į" + ], + [ + "å³", + "°" + ], + [ + "D", + "P" + ], + [ + "Ġp", + "red" + ], + [ + "Ġ20", + "22" + ], + [ + "æıĴ", + "åħ¥" + ], + [ + "l", + "st" + ], + [ + "Ġ2", + "1" + ], + [ + "åī", + "²" + ], + [ + "èµĦæºIJ", + "çļĦ" + ], + [ + "éĤ", + "Ģ" + ], + [ + "è¿Ļ", + "åı¥è¯Ŀ" + ], + [ + "è¾ĥ", + "é«ĺ" + ], + [ + "çļĦæ¸", + "¸" + ], + [ + "æ±", + "ģ" + ], + [ + "ill", + "s" + ], + [ + "æĬķèµĦ", + "èĢħ" + ], + [ + "åı¯èĥ½", + "导èĩ´" + ], + [ + "ç±»", + "åĪ«" + ], + [ + "éĿŀ常", + "éĩįè¦ģçļĦ" + ], + [ + "Ġpass", + "word" + ], + [ + "Ġear", + "ly" + ], + [ + "Ġha", + "ven" + ], + [ + "åĻ", + "ª" + ], + [ + "éģĵ", + "è·¯" + ], + [ + "Ġlear", + "n" + ], + [ + "æ¿Ģ", + "åĬ±" + ], + [ + "åĩº", + "è¡Į" + ], + [ + "e", + "e" + ], + [ + "ra", + "ft" + ], + [ + "pl", + "ot" + ], + [ + "åĴĮ", + "çIJĨ解" + ], + [ + "Ġele", + "ct" + ], + [ + "n", + "ect" + ], + [ + "Ġfig", + "ure" + ], + [ + "çļĦæĶ¯", + "æĮģ" + ], + [ + "Ġspec", + "ify" + ], + [ + "ate", + "g" + ], + [ + "大", + "èĦij" + ], + [ + "ĠH", + "angzhou" + ], + [ + "æĢ»", + "æĺ¯" + ], + [ + "Ġmod", + "ern" + ], + [ + "ëĬ", + "Ķ" + ], + [ + "Ġ\\)", + ":" + ], + [ + "Ġcru", + "cial" + ], + [ + "æľĢ", + "æĸ°" + ], + [ + "m", + "p" + ], + [ + "解", + "çŃĶ" + ], + [ + "ir", + "d" + ], + [ + "ĠĠĠĠ", + "ĠĠĠĠ" + ], + [ + "åı¯ä»¥", + "被" + ], + [ + "æŁIJ", + "ç§į" + ], + [ + "ç³", + "ķ" + ], + [ + "Ġconst", + "ant" + ], + [ + "4", + "8" + ], + [ + "A", + "C" + ], + [ + "éľĢè¦ģ", + "èĢĥèĻij" + ], + [ + "Ġex", + "ec" + ], + [ + "æľŁ", + "éĹ´" + ], + [ + "ä¼ĺ", + "æĥł" + ], + [ + "Ġe", + "t" + ], + [ + "缸", + "ä¿¡" + ], + [ + "ĠE", + "l" + ], + [ + "Ġper", + "iod" + ], + [ + "Ġse", + "cre" + ], + [ + "æĪijæĥ³", + "äºĨ解ä¸Ģä¸ĭ" + ], + [ + "带æĿ¥", + "äºĨ" + ], + [ + "Ġapplic", + "ation" + ], + [ + "çļĦ", + "åĩĨç¡®" + ], + [ + "åįķ", + "åħĥ" + ], + [ + "èĢĥ", + "è¯ķ" + ], + [ + "ĠU", + "S" + ], + [ + "åĪĽ", + "ä¸ļ" + ], + [ + "æĦŁ", + "åĴĮ" + ], + [ + "询", + "éĹ®" + ], + [ + "éĽĨ", + "ä¸Ń" + ], + [ + "ap", + "an" + ], + [ + "P", + "r" + ], + [ + "æĹ", + "§" + ], + [ + "è´", + "Ŀ" + ], + [ + "à¸", + "²" + ], + [ + "æıIJ", + "éĨĴ" + ], + [ + "è°ĥ", + "èĬĤ" + ], + [ + "åı¤", + "èĢģ" + ], + [ + "Ġrecogn", + "ized" + ], + [ + "稳å®ļ", + "æĢ§" + ], + [ + "åıij", + "æĮ¥" + ], + [ + "I", + "T" + ], + [ + "å", + "ĺ" + ], + [ + "åľ¨", + "使çĶ¨" + ], + [ + "åıijå±ķ", + "çļĦ" + ], + [ + "æĭ", + "Į" + ], + [ + "çĿ", + "Ľ" + ], + [ + "Ġch", + "ild" + ], + [ + "ï¼Į", + "\"" + ], + [ + "çļĦæ", + "Ľ" + ], + [ + "å¿«", + "ä¹IJ" + ], + [ + "Ġag", + "ain" + ], + [ + "Ġf", + "requ" + ], + [ + "çļĦæ", + "ĵ" + ], + [ + "为", + "ä½ł" + ], + [ + "审", + "æŁ¥" + ], + [ + "ĠI", + "s" + ], + [ + "åIJİ", + "ç»Ń" + ], + [ + "Ġtra", + "ditional" + ], + [ + "Ġc", + "rit" + ], + [ + "ä¸Ģ", + "æŦ" + ], + [ + "èĥ½åĬĽ", + "åĴĮ" + ], + [ + "he", + "ad" + ], + [ + "å¿ĥçIJĨ", + "åģ¥åº·" + ], + [ + "Ġex", + "ist" + ], + [ + "Ġsu", + "ccess" + ], + [ + "强", + "大" + ], + [ + "æĿĥ", + "çĽĬ" + ], + [ + "è¾¹", + "ç¼ĺ" + ], + [ + "at", + "ch" + ], + [ + "ol", + "l" + ], + [ + "åĶ¯", + "ä¸Ģ" + ], + [ + "çļĦçĬ", + "¶" + ], + [ + "Ġn", + "on" + ], + [ + "éĢļè¿ĩ", + "åĪĨæŀIJ" + ], + [ + "æ°´", + "æŀľ" + ], + [ + "çļĦæĮĩ", + "导" + ], + [ + "G", + "oogle" + ], + [ + "çļĦéĩįè¦ģ", + "æĢ§" + ], + [ + "Ġb", + "en" + ], + [ + "s", + "in" + ], + [ + "å¤", + "«" + ], + [ + "çĥ", + "Ł" + ], + [ + "æıIJ", + "交" + ], + [ + "æĽ´", + "æľīæķĪåľ°" + ], + [ + "æĹł", + "éľĢ" + ], + [ + "Ġse", + "arch" + ], + [ + "çIJĨ", + "念" + ], + [ + "Ġqu", + "adr" + ], + [ + "èģĮ", + "ä½į" + ], + [ + "Ġsumm", + "ar" + ], + [ + "Ġchall", + "eng" + ], + [ + "å°±", + "ä¸ļ" + ], + [ + "c", + "m" + ], + [ + "æĦŁ", + "æŁĵ" + ], + [ + "åij", + "¢" + ], + [ + "Ġderiv", + "ative" + ], + [ + "èµ", + "ı" + ], + [ + "Ġcon", + "sole" + ], + [ + "çĶµ", + "åķĨ" + ], + [ + "ç¥ŀ", + "ç§ĺ" + ], + [ + "Ġdist", + "ance" + ], + [ + "ĠSp", + "anish" + ], + [ + "Ġeffic", + "ient" + ], + [ + "ç½", + "ª" + ], + [ + "åĢ¼", + "å¾Ĺ" + ], + [ + "åĽ¢éĺŁ", + "çļĦ" + ], + [ + "æĶ¿", + "æ²»" + ], + [ + "hel", + "lo" + ], + [ + "a", + "ction" + ], + [ + "le", + "x" + ], + [ + "oo", + "se" + ], + [ + "Ġfound", + "er" + ], + [ + "大", + "äºİ" + ], + [ + "Ġstud", + "ents" + ], + [ + "çĦ¦", + "èĻij" + ], + [ + "ä½Ĩ", + "åľ¨" + ], + [ + "Ġcustom", + "er" + ], + [ + ":", + "//" + ], + [ + "A", + "M" + ], + [ + "å§", + "ĵ" + ], + [ + "æ°ĶåĢĻ", + "åıĺåĮĸ" + ], + [ + "Ġeconom", + "ic" + ], + [ + "s", + "ion" + ], + [ + "ä¸Ģ", + "å®ļçļĦ" + ], + [ + "éħ", + "±" + ], + [ + "Ġg", + "en" + ], + [ + "Ġd", + "ance" + ], + [ + "èĦ", + "ī" + ], + [ + "表", + "æĺİ" + ], + [ + "èĩªçĦ¶", + "çļĦ" + ], + [ + "Ġcont", + "inu" + ], + [ + "à", + "¦" + ], + [ + "ener", + "ate" + ], + [ + "éĤĢ", + "请" + ], + [ + "Ġqu", + "ant" + ], + [ + "åķĨ", + "åĬ¡" + ], + [ + "认", + "çŁ¥" + ], + [ + "Ġke", + "ep" + ], + [ + "en", + "n" + ], + [ + "éĺ¿éĩĮäºij", + "çļĦ" + ], + [ + "Ġindividual", + "s" + ], + [ + "Ġc", + "ertain" + ], + [ + "ad", + "d" + ], + [ + "Ġsu", + "st" + ], + [ + "end", + "er" + ], + [ + "Ġ", + "æµĭè¯ķ" + ], + [ + "è¿", + "Ł" + ], + [ + "åī", + "¯" + ], + [ + "us", + "ion" + ], + [ + "so", + "ft" + ], + [ + "Ġlim", + "it" + ], + [ + "Ñ", + "ĩ" + ], + [ + "è¿", + "ģ" + ], + [ + "æĿ¥", + "计ç®Ĺ" + ], + [ + "ight", + "s" + ], + [ + "æ³", + "³" + ], + [ + "çļĦæĻº", + "èĥ½" + ], + [ + "çļĦæĬķ", + "èµĦ" + ], + [ + "N", + "o" + ], + [ + "帮åĬ©", + "ä¼ģä¸ļ" + ], + [ + "Ġresult", + "s" + ], + [ + "æ", + "ĩ" + ], + [ + "æķ°æį®", + "å¤ĦçIJĨ" + ], + [ + "ĠP", + "er" + ], + [ + "Ġfam", + "ous" + ], + [ + "åħ¬", + "åĽŃ" + ], + [ + "çļĦæ", + "Ł" + ], + [ + "åħ³", + "éĹŃ" + ], + [ + "请", + "åijĬè¯īæĪij" + ], + [ + "os", + "s" + ], + [ + "Ġ<", + "=" + ], + [ + "çł´", + "åĿı" + ], + [ + "çĶŁ", + "ç´ł" + ], + [ + "æĪij们", + "使çĶ¨" + ], + [ + "w", + "here" + ], + [ + "ä¸Ģ", + "天" + ], + [ + "æİ¥", + "è¿ij" + ], + [ + "EC", + "S" + ], + [ + "c", + "ient" + ], + [ + "ĠS", + "he" + ], + [ + "f", + "ile" + ], + [ + "olog", + "y" + ], + [ + "ç¼ĸåĨĻ", + "ä¸Ģ个" + ], + [ + "}", + "}{" + ], + [ + "åı¯èĥ½", + "åľ¨" + ], + [ + "Ñģ", + "ÑĤ" + ], + [ + "Ġconvers", + "ation" + ], + [ + "Ġmus", + "ic" + ], + [ + "çļĦ", + "åİĨåı²" + ], + [ + "缸åºĶ", + "çļĦ" + ], + [ + "è¿Ļ", + "åĮħæĭ¬" + ], + [ + "æŃ", + "¦" + ], + [ + "åħ³éĶ®", + "è¯į" + ], + [ + "Ġtra", + "vel" + ], + [ + "Ġsust", + "ain" + ], + [ + "æµıè§Ī", + "åĻ¨" + ], + [ + "\"", + "\"" + ], + [ + "çļĦç", + "¥" + ], + [ + "Ġdes", + "cript" + ], + [ + "Ġproduct", + "s" + ], + [ + "f", + "ect" + ], + [ + "Ġab", + "ove" + ], + [ + "Ġsub", + "ject" + ], + [ + "Ġre", + "ce" + ], + [ + "æĪIJ", + "çĨŁ" + ], + [ + "è¾¾", + "æij©" + ], + [ + "è´¦", + "åı·" + ], + [ + "L", + "L" + ], + [ + "Ġh", + "o" + ], + [ + "èį", + "·" + ], + [ + "åĽłä¸º", + "å®ĥ" + ], + [ + "in", + "a" + ], + [ + "ç´§", + "å¼ł" + ], + [ + "è°", + "ľ" + ], + [ + "æĶ¾", + "åľ¨" + ], + [ + "Ġs", + "at" + ], + [ + "Ġcan", + "not" + ], + [ + "èĤĮ", + "èĤī" + ], + [ + "a", + "us" + ], + [ + "æıIJ", + "示" + ], + [ + "åľ¨è¿Ļ个", + "ä¾ĭåŃIJä¸Ń" + ], + [ + "Ġc", + "y" + ], + [ + "Ġ3", + "6" + ], + [ + "ad", + "em" + ], + [ + "èıľ", + "åįķ" + ], + [ + "çļĦæĵ", + "įä½ľ" + ], + [ + "cl", + "ass" + ], + [ + "è¾ĥ", + "ä½İ" + ], + [ + "g", + "ative" + ], + [ + "p", + "o" + ], + [ + "å¼", + "ĥ" + ], + [ + "Ù", + "Ī" + ], + [ + "ç", + "Ĵ" + ], + [ + "re", + "l" + ], + [ + "ent", + "ly" + ], + [ + "æ¤", + "Ĵ" + ], + [ + "涵", + "çĽĸ" + ], + [ + "an", + "n" + ], + [ + "便", + "åĪ©" + ], + [ + "çļĦæŃ¥", + "骤" + ], + [ + "6", + "7" + ], + [ + "Ġ", + "." + ], + [ + "åĢ", + "Ł" + ], + [ + "çļĦ", + "çݯå¢ĥ" + ], + [ + "åĽŀ", + "å½Ĵ" + ], + [ + "Ġcol", + "ors" + ], + [ + "Ġfriend", + "s" + ], + [ + "ht", + "ml" + ], + [ + "æĭ", + "ĵ" + ], + [ + "éĿŀ常", + "æľī" + ], + [ + "æĪĸ", + "æľįåĬ¡" + ], + [ + "è", + "©" + ], + [ + "ä¹", + "ı" + ], + [ + "åľ¨", + "ä¸įåIJĮ" + ], + [ + "审", + "计" + ], + [ + "1", + "23" + ], + [ + "åĵª", + "个" + ], + [ + "å²", + "Ľ" + ], + [ + "Ġcomple", + "te" + ], + [ + "Ġprocess", + "ing" + ], + [ + "ĠSh", + "ang" + ], + [ + "Ġcomprehen", + "sive" + ], + [ + "ou", + "se" + ], + [ + "å½", + "¼" + ], + [ + "åIJĪ", + "约" + ], + [ + "æ¡", + "Į" + ], + [ + "æĮģç»Ń", + "åŃ¦ä¹ł" + ], + [ + "Ġind", + "ic" + ], + [ + "Ġbu", + "ild" + ], + [ + "HT", + "ML" + ], + [ + "æ³Ħ", + "éľ²" + ], + [ + "Ġcom", + "b" + ], + [ + "ç", + "ĺ" + ], + [ + "çĶ¨", + "æĢ§" + ], + [ + "å·²", + "çŁ¥" + ], + [ + "ĠY", + "ork" + ], + [ + "æĽ´", + "容æĺĵ" + ], + [ + "符", + "åı·" + ], + [ + "æ¶Ī", + "æģ¯" + ], + [ + "Ġw", + "arm" + ], + [ + "çº", + "¯" + ], + [ + "åĮ»", + "åѦ" + ], + [ + "å¤Ħ", + "äºİ" + ], + [ + "d", + "a" + ], + [ + "Ġf", + "all" + ], + [ + "Ġl", + "iter" + ], + [ + "ĠP", + "l" + ], + [ + "Ġc", + "ur" + ], + [ + "çļĦ", + "设计" + ], + [ + "ĠO", + "r" + ], + [ + "ç»ĦæĪIJ", + "éĥ¨åĪĨ" + ], + [ + "ç¼ĵ", + "解" + ], + [ + "Ġprim", + "arily" + ], + [ + "åĨħ", + "å¿ĥ" + ], + [ + "çIJĨ", + "æĥ³" + ], + [ + "åĩº", + "çļĦ" + ], + [ + "æ±", + "¤" + ], + [ + "表", + "æł¼" + ], + [ + "erm", + "an" + ], + [ + "ht", + "t" + ], + [ + "è¿Ļ", + "使å¾Ĺ" + ], + [ + "æ¡Ī", + "ä¾ĭ" + ], + [ + "å©", + "ļ" + ], + [ + "éľĢè¦ģ注æĦı", + "çļĦæĺ¯" + ], + [ + "Ġint", + "rodu" + ], + [ + "åİ»", + "éĻ¤" + ], + [ + "Ġcompan", + "ies" + ], + [ + "d", + "own" + ], + [ + "Ġex", + "change" + ], + [ + "åĴĮ", + "åıijå±ķ" + ], + [ + "å¾Ī", + "ä¹IJæĦı" + ], + [ + "é¸", + "£" + ], + [ + "Ġsubs", + "idi" + ], + [ + "æĽ´", + "é«ĺæķĪ" + ], + [ + "个人", + "ä¿¡æģ¯" + ], + [ + "Python", + "代çłģ" + ], + [ + "ä¸ĭéĿ¢", + "æĺ¯" + ], + [ + "g", + "ing" + ], + [ + "æĮĩ", + "å®ļ" + ], + [ + "åį«", + "çĶŁ" + ], + [ + "æĻ", + "¨" + ], + [ + "çļĦç", + "Ŀ" + ], + [ + "åºĶçĶ¨", + "åľºæĻ¯" + ], + [ + "i", + "or" + ], + [ + "æī", + "«" + ], + [ + "çĶ¨æĪ·", + "åı¯ä»¥" + ], + [ + "ĠG", + "oogle" + ], + [ + "o", + "in" + ], + [ + "ä¸į", + "éľĢè¦ģ" + ], + [ + "æıIJä¾Ľ", + "帮åĬ©" + ], + [ + "es", + "p" + ], + [ + "æıIJä¾Ľ", + "çļĦ" + ], + [ + "ç»Ļ", + "äºĪ" + ], + [ + "ĠZ", + "hang" + ], + [ + "ëĭ", + "Ī" + ], + [ + "åı¯èĥ½", + "æľī" + ], + [ + "Ġst", + "yle" + ] + ] + } +} \ No newline at end of file diff --git a/chronos/assets/tokenizers/legacy_6400/tokenizer_config.json b/chronos/assets/tokenizers/legacy_6400/tokenizer_config.json new file mode 100644 index 0000000..e46c14a --- /dev/null +++ b/chronos/assets/tokenizers/legacy_6400/tokenizer_config.json @@ -0,0 +1,335 @@ +{ + "add_bos_token": false, + "add_eos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "0": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "5": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "6": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "7": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "8": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "9": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "10": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "11": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "12": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "13": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "14": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "15": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "16": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "<|buffer1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "<|buffer2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "<|buffer3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "<|buffer4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "<|buffer5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "<|buffer6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "<|buffer7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "<|buffer8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "<|buffer9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "<|audio_pad|>", + "", + "", + "", + "" + ], + "bos_token": "<|im_start|>", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "legacy": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "unk_token": "<|endoftext|>", + "image_token": "<|image_pad|>", + "audio_token": "<|audio_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>", + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if true %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if open_thinking is defined and open_thinking is true %}\n {{- '\\n' }}\n {%- else %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}", + "tokenizer_class": "PreTrainedTokenizerFast" +} \ No newline at end of file diff --git a/chronos/data/flexible_dataset.py b/chronos/data/flexible_dataset.py index 2eb7793..5366e99 100644 --- a/chronos/data/flexible_dataset.py +++ b/chronos/data/flexible_dataset.py @@ -9,8 +9,8 @@ instruction+output, conversations, messages). StreamingSFTDataset — supervised fine-tuning. Uses the tokenizer's chat template; only assistant tokens contribute - to the loss (label = -100 elsewhere). Drop-in - for minimind's SFTDataset. + to the loss (label = -100 elsewhere). Reads the + common `conversations` jsonl layout. StreamingDPODataset — preference pairs. Returns the dict shape DPO / ORPO trainers expect. @@ -605,7 +605,7 @@ def _pad_embeddings(values): # ── SFT (conversations → assistant-token loss mask) ───────────────── class StreamingSFTDataset(_StreamingJSONLBase): - """Streaming counterpart to minimind's SFTDataset. + """Streaming chat-SFT dataset (``conversations`` jsonl records). Reads `conversations: [{role, content, ...}, ...]` records, applies the tokenizer's chat template, and masks every position outside an @@ -617,7 +617,7 @@ def __init__(self, data_path: str, tokenizer, max_length: int = 1024): self.tokenizer = tokenizer self.max_length = max_length # Sentinels marking the assistant turn boundary in encoded form. - # Same formulation minimind uses (`bos`+'assistant\n' / `eos`+'\n'). + # ChatML-style turn markers (`bos`+'assistant\n' / `eos`+'\n'). self.bos_id = tokenizer( f'{tokenizer.bos_token}assistant\n', add_special_tokens=False ).input_ids @@ -741,7 +741,7 @@ def __getitem__(self, index): # ── DPO / ORPO (chosen + rejected pair) ───────────────────────────── class StreamingDPODataset(_StreamingJSONLBase): - """Streaming counterpart to minimind's DPODataset. + """Streaming preference-pair dataset (``chosen`` / ``rejected`` jsonl records). Returns the dict shape the DPO/ORPO trainers expect: {x_chosen, y_chosen, mask_chosen, x_rejected, y_rejected, mask_rejected} diff --git a/chronos/deps.py b/chronos/deps.py index 7a01a4a..8fdba29 100644 --- a/chronos/deps.py +++ b/chronos/deps.py @@ -1,80 +1,38 @@ -""" -chronos/deps.py — Automatic dependency bootstrap for minimind-master kernel. - -When minimind-master is not found locally, this module downloads it from -GitHub and makes it importable. Called automatically on first import of -any chronos.model module. +"""Compatibility shim (0.3.2+): Chronos has no external kernel dependency. -Legal notice: minimind is licensed under Apache-2.0. -See THIRD_PARTY_NOTICES.md for full attribution. +Before 0.3.2 this module git-cloned minimind from an unpinned ``master`` at +import time and inserted it at the front of ``sys.path``. The kernel pieces +Chronos used now live in :mod:`chronos.model.kernel`; nothing is downloaded +and ``sys.path`` is left alone. ``import chronos.deps`` keeps working for old +scripts, and :func:`get_tokenizer_path` returns the bundled legacy tokenizer. """ -import sys +from __future__ import annotations + import os -import subprocess -import importlib from pathlib import Path -MINIMIND_REPO = "https://github.com/jingyaogong/minimind" -MINIMIND_COMMIT = "master" # pin to a tag/commit for reproducibility +LEGACY_TOKENIZER_DIR = Path(__file__).resolve().parent / "assets" / "tokenizers" / "legacy_6400" -# Candidate local paths (in priority order) -_CANDIDATE_PATHS = [ - Path(__file__).parent.parent.parent / "minimind-master", # sibling dir - Path(__file__).parent.parent / "minimind-master", - Path.home() / ".cache" / "chronos" / "minimind-master", -] +def get_tokenizer_path() -> str: + """Default tokenizer for checkpoints that never recorded one. -def _find_local() -> Path | None: - # A bundle ships a vendored copy (``/third_party/minimind``) and - # points here, so a recipient never needs git or network access. - override = os.environ.get("CHRONOS_MINIMIND_DIR", "").strip() + ``CHRONOS_DEFAULT_TOKENIZER`` overrides it; otherwise the bundled legacy + 6400-token vocabulary (data files only, Apache-2.0, see its NOTICE) is + used. New models record their own tokenizer in the checkpoint sidecar. + """ + override = os.environ.get("CHRONOS_DEFAULT_TOKENIZER", "").strip() if override: p = Path(override).expanduser() - if not (p / "model" / "model_minimind.py").exists(): - raise FileNotFoundError(f"CHRONOS_MINIMIND_DIR={override!r} has no model/model_minimind.py") - return p - for p in _CANDIDATE_PATHS: - if (p / "model" / "model_minimind.py").exists(): - return p - return None - + if not (p / "tokenizer.json").is_file(): + raise FileNotFoundError(f"CHRONOS_DEFAULT_TOKENIZER={override!r} has no tokenizer.json") + return str(p) + return str(LEGACY_TOKENIZER_DIR) -def _download(target: Path) -> Path: - target.parent.mkdir(parents=True, exist_ok=True) - print(f"[Chronos] minimind not found locally. Downloading from {MINIMIND_REPO} ...") - print(f"[Chronos] Target: {target}") - print(f"[Chronos] License: Apache-2.0 | Copyright jingyaogong") - subprocess.run( - ["git", "clone", "--depth", "1", "--branch", MINIMIND_COMMIT, - MINIMIND_REPO, str(target)], - check=True, - ) - print(f"[Chronos] minimind downloaded to {target}") - return target - -def ensure_minimind() -> Path: - """ - Ensure minimind-master is available and on sys.path. - Returns the path to the minimind root directory. - """ - local = _find_local() - if local is None: - cache_dir = Path.home() / ".cache" / "chronos" / "minimind-master" - local = _download(cache_dir) - - root = str(local) - if root not in sys.path: - sys.path.insert(0, root) - return local - - -def get_tokenizer_path() -> str: - """Return the path to the minimind tokenizer directory.""" - root = ensure_minimind() - return str(root / "model") - - -# Run on import -_minimind_root = ensure_minimind() +def ensure_minimind(): + """Removed in 0.3.2: Chronos no longer needs minimind. Returns ``None``.""" + import warnings + warnings.warn("chronos.deps.ensure_minimind() is a no-op since 0.3.2; Chronos ships its own kernel " + "(chronos.model.kernel).", DeprecationWarning, stacklevel=2) + return None diff --git a/chronos/eval/benchmark.py b/chronos/eval/benchmark.py index 75be33f..393a5aa 100644 --- a/chronos/eval/benchmark.py +++ b/chronos/eval/benchmark.py @@ -14,8 +14,7 @@ --device cpu \ --max_new_tokens 64 """ -import sys -import chronos.deps # ensure minimind on sys.path +import chronos.deps # default tokenizer path import argparse import math @@ -108,7 +107,7 @@ def run_benchmark(args): chronos.deps.get_tokenizer_path() ) - from dataset.lm_dataset import PretrainDataset + from chronos.data.flexible_dataset import FlexibleDataset as PretrainDataset dataset = PretrainDataset(args.data_path, tokenizer, max_length=args.max_seq_len) loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False) diff --git a/chronos/eval/io_profiler.py b/chronos/eval/io_profiler.py index 1c08343..1ca8c3d 100644 --- a/chronos/eval/io_profiler.py +++ b/chronos/eval/io_profiler.py @@ -16,8 +16,7 @@ - Mean L2 routing shift (lower = more cache-friendly) - Cache hit rate estimate given VRAM budget """ -import sys -import chronos.deps # ensure minimind on sys.path +import chronos.deps # default tokenizer path import argparse import json @@ -221,7 +220,7 @@ def main(): chronos.deps.get_tokenizer_path() ) - from dataset.lm_dataset import PretrainDataset + from chronos.data.flexible_dataset import FlexibleDataset as PretrainDataset dataset = PretrainDataset(args.data_path, tokenizer, max_length=args.max_seq_len) loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False) diff --git a/chronos/io/cluster_layout.py b/chronos/io/cluster_layout.py index 8f9ef54..d473591 100644 --- a/chronos/io/cluster_layout.py +++ b/chronos/io/cluster_layout.py @@ -17,8 +17,7 @@ --data_path ./dataset/calib.jsonl \ --output_dir ./expert_cache_clustered """ -import sys -import chronos.deps # ensure minimind on sys.path +import chronos.deps # default tokenizer path import os import json @@ -406,7 +405,7 @@ def build_clustered_expert_cache( infer expert co-occurrence, cluster experts, and write .ctsr shards plus cluster_manifest.json for Chronos lazy loading. """ - import chronos.deps + import chronos.deps # default tokenizer path from chronos.data.flexible_dataset import FlexibleDataset from chronos.model.moe_chronos import ChronosMOEFeedForward diff --git a/chronos/io/expert_store.py b/chronos/io/expert_store.py index cb38b3a..b7dc53a 100644 --- a/chronos/io/expert_store.py +++ b/chronos/io/expert_store.py @@ -9,9 +9,6 @@ - Evict VRAM weights via LRU when budget is exceeded - Track per-expert residency state for soft-gating mask generation """ -import sys -import chronos.deps # ensure minimind on sys.path - import contextlib import os import threading @@ -313,7 +310,7 @@ def _expert_dtype(self) -> torch.dtype: return torch.float32 def _build_live_expert(self): - from model.model_minimind import FeedForward + from chronos.model.kernel import FeedForward # No random initialization or temporary full allocation on a cache miss. with torch.device("meta"): return FeedForward(self.config, intermediate_size=self.config.moe_intermediate_size) diff --git a/chronos/mlx/__init__.py b/chronos/mlx/__init__.py index d881163..d949318 100644 --- a/chronos/mlx/__init__.py +++ b/chronos/mlx/__init__.py @@ -13,7 +13,8 @@ Key differences from the CUDA backend: - No ExpertStore.promote_to_vram() — mx.eval() pins to Metal on first use - No H2D stream — Metal handles async via its own command queue - - torch.compile unavailable; mx.compile() used instead + - torch.compile unavailable; the training micro step uses mx.compile() + (chronos/mlx/training/compiled.py, CHRONOS_MLX_COMPILE=auto|on|off) - All modules are mlx.nn.Module, not torch.nn.Module """ __all__ = [ diff --git a/chronos/mlx/attention.py b/chronos/mlx/attention.py index 02f1100..7d7b281 100644 --- a/chronos/mlx/attention.py +++ b/chronos/mlx/attention.py @@ -9,7 +9,8 @@ boolean mask built here; softmax is FP32 inside the kernel, so the low-precision chain has no score/probability casts. - SlidingWindow: slice oldest tokens with array indexing, not assignment. -- mx.compile()-friendly: no Python branches on runtime tensor shapes. +- RoPE rotation is one shapeless-compiled kernel; the attention path itself + is compiled as part of the training step (chronos/mlx/training/compiled.py). """ import math from functools import partial @@ -17,6 +18,8 @@ import mlx.core as mx import mlx.nn as nn +from chronos.mlx.checkpointing import Dropout as TapeDropout # replayed under recomputation + from chronos.mlx.kv_quant import kv_read, kv_store @@ -26,7 +29,7 @@ def _rope_freqs( base: float = 1_000_000.0, rope_scaling: dict | None = None, ) -> mx.array: - """Precompute MiniMind/PyTorch-compatible RoPE tables [max_len, dim]. + """Precompute RoPE tables [max_len, dim] matching ``chronos.model.kernel.precompute_freqs_cis``. The PyTorch path uses duplicated cos/sin columns and rotate-half over the whole last dimension. MLX must mirror that exactly for checkpoint parity. @@ -210,8 +213,8 @@ def __init__(self, config): self.k_rope_proj = nn.Linear(self.kv_latent_dim, self.n_kv_heads * self.rope_dim, bias=False) self.v_proj = nn.Linear(self.kv_latent_dim, self.n_kv_heads * self.head_dim, bias=False) self.o_proj = nn.Linear(self.n_heads * self.head_dim, H, bias=False) - self.attn_dropout = nn.Dropout(config.dropout) - self.resid_dropout = nn.Dropout(config.dropout) + self.attn_dropout = TapeDropout(config.dropout) + self.resid_dropout = TapeDropout(config.dropout) self.dropout_p = float(config.dropout) self._init_rope_cache(config, self.head_dim, rotated_dim=self.rope_dim) @@ -338,8 +341,8 @@ def __init__(self, config): self.o_proj = nn.Linear(self.n_heads * self.head_dim, H, bias=False) self.q_norm = nn.RMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = nn.RMSNorm(self.head_dim, eps=config.rms_norm_eps) - self.attn_dropout = nn.Dropout(config.dropout) - self.resid_dropout = nn.Dropout(config.dropout) + self.attn_dropout = TapeDropout(config.dropout) + self.resid_dropout = TapeDropout(config.dropout) self.dropout_p = float(config.dropout) self._init_rope_cache(config, self.head_dim) @@ -388,7 +391,9 @@ def __call__( # attention leaks future tokens into early queries during long prefill. q_positions = mx.arange(S)[:, None] + cached_len k_positions = mx.arange(T_full)[None, :] - allowed = (k_positions <= q_positions) & (k_positions > q_positions-self.window_size) + allowed = k_positions <= q_positions + if self.window_size is not None: # None: dense (use_hybrid_attention=False) + allowed = allowed & (k_positions > q_positions - self.window_size) scale = 1.0 / math.sqrt(self.head_dim) out = self._attend(q.transpose(0, 2, 1, 3), k.transpose(0, 2, 1, 3), v.transpose(0, 2, 1, 3), @@ -396,15 +401,15 @@ def __call__( out = out.transpose(0, 2, 1, 3).reshape(B, S, -1) # Cache stores [B, T, nKV, d] (pre-repeat, pre-transpose) - keep = self.window_size + int(self.cache_slack) + keep = None if self.window_size is None else self.window_size + int(self.cache_slack) if self.kv_int8 and kv_override is None: # Quantize only the new tokens (each token once), then evict. qk = kv_store(k_new, cache[0] if cache is not None else None, self.kv_group) qv = kv_store(v_new, cache[1] if cache is not None else None, self.kv_group) - start = max(0, qk.shape[1] - keep) + start = 0 if keep is None else max(0, qk.shape[1] - keep) new_cache = (qk[:, start:], qv[:, start:], past_len + S) else: - new_cache = (k[:, -keep:], v[:, -keep:], past_len + S) + new_cache = (k, v, past_len + S) if keep is None else (k[:, -keep:], v[:, -keep:], past_len + S) if return_shared_kv: return self.resid_dropout(self.o_proj(out)), new_cache, (k, v, past_len + S) return self.resid_dropout(self.o_proj(out)), new_cache diff --git a/chronos/mlx/checkpointing.py b/chronos/mlx/checkpointing.py new file mode 100644 index 0000000..719a8f2 --- /dev/null +++ b/chronos/mlx/checkpointing.py @@ -0,0 +1,246 @@ +"""Activation recomputation (``config.activation_checkpointing``) for MLX training. + +Each block call becomes an ``mx.checkpoint`` region: the backward pass keeps +only the block's input and output and recomputes everything in between. +Peak memory in Chronos training is dominated by per-virtual-layer +activations, and a recurrent group runs one physical block once per +iteration, so this is where the memory goes. + +A plain ``mx.checkpoint`` would be silently wrong for these blocks, for two +reasons: + +* **Random draws.** Dropout and the MoE cache-miss simulation + (``fallback_mask_prob``) draw fresh bits when the region is recomputed, so + the backward pass would differentiate a different network than the one the + forward pass ran (measured on mlx 0.32.4: same loss, different gradients). + Every draw goes through :func:`draw`, which records it on the region's tape + during the forward pass and replays the recorded array during the + recomputation. Draws are consumed in the same order and count as without + checkpointing, so the global RNG stream (and resume) is unchanged. +* **Side effects.** The MoE appends its router probabilities once per call, + and the regularized loss reads them. The recomputation must not append a + second copy, and the loss must read the forward-pass arrays (the + recomputed ones are internal to the backward pass). :func:`recomputing` + tells modules that the current call is a recomputation. + +Only arrays cross a region boundary, packed into one output per dtype so +no two outputs can be the same graph node (see :func:`_pack`); +:func:`call_block` rebuilds the non-array parts of a block's return value +(e.g. the SWA absolute position) around them. +""" +from __future__ import annotations + +import threading +from typing import Callable + +import mlx.core as mx +import mlx.nn as nn + +_STATE = threading.local() + + +class _Tape: + __slots__ = ("draws", "pos", "replay", "calls") + + def __init__(self): + self.draws: list = [] + self.pos = 0 + self.replay = False + self.calls = 0 + + +def _tape() -> _Tape | None: + return getattr(_STATE, "tape", None) + + +def recomputing() -> bool: + """True while a checkpointed region is being recomputed for its gradient.""" + tape = _tape() + return tape is not None and tape.replay + + +def recorded(fn: Callable[[], object]): + """``fn()`` in the forward pass; the same value again in its recomputation. + + For values that carry no gradient but decide what a block computes: + random draws, routing indices, and module state read inside a block + (another forward may change that state before the backward pass runs). + Outside a checkpointed region this is just ``fn()``. + """ + tape = _tape() + if tape is None: + return fn() + if tape.replay: + if tape.pos >= len(tape.draws): + raise RuntimeError("checkpoint recomputation recorded more values than its forward pass") + value = tape.draws[tape.pos] + else: + value = fn() + if isinstance(value, mx.array): + value = mx.stop_gradient(value) + tape.draws.append(value) + tape.pos += 1 + return value + + +draw = recorded # random arrays: the common case + + +def dropout(module, x: mx.array) -> mx.array: + """``nn.Dropout`` semantics with a tape-recorded mask.""" + keep = float(getattr(module, "_p_1", 1.0)) + if keep == 1.0 or not module.training: + return x + mask = draw(lambda: mx.random.bernoulli(keep, x.shape)) + return (mask * x) * (1.0 / keep) + + +def checkpointed(module, fn: Callable[..., tuple]) -> Callable[..., tuple]: + """``fn(*arrays)`` recomputed in the backward pass, with tape replay. + + ``module`` owns the trainable parameters ``fn`` uses (they are passed + through the region so their gradients are recomputed too). ``fn`` must + take and return arrays (or tuples of arrays) only. + """ + tape = _Tape() + + def region(params, *args): + prev = _tape() + tape.replay = tape.calls > 0 # 1st call: forward; later calls: recomputation + tape.calls += 1 + tape.pos = 0 + # The forward call receives the module's own arrays; a recomputation + # receives its own primals, which must be the ones the ops capture. + # They are swapped back afterwards so the module never keeps them. + saved = module.trainable_parameters() if tape.replay else None + if saved is not None: + module.update(params) + _STATE.tape = tape + try: + return fn(*args) + finally: + _STATE.tape = prev + if saved is not None: + module.update(saved) + + run = mx.checkpoint(region) + + def call(*args): + return run(module.trainable_parameters(), *args) + + return call + + +class Dropout(nn.Dropout): + """``nn.Dropout`` whose mask is replayed inside a recomputation.""" + + def __call__(self, x: mx.array) -> mx.array: + return dropout(self, x) + + +def side_output_sink() -> list | None: + """Inside a checkpointed block: list collecting ``(owner, kind, array)`` side outputs.""" + return getattr(_STATE, "sink", None) + + +class _Slot: + pass + + +def _split(obj, arrays: list): + """Replace every array leaf with ``_Slot`` and collect it (tuples/lists/dicts).""" + if isinstance(obj, mx.array): + arrays.append(obj) + return _Slot + if isinstance(obj, (tuple, list)): + return type(obj)(_split(o, arrays) for o in obj) + if isinstance(obj, dict): + return {k: _split(v, arrays) for k, v in obj.items()} + return obj # None / int / float / bool / str + + +def _join(spec, it): + if spec is _Slot: + return next(it) + if isinstance(spec, (tuple, list)): + return type(spec)(_join(s, it) for s in spec) + if isinstance(spec, dict): + return {k: _join(v, it) for k, v in spec.items()} + return spec + + +def _pack(arrays: list) -> tuple[tuple, list]: + """Region outputs as one array per dtype (a flat concatenation). + + mlx 0.32.4 keeps the cotangent of only the first copy when a checkpointed + function returns the same graph node more than once, silently dropping + the others' gradients. Aliases are not visible from Python: a block's + new cache and its shared KV view may be one object, but a full-length + slice (``k[:, -keep:]`` with ``keep >= T``) or a same-dtype ``astype`` is + a *new* Python object over the *same* node. Packing makes every output + a distinct node by construction; the concatenation's backward splits + the cotangent exactly, and an aliased array receives the sum of its + copies' cotangents through ordinary autodiff inside the recomputation. + """ + groups: dict = {} + layout = [] + for a in arrays: + members = groups.setdefault(a.dtype, []) + layout.append((a.dtype, len(members), tuple(a.shape), a.size)) + members.append(a) + order = list(groups) + packed = tuple(groups[d][0] if len(groups[d]) == 1 else + mx.concatenate([a.reshape(-1) for a in groups[d]]) for d in order) + starts: dict = {} + for d in order: + offset, starts[d] = 0, [] + for a in groups[d]: + starts[d].append(offset) + offset += a.size + return packed, [(order.index(d), len(groups[d]) == 1, starts[d][j], shape, size) + for d, j, shape, size in layout] + + +def _unpack(packed: tuple, layout: list) -> list: + out = [] + for g, single, start, shape, size in layout: + out.append(packed[g] if single else packed[g][start:start + size].reshape(shape)) + return out + + +def call_block(block, hidden: mx.array, **kwargs): + """``block(hidden, **kwargs)`` as one recomputed region. + + ``kwargs`` and the block's return value may mix arrays with plain Python + values (``None``, ints such as the SWA absolute position); only the + arrays cross the region boundary. Side outputs a module hands to + :func:`side_output_sink` (MoE router probabilities) are returned as + region outputs and delivered to their owner via ``owner.accept_side_output``, + so a loss term that reads them is differentiated through the + recomputation instead of keeping the block's activations alive. + """ + arrays = [hidden] + in_spec = _split(kwargs, arrays) + meta: dict = {} + + def fn(h, *rest): + prev = side_output_sink() + sink: list = [] + _STATE.sink = sink + try: + out = block(h, **_join(in_spec, iter(rest))) + finally: + _STATE.sink = prev + flat: list = [] + meta["spec"] = _split(out, flat) + meta["owners"] = [(owner, kind) for owner, kind, _ in sink] + flat += [value for _, _, value in sink] + packed, meta["layout"] = _pack(flat) + return packed + + outputs = _unpack(checkpointed(block, fn)(*arrays), meta["layout"]) + owners = meta["owners"] + n_main = len(outputs) - len(owners) + for (owner, kind), value in zip(owners, outputs[n_main:]): + owner.accept_side_output(kind, value) + return _join(meta["spec"], iter(outputs[:n_main])) diff --git a/chronos/mlx/inference.py b/chronos/mlx/inference.py index 43f9c2c..5a2fe1e 100644 --- a/chronos/mlx/inference.py +++ b/chronos/mlx/inference.py @@ -4,7 +4,9 @@ Mirrors ChronosInferenceEngine (PyTorch) but uses: - ChronosMLXModel for forward passes - MLXExpertStore (unified memory, no H2D stream) -- mx.compile() for static-graph acceleration +- Lazy MLX graphs evaluated per decode step (decode shapes change every + token, so the step is not traced with mx.compile; RoPE uses a shapeless + compiled kernel) - Async prefetch via Python threading (SSD → unified memory) Usage: diff --git a/chronos/mlx/model.py b/chronos/mlx/model.py index 54a68bb..4300518 100644 --- a/chronos/mlx/model.py +++ b/chronos/mlx/model.py @@ -6,7 +6,9 @@ - MLAAttentionMLX (even layers) / SlidingWindowAttentionMLX (odd layers) - ChronosMLXMOE for feed-forward - LookaheadRouter as a plain Linear projection -- mx.compile()-friendly: no Python control flow on runtime shapes +- Training micro steps are compiled by chronos/mlx/training/compiled.py + (one graph per depth/shape/config signature; Python decisions such as + the executed depth are resolved outside the trace) On Apple Silicon, mx.eval() materialises arrays into Metal shared memory. Prefetch = eagerly eval expert weights during the prefetch window. @@ -16,6 +18,7 @@ import mlx.nn as nn from chronos.mlx.attention import MLAAttentionMLX, SlidingWindowAttentionMLX +from chronos.mlx.checkpointing import call_block from chronos.mlx.moe import ChronosMLXMOE, FeedForwardMLX from chronos.mlx.recurrent import RecurrentControllerMLX, auxiliary_state from chronos.config_schema import resolve_recurrent_attention_mode, recurrent_cache_layout @@ -79,7 +82,7 @@ def __init__(self, layer_id: int, config): self.self_attn = SlidingWindowAttentionMLX(config) else: self.self_attn = SlidingWindowAttentionMLX(config) - self.self_attn.window_size = 2**63 - 1 + self.self_attn.window_size = None # dense causal GQA (a 2**63 window overflowed int32) if getattr(config, 'use_moe', True): self.mlp = ChronosMLXMOE(config) @@ -197,9 +200,14 @@ def _forward(self, input_ids, cache, available_expert_masks, logits_dtype, depth cfg = self.config object.__setattr__(self, "last_recurrent_state", None) + # Activation recomputation (Torch parity: training, no incoming cache). + recompute = self.training and bool(getattr(cfg, "activation_checkpointing", False)) + def run(i, hidden, slot=None, **kw): avail = available_expert_masks[i] if available_expert_masks is not None else None c = cache[i if slot is None else slot] if "kv_override" not in kw else None + if recompute and c is None: + return call_block(self.layers[i], hidden, mask=mask, cache=None, available_expert_mask=avail, **kw) return self.layers[i](hidden, mask=mask, cache=c, available_expert_mask=avail, **kw) if cfg.recurrent_enabled and cfg.recurrent_depth > 1: diff --git a/chronos/mlx/moe.py b/chronos/mlx/moe.py index 03b3051..a1d641c 100644 --- a/chronos/mlx/moe.py +++ b/chronos/mlx/moe.py @@ -4,11 +4,13 @@ Soft gating is expressed as weighted sum (compile-safe): out = avail[i] * expert_out + (1 - avail[i]) * shared_out -mx.compile() traces a single graph — no Python branches on mask values. +No Python branches on mask values, so a compiled training step +(chronos/mlx/training/compiled.py) keeps one graph per signature. """ import mlx.core as mx import mlx.nn as nn +from chronos.mlx.checkpointing import draw, recomputing, recorded, side_output_sink from chronos.mlx.moe_kernels import ( gather_mm_supported, grouped_swiglu, sparse_dispatch_ok, stack_expert_weights, training_availability, ) @@ -22,7 +24,7 @@ def _sparse_ok(dtype) -> bool: class FeedForwardMLX(nn.Module): - """SwiGLU FFN matching minimind's FeedForward structure.""" + """SwiGLU FFN matching ``chronos.model.kernel.FeedForward``.""" def __init__(self, hidden_size: int, intermediate_size: int): super().__init__() @@ -88,6 +90,13 @@ def last_router_probs(self): def iteration_router_probs(self): return self._iteration_router_probs + def accept_side_output(self, kind, value): + """Record one dispatch's router probabilities ``[B, S, E]``.""" + if kind != "probs": + raise ValueError(f"unknown MoE side output {kind!r}") + object.__setattr__(self, "_last_router_probs", value) + self._iteration_router_probs.append(value) + def reset_iteration_routing(self): object.__setattr__(self, "_iteration_router_probs", []) object.__setattr__(self, "_shared_topk_idx", None) @@ -100,7 +109,9 @@ def close_pack_scope(self): def _expert_stacks(self): scope = self._pack_scope - if scope is None: + if scope is None or recomputing(): + # A recomputation stacks its own primals, so the expert-weight + # gradients reach the region's inputs. return stack_expert_weights(self.experts) # Keyed on the live weight arrays, so an expert swapped in mid-forward # (lazy promotion) can never be served from a stale stack. @@ -188,21 +199,28 @@ def __call__( x_flat = x.reshape(-1, H) # [N, H] scores = mx.softmax(self.gate(x_flat).astype(mx.float32), axis=-1) # [N, E] - object.__setattr__(self, "_last_router_probs", scores.reshape(B, S, self.num_experts)) - self._iteration_router_probs.append(self._last_router_probs) - - # top-k routing - shared_idx = self._shared_topk_idx - if (self.router_scope == "shared" and shared_idx is not None - and shared_idx.shape[0] == scores.shape[0] and len(self._iteration_router_probs) > 1): + probs = scores.reshape(B, S, self.num_experts) + # Inside a recomputed block (``chronos.mlx.checkpointing``) side + # outputs leave the region as region outputs; the loss then reads the + # forward-pass arrays and never a recomputed copy. + sink = side_output_sink() + + # top-k routing (router_scope="shared": later iterations reuse the + # first dispatch's ids). Indices are discrete: gradients flow through + # topk_weight (gathered scores) only. ``recorded`` makes a + # recomputation follow the forward pass's routing exactly. + shared_idx = recorded(lambda: self._shared_topk_idx if self.router_scope == "shared" else None) + if shared_idx is not None and shared_idx.shape[0] == scores.shape[0]: topk_idx = shared_idx else: - topk_idx = mx.argpartition(-scores, kth=self.num_experts_per_tok - 1, axis=-1) - # Routing indices are discrete: gradients flow through topk_weight - # (gathered scores) only, never through the integer indices. - topk_idx = mx.stop_gradient(topk_idx[:, :self.num_experts_per_tok]) # [N, K] - if self.router_scope == "shared": + k = self.num_experts_per_tok + topk_idx = recorded(lambda: mx.stop_gradient(mx.argpartition(-scores, kth=k - 1, axis=-1)[:, :k])) + if self.router_scope == "shared" and self._shared_topk_idx is None and not recomputing(): object.__setattr__(self, "_shared_topk_idx", topk_idx) + if sink is not None: + sink.append((self, "probs", probs)) + else: + self.accept_side_output("probs", probs) topk_weight = mx.take_along_axis(scores, topk_idx, axis=-1) # [N, K] if self.norm_topk_prob: topk_weight = topk_weight / (topk_weight.sum(axis=-1, keepdims=True) + 1e-20) @@ -218,7 +236,7 @@ def __call__( if self.training and self.fallback_mask_prob > 0.0 and shared is not None: # Cache-miss simulation (PyTorch parity): an unavailable routed # expert is served by the shared expert, as at offload inference. - fallback = (training_availability(self.num_experts, self.fallback_mask_prob), shared) + fallback = (draw(lambda: training_availability(self.num_experts, self.fallback_mask_prob)), shared) y = grouped_swiglu(x_flat, topk_idx, topk_weight, *self._expert_stacks(), fallback=fallback) if shared is not None: diff --git a/chronos/mlx/recurrent.py b/chronos/mlx/recurrent.py index cc8b795..0002155 100644 --- a/chronos/mlx/recurrent.py +++ b/chronos/mlx/recurrent.py @@ -31,6 +31,9 @@ def adaptive_active(self, cfg, training): or self.training_progress >= cfg.depth_schedule_warmup_fraction) def execution_depth(self, cfg, training): + pinned = getattr(self, "_pinned_depth", None) + if pinned is not None and training: + return pinned # sampled once per step by a compiled training step ceiling = cfg.recurrent_depth if training else cfg.max_inference_depth if not training or cfg.recurrent_strategy == "fixed" or cfg.recurrent_depth_jitter == 0: return ceiling diff --git a/chronos/mlx/training/compiled.py b/chronos/mlx/training/compiled.py new file mode 100644 index 0000000..b4422af --- /dev/null +++ b/chronos/mlx/training/compiled.py @@ -0,0 +1,242 @@ +"""``mx.compile`` for one MLX training micro step (loss + gradients). + +Measured on mlx 0.32.4 (see the 0.3.2a0 CHANGELOG): the compiled step gives +the eager loss and gradients up to FP32 summation order (<= 1.7e-5 relative +on CPU FP32), consumes the RNG stream identically, and runs 1.08-1.18x +faster on the 50M en->zh config because elementwise chains (RMSNorm, +SwiGLU, softmax/CE, router regularizers) fuse into fewer kernels. + +What compiling needs here, and how it is handled: + +* **Mutable state.** Model parameters and ``mx.random.state`` are declared + as compile inputs *and* outputs, so a step reads the live weights and the + dropout / cache-miss draws advance the global RNG exactly as eagerly. +* **Python decisions are part of the graph.** The executed recurrent depth, + whether ACT mixing is active, the training flag, input shapes and dtypes + all change the traced graph. Each distinct signature gets its own + compiled function; ``depth`` is sampled once per step outside the trace + and pinned for the call (``depth_override``-free: see ``_pinned_depth``). +* **Side outputs.** The loss reads, and telemetry logs, arrays the forward + pass stores on modules (router probabilities, loss components, recurrent + diagnostics). Inside a trace those are placeholders, so they are returned + as explicit outputs and written back after the call. +* **Fallback.** Anything else (multimodal inputs, a lazy expert, an + unexpected trace error) runs eagerly; ``mode="auto"`` logs why once. +""" +from __future__ import annotations + +import contextlib +from typing import Callable + +import mlx.core as mx +import mlx.nn as nn + +# ── side outputs the regularized loss and the telemetry read ────────────── + +_MODEL_SIDE = ("_last_loss_components", "_last_stage_metrics", "last_recurrent_state", "_last_lookahead_probs") + + +def _moes(model): + from chronos.mlx.moe import ChronosMLXMOE + return [layer.mlp for layer in getattr(model, "layers", []) if isinstance(getattr(layer, "mlp", None), ChronosMLXMOE)] + + +def _split(obj, arrays: list): + """Array leaves -> index placeholders; everything else kept (bools, ints).""" + if isinstance(obj, mx.array): + arrays.append(obj) + return ("__array__", len(arrays) - 1) + if isinstance(obj, dict): + return {k: _split(v, arrays) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return type(obj)(_split(v, arrays) for v in obj) + return obj + + +def _join(spec, arrays): + if isinstance(spec, tuple) and len(spec) == 2 and spec[0] == "__array__": + return arrays[spec[1]] + if isinstance(spec, dict): + return {k: _join(v, arrays) for k, v in spec.items()} + if isinstance(spec, (list, tuple)): + return type(spec)(_join(v, arrays) for v in spec) + return spec + + +def _side_get(model) -> dict: + return {"model": {k: getattr(model, k, None) for k in _MODEL_SIDE}, + "moe": [[m._last_router_probs, list(m._iteration_router_probs), m._shared_topk_idx] for m in _moes(model)]} + + +def _side_set(model, side: dict) -> None: + for k, v in side["model"].items(): + object.__setattr__(model, k, v) + for m, (last, probs, topk) in zip(_moes(model), side["moe"]): + object.__setattr__(m, "_last_router_probs", last) + object.__setattr__(m, "_iteration_router_probs", list(probs)) + object.__setattr__(m, "_shared_topk_idx", topk) + + +def _static(value): + """Hashable stand-in for a non-array argument, or ``None`` when unsupported.""" + if value is None or isinstance(value, (bool, int, float, str)): + return ("v", value) + if isinstance(value, nn.Module): + return ("m", id(value)) + return None + + +def env_mode(default: str = "auto") -> str: + import os + mode = (os.environ.get("CHRONOS_MLX_COMPILE", "") or default).strip().lower() + return {"1": "on", "true": "on", "0": "off", "false": "off", "none": "off"}.get(mode, mode) + + +class CompiledStep: + """``nn.value_and_grad(model, loss_fn)`` with a compiled graph per signature. + + Call it exactly like the eager ``grad_fn``: ``loss, grads = step(*args)``. + ``args`` may mix arrays, ``None``/scalars and the model itself. After the + call the model's side outputs (router probabilities, loss components, + recurrent diagnostics) hold real arrays, as after an eager call. + + ``mode``: ``"auto"`` compiles and falls back to eager (logging the + reason once) when a signature cannot be compiled; ``"on"`` raises + instead; ``"off"`` is the eager ``grad_fn``. + """ + + def __init__(self, model, loss_fn: Callable, *, mode: str = "auto", log=print, max_graphs: int = 128): + from collections import OrderedDict + mode = str(mode or "auto").lower() + if mode not in {"auto", "on", "off"}: + raise ValueError(f"mlx compile mode must be auto|on|off, got {mode!r}") + self.model, self.mode, self.log, self.max_graphs = model, mode, log, int(max_graphs) + self.eager = nn.value_and_grad(model, loss_fn) + self._graphs: OrderedDict = OrderedDict() + self._specs: dict = {} + self._failed: set = set() + self._logged: set = set() + self.stats = {"compiled": 0, "eager": 0, "traces": 0} + + # -- eligibility --------------------------------------------------------- + def _eager_reason(self, args) -> str | None: + if self.mode == "off": + return "disabled" + from chronos.mlx.moe import LazyFeedForwardMLX + model = self.model + if getattr(model, "multimodal_fusion", None) is not None: + return "multimodal model (encoders run eagerly)" + if any(isinstance(e, LazyFeedForwardMLX) for m in _moes(model) for e in m.experts): + return "lazy (paged) experts" + if any(not isinstance(a, mx.array) and _static(a) is None for a in args): + return "non-array batch argument" + return None + + def _note(self, reason: str) -> None: + if reason not in self._logged: + self._logged.add(reason) + self.log(f"[mlx-compile] eager step: {reason}") + + def _plan(self, args): + """(graph key, depth to pin) for this call; Python decisions included.""" + model, cfg = self.model, self.model.config + depth = adaptive = None + if cfg.recurrent_enabled and cfg.recurrent_depth > 1: + controller = model.recurrent_controller + depth = int(controller.execution_depth(cfg, model.training)) + adaptive = bool(controller.adaptive_active(cfg, model.training)) + sig = tuple(("a", tuple(a.shape), str(a.dtype)) if isinstance(a, mx.array) else _static(a) for a in args) + # Plain config values decide branches too (and callers may toggle them). + conf = tuple(sorted((k, v) for k, v in vars(cfg).items() + if v is None or isinstance(v, (bool, int, float, str)))) + return (sig, bool(model.training), depth, adaptive, conf), depth + + def _prewarm(self, args) -> None: + """Grow RoPE tables before tracing, so no module stores a traced table.""" + length = max([a.shape[1] for a in args if isinstance(a, mx.array) and a.ndim == 2] or [0]) + if length: + for _, module in self.model.named_modules(): + grow = getattr(module, "_ensure_rope_cache", None) + if grow is not None: + grow(length) + + # -- compiled graph ------------------------------------------------------ + def _build(self, key, args): + model = self.model + spec = [None if isinstance(a, mx.array) else a for a in args] + slots = [i for i, a in enumerate(args) if isinstance(a, mx.array)] + meta: dict = {} + state = [model.state, mx.random.state] + + def step(*arrays): + full = list(spec) + for i, a in zip(slots, arrays): + full[i] = a + loss, grads = self.eager(*full) + flat: list = [] + meta["side"] = _split(_side_get(model), flat) + return loss, grads, flat + + fn = mx.compile(step, inputs=state, outputs=state) + return fn, meta + + @contextlib.contextmanager + def _pin(self, depth): + controller = getattr(self.model, "recurrent_controller", None) + if controller is None or depth is None: + yield + return + object.__setattr__(controller, "_pinned_depth", depth) + try: + yield + finally: + object.__setattr__(controller, "_pinned_depth", None) + + def __call__(self, *args): + reason = self._eager_reason(args) + key = depth = None + if reason is None: + key, depth = self._plan(args) + if key in self._failed: + reason = "signature failed to compile earlier" + if reason is not None: + if reason != "disabled": + self._note(reason) + self.stats["eager"] += 1 + with self._pin(depth): + return self.eager(*args) + hit = self._graphs.get(key) + if hit is None: + self._prewarm(args) + hit = self._build(key, args) + self._graphs[key] = hit + self.stats["traces"] += 1 + while len(self._graphs) > self.max_graphs: + self._graphs.popitem(last=False) + else: + self._graphs.move_to_end(key) + fn, meta = hit + arrays = [a for a in args if isinstance(a, mx.array)] + try: + with self._pin(depth): + loss, grads, flat = fn(*arrays) + except Exception as exc: # trace-time failure: never lose the step + self._graphs.pop(key, None) + if self.mode == "on": + raise + self._failed.add(key) + self._note(f"compile failed ({type(exc).__name__}: {str(exc)[:160]})") + self.stats["eager"] += 1 + with self._pin(depth): + return self.eager(*args) + _side_set(self.model, _join(meta["side"], flat)) + self.stats["compiled"] += 1 + return loss, grads + + +def make_grad_fn(model, loss_fn: Callable, *, mode: str | None = None, log=print): + """Training ``grad_fn``: compiled (``CHRONOS_MLX_COMPILE``/``mode``, default auto) or eager.""" + mode = env_mode() if mode is None else str(mode).lower() + if mode == "off": + return nn.value_and_grad(model, loss_fn) + return CompiledStep(model, loss_fn, mode=mode, log=log) diff --git a/chronos/mlx/training/longrun.py b/chronos/mlx/training/longrun.py index b4f1d63..b4730ea 100644 --- a/chronos/mlx/training/longrun.py +++ b/chronos/mlx/training/longrun.py @@ -342,6 +342,10 @@ def __init__(self, spec: RunSpec, log=print): self.telemetry = telemetry self.heartbeat = telemetry.Heartbeat(spec.run_dir) self.precision = getattr(self.model, "_chronos_precision", None) + # mx.compile of the micro step: recipe ``extra.mlx_compile`` (auto|on|off), + # else CHRONOS_MLX_COMPILE, else auto. + from chronos.mlx.training.compiled import env_mode + self.compile_mode = env_mode(str((spec.extra or {}).get("mlx_compile") or "auto")) self.runtime = telemetry.runtime_info(self.model, spec, self.precision) if self.cache_limit is not None: self.runtime["mlx_cache_limit_gb"] = self.cache_limit["limit_gb"] @@ -494,7 +498,10 @@ def run_phase(self, pi: int, deadline_s: float, max_steps: int | None = None): if self.state["rng"].get(phase.name): source.set_rng_state(self.state["rng"][phase.name]) accum = max(1, phase.tokens_per_step // phase.micro_tokens) - grad_fn = nn.value_and_grad(self.model, self._loss_fn) + if getattr(self, "_grad_fn", None) is None: # one compiled step (and its graphs) per run + from chronos.mlx.training.compiled import make_grad_fn + self._grad_fn = make_grad_fn(self.model, self._loss_fn, mode=self.compile_mode, log=self.log) + grad_fn = self._grad_fn controller = self.model.recurrent_controller top_k = int(self.config.num_experts_per_tok) t_phase = time.monotonic() diff --git a/chronos/mlx/training/trainer.py b/chronos/mlx/training/trainer.py index dc4b0b6..0e0631d 100644 --- a/chronos/mlx/training/trainer.py +++ b/chronos/mlx/training/trainer.py @@ -1172,7 +1172,23 @@ def loss_fn(m, batch): return _grpo_stage_loss(m, reference, batch, beta=beta, router_ref=router_ref) raise ValueError(f"unknown MLX stage: {stage}") - grad_fn = nn.value_and_grad(model, loss_fn) + eager_grad = nn.value_and_grad(model, loss_fn) + text_grad = None + if stage in {"pretrain", "sft"}: + # Text batches take the compiled micro step (``CHRONOS_MLX_COMPILE``, + # default auto); media batches and other stages stay eager. + from chronos.mlx.training.compiled import make_grad_fn + text_grad = make_grad_fn(model, lambda m, ids, labels: _ce_stage_loss(m, ids, labels), + mode=getattr(args, "mlx_compile", None) if args is not None else None, + log=print) + + def grad_fn(m, batch): + if text_grad is not None: + ids, labels, media, _mask = batch_modalities(batch) + if not media: + return text_grad(m, _ids_to_mx(ids), _ids_to_mx(labels)) + return eager_grad(m, batch) + _emit({ "event": "start", "stage": stage, diff --git a/chronos/model/checkpoint.py b/chronos/model/checkpoint.py index 3640ffa..4b3e22f 100644 --- a/chronos/model/checkpoint.py +++ b/chronos/model/checkpoint.py @@ -103,7 +103,8 @@ def resolve_checkpoint_tokenizer(checkpoint_path: str | None = None, explicit: s 1. ``explicit``; 2. ``/tokenizer`` (bundle layout); 3. the ``tokenizer.path`` recorded in the sidecar config; - 4. the minimind tokenizer, only when ``fallback`` is set. + 4. the bundled legacy 6400-token tokenizer (``chronos.deps.get_tokenizer_path``), + only when ``fallback`` is set. A path that exists but holds no ``tokenizer.json`` raises an error. It is never silently skipped, because a wrong vocabulary corrupts training without failing. diff --git a/chronos/model/config.py b/chronos/model/config.py index c9cdfaf..8968338 100644 --- a/chronos/model/config.py +++ b/chronos/model/config.py @@ -1,12 +1,9 @@ -import sys -import chronos.deps # ensure minimind on sys.path +from chronos.model.kernel import ChronosBaseConfig -from model.model_minimind import MiniMindConfig - -class ChronosConfig(MiniMindConfig): +class ChronosConfig(ChronosBaseConfig): """ - Extends MiniMindConfig with Project Chronos-specific fields: + Extends ChronosBaseConfig with Project Chronos-specific fields: - lookahead_steps: how many future token steps to predict expert routing for - num_shared_experts: experts always resident in VRAM as fallback - lambda_balance / lambda_temporal: loss coefficients @@ -231,7 +228,7 @@ def __init__(self, **kwargs): self.offload_miss_policy_default: str = kwargs.pop("offload_miss_policy_default", "on_demand") self.recommended_resident_experts: int | None = kwargs.pop("recommended_resident_experts", None) self.offload_working_set_experts: int | None = kwargs.pop("offload_working_set_experts", None) - # M7: regularization defaults. MiniMindConfig defaults dropout=0.0 + # M7: regularization defaults. ChronosBaseConfig defaults dropout=0.0 # and weight_decay isn't even a config field — both major causes # of overfitting on small corpora. Override here. kwargs.setdefault("dropout", 0.1) diff --git a/chronos/model/hf_io.py b/chronos/model/hf_io.py index e7f4409..7c4e363 100644 --- a/chronos/model/hf_io.py +++ b/chronos/model/hf_io.py @@ -68,7 +68,7 @@ def save_chronos_pretrained( if paging is not None: paging.materialize_all() - # MiniMind/Chronos tie ``lm_head.weight`` to ``model.embed_tokens.weight`` + # Chronos ties ``lm_head.weight`` to ``model.embed_tokens.weight`` # but doesn't declare it via ``_tied_weights_keys``, which trips # safetensors' shared-tensor detector. Clone lm_head once before saving; # restore the tie after. @@ -115,7 +115,7 @@ def load_chronos_pretrained( from chronos.model.model_chronos import ChronosForCausalLM model = ChronosForCausalLM.from_pretrained(save_directory, **from_pretrained_kwargs) - # MiniMind/Chronos precompute RoPE frequencies as *persistent* buffers. + # Chronos precomputes RoPE frequencies from config (FP32, not buffers). # When the checkpoint is saved as fp16 and reloaded, these deterministic # buffers get truncated to zeros/NaNs. Recompute them from scratch — # they are a pure function of config. @@ -131,22 +131,15 @@ def load_chronos_pretrained( def _rehydrate_rope_buffers(model) -> None: - """Recompute ``freqs_cos`` and ``freqs_sin`` on the inner ChronosModel. - These are deterministic from config; the precompute logic lives inside - ``ChronosModel.__init__``. We invoke it via a fresh ChronosModel and - copy the buffers across. Cheap — only runs once per load.""" - try: - inner = getattr(model, "model", None) - if inner is None: - return - cfg = model.config - from chronos.model.model_chronos import ChronosModel - fresh = ChronosModel(cfg) - if hasattr(inner, "freqs_cos") and hasattr(fresh, "freqs_cos"): - inner.freqs_cos = fresh.freqs_cos - if hasattr(inner, "freqs_sin") and hasattr(fresh, "freqs_sin"): - inner.freqs_sin = fresh.freqs_sin - except Exception: - # Rehydration is best-effort; fall through silently so an unexpected - # config shape doesn't block the load. - pass + """Point the inner model's ``freqs_cos/sin`` at fresh FP32 tables. + + They are a pure function of config (``kernel.rope_tables``), so no model + is rebuilt; a half-precision save can never truncate them. + """ + inner = getattr(model, "model", None) + if inner is None or not hasattr(inner, "freqs_cos"): + return + from chronos.model.kernel import rope_tables + cfg = model.config + inner.freqs_cos, inner.freqs_sin = rope_tables( + cfg.head_dim, cfg.max_position_embeddings, cfg.rope_theta, cfg.rope_scaling, inner.freqs_cos.device) diff --git a/chronos/model/hybrid_attention.py b/chronos/model/hybrid_attention.py index dc69dde..4a16c21 100644 --- a/chronos/model/hybrid_attention.py +++ b/chronos/model/hybrid_attention.py @@ -13,9 +13,6 @@ Even layers: MLA for global context with compressed cache Odd layers: SlidingWindow for local patterns with bounded cache """ -import sys -import chronos.deps # ensure minimind on sys.path - import math import torch import torch.nn as nn @@ -295,6 +292,9 @@ def _absorbed_attention(self, q_nope, q_rope, c_kv, k_rope, mask, is_causal, dro # ── Sliding Window Attention ────────────────────────────────────── +_CONFIG_WINDOW = object() # sentinel: take ``sliding_window_size`` from the config + + class SlidingWindowAttention(_RopeTables, nn.Module): """ Local sliding window attention. KV cache is capped at window_size tokens, @@ -304,14 +304,15 @@ class SlidingWindowAttention(_RopeTables, nn.Module): During decode: KV cache evicts oldest tokens beyond window_size. """ - def __init__(self, config: ChronosConfig): + def __init__(self, config: ChronosConfig, window_size=_CONFIG_WINDOW): super().__init__() self.n_heads = config.num_attention_heads self.n_kv_heads = config.num_key_value_heads or config.num_attention_heads self.head_dim = config.head_dim self.n_rep = self.n_heads // self.n_kv_heads self.dropout = config.dropout - self.window_size = getattr(config, 'sliding_window_size', 2048) + # ``None`` = no window (GlobalAttention): full causal GQA, no eviction. + self.window_size = getattr(config, 'sliding_window_size', 2048) if window_size is _CONFIG_WINDOW else window_size # Extra retained cache entries (outside the attention window, masked # out) so speculative decoding can roll back rejected draft tokens. self.cache_slack = 0 @@ -362,16 +363,18 @@ def forward(self, x, position_embeddings, past_key_value=None, use_cache=False, # Keep full keys for this chunk's queries, then bound the persistent # cache. Track absolute positions after eviction for correct RoPE. - keep = self.window_size + int(self.cache_slack) + keep = None if self.window_size is None else self.window_size + int(self.cache_slack) past_kv = None if use_cache and self.kv_int8 and kv_override is None: # Quantize only the new tokens (each token once); evict afterwards. prev = past_key_value if past_key_value is not None else (None, None) qk = kv_store(new_k, prev[0], True, self.kv_group) qv = kv_store(new_v, prev[1], True, self.kv_group) - past_kv = (qk[:, max(0, qk.shape[1] - keep):], qv[:, max(0, qv.shape[1] - keep):], past_len + S) + if keep is not None: + qk, qv = qk[:, max(0, qk.shape[1] - keep):], qv[:, max(0, qv.shape[1] - keep):] + past_kv = (qk, qv, past_len + S) elif use_cache: - past_kv = (xk[:, -keep:], xv[:, -keep:], past_len+S) + past_kv = (xk, xv, past_len + S) if keep is None else (xk[:, -keep:], xv[:, -keep:], past_len + S) shared_kv = (xk, xv, past_len + S) T = xk.shape[1] @@ -386,12 +389,28 @@ def forward(self, x, position_embeddings, past_key_value=None, use_cache=False, return self.resid_dropout(self.o_proj(out)), past_kv +class GlobalAttention(SlidingWindowAttention): + """Full causal GQA, used when ``use_hybrid_attention=False``. + + It is the sliding-window kernel without a window. Parameter names + (``q/k/v/o_proj``, ``q_norm``, ``k_norm``) and the ``(k, v, abs_pos)`` + cache are the same, so every recurrent mode works on it, as do int8 KV, + padding masks and speculative rollback. + """ + + def __init__(self, config: ChronosConfig): + super().__init__(config, window_size=None) + + def make_attention(layer_id: int, config: ChronosConfig) -> nn.Module: """ - Layer assignment: + Layer assignment (``use_hybrid_attention``, default True): even layers → MLAAttention (global context, compressed KV cache) odd layers → SlidingWindowAttention (local patterns, bounded KV cache) + With hybrid attention off, every layer is GlobalAttention (full GQA). """ + if not getattr(config, "use_hybrid_attention", True): + return GlobalAttention(config) if layer_id % 2 == 0: return MLAAttention(config) return SlidingWindowAttention(config) diff --git a/chronos/model/kernel.py b/chronos/model/kernel.py new file mode 100644 index 0000000..bb980de --- /dev/null +++ b/chronos/model/kernel.py @@ -0,0 +1,317 @@ +"""Chronos native kernel primitives (replaces the former minimind dependency). + +Before 0.3.2 Chronos imported its config base, RoPE tables, SwiGLU FFN and a +dense GQA attention from a minimind checkout that ``chronos.deps`` cloned +from an unpinned ``master`` at import time and pushed onto ``sys.path``. +This module owns those pieces. Field names, defaults, parameter names and +RoPE numerics are kept identical, so every existing checkpoint and config +sidecar loads unchanged (``tests/test_native_kernel.py`` checks this). + +What changed by design: + +* RoPE tables are FP32 and bit-identical to the old ones, but one table per + ``(dim, base, scaling, device)`` is shared by every layer (``rope_tables``) + instead of each attention layer holding its own copy. +* ``SwiGLU`` uses ``F.silu`` directly (no ``ACT2FN`` lookup per call) and keeps + the ``gate_proj/up_proj/down_proj`` names expert pages are stored under. +* Logging is a sink registry (``add_log_sink``), so callers capture trainer + output without monkey-patching a module attribute. +""" +from __future__ import annotations + +import math +import os +import random +import threading +from contextlib import contextmanager +from typing import Callable + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import PretrainedConfig + +# ── Config base ─────────────────────────────────────────────────────────── + +YARN_DEFAULT = { + "beta_fast": 32, "beta_slow": 1, "factor": 16, + "original_max_position_embeddings": 2048, "attention_factor": 1.0, "type": "yarn", +} + + +class ChronosBaseConfig(PretrainedConfig): + """Dense-transformer fields every Chronos model shares. + + Names and defaults match the pre-0.3.2 base so ``config.json`` sidecars + written by any earlier release resolve to the same values. + """ + + model_type = "chronos_base" + + def __init__(self, hidden_size: int = 768, num_hidden_layers: int = 8, use_moe: bool = False, **kwargs): + super().__init__(**kwargs) + g = kwargs.get + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.use_moe = use_moe + self.dropout = g("dropout", 0.0) + self.vocab_size = g("vocab_size", 6400) + self.bos_token_id = g("bos_token_id", 1) + self.eos_token_id = g("eos_token_id", 2) + self.flash_attn = g("flash_attn", True) + self.num_attention_heads = g("num_attention_heads", 8) + self.num_key_value_heads = g("num_key_value_heads", 4) + self.head_dim = g("head_dim", self.hidden_size // self.num_attention_heads) + self.hidden_act = g("hidden_act", "silu") + self.intermediate_size = g("intermediate_size", math.ceil(hidden_size * math.pi / 64) * 64) + self.max_position_embeddings = g("max_position_embeddings", 32768) + self.rms_norm_eps = g("rms_norm_eps", 1e-6) + self.rope_theta = g("rope_theta", 1e6) + self.tie_word_embeddings = g("tie_word_embeddings", True) + self.inference_rope_scaling = g("inference_rope_scaling", False) + # YaRN is switched on by ``inference_rope_scaling``; transformers 5 + # validates an explicit ``rope_scaling`` kwarg against its own schema. + self.rope_scaling = dict(YARN_DEFAULT) if self.inference_rope_scaling else None + # MoE fields (ignored when use_moe is False). + self.num_experts = g("num_experts", 4) + self.num_experts_per_tok = g("num_experts_per_tok", 1) + self.moe_intermediate_size = g("moe_intermediate_size", self.intermediate_size) + self.norm_topk_prob = g("norm_topk_prob", True) + self.router_aux_loss_coef = g("router_aux_loss_coef", 5e-4) + + +# ── RoPE (FP32, rotate-half layout, optional YaRN) ──────────────────────── + +def _inv_freq(dim: int, base: float, scaling: dict | None, end: int) -> tuple[torch.Tensor, float]: + cpu = torch.device("cpu") # explicit: callers may run under ``with torch.device('meta')`` + freqs = 1.0 / (base ** (torch.arange(0, dim, 2, device=cpu)[: dim // 2].float() / dim)) + attn_factor = 1.0 + if scaling is not None: + # YaRN: f'(i) = f(i) * ((1 - g) + g / s), g a linear ramp over the band. + orig_max = scaling.get("original_max_position_embeddings", 2048) + factor = scaling.get("factor", 16) + beta_fast, beta_slow = scaling.get("beta_fast", 32.0), scaling.get("beta_slow", 1.0) + attn_factor = scaling.get("attention_factor", 1.0) + if end / orig_max > 1.0: + def inv_dim(b): + return (dim * math.log(orig_max / (b * 2 * math.pi))) / (2 * math.log(base)) + low = max(math.floor(inv_dim(beta_fast)), 0) + high = min(math.ceil(inv_dim(beta_slow)), dim // 2 - 1) + ramp = torch.clamp((torch.arange(dim // 2, device=cpu).float() - low) / max(high - low, 0.001), 0, 1) + freqs = freqs * (1 - ramp + ramp / factor) + return freqs, attn_factor + + +def precompute_freqs_cis(dim: int, end: int = 32 * 1024, rope_base: float = 1e6, + rope_scaling: dict | None = None) -> tuple[torch.Tensor, torch.Tensor]: + """FP32 duplicated cos/sin tables ``[end, dim]`` (rotate-half layout), on CPU. + + Same signature and bit-identical output as the pre-0.3.2 helper, so the + MLX mirror (``chronos.mlx.attention._rope_freqs``) and every trained + checkpoint keep their RoPE. + """ + freqs, attn_factor = _inv_freq(int(dim), float(rope_base), rope_scaling, int(end)) + angles = torch.outer(torch.arange(int(end), device=freqs.device), freqs).float() + # One thread: torch splits cos/sin across threads above a size threshold, + # and the vector/scalar split points then differ by one ulp from a + # shorter table (measured on torch 2.10, 18 threads). Single-threaded, + # every row depends on its angle only, so a cached table's prefix equals + # a freshly computed shorter one bit for bit. Built once per process. + threads = torch.get_num_threads() + torch.set_num_threads(1) + try: + c, s = torch.cos(angles), torch.sin(angles) + finally: + torch.set_num_threads(threads) + cos = torch.cat([c, c], dim=-1) * attn_factor + sin = torch.cat([s, s], dim=-1) * attn_factor + return cos, sin + + +_ROPE_LOCK = threading.Lock() +_ROPE_TABLES: dict[tuple, tuple[torch.Tensor, torch.Tensor]] = {} + + +def _scaling_key(scaling: dict | None): + return None if not scaling else tuple(sorted((str(k), str(v)) for k, v in scaling.items())) + + +def rope_tables(dim: int, length: int, base: float, scaling, device) -> tuple[torch.Tensor, torch.Tensor]: + """Shared FP32 cos/sin tables with at least ``length`` rows on ``device``. + + One table per ``(dim, base, scaling, device)`` serves every layer and + grows geometrically. YaRN rescales the frequencies only when the table + is longer than ``original_max_position_embeddings``, so scaled tables are + keyed by that regime as well, and growth never crosses the boundary. Rows + are therefore bit-identical to ``precompute_freqs_cis(dim, length)``. + """ + dim, length = int(dim), int(length) + dev = torch.device(device) if device is not None else torch.device("cpu") + cap = None + regime = None + if scaling: + orig = scaling.get("original_max_position_embeddings", 2048) + regime = length / orig > 1.0 + cap = None if regime else int(orig) # unscaled regime stops at orig rows + key = (dim, float(base), _scaling_key(scaling), str(dev), regime) + with _ROPE_LOCK: + hit = _ROPE_TABLES.get(key) + if hit is not None and hit[0].shape[0] >= length: + return hit + rows = max(length, 2 * (hit[0].shape[0] if hit is not None else 0)) + if cap is not None: + rows = max(length, min(rows, cap)) + cos, sin = precompute_freqs_cis(dim, rows, base, scaling) + out = (cos.to(device=dev, dtype=torch.float32), sin.to(device=dev, dtype=torch.float32)) + _ROPE_TABLES[key] = out + return out + + +def clear_rope_tables() -> None: + with _ROPE_LOCK: + _ROPE_TABLES.clear() + + +def rotate_half(x: torch.Tensor) -> torch.Tensor: + half = x.shape[-1] // 2 + return torch.cat((-x[..., half:], x[..., :half]), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim: int = 1): + """Rotate q/k with tables broadcast on ``unsqueeze_dim``; one rounding each.""" + c, s = cos.unsqueeze(unsqueeze_dim), sin.unsqueeze(unsqueeze_dim) + return (q * c + rotate_half(q) * s).to(q.dtype), (k * c + rotate_half(k) * s).to(k.dtype) + + +def repeat_kv(x: torch.Tensor, n_rep: int) -> torch.Tensor: + """[B, T, nKV, d] -> [B, T, nKV * n_rep, d] (only for kernels without native GQA).""" + if n_rep == 1: + return x + b, t, h, d = x.shape + return x[:, :, :, None, :].expand(b, t, h, n_rep, d).reshape(b, t, h * n_rep, d) + + +# ── SwiGLU feed-forward ─────────────────────────────────────────────────── + +_ACTIVATIONS: dict[str, Callable[[torch.Tensor], torch.Tensor]] = { + "silu": F.silu, "swish": F.silu, "gelu": F.gelu, "relu": F.relu, + "gelu_pytorch_tanh": lambda x: F.gelu(x, approximate="tanh"), +} + + +def activation(name: str) -> Callable[[torch.Tensor], torch.Tensor]: + if name in _ACTIVATIONS: + return _ACTIVATIONS[name] + from transformers.activations import ACT2FN # rare activations only + return ACT2FN[name] + + +class FeedForward(nn.Module): + """Gated FFN ``down(act(gate(x)) * up(x))``. + + Parameter names are the on-disk contract (dense FFN, routed and shared + experts, ``.ctsr`` expert pages), so they must not change. + """ + + def __init__(self, config, intermediate_size: int | None = None): + super().__init__() + inter = int(intermediate_size or config.intermediate_size) + self.gate_proj = nn.Linear(config.hidden_size, inter, bias=False) + self.down_proj = nn.Linear(inter, config.hidden_size, bias=False) + self.up_proj = nn.Linear(config.hidden_size, inter, bias=False) + self.act_fn = activation(getattr(config, "hidden_act", "silu")) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + +SwiGLU = FeedForward + + +# ── Training utilities ──────────────────────────────────────────────────── + +def is_main_process() -> bool: + import torch.distributed as dist + return not (dist.is_available() and dist.is_initialized()) or dist.get_rank() == 0 + + +_LOG_SINKS: list[Callable[[str], None]] = [] + + +def Logger(content) -> None: # noqa: N802 (historic trainer API name) + """Print on the main process and forward the line to registered sinks.""" + if not is_main_process(): + return + print(content) + for sink in list(_LOG_SINKS): + try: + sink(str(content)) + except Exception: + pass + + +@contextmanager +def add_log_sink(sink: Callable[[str], None]): + """Receive every ``Logger`` line while the context is active.""" + _LOG_SINKS.append(sink) + try: + yield sink + finally: + try: + _LOG_SINKS.remove(sink) + except ValueError: + pass + + +def setup_seed(seed: int) -> None: + """Seed Python, NumPy and torch (all devices); deterministic cuDNN.""" + import numpy as np + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def _unwrap(module): + module = getattr(module, "module", module) # DDP + return getattr(module, "_orig_mod", module) # torch.compile + + +def save_resume_state(path: str, *, model, optimizer=None, epoch: int = 0, step: int = 0, **extra) -> str: + """Atomically write a resume file (weights in their dtype + optimizer + counters). + + The release checkpoint itself is written by + ``chronos.model.checkpoint.save_state_dict_with_config``; this file only + carries what is needed to continue training. + """ + import torch.distributed as dist + os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True) + state = { + "model": {k: v.detach().cpu() for k, v in _unwrap(model).state_dict().items()}, + "optimizer": optimizer.state_dict() if optimizer is not None else None, + "epoch": int(epoch), "step": int(step), + "world_size": dist.get_world_size() if dist.is_available() and dist.is_initialized() else 1, + } + for key, value in extra.items(): + if value is not None: + state[key] = _unwrap(value).state_dict() if hasattr(value, "state_dict") else value + tmp = path + ".tmp" + torch.save(state, tmp) + os.replace(tmp, path) + return path + + +def load_resume_state(path: str): + """Load a resume file; ``step`` is rescaled when the world size changed.""" + import torch.distributed as dist + if not os.path.exists(path): + return None + data = torch.load(path, map_location="cpu", weights_only=False) + saved = int(data.get("world_size", 1) or 1) + current = dist.get_world_size() if dist.is_available() and dist.is_initialized() else 1 + if saved != current: + data["step"] = int(data.get("step", 0)) * saved // current + return data diff --git a/chronos/model/model_chronos.py b/chronos/model/model_chronos.py index a7ec7d7..c0d06fb 100644 --- a/chronos/model/model_chronos.py +++ b/chronos/model/model_chronos.py @@ -1,15 +1,10 @@ -import sys -import chronos.deps # ensure minimind on sys.path - import math import torch import torch.nn as nn import torch.nn.functional as F from transformers import PreTrainedModel, GenerationMixin -from model.model_minimind import ( - precompute_freqs_cis, Attention, FeedForward, -) +from .kernel import FeedForward, rope_tables from .config import ChronosConfig from .precision_ops import RMSNorm, lm_head_logits from .lookahead_router import LookaheadRouter @@ -60,10 +55,7 @@ class ChronosBlock(nn.Module): def __init__(self, layer_id: int, config: ChronosConfig): super().__init__() self.layer_id = layer_id - if getattr(config, 'use_hybrid_attention', True): - self.self_attn = make_attention(layer_id, config) - else: - self.self_attn = Attention(config) + self.self_attn = make_attention(layer_id, config) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.mlp = ChronosMOEFeedForward(config) if config.use_moe else FeedForward(config) @@ -133,7 +125,7 @@ def apply_repetition_penalty(logits, input_ids, penalty: float): class ChronosModel(nn.Module): """ - Backbone: identical to MiniMindModel but: + Backbone: pre-norm decoder (native kernel, ``chronos.model.kernel``) that: - Uses ChronosBlock (with ChronosMOEFeedForward) - Inserts LookaheadRouter after block 0 - Exposes router_probs_seq for temporal loss computation @@ -154,16 +146,11 @@ def __init__(self, config: ChronosConfig): self.recurrent_controller = RecurrentController(config) # Pure-text models carry no image/audio encoder parameters or compute. self.multimodal_fusion = MultimodalFusion(config) if config.modalities else None - freqs_cos, freqs_sin = precompute_freqs_cis( - dim=config.head_dim, - end=config.max_position_embeddings, - rope_base=config.rope_theta, - rope_scaling=config.rope_scaling, - ) # Plain FP32 attributes (not buffers): ``model.to(bf16)`` must not - # round them. Attention layers own their own FP32 tables; these are - # kept for callers that read the model-level RoPE. - self.freqs_cos, self.freqs_sin = freqs_cos.float(), freqs_sin.float() + # round them. The table is the shared one every SWA/global layer + # uses (``kernel.rope_tables``), so it costs no extra memory. + self.freqs_cos, self.freqs_sin = rope_tables( + config.head_dim, config.max_position_embeddings, config.rope_theta, config.rope_scaling, "cpu") self.last_recurrent_state = None def _run_block( @@ -385,14 +372,8 @@ def _ensure_rope_cache(self, needed_len: int, device: torch.device) -> None: if needed_len <= current_len and self.freqs_cos.device == device: return target_len = max(needed_len, current_len * 2, int(self.config.max_position_embeddings)) - freqs_cos, freqs_sin = precompute_freqs_cis( - dim=self.config.head_dim, - end=target_len, - rope_base=self.config.rope_theta, - rope_scaling=self.config.rope_scaling, - ) - self.freqs_cos = freqs_cos.to(device=device) - self.freqs_sin = freqs_sin.to(device=device) + self.freqs_cos, self.freqs_sin = rope_tables( + self.config.head_dim, target_len, self.config.rope_theta, self.config.rope_scaling, device) self.config.max_position_embeddings = max( int(self.config.max_position_embeddings), target_len, diff --git a/chronos/model/moe_chronos.py b/chronos/model/moe_chronos.py index b637c7f..8343a29 100644 --- a/chronos/model/moe_chronos.py +++ b/chronos/model/moe_chronos.py @@ -1,12 +1,9 @@ -import sys -import chronos.deps # ensure minimind on sys.path - import torch import torch.nn as nn import torch.nn.functional as F from transformers.activations import ACT2FN -from model.model_minimind import FeedForward, MOEFeedForward +from .kernel import FeedForward from .config import ChronosConfig @@ -38,13 +35,13 @@ def _run_expert(expert: nn.Module, x: torch.Tensor) -> torch.Tensor: class ChronosMOEFeedForward(nn.Module): """ - Extends MOEFeedForward with: + Routed SwiGLU experts plus: 1. shared_experts: always resident in VRAM, used as soft fallback 2. soft_gate: when target experts are unavailable (cache miss), blend their routing weight into shared experts instead of blocking on I/O - Training (available_expert_mask=None): identical to MOEFeedForward, - plus stores per-token routing probs for temporal_locality_loss. + Training (available_expert_mask=None): softmax top-k dispatch with a + load-balance term, plus stores per-token routing probs for temporal_locality_loss. Inference (available_expert_mask provided): routes cache-miss tokens to shared experts with zero latency. diff --git a/chronos/model/precision_ops.py b/chronos/model/precision_ops.py index 70e55de..2ad9205 100644 --- a/chronos/model/precision_ops.py +++ b/chronos/model/precision_ops.py @@ -30,8 +30,8 @@ class RMSNorm(nn.Module): """RMSNorm with FP32 statistics and a single rounding to the input dtype. - Parameter name (``weight``) and class-name suffix match minimind's - RMSNorm, so checkpoints and the no-decay optimizer grouping are unchanged. + Parameter name (``weight``) and the ``RMSNorm`` class-name suffix are the + checkpoint and no-decay optimizer-grouping contract. """ def __init__(self, dim: int, eps: float = 1e-5): @@ -48,11 +48,9 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: def rope_tables(dim: int, length: int, base: float, scaling, device) -> tuple[torch.Tensor, torch.Tensor]: - """FP32 duplicated cos/sin tables [length, dim] (minimind layout).""" - import chronos.deps # noqa: F401 (minimind on sys.path) - from model.model_minimind import precompute_freqs_cis - cos, sin = precompute_freqs_cis(dim=int(dim), end=int(length), rope_base=base, rope_scaling=scaling) - return cos.to(device=device, dtype=torch.float32), sin.to(device=device, dtype=torch.float32) + """FP32 duplicated cos/sin tables [>=length, dim], shared by every layer.""" + from chronos.model.kernel import rope_tables as _shared + return _shared(dim, length, base, scaling, device) def _rotate_half(x: torch.Tensor) -> torch.Tensor: diff --git a/chronos/model/temporal_loss.py b/chronos/model/temporal_loss.py index b24141d..e460cd3 100644 --- a/chronos/model/temporal_loss.py +++ b/chronos/model/temporal_loss.py @@ -243,7 +243,7 @@ def load_balance_loss( num_experts_per_tok: int, ) -> torch.Tensor: """ - Standard auxiliary load-balancing loss (same formulation as minimind). + Standard auxiliary load-balancing loss (Switch-Transformer style, E·Σ f_i·P_i over top-k). Args: router_logits: [N, num_experts] (N = B*S flattened) diff --git a/chronos/runtime/inference_engine.py b/chronos/runtime/inference_engine.py index 2b24b21..0f253af 100644 --- a/chronos/runtime/inference_engine.py +++ b/chronos/runtime/inference_engine.py @@ -13,9 +13,6 @@ output_ids = engine.generate(input_ids, max_new_tokens=200) print(engine.last_stats) """ -import sys -import chronos.deps # ensure minimind on sys.path - import time from typing import Dict, Hashable, List, Optional diff --git a/chronos/serving/adapter.py b/chronos/serving/adapter.py index 033728b..2da9ef8 100644 --- a/chronos/serving/adapter.py +++ b/chronos/serving/adapter.py @@ -12,6 +12,8 @@ import torch +from chronos.tokenizer.visible import decode_visible + from .api_compat import ( message_text, openai_chat_response, @@ -60,7 +62,7 @@ def generate(self, *, messages=None, prompt=None, max_tokens: int = 128, modality_mask=(torch.ones(ids.shape[0]) if modalities else None), ) generated = output[:, ids.shape[1]:] - text = self.tokenizer.decode(generated[0].tolist(), skip_special_tokens=True) + text = decode_visible(self.tokenizer, generated[0].tolist()) stats = dict(getattr(self.engine, "last_stats", {}) or {}) stats.setdefault("prompt_tokens", int(ids.shape[1])) stats.setdefault("completion_tokens", int(generated.shape[1])) diff --git a/chronos/serving/runtime.py b/chronos/serving/runtime.py index 0a52226..96c72fc 100644 --- a/chronos/serving/runtime.py +++ b/chronos/serving/runtime.py @@ -17,6 +17,8 @@ import torch +from chronos.tokenizer.visible import decode_visible + def _rss_mb() -> float: try: @@ -297,7 +299,7 @@ def on_token(t): if t in eos: return generated.append(t) - text = tok.decode(generated, skip_special_tokens=True) + text = decode_visible(tok, generated) # keeps even if special if text.endswith("�"): return # wait for the rest of a multi-byte char delta, emitted_text = text[len(emitted_text):], text diff --git a/chronos/tokenizer/visible.py b/chronos/tokenizer/visible.py new file mode 100644 index 0000000..4c5e405 --- /dev/null +++ b/chronos/tokenizer/visible.py @@ -0,0 +1,72 @@ +"""Decode generated ids for display without losing structural tags. + +``skip_special_tokens=True`` strips every *special* token. A custom HF +tokenizer may register ```` / ```` (and ````) +as special or added tokens; plain skipping then deletes the tags and the +serving tool-call parser sees bare JSON. :func:`decode_visible` drops only +control tokens (``<|im_end|>``, padding, BOS/EOS, ...) and keeps the tags as +text, for HF tokenizers and the native ``ChronosTokenizer`` alike. +""" +from __future__ import annotations + +from typing import Iterable + +# Tags the chat template writes as text and the server parses back. +STRUCTURAL_TAGS: tuple[str, ...] = ("", "", "", "") + + +def _token_id(tokenizer, token: str) -> int | None: + vocab_lookup = getattr(tokenizer, "convert_tokens_to_ids", None) + if vocab_lookup is not None: + tid = vocab_lookup(token) + unk = getattr(tokenizer, "unk_token_id", None) + if isinstance(tid, int) and tid >= 0 and (unk is None or tid != unk or token == getattr(tokenizer, "unk_token", None)): + return tid + return None + vocab = getattr(tokenizer, "vocab", None) + if isinstance(vocab, dict): + tid = vocab.get(token) + return int(tid) if tid is not None else None + return None + + +def control_token_ids(tokenizer, keep: Iterable[str] = STRUCTURAL_TAGS) -> frozenset[int]: + """Ids of special/added tokens to hide, i.e. all of them except ``keep``. + + Cached on the tokenizer object; the set only changes if tokens are added + later, which the cache key (vocab size) detects. + """ + keep = tuple(keep) + size = len(tokenizer) if hasattr(tokenizer, "__len__") else None + cache = getattr(tokenizer, "_chronos_control_ids", None) + if cache is not None and cache[0] == (size, keep): + return cache[1] + keep_ids = {i for i in (_token_id(tokenizer, t) for t in keep) if i is not None} + hidden: set[int] = set(int(i) for i in (getattr(tokenizer, "all_special_ids", None) or ())) + decoder = getattr(tokenizer, "added_tokens_decoder", None) + if isinstance(decoder, dict): # HF: added tokens flagged ``special`` + hidden.update(int(i) for i, tok in decoder.items() if getattr(tok, "special", False)) + special = getattr(tokenizer, "special_tokens", None) + if isinstance(special, (list, tuple)): # native ChronosTokenizer + hidden.update(i for i in (_token_id(tokenizer, t) for t in special) if i is not None) + for attr in ("bos_token_id", "eos_token_id", "pad_token_id", "unk_token_id"): + tid = getattr(tokenizer, attr, None) # minimal tokenizers expose only these + if isinstance(tid, int): + hidden.add(tid) + ids = frozenset(hidden - keep_ids) + try: + tokenizer._chronos_control_ids = ((size, keep), ids) + except Exception: # slotted / frozen tokenizer objects + pass + return ids + + +def decode_visible(tokenizer, ids, keep: Iterable[str] = STRUCTURAL_TAGS) -> str: + """Decode ``ids`` hiding control tokens but keeping structural tags.""" + hidden = control_token_ids(tokenizer, keep) + kept = [int(i) for i in ids if int(i) not in hidden] + if not kept: + return "" + if hasattr(tokenizer, "special_tokens") and not hasattr(tokenizer, "all_special_ids"): + return tokenizer.decode(kept, skip_special=False) # ChronosTokenizer + return tokenizer.decode(kept, skip_special_tokens=False) diff --git a/chronos/trainer/chronos_trainer.py b/chronos/trainer/chronos_trainer.py index 0cdf69d..536b872 100644 --- a/chronos/trainer/chronos_trainer.py +++ b/chronos/trainer/chronos_trainer.py @@ -1,6 +1,3 @@ -import sys -import chronos.deps # ensure minimind on sys.path - import os import time import torch @@ -8,10 +5,7 @@ from torch import optim from torch.utils.data import DataLoader -from model.model_minimind import MiniMindConfig -from trainer.trainer_utils import ( - Logger, is_main_process, lm_checkpoint, setup_seed, -) +from chronos.model.kernel import Logger, is_main_process, save_resume_state, setup_seed from chronos.trainer.optim_utils import get_lr # warmup→cosine override from chronos.data.flexible_dataset import ( FlexibleDataset as PretrainDataset, @@ -39,7 +33,7 @@ class ChronosTrainer: """ Training orchestrator for Project Chronos. - Wraps minimind's training loop and injects: + Pretraining loop (native kernel utilities) that injects: - temporal_locality_loss via λ2 * ||E_t - E_{t-1}||^2 - load_balance_loss via λ1 (already computed inside ChronosMOEFeedForward) - LookaheadRouter accuracy logging @@ -191,10 +185,11 @@ def _save(self, epoch, step, wandb=None): self.config, stage="chronos", ) - lm_checkpoint( - self.config, weight='chronos', model=self.model, - optimizer=self.optimizer, scaler=self.scaler, - epoch=epoch, step=step, wandb=wandb, - save_dir=self.args.save_dir, + # Resume state only; the release checkpoint above keeps its dtype + # policy (BF16 stays BF16) and is not rewritten here. + save_resume_state( + f'{self.args.save_dir}/chronos_{self.config.hidden_size}_moe_resume.pth', + model=self.model, optimizer=self.optimizer, epoch=epoch, step=step, + scaler=self.scaler, wandb_id=getattr(wandb, 'id', None) if wandb else None, ) self.model.train() diff --git a/chronos/trainer/distill_trainer.py b/chronos/trainer/distill_trainer.py index caf64ff..4f5d699 100644 --- a/chronos/trainer/distill_trainer.py +++ b/chronos/trainer/distill_trainer.py @@ -6,8 +6,8 @@ L_label = cross_entropy(student_logits, labels, ignore_index=-100) L_total = α · L_KD + (1 − α) · L_label + chronos_loss_term (lookahead etc.) -The teacher is any frozen model that returns ``.logits`` on forward (Chronos -or MiniMind checkpoints both work). Router KL anchor still applies — Stage +The teacher is any frozen model that returns ``.logits`` on forward (any Chronos +checkpoint works). Router KL anchor still applies — Stage 6 shouldn't corrupt the cache layout any more than SFT/DPO can. """ from __future__ import annotations @@ -20,8 +20,7 @@ from torch import optim from torch.utils.data import DataLoader -import chronos.deps # noqa: F401 -from trainer.trainer_utils import Logger # type: ignore +from chronos.model.kernel import Logger from chronos.data.flexible_dataset import ( StreamingSFTDataset as SFTDataset, MultimodalStreamingSFTDataset, multimodal_collate, diff --git a/chronos/trainer/dpo_trainer.py b/chronos/trainer/dpo_trainer.py index 7d6e537..b0d1634 100644 --- a/chronos/trainer/dpo_trainer.py +++ b/chronos/trainer/dpo_trainer.py @@ -19,8 +19,7 @@ from torch import optim from torch.utils.data import DataLoader -import chronos.deps # noqa -from trainer.trainer_utils import Logger, is_main_process # type: ignore +from chronos.model.kernel import Logger, is_main_process from chronos.data.flexible_dataset import StreamingDPODataset as DPODataset from chronos.model.config import ChronosConfig diff --git a/chronos/trainer/grpo_trainer.py b/chronos/trainer/grpo_trainer.py index d15fdff..973c732 100644 --- a/chronos/trainer/grpo_trainer.py +++ b/chronos/trainer/grpo_trainer.py @@ -31,8 +31,7 @@ from torch import optim from torch.utils.data import DataLoader -import chronos.deps # noqa -from trainer.trainer_utils import Logger # type: ignore +from chronos.model.kernel import Logger from chronos.model.config import ChronosConfig from chronos.model.model_chronos import ChronosForCausalLM diff --git a/chronos/trainer/loss_mixin.py b/chronos/trainer/loss_mixin.py index 691cf9f..04a91ba 100644 --- a/chronos/trainer/loss_mixin.py +++ b/chronos/trainer/loss_mixin.py @@ -309,7 +309,7 @@ def capture_reference_routing(model, calibration_batch, device, *, modality_inpu try: _ = model(ids, use_cache=False, modality_inputs=modality_inputs, modality_mask=modality_mask) except TypeError: - _ = model(ids) # minimind models don't accept use_cache kwarg here + _ = model(ids) # plain HF-style models without the Chronos kwargs router_4d = collect_router_probs(model) if router_4d is None: ref = None diff --git a/chronos/trainer/optim_utils.py b/chronos/trainer/optim_utils.py index cf40c8d..2185d8d 100644 --- a/chronos/trainer/optim_utils.py +++ b/chronos/trainer/optim_utils.py @@ -12,7 +12,7 @@ this in one place means every trainer benefits from the same correct recipe and we don't get drift. -2. Warmup-then-cosine schedule. minimind's get_lr is pure cosine (effective +2. Warmup-then-cosine schedule. The pre-0.2 schedule was pure cosine (effective LR at step 0 is already 0.55·base_lr), which destabilizes the first few hundred steps with small batches. Linear warmup over the first ~5% of steps then cosine decay to 10% of base is the boring-and-correct choice. diff --git a/chronos/trainer/orpo_trainer.py b/chronos/trainer/orpo_trainer.py index 7d75377..00c0686 100644 --- a/chronos/trainer/orpo_trainer.py +++ b/chronos/trainer/orpo_trainer.py @@ -25,8 +25,7 @@ from torch import optim from torch.utils.data import DataLoader -import chronos.deps # noqa -from trainer.trainer_utils import Logger, is_main_process # type: ignore +from chronos.model.kernel import Logger, is_main_process from chronos.data.flexible_dataset import StreamingDPODataset as DPODataset from chronos.model.config import ChronosConfig diff --git a/chronos/trainer/reward.py b/chronos/trainer/reward.py index 23d20b0..06c89db 100644 --- a/chronos/trainer/reward.py +++ b/chronos/trainer/reward.py @@ -7,8 +7,7 @@ No external dependencies, used in smoke tests and when no real reward model is configured. - ``LMRewardModel`` — wraps an HF AutoModel that exposes a ``.get_score(...)`` - method (the minimind convention, see - ``minimind-master/trainer/trainer_utils.py:160``). Loads lazily so + method (the ABI of InternLM2-style reward checkpoints). Loads lazily so importing this file never forces an HF network download. Any callable ``score(prompt: str, response_tokens: List[int], tokenizer) -> float`` @@ -19,6 +18,8 @@ from typing import List, Optional +from chronos.tokenizer.visible import decode_visible + TOY_VOCAB = { "the", "quick", "brown", "fox", "machine", "learning", "model", "expert", @@ -42,7 +43,7 @@ def __call__(self, prompt: str, response_tokens: List[int], tokenizer) -> float: return self.score(prompt, response_tokens, tokenizer) def score(self, prompt: str, response_tokens: List[int], tokenizer) -> float: - text = tokenizer.decode(response_tokens, skip_special_tokens=True) + text = decode_visible(tokenizer, response_tokens) words = text.split() if not words: return -1.0 @@ -58,7 +59,7 @@ class LMRewardModel: """HF-backed reward model. Wraps any ``AutoModel`` that exposes ``get_score(tokenizer, messages) -> float`` - (the reward-model ABI used by minimind and several open reward-model + (the ABI of InternLM2-style and several other open reward-model checkpoints). The model is loaded lazily on the first call so that importing this module is free. @@ -67,8 +68,7 @@ class LMRewardModel: reward = LMRewardModel("path/to/internlm2-1_8b-reward") r = reward.score(prompt, response_tokens, tokenizer) - The score is clipped to [-3.0, 3.0] to keep GRPO advantages well-scaled, - matching the minimind convention. + The score is clipped to [-3.0, 3.0] to keep GRPO advantages well-scaled. """ def __init__(self, model_path: str, device: str = "cpu", @@ -98,7 +98,7 @@ def __call__(self, prompt: str, response_tokens: List[int], tokenizer) -> float: def score(self, prompt: str, response_tokens: List[int], tokenizer) -> float: self._load() - response = tokenizer.decode(response_tokens, skip_special_tokens=True) + response = decode_visible(tokenizer, response_tokens) # tool tags survive messages = [ {"role": "user", "content": prompt}, {"role": "assistant", "content": response}, diff --git a/chronos/trainer/sft_trainer.py b/chronos/trainer/sft_trainer.py index 5f430cc..5ea16aa 100644 --- a/chronos/trainer/sft_trainer.py +++ b/chronos/trainer/sft_trainer.py @@ -3,13 +3,12 @@ Chronos-aware supervised fine-tuning trainer. -Built around minimind's SFTDataset (jsonl with `conversations` field, -tokenizer.apply_chat_template + per-assistant-token loss mask) so the data -pipeline is identical to the one in minimind-master/trainer/train_full_sft.py. -What we add: +Data: jsonl with a ``conversations`` field, rendered with +``tokenizer.apply_chat_template`` and a per-assistant-token loss mask +(``StreamingSFTDataset``; the common chat-SFT jsonl layout). On top of CE: -1. The model returns ``(outputs, lookahead_probs)``, which minimind's trainer - does not know about. We forward it explicitly. +1. The model returns ``(outputs, lookahead_probs)``; the lookahead output is + forwarded explicitly. 2. ``chronos_loss_term`` mixes in λ_balance·aux + λ_temporal·temporal + λ_lookahead·lookahead on top of the base CE. 3. Optional ``router_kl_anchor`` pulls the current router distribution back @@ -25,9 +24,8 @@ from torch import optim from torch.utils.data import DataLoader -import chronos.deps # ensure minimind on sys.path -from trainer.trainer_utils import Logger, is_main_process # type: ignore +from chronos.model.kernel import Logger, is_main_process from chronos.data.flexible_dataset import ( StreamingSFTDataset, MultimodalStreamingSFTDataset, multimodal_collate, ) diff --git a/chronos/trainer/stage_utils.py b/chronos/trainer/stage_utils.py index 9851dca..2ca6551 100644 --- a/chronos/trainer/stage_utils.py +++ b/chronos/trainer/stage_utils.py @@ -68,7 +68,7 @@ def add_topology_args(parser, *, defaults: bool = False): parser.add_argument("--vocab_size", type=int, default=(6400 if defaults else None)) parser.add_argument("--tokenizer_path", default=None, help="Tokenizer directory (tokenizer.json). Default: the checkpoint's own tokenizer " - "(/tokenizer or its config metadata), else minimind.") + "(/tokenizer or its config metadata), else the bundled legacy 6400 tokenizer.") parser.add_argument("--recurrent_groups", type=int, default=(2 if defaults else None)) parser.add_argument("--recurrent_depth", type=int, default=(4 if defaults else None)) parser.add_argument("--min_recurrent_depth", type=int, default=(2 if defaults else None)) @@ -225,8 +225,8 @@ def load_stage_tokenizer(args, config, checkpoint_path: str | None = None): """Load the tokenizer that matches ``checkpoint_path`` (or ``--tokenizer_path``). Raises an error when the tokenizer has more entries than ``config.vocab_size``. - A larger tokenizer would emit ids the embedding cannot hold. The minimind - tokenizer (6400 entries) used on a 16k-vocabulary checkpoint fails the other + A larger tokenizer would emit ids the embedding cannot hold. The legacy + 6400-entry tokenizer used on a 16k-vocabulary checkpoint fails the other way: it trains silently on the wrong ids. """ from transformers import AutoTokenizer diff --git a/chronos/tuning/chronos_auto_tuner.py b/chronos/tuning/chronos_auto_tuner.py index efa4ad5..d653239 100644 --- a/chronos/tuning/chronos_auto_tuner.py +++ b/chronos/tuning/chronos_auto_tuner.py @@ -10,7 +10,6 @@ from dataclasses import dataclass, field from typing import List -import chronos.deps # ensure minimind on sys.path from chronos.tuning._base_tuner import ( AutoTuner, SearchSpaceConfig, AutoTuneResult, _gc_collect, diff --git a/chronos/verify.py b/chronos/verify.py index 418b00f..0f5e2c2 100644 --- a/chronos/verify.py +++ b/chronos/verify.py @@ -8,7 +8,6 @@ import torch import torch.nn.functional as F -import chronos.deps # noqa: F401 from chronos.backend import available as available_backends from chronos.model.checkpoint import load_checkpoint_state_dict, load_state_dict_controlled from chronos.model.model_chronos import ChronosForCausalLM diff --git a/chronos/webui/i18n/en.json b/chronos/webui/i18n/en.json index cdeca8b..1ae7888 100644 --- a/chronos/webui/i18n/en.json +++ b/chronos/webui/i18n/en.json @@ -3,7 +3,7 @@ "app.title": "Chronos Workbench", "bench.better": "Better", "bench.higher": "Higher is better", - "bench.intro": "Compare MiniMind and Chronos on parameters, memory and decode speed with benchmark_compare.py.", + "bench.intro": "Measure Chronos parameters, memory and decode speed with benchmark_compare.py; an external dense baseline is optional.", "bench.lower": "Lower is better", "bench.measured_note": "Measured by benchmark_compare.py on this machine.", "bench.metric": "Metric", @@ -13,7 +13,7 @@ "bench.ram_model": "Model RAM", "bench.ram_train": "Training RAM", "bench.run": "Run benchmark", - "bench.title": "Benchmark: MiniMind vs Chronos", + "bench.title": "Benchmark: Chronos vs a dense baseline", "bench.tps": "Decode tokens/s", "bench.unavailable": "benchmark_compare.py is not in this installation.", "choice.absorbed": "Absorbed", @@ -678,7 +678,7 @@ "preset.b1": "1B (64 GB)", "preset.b1.summary": "1B MoE with 64 fine-grained experts and aggressive memory saving.", "preset.m200": "200M (32 GB)", - "preset.m200.summary": "General 200M MoE with balanced memory saving and the 6400-token minimind vocabulary.", + "preset.m200.summary": "General 200M MoE with balanced memory saving and the legacy 6400-token vocabulary.", "preset.m50": "50M (16k vocabulary)", "preset.m50.summary": "50M MoE with 36 experts (top-2) for a 16k Chronos tokenizer; no memory saving.", "preset.m500": "500M (32 GB)", diff --git a/chronos/webui/i18n/ja.json b/chronos/webui/i18n/ja.json index 0480eba..b8d04de 100644 --- a/chronos/webui/i18n/ja.json +++ b/chronos/webui/i18n/ja.json @@ -3,7 +3,7 @@ "app.title": "Chronos ワークベンチ", "bench.better": "優位", "bench.higher": "高いほど良い", - "bench.intro": "benchmark_compare.py で MiniMind と Chronos のパラメータ数・メモリ・デコード速度を比較します。", + "bench.intro": "benchmark_compare.py で Chronos のパラメータ数・メモリ・デコード速度を測定します。外部の dense ベースラインとの比較は任意です。", "bench.lower": "低いほど良い", "bench.measured_note": "benchmark_compare.py がこのマシンで実測。", "bench.metric": "指標", @@ -13,7 +13,7 @@ "bench.ram_model": "モデルの RAM", "bench.ram_train": "学習時の RAM", "bench.run": "ベンチマークを実行", - "bench.title": "ベンチマーク：MiniMind と Chronos", + "bench.title": "ベンチマーク：Chronos と dense ベースライン", "bench.tps": "デコード トークン/秒", "bench.unavailable": "このインストールには benchmark_compare.py がありません。", "choice.absorbed": "吸収", @@ -678,7 +678,7 @@ "preset.b1": "1B（64 GB）", "preset.b1.summary": "1B MoE。64 個の細粒度エキスパート、積極的なメモリ節約。", "preset.m200": "200M（32 GB）", - "preset.m200.summary": "汎用 200M MoE。メモリ節約はバランス型で、minimind 6400 語彙を使用。", + "preset.m200.summary": "汎用 200M MoE。メモリ節約はバランス型で、旧 6400 語彙を使用。", "preset.m50": "50M（16k 語彙）", "preset.m50.summary": "36 エキスパート（top-2）の 50M MoE、16k Chronos トークナイザー向け。メモリ節約なし。", "preset.m500": "500M（32 GB）", diff --git a/chronos/webui/i18n/zh-Hans.json b/chronos/webui/i18n/zh-Hans.json index 06a2bf8..4dd0950 100644 --- a/chronos/webui/i18n/zh-Hans.json +++ b/chronos/webui/i18n/zh-Hans.json @@ -3,7 +3,7 @@ "app.title": "Chronos 工作台", "bench.better": "更优", "bench.higher": "越高越好", - "bench.intro": "用 benchmark_compare.py 比较 MiniMind 与 Chronos 的参数量、内存和解码速度。", + "bench.intro": "用 benchmark_compare.py 测量 Chronos 的参数量、内存和解码速度；外部 dense 基线的对比是可选的。", "bench.lower": "越低越好", "bench.measured_note": "由 benchmark_compare.py 在本机实测。", "bench.metric": "指标", @@ -13,7 +13,7 @@ "bench.ram_model": "模型内存", "bench.ram_train": "训练内存", "bench.run": "运行基准测试", - "bench.title": "基准测试：MiniMind 与 Chronos", + "bench.title": "基准测试：Chronos 与 dense 基线", "bench.tps": "解码 token/秒", "bench.unavailable": "当前安装中没有 benchmark_compare.py。", "choice.absorbed": "吸收", @@ -678,7 +678,7 @@ "preset.b1": "1B（64 GB）", "preset.b1.summary": "1B MoE，64 个细粒度专家，激进省内存。", "preset.m200": "200M（32 GB）", - "preset.m200.summary": "通用 200M MoE，均衡省内存，使用 minimind 6400 词表。", + "preset.m200.summary": "通用 200M MoE，均衡省内存，使用旧版 6400 词表。", "preset.m50": "50M（16k 词表）", "preset.m50.summary": "50M MoE，36 个专家（top-2），配 16k Chronos 分词器；不启用省内存选项。", "preset.m500": "500M（32 GB）", diff --git a/chronos/webui/i18n/zh-Hant.json b/chronos/webui/i18n/zh-Hant.json index 56f3ebb..49d0faf 100644 --- a/chronos/webui/i18n/zh-Hant.json +++ b/chronos/webui/i18n/zh-Hant.json @@ -3,7 +3,7 @@ "app.title": "Chronos 工作台", "bench.better": "較佳者", "bench.higher": "越高越好", - "bench.intro": "以 benchmark_compare.py 比較 MiniMind 與 Chronos 的參數量、記憶體與解碼速度。", + "bench.intro": "以 benchmark_compare.py 量測 Chronos 的參數量、記憶體與解碼速度；外部 dense 基線的比較為可選。", "bench.lower": "越低越好", "bench.measured_note": "由 benchmark_compare.py 在本機實測。", "bench.metric": "指標", @@ -13,7 +13,7 @@ "bench.ram_model": "模型記憶體", "bench.ram_train": "訓練記憶體", "bench.run": "執行基準測試", - "bench.title": "基準測試：MiniMind 與 Chronos", + "bench.title": "基準測試：Chronos 與 dense 基線", "bench.tps": "解碼 token/秒", "bench.unavailable": "目前安裝中沒有 benchmark_compare.py。", "choice.absorbed": "吸收", @@ -678,7 +678,7 @@ "preset.b1": "1B（64 GB）", "preset.b1.summary": "1B MoE，64 個細粒度專家，積極節省記憶體。", "preset.m200": "200M（32 GB）", - "preset.m200.summary": "通用 200M MoE，均衡節省記憶體，使用 minimind 6400 詞表。", + "preset.m200.summary": "通用 200M MoE，均衡節省記憶體，使用舊版 6400 詞表。", "preset.m50": "50M（16k 詞表）", "preset.m50.summary": "50M MoE，36 個專家（top-2），配 16k Chronos 分詞器；不啟用省記憶體選項。", "preset.m500": "500M（32 GB）", diff --git a/chronos/webui/server.py b/chronos/webui/server.py index b03fcb6..7513042 100644 --- a/chronos/webui/server.py +++ b/chronos/webui/server.py @@ -884,7 +884,7 @@ def serve_state(): return {"running": running, "url": url, "addr": addr, "auth": ui.serve.auth if running else False, "log": log} - # ── benchmark (MiniMind vs Chronos script in the repo) ────────────── + # ── benchmark (benchmark_compare.py; external baseline optional) ── bench_script = os.path.join(REPO_ROOT, "benchmark_compare.py") bench_results = os.path.join(REPO_ROOT, "benchmark_results.json") diff --git a/chronos/workbench/data.py b/chronos/workbench/data.py index e2a95d2..301b343 100644 --- a/chronos/workbench/data.py +++ b/chronos/workbench/data.py @@ -12,7 +12,6 @@ import statistics from collections import Counter -import chronos.deps # noqa: F401 _CHAT_KEYS = ("conversations", "messages", "dialog", "dialogue") _PROMPT_KEYS = ("prompt", "question", "query", "input", "instruction") diff --git a/chronos/workbench/inference.py b/chronos/workbench/inference.py index b533aae..8101608 100644 --- a/chronos/workbench/inference.py +++ b/chronos/workbench/inference.py @@ -8,8 +8,8 @@ import os import time -import chronos.deps # noqa: F401 (auto-bootstrap minimind on sys.path) from chronos.trainer.device_utils import configure_cpu_threads, cpu_thread_snapshot +from chronos.tokenizer.visible import decode_visible INFERENCE_BACKEND_CHOICES = ("mlx", "cuda", "xpu", "mps", "cpu") @@ -215,7 +215,7 @@ def _decode_generation_diagnostics(tokenizer, tokens: list[int], stats: dict | N "bos": getattr(tokenizer, "bos_token_id", None), } raw_text = tokenizer.decode(tokens, skip_special_tokens=False) if tokens else "" - clean_text = tokenizer.decode(tokens, skip_special_tokens=True) if tokens else "" + clean_text = decode_visible(tokenizer, tokens) if tokens else "" first_token_id = tokens[0] if tokens else None first_token_kind = "" for name, token_id in special_ids.items(): @@ -1260,7 +1260,7 @@ def _inference_tokenizer_dir(cfg, model_path_val: str) -> str: """The tokenizer that belongs to the checkpoint, not a fixed default. Order: config ``tokenizer_path``; the checkpoint's bundled ``tokenizer/`` - or the path in its sidecar; the minimind tokenizer only as the last resort + or the path in its sidecar; the bundled legacy 6400 tokenizer only as the last resort (older checkpoints that never recorded one). """ from chronos.model.checkpoint import resolve_checkpoint_tokenizer diff --git a/chronos/workbench/training.py b/chronos/workbench/training.py index 0760b35..ff945dd 100644 --- a/chronos/workbench/training.py +++ b/chronos/workbench/training.py @@ -17,8 +17,9 @@ from types import SimpleNamespace -import chronos.deps # auto-bootstrap minimind on sys.path -from ui.presets import MINIMIND_MOE_DEFAULTS +import chronos.deps # default tokenizer path +from chronos.tokenizer.visible import decode_visible +from ui.presets import BASE_MOE_DEFAULTS from chronos.backend import training_available, resolve_training_device from chronos.trainer.device_utils import ( autocast_context, @@ -597,7 +598,7 @@ def _prepare_training_run_config( teacher_path: str | None, sample_prompt: str | None, ) -> tuple[dict, dict]: - merged = dict(MINIMIND_MOE_DEFAULTS) + merged = dict(BASE_MOE_DEFAULTS) merged.update(dict(cfg) if cfg else {}) # Per-stage hyperparameters (Config tab → auto plan) override the shared # values for this stage only; architecture keys are never stage-specific. @@ -859,23 +860,14 @@ def _cleanup_training_dataset_artifact(path: str | None) -> dict: @contextmanager def _trainer_logger_sink(put_line): - from trainer import trainer_utils as _tu # type: ignore + """Forward every trainer ``Logger`` line to ``put_line`` while active. - orig_logger = _tu.Logger - - def _hook(content): - line = str(content) - put_line(line) - try: - orig_logger(content) - except Exception: - pass - - _tu.Logger = _hook - try: + The old version swapped a module attribute, which trainers that had + already bound ``Logger`` by name never saw; the sink registry reaches all. + """ + from chronos.model.kernel import add_log_sink + with add_log_sink(put_line): yield - finally: - _tu.Logger = orig_logger def _sniff_checkpoint(path: str) -> dict: @@ -1186,7 +1178,7 @@ def _generate_live_sample(self, model, device: str, cfg: dict, mode: str): gen = torch.cat([gen, nxt], dim=-1) if int(nxt.item()) == (tokenizer.eos_token_id or -1): break - txt = tokenizer.decode(gen[0, ids.shape[1]:].tolist(), skip_special_tokens=True) + txt = decode_visible(tokenizer, gen[0, ids.shape[1]:].tolist()) if was_training: model.train() self.last_sample = txt @@ -2024,7 +2016,7 @@ def _eval_val() -> float: gen = torch.cat([gen, nxt], dim=-1) if int(nxt.item()) == (tokenizer.eos_token_id or -1): break - txt = tokenizer.decode(gen[0, ids.shape[1]:].tolist(), skip_special_tokens=True) + txt = decode_visible(tokenizer, gen[0, ids.shape[1]:].tolist()) model.train() self.last_sample = txt self.last_sample_step = global_step @@ -2083,7 +2075,7 @@ def _load_tokenizer(self): def resolve_tokenizer_dir(configured: str | None) -> str: - """Config ``tokenizer_path`` (a Chronos Tokenizer Builder output) wins; else minimind.""" + """Config ``tokenizer_path`` (a Chronos Tokenizer Builder output) wins; else the bundled legacy one.""" configured = os.path.expanduser(str(configured or "").strip()) if configured: if not os.path.isfile(os.path.join(configured, "tokenizer.json")): diff --git a/demos/translation/README.md b/demos/translation/README.md index d3bdf9c..0f66f00 100644 --- a/demos/translation/README.md +++ b/demos/translation/README.md @@ -25,18 +25,18 @@ while 0.3.1 is in beta) or from the same checkout. ```sh git clone https://github.com/FonaTech/Project_Chronos.git && cd Project_Chronos -pip install -e '.[mlx,serve]' # the kernel (or: pip install --pre 'Project_Chronos[mlx,serve]==0.3.1a3') +pip install -e '.[mlx,serve]' # the kernel (or: pip install --pre 'Project_Chronos[mlx,serve]==0.3.2a0') pip install -e demos/translation # this extension (sacrebleu, sacremoses, pyarrow pinned) chronos-translate --version ``` -The extension pins `Project_Chronos==0.3.1a3`. Use the checkout's tag -(`git checkout v0.3.1a3`) so both sides match. A model bundle needs the two +The extension pins `Project_Chronos==0.3.2a0`. Use the checkout's tag +(`git checkout v0.3.2a0`) so both sides match. A model bundle needs the two as wheels; build them from the checkout: ```sh -python -m build --outdir dist . # dist/project_chronos-0.3.1a3-py3-none-any.whl -python -m build --outdir dist demos/translation # dist/chronos_translate-0.3.1a3-py3-none-any.whl +python -m build --outdir dist . # dist/project_chronos-0.3.2a0-py3-none-any.whl +python -m build --outdir dist demos/translation # dist/chronos_translate-0.3.2a0-py3-none-any.whl ``` The extension's tests run from the repo root: `pytest demos/translation/tests`. diff --git a/demos/translation/README_zh.md b/demos/translation/README_zh.md index 87bf02b..ca28a9b 100644 --- a/demos/translation/README_zh.md +++ b/demos/translation/README_zh.md @@ -23,17 +23,17 @@ Chronos 内核（`chronos/`）用于训练通用模型，不含任何翻译特 ```sh git clone https://github.com/FonaTech/Project_Chronos.git && cd Project_Chronos -pip install -e '.[mlx,serve]' # 内核（或：pip install --pre 'Project_Chronos[mlx,serve]==0.3.1a3'） +pip install -e '.[mlx,serve]' # 内核（或：pip install --pre 'Project_Chronos[mlx,serve]==0.3.2a0'） pip install -e demos/translation # 本扩展（sacrebleu、sacremoses、pyarrow 固定版本） chronos-translate --version ``` -扩展固定依赖 `Project_Chronos==0.3.1a3`。请检出对应标签（`git checkout v0.3.1a3`），使两边版本一致。 +扩展固定依赖 `Project_Chronos==0.3.2a0`。请检出对应标签（`git checkout v0.3.2a0`），使两边版本一致。 模型包需要这两个 wheel，可以从检出目录构建： ```sh -python -m build --outdir dist . # dist/project_chronos-0.3.1a3-py3-none-any.whl -python -m build --outdir dist demos/translation # dist/chronos_translate-0.3.1a3-py3-none-any.whl +python -m build --outdir dist . # dist/project_chronos-0.3.2a0-py3-none-any.whl +python -m build --outdir dist demos/translation # dist/chronos_translate-0.3.2a0-py3-none-any.whl ``` 扩展测试在仓库根目录运行：`pytest demos/translation/tests`。 diff --git a/demos/translation/chronos_translate/bundle/builder.py b/demos/translation/chronos_translate/bundle/builder.py index f55498e..0e432ae 100644 --- a/demos/translation/chronos_translate/bundle/builder.py +++ b/demos/translation/chronos_translate/bundle/builder.py @@ -14,8 +14,6 @@ "chronos_tokenizer.json", "tokenizer_report.json") FLORES_REL = os.path.join("facebook__flores__eng-zho", "data", "pair", "eng_Latn-zho_Hans") FLORES_FILES = ("dev-00000-of-00001.parquet", "devtest-00000-of-00001.parquet") -# minimind sub-packages Chronos imports (model.*, trainer.trainer_utils, dataset.lm_dataset). -MINIMIND_PARTS = ("model", "trainer", "dataset", "LICENSE") # Never shipped: they hold absolute local paths, optimizer state or caches. _SKIP_NAMES = {"__pycache__", ".DS_Store"} @@ -149,7 +147,8 @@ def build_bundle(run_dir: str, out_dir: str | None = None, *, phase: str = "pret """Assemble a bundle from a run directory (read-only on the run). ``wheel``: one path or a list of paths (the kernel and the - ``chronos_translate`` wheels) copied to ``wheels/`` for ``bin/setup.sh``.""" + ``chronos_translate`` wheels) copied to ``wheels/`` for ``bin/setup.sh``. + ``minimind_dir`` is ignored since 0.3.2 (the kernel is self-contained).""" from chronos.model.checkpoint import checkpoint_config_path, tokenizer_metadata from . import docs, recommend run_dir = os.path.abspath(run_dir) @@ -204,16 +203,7 @@ def build_bundle(run_dir: str, out_dir: str | None = None, *, phase: str = "pret valid_src = os.path.join(spec["shards_dir"], "sft", "valid") if os.path.isdir(valid_src): _copytree(valid_src, os.path.join(out_dir, "data", "sft_valid")) - # minimind (Apache-2.0) so the recipient needs neither git nor network. - from chronos.deps import ensure_minimind - mm = minimind_dir or str(ensure_minimind()) - for part in MINIMIND_PARTS: - src = os.path.join(mm, part) - if os.path.isdir(src): - _copytree(src, os.path.join(out_dir, "third_party", "minimind", part)) - elif os.path.isfile(src): - _copy(src, os.path.join(out_dir, "third_party", "minimind", part)) - mm_rev = _git_head(mm) + del minimind_dir # kept for callers of the pre-0.3.2 signature for whl in ([wheel] if isinstance(wheel, str) else list(wheel or ())): _copy(whl, os.path.join(out_dir, "wheels", os.path.basename(whl))) # Recipes: the exact spec that produced these weights (paths made relative). @@ -238,7 +228,6 @@ def build_bundle(run_dir: str, out_dir: str | None = None, *, phase: str = "pret "tokenizer": {"dir": "model/tokenizer", **{k: v for k, v in tok_meta.items() if k != "path"}}, "training": prov, "eval": {"flores_plus": flores, "licence": "CC BY-SA 4.0", "root": "eval"}, - "third_party": {"minimind": {"dir": "third_party/minimind", "licence": "Apache-2.0", "commit": mm_rev}}, "chronos_version": _chronos_version(), "chronos_translate_version": _extension_version(), } @@ -252,17 +241,6 @@ def build_bundle(run_dir: str, out_dir: str | None = None, *, phase: str = "pret return out_dir -def _git_head(path: str) -> str | None: - head = os.path.join(path, ".git", "HEAD") - try: - ref = open(head, encoding="utf-8").read().strip() - if ref.startswith("ref: "): - return open(os.path.join(path, ".git", ref[5:]), encoding="utf-8").read().strip() - return ref - except OSError: - return None - - def _chronos_version() -> str: import chronos return getattr(chronos, "__version__", "unknown") diff --git a/demos/translation/chronos_translate/bundle/docs.py b/demos/translation/chronos_translate/bundle/docs.py index 75673b1..832d86b 100644 --- a/demos/translation/chronos_translate/bundle/docs.py +++ b/demos/translation/chronos_translate/bundle/docs.py @@ -65,7 +65,6 @@ def write_examples(out_dir: str) -> None: # bundle …`); every other command goes to the Chronos kernel CLI. HERE="$(cd "$(dirname "$0")/.." && pwd)" export CHRONOS_BUNDLE="${CHRONOS_BUNDLE:-$HERE}" -export CHRONOS_MINIMIND_DIR="${CHRONOS_MINIMIND_DIR:-$HERE/third_party/minimind}" BIN=chronos if [ "${1:-}" = bundle ]; then BIN=chronos-translate; fi if [ -x "$HERE/.venv/bin/$BIN" ]; then exec "$HERE/.venv/bin/$BIN" "$@"; fi @@ -171,9 +170,8 @@ def _readme_en(m: dict, recs: dict) -> str: "`--residency planned` (default) evicts the page needed latest; `window` keeps LRU order. " "`--decay`, `--ram-ratio`, `--decode-wait` and `--verify-wait` tune it; `--help` lists the defaults. " "The page counts are printed to stderr.", "", - "`bin/chronos` sets `CHRONOS_BUNDLE` and `CHRONOS_MINIMIND_DIR` (the vendored minimind in " - "`third_party/`), so nothing is fetched from the network. Without `bin/`, export both variables and call " - "`chronos-translate bundle …` directly.", "", + "`bin/chronos` sets `CHRONOS_BUNDLE`; nothing is fetched from the network. Without `bin/`, export " + "`CHRONOS_BUNDLE` and call `chronos-translate bundle …` directly.", "", "## Next stage: SFT (fine-tuning) on Apple silicon (MLX)", "", "```sh", "# 1. your pairs: .jsonl {\"en\":…, \"zh\":…} | .csv | .tsv | .parquet", @@ -213,7 +211,7 @@ def _readme_en(m: dict, recs: dict) -> str: "## Files", "", "`bundle.json` (identity, provenance, sha256 of every file) · `model/` (weights, config sidecar, tokenizer) · " "`recipes/source_run_spec.json` (the exact training spec) · `recommended.json` (next-stage parameters) · " - "`eval/` (FLORES+) · `data/sft_valid/` (held-out SFT validation shards) · `examples/` · `third_party/` · " + "`eval/` (FLORES+) · `data/sft_valid/` (held-out SFT validation shards) · `examples/` · " "`wheels/` · `bin/`. Verify integrity any time with `bin/chronos bundle check`.", "", "See `NOTICE.md` before redistributing: the weights are for **research use only**.", "", ] @@ -264,8 +262,8 @@ def _readme_zh(m: dict, recs: dict) -> str: "`--residency planned`（默认）优先淘汰最晚才会用到的页，`window` 保持 LRU 顺序。" "可用 `--decay`、`--ram-ratio`、`--decode-wait`、`--verify-wait` 调整，默认值见 `--help`。" "页读取统计打印到 stderr。", "", - "`bin/chronos` 会设置 `CHRONOS_BUNDLE` 和 `CHRONOS_MINIMIND_DIR`（指向 `third_party/` 里随包附带的 minimind），" - "全程不需要联网。不用 `bin/` 时，手动导出这两个变量后直接调用 `chronos-translate bundle …`。", "", + "`bin/chronos` 会设置 `CHRONOS_BUNDLE`，全程不需要联网。不用 `bin/` 时，手动导出 `CHRONOS_BUNDLE` " + "后直接调用 `chronos-translate bundle …`。", "", "## 下一阶段：SFT 微调（Apple silicon / MLX）", "", "```sh", "# 1. 准备平行语料：.jsonl {\"en\":…, \"zh\":…} | .csv | .tsv | .parquet", @@ -301,7 +299,7 @@ def _readme_zh(m: dict, recs: dict) -> str: "## 文件", "", "`bundle.json`（身份、来源、每个文件的 sha256）· `model/`（权重、配置 sidecar、分词器）· " "`recipes/source_run_spec.json`（产生这份权重的训练配置）· `recommended.json`（下一阶段参数）· " - "`eval/`（FLORES+）· `data/sft_valid/`（留出的 SFT 验证分片）· `examples/` · `third_party/` · `wheels/` · " + "`eval/`（FLORES+）· `data/sft_valid/`（留出的 SFT 验证分片）· `examples/` · `wheels/` · " "`bin/`。随时可用 `bin/chronos bundle check` 校验完整性。", "", "再分发前请阅读 `NOTICE.md`：权重**仅供研究使用**。", "", ] @@ -309,7 +307,6 @@ def _readme_zh(m: dict, recs: dict) -> str: def _notice(m: dict) -> str: - mm = (m.get("third_party") or {}).get("minimind") or {} return "\n".join([ "# NOTICE", "", "## Weights: research use only", "", @@ -324,9 +321,8 @@ def _notice(m: dict) -> str: "evaluation only**. Do not use them commercially or redistribute them as a product without checking the " "licences of the underlying corpora yourself.", "", "## Code", "", - "- Chronos (`wheels/`, `bin/`): Apache-2.0.", - f"- minimind (`third_party/minimind/`, commit `{mm.get('commit') or 'unknown'}`): Apache-2.0, " - "© jingyaogong. Its `LICENSE` file is included.", + "- Chronos (`wheels/`, `bin/`): Apache-2.0. Since 0.3.2 the kernel has no third-party model code; " + "see `THIRD_PARTY_NOTICES.md` in the Chronos wheel.", "- FLORES+ files in `eval/` keep their CC BY-SA 4.0 licence (attribution: the FLORES+ / OLDI project).", "", "## No warranty", "", "Provided as is. Machine translation output can be wrong; check it before relying on it.", "", diff --git a/demos/translation/pyproject.toml b/demos/translation/pyproject.toml index 8b93640..d62566a 100644 --- a/demos/translation/pyproject.toml +++ b/demos/translation/pyproject.toml @@ -4,21 +4,21 @@ build-backend = "setuptools.build_meta" [project] name = "chronos-translate" -version = "0.3.1a3" +version = "0.3.2a0" description = "English-Chinese translation demo for the Chronos kernel: corpus pipeline, BLEU/chrF runs, model bundles" readme = "README.md" license = "Apache-2.0" requires-python = ">=3.10" dependencies = [ - "Project_Chronos==0.3.1a3", + "Project_Chronos==0.3.2a0", "sacrebleu==2.6.0", "sacremoses==0.2.0", "pyarrow==23.0.1", ] [project.optional-dependencies] -mlx = ["Project_Chronos[mlx]==0.3.1a3"] -serve = ["Project_Chronos[serve]==0.3.1a3"] +mlx = ["Project_Chronos[mlx]==0.3.2a0"] +serve = ["Project_Chronos[serve]==0.3.2a0"] [project.scripts] chronos-translate = "chronos_translate.cli:main" diff --git a/demos/translation/scripts/make_docmix_packages.py b/demos/translation/scripts/make_docmix_packages.py index 0fe74b6..80112ec 100644 --- a/demos/translation/scripts/make_docmix_packages.py +++ b/demos/translation/scripts/make_docmix_packages.py @@ -1,7 +1,7 @@ """Self-contained MLX training packages for the doc-mix sets (A: 15%/4k/24h, B: 5%/1k/8h, C: 1%/1k/1h). -Each package carries its data, tokenizer, eval sets, Chronos source, vendored -minimind, a recipe with relative paths, pinned requirements, launch/monitor +Each package carries its data, tokenizer, eval sets, Chronos source (no +third-party kernel since 0.3.2), a recipe with relative paths, pinned requirements, launch/monitor scripts and SHA256SUMS. Usage:: python demos/translation/scripts/make_docmix_packages.py OUT_DIR [A B C] @@ -19,14 +19,13 @@ DEMO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) REPO = os.path.abspath(os.environ.get("CHRONOS_REPO") or os.path.dirname(os.path.dirname(DEMO))) sys.path[:0] = [REPO, DEMO] -from chronos_translate.bundle.builder import TOKENIZER_FILES, FLORES_REL, FLORES_FILES, MINIMIND_PARTS, sha256_file # noqa: E402 +from chronos_translate.bundle.builder import TOKENIZER_FILES, FLORES_REL, FLORES_FILES, sha256_file # noqa: E402 from chronos.config_schema import model_config_kwargs # noqa: E402 DATA = os.path.join(REPO, "Translation Training Data") DOCMIX = os.path.join(DATA, "docmix_v1") RAW = os.path.join(DATA, "raw") TOKENIZER = os.path.join(DATA, "tokenizer_v1") -MINIMIND = os.path.expanduser("~/.cache/chronos/minimind-master") PLAN = os.path.join(DEMO, "recipes", "docmix_plan.json") SKIP = {"__pycache__", ".DS_Store"} # The extension's own recipes and tests are not needed inside a package. @@ -232,7 +231,6 @@ def shards_manifest(cfg, docmix): fi fi export PYTHONPATH="$HERE/src" -export CHRONOS_MINIMIND_DIR="$HERE/third_party/minimind" export TOKENIZERS_PARALLELISM=false export PYTHONNOUSERSITE=1 "$PY" -c 'import mlx.core as mx; i = mx.device_info(mx.gpu); print("[pkg] GPU:", i.get("device_name"), "| 内存", round(i.get("memory_size", 0) / 2**30), "GiB | 推荐工作集", round(i.get("max_recommended_working_set_size", 0) / 2**30), "GiB")' @@ -256,7 +254,6 @@ def shards_manifest(cfg, docmix): cd "$HERE" PY="${CHRONOS_PY:-$HERE/.venv/bin/python}" export PYTHONPATH="$HERE/src" -export CHRONOS_MINIMIND_DIR="$HERE/third_party/minimind" export PYTHONNOUSERSITE=1 exec "$PY" -m chronos.monitor.server --runs-dir "$HERE" --host 127.0.0.1 --port "${PORT:-7870}" "$@" """ @@ -370,9 +367,6 @@ def build(key, out_root, plan): _copytree(os.path.join(DEMO, "chronos_translate"), os.path.join(tmp, "src", "chronos_translate")) for name in ("LICENSE", "THIRD_PARTY_NOTICES.md", "pyproject.toml"): _copy(os.path.join(REPO, name), os.path.join(tmp, "src", name)) - for part in MINIMIND_PARTS: - s = os.path.join(MINIMIND, part) - (_copytree if os.path.isdir(s) else _copy)(s, os.path.join(tmp, "third_party", "minimind", part)) rec = recipe(key, cfg, plan[key], docmix) _json(os.path.join(tmp, "recipe.json"), rec) _write(os.path.join(tmp, "requirements-train.txt"), REQUIREMENTS) diff --git a/demos/translation/tests/test_bundle.py b/demos/translation/tests/test_bundle.py index cebd6f8..1b66533 100644 --- a/demos/translation/tests/test_bundle.py +++ b/demos/translation/tests/test_bundle.py @@ -97,7 +97,8 @@ def test_build_records_best_validation_and_leaks_no_paths(bundle, run_dir): assert m["training"]["step"] == 20 and m["training"]["valid_ce"] == 2.5 # min valid CE, not the last assert m["training"]["tokens_seen"] == 2000 and m["prompt_format"] == "pretrain_tags" assert m["eval"]["flores_plus"] == ["dev-00000-of-00001.parquet", "devtest-00000-of-00001.parquet"] - assert "model" in os.listdir(os.path.join(bundle, "third_party", "minimind")) + assert not os.path.exists(os.path.join(bundle, "third_party")) # no vendored kernel since 0.3.2 + assert "third_party" not in m home = os.path.expanduser("~") for rel in ("recipes/source_run_spec.json", m["model"]["config"], "bundle.json", "recommended.json"): text = open(os.path.join(bundle, rel), encoding="utf-8").read() diff --git a/diagnose_checkpoint.py b/diagnose_checkpoint.py index ab6e65e..aa9aa3b 100644 --- a/diagnose_checkpoint.py +++ b/diagnose_checkpoint.py @@ -22,7 +22,7 @@ from torch.utils.data import DataLoader from transformers import AutoTokenizer -import chronos.deps # noqa: F401 +import chronos.deps # default tokenizer path from chronos.backend import resolve_training_device from chronos.data.flexible_dataset import FlexibleDataset, StreamingSFTDataset from chronos.eval.io_profiler import evaluate_lookahead_accuracy, estimate_cache_hit_rate diff --git a/pyproject.toml b/pyproject.toml index e35c8d7..1663d21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "Project_Chronos" -version = "0.3.1a3" +version = "0.3.2a0" description = "Storage-aware recurrent-depth MoE: lookahead expert prefetch, SSD-lazy experts, hybrid MLA/SWA attention, self-speculative decoding" readme = "README.md" license = "Apache-2.0" @@ -68,6 +68,7 @@ include = ["chronos*", "ui*"] ui = ["event_contract.js", "presets_v3.json"] "chronos.monitor" = ["static/*.html", "static/*.js", "static/*.css"] "chronos.webui" = ["static/*.html", "static/*.js", "static/*.css", "static/*.svg", "i18n/*.json"] +"chronos.assets" = ["tokenizers/*/*.json", "tokenizers/*/LICENSE", "tokenizers/*/NOTICE"] [tool.black] line-length = 100 diff --git a/tests/test_interface_regressions.py b/tests/test_interface_regressions.py index f80c311..42696bb 100644 --- a/tests/test_interface_regressions.py +++ b/tests/test_interface_regressions.py @@ -168,7 +168,9 @@ def inject(model, ids, labels): events = [] result = trainer.run_mlx_stage(stage="pretrain", config=cfg, checkpoint_path=None, save_dir=str(tmp_path), loader=DataLoader(TensorDataset(ids, ids), batch_size=1), - args=SimpleNamespace(steps=3, epochs=1, log_interval=1, dtype="bfloat16", grad_clip=1.), + # mlx_compile off: the injected loss changes per Python call, which a + # compiled graph (traced once) would not replay. + args=SimpleNamespace(steps=3, epochs=1, log_interval=1, dtype="bfloat16", grad_clip=1., mlx_compile="off"), progress_callback=events.append) assert result.failure_reason == "finite_loss_divergence" assert result.stopped and result.checkpoint_path.endswith(".recovery.pth") diff --git a/tests/test_lazy_experts.py b/tests/test_lazy_experts.py index 45db7c0..f85de48 100644 --- a/tests/test_lazy_experts.py +++ b/tests/test_lazy_experts.py @@ -120,8 +120,8 @@ def test_class_default_is_off_and_new_config_entry_points_turn_it_on(monkeypatch import train_chronos monkeypatch.setattr(sys, "argv", ["train_chronos.py", "--data_path", "unused.jsonl"]) assert train_chronos.parse_args().lambda_cluster_locality == DEFAULT_LAMBDA - from ui.presets import MINIMIND_MOE_DEFAULTS - assert MINIMIND_MOE_DEFAULTS["lambda_cluster_locality"] == DEFAULT_LAMBDA + from ui.presets import BASE_MOE_DEFAULTS + assert BASE_MOE_DEFAULTS["lambda_cluster_locality"] == DEFAULT_LAMBDA def test_finetune_spec_turns_cluster_locality_on_by_default(tmp_path): diff --git a/tests/test_longrun.py b/tests/test_longrun.py index 7ff0533..b8a3614 100644 --- a/tests/test_longrun.py +++ b/tests/test_longrun.py @@ -39,6 +39,7 @@ def _train(run, steps): import time if run.state["phase_total"] is None: run.state["phase_total"] = 12 # fixed schedule (skip the throughput probe) + run.spec.schedule = "steps" # LR by step, not by wall clock (compile traces take time) run.run_phase(0, time.monotonic() + 3600, max_steps=steps) @@ -64,6 +65,63 @@ def test_resume_is_bit_exact(tmp_path): assert worst == 0.0, worst +def test_compiled_and_recomputed_steps_match_eager_inside_longrun(tmp_path, monkeypatch): + """Inside the real LongRun loop (accumulation, clipping, AdamW, dropout + and the cache-miss simulation on), every micro step's gradients with + ``mx.compile`` and/or activation recomputation equal the eager ones on + the same weights, batch and RNG state. + + (Whole trajectories are not compared: AdamW's first updates are close to + sign(g), so FP32 rounding differences of 1e-7 between fused and unfused + kernels flip tiny components and the weights drift apart by ~lr.)""" + import random + from mlx.utils import tree_flatten + from chronos.mlx.training.compiled import CompiledStep + from chronos.mlx.training.longrun import LongRun + monkeypatch.delenv("CHRONOS_MLX_COMPILE", raising=False) + shards = str(tmp_path / "shards"); _tiny_shards(shards) + spec = _spec(str(tmp_path / "run"), shards) + spec.model_config.update(dropout=0.1, fallback_mask_prob=0.2) + spec.extra = {"mlx_compile": "off"} + worst = {} + with mx.stream(mx.cpu): + run = LongRun(spec, log=lambda *_: None) + model, plain = run.model, run._micro_step + compiled = CompiledStep(model, run._loss_fn, mode="on") + + def flat(g): + return {k: mx.array(v) for k, v in tree_flatten(g)} + + def check(grad_fn, ids, labels): + seed, state = int(mx.random.randint(0, 2 ** 30, ()).item()), random.getstate() + variants = {} + for name, fn, ckpt in (("recompute", grad_fn, True), ("compiled", compiled, False), + ("both", compiled, True)): + mx.random.seed(seed); random.setstate(state) + model.config.activation_checkpointing = ckpt + variants[name] = flat(plain(fn, ids, labels)[1]) + model.config.activation_checkpointing = False + mx.random.seed(seed); random.setstate(state) + loss, grads = plain(grad_fn, ids, labels) + ref = flat(grads) + norm = sum(float(mx.sum(v * v)) for v in ref.values()) ** 0.5 + for name, g in variants.items(): + err = sum(float(mx.sum((g[k] - ref[k]) ** 2)) for k in ref) ** 0.5 / norm + worst[name] = max(worst.get(name, 0.0), err) + return loss, grads + + run._micro_step = check + _train(run, 3) + assert compiled.stats["eager"] == 0 and compiled.stats["compiled"] == 12 + assert set(worst) == {"recompute", "compiled", "both"} + # Separately each is a reordering of the same FP32 sums (measured + # <= 4e-7 / 1.7e-5). Together the fused recomputed MLA backward reorders + # more of them: measured up to 1e-4 relative, while scaling these weights + # by (1 + 1e-7 * N(0,1)) alone moves the eager gradient by 1.8e-4. A + # structural error (a dropped cotangent, a redrawn mask) is >= 1e-1. + assert worst["recompute"] <= 1e-5 and worst["compiled"] <= 1e-4 and worst["both"] <= 1e-3, worst + + def test_task_eval_hook_feeds_validation_and_final_rows(tmp_path): """A task extension adds metrics by overriding ``task_eval``; the kernel has none.""" import json diff --git a/tests/test_mlx_checkpointing.py b/tests/test_mlx_checkpointing.py new file mode 100644 index 0000000..35bc7b1 --- /dev/null +++ b/tests/test_mlx_checkpointing.py @@ -0,0 +1,225 @@ +"""MLX activation recomputation and the compiled training step: exactness. + +Both are training-only transforms, so every test compares against the +plain eager step on the same weights, batch and RNG seed: loss, every +gradient, router side outputs, loss components and the RNG stream. +""" +from __future__ import annotations + +import os +import sys + +import pytest + +_repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if _repo_root not in sys.path: + sys.path.insert(0, _repo_root) + +mx = pytest.importorskip("mlx.core") +nn = pytest.importorskip("mlx.nn") +from mlx.utils import tree_flatten # noqa: E402 + +CASES = { + "shared_first": {}, + "per_iteration": dict(recurrent_attention_mode="per_iteration"), + "first_only": dict(recurrent_attention_mode="first_only"), + "router_shared": dict(router_scope="shared"), + "no_recurrence": dict(recurrent_depth=1), + "dense_attention": dict(use_hybrid_attention=False), + # SWA window >= sequence: the cache slice k[:, -keep:] is the same graph + # node as the shared KV view (short SFT buckets hit this). + "window_covers_sequence": dict(sliding_window_size=64), +} + + +def _config(**kw): + from chronos.model.config import ChronosConfig + base = dict(architecture_version=3, hidden_size=64, num_hidden_layers=4, prelude_layers=1, coda_layers=1, + recurrent_depth=3, min_recurrent_depth=2, num_attention_heads=4, num_key_value_heads=2, + kv_latent_dim=16, rope_dim=8, num_experts=6, num_experts_per_tok=2, num_shared_experts=1, + moe_intermediate_size=32, vocab_size=97, max_position_embeddings=64, sliding_window_size=8, + dropout=0.1, fallback_mask_prob=0.2, adaptive_halting=True, lookahead_steps=2, + layer_aware_lookahead=True) + base.update(kw) + return ChronosConfig(**base) + + +def _model(cfg): + from chronos.mlx.model import ChronosMLXModel + mx.random.seed(0) + model = ChronosMLXModel(cfg) + mx.eval(model.parameters()) + model.train() + return model + + +def _loss(model, ids, labels): + from chronos.mlx.training.trainer import _chronos_regularized_loss + logits, lookahead, _ = model(ids, logits_dtype=mx.float32) + ce = nn.losses.cross_entropy(logits.reshape(-1, logits.shape[-1]), labels.reshape(-1), reduction="mean") + return _chronos_regularized_loss(model, ce, lookahead, model.config, metrics={"ce": ce, "primary": ce}) + + +def _batch(cfg, b=2, s=12): + mx.random.seed(1) + return mx.random.randint(3, cfg.vocab_size, (b, s)), mx.random.randint(3, cfg.vocab_size, (b, s)) + + +def _step(model, grad_fn, ids, labels, seed=3): + from chronos.mlx.training.trainer import _collect_router_probs + mx.random.seed(seed) + loss, grads = grad_fn(model, ids, labels) + probs = _collect_router_probs(model) + comps = dict(model._last_loss_components) + after = mx.random.state[0] + mx.eval(loss, grads, probs, comps, after) + return dict(loss=loss, grads=dict(tree_flatten(grads)), probs=probs, comps=comps, rng=after, + lens=[len(layer.mlp.iteration_router_probs) for layer in model.layers]) + + +def _assert_same(a, b, tol): + assert abs(float(a["loss"]) - float(b["loss"])) <= tol + assert a["grads"].keys() == b["grads"].keys() + worst = max(float(mx.abs(a["grads"][k] - b["grads"][k]).max()) for k in a["grads"]) + assert worst <= tol, worst + assert a["lens"] == b["lens"] + assert float(mx.abs(a["probs"] - b["probs"]).max()) <= tol + assert a["comps"].keys() == b["comps"].keys() + for k in a["comps"]: + assert float(mx.abs(a["comps"][k] - b["comps"][k]).max()) <= tol, k + assert bool(mx.all(a["rng"] == b["rng"])), "RNG stream consumed differently" + + +@pytest.fixture(autouse=True) +def _cpu(): + with mx.stream(mx.cpu): + yield + + +@pytest.mark.parametrize("case", sorted(CASES)) +def test_recomputation_matches_plain_backward(case): + """Dropout and the cache-miss simulation are replayed, so the gradients + are those of the network the forward pass ran (a plain ``mx.checkpoint`` + redraws them: same loss, different gradients).""" + cfg = _config(**CASES[case]) + model = _model(cfg) + ids, labels = _batch(cfg) + grad_fn = nn.value_and_grad(model, _loss) + plain = _step(model, grad_fn, ids, labels) + model.config.activation_checkpointing = True + recomputed = _step(model, grad_fn, ids, labels) + _assert_same(plain, recomputed, 1e-6) + zero = [k for k, g in recomputed["grads"].items() if float(mx.abs(g).max()) == 0.0] + assert zero == [k for k, g in plain["grads"].items() if float(mx.abs(g).max()) == 0.0] + + +def test_repeated_region_outputs_keep_their_gradients(): + """mlx 0.32.4 drops the cotangent of a repeated checkpoint output; the + shared_first block returns its KV as both new cache and shared view, and + a full-length cache slice is the same node without being the same object.""" + from chronos.mlx.checkpointing import call_block + + class Twice(nn.Module): + def __init__(self): + super().__init__() + self.w = mx.array([0.5, -1.5]) + + def __call__(self, x): + y = mx.sin(x * self.w) + # Same object, a full-length slice and a same-dtype cast: in mlx + # 0.32.4 all three are the same graph node as ``y``. + return y, (y, y[-10:], y.astype(y.dtype), 2) + + block, x = Twice(), mx.array([0.3, 1.2]) + + def loss(m, x, wrap): + out, (again, sliced, cast, k) = call_block(m, x) if wrap else m(x) + assert k == 2 + return (out + 3 * again + 5 * sliced + 7 * cast).sum() + + ref = nn.value_and_grad(block, lambda m, x: loss(m, x, False))(block, x)[1]["w"] + got = nn.value_and_grad(block, lambda m, x: loss(m, x, True))(block, x)[1]["w"] + assert float(mx.abs(ref - got).max()) < 1e-6 and float(mx.abs(ref).max()) > 0 + + +def test_recomputation_skips_cached_calls_and_inference(): + """A block that receives a KV cache (GRPO rollouts in train mode) and any + eval-mode forward run eagerly and return real caches.""" + cfg = _config(activation_checkpointing=True, dropout=0.0, fallback_mask_prob=0.0) + model = _model(cfg) + ids, _ = _batch(cfg, b=1, s=6) + full, _, _ = model(ids) + _, _, cache = model(ids[:, :5]) + step, _, _ = model(ids[:, 5:], cache=cache) + assert float(mx.abs(step[:, -1] - full[:, -1]).max()) < 1e-4 + model.eval() + ref, _, _ = model(ids) + model.config.activation_checkpointing = False + assert float(mx.abs(ref - model(ids)[0]).max()) == 0.0 + + +@pytest.mark.parametrize("checkpointing", [False, True]) +@pytest.mark.parametrize("case", ["shared_first", "per_iteration", "router_shared", "dense_attention"]) +def test_compiled_step_matches_eager(case, checkpointing): + from chronos.mlx.training.compiled import CompiledStep + cfg = _config(**CASES[case], activation_checkpointing=checkpointing) + model = _model(cfg) + ids, labels = _batch(cfg) + eager = nn.value_and_grad(model, _loss) + step = CompiledStep(model, _loss, mode="on") + for seed in (3, 4): # trace, then replay + _assert_same(_step(model, eager, ids, labels, seed), _step(model, step, ids, labels, seed), 1e-6) + assert step.stats == {"compiled": 2, "eager": 0, "traces": 1} + + +def test_compiled_step_retraces_on_python_decisions(): + """Shapes, the ACT switch (training progress) and a toggled config field + each select their own graph; replays never reuse a stale one.""" + from chronos.mlx.training.compiled import CompiledStep + cfg = _config(depth_schedule="fixed_then_adaptive", depth_schedule_warmup_fraction=0.5) + model = _model(cfg) + eager = nn.value_and_grad(model, _loss) + step = CompiledStep(model, _loss, mode="on") + controller = model.recurrent_controller + for progress, shape, coef in ((0.1, (2, 12), 0.0), (0.9, (2, 12), 0.0), (0.9, (3, 8), 0.0), + (0.9, (2, 12), 0.01), (0.1, (2, 12), 0.0)): + controller.set_training_progress(progress, 1.0) + model.config.recurrent_distill_coef = coef + ids, labels = _batch(cfg, *shape) + _assert_same(_step(model, eager, ids, labels), _step(model, step, ids, labels), 1e-6) + assert step.stats["traces"] == 4 and step.stats["compiled"] == 5 + + +def test_compiled_step_pins_one_sampled_depth_per_step(): + """With depth jitter the depth is drawn once (Python RNG) and the graph + for that depth is used, so eager and compiled see the same depth.""" + import random + from chronos.mlx.training.compiled import CompiledStep + cfg = _config(recurrent_depth=4, min_recurrent_depth=1, recurrent_depth_jitter=1.0, + depth_schedule_warmup_fraction=0.0) + model = _model(cfg) + model.recurrent_controller.set_training_progress(1.0, 1.0) + ids, labels = _batch(cfg) + eager = nn.value_and_grad(model, _loss) + step = CompiledStep(model, _loss, mode="on") + for seed in range(6): + random.seed(seed) + a = _step(model, eager, ids, labels, seed) + random.seed(seed) + b = _step(model, step, ids, labels, seed) + _assert_same(a, b, 1e-6) + assert step.stats["traces"] >= 2 # several depths were drawn + + +def test_compiled_step_falls_back_for_unsupported_inputs(): + from chronos.mlx.training.compiled import CompiledStep + cfg = _config() + model = _model(cfg) + lines = [] + step = CompiledStep(model, lambda m, batch: _loss(m, *batch), mode="auto", log=lines.append) + ids, labels = _batch(cfg) + loss, _ = step(model, (ids, labels)) + mx.eval(loss) + assert step.stats["eager"] == 1 and lines == ["[mlx-compile] eager step: non-array batch argument"] + with pytest.raises(ValueError): + CompiledStep(model, _loss, mode="sometimes") diff --git a/tests/test_native_kernel.py b/tests/test_native_kernel.py new file mode 100644 index 0000000..d559e6b --- /dev/null +++ b/tests/test_native_kernel.py @@ -0,0 +1,236 @@ +"""Native kernel (0.3.2+): no minimind, same numerics, working dense mode.""" +from __future__ import annotations + +import math +import os +import subprocess +import sys + +import pytest +import torch + +_repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if _repo_root not in sys.path: + sys.path.insert(0, _repo_root) + +from chronos.model import kernel # noqa: E402 +from chronos.model.config import ChronosConfig # noqa: E402 +from chronos.model.model_chronos import ChronosForCausalLM # noqa: E402 + +MODES = ("shared_first", "per_iteration", "first_only") + + +def test_import_needs_no_external_kernel(tmp_path): + """``import chronos`` must not clone, download or edit sys.path.""" + code = ( + "import sys; before = list(sys.path); import chronos, chronos.model.model_chronos, chronos.deps;" + "import chronos.trainer.chronos_trainer, chronos.trainer.sft_trainer;" + "assert sys.path == before, set(sys.path) ^ set(before);" + "bad = [m for m in ('model', 'trainer', 'dataset', 'model.model_minimind') if m in sys.modules];" + "assert not bad, bad;" + "assert not any('minimind' in p.lower() for p in sys.path)" + ) + env = {**os.environ, "CHRONOS_BACKEND": "cpu", "PYTHONPATH": _repo_root, + "HOME": str(tmp_path)} # no ~/.cache/chronos checkout to find + env.pop("CHRONOS_MINIMIND_DIR", None) + out = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=300) + assert out.returncode == 0, out.stderr[-2000:] + + +def test_base_config_defaults_match_previous_releases(): + c = kernel.ChronosBaseConfig() + assert (c.hidden_size, c.num_hidden_layers, c.vocab_size, c.bos_token_id, c.eos_token_id) == (768, 8, 6400, 1, 2) + assert (c.num_attention_heads, c.num_key_value_heads, c.head_dim) == (8, 4, 96) + assert c.intermediate_size == math.ceil(768 * math.pi / 64) * 64 == 2432 + assert (c.max_position_embeddings, c.rms_norm_eps, c.rope_theta) == (32768, 1e-6, 1e6) + assert c.rope_scaling is None and c.tie_word_embeddings is True and c.hidden_act == "silu" + assert (c.num_experts, c.num_experts_per_tok, c.norm_topk_prob, c.router_aux_loss_coef) == (4, 1, True, 5e-4) + y = kernel.ChronosBaseConfig(hidden_size=512, num_attention_heads=8, inference_rope_scaling=True) + assert y.head_dim == 64 and y.intermediate_size == 1664 and y.moe_intermediate_size == 1664 + # transformers >= 5.2x adds its own ``rope_type``/``rope_theta`` keys; the YaRN + # fields (all the RoPE tables read) are the defaults, and the shared dict is untouched. + assert {k: y.rope_scaling[k] for k in kernel.YARN_DEFAULT} == kernel.YARN_DEFAULT + assert y.rope_scaling is not kernel.YARN_DEFAULT and set(kernel.YARN_DEFAULT) == { + "beta_fast", "beta_slow", "factor", "original_max_position_embeddings", "attention_factor", "type"} + long = 3 * kernel.YARN_DEFAULT["original_max_position_embeddings"] + assert torch.equal(kernel.precompute_freqs_cis(32, long, 1e6, y.rope_scaling)[0], + kernel.precompute_freqs_cis(32, long, 1e6, kernel.YARN_DEFAULT)[0]) + assert isinstance(ChronosConfig(), kernel.ChronosBaseConfig) + + +def test_rope_closed_form_and_layout(): + cos, sin = kernel.precompute_freqs_cis(8, 16, rope_base=10000.0) + assert cos.dtype == torch.float32 and cos.shape == (16, 8) + inv = 1.0 / (10000.0 ** (torch.arange(0, 8, 2).float() / 8)) + ang = torch.arange(16).float()[:, None] * inv[None] + torch.testing.assert_close(cos[:, :4], torch.cos(ang), rtol=0, atol=1e-6) + torch.testing.assert_close(sin[:, 4:], torch.sin(ang), rtol=0, atol=1e-6) # duplicated halves + assert torch.equal(cos[0], torch.ones(8)) and torch.equal(sin[0], torch.zeros(8)) + + +@pytest.mark.parametrize("scaling", [None, kernel.YARN_DEFAULT]) +def test_shared_rope_tables_are_bit_identical_and_shared(scaling): + kernel.clear_rope_tables() + for length in (5, 64, 2048, 2049, 4100, 9000, 100): + cos, sin = kernel.rope_tables(32, length, 1e6, scaling, "cpu") + ref_cos, ref_sin = kernel.precompute_freqs_cis(32, length, 1e6, scaling) + assert cos.shape[0] >= length + assert torch.equal(cos[:length], ref_cos) and torch.equal(sin[:length], ref_sin), length + a = kernel.rope_tables(32, 10, 1e6, scaling, "cpu")[0] + b = kernel.rope_tables(32, 20, 1e6, scaling, "cpu")[0] + assert a.data_ptr() == b.data_ptr() # one table, not a copy + + +def test_model_layers_share_one_rope_table(): + kernel.clear_rope_tables() + cfg = ChronosConfig(hidden_size=64, num_hidden_layers=4, num_attention_heads=4, num_key_value_heads=2, + vocab_size=97, num_experts=4, num_experts_per_tok=2, num_shared_experts=1, + max_position_embeddings=128, recurrent_depth=2, prelude_layers=1, coda_layers=1, + dropout=0.0, use_hybrid_attention=False) + model = ChronosForCausalLM(cfg).eval() + with torch.no_grad(): + model(torch.randint(3, 97, (1, 12))) + ptrs = {m._rope_cos.data_ptr() for m in model.modules() if getattr(m, "_rope_cos", None) is not None} + assert len(ptrs) == 1 + assert model.model.freqs_cos.data_ptr() in ptrs + model.to(torch.bfloat16) + assert model.model.freqs_cos.dtype == torch.float32 # never rounded + + +def dense_model(mode, **kw): + torch.manual_seed(7) + cfg = ChronosConfig(hidden_size=64, num_hidden_layers=4, num_attention_heads=4, num_key_value_heads=2, + vocab_size=96, num_experts=4, num_experts_per_tok=2, num_shared_experts=1, + moe_intermediate_size=64, max_position_embeddings=128, recurrent_depth=3, + prelude_layers=1, coda_layers=1, dropout=0.0, fallback_mask_prob=0.0, + use_hybrid_attention=False, recurrent_attention_mode=mode, **kw) + return ChronosForCausalLM(cfg).eval() + + +@pytest.mark.parametrize("mode", MODES) +def test_dense_gqa_mode_prefill_decode_matches_full(mode): + """``use_hybrid_attention=False`` crashed before 0.3.2 in every recurrent mode.""" + from chronos.model.hybrid_attention import GlobalAttention + model = dense_model(mode) + assert all(isinstance(layer.self_attn, GlobalAttention) for layer in model.model.layers) + ids = torch.randint(3, 96, (2, 20)) + with torch.no_grad(): + full = model(ids).logits + out = model(ids[:, :12], use_cache=True) + cache, steps = out.past_key_values, [] + for t in range(12, 20): + out = model(ids[:, t:t + 1], past_key_values=cache, use_cache=True) + cache = out.past_key_values + steps.append(out.logits[:, -1]) + mask = torch.ones_like(ids) + mask[1, :5] = 0 + masked = model(ids, attention_mask=mask).logits + torch.testing.assert_close(torch.stack(steps, 1), full[:, 12:20], rtol=0, atol=1e-5) + assert torch.isfinite(masked).all() + torch.testing.assert_close(masked[0], full[0], rtol=0, atol=1e-5) # unpadded row unchanged + + +def test_dense_gqa_int8_kv_and_speculative(): + from chronos.runtime.speculative import speculative_generate + ref_model, q_model = dense_model("shared_first"), dense_model("shared_first", kv_cache_dtype="int8") + ids = torch.randint(3, 96, (1, 9)) + with torch.no_grad(): + full = ref_model(ids).logits[:, -1] + pre = q_model(ids[:, :8], use_cache=True) + step = q_model(ids[:, 8:], past_key_values=pre.past_key_values, use_cache=True).logits[:, -1] + assert (step - full).abs().max() < 5e-2 + seq = ids[0].tolist() + out = ref_model(ids, use_cache=True) + cache = out.past_key_values + for _ in range(12): + seq.append(int(out.logits[0, -1].argmax())) + out = ref_model(torch.tensor([[seq[-1]]]), past_key_values=cache, use_cache=True) + cache = out.past_key_values + spec, _stats = speculative_generate(ref_model, ids, max_new_tokens=12, gamma=4, draft_depth=1, gamma_max=8) + assert spec[0].tolist() == seq + + +def test_logger_sinks_receive_lines(capsys): + lines = [] + with kernel.add_log_sink(lines.append): + kernel.Logger("step 1") + kernel.Logger({"loss": 2}) + kernel.Logger("after") + assert lines == ["step 1", "{'loss': 2}"] + assert "after" in capsys.readouterr().out + + +def test_trainer_modules_use_the_shared_logger(): + import chronos.trainer.chronos_trainer as ct + import chronos.trainer.sft_trainer as st + from chronos.workbench.training import _trainer_logger_sink + assert ct.Logger is kernel.Logger and st.Logger is kernel.Logger + seen = [] + with _trainer_logger_sink(seen.append): + ct.Logger("from trainer") # name bound at import time + assert seen == ["from trainer"] + + +def test_resume_state_roundtrip_keeps_dtype(tmp_path): + net = torch.nn.Linear(4, 3).to(torch.bfloat16) + opt = torch.optim.AdamW(net.parameters(), lr=1e-3) + net(torch.randn(2, 4, dtype=torch.bfloat16)).float().sum().backward() + opt.step() + path = kernel.save_resume_state(str(tmp_path / "sub" / "r.pth"), model=net, optimizer=opt, + epoch=2, step=17, wandb_id="abc", scaler=None) + assert not os.path.exists(path + ".tmp") + data = kernel.load_resume_state(path) + assert (data["epoch"], data["step"], data["world_size"], data["wandb_id"]) == (2, 17, 1, "abc") + assert "scaler" not in data + assert data["model"]["weight"].dtype == torch.bfloat16 + assert torch.equal(data["model"]["weight"], net.weight.detach()) + opt.load_state_dict(data["optimizer"]) + assert kernel.load_resume_state(str(tmp_path / "missing.pth")) is None + + +def test_legacy_tokenizer_is_bundled(monkeypatch, tmp_path): + import chronos.deps as deps + monkeypatch.delenv("CHRONOS_DEFAULT_TOKENIZER", raising=False) + path = deps.get_tokenizer_path() + for name in ("tokenizer.json", "tokenizer_config.json", "LICENSE", "NOTICE"): + assert os.path.isfile(os.path.join(path, name)), name + transformers = pytest.importorskip("transformers") + tok = transformers.AutoTokenizer.from_pretrained(path) + assert len(tok) == 6400 + assert tok.decode(tok("hello 世界")["input_ids"], skip_special_tokens=True).strip() == "hello 世界" + monkeypatch.setenv("CHRONOS_DEFAULT_TOKENIZER", str(tmp_path)) + with pytest.raises(FileNotFoundError): + deps.get_tokenizer_path() + with pytest.warns(DeprecationWarning): + assert deps.ensure_minimind() is None + + +def test_pretrain_save_keeps_bf16_release_checkpoint(tmp_path): + """Before 0.3.2 the resume helper re-wrote chronos__moe.pth as FP16.""" + import types + from chronos.trainer.chronos_trainer import ChronosTrainer + cfg = ChronosConfig(hidden_size=64, num_hidden_layers=4, num_attention_heads=4, num_key_value_heads=2, + vocab_size=97, num_experts=4, num_experts_per_tok=2, num_shared_experts=1, + max_position_embeddings=64, recurrent_depth=2, prelude_layers=1, coda_layers=1, dropout=0.0) + args = types.SimpleNamespace(device="cpu", dtype="bfloat16", learning_rate=1e-4, save_dir=str(tmp_path), + epochs=1, accumulation_steps=1, grad_clip=1.0, log_interval=1, save_interval=1) + trainer = ChronosTrainer(cfg, args) + trainer._save(0, 3) + state = torch.load(tmp_path / "chronos_64_moe.pth", map_location="cpu", weights_only=False) + state = state.get("model", state) + assert {v.dtype for v in state.values() if torch.is_tensor(v) and v.is_floating_point()} == {torch.bfloat16} + resume = kernel.load_resume_state(str(tmp_path / "chronos_64_moe_resume.pth")) + assert resume["step"] == 3 and resume["optimizer"] is not None + + +def test_rope_tables_under_meta_device_context(): + """The planner counts parameters under ``torch.device('meta')``.""" + kernel.clear_rope_tables() + with torch.device("meta"): + cos, _ = kernel.rope_tables(16, 32, 1e6, None, "cpu") + model = ChronosForCausalLM(ChronosConfig(hidden_size=64, num_hidden_layers=4, num_attention_heads=4, + num_key_value_heads=2, vocab_size=97, num_experts=4, + num_experts_per_tok=2, max_position_embeddings=64)) + assert cos.device.type == "cpu" and torch.isfinite(cos).all() + assert model.model.freqs_cos.device.type == "cpu" + assert next(model.parameters()).device.type == "meta" diff --git a/tests/test_planner_ui.py b/tests/test_planner_ui.py index 26b2233..d7676ea 100644 --- a/tests/test_planner_ui.py +++ b/tests/test_planner_ui.py @@ -138,7 +138,7 @@ def test_v3_presets_build_and_fill_every_widget(): CONFIG_INPUT_ORDER, DEFAULT_PRESET, V3_PRESET_META, get_preset, preset_names, values_in_input_order, ) names = preset_names() - assert DEFAULT_PRESET in names and "MiniMind-MoE (default)" in names + assert DEFAULT_PRESET in names and "Chronos-v2 legacy (768H, 6400 vocab)" in names assert sum(n.startswith("Chronos-v3") for n in names) == 5 for name in names: cfg = get_preset(name) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index eb20d75..46c9aaf 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -9,7 +9,6 @@ if _repo_root not in sys.path: sys.path.insert(0, _repo_root) -import chronos.deps # auto-bootstrap minimind on sys.path import torch from torch.utils.data import DataLoader, TensorDataset @@ -1742,6 +1741,55 @@ class Args: assert any(e.get("event") == "save" and e.get("reason") == "final" for e in events) +def test_mlx_compiled_step_nan_rolls_back(tmp_path, monkeypatch): + """The default (compiled) step: a NaN that comes from the data, not from + a swapped Python function, still trips the rollback, and the run keeps + compiling afterwards.""" + if not _mlx_available(): + return + import mlx.core as mx + from chronos.model.config import ChronosConfig + import chronos.mlx.training.trainer as trainer_mod + from chronos.mlx.training import run_mlx_stage + + monkeypatch.delenv("CHRONOS_MLX_COMPILE", raising=False) + cfg = ChronosConfig(hidden_size=32, num_hidden_layers=1, num_experts=2, + num_experts_per_tok=1, num_shared_experts=1, + num_attention_heads=2, num_key_value_heads=2, + kv_latent_dim=8, rope_dim=4, max_seq_len=16, + vocab_size=64, use_moe=True, + lambda_balance=0.0, lambda_temporal=0.0, + lambda_lookahead=0.0, lambda_lookahead_topk=0.0) + ids = torch.randint(1, cfg.vocab_size, (3, 8), dtype=torch.long) + ids[1, 0] = 0 # token 0 marks the poisoned batch + loader = DataLoader(TensorDataset(ids, ids.clone()), batch_size=1) + original_loss = trainer_mod._ce_stage_loss + + def poisoned_loss(model, input_ids, labels): # traced once; NaN decided by the data + loss = original_loss(model, input_ids, labels) + return mx.where(mx.any(input_ids == 0), mx.array(float("nan"), dtype=loss.dtype), loss) + + class Args: + learning_rate = 1e-4 + weight_decay = 0.0 + steps = 3 + epochs = 1 + max_seq_len = 8 + log_interval = 1 + save_interval = 100 + accumulation_steps = 1 + grad_clip = 1.0 + mlx_rollback_limit = 3 + mlx_compile = "on" + + monkeypatch.setattr(trainer_mod, "_ce_stage_loss", poisoned_loss) + events = [] + result = run_mlx_stage(stage="pretrain", config=cfg, checkpoint_path=None, save_dir=str(tmp_path), + loader=loader, args=Args(), progress_callback=lambda event: events.append(dict(event))) + assert result.rollbacks == 1 and result.checkpoint_saved is True + assert any(e.get("event") == "rollback" for e in events) + + def test_mlx_training_nan_rollback_saves_last_finite(tmp_path, monkeypatch): if not _mlx_available(): return @@ -1775,6 +1823,7 @@ class Args: steps = 3 epochs = 1 max_seq_len = 8 + mlx_compile = "off" # the NaN is injected by swapping the Python loss, which a compiled graph calls only while tracing log_interval = 1 save_interval = 100 accumulation_steps = 1 @@ -2132,7 +2181,7 @@ def test_config_input_order_matches_autotune_mapping(): def test_config_schema_includes_unlimited_max_steps_default(): from ui.presets import CONFIG_INPUT_ORDER, get_preset, values_in_input_order - cfg = get_preset("MiniMind-MoE (default)") + cfg = get_preset("Chronos-v2 legacy (768H, 6400 vocab)") assert cfg["max_steps"] == 0 idx = CONFIG_INPUT_ORDER.index("max_steps") assert values_in_input_order(cfg)[idx] == 0 diff --git a/tests/test_smoke_cuda.py b/tests/test_smoke_cuda.py index 06b16fb..551ad67 100644 --- a/tests/test_smoke_cuda.py +++ b/tests/test_smoke_cuda.py @@ -21,7 +21,6 @@ def main(): sys.exit(0) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - import chronos.deps # noqa: F401 from chronos.model.config import ChronosConfig from chronos.model.model_chronos import ChronosForCausalLM from chronos.runtime.cache_manager import CacheManager diff --git a/tests/test_tool_tags_special_tokens.py b/tests/test_tool_tags_special_tokens.py new file mode 100644 index 0000000..1f1a574 --- /dev/null +++ b/tests/test_tool_tags_special_tokens.py @@ -0,0 +1,102 @@ +"""Tool-call tags survive decoding when a custom HF tokenizer marks them special.""" +from __future__ import annotations + +import json +import os +import sys + +import pytest + +_repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if _repo_root not in sys.path: + sys.path.insert(0, _repo_root) + +from chronos.tokenizer.visible import control_token_ids, decode_visible # noqa: E402 + +TOOLS = [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}] +ANSWER = 'Let me check.\n{"name": "get_weather", "arguments": {"city": "Paris"}}' + + +def _hf_tokenizer(tags_special: bool = True): + pytest.importorskip("tokenizers") + from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers + from transformers import PreTrainedTokenizerFast + tk = Tokenizer(models.BPE(unk_token=None)) + tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) + tk.decoder = decoders.ByteLevel() + trainer = trainers.BpeTrainer(vocab_size=300, show_progress=False, + special_tokens=["<|endoftext|>", "<|im_start|>", "<|im_end|>"], + initial_alphabet=pre_tokenizers.ByteLevel.alphabet()) + tk.train_from_iterator([ANSWER, "Let me check the weather."], trainer) + tok = PreTrainedTokenizerFast(tokenizer_object=tk, eos_token="<|im_end|>", pad_token="<|endoftext|>") + tags = ["", "", "", ""] + if tags_special: + tok.add_special_tokens({"additional_special_tokens": tags}) + else: + tok.add_tokens(tags) + return tok + + +@pytest.mark.parametrize("tags_special", [True, False]) +def test_decode_visible_keeps_tags_drops_control(tags_special): + tok = _hf_tokenizer(tags_special) + ids = tok(ANSWER + "<|im_end|>", add_special_tokens=False)["input_ids"] + assert tok.convert_tokens_to_ids("") in ids # registered as one token + if tags_special: + # The bug this guards against: plain skipping deletes the tags. + assert "" not in tok.decode(ids, skip_special_tokens=True) + assert decode_visible(tok, ids) == ANSWER + hidden = control_token_ids(tok) + assert tok.convert_tokens_to_ids("<|im_end|>") in hidden + assert tok.convert_tokens_to_ids("") not in hidden + + +def test_decode_visible_native_tokenizer(): + from chronos.tokenizer.model import ChronosTokenizer, special_tokens_with + from chronos.tokenizer.trainer import BPETrainerConfig, train_bpe + special = special_tokens_with(("", "")) + trained = train_bpe({"en": [ANSWER] * 4}, BPETrainerConfig(vocab_size=300 + len(special), special_tokens=special)) + tok = ChronosTokenizer.from_trained(trained, special) + ids = tok.encode(ANSWER + "<|im_end|>") + assert tok.vocab[""] in ids + assert decode_visible(tok, ids) == ANSWER + + +def test_serving_stream_parses_special_tool_tags(monkeypatch): + """generate_stream → /api/chat and /v1/chat/completions with special tags.""" + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + import chronos.runtime.speculative as spec + from chronos.serving.ollama_server import create_app + from chronos.serving.runtime import ChronosRuntime, LoadedModel + + tok = _hf_tokenizer(True) + script = tok(ANSWER, add_special_tokens=False)["input_ids"] + [tok.convert_tokens_to_ids("<|im_end|>")] + + def fake_generate(model, input_ids, *, on_token=None, **_kw): + for t in script: # one token at a time, as decoding does + on_token(int(t)) + return input_ids, spec.SpecStats() + + monkeypatch.setattr(spec, "speculative_generate", fake_generate) + cfg = type("Cfg", (), {"recurrent_depth": 1, "max_inference_depth": None})() + rt = ChronosRuntime(None) + rt.loaded["m"] = LoadedModel(name="m", path="m.pth", config=cfg, model=None, tokenizer=tok, + engine=None, device="cpu", lazy_experts=False) + rt.encode_chat = lambda lm, messages, tools=None: [1, 2, 3] + client = TestClient(create_app(rt)) + body = {"model": "m", "messages": [{"role": "user", "content": "weather?"}], "tools": TOOLS, "stream": False} + + msg = client.post("/api/chat", json=body).json()["message"] + assert msg["content"] == "Let me check." + assert msg["tool_calls"] == [{"function": {"name": "get_weather", "arguments": {"city": "Paris"}}}] + + out = client.post("/v1/chat/completions", json=body).json()["choices"][0] + assert out["finish_reason"] == "tool_calls" + assert json.loads(out["message"]["tool_calls"][0]["function"]["arguments"]) == {"city": "Paris"} + + events = [json.loads(x[6:]) for x in client.post("/v1/chat/completions", json={**body, "stream": True}).text.split("\n\n") + if x.startswith("data: {")] + streamed = [c for e in events for c in e["choices"][0]["delta"].get("tool_calls", [])] + assert [c["function"]["name"] for c in streamed] == ["get_weather"] + assert not any(" str | None: + """Pop ``--minimind-dir PATH`` / ``--minimind-dir=PATH`` from ``argv``.""" + argv = sys.argv if argv is None else argv + for i, arg in enumerate(list(argv)): + if arg == "--minimind-dir" and i + 1 < len(argv): + value = argv[i + 1] + del argv[i:i + 2] + return value + if arg.startswith("--minimind-dir="): + del argv[i] + return arg.split("=", 1)[1] + return None + + +def require_minimind(path: str | None = None) -> str: + """Put a user-provided minimind checkout on ``sys.path`` and return it.""" + path = path or minimind_dir_from_argv() or os.environ.get("CHRONOS_MINIMIND_DIR", "").strip() + if not path: + raise SystemExit( + "This comparison needs an external minimind checkout as the baseline.\n" + "Chronos no longer bundles or downloads it (since 0.3.2). Clone it yourself, e.g.\n" + " git clone https://github.com/jingyaogong/minimind /path/to/minimind\n" + "and pass --minimind-dir /path/to/minimind (or set CHRONOS_MINIMIND_DIR).\n" + "For a baseline with no external code, run benchmark_compare.py.") + path = os.path.abspath(os.path.expanduser(path)) + if not os.path.isfile(os.path.join(path, "model", "model_minimind.py")): + raise SystemExit(f"{path} is not a minimind checkout (model/model_minimind.py is missing).") + if path not in sys.path: + sys.path.append(path) # after Chronos, so it never shadows chronos.* modules + return path diff --git a/tools/benchmark_chronos_v2.py b/tools/benchmark_chronos_v2.py index 183af01..2c4ea3d 100644 --- a/tools/benchmark_chronos_v2.py +++ b/tools/benchmark_chronos_v2.py @@ -30,7 +30,6 @@ import torch import torch.nn.functional as F -import chronos.deps # noqa: F401 - registers the bundled MiniMind package from chronos.model.config import ChronosConfig from chronos.model.model_chronos import ChronosForCausalLM from chronos.runtime.inference_engine import ChronosInferenceEngine diff --git a/tools/compare_minimind_chronos.py b/tools/compare_minimind_chronos.py index 6e8ce03..01b53bd 100644 --- a/tools/compare_minimind_chronos.py +++ b/tools/compare_minimind_chronos.py @@ -18,7 +18,7 @@ 5. final_ce_loss — sanity that Chronos is in the same ballpark. Run: - python tools/compare_minimind_chronos.py --steps 200 --device cpu \ + python tools/compare_minimind_chronos.py --minimind-dir /path/to/minimind --steps 200 --device cpu \ --output results/compare_results.json """ from __future__ import annotations @@ -42,7 +42,11 @@ if _REPO not in sys.path: sys.path.insert(0, _REPO) -import chronos.deps # noqa: E402 -- ensures minimind on sys.path +from tools._minimind_baseline import require_minimind # noqa: E402 + +require_minimind() # external baseline: --minimind-dir or CHRONOS_MINIMIND_DIR + +import chronos.deps # noqa: E402 (tokenizer path) from transformers import AutoTokenizer # noqa: E402 diff --git a/tools/compare_minimind_chronos_v2.py b/tools/compare_minimind_chronos_v2.py index c6af29c..f99a47d 100644 --- a/tools/compare_minimind_chronos_v2.py +++ b/tools/compare_minimind_chronos_v2.py @@ -31,7 +31,11 @@ if _REPO not in sys.path: sys.path.insert(0, _REPO) -import chronos.deps # noqa: E402 +from tools._minimind_baseline import require_minimind # noqa: E402 + +require_minimind() # external baseline: --minimind-dir or CHRONOS_MINIMIND_DIR + +import chronos.deps # noqa: E402 (tokenizer path) from transformers import AutoTokenizer # noqa: E402 diff --git a/tools/compare_minimind_chronos_v3.py b/tools/compare_minimind_chronos_v3.py index 284aed0..0384ebd 100644 --- a/tools/compare_minimind_chronos_v3.py +++ b/tools/compare_minimind_chronos_v3.py @@ -8,6 +8,7 @@ Run:: python tools/compare_minimind_chronos_v3.py \\ + --minimind-dir /path/to/minimind \\ --pretrain_steps 150 --align_steps 30 --distill_steps 30 \\ --simulated_ssd_ms 30 --device cpu \\ --output results/compare_results_v3.json @@ -28,7 +29,7 @@ if _REPO not in sys.path: sys.path.insert(0, _REPO) -import chronos.deps # noqa: E402 +import chronos.deps # noqa: E402 (tokenizer path) from chronos.backend import BackendDispatcher # noqa: E402 # Reuse the v2 harness entirely, then add stages. diff --git a/train_chronos.py b/train_chronos.py index f4999ce..0f6c412 100644 --- a/train_chronos.py +++ b/train_chronos.py @@ -12,13 +12,12 @@ --device cuda:0 """ import sys -import chronos.deps # ensure minimind on sys.path import argparse from torch.utils.data import DataLoader from chronos.backend import resolve_training_device -from trainer.trainer_utils import setup_seed, Logger +from chronos.model.kernel import Logger, setup_seed from chronos.data.flexible_dataset import ( FlexibleDataset as PretrainDataset, MultimodalStreamingDataset, diff --git a/train_chronos_distill.py b/train_chronos_distill.py index f4ebc4a..08d8d1e 100644 --- a/train_chronos_distill.py +++ b/train_chronos_distill.py @@ -2,7 +2,6 @@ import argparse import os -import chronos.deps # noqa from chronos.backend import resolve_training_device from chronos.model.model_chronos import ChronosForCausalLM @@ -25,7 +24,7 @@ def main(): p = argparse.ArgumentParser() p.add_argument("--data_path", required=True) p.add_argument("--teacher_path", required=True, - help="Path to a Chronos or MiniMind .pth teacher checkpoint.") + help="Path to a Chronos .pth teacher checkpoint.") p.add_argument("--from_weight", default="grpo", help="Student starting checkpoint stem.") p.add_argument("--save_dir", default="out") diff --git a/train_chronos_dpo.py b/train_chronos_dpo.py index 2a19245..2ff7332 100644 --- a/train_chronos_dpo.py +++ b/train_chronos_dpo.py @@ -1,7 +1,6 @@ """train_chronos_dpo.py — Stage 3 (DPO) entry point.""" import argparse -import chronos.deps # noqa import torch from chronos.backend import resolve_training_device diff --git a/train_chronos_grpo.py b/train_chronos_grpo.py index 7946e98..0a91521 100644 --- a/train_chronos_grpo.py +++ b/train_chronos_grpo.py @@ -1,7 +1,6 @@ """train_chronos_grpo.py — Stage 5 (GRPO) entry point.""" import argparse -import chronos.deps # noqa from chronos.backend import resolve_training_device from chronos.model.model_chronos import ChronosForCausalLM diff --git a/train_chronos_orpo.py b/train_chronos_orpo.py index c262b11..3f0fc6d 100644 --- a/train_chronos_orpo.py +++ b/train_chronos_orpo.py @@ -1,7 +1,6 @@ """train_chronos_orpo.py — Stage 4 (ORPO) entry point.""" import argparse -import chronos.deps # noqa import torch from chronos.backend import resolve_training_device diff --git a/train_chronos_sft.py b/train_chronos_sft.py index 4ea06ef..e707c2e 100644 --- a/train_chronos_sft.py +++ b/train_chronos_sft.py @@ -1,7 +1,6 @@ """train_chronos_sft.py — Stage 2 (SFT) entry point.""" import argparse -import chronos.deps # noqa from chronos.backend import resolve_training_device from chronos.model.model_chronos import ChronosForCausalLM diff --git a/ui/estimator.py b/ui/estimator.py index 8beac7d..80152bf 100644 --- a/ui/estimator.py +++ b/ui/estimator.py @@ -15,8 +15,8 @@ } -def _minimind_intermediate(hidden_size: int) -> int: - """Replicates MiniMindConfig's auto-derivation rule.""" +def _base_intermediate(hidden_size: int) -> int: + """ChronosBaseConfig auto-derivation: ceil(h·π/64)·64.""" return int(math.ceil(hidden_size * math.pi / 64) * 64) @@ -28,7 +28,7 @@ class ArchConfig: num_key_value_heads: int = 4 vocab_size: int = 6400 # intermediate_size / moe_intermediate_size: when None, auto-derive from - # hidden_size via the MiniMind rule so the estimate matches the trainer. + # hidden_size via the ChronosBaseConfig rule so the estimate matches the trainer. intermediate_size: Optional[int] = None moe_intermediate_size: Optional[int] = None head_dim: Optional[int] = None diff --git a/ui/i18n.py b/ui/i18n.py index 8027fd3..f7ac7e6 100644 --- a/ui/i18n.py +++ b/ui/i18n.py @@ -152,7 +152,7 @@ # ── M8d: M6/M7/M8 widget labels ───────────────────────────── "config.preset": "预设模板", "config.load_preset": "📥 载入预设", - "config.reset_minimind": "↻ 重置为 MiniMind-MoE", + "config.reset_minimind": "↻ 重置为默认（v2 旧版形状）", "config.cfg_path": "配置文件路径 (.json)", "config.save_cfg": "💾 保存配置到文件", "config.load_cfg": "📂 从文件载入配置", @@ -332,7 +332,7 @@ # ── M8d ─────────────────────────────────────────────────── "config.preset": "預設範本", "config.load_preset": "📥 載入預設", - "config.reset_minimind": "↻ 重置為 MiniMind-MoE", + "config.reset_minimind": "↻ 重置為預設（v2 舊版形狀）", "config.cfg_path": "配置檔案路徑 (.json)", "config.save_cfg": "💾 儲存配置到檔案", "config.load_cfg": "📂 從檔案載入配置", @@ -512,7 +512,7 @@ # ── M8d ─────────────────────────────────────────────────── "config.preset": "Preset", "config.load_preset": "📥 Load Preset", - "config.reset_minimind": "↻ Reset to MiniMind-MoE", + "config.reset_minimind": "↻ Reset to defaults (v2 legacy shape)", "config.cfg_path": "Config file path (.json)", "config.save_cfg": "💾 Save Config to File", "config.load_cfg": "📂 Load Config from File", @@ -692,7 +692,7 @@ # ── M8d ─────────────────────────────────────────────────── "config.preset": "プリセット", "config.load_preset": "📥 プリセット読込", - "config.reset_minimind": "↻ MiniMind-MoE にリセット", + "config.reset_minimind": "↻ 既定値（v2 旧形状）にリセット", "config.cfg_path": "設定ファイルパス (.json)", "config.save_cfg": "💾 設定をファイルに保存", "config.load_cfg": "📂 ファイルから設定を読込", diff --git a/ui/presets.py b/ui/presets.py index 4b5d8be..6a0a760 100644 --- a/ui/presets.py +++ b/ui/presets.py @@ -1,16 +1,17 @@ """ ui/presets.py -Default + preset configurations matching minimind's MoE defaults. +Default + preset configurations for the Config tab. Each preset is a complete dict that can be: 1. Loaded into the Config tab sliders via the "Load Preset" button. - 2. Used as the initial value when the user clicks "Reset to MiniMind". + 2. Used as the initial value when the user clicks "Reset to defaults". 3. Persisted to / loaded from JSON via the Save/Load Config buttons. -The single source of truth for the *minimind* preset is what -``MiniMindConfig(use_moe=True)`` actually produces — we mirror those values -here so the UI shows the same model the trainer would build. +``BASE_MOE_DEFAULTS`` mirrors what ``ChronosConfig`` builds from the +native kernel base (``chronos.model.kernel.ChronosBaseConfig``; the same +field defaults as releases before 0.3.2), so the UI shows the model the +trainer would build. """ from __future__ import annotations @@ -19,11 +20,11 @@ from typing import Dict, List -# ── MiniMind MoE — exact defaults from model/model_minimind.py ───── -# These values match MiniMindConfig(hidden_size=768, num_hidden_layers=8, -# use_moe=True) field-for-field, with Chronos-specific extensions left at -# their conservative defaults (no router-anchor by default). -MINIMIND_MOE_DEFAULTS: Dict = {'architecture_version': 2, +# ── Base MoE defaults (768H × 8L, 6400 vocab) ────────────────────── +# Field-for-field the ChronosBaseConfig(hidden_size=768, num_hidden_layers=8, +# use_moe=True) defaults, with Chronos-specific extensions left at their +# conservative defaults (no router-anchor by default). +BASE_MOE_DEFAULTS: Dict = {'architecture_version': 2, 'recurrent_enabled': True, 'recurrent_groups': 2, 'recurrent_depth': 4, @@ -200,6 +201,8 @@ # Generated by the adaptive planner (chronos.model.planner) and stored as JSON # so importing the UI never runs the search. ``python -m ui.presets`` # regenerates the file from V3_TIER_REQUESTS. +MINIMIND_MOE_DEFAULTS = BASE_MOE_DEFAULTS # name used before 0.3.2 + V3_PRESETS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "presets_v3.json") V3_TIER_REQUESTS = [ @@ -209,7 +212,7 @@ ("Chronos-v3 50M (16k vocab)", "50M MoE (36 experts, top-2) for a 16k Chronos tokenizer; no memory saving.", dict(target_total=50e6, num_experts=36, active_experts=2, vocab_size=16384, seq_len=512, memory_saving="none", ram_gb=32)), - ("Chronos-v3 200M (32 GB)", "General 200M MoE, balanced memory saving, minimind 6400 vocab.", + ("Chronos-v3 200M (32 GB)", "General 200M MoE, balanced memory saving, legacy 6400-token vocabulary.", dict(target_total=200e6, num_experts=32, active_experts=2, vocab_size=6400, seq_len=1024, memory_saving="balanced", ram_gb=32)), ("Chronos-v3 500M (32 GB)", "500M MoE, aggressive memory saving (int8 KV, shared routing, checkpointing).", @@ -228,7 +231,7 @@ def build_v3_presets() -> dict: for name, blurb, request in V3_TIER_REQUESTS: plan = plan_training(PlanRequest(**request)) cfg = dict(plan["config"]) - presets[name] = {k: v for k, v in cfg.items() if MINIMIND_MOE_DEFAULTS.get(k, object()) != v} + presets[name] = {k: v for k, v in cfg.items() if BASE_MOE_DEFAULTS.get(k, object()) != v} arch, mem = plan["report"]["architecture"], plan["report"]["memory"] meta[name] = dict( summary=blurb, request={k: (list(v) if isinstance(v, tuple) else v) for k, v in request.items()}, @@ -248,17 +251,17 @@ def _load_v3_presets() -> tuple[Dict[str, Dict], Dict[str, Dict]]: doc = json.load(fh) except (OSError, ValueError): return {}, {} - return ({name: {**MINIMIND_MOE_DEFAULTS, **cfg} for name, cfg in doc.get("presets", {}).items()}, + return ({name: {**BASE_MOE_DEFAULTS, **cfg} for name, cfg in doc.get("presets", {}).items()}, dict(doc.get("_meta", {}))) V3_PRESETS, V3_PRESET_META = _load_v3_presets() -DEFAULT_PRESET = "Chronos-v3 200M (32 GB)" if "Chronos-v3 200M (32 GB)" in V3_PRESETS else "MiniMind-MoE (default)" +DEFAULT_PRESET = "Chronos-v3 200M (32 GB)" if "Chronos-v3 200M (32 GB)" in V3_PRESETS else "Chronos-v2 legacy (768H, 6400 vocab)" PRESETS: Dict[str, Dict] = { **V3_PRESETS, - # Legacy v2 shape kept for minimind compatibility (first_only recurrence). - "MiniMind-MoE (default)": dict(MINIMIND_MOE_DEFAULTS), + # Legacy v2 shape kept for old checkpoints (first_only recurrence). + "Chronos-v2 legacy (768H, 6400 vocab)": dict(BASE_MOE_DEFAULTS), } # Pre-v3 preset names (saved sessions, docs) resolve to the closest v3 tier. @@ -270,6 +273,7 @@ def _load_v3_presets() -> tuple[Dict[str, Dict], Dict[str, Dict]]: "Chronos-v3 50M (en\u2192zh recipe)": "Chronos-v3 50M (16k vocab)", # 0.3.1a1 tier name "Medium (512H × 8L × 8E)": "Chronos-v3 200M (32 GB)", "Large (1024H × 16L × 16E)": "Chronos-v3 1B (64 GB)", + "MiniMind-MoE (default)": "Chronos-v2 legacy (768H, 6400 vocab)", # before 0.3.2 } @@ -280,7 +284,7 @@ def preset_names() -> List[str]: def get_preset(name: str) -> Dict: """Return a copy of the named preset (avoids accidental mutation).""" name = _PRESET_ALIASES.get(name, name) - cfg = dict(MINIMIND_MOE_DEFAULTS) + cfg = dict(BASE_MOE_DEFAULTS) cfg.update(json.loads(json.dumps(PRESETS.get(name, {})))) # nested stage_overrides copied too return cfg @@ -288,11 +292,11 @@ def get_preset(name: str) -> Dict: def values_in_input_order(cfg: Dict) -> List: """Return preset values in the same order as Config tab's all_inputs. - For keys missing from `cfg`, falls back to MINIMIND_MOE_DEFAULTS. If a + For keys missing from `cfg`, falls back to BASE_MOE_DEFAULTS. If a value is ``None`` (missing in both), it is replaced by a safe-default per field type so Gradio's slider/number widgets don't reject it. """ - # Hardcoded fallbacks for fields whose MiniMind default is None or absent. + # Hardcoded fallbacks for fields whose base default is None or absent. # Values chosen to match the slider minima where applicable. SAFE = { "hidden_size": 768, "num_hidden_layers": 8, @@ -330,7 +334,7 @@ def values_in_input_order(cfg: Dict) -> List: } out = [] for k in CONFIG_INPUT_ORDER: - v = cfg.get(k) if k in cfg else MINIMIND_MOE_DEFAULTS.get(k) + v = cfg.get(k) if k in cfg else BASE_MOE_DEFAULTS.get(k) if v is None: v = SAFE.get(k, 0) if k == "dtype": @@ -362,7 +366,7 @@ def load_config(path: str) -> Dict: """Read JSON and return the dict. Raises FileNotFoundError if missing.""" with open(os.path.abspath(path)) as f: loaded = json.load(f) - cfg = dict(MINIMIND_MOE_DEFAULTS) + cfg = dict(BASE_MOE_DEFAULTS) if not isinstance(loaded, dict): raise ValueError("Configuration JSON must be an object") if "max_position_embeddings" in loaded and "max_seq_len" not in loaded: diff --git a/ui/tabs/autotune_tab.py b/ui/tabs/autotune_tab.py index 818c592..e11fc6e 100644 --- a/ui/tabs/autotune_tab.py +++ b/ui/tabs/autotune_tab.py @@ -18,7 +18,6 @@ from ui.gradio_compat import gr -import chronos.deps # noqa: F401 from ui.i18n import t, register_translatable from ui.presets import CONFIG_INPUT_ORDER from chronos.tuning.chronos_auto_tuner import ChronosAutoTuner, ChronosSearchSpaceConfig diff --git a/ui/tabs/benchmark_tab.py b/ui/tabs/benchmark_tab.py index 52323e8..9c2872c 100644 --- a/ui/tabs/benchmark_tab.py +++ b/ui/tabs/benchmark_tab.py @@ -1,5 +1,5 @@ """ -ui/tabs/benchmark_tab.py — MiniMind vs Chronos comparison view. +ui/tabs/benchmark_tab.py — dense baseline vs Chronos comparison view. Renders the comparison_results.json as: @@ -14,7 +14,6 @@ from ui.gradio_compat import gr import pandas as pd -import chronos.deps # noqa: F401 from ui.i18n import t, register_translatable from chronos.trainer.device_utils import configure_cpu_threads, cpu_thread_snapshot @@ -37,10 +36,10 @@ def _format_table(data: dict) -> str: if not data: return "_No benchmark data yet. Click **Run** to start._" - mm = data.get("minimind", {}) or {} + mm = data.get("baseline") or data.get("minimind") or {} # "minimind": files before 0.3.2 ch = data.get("chronos", {}) or {} - rows = ["| Metric | MiniMind | Chronos | Δ (Chronos − MiniMind) | Better |", + rows = ["| Metric | Baseline | Chronos | Δ (Chronos − Baseline) | Better |", "|---|---|---|---|---|"] for label, key, unit, lower_is_better in CHART_METRICS: mv = mm.get(key) @@ -56,9 +55,9 @@ def _format_table(data: dict) -> str: sign = "−" if d < 0 else ("+" if d > 0 else "±") delta = f"{sign}{abs(d):.4g} {unit}" if lower_is_better: - better = "**Chronos**" if cv < mv else ("MiniMind" if cv > mv else "tie") + better = "**Chronos**" if cv < mv else ("Baseline" if cv > mv else "tie") else: - better = "**Chronos**" if cv > mv else ("MiniMind" if cv < mv else "tie") + better = "**Chronos**" if cv > mv else ("Baseline" if cv < mv else "tie") rows.append(f"| {label} | {mvs} | {cvs} | {delta} | {better} |") extra = [] @@ -82,7 +81,7 @@ def _format_table(data: dict) -> str: def _to_chart_df(data: dict) -> pd.DataFrame: """Convert benchmark JSON into a long-format DataFrame for gr.BarPlot.""" rows = [] - for model_key, label_key in (("minimind", "MiniMind"), ("chronos", "Chronos")): + for model_key, label_key in (("baseline", "Baseline"), ("minimind", "MiniMind"), ("chronos", "Chronos")): d = data.get(model_key, {}) or {} for label, key, unit, _lower in CHART_METRICS: v = d.get(key) @@ -136,7 +135,7 @@ def build_benchmark_tab(): chart = gr.BarPlot( value=initial_df, x="metric", y="value", color="model", - title="MiniMind vs Chronos — comparison metrics", + title="Baseline vs Chronos — comparison metrics", tooltip=["model", "metric", "value", "unit"], height=320, ) diff --git a/ui/tabs/config_tab.py b/ui/tabs/config_tab.py index 9150c85..bd2f2ca 100644 --- a/ui/tabs/config_tab.py +++ b/ui/tabs/config_tab.py @@ -19,7 +19,7 @@ ) from ui.planner_panel import build_planner_panel from ui.presets import ( - CONFIG_INPUT_ORDER, DEFAULT_PRESET, MINIMIND_MOE_DEFAULTS, preset_names, get_preset, values_in_input_order, + CONFIG_INPUT_ORDER, DEFAULT_PRESET, BASE_MOE_DEFAULTS, preset_names, get_preset, values_in_input_order, save_config, load_config, ) @@ -58,7 +58,7 @@ def _estimate(cfg_dict: dict): """Compute estimator outputs from a config dict. A moe_intermediate_size of 0 (or missing) is treated as "auto-derive - from hidden_size", matching MiniMindConfig's default rule. This keeps + from hidden_size", matching the ChronosBaseConfig default rule. This keeps the estimator consistent with what the trainer actually builds. """ arch = ArchConfig.from_config(cfg_dict) @@ -91,7 +91,7 @@ def merge_config_inputs(previous: dict, values: list) -> dict: """Apply editor fields by name while retaining loaded fields without widgets.""" if len(values) != len(CONFIG_INPUT_ORDER): raise ValueError(f"Config editor expects {len(CONFIG_INPUT_ORDER)} values, received {len(values)}") - cfg = dict(previous or MINIMIND_MOE_DEFAULTS) + cfg = dict(previous or BASE_MOE_DEFAULTS) for key, value in zip(CONFIG_INPUT_ORDER, values): if key == "modalities": value = value if isinstance(value, list) else str(value or "").split(",") @@ -99,7 +99,7 @@ def merge_config_inputs(previous: dict, values: list) -> dict: elif key == "dtype": value = _normalize_dtype_name(value) else: - default = MINIMIND_MOE_DEFAULTS.get(key) + default = BASE_MOE_DEFAULTS.get(key) if value is None: value = default if isinstance(default, bool): @@ -510,7 +510,7 @@ def apply_preset(name): return config_result(get_preset(name)) def reset_minimind(): - return config_result(get_preset("MiniMind-MoE (default)")) + return config_result(get_preset("Chronos-v2 legacy (768H, 6400 vocab)")) def do_save(previous, path, *values): try: diff --git a/ui/tabs/data_tab.py b/ui/tabs/data_tab.py index 55bdfd4..c3dbd93 100644 --- a/ui/tabs/data_tab.py +++ b/ui/tabs/data_tab.py @@ -20,7 +20,6 @@ from ui.gradio_compat import gr -import chronos.deps # noqa: F401 from ui.i18n import t, register_translatable from chronos.workbench.data import ( # noqa: F401 (re-exported) _ANSWER_KEYS, _CHAT_KEYS, _PROMPT_KEYS, _extract_text, _fields, _scan, record_schema, diff --git a/ui/tabs/inference_tab.py b/ui/tabs/inference_tab.py index 19bbb1e..532dcff 100644 --- a/ui/tabs/inference_tab.py +++ b/ui/tabs/inference_tab.py @@ -238,7 +238,7 @@ def _sniff_checkpoint(path: str) -> dict: # qw: (n_heads * head_dim, hidden); kw: (n_kv * head_dim, hidden) # head_dim is typically hidden_size // n_heads, but we can't # split q without knowing head_dim. Conservative: assume the - # ratio matches MiniMind defaults (head_dim = h // num_heads). + # ratio matches the ChronosBaseConfig defaults (head_dim = h // num_heads). # Solve via the kw / qw ratio assuming both share head_dim. out["num_attention_heads_total_dim"] = int(qw.shape[0]) out["num_kv_heads_total_dim"] = int(kw.shape[0]) @@ -256,7 +256,7 @@ def _sniff_checkpoint(path: str) -> dict: total_q = int(qnope.shape[0] + qrope.shape[0]) h = out["hidden_size"] if total_q > 0 and h % total_q == 0: - # For MiniMind/Chronos checkpoints head_dim is + # For Chronos checkpoints head_dim is # hidden_size / num_attention_heads, so total_q equals # hidden_size and n_heads is recovered by divisibility. pass diff --git a/ui/tabs/train_tab.py b/ui/tabs/train_tab.py index df27efb..c15a4a9 100644 --- a/ui/tabs/train_tab.py +++ b/ui/tabs/train_tab.py @@ -6,7 +6,6 @@ from ui.gradio_compat import gr -import chronos.deps # noqa: F401 (auto-bootstrap minimind on sys.path) from ui.i18n import t, register_translatable, get_current_lang from chronos.backend import training_available, resolve_training_device # noqa: F401 import chronos.workbench.training as _training