diff --git a/.github/workflows/trl-quick-start.yml b/.github/workflows/trl-quick-start.yml new file mode 100644 index 000000000..317720749 --- /dev/null +++ b/.github/workflows/trl-quick-start.yml @@ -0,0 +1,34 @@ +# trl quick start guard - thin trigger over quick-start-template.yml; the engine owns the monitor loop, cache I/O and result publishing. +name: trl-quick-start + +concurrency: + group: ${{ github.event_name == 'schedule' && 'trl-quick-start-schedule' || format('manual-{0}', github.run_id) }} + cancel-in-progress: false + +on: + schedule: + - cron: '0 */3 * * *' + workflow_dispatch: + pull_request: + branches: [main] + paths: + - 'sources/trl/**' + - 'tests/trl/**' + +permissions: + contents: read + +jobs: + trl-quick-start: + uses: ./.github/workflows/quick-start-template.yml + with: + project: trl + test_runner: '["linux-aarch64-a2-1"]' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + container_options: >- + --volume=/data/ci-cache/modelscope/trl:/root/.cache/modelscope + timeout_minutes: 180 + upstream_repo: huggingface/trl + doc_url: 'https://raw.githubusercontent.com/Ascend/docs/{0}/sources/trl/quick_start.md' + doc_path: https://github.com/Ascend/docs/blob/main/sources/trl/quick_start.md + test_command: python -m unittest tests.trl.test_quick_start_ascend -v 2>&1 diff --git a/conf.py b/conf.py index 5bc93024e..0e54ad91c 100644 --- a/conf.py +++ b/conf.py @@ -77,7 +77,8 @@ 'sources/llama_cpp/quick_start.md', 'sources/whisper_cpp/quick_start.md', 'sources/llm_compressor/quick_start.md', - 'sources/axolotl/quick_start.md'] + 'sources/axolotl/quick_start.md', + 'sources/trl/quick_start.md'] # -- Options for HTML output ------------------------------------------------- diff --git a/index.rst b/index.rst index 9c4356f98..a4111eca9 100644 --- a/index.rst +++ b/index.rst @@ -214,7 +214,7 @@

Transformer Reinforcement Learning

适用于 SFT、PPO、DPO 等方法的模型后训练库。

-
官方链接|安装指南|快速上手
+
官方链接|快速上手
diff --git a/sources/trl/images/image.png b/sources/trl/images/image.png deleted file mode 100644 index bf674bed6..000000000 Binary files a/sources/trl/images/image.png and /dev/null differ diff --git a/sources/trl/index.rst b/sources/trl/index.rst index 2f7ce0103..89892f3b6 100644 --- a/sources/trl/index.rst +++ b/sources/trl/index.rst @@ -1,8 +1,2 @@ -Transformer Reinforcement Learning -=================================================== - -.. toctree:: - :maxdepth: 2 - - install.rst - quick_start.rst +.. include:: quick_start.md + :parser: myst_parser.sphinx_ diff --git a/sources/trl/install.rst b/sources/trl/install.rst deleted file mode 100644 index f1c74e9d3..000000000 --- a/sources/trl/install.rst +++ /dev/null @@ -1,38 +0,0 @@ -安装指南 -=============== - -本教程面向使用 TRL (Transformer Reinforcement Learning) & 昇腾的开发者，帮助完成昇腾环境下 TRL 的安装。 - -昇腾环境安装 ---------------- - -请根据已有昇腾产品型号及 CPU 架构等按照 :doc:`快速安装昇腾环境指引 <../ascend/quick_install>` 进行昇腾环境安装。 - -.. warning:: - CANN 最低版本为 8.0.rc1，安装 CANN 时，请同时安装 Kernel 算子包。 - -TRL 下载安装 ---------------- - -1. 安装项目所需依赖 - -.. code-block:: shell - - pip install trl -i https://pypi.tuna.tsinghua.edu.cn/simple - -另外，本项目需要手动安装 transformers 仓库的最新 main 分支，否则可能会出现如下错误： - -.. image:: ./images/image.png - -.. code-block:: shell - - pip install git+https://github.com/huggingface/transformers.git - -2. 安装 torch_npu - -.. code-block:: shell - - pip install torch==2.1.0 torch_npu==2.1.0.post6 -i https://pypi.tuna.tsinghua.edu.cn/simple - -.. hint:: - torch_npu 的版本需要匹配 torch 的版本，详细信息请参考：`Ascend Extension for PyTorch `_。 diff --git a/sources/trl/quick_start.md b/sources/trl/quick_start.md new file mode 100644 index 000000000..001daf7c7 --- /dev/null +++ b/sources/trl/quick_start.md @@ -0,0 +1,191 @@ +# TRL + +TRL 用统一的 `Trainer` / `Config` API 支持多种模型后训练方法。本示例在单卡昇腾 NPU 上，用 Qwen2.5-0.5B-Instruct 分别运行 SFT 和 DPO LoRA。 + +## 前置条件 + +### 硬件 + +Atlas 900 A2 / A3 训练系列产品或者 Ascend 950 系列产品，并按需完成物理机或容器内的设备挂载。 + +### 基础软件 + +在运行本文档示例之前，你的机器上需要已经装好并可用： + +- 可用的 Python 环境 +- 可用的 CANN（参考[快速安装昇腾环境](https://ascend.github.io/docs/sources/ascend/quick_install.html)） +- 根据 CANN 版本安装匹配的 `torch_npu`（参考 [Ascend PyTorch 安装文档](https://gitcode.com/Ascend/pytorch)） + +本文档示例在 Python 3.12、CANN 9.1.0、`torch_npu` 2.9.0.post2 环境下验证通过。 + +## 加载 CANN 环境 + +```shell +source /usr/local/Ascend/ascend-toolkit/set_env.sh +``` + +## 安装 TRL + +安装 TRL 并查看安装版本： + +```shell #test id="install-trl" +python -m pip install trl +python -c "import trl; print('trl', trl.__version__)" +``` + +输出结果如下，其中 `xxx` 为实际安装的 TRL 版本： + +```shell #test-result id="install-trl" fuzzy='...' fuzzy='xxx' +... +trl xxx +``` + +## 示例一：SFT LoRA 后训练 + +用 Qwen2.5-0.5B-Instruct 和 ModelScope 的 `HuggingFaceH4/ultrafeedback_binarized` SFT 子集进行 5 步 LoRA 微调，模型与数据集会自动下载，适配器保存到 `output/trl-sft-lora`。 + +安装示例依赖： + +```shell #test-setup +python -m pip install peft "transformers>=4.56.2,<5.0" datasets "modelscope==1.37.0" +``` +运行 SFT 训练脚本： +```python #test id="sft-lora" +import os +import shutil +import torch +import torch_npu +from datasets import load_dataset +from modelscope import snapshot_download +from peft import LoraConfig, TaskType +from trl import SFTConfig, SFTTrainer + +print("TRL_SFT_BEGIN") + +ds_path = snapshot_download( + 'HuggingFaceH4/ultrafeedback_binarized', repo_type='dataset', +) +data_dir = './ultrafeedback_sft' +if os.path.isdir(data_dir): + shutil.rmtree(data_dir) +os.makedirs(data_dir, exist_ok=True) +for name in os.listdir(os.path.join(ds_path, 'data')): + if name.startswith('train_sft-'): + shutil.copy2(os.path.join(ds_path, 'data', name), data_dir) +train_dataset = load_dataset( + 'parquet', data_files=os.path.join(data_dir, 'train_sft-*.parquet'), + split='train', +).select_columns(['messages']) + +model = snapshot_download('Qwen/Qwen2.5-0.5B-Instruct') + +trainer = SFTTrainer( + model=model, + train_dataset=train_dataset, + peft_config=LoraConfig(r=8, lora_alpha=32, task_type=TaskType.CAUSAL_LM), + args=SFTConfig( + output_dir="output/trl-sft-lora", + max_steps=5, + per_device_train_batch_size=1, + gradient_accumulation_steps=1, + learning_rate=1e-4, + max_length=512, + logging_steps=1, + save_strategy="no", + report_to="none", + model_init_kwargs={"dtype": torch.bfloat16}, + ), +) +print("model device:", next(trainer.model.parameters()).device) +trainer.train() +trainer.save_model("output/trl-sft-lora") +print("LoRA adapter saved to: output/trl-sft-lora") +print("TRL_SFT_DONE") +``` + +输出结果类似如下： + +```shell #test-result id="sft-lora" +... +LoRA adapter saved to: output/trl-sft-lora +TRL_SFT_DONE +``` + +## 示例二：偏好优化 DPO LoRA + +再用相同模型和数据集运行 3 步 DPO LoRA，适配器保存到 `output/trl-dpo-lora`。 + +```python #test id="dpo-lora" +import os +import shutil +import torch +import torch_npu +from datasets import load_dataset +from modelscope import snapshot_download +from peft import LoraConfig, TaskType +from transformers import AutoModelForCausalLM, AutoTokenizer +from trl import DPOConfig, DPOTrainer + +print("TRL_DPO_BEGIN") + +ds_path = snapshot_download( + 'HuggingFaceH4/ultrafeedback_binarized', repo_type='dataset', +) +data_dir = './ultrafeedback_prefs' +if os.path.isdir(data_dir): + shutil.rmtree(data_dir) +os.makedirs(data_dir, exist_ok=True) +for name in os.listdir(os.path.join(ds_path, 'data')): + if name.startswith('train_prefs-'): + shutil.copy2(os.path.join(ds_path, 'data', name), data_dir) +train_dataset = load_dataset( + 'parquet', data_files=os.path.join(data_dir, 'train_prefs-*.parquet'), + split='train', +) + +# prompt 列是纯字符串，chosen / rejected 是 messages 列表；把 prompt 转成 +# 单条 user 消息即可让 DPOTrainer 按 conversational 格式处理 +def to_conversational(example): + example['prompt'] = [{'role': 'user', 'content': example['prompt']}] + return example + +train_dataset = train_dataset.map(to_conversational) + +model_path = snapshot_download('Qwen/Qwen2.5-0.5B-Instruct') +model = AutoModelForCausalLM.from_pretrained(model_path, dtype=torch.bfloat16) +tokenizer = AutoTokenizer.from_pretrained(model_path) + +trainer = DPOTrainer( + model=model, + ref_model=None, + processing_class=tokenizer, + train_dataset=train_dataset, + peft_config=LoraConfig(r=8, lora_alpha=32, task_type=TaskType.CAUSAL_LM), + args=DPOConfig( + output_dir="output/trl-dpo-lora", + max_steps=3, + per_device_train_batch_size=1, + gradient_accumulation_steps=1, + learning_rate=1e-4, + max_length=512, + logging_steps=1, + save_strategy="no", + report_to="none", + ), +) +print("model device:", next(trainer.model.parameters()).device) +trainer.train() +trainer.save_model("output/trl-dpo-lora") +print("LoRA adapter saved to: output/trl-dpo-lora") +print("TRL_DPO_DONE") +``` + +输出结果类似如下： + +```shell #test-result id="dpo-lora" +... +LoRA adapter saved to: output/trl-dpo-lora +TRL_DPO_DONE +``` + +更多方法（GRPO / PPO / Reward / KTO 等）见 [TRL examples](https://github.com/huggingface/trl/tree/main/examples)。 diff --git a/sources/trl/quick_start.rst b/sources/trl/quick_start.rst deleted file mode 100644 index 987b5b014..000000000 --- a/sources/trl/quick_start.rst +++ /dev/null @@ -1,49 +0,0 @@ -快速开始 -=============== - -.. note:: - - 阅读本篇前，请确保已按照 :doc:`安装教程 <./install>` 准备好昇腾环境及 TRL (Transformer Reinforcement Learning) ！ - -本教程以 DPO 方法为例，讲述如何使用 TRL 在昇腾 NPU 上进行模型的后训练。 - -前置准备 ---------------- - -本篇样例代码为 TRL 官方样例，需提前进行下载： - -.. code-block:: - - git clone https://github.com/huggingface/trl.git - -模型训练 ---------------- - -进入 TRL 项目目录，依次执行如下命令： - -.. code-block:: - - cd examples/scripts - python dpo.py - -出现如下日志则代表训练成功： - -:: - - Tokenizing train dataset: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 62135/62135 [07:11<00:00, 143.85 examples/s] - Tokenizing eval dataset: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1000/1000 [00:06<00:00, 144.73 examples/s] - Detected kernel version 4.19.90, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. - 0%| ... | 0/3883 [00:00 bool: + if not value: + return False + return value.strip().lower() == 'true' + + +def _e2e_enabled() -> bool: + return _is_truthy(os.environ.get('NPU_READY')) + + +class TestQuickStartAscend(MarkdownDocTestBase, unittest.TestCase): + DEFAULT_COMMAND_TIMEOUT = 7200 + USER_AGENT = 'cosdt-ci-test/quick-start' + ERROR_MARKERS = ( + *MarkdownDocTestBase.ERROR_MARKERS, + 'applicaiton exception', + 'ERR99999', + ) + + _CUDA_CONSTRAINTS = ( + 'cuda-toolkit<0', + 'cuda-python<0', + 'cuda-bindings<0', + 'cuda-core<0', + 'cuda-pathfinder<0', + 'flashinfer-python<0', + 'nvidia-cublas<0', + 'nvidia-cuda-runtime<0', + 'nvidia-cuda-nvrtc<0', + 'nvidia-cuda-cupti<0', + 'nvidia-cudnn<0', + 'nvidia-cudnn-frontend<0', + 'nvidia-cufft<0', + 'nvidia-curand<0', + 'nvidia-cusolver<0', + 'nvidia-cusparse<0', + 'nvidia-cutlass-dsl<0', + 'nvidia-cutlass-dsl-libs-base<0', + 'nvidia-cutlass-dsl-libs-core<0', + 'nvidia-cutlass-dsl-libs-cu12<0', + 'nvidia-ml-py<0', + 'nvidia-nccl<0', + 'nvidia-nvjitlink<0', + 'nvidia-nvtx<0', + 'nvidia-cublas-cu12<0', + 'nvidia-cuda-nvdisasm<0', + 'nvidia-cuda-runtime-cu12<0', + 'nvidia-cuda-nvrtc-cu12<0', + 'nvidia-cuda-cupti-cu12<0', + 'nvidia-cudnn-cu12<0', + 'nvidia-cufft-cu12<0', + 'nvidia-curand-cu12<0', + 'nvidia-cusolver-cu12<0', + 'nvidia-cusparse-cu12<0', + 'nvidia-cusparselt-cu12<0', + 'nvidia-nccl-cu12<0', + 'nvidia-nvjitlink-cu12<0', + 'nvidia-nvtx-cu12<0', + ) + _CONSTRAINTS_FILE = '/tmp/trl_npu_constraints.txt' + + _CLUSTER_INDEX = 'http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple' + _ASCEND_EXTRA = 'https://repo.huaweicloud.com/ascend/repos/pypi' + + _CANN_SET_ENV = '/usr/local/Ascend/ascend-toolkit/set_env.sh' + + @classmethod + def prepare_environment(cls) -> None: + if os.path.isfile(cls._CANN_SET_ENV): + merged = subprocess.run( + ['bash', '-c', f'source {cls._CANN_SET_ENV} >/dev/null 2>&1; env'], + capture_output=True, text=True, check=True, + ) + for line in merged.stdout.splitlines(): + if '=' not in line: + continue + key, _, value = line.partition('=') + os.environ.setdefault(key, value) + print('setup: sourced CANN env from set_env.sh') + else: + print( + f'setup: skipping CANN env source ({cls._CANN_SET_ENV} not present)' + ) + + with open(cls._CONSTRAINTS_FILE, 'w', encoding='utf-8') as fh: + fh.write('\n'.join(cls._CUDA_CONSTRAINTS) + '\n') + os.environ['PIP_CONSTRAINT'] = cls._CONSTRAINTS_FILE + + os.environ.setdefault('ASCEND_RT_VISIBLE_DEVICES', '0') + + _PROBE_SCRIPT = ( + 'import torch, torch_npu\n' + "raise SystemExit(0 if " + "torch.__version__.startswith('2.9.0') " + "and torch_npu.__version__.startswith('2.9.0') " + "else 1)" + ) + probe = subprocess.run( + ['python', '-c', _PROBE_SCRIPT], + capture_output=True, + check=False, + ) + if probe.returncode == 0: + _VERSIONS_SCRIPT = ( + 'import torch, torch_npu; ' + 'print(torch.__version__, torch_npu.__version__)' + ) + versions = subprocess.run( + ['python', '-c', _VERSIONS_SCRIPT], + capture_output=True, text=True, check=True, + ) + print(f'setup: reusing image torch stack ({versions.stdout.strip()})') + else: + print('setup: installing torch==2.9.0 torch_npu==2.9.0.post2') + subprocess.run( + [ + 'python', '-m', 'pip', 'install', + '--index-url', cls._CLUSTER_INDEX, + '--extra-index-url', cls._ASCEND_EXTRA, + 'torch==2.9.0', 'torch_npu==2.9.0.post2', + ], + check=True, + ) + + ensure_safetensors() + + purge_modelscope_corrupt(resolve_modelscope_cache()) + + @classmethod + def setUpClass(cls) -> None: + if _e2e_enabled(): + cls.prepare_environment() + + @unittest.skipIf( + not _e2e_enabled(), + 'end-to-end requires NPU runner; set NPU_READY=true', + ) + def test_runs_doc(self) -> None: + self.run_template() + + +if __name__ == '__main__': + unittest.main()