第 46 章：中文 Head 微调实验

本文整理自 Datawhale 开源项目 datawhalechina/jev-cookbook（CC BY-NC-SA 4.0），源文件：main/10_本地模型/notebooks/zh_head_finetuning.ipynb。

本 Notebook 按照 self-llm 教程的逐步讲解形式组织：确认并下载权重、准备环境、检查数据、训练、查看指标、加载微调结果。内容针对 Laya 自己的模型结构和训练代码编写，可从仓库根目录或本目录逐格运行。

本实验复现仓库中的 112 条合成中文工单试跑。它适合学习和讨论流程；标签不是人工金标，不能据此推断生产效果。

训练方式： Laya 是多语言 encoder 加结构化决策 head 的模型。当前脚本冻结 encoder 和 act head，只用监督交叉熵更新决策 head。它不是 causal LM，也不使用 MiniCPM 的 LoRA 配置；这也不是 Laya 上游 RLCD 训练。

预计产物： GPU 微调 checkpoint、逐轮 CSV/JSONL 日志、实验元数据和离线 HTML 曲线。训练输出写到系统临时目录，不会覆盖下载的基座权重。

from pathlib import Path
import sys

ROOT = next(
    (parent for parent in (Path.cwd(), *Path.cwd().parents)
     if (parent / "laya" / "finetune_reviewed_jsonl.py").is_file()),
    None,
)
if ROOT is None:
    raise FileNotFoundError("请从 jev-cookbook 仓库打开本 Notebook")

DATA_PATH = ROOT / "laya" / "experiments" / "zh-pilot-112" / "train-dev.jsonl"
MODEL_DIR = ROOT / "laya" / "models" / "multilingual"
sys.path.insert(0, str(ROOT))
print("仓库根目录:", ROOT)
print("训练数据:", DATA_PATH)
print("多语言权重目录:", MODEL_DIR)

1. 先下载 Laya 多语言权重

中文数据使用 convaiinnovations/laya 的 multilingual checkpoint。只下载训练需要的多语言目录；首次下载需要网络，下载后 Notebook 会复用本地文件。完整说明见 laya/models/README.md。

import os
import shutil
import subprocess
import sysconfig

required_files = [
    MODEL_DIR / "model.safetensors",
    MODEL_DIR / "rl_agent_config.json",
    MODEL_DIR / "encoder",
    MODEL_DIR / "tokenizer",
]
missing = [str(path) for path in required_files if not path.exists()]
if missing:
    subprocess.run([sys.executable, "-m", "pip", "install", "modelscope-hub"], check=True)
    cli_name = "ms-hub.exe" if os.name == "nt" else "ms-hub"
    cli = Path(sysconfig.get_path("scripts")) / cli_name
    if not cli.is_file():
        found = shutil.which("ms-hub")
        if found:
            cli = Path(found)
        else:
            raise FileNotFoundError("ModelScope Hub CLI 未安装成功；请重启 kernel 后重新运行此格")
    subprocess.run(
        [str(cli), "download", "convaiinnovations/laya",
         "--local-dir", str(ROOT / "laya" / "models"),
         "--include", "multilingual/**"],
        check=True,
    )

missing = [str(path) for path in required_files if not path.exists()]
if missing:
    raise FileNotFoundError("模型文件仍不完整: " + ", ".join(missing))
print("多语言 checkpoint 已就绪:", MODEL_DIR)

2. 安装项目依赖并检查 CUDA

请在启用了 NVIDIA CUDA 的 Jupyter kernel 中运行。仓库的 requirements 文件包含 PyTorch、Transformers、Safetensors 和数据处理依赖；如果当前 Python 环境已经匹配，可跳过安装并直接检查 GPU。安装或升级 PyTorch 后需要重启 kernel，再从 GPU 检查开始继续运行。

import subprocess

subprocess.run(
    [sys.executable, "-m", "pip", "install", "-r", str(ROOT / "laya" / "requirements.txt")],
    check=True,
)

import torch
import transformers
import safetensors
import numpy as np

print("Python:", sys.version.split()[0])
print("PyTorch:", torch.__version__)
print("CUDA runtime:", torch.version.cuda)
print("Transformers:", transformers.__version__)
print("Safetensors:", safetensors.__version__)
print("NumPy:", np.__version__)
print("CUDA available:", torch.cuda.is_available())
if not torch.cuda.is_available():
    raise RuntimeError("当前 kernel 没有可用 CUDA；请安装与驱动匹配的 PyTorch CUDA 版本并重启 kernel")
print("GPU:", torch.cuda.get_device_name(0))

3. 读取 JSONL 并检查切分

每行是一条 state 记录，qs 中包含一道或多道 choice / noul / score 问题。训练器会检查审核状态、重复 ID，以及 source_group_id 是否跨 train/dev；这里先查看数据量和题型分布。

这份示例记录为 assistant_reviewed_pilot。它代表经过辅助抽查的合成试跑数据，仍不是人工 gold。

import hashlib
import json
from collections import Counter

records = [json.loads(line) for line in DATA_PATH.read_text(encoding="utf-8").splitlines() if line.strip()]
train_records = [record for record in records if record["split"] == "train"]
dev_records = [record for record in records if record["split"] == "dev"]
train_groups = {record["source_group_id"] for record in train_records}
dev_groups = {record["source_group_id"] for record in dev_records}
assert train_records and dev_records
assert not (train_groups & dev_groups), "source_group_id 泄漏到 train/dev 两侧"
assert all(record["metadata"]["review_status"] == "assistant_reviewed_pilot" for record in records)
reference_experiment = json.loads((DATA_PATH.parent / "experiment.json").read_text(encoding="utf-8"))
data_sha256 = hashlib.sha256(DATA_PATH.read_bytes()).hexdigest()
assert data_sha256 == reference_experiment["data_sha256"], "数据文件与已记录实验哈希不一致"

print("records:", len(records), "train:", len(train_records), "dev:", len(dev_records))
print("questions:", sum(len(r["qs"]) for r in train_records), "train /", sum(len(r["qs"]) for r in dev_records), "dev")
print("train/dev source groups overlap:", bool(train_groups & dev_groups))
print("question types:", dict(Counter(q["t"] for r in records for q in r["qs"])))
print("review states:", dict(Counter(r["metadata"]["review_status"] for r in records)))

4. 配置并启动 head-only 微调

默认参数与已提交的 GPU 试跑对齐：随机种子 42、head 学习率 1e-4、最多 8 轮、dev soft cross-entropy 连续 2 轮不改善则早停、token budget 4096。训练器按 dev soft cross-entropy 选最佳 checkpoint。

由于示例数据标记为 assistant_reviewed_pilot，命令显式启用 --allow-assistant-reviewed-pilot。对已由人工审核的正式数据，应去掉该开关并使用 approved 或 human_reviewed 状态。

import tempfile
import uuid

RUN_DIR = Path(tempfile.gettempdir()) / ("laya-zh-notebook-" + uuid.uuid4().hex[:8])
command = [
    sys.executable,
    str(ROOT / "laya" / "finetune_reviewed_jsonl.py"),
    "--data", str(DATA_PATH),
    "--model-dir", str(MODEL_DIR),
    "--output-dir", str(RUN_DIR),
    "--epochs", "8",
    "--patience", "2",
    "--head-lr", "1e-4",
    "--max-tokens", "4096",
    "--max-seqs", "8",
    "--seed", "42",
    "--allow-assistant-reviewed-pilot",
]
print("输出目录:", RUN_DIR)
subprocess.run(command, cwd=ROOT, check=True)

5. 查看训练曲线和评估结果

每轮训练和验证指标保存在 training_log.csv 与 training_log.jsonl；experiment.json 记录数据 / 权重 SHA256、实际软件版本、训练参数、显卡和验证结果。下面用同一个可视化脚本生成自包含 HTML。

subprocess.run(
    [sys.executable, str(ROOT / "laya" / "visualize_training.py"), "--run-dir", str(RUN_DIR)],
    cwd=ROOT,
    check=True,
)

experiment = json.loads((RUN_DIR / "experiment.json").read_text(encoding="utf-8"))
summary = {
    "gpu": experiment["gpu"],
    "best_epoch": experiment["best_epoch"],
    "dev_before": experiment["dev_before"],
    "dev_after": experiment["dev_after"],
    "peak_vram_gib": experiment["peak_vram_gib"],
    "training_seconds": experiment["training_seconds"],
    "artifacts": ["training_report.html", "training_log.csv", "training_log.jsonl", "experiment.json"],
}
summary

from IPython.display import HTML, display

display(HTML(filename=str(RUN_DIR / "training_report.html")))

6. 加载基座与微调模型做一次推理对照

这一步检查保存的 checkpoint 可否被项目客户端加载，并观察相同 dev state 上的输出。单条样本是加载 smoke check，不是准确率或泛化评估；量化指标以上一格的完整 dev 指标为准。

import gc
from laya.client import LayaClient

sample = dev_records[0]
questions = {
    question["id"]: {
        "type": question["t"],
        "instructions": question["ins"],
        "criteria": question["crit"],
    }
    for question in sample["qs"]
}

base_client = LayaClient(MODEL_DIR, device="cuda")
base_answers = base_client.system_one(sample["state"], questions)["answers"]
del base_client
gc.collect()
torch.cuda.empty_cache()

tuned_client = LayaClient(RUN_DIR, device="cuda")
tuned_answers = tuned_client.system_one(sample["state"], questions)["answers"]
print("标签索引:", {q["id"]: q["y"] for q in sample["qs"]})
print("基座输出:", json.dumps(base_answers, ensure_ascii=False, indent=2))
print("微调输出:", json.dumps(tuned_answers, ensure_ascii=False, indent=2))

7. 复现结果说明

本次提交的数据、日志、实验摘要和 HTML 报告位于 laya/experiments/zh-pilot-112/。相同软件、驱动和随机种子可获得接近的结果；CUDA kernel 与库版本差异可能造成数值偏差。

样本量只有 112 条，且 dev 来自同一任务规范。请把本 Notebook 当作训练流程教程，讨论模型效果前应补充真实人工审核数据、独立 test / OOD split 和概率校准评估。完整操作说明见 laya/FINETUNING.md。