RLHF 实验分析：从训练到部署

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from transformers import GPT2LMHeadModel, GPT2Tokenizer
import torch
import wandb

# 设置随机种子
np.random.seed(42)
torch.manual_seed(42)

print("环境设置完成")
print(f"PyTorch 版本: {torch.__version__}")
print(f"CUDA 可用: {torch.cuda.is_available()}")

# 加载训练日志数据
training_logs = pd.read_csv('rlhf_training_logs.csv')

print(f"训练日志形状: {training_logs.shape}")
print("\n前5行数据:")
print(training_logs.head())

print("\n数据列信息:")
print(training_logs.info())

# 绘制训练损失曲线
plt.figure(figsize=(15, 10))

# 创建子图
fig, axes = plt.subplots(2, 2, figsize=(15, 10))
fig.suptitle('RLHF 训练监控面板', fontsize=16, fontweight='bold')

# 策略损失
axes[0, 0].plot(training_logs['step'], training_logs['policy_loss'], 
               color='blue', linewidth=2, alpha=0.8)
axes[0, 0].set_title('策略损失 (Policy Loss)')
axes[0, 0].set_xlabel('训练步数')
axes[0, 0].set_ylabel('损失值')
axes[0, 0].grid(True, alpha=0.3)

# 价值损失
axes[0, 1].plot(training_logs['step'], training_logs['value_loss'], 
               color='red', linewidth=2, alpha=0.8)
axes[0, 1].set_title('价值损失 (Value Loss)')
axes[0, 1].set_xlabel('训练步数')
axes[0, 1].set_ylabel('损失值')
axes[0, 1].grid(True, alpha=0.3)

# KL散度
axes[1, 0].plot(training_logs['step'], training_logs['kl_divergence'], 
               color='green', linewidth=2, alpha=0.8)
axes[1, 0].set_title('KL散度 (KL Divergence)')
axes[1, 0].set_xlabel('训练步数')
axes[1, 0].set_ylabel('KL散度值')
axes[1, 0].grid(True, alpha=0.3)

# 奖励分数
axes[1, 1].plot(training_logs['step'], training_logs['reward_score'], 
               color='purple', linewidth=2, alpha=0.8)
axes[1, 1].set_title('奖励分数 (Reward Score)')
axes[1, 1].set_xlabel('训练步数')
axes[1, 1].set_ylabel('奖励值')
axes[1, 1].grid(True, alpha=0.3)

plt.tight_layout()
plt.show()

# 打印训练统计
print("\n训练统计信息:")
print(f"最终策略损失: {training_logs['policy_loss'].iloc[-1]:.4f}")
print(f"最终价值损失: {training_logs['value_loss'].iloc[-1]:.4f}")
print(f"最终KL散度: {training_logs['kl_divergence'].iloc[-1]:.6f}")
print(f"最终奖励分数: {training_logs['reward_score'].iloc[-1]:.4f}")

# 计算训练效率指标
def calculate_training_metrics(logs):
    """计算训练效率相关指标"""
    
    # 损失下降率
    initial_policy_loss = logs['policy_loss'].iloc[0]
    final_policy_loss = logs['policy_loss'].iloc[-1]
    policy_loss_reduction = (initial_policy_loss - final_policy_loss) / initial_policy_loss * 100
    
    # 奖励改进幅度
    initial_reward = logs['reward_score'].iloc[0]
    final_reward = logs['reward_score'].iloc[-1]
    reward_improvement = final_reward - initial_reward
    
    # 训练稳定性（损失方差）
    policy_loss_variance = logs['policy_loss'].var()
    
    return {
        'policy_loss_reduction_pct': policy_loss_reduction,
        'reward_improvement': reward_improvement,
        'training_stability': 1.0 / (1.0 + policy_loss_variance)  # 越大越稳定
    }

metrics = calculate_training_metrics(training_logs)

print("📊 训练效率指标:")
print("=" * 40)
print(f"策略损失下降: {metrics['policy_loss_reduction_pct']:.2f}%")
print(f"奖励改进幅度: {metrics['reward_improvement']:.4f}")
print(f"训练稳定性: {metrics['training_stability']:.4f}")

# 绘制移动平均图
window_size = 50
training_logs['policy_loss_ma'] = training_logs['policy_loss'].rolling(window=window_size).mean()
training_logs['reward_score_ma'] = training_logs['reward_score'].rolling(window=window_size).mean()

plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(training_logs['step'], training_logs['policy_loss'], alpha=0.3, color='blue', label='原始数据')
plt.plot(training_logs['step'], training_logs['policy_loss_ma'], color='blue', linewidth=2, label=f'{window_size}步移动平均')
plt.title('策略损失趋势')
plt.xlabel('训练步数')
plt.ylabel('损失值')
plt.legend()
plt.grid(True, alpha=0.3)

plt.subplot(1, 2, 2)
plt.plot(training_logs['step'], training_logs['reward_score'], alpha=0.3, color='purple', label='原始数据')
plt.plot(training_logs['step'], training_logs['reward_score_ma'], color='purple', linewidth=2, label=f'{window_size}步移动平均')
plt.title('奖励分数趋势')
plt.xlabel('训练步数')
plt.ylabel('奖励值')
plt.legend()
plt.grid(True, alpha=0.3)

plt.tight_layout()
plt.show()