AI集成学习：Bagging、Boosting与Stacking

import numpy as np
import pandas as pd
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb
import lightgbm as lgb
from catboost import CatBoostClassifier
import joblib
import os

class EnsembleModel:
    def __init__(self, n_folds=5, random_state=42):
        self.n_folds = n_folds
        self.random_state = random_state
        self.models = {
            'xgboost': xgb.XGBClassifier(
                n_estimators=200,
                max_depth=6,
                learning_rate=0.1,
                subsample=0.8,
                colsample_bytree=0.8,
                random_state=random_state,
                eval_metric='auc'
            ),
            'lightgbm': lgb.LGBMClassifier(
                n_estimators=200,
                max_depth=6,
                learning_rate=0.1,
                subsample=0.8,
                colsample_bytree=0.8,
                random_state=random_state
            ),
            'catboost': CatBoostClassifier(
                iterations=200,
                depth=6,
                learning_rate=0.1,
                random_seed=random_state,
                verbose=False
            ),
            'rf': RandomForestClassifier(
                n_estimators=100,
                max_depth=10,
                min_samples_split=20,
                random_state=random_state
            )
        }
        self.trained_models = {}
        self.weights = {}

    def fit(self, X, y, model_dir='models'):
        """训练所有基模型"""
        os.makedirs(model_dir, exist_ok=True)

        # 使用分层 K 折交叉验证
        skf = StratifiedKFold(n_splits=self.n_folds, shuffle=True, random_state=self.random_state)

        # 存储每个模型在每个 fold 的验证集预测结果
        oof_preds = {name: np.zeros(len(y)) for name in self.models.keys()}

        for fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):
            print(f"Fold {fold + 1}/{self.n_folds}")

            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
            y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]

            for model_name, model in self.models.items():
                print(f"  Training {model_name}...")

                # 训练模型
                model.fit(X_train, y_train, eval_set=[(X_val, y_val)], verbose=False)

                # 预测验证集
                if model_name == 'catboost':
                    val_pred = model.predict_proba(X_val)[:, 1]
                else:
                    val_pred = model.predict_proba(X_val)[:, 1]

                oof_preds[model_name][val_idx] = val_pred

                # 保存模型
                model_path = os.path.join(model_dir, f'{model_name}_fold{fold}.pkl')
                joblib.dump(model, model_path)

        # 计算每个模型的权重（基于 OOF 预测的 AUC）
        for model_name, preds in oof_preds.items():
            auc = roc_auc_score(y, preds)
            self.weights[model_name] = auc
            print(f"{model_name} OOF AUC: {auc:.4f}")

        # 归一化权重
        total_weight = sum(self.weights.values())
        for name in self.weights:
            self.weights[name] /= total_weight

        print(f"Final weights: {self.weights}")

    def predict_proba(self, X, model_dir='models'):
        """预测"""
        fold_preds = {name: [] for name in self.models.keys()}

        # 加载所有 fold 的模型
        for fold in range(self.n_folds):
            for model_name in self.models.keys():
                model_path = os.path.join(model_dir, f'{model_name}_fold{fold}.pkl')
                model = joblib.load(model_path)

                if model_name == 'catboost':
                    pred = model.predict_proba(X)[:, 1]
                else:
                    pred = model.predict_proba(X)[:, 1]

                fold_preds[model_name].append(pred)

        # 对每个模型的预测结果取平均
        avg_preds = {}
        for model_name, preds_list in fold_preds.items():
            avg_preds[model_name] = np.mean(preds_list, axis=0)

        # 加权融合
        final_pred = np.zeros(len(X))
        for model_name, weight in self.weights.items():
            final_pred += weight * avg_preds[model_name]

        return final_pred

# 使用示例
# 假设 X_train, y_train, X_test 已经准备好
ensemble = EnsembleModel(n_folds=5, random_state=42)
ensemble.fit(X_train, y_train, model_dir='saved_models')

# 预测
test_preds = ensemble.predict_proba(X_test, model_dir='saved_models')

from sklearn.model_selection import GridSearchCV

def optimize_xgboost(X, y):
    param_grid = {
        'n_estimators': [100, 200, 300],
        'max_depth': [4, 6, 8],
        'learning_rate': [0.05, 0.1, 0.15],
        'subsample': [0.7, 0.8, 0.9],
        'colsample_bytree': [0.7, 0.8, 0.9]
    }

    xgb_model = xgb.XGBClassifier(random_state=42, eval_metric='auc')
    grid_search = GridSearchCV(xgb_model, param_grid, cv=3, scoring='roc_auc', n_jobs=-1)
    grid_search.fit(X, y)

    return grid_search.best_params_

def calculate_adaptive_weights(oof_preds, y, recent_window=0.3):
    """
    根据最近表现动态调整权重
    recent_window: 只考虑表现最好的前 N% 的样本
    """
    adaptive_weights = {}

    for model_name, preds in oof_preds.items():
        # 计算每个样本的预测准确度（二分类可以用 logloss）
        epsilon = 1e-15
        logloss = - (y * np.log(preds + epsilon) + (1 - y) * np.log(1 - preds + epsilon))

        # 只考虑表现最好的样本
        threshold = np.quantile(logloss, 1 - recent_window)
        mask = logloss <= threshold

        # 在优质样本上的表现
        adaptive_weights[model_name] = roc_auc_score(y[mask], preds[mask])

    # 归一化
    total = sum(adaptive_weights.values())
    for name in adaptive_weights:
        adaptive_weights[name] /= total

    return adaptive_weights

from sklearn.calibration import CalibratedClassifierCV

def calibrate_model(model, X_cal, y_cal, method='isotonic'):
    """概率校准"""
    calibrated = CalibratedClassifierCV(model, method=method, cv='prefit')
    calibrated.fit(X_cal, y_cal)
    return calibrated

# 在每个 fold 训练后校准
for model_name, model in self.models.items():
    model.fit(X_train, y_train)

    # 使用验证集进行校准
    calibrated_model = calibrate_model(model, X_val, y_val, method='isotonic')
    val_pred = calibrated_model.predict_proba(X_val)[:, 1]

from sklearn.model_selection import TimeSeriesSplit

def time_based_split(X, y, n_splits=5):
    """时间序列切分"""
    tscv = TimeSeriesSplit(n_splits=n_splits)

    for train_idx, val_idx in tscv.split(X):
        yield train_idx, val_idx

def knowledge_distillation(teacher_model, X_train, y_train, student_model=None):
    """知识蒸馏"""
    if student_model is None:
        student_model = xgb.XGBClassifier(
            n_estimators=50,  # 更少的树
            max_depth=4,      # 更浅的深度
            learning_rate=0.1
        )

    # 获取教师模型的软标签
    teacher_probs = teacher_model.predict_proba(X_train)[:, 1]

    # 训练学生模型（使用软标签）
    student_model.fit(X_train, teacher_probs, eval_metric='auc')

    return student_model

from sklearn.preprocessing import LabelEncoder
from sklearn.impute import SimpleImputer

class FeaturePreprocessor:
    def __init__(self):
        self.label_encoders = {}
        self.imputer = SimpleImputer(strategy='median')
        self.categorical_cols = []
        self.numerical_cols = []

    def fit(self, X):
        """训练预处理器"""
        # 区分数值型和类别型特征
        for col in X.columns:
            if X[col].dtype == 'object':
                self.categorical_cols.append(col)
            else:
                self.numerical_cols.append(col)

        # 标签编码类别变量
        for col in self.categorical_cols:
            le = LabelEncoder()
            X[col] = X[col].astype(str)
            le.fit(X[col])
            self.label_encoders[col] = le

        # 拟合缺失值填充
        self.imputer.fit(X[self.numerical_cols])

        return self

    def transform(self, X):
        """转换数据"""
        X = X.copy()

        # 处理类别变量
        for col in self.categorical_cols:
            X[col] = X[col].astype(str)
            X[col] = self.label_encoders[col].transform(X[col])

        # 处理数值变量
        X[self.numerical_cols] = self.imputer.transform(X[self.numerical_cols])

        return X

    def transform_for_lgb(self, X):
        """LightGBM 可以直接使用类别变量"""
        X = X.copy()

        # 标记类别列
        for col in self.categorical_cols:
            X[col] = X[col].astype('category')

        return X

from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, roc_auc_score
import pandas as pd

# 读取数据（假设已经做了预处理）
df = pd.read_csv('/home/liqinsi/data/risk_control_data.csv')
X = df.drop('label', axis=1)
y = df['label']

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# 基模型用决策树
base_clf = DecisionTreeClassifier(max_depth=5, min_samples_split=10)

# Bagging 集成
bagging_clf = BaggingClassifier(
    estimator=base_clf,
    n_estimators=100,
    max_samples=0.8,
    max_features=0.8,
    random_state=42,
    n_jobs=-1
)

bagging_clf.fit(X_train, y_train)
y_pred = bagging_clf.predict(X_test)
y_proba = bagging_clf.predict_proba(X_test)[:, 1]

print(f"Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print(f"AUC: {roc_auc_score(y_test, y_proba):.4f}")

# 对比不同基模型深度
for depth in [3, 5, 8, 12, None]:
    base = DecisionTreeClassifier(max_depth=depth, random_state=42)
    bagging = BaggingClassifier(
        estimator=base,
        n_estimators=100,
        max_samples=0.8,
        random_state=42,
        n_jobs=-1
    )
    bagging.fit(X_train, y_train)
    auc = roc_auc_score(y_test, bagging.predict_proba(X_test)[:, 1])
    print(f"Depth {depth}: AUC = {auc:.4f}")

from sklearn.ensemble import RandomForestClassifier

rf_clf = RandomForestClassifier(
    n_estimators=200,
    max_depth=6,
    min_samples_split=10,
    min_samples_leaf=5,
    max_features='sqrt',
    random_state=42,
    n_jobs=-1,
    class_weight='balanced'  # 处理样本不平衡
)

rf_clf.fit(X_train, y_train)
y_pred = rf_clf.predict(X_test)
y_proba = rf_clf.predict_proba(X_test)[:, 1]

print(f"Random Forest - Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print(f"Random Forest - AUC: {roc_auc_score(y_test, y_proba):.4f}")

from sklearn.ensemble import AdaBoostClassifier

ada_clf = AdaBoostClassifier(
    estimator=DecisionTreeClassifier(max_depth=3),
    n_estimators=100,
    learning_rate=0.1,
    random_state=42
)

ada_clf.fit(X_train, y_train)
y_pred = ada_clf.predict(X_test)
y_proba = ada_clf.predict_proba(X_test)[:, 1]

print(f"AdaBoost - Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print(f"AdaBoost - AUC: {roc_auc_score(y_test, y_proba):.4f}")

import xgboost as xgb

# 转换成 DMatrix 格式（XGBoost 的专用数据结构）
dtrain = xgb.DMatrix(X_train, label=y_train)
dtest = xgb.DMatrix(X_test, label=y_test)

params = {
    'objective': 'binary:logistic',
    'eval_metric': 'auc',
    'max_depth': 6,
    'eta': 0.1,  # 学习率
    'subsample': 0.8,
    'colsample_bytree': 0.8,
    'min_child_weight': 5,
    'gamma': 0.1,
    'lambda': 1.5,  # L2 正则化
    'alpha': 0.5,   # L1 正则化
    'scale_pos_weight': 20,  # 处理样本不平衡
    'seed': 42
}

watchlist = [(dtrain, 'train'), (dtest, 'eval')]

# 训练
xgb_model = xgb.train(
    params,
    dtrain,
    num_boost_round=500,
    evals=watchlist,
    early_stopping_rounds=30,
    verbose_eval=50
)

# 预测
y_proba = xgb_model.predict(dtest)
y_pred = (y_proba > 0.5).astype(int)

print(f"XGBoost - Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print(f"XGBoost - AUC: {roc_auc_score(y_test, y_proba):.4f}")

from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler

# 训练不同模型
rf_clf.fit(X_train, y_train)
xgb_model = xgb.train(params, dtrain, num_boost_round=500, evals=watchlist, early_stopping_rounds=30, verbose_eval=False)
lr_clf = LogisticRegression(max_iter=1000, class_weight='balanced')
lr_clf.fit(X_train, y_train)

# 获取预测概率
rf_proba = rf_clf.predict_proba(X_test)[:, 1]
xgb_proba = xgb_model.predict(dtest)
lr_proba = lr_clf.predict_proba(X_test)[:, 1]

# 简单平均
ensemble_proba = (rf_proba + xgb_proba + lr_proba) / 3
ensemble_pred = (ensemble_proba > 0.5).astype(int)

print(f"Ensemble (Average) - Accuracy: {accuracy_score(y_test, ensemble_pred):.4f}")
print(f"Ensemble (Average) - AUC: {roc_auc_score(y_test, ensemble_proba):.4f}")

import numpy as np
from sklearn.model_selection import KFold

# 用 K-Fold 生成训练集的 meta 特征（避免过拟合）
kf = KFold(n_splits=5, shuffle=True, random_state=42)
meta_train = np.zeros((X_train.shape[0], 3))
meta_test = np.zeros((X_test.shape[0], 3))

for i, (train_idx, val_idx) in enumerate(kf.split(X_train)):
    X_tr, X_val = X_train.iloc[train_idx], X_train.iloc[val_idx]
    y_tr, y_val = y_train.iloc[train_idx], y_train.iloc[val_idx]

    # 训练三个基模型
    rf = RandomForestClassifier(max_depth=6, n_estimators=100, random_state=42, n_jobs=-1)
    rf.fit(X_tr, y_tr)
    meta_train[val_idx, 0] = rf.predict_proba(X_val)[:, 1]

    dtrain_fold = xgb.DMatrix(X_tr, label=y_tr)
    dval_fold = xgb.DMatrix(X_val)
    xgb_fold = xgb.train(params, dtrain_fold, num_boost_round=200, verbose_eval=False)
    meta_train[val_idx, 1] = xgb_fold.predict(dval_fold)

    lr = LogisticRegression(max_iter=1000, class_weight='balanced')
    lr.fit(X_tr, y_tr)
    meta_train[val_idx, 2] = lr.predict_proba(X_val)[:, 1]

    # 在测试集上预测
    meta_test[:, 0] += rf.predict_proba(X_test)[:, 1]
    meta_test[:, 1] += xgb_fold.predict(dtest)
    meta_test[:, 2] += lr.predict_proba(X_test)[:, 1]

meta_test /= 5  # 取平均

# 用 meta-learner 学习权重
meta_learner = LogisticRegression()
meta_learner.fit(meta_train, y_train)

final_proba = meta_learner.predict_proba(meta_test)[:, 1]
final_pred = (final_proba > 0.5).astype(int)

print(f"Stacking - Accuracy: {accuracy_score(y_test, final_pred):.4f}")
print(f"Stacking - AUC: {roc_auc_score(y_test, final_proba):.4f}")
print(f"Meta-learner weights: {meta_learner.coef_}")

# 快速训练版本
params_fast = params.copy()
params_fast['tree_method'] = 'hist'
params_fast['eta'] = 0.3

xgb_model_fast = xgb.train(
    params_fast,
    dtrain,
    num_boost_round=200,
    evals=watchlist,
    early_stopping_rounds=20,
    verbose_eval=False
)

from joblib import Parallel, delayed

def predict_single(model, X):
    return model.predict_proba(X)[:, 1]

# 并行预测多个模型
probabilities = Parallel(n_jobs=4)(
    delayed(predict_single)(clf, X_test) for clf in bagging_clf.estimators_
)
ensemble_proba = np.mean(probabilities, axis=0)

# 监控各基模型在验证集上的表现
base_scores = []
for i, estimator in enumerate(bagging_clf.estimators_):
    pred = estimator.predict_proba(X_test)[:, 1]
    auc = roc_auc_score(y_test, pred)
    base_scores.append(auc)

print(f"Base model AUCs: mean={np.mean(base_scores):.4f}, std={np.std(base_scores):.4f}")
print(f"Worst base model: {np.min(base_scores):.4f}, best base model: {np.max(base_scores):.4f}")