search

miguelgfierro / ai_projects

AI projects
https://miguelgfierro.com/
Other
773 stars 175 forks source link

Check why I have the performance drop in XGBoost hist #117

Closed miguelgfierro closed 2 years ago

miguelgfierro commented 2 years ago

I added a comment in XGBoost discuss channel: https://discuss.xgboost.ai/t/xgboost-xgboost-hist-and-lightgbm-benchmark-xgboost-hist-performance-drop/2719

Issue in 1.5.0, 1.5.1 and 1.5.2 -> https://github.com/dmlc/xgboost/issues/7783

miguelgfierro commented 2 years ago

Test with XGBoost==1.5.0

{
    "lgbm": {
        "performance": {
            "AUC": 0.9099440398404464,
            "Accuracy": 0.8288,
            "F1": 0.8528830454584515,
            "Precision": 0.8450404427415922,
            "Recall": 0.8608725821840576
        },
        "test_time": 0.06698051699999041,
        "train_time": 1.3244877409999845
    },
    "xgb": {
        "performance": {
            "AUC": 0.9030499315502538,
            "Accuracy": 0.78585,
            "F1": 0.8344606346384262,
            "Precision": 0.752579475738985,
            "Recall": 0.9363344609246249
        },
        "test_time": 0.024577681000010898,
        "train_time": 6.808380440999997
    },
    "xgb_hist": {
        "performance": {
            "AUC": 0.7055771673038684,
            "Accuracy": 0.57695,
            "F1": 0.7315417076498398,
            "Precision": 0.5767460476285772,
            "Recall": 0.9999132622083442
        },
        "test_time": 0.007026228000000856,
        "train_time": 0.456990138000009
    }
}

test xgboost==1.5.1

{
    "lgbm": {
        "performance": {
            "AUC": 0.9079323396507061,
            "Accuracy": 0.82545,
            "F1": 0.8514278418521514,
            "Precision": 0.84136596854235,
            "Recall": 0.8617332873880083
        },
        "test_time": 0.05388455799999292,
        "train_time": 1.0370406399999865
    },
    "xgb": {
        "performance": {
            "AUC": 0.9009313475174743,
            "Accuracy": 0.7839,
            "F1": 0.8335130970724192,
            "Precision": 0.7538322185061316,
            "Recall": 0.9320296347346657
        },
        "test_time": 0.02538965299999063,
        "train_time": 6.40890938299998
    },
    "xgb_hist": {
        "performance": {
            "AUC": 0.7059850769808664,
            "Accuracy": 0.5808,
            "F1": 0.7346499556905938,
            "Precision": 0.5806483890334201,
            "Recall": 0.9998277050310131
        },
        "test_time": 0.00643289000004188,
        "train_time": 0.4133953040000051
    }
}

test xgboost==1.6.0rc1

{
    "lgbm": {
        "performance": {
            "AUC": 0.9107592746802494,
            "Accuracy": 0.83225,
            "F1": 0.8560147633148792,
            "Precision": 0.8479000170039109,
            "Recall": 0.8642863333044458
        },
        "test_time": 0.0745178820000092,
        "train_time": 0.9469629170000644
    },
    "xgb": {
        "performance": {
            "AUC": 0.9033527393352302,
            "Accuracy": 0.7873,
            "F1": 0.8353460287970275,
            "Precision": 0.7547737287542842,
            "Recall": 0.9351763584366063
        },
        "test_time": 0.020666641000048003,
        "train_time": 7.129535925999903
    },
    "xgb_hist": {
        "performance": {
            "AUC": 0.9079546106230758,
            "Accuracy": 0.79705,
            "F1": 0.8405922318658446,
            "Precision": 0.7686009767308245,
            "Recall": 0.9274633850420314
        },
        "test_time": 0.027357233000088854,
        "train_time": 1.9496805300000233
    }
}