#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Tue Mar 15 11:09:50 2022 @author: tanu """ from sklearn.metrics import make_scorer, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score, classification_report from sklearn.model_selection import train_test_split, cross_validate, cross_val_score from sklearn.model_selection import StratifiedKFold,RepeatedStratifiedKFold, RepeatedKFold from sklearn.pipeline import Pipeline, make_pipeline #%% GLOBALS rs = {'random_state': 42} njobs = {'n_jobs': 10} scoring_fn = ({'accuracy' : make_scorer(accuracy_score) , 'fscore' : make_scorer(f1_score) , 'mcc' : make_scorer(matthews_corrcoef) , 'precision' : make_scorer(precision_score) , 'recall' : make_scorer(recall_score) , 'roc_auc' : make_scorer(roc_auc_score) , 'jcc' : make_scorer(jaccard_score) }) skf_cv = StratifiedKFold(n_splits = 10 #, shuffle = False, random_state= None) , shuffle = True,**rs) rskf_cv = RepeatedStratifiedKFold(n_splits = 10 , n_repeats = 3 , **rs) mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)} jacc_score_fn = {'jcc': make_scorer(jaccard_score)} ############################################################################### #%% MultModelsCl: function call() mm_skf_scoresD = MultModelsCl(input_df = X , target = y , var_type = 'mixed' , skf_cv = skf_cv , blind_test_input_df = X_bts , blind_test_target = y_bts) baseline_all = pd.DataFrame(mm_skf_scoresD) baseline_all = baseline_all.T #baseline_train = baseline_all.filter(like='train_', axis=1) baseline_CT = baseline_all.filter(like='test_', axis=1) baseline_CT.sort_values(by = ['test_mcc'], ascending = False, inplace = True) baseline_BT = baseline_all.filter(like='bts_', axis=1) baseline_BT.sort_values(by = ['bts_mcc'], ascending = False, inplace = True) #%% SMOTE NC: Oversampling [Numerical + categorical] mm_skf_scoresD7 = MultModelsCl(input_df = X_smnc , target = y_smnc , var_type = 'mixed' , skf_cv = skf_cv , blind_test_input_df = X_bts , blind_test_target = y_bts) smnc_all = pd.DataFrame(mm_skf_scoresD7) smnc_all = smnc_all.T smnc_CT = smnc_all.filter(like='test_', axis=1) smnc_CT.sort_values(by = ['test_mcc'], ascending = False, inplace = True) smnc_BT = smnc_all.filter(like='bts_', axis=1) smnc_BT.sort_values(by = ['bts_mcc'], ascending = False, inplace = True) #%% ROS: Numerical + categorical mm_skf_scoresD3 = MultModelsCl(input_df = X_ros , target = y_ros , var_type = 'mixed' , skf_cv = skf_cv , blind_test_input_df = X_bts , blind_test_target = y_bts) ros_all = pd.DataFrame(mm_skf_scoresD3) ros_all = ros_all.T ros_CT = ros_all.filter(like='test_', axis=1) ros_CT.sort_values(by = ['test_mcc'], ascending = False, inplace = True) ros_BT = ros_all.filter(like='bts_', axis=1) ros_BT.sort_values(by = ['bts_mcc'], ascending = False, inplace = True) #%% RUS: Numerical + categorical mm_skf_scoresD4 = MultModelsCl(input_df = X_rus , target = y_rus , var_type = 'mixed' , skf_cv = skf_cv , blind_test_input_df = X_bts , blind_test_target = y_bts) rus_all = pd.DataFrame(mm_skf_scoresD4) rus_all = rus_all.T rus_CT = rus_all.filter(like='test_', axis=1) rus_CT.sort_values(by = ['test_mcc'], ascending = False, inplace = True) rus_BT = rus_all.filter(like='bts_' , axis=1) rus_BT.sort_values(by = ['bts_mcc'], ascending = False, inplace = True) #%% ROS + RUS Combined: Numerical + categorical mm_skf_scoresD8 = MultModelsCl(input_df = X_rouC , target = y_rouC , var_type = 'mixed' , skf_cv = skf_cv , blind_test_input_df = X_bts , blind_test_target = y_bts) rouC_all = pd.DataFrame(mm_skf_scoresD8) rouC_all = rouC_all.T rouC_CT = rouC_all.filter(like='test_', axis=1) rouC_CT.sort_values(by = ['test_mcc'], ascending = False, inplace = True) rouC_BT = rouC_all.filter(like='bts_', axis=1) rouC_BT.sort_values(by = ['bts_mcc'], ascending = False, inplace = True) #%% SMOTE OS: Numerical only # mm_skf_scoresD2 = MultModelsCl(input_df = X_sm # , target = y_sm # , var_type = 'numerical' # , skf_cv = skf_cv) # sm_all = pd.DataFrame(mm_skf_scoresD2) # sm_all = sm_all.T # sm_CT = sm_all.filter(like='test_', axis=1) #sm_CT.sort_values(by = ['test_mcc'], ascending = False, inplace = True) # sm_BT = sm_all.filter(like='bts_', axis=1) #sm_BT.sort_values(by = ['bts_mcc'], ascending = False, inplace = True) #%% SMOTE ENN: Over + Undersampling combined: Numerical ONLY # mm_skf_scoresD5 = MultModelsCl(input_df = X_enn # , target = y_enn # , var_type = 'numerical' # , skf_cv = skf_cv # , blind_test_input_df = X_bts # , blind_test_target = y_bts) # enn_all = pd.DataFrame(mm_skf_scoresD5) # enn_all = enn_all.T # enn_CT = enn_all.filter(like='test_', axis=1) #enn_CT.sort_values(by = ['test_mcc'], ascending = False, inplace = True) # enn_BT = enn_all.filter(like='bts_', axis=1) #enn_BT.sort_values(by = ['bts_mcc'], ascending = False, inplace = True) #%% Repeated ENN # mm_skf_scoresD6 = MultModelsCl(input_df = X_renn # , target = y_renn # , var_type = 'numerical' # , skf_cv = skf_cv # , blind_test_input_df = X_bts # , blind_test_target = y_bts) # renn_all = pd.DataFrame(mm_skf_scoresD6) # renn_all = renn_all.T # renn_CT = renn_all.filter(like='test_', axis=1) #renn_CT.sort_values(by = ['test_mcc'], ascending = False, inplace = True) # renn_BT = renn_all.filter(like='bts_', axis=1) # renn_BT.sort_values(by = ['bts_mcc'], ascending = False, inplace = True)