diff --git a/scripts/ml/combined_model/cm_datai.py b/scripts/ml/combined_model/cm_datai.py deleted file mode 100644 index b41e980..0000000 --- a/scripts/ml/combined_model/cm_datai.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Wed Jun 29 19:44:06 2022 - -@author: tanu -""" -import sys, os -import pandas as pd -import numpy as np -import re -from copy import deepcopy -from sklearn import linear_model -from sklearn import datasets -from collections import Counter -############################################################################### -homedir = os.path.expanduser("~") -sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions') -sys.path -############################################################################### -outdir = homedir + '/git/LSHTM_ML/output/combined/' - -#==================== -# Import ML functions -#==================== -from ml_data_combined import * -from MultClfs_logo_skf import * -#from GetMLData import * -#from SplitTTS import * - -skf_cv = StratifiedKFold(n_splits = 10 , shuffle = True, random_state = 42) - -#logo = LeaveOneGroupOut() - -######################################################################## -# COMPLETE data: No tts_split -######################################################################## -#%% -def CMLogoData(cm_input_df = pd.DataFrame() - , all_genes = ["embb", "katg", "rpob", "pnca", "gid", "alr"] - , bts_genes = ["embb", "katg" - , "rpob", "pnca", "gid" - ] - , cols_to_drop = ['dst', 'dst_mode', 'gene_name'] - , target_var = 'dst_mode' - , gene_group = 'gene_name' - , std_gene_omit = [] - ): - - cm_dataD = {} - for bts_gene in bts_genes: - print('\n BTS gene:', bts_gene) - - if not std_gene_omit: - training_genesL = ['alr'] - else: - training_genesL = [] - - tr_gene_omit = std_gene_omit + [bts_gene] - n_tr_genes = (len(bts_genes) - (len(std_gene_omit))) - #n_total_genes = (len(bts_genes) - len(std_gene_omit)) - n_total_genes = len(all_genes) - - training_genesL = training_genesL + list(set(bts_genes) - set(tr_gene_omit)) - #training_genesL = [element for element in bts_genes if element not in tr_gene_omit] - - print('\nTotal genes: ', n_total_genes - ,'\nTraining on:', n_tr_genes - ,'\nTraining on genes:', training_genesL - , '\nOmitted genes:', tr_gene_omit - , '\nBlind test gene:', bts_gene) - - tts_split_type = "logo_skf_BT_" + bts_gene - - outFile = outdir + str(n_tr_genes+1) + "genes_" + tts_split_type + ".csv" - - print(outFile) - - bts_geneD = {} - #------- - # training - #------ - cm_training_df = cm_input_df[~cm_input_df['gene_name'].isin(tr_gene_omit)] - - cm_X = cm_training_df.drop(cols_to_drop, axis=1, inplace=False) - #cm_y = cm_training_df.loc[:,'dst_mode'] - cm_y = cm_training_df.loc[:, target_var] - - gene_group = cm_training_df.loc[:,'gene_name'] - - print('\nTraining data dim:', cm_X.shape - , '\nTraining Target dim:', cm_y.shape) - - if all(cm_X.columns.isin(cols_to_drop) == False): - print('\nChecked training df does NOT have Target var') - else: - sys.exit('\nFAIL: training data contains Target var') - - #--------------- - # BTS: genes - #--------------- - cm_test_df = cm_input_df[cm_input_df['gene_name'].isin([bts_gene])] - - cm_bts_X = cm_test_df.drop(cols_to_drop, axis = 1, inplace = False) - #cm_bts_y = cm_test_df.loc[:, 'dst_mode'] - cm_bts_y = cm_test_df.loc[:, target_var] - - print('\nTEST data dim:', cm_bts_X.shape - , '\nTEST Target dim:', cm_bts_y.shape) - - - bts_geneD = {'cm_X' : cm_X - , 'cm_y' : cm_y - , 'cm_bts_X': cm_bts_X - , 'cm_bts_y': cm_bts_y} - - cm_dataD[bts_gene] = bts_geneD - - return(cm_dataD) - -#%% -df_complete_6g = CMLogoData(cm_input_df = combined_df, std_gene_omit=[] ) -df_complete_5g = CMLogoData(cm_input_df = combined_df, std_gene_omit=['alr']) -# checks - -len(df_complete_6g['embb']['cm_X']) -#len(df_complete_6g['embb']['cm_y']) - -len(df_complete_5g['embb']['cm_X']) -#len(df_complete_5g['embb']['cm_y']) - -df_actual_6g = CMLogoData(cm_input_df = combined_df_actual, std_gene_omit=[] ) -df_actual_5g = CMLogoData(cm_input_df = combined_df_actual, std_gene_omit=['alr']) - -len(df_actual_6g['embb']['cm_X']) -len(df_actual_5g['embb']['cm_X']) diff --git a/scripts/ml/combined_model/cm_logo_skf.py b/scripts/ml/combined_model/cm_logo_skf.py index 8fde3b5..f57ae43 100755 --- a/scripts/ml/combined_model/cm_logo_skf.py +++ b/scripts/ml/combined_model/cm_logo_skf.py @@ -176,7 +176,8 @@ def CMLogoSkf(cm_input_df print("Running Multiple models on LOGO with SKF") #%%:Running Multiple models on LOGO with SKF - cD3_v2 = MultModelsCl_logo_skf(input_df = cm_X +# cD3_v2 = MultModelsCl_logo_skf(input_df = cm_X # two func were identical excpet for name + cD3_v2 = MultModelsCl(input_df = cm_X , target = cm_y , sel_cv = skf_cv , tts_split_type = tts_split_type diff --git a/scripts/ml/ml_functions/CHECK_model.py b/scripts/ml/ml_functions/CHECK_model.py deleted file mode 100644 index bf740da..0000000 --- a/scripts/ml/ml_functions/CHECK_model.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Jul 7 22:18:14 2022 - -@author: tanu -""" - - -# Create a pipeline that standardizes the data then creates a model -import pandas as pd - -from pandas import read_csv -from sklearn.model_selection import KFold -from sklearn.model_selection import cross_val_score -from sklearn.preprocessing import StandardScaler -from sklearn.pipeline import Pipeline -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis - -# load data -url = "https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv" -names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class'] -dataframe = read_csv(url, names=names) -array = dataframe.values -X = array[:,0:8] -X = pd.DataFrame(X) - -Y = array[:,8] -Y = pd.DataFrame(Y) -kfold = KFold(n_splits=1, random_state=None) -spl_type = "check" -fooD1 = MultModelsCl(input_df = X - , target = Y - , sel_cv = kfold - , run_blind_test = False - #, blind_test_df = df2['X_bts'] - #, blind_test_target = df2['y_bts'] - , add_cm = False - , add_yn = False - , tts_split_type = spl_type - , resampling_type = 'none' # default - , var_type = ['mixed'] - , scale_numeric = ['std'] - , return_formatted_output = True - ) - -# create pipeline -estimators = [] -estimators.append(('standardize', StandardScaler())) -estimators.append(('lda', LinearDiscriminantAnalysis())) -model = Pipeline(estimators) - -# evaluate pipeline -seed = 7 -#kfold = KFold(n_splits=10, random_state=seed) -kfold = KFold(n_splits=10, random_state=None) - -results = cross_val_score(model, X, Y, cv=kfold) -print(results.mean()) -results_A = round(results.mean(),2) - -results2 = cross_val_score(model, X, Y, cv=kfold, scoring = "recall") -print(results2.mean()) -results_R = round(results2.mean(),2) - -results3 = cross_val_score(model, X, Y, cv=kfold, scoring = "precision") -print(results3.mean()) -results_P = round(results3.mean(),2) - -results4 = cross_val_score(model, X, Y, cv=kfold, scoring = "f1") -print(results4.mean()) -results_f1 = round(results4.mean(),2) - -results5 = cross_val_score(model, X, Y, cv=kfold, scoring = "jaccard") -print(results5.mean()) -results_J = round(results5.mean(),2) - -results6 = cross_val_score(model, X, Y, cv=kfold, scoring = "matthews_corrcoef") -print(results6.mean()) -results_mcc = round(results6.mean(),2) - - -#%% -import numpy as np -from sklearn.compose import ColumnTransformer -from sklearn.datasets import fetch_openml -from sklearn.pipeline import Pipeline, make_pipeline -from sklearn.decomposition import PCA -from sklearn.impute import SimpleImputer, KNNImputer -from sklearn.preprocessing import RobustScaler, OneHotEncoder -from sklearn.ensemble import RandomForestClassifier -from sklearn.model_selection import train_test_split, cross_val_score, RandomizedSearchCV - -X_train, X_test, y_train, y_test = train_test_split(X, Y, stratify=Y, test_size=0.2) - -fooD2 = MultModelsCl(input_df = X_train - , target = y_train - , sel_cv = kfold - , run_blind_test = True - , blind_test_df = X_test - , blind_test_target = y_test - , add_cm = False - , add_yn = False - , tts_split_type = spl_type - , resampling_type = 'none' # default - , var_type = ['mixed'] - , scale_numeric = ['std'] - , return_formatted_output = True - ) -# fitting and predicting on test -model.fit(X_train, y_train) -y_pred = model.predict(X_test) - - -results_A = round(cross_val_score(model, X_train, y_train, cv=kfold).mean(),2) -print(results_A) - -results_P = round(cross_val_score(model, X_train, y_train, cv=kfold, scoring = "precision").mean(),2) -print(results_P) - -results_R = round(cross_val_score(model, X_train, y_train, cv=kfold, scoring = "recall").mean(),2) -print(results_R) - -results_F = round(cross_val_score(model, X_train, y_train, cv=kfold, scoring = "f1").mean(),2) -print(results_F) - -results_J = round(cross_val_score(model, X_train, y_train, cv=kfold, scoring = "jaccard").mean(),2) -print(results_J) - -results_M = round(cross_val_score(model, X_train, y_train, cv=kfold, scoring = "matthews_corrcoef").mean(),2) -print(results_M) - -print('\nCV example accuracy:', results_P) -print('BTS example accuracy:', round(precision_score(y_test, y_pred),2)) - -print('\nCV example accuracy:', results_J) -print('BTS example accuracy:', round(jaccard_score(y_test, y_pred),2)) - -print('\nCV example accuracy:', results_R) -print('BTS example accuracy:', round(recall_score(y_test, y_pred),2)) - -print('\nCV example accuracy:', results_F) -print('BTS example accuracy:', round(f1_score(y_test, y_pred),2)) - -print('\nCV example accuracy:', results_A) -print('BTS example accuracy:', round(accuracy_score(y_test, y_pred),2)) - -print('\nCV example accuracy:', results_M) -print('BTS example accuracy:', round(matthews_corrcoef(y_test, y_pred),2)) - - - - diff --git a/scripts/ml/ml_functions/MultClfs.py b/scripts/ml/ml_functions/MultClfs.py index c22e41b..5366a6c 100755 --- a/scripts/ml/ml_functions/MultClfs.py +++ b/scripts/ml/ml_functions/MultClfs.py @@ -92,10 +92,10 @@ scoring_fn = ({ 'mcc' : make_scorer(matthews_corrcoef) , 'roc_auc' : make_scorer(roc_auc_score) , 'jcc' : make_scorer(jaccard_score) }) - +# for sel_cv INSIDE FUNCTION CALL NOW #skf_cv = StratifiedKFold(n_splits = 10 # #, shuffle = False, random_state= None) -# , shuffle = True,**rs) +# , shuffle = True, **rs) #rskf_cv = RepeatedStratifiedKFold(n_splits = 10 # , n_repeats = 3 @@ -149,25 +149,26 @@ scoreBT_mapD = {'bts_mcc' : 'MCC' # Run Multiple Classifiers ############################ # Multiple Classification - Model Pipeline -def MultModelsCl(input_df, target - , sel_cv - , tts_split_type - , resampling_type - #, group = None - - , add_cm = True # adds confusion matrix based on cross_val_predict - , add_yn = True # adds target var class numbers - , var_type = ['numerical', 'categorical','mixed'] - , scale_numeric = ['min_max', 'std', 'min_max_neg', 'none'] +def MultModelsCl(input_df + , target + , sel_cv + , tts_split_type + , resampling_type + #, group = None + + , add_cm = True # adds confusion matrix based on cross_val_predict + , add_yn = True # adds target var class numbers + , var_type = ['numerical', 'categorical','mixed'] + , scale_numeric = ['min_max', 'std', 'min_max_neg', 'none'] - , run_blind_test = True - , blind_test_df = pd.DataFrame() - , blind_test_target = pd.Series(dtype = int) - , return_formatted_output = True + , run_blind_test = True + , blind_test_df = pd.DataFrame() + , blind_test_target = pd.Series(dtype = int) + , return_formatted_output = True - , random_state = 42 - , n_jobs = os.cpu_count() # the number of jobs should equal the number of CPU cores - ): + , random_state = 42 + , n_jobs = os.cpu_count() # the number of jobs should equal the number of CPU cores + ): ''' @ param input_df: input features @@ -357,10 +358,9 @@ def MultModelsCl(input_df, target y_pred = cross_val_predict(model_pipeline , input_df , target - #, commented out thing, - , cv=sel_cv - , **njobs - ) + , cv = sel_cv + #, groups = group + , **njobs) #_tn, _fp, _fn, _tp = confusion_matrix(y_pred, y).ravel() # internally tn, fp, fn, tp = confusion_matrix(y_pred, target).ravel() diff --git a/scripts/ml/ml_functions/MultClfs_SIMPLE.py b/scripts/ml/ml_functions/MultClfs_SIMPLE.py deleted file mode 100644 index 7e0fdba..0000000 --- a/scripts/ml/ml_functions/MultClfs_SIMPLE.py +++ /dev/null @@ -1,553 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Fri Mar 4 15:25:33 2022 - -@author: tanu -""" -#%% -import os, sys -import pandas as pd -import numpy as np -import pprint as pp -from copy import deepcopy -from sklearn import linear_model -from sklearn import datasets -from collections import Counter - -from sklearn.linear_model import LogisticRegression, LogisticRegressionCV -from sklearn.linear_model import RidgeClassifier, RidgeClassifierCV, SGDClassifier, PassiveAggressiveClassifier - -from sklearn.naive_bayes import BernoulliNB -from sklearn.neighbors import KNeighborsClassifier -from sklearn.svm import SVC -from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier -from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier, BaggingClassifier -from sklearn.naive_bayes import GaussianNB -from sklearn.gaussian_process import GaussianProcessClassifier, kernels -from sklearn.gaussian_process.kernels import RBF, DotProduct, Matern, RationalQuadratic, WhiteKernel - -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis -from sklearn.neural_network import MLPClassifier - -from sklearn.svm import SVC -from xgboost import XGBClassifier -from sklearn.naive_bayes import MultinomialNB -from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder - -from sklearn.compose import ColumnTransformer -from sklearn.compose import make_column_transformer - -from sklearn.metrics import make_scorer, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score -from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score, classification_report - -# added -from sklearn.model_selection import train_test_split, cross_validate, cross_val_score, LeaveOneOut, KFold, RepeatedKFold, cross_val_predict - -from sklearn.model_selection import train_test_split, cross_validate, cross_val_score -from sklearn.model_selection import StratifiedKFold,RepeatedStratifiedKFold, RepeatedKFold - -from sklearn.pipeline import Pipeline, make_pipeline - -from sklearn.feature_selection import RFE, RFECV - -import itertools -import seaborn as sns -import matplotlib.pyplot as plt - -from statistics import mean, stdev, median, mode - -from imblearn.over_sampling import RandomOverSampler -from imblearn.under_sampling import RandomUnderSampler -from imblearn.over_sampling import SMOTE -from sklearn.datasets import make_classification -from imblearn.combine import SMOTEENN -from imblearn.combine import SMOTETomek - -from imblearn.over_sampling import SMOTENC -from imblearn.under_sampling import EditedNearestNeighbours -from imblearn.under_sampling import RepeatedEditedNearestNeighbours - -from sklearn.model_selection import GridSearchCV -from sklearn.base import BaseEstimator -from sklearn.impute import KNNImputer as KNN -import json -import argparse -import re -from sklearn.decomposition import PCA -#%% GLOBALS -rs = {'random_state': 42} -njobs = {'n_jobs': os.cpu_count() } # the number of jobs should equal the number of CPU cores - -scoring_fn = ({ 'mcc' : make_scorer(matthews_corrcoef) - , 'fscore' : make_scorer(f1_score) - , 'precision' : make_scorer(precision_score) - , 'recall' : make_scorer(recall_score) - , 'accuracy' : make_scorer(accuracy_score) - , 'roc_auc' : make_scorer(roc_auc_score) - , 'jcc' : make_scorer(jaccard_score) - }) - -skf_cv = StratifiedKFold(n_splits = 10 - #, shuffle = False, random_state= None) - , shuffle = True,**rs) - -rskf_cv = RepeatedStratifiedKFold(n_splits = 10 - , n_repeats = 3 - , **rs) - -mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)} -jacc_score_fn = {'jcc': make_scorer(jaccard_score)} - -############################################################################### -score_type_ordermapD = { 'mcc' : 1 - , 'fscore' : 2 - , 'jcc' : 3 - , 'precision' : 4 - , 'recall' : 5 - , 'accuracy' : 6 - , 'roc_auc' : 7 - , 'TN' : 8 - , 'FP' : 9 - , 'FN' : 10 - , 'TP' : 11 - , 'trainingY_neg': 12 - , 'trainingY_pos': 13 - , 'blindY_neg' : 14 - , 'blindY_pos' : 15 - , 'fit_time' : 16 - , 'score_time' : 17 - } - -scoreCV_mapD = {'test_mcc' : 'MCC' - , 'test_fscore' : 'F1' - , 'test_precision' : 'Precision' - , 'test_recall' : 'Recall' - , 'test_accuracy' : 'Accuracy' - , 'test_roc_auc' : 'ROC_AUC' - , 'test_jcc' : 'JCC' - } - -scoreBT_mapD = {'bts_mcc' : 'MCC' - , 'bts_fscore' : 'F1' - , 'bts_precision' : 'Precision' - , 'bts_recall' : 'Recall' - , 'bts_accuracy' : 'Accuracy' - , 'bts_roc_auc' : 'ROC_AUC' - , 'bts_jcc' : 'JCC' - } - -#%%############################################################################ -############################ -# MultModelsCl() -# Run Multiple Classifiers -############################ -# Multiple Classification - Model Pipeline -def MultModelsCl(input_df, target - , sel_cv - , blind_test_df - , blind_test_target - , tts_split_type - - , resampling_type = 'none' # default - , add_cm = True # adds confusion matrix based on cross_val_predict - , add_yn = True # adds target var class numbers - , var_type = ['numerical', 'categorical','mixed'] - , scale_numeric = ['min_max', 'std', 'min_max_neg', 'none'] - , run_blind_test = True - , return_formatted_output = True): - - ''' - @ param input_df: input features - @ type: df with input features WITHOUT the target variable - - @param target: target (or output) feature - @type: df or np.array or Series - - @param skv_cv: stratifiedK fold int or object to allow shuffle and random state to pass - @type: int or StratifiedKfold() - - @var_type: numerical, categorical and mixed to determine what col_transform to apply (MinMaxScalar and/or one-hot encoder) - @type: list - - returns - Dict containing multiple classification scores for each model and mean of each Stratified Kfold including training - ''' - - #====================================================== - # Determine categorical and numerical features - #====================================================== - numerical_ix = input_df.select_dtypes(include=['int64', 'float64']).columns - numerical_ix - categorical_ix = input_df.select_dtypes(include=['object', 'bool']).columns - categorical_ix - - #====================================================== - # Determine preprocessing steps ~ var_type - #====================================================== - - # if var_type == 'numerical': - # t = [('num', MinMaxScaler(), numerical_ix)] - - # if var_type == 'categorical': - # t = [('cat', OneHotEncoder(), categorical_ix)] - - # # if var_type == 'mixed': - # # t = [('num', MinMaxScaler(), numerical_ix) - # # , ('cat', OneHotEncoder(), categorical_ix) ] - - # if var_type == 'mixed': - # t = [('cat', OneHotEncoder(), categorical_ix) ] - if type(var_type) == list: - var_type = str(var_type[0]) - else: - var_type = var_type - - if var_type in ['numerical','mixed']: - if scale_numeric == ['none']: - t = [('cat', OneHotEncoder(), categorical_ix)] - if scale_numeric != ['none']: - if scale_numeric == ['min_max']: - scaler = MinMaxScaler() - if scale_numeric == ['min_max_neg']: - scaler = MinMaxScaler(feature_range=(-1, 1)) - if scale_numeric == ['std']: - scaler = StandardScaler() - - t = [('num', scaler, numerical_ix) - , ('cat', OneHotEncoder(), categorical_ix)] - - - if var_type == 'categorical': - t = [('cat', OneHotEncoder(), categorical_ix)] - - - col_transform = ColumnTransformer(transformers = t - , remainder='passthrough') - - - #====================================================== - # Specify multiple Classification Models - #====================================================== - models = [('AdaBoost Classifier' , AdaBoostClassifier(**rs) ) - , ('Bagging Classifier' , BaggingClassifier(**rs, **njobs, bootstrap = True, oob_score = True, verbose = 3, n_estimators = 100) ) - , ('Decision Tree' , DecisionTreeClassifier(**rs) ) - , ('Extra Tree' , ExtraTreeClassifier(**rs) ) - , ('Extra Trees' , ExtraTreesClassifier(**rs) ) - , ('Gradient Boosting' , GradientBoostingClassifier(**rs) ) - , ('Gaussian NB' , GaussianNB() ) - , ('Gaussian Process' , GaussianProcessClassifier(**rs) ) - , ('K-Nearest Neighbors' , KNeighborsClassifier() ) - , ('LDA' , LinearDiscriminantAnalysis() ) - , ('Logistic Regression' , LogisticRegression(**rs) ) - , ('Logistic RegressionCV' , LogisticRegressionCV(cv = 3, **rs)) - , ('MLP' , MLPClassifier(max_iter = 500, **rs) ) - , ('Multinomial' , MultinomialNB() ) - , ('Naive Bayes' , BernoulliNB() ) - , ('Passive Aggresive' , PassiveAggressiveClassifier(**rs, **njobs) ) - , ('QDA' , QuadraticDiscriminantAnalysis() ) - # , ('Random Forest' , RandomForestClassifier(**rs, n_estimators = 1000, **njobs ) ) - , ('Random Forest2' , RandomForestClassifier(min_samples_leaf = 5 - , n_estimators = 1000 - , bootstrap = True - , oob_score = True - , **njobs - , **rs - , max_features = 'auto') ) - , ('Ridge Classifier' , RidgeClassifier(**rs) ) - , ('Ridge ClassifierCV' , RidgeClassifierCV(cv = 3) ) - , ('SVC' , SVC(**rs) ) - , ('Stochastic GDescent' , SGDClassifier(**rs, **njobs) ) - , ('XGBoost' , XGBClassifier(**rs, verbosity = 0, use_label_encoder =False, **njobs) ) - - ] - - mm_skf_scoresD = {} - - print('\n==============================================================\n' - , '\nRunning several classification models (n):', len(models) - ,'\nList of models:') - for m in models: - print(m) - print('\n================================================================\n') - - index = 1 - for model_name, model_fn in models: - print('\nRunning classifier:', index - , '\nModel_name:' , model_name - , '\nModel func:' , model_fn) - index = index+1 - - model_pipeline = Pipeline([ - ('prep' , col_transform) - , ('model' , model_fn)]) - - # model_pipeline = Pipeline([ - # ('prep' , col_transform) - # , ('pca' , PCA(n_components = 2)) - # , ('model' , model_fn)]) - - - print('\nRunning model pipeline:', model_pipeline) - skf_cv_modD = cross_validate(model_pipeline - , input_df - , target - , cv = sel_cv - , scoring = scoring_fn - , return_train_score = True) - #============================== - # Extract mean values for CV - #============================== - mm_skf_scoresD[model_name] = {} - - for key, value in skf_cv_modD.items(): - print('\nkey:', key, '\nvalue:', value) - print('\nmean value:', np.mean(value)) - mm_skf_scoresD[model_name][key] = round(np.mean(value),2) - - # ADD more info: meta data related to input df - mm_skf_scoresD[model_name]['resampling'] = resampling_type - mm_skf_scoresD[model_name]['n_training_size'] = len(input_df) - mm_skf_scoresD[model_name]['n_trainingY_ratio'] = round(Counter(target)[0]/Counter(target)[1], 2) - mm_skf_scoresD[model_name]['n_features'] = len(input_df.columns) - mm_skf_scoresD[model_name]['tts_split'] = tts_split_type - - ####################################################################### - #====================================================== - # Option: Add confusion matrix from cross_val_predict - # Understand and USE with caution - #====================================================== - if add_cm: - cmD = {} - - # Calculate cm - y_pred = cross_val_predict(model_pipeline, input_df, target, cv = sel_cv, **njobs) - #_tn, _fp, _fn, _tp = confusion_matrix(y_pred, y).ravel() # internally - tn, fp, fn, tp = confusion_matrix(y_pred, target).ravel() - - # Build cm dict - cmD = {'TN' : tn - , 'FP': fp - , 'FN': fn - , 'TP': tp} - - # Update cv dict cmD - mm_skf_scoresD[model_name].update(cmD) - - #============================================= - # Option: Add targety numbers for data - #============================================= - if add_yn: - tnD = {} - - # Build tn numbers dict - tnD = {'n_trainingY_neg' : Counter(target)[0] - , 'n_trainingY_pos' : Counter(target)[1] } - - # Update cv dict with cmD and tnD - mm_skf_scoresD[model_name].update(tnD) - -#%% - #========================= - # Option: Blind test (bts) - #========================= - if run_blind_test: - btD = {} - - # Build bts numbers dict - btD = {'n_blindY_neg' : Counter(blind_test_target)[0] - , 'n_blindY_pos' : Counter(blind_test_target)[1] - , 'n_testY_ratio' : round(Counter(blind_test_target)[0]/Counter(blind_test_target)[1], 2) - , 'n_test_size' : len(blind_test_df) } - - # Update cmD+tnD dicts with btD - mm_skf_scoresD[model_name].update(btD) - - #-------------------------------------------------------- - # Build the final results with all scores for the model - #-------------------------------------------------------- - #bts_predict = gscv_fs.predict(blind_test_df) - model_pipeline.fit(input_df, target) - bts_predict = model_pipeline.predict(blind_test_df) - - bts_mcc_score = round(matthews_corrcoef(blind_test_target, bts_predict),2) - print('\nMCC on Blind test:' , bts_mcc_score) - #print('\nAccuracy on Blind test:', round(accuracy_score(blind_test_target, bts_predict),2)) - print('\nMCC on Training:' , mm_skf_scoresD[model_name]['test_mcc'] ) - - mm_skf_scoresD[model_name]['bts_mcc'] = bts_mcc_score - mm_skf_scoresD[model_name]['bts_fscore'] = round(f1_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_precision'] = round(precision_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_recall'] = round(recall_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_accuracy'] = round(accuracy_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_roc_auc'] = round(roc_auc_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_jcc'] = round(jaccard_score(blind_test_target, bts_predict),2) - #mm_skf_scoresD[model_name]['diff_mcc'] = train_test_diff_MCC - - - #return(mm_skf_scoresD) - #============================ - # Process the dict to have WF - #============================ - if return_formatted_output: - CV_BT_metaDF = ProcessMultModelsCl(mm_skf_scoresD) - return(CV_BT_metaDF) - else: - return(mm_skf_scoresD) - -#%% Process output function ################################################### -############################ -# ProcessMultModelsCl() -############################ -#Processes the dict from above if use_formatted_output = True - -def ProcessMultModelsCl(inputD = {}, blind_test_data = True): - - scoresDF = pd.DataFrame(inputD) - - #------------------------ - # Extracting split_name - #----------------------- - tts_split_nameL = [] - for k,v in inputD.items(): - tts_split_nameL = tts_split_nameL + [v['tts_split']] - - if len(set(tts_split_nameL)) == 1: - tts_split_name = str(list(set(tts_split_nameL))[0]) - print('\nExtracting tts_split_name:', tts_split_name) - - #---------------------- - # WF: CV results - #---------------------- - scoresDFT = scoresDF.T - - scoresDF_CV = scoresDFT.filter(regex='^test_.*$', axis = 1); scoresDF_CV.columns - # map colnames for consistency to allow concatenting - scoresDF_CV.columns = scoresDF_CV.columns.map(scoreCV_mapD); scoresDF_CV.columns - scoresDF_CV['source_data'] = 'CV' - - #---------------------- - # WF: Meta data - #---------------------- - metaDF = scoresDFT.filter(regex='^(?!test_.*$|bts_.*$|train_.*$).*'); metaDF.columns - - print('\nTotal cols in each df:' - , '\nCV df:', len(scoresDF_CV.columns) - , '\nmetaDF:', len(metaDF.columns)) - - #------------------------------------- - # Combine WF: CV + Metadata - #------------------------------------- - - combDF = pd.merge(scoresDF_CV, metaDF, left_index = True, right_index = True) - print('\nAdding column: Model_name') - combDF['Model_name'] = combDF.index - - #---------------------- - # WF: BTS results - #---------------------- - if blind_test_data: - - scoresDF_BT = scoresDFT.filter(regex='^bts_.*$', axis = 1); scoresDF_BT.columns - # map colnames for consistency to allow concatenting - scoresDF_BT.columns = scoresDF_BT.columns.map(scoreBT_mapD); scoresDF_BT.columns - scoresDF_BT['source_data'] = 'BT' - - - print('\nTotal cols in bts df:' - , '\nBT_df:', len(scoresDF_BT.columns)) - - if len(scoresDF_CV.columns) == len(scoresDF_BT.columns): - print('\nFirst proceeding to rowbind CV and BT dfs:') - expected_ncols_out = len(scoresDF_BT.columns) + len(metaDF.columns) - print('\nFinal output should have:', expected_ncols_out, 'columns' ) - - #----------------- - # Combine WF - #----------------- - dfs_combine_wf = [scoresDF_CV, scoresDF_BT] - - print('\nCombinig', len(dfs_combine_wf), 'using pd.concat by row ~ rowbind' - , '\nChecking Dims of df to combine:' - , '\nDim of CV:', scoresDF_CV.shape - , '\nDim of BT:', scoresDF_BT.shape) - #print(scoresDF_CV) - #print(scoresDF_BT) - - dfs_nrows_wf = [] - for df in dfs_combine_wf: - dfs_nrows_wf = dfs_nrows_wf + [len(df)] - dfs_nrows_wf = max(dfs_nrows_wf) - - dfs_ncols_wf = [] - for df in dfs_combine_wf: - dfs_ncols_wf = dfs_ncols_wf + [len(df.columns)] - dfs_ncols_wf = max(dfs_ncols_wf) - print(dfs_ncols_wf) - - expected_nrows_wf = len(dfs_combine_wf) * dfs_nrows_wf - expected_ncols_wf = dfs_ncols_wf - - common_cols_wf = list(set.intersection(*(set(df.columns) for df in dfs_combine_wf))) - print('\nNumber of Common columns:', dfs_ncols_wf - , '\nThese are:', common_cols_wf) - - if len(common_cols_wf) == dfs_ncols_wf : - combined_baseline_wf = pd.concat([df[common_cols_wf] for df in dfs_combine_wf], ignore_index=False) - print('\nConcatenating dfs with different resampling methods [WF]:' - , '\nSplit type:', tts_split_name - , '\nNo. of dfs combining:', len(dfs_combine_wf)) - #print('\n================================================^^^^^^^^^^^^') - if len(combined_baseline_wf) == expected_nrows_wf and len(combined_baseline_wf.columns) == expected_ncols_wf: - #print('\n================================================^^^^^^^^^^^^') - - print('\nPASS:', len(dfs_combine_wf), 'dfs successfully combined' - , '\nnrows in combined_df_wf:', len(combined_baseline_wf) - , '\nncols in combined_df_wf:', len(combined_baseline_wf.columns)) - else: - print('\nFAIL: concatenating failed' - , '\nExpected nrows:', expected_nrows_wf - , '\nGot:', len(combined_baseline_wf) - , '\nExpected ncols:', expected_ncols_wf - , '\nGot:', len(combined_baseline_wf.columns)) - sys.exit('\nFIRST IF FAILS') - ## - c1L = list(set(combined_baseline_wf.index)) - c2L = list(metaDF.index) - - #if set(c1L) == set(c2L): - if set(c1L) == set(c2L) and all(x in c2L for x in c1L) and all(x in c1L for x in c2L): - print('\nPASS: proceeding to merge metadata with CV and BT dfs') - combDF = pd.merge(combined_baseline_wf, metaDF, left_index = True, right_index = True) - print('\nAdding column: Model_name') - combDF['Model_name'] = combDF.index - - else: - sys.exit('\nFAIL: Could not merge metadata with CV and BT dfs') - - else: - print('\nConcatenting dfs not possible [WF],check numbers ') - - #------------------------------------- - # Combine WF+Metadata: Final output - #------------------------------------- - - # if len(combDF.columns) == expected_ncols_out: - # print('\nPASS: Combined df has expected ncols') - # else: - # sys.exit('\nFAIL: Length mismatch for combined_df') - - # print('\nAdding column: Model_name') - # combDF['Model_name'] = combDF.index - - print('\n=========================================================' - , '\nSUCCESS: Ran multiple classifiers' - , '\n=======================================================') - - #resampling_methods_wf = combined_baseline_wf[['resampling']] - #resampling_methods_wf = resampling_methods_wf.drop_duplicates() - #, '\n', resampling_methods_wf) - - return combDF - -############################################################################### diff --git a/scripts/ml/ml_functions/MultClfs_fi.py b/scripts/ml/ml_functions/MultClfs_fi.py deleted file mode 100644 index 89562e2..0000000 --- a/scripts/ml/ml_functions/MultClfs_fi.py +++ /dev/null @@ -1,306 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Fri Mar 4 15:25:33 2022 - -@author: tanu -""" -#%% -import os, sys -import pandas as pd -import numpy as np -import pprint as pp -from copy import deepcopy -from sklearn import linear_model -from sklearn import datasets -from collections import Counter - -from sklearn.linear_model import LogisticRegression, LogisticRegressionCV -from sklearn.linear_model import RidgeClassifier, RidgeClassifierCV, SGDClassifier, PassiveAggressiveClassifier - -from sklearn.naive_bayes import BernoulliNB -from sklearn.neighbors import KNeighborsClassifier -from sklearn.svm import SVC -from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier -from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier, BaggingClassifier -from sklearn.naive_bayes import GaussianNB -from sklearn.gaussian_process import GaussianProcessClassifier, kernels -from sklearn.gaussian_process.kernels import RBF, DotProduct, Matern, RationalQuadratic, WhiteKernel - -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis -from sklearn.neural_network import MLPClassifier - -from sklearn.svm import SVC -from xgboost import XGBClassifier -from sklearn.naive_bayes import MultinomialNB -from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder - -from sklearn.compose import ColumnTransformer -from sklearn.compose import make_column_transformer - -from sklearn.metrics import make_scorer, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score -from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score, classification_report - -# added -from sklearn.model_selection import train_test_split, cross_validate, cross_val_score, LeaveOneOut, KFold, RepeatedKFold, cross_val_predict - -from sklearn.model_selection import train_test_split, cross_validate, cross_val_score -from sklearn.model_selection import StratifiedKFold,RepeatedStratifiedKFold, RepeatedKFold - -from sklearn.pipeline import Pipeline, make_pipeline - -from sklearn.feature_selection import RFE, RFECV - -import itertools -import seaborn as sns -import matplotlib.pyplot as plt - -from statistics import mean, stdev, median, mode - -from imblearn.over_sampling import RandomOverSampler -from imblearn.under_sampling import RandomUnderSampler -from imblearn.over_sampling import SMOTE -from sklearn.datasets import make_classification -from imblearn.combine import SMOTEENN -from imblearn.combine import SMOTETomek - -from imblearn.over_sampling import SMOTENC -from imblearn.under_sampling import EditedNearestNeighbours -from imblearn.under_sampling import RepeatedEditedNearestNeighbours - -from sklearn.model_selection import GridSearchCV -from sklearn.base import BaseEstimator -from sklearn.impute import KNNImputer as KNN -import json -import argparse -import re -from sklearn.model_selection import LeaveOneGroupOut - -#%% GLOBALS -rs = {'random_state': 42} -njobs = {'n_jobs': os.cpu_count() } # the number of jobs should equal the number of CPU cores - -scoring_fn = ({ 'mcc' : make_scorer(matthews_corrcoef) - , 'fscore' : make_scorer(f1_score) - , 'precision' : make_scorer(precision_score) - , 'recall' : make_scorer(recall_score) - , 'accuracy' : make_scorer(accuracy_score) - , 'roc_auc' : make_scorer(roc_auc_score) - , 'jcc' : make_scorer(jaccard_score) - }) - -skf_cv = StratifiedKFold(n_splits = 10 - #, shuffle = False, random_state= None) - , shuffle = True,**rs) - -rskf_cv = RepeatedStratifiedKFold(n_splits = 10 - , n_repeats = 3 - , **rs) - -logo = LeaveOneGroupOut() - - -mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)} -jacc_score_fn = {'jcc': make_scorer(jaccard_score)} -############################################################################### -homedir = os.path.expanduser("~") -sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions') -sys.path -############################################################################### -outdir = homedir - -from GetMLData import * -from SplitTTS import * - - -def remove(string): - return(string.replace(" ", "")) -#%%############################################################################ -############################ -# MultModelsCl() -# Run Multiple Classifiers -############################ -# Multiple Classification - Model Pipeline -def MultClfs_fi(input_df, target, sel_cv - , blind_test_df - , blind_test_target - , tts_split_type - - , resampling_type = 'none' # default - #, add_cm = True # adds confusion matrix based on cross_val_predict - #, add_yn = True # adds target var class numbers - , var_type = ['numerical', 'categorical','mixed'] - , run_blind_test = True - #, return_formatted_output = True - ): - - ''' - @ param input_df: input features - @ type: df with input features WITHOUT the target variable - - @param target: target (or output) feature - @type: df or np.array or Series - - @param skv_cv: stratifiedK fold int or object to allow shuffle and random state to pass - @type: int or StratifiedKfold() - - @var_type: numerical, categorical and mixed to determine what col_transform to apply (MinMaxScalar and/or one-ho t encoder) - @type: list - - returns - Dict containing multiple classification scores for each model and mean of each Stratified Kfold including training - ''' - - #====================================================== - # Determine categorical and numerical features - #====================================================== - numerical_ix = input_df.select_dtypes(include=['int64', 'float64']).columns - numerical_ix - categorical_ix = input_df.select_dtypes(include=['object', 'bool']).columns - categorical_ix - - #====================================================== - # Determine preprocessing steps ~ var_type - #====================================================== - if var_type == 'numerical': - t = [('num', MinMaxScaler(), numerical_ix)] - - if var_type == 'categorical': - t = [('cat', OneHotEncoder(), categorical_ix)] - - if var_type == 'mixed': - t = [('num', MinMaxScaler(), numerical_ix) - , ('cat', OneHotEncoder(), categorical_ix) ] - - col_transform = ColumnTransformer(transformers = t - , remainder='passthrough') - - - - #====================================================== - # Specify multiple Classification Models - #====================================================== - models = [('AdaBoost Classifier' , AdaBoostClassifier(**rs) ) - # , ('Bagging Classifier' , BaggingClassifier(**rs, **njobs, bootstrap = True, oob_score = True) ) - # , ('Decision Tree' , DecisionTreeClassifier(**rs) ) - # , ('Extra Tree' , ExtraTreeClassifier(**rs) ) - # , ('Extra Trees' , ExtraTreesClassifier(**rs) ) - # , ('Gradient Boosting' , GradientBoostingClassifier(**rs) ) - # , ('Gaussian NB' , GaussianNB() ) - # , ('Gaussian Process' , GaussianProcessClassifier(**rs) ) - # , ('K-Nearest Neighbors' , KNeighborsClassifier() ) - # , ('LDA' , LinearDiscriminantAnalysis() ) - # , ('Logistic Regression' , LogisticRegression(**rs) ) - # , ('Logistic RegressionCV' , LogisticRegressionCV(cv = 3, **rs)) - # , ('MLP' , MLPClassifier(max_iter = 500, **rs) ) - # , ('Multinomial' , MultinomialNB() ) - # , ('Naive Bayes' , BernoulliNB() ) - # , ('Passive Aggresive' , PassiveAggressiveClassifier(**rs, **njobs) ) - # , ('QDA' , QuadraticDiscriminantAnalysis() ) - # , ('Random Forest' , RandomForestClassifier(**rs, n_estimators = 1000, **njobs ) ) - # # , ('Random Forest2' , RandomForestClassifier(min_samples_leaf = 5 - # # , n_estimators = 1000 - # # , bootstrap = True - # # , oob_score = True - # # , **njobs - # # , **rs - # # , max_features = 'auto') ) - # , ('Ridge Classifier' , RidgeClassifier(**rs) ) - # , ('Ridge ClassifierCV' , RidgeClassifierCV(cv = 3) ) - # , ('SVC' , SVC(**rs) ) - , ('Stochastic GDescent' , SGDClassifier(**rs, **njobs) ) - , ('XGBoost' , XGBClassifier(**rs, verbosity = 0, use_label_encoder =False, **njobs) ) - ] - - mm_skf_scoresD = {} - - print('\n==============================================================\n' - , '\nRunning several classification models (n):', len(models) - ,'\nList of models:') - for m in models: - print(m) - print('\n================================================================\n') - - index = 1 - for model_name, model_fn in models: - print('\nRunning classifier:', index - , '\nModel_name:' , model_name - , '\nModel func:' , model_fn) - index = index+1 - - model_pipeline = Pipeline([ - ('prep' , col_transform) - , ('model' , model_fn)]) - - print('\nRunning model pipeline:', model_pipeline) - skf_cv_modD = cross_validate(model_pipeline - , input_df - , target - , cv = sel_cv - , scoring = scoring_fn) - #============================== - # Extract mean values for CV - #============================== - mm_skf_scoresD[model_name] = {} - - for key, value in skf_cv_modD.items(): - print('\nkey:', key, '\nvalue:', value) - print('\nmean value:', np.mean(value)) - mm_skf_scoresD[model_name][key] = round(np.mean(value),2) - - # ADD more info: meta data related to input df - mm_skf_scoresD[model_name]['resampling'] = resampling_type - mm_skf_scoresD[model_name]['n_training_size'] = len(input_df) - mm_skf_scoresD[model_name]['n_trainingY_ratio'] = round(Counter(target)[0]/Counter(target)[1], 2) - mm_skf_scoresD[model_name]['n_features'] = len(input_df.columns) - mm_skf_scoresD[model_name]['tts_split'] = tts_split_type - - # FS - #mnf = remove(model_name) - #model_pipeline.fit(input_df, target) - #print('\nFeature importance:', (model_pipeline.named_steps.model.feature_importances_)) - #allf_xgboost = model_pipeline.feature_names_in_ - #fsi_model = model_pipeline.named_steps.model.feature_importances_ - #mm_skf_scoresD[model_name]['fs_importance'] = fsi_model - # TODO: add this as a key - #Add - - #pyplot.bar(range(len(model_pipeline.named_steps.model.feature_importances_)), model_pipeline.named_steps.model.feature_importances_) - #pyplot.show() - #plot_importance(model_pipeline.named_steps.model.feature_importances_) - #pyplot.show() - - - if run_blind_test: - btD = {} - - # Build bts numbers dict - btD = {'n_blindY_neg' : Counter(blind_test_target)[0] - , 'n_blindY_pos' : Counter(blind_test_target)[1] - , 'n_testY_ratio' : round(Counter(blind_test_target)[0]/Counter(blind_test_target)[1], 2) - , 'n_test_size' : len(blind_test_df) } - - # Update cmD+tnD dicts with btD - mm_skf_scoresD[model_name].update(btD) - - #-------------------------------------------------------- - # Build the final results with all scores for the model - #-------------------------------------------------------- - #bts_predict = gscv_fs.predict(blind_test_df) - model_pipeline.fit(input_df, target) - bts_predict = model_pipeline.predict(blind_test_df) - - bts_mcc_score = round(matthews_corrcoef(blind_test_target, bts_predict),2) - print('\nMCC on Blind test:' , bts_mcc_score) - #print('\nAccuracy on Blind test:', round(accuracy_score(blind_test_target, bts_predict),2)) - - mm_skf_scoresD[model_name]['bts_mcc'] = bts_mcc_score - mm_skf_scoresD[model_name]['bts_fscore'] = round(f1_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_precision'] = round(precision_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_recall'] = round(recall_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_accuracy'] = round(accuracy_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_roc_auc'] = round(roc_auc_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_jcc'] = round(jaccard_score(blind_test_target, bts_predict),2) - - return(mm_skf_scoresD) -#%% \ No newline at end of file diff --git a/scripts/ml/ml_functions/MultClfs_logo.py b/scripts/ml/ml_functions/MultClfs_logo.py deleted file mode 100755 index bf03382..0000000 --- a/scripts/ml/ml_functions/MultClfs_logo.py +++ /dev/null @@ -1,528 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Fri Mar 4 15:25:33 2022 - -@author: tanu -""" -#%% -import os, sys -import pandas as pd -import numpy as np -import pprint as pp -from copy import deepcopy -from sklearn import linear_model -from sklearn import datasets -from collections import Counter - -from sklearn.linear_model import LogisticRegression, LogisticRegressionCV -from sklearn.linear_model import RidgeClassifier, RidgeClassifierCV, SGDClassifier, PassiveAggressiveClassifier - -from sklearn.naive_bayes import BernoulliNB -from sklearn.neighbors import KNeighborsClassifier -from sklearn.svm import SVC -from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier -from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier, BaggingClassifier -from sklearn.naive_bayes import GaussianNB -from sklearn.gaussian_process import GaussianProcessClassifier, kernels -from sklearn.gaussian_process.kernels import RBF, DotProduct, Matern, RationalQuadratic, WhiteKernel - -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis -from sklearn.neural_network import MLPClassifier - -from sklearn.svm import SVC -from xgboost import XGBClassifier -from sklearn.naive_bayes import MultinomialNB -from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder - -from sklearn.compose import ColumnTransformer -from sklearn.compose import make_column_transformer - -from sklearn.metrics import make_scorer, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score -from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score, classification_report - -# added -from sklearn.model_selection import train_test_split, cross_validate, cross_val_score, LeaveOneOut, KFold, RepeatedKFold, cross_val_predict - -from sklearn.model_selection import train_test_split, cross_validate, cross_val_score -from sklearn.model_selection import StratifiedKFold,RepeatedStratifiedKFold, RepeatedKFold - -from sklearn.pipeline import Pipeline, make_pipeline - -from sklearn.feature_selection import RFE, RFECV - -import itertools -import seaborn as sns -import matplotlib.pyplot as plt - -from statistics import mean, stdev, median, mode - -from imblearn.over_sampling import RandomOverSampler -from imblearn.under_sampling import RandomUnderSampler -from imblearn.over_sampling import SMOTE -from sklearn.datasets import make_classification -from imblearn.combine import SMOTEENN -from imblearn.combine import SMOTETomek - -from imblearn.over_sampling import SMOTENC -from imblearn.under_sampling import EditedNearestNeighbours -from imblearn.under_sampling import RepeatedEditedNearestNeighbours - -from sklearn.model_selection import GridSearchCV -from sklearn.base import BaseEstimator -from sklearn.impute import KNNImputer as KNN -import json -import argparse -import re -import itertools -from sklearn.model_selection import LeaveOneGroupOut -#%% GLOBALS -rs = {'random_state': 42} -njobs = {'n_jobs': os.cpu_count() } # the number of jobs should equal the number of CPU cores - -scoring_fn = ({ 'mcc' : make_scorer(matthews_corrcoef) - , 'fscore' : make_scorer(f1_score) - , 'precision' : make_scorer(precision_score) - , 'recall' : make_scorer(recall_score) - , 'accuracy' : make_scorer(accuracy_score) - , 'roc_auc' : make_scorer(roc_auc_score) - , 'jcc' : make_scorer(jaccard_score) - }) - -skf_cv = StratifiedKFold(n_splits = 10 - #, shuffle = False, random_state= None) - , shuffle = True,**rs) - -rskf_cv = RepeatedStratifiedKFold(n_splits = 10 - , n_repeats = 3 - , **rs) -logo = LeaveOneGroupOut() - -mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)} -jacc_score_fn = {'jcc': make_scorer(jaccard_score)} - -############################################################################### -score_type_ordermapD = { 'mcc' : 1 - , 'fscore' : 2 - , 'jcc' : 3 - , 'precision' : 4 - , 'recall' : 5 - , 'accuracy' : 6 - , 'roc_auc' : 7 - , 'TN' : 8 - , 'FP' : 9 - , 'FN' : 10 - , 'TP' : 11 - , 'trainingY_neg': 12 - , 'trainingY_pos': 13 - , 'blindY_neg' : 14 - , 'blindY_pos' : 15 - , 'fit_time' : 16 - , 'score_time' : 17 - } - -scoreCV_mapD = {'test_mcc' : 'MCC' - , 'test_fscore' : 'F1' - , 'test_precision' : 'Precision' - , 'test_recall' : 'Recall' - , 'test_accuracy' : 'Accuracy' - , 'test_roc_auc' : 'ROC_AUC' - , 'test_jcc' : 'JCC' - } - -scoreBT_mapD = {'bts_mcc' : 'MCC' - , 'bts_fscore' : 'F1' - , 'bts_precision' : 'Precision' - , 'bts_recall' : 'Recall' - , 'bts_accuracy' : 'Accuracy' - , 'bts_roc_auc' : 'ROC_AUC' - , 'bts_jcc' : 'JCC' - } - -#gene_group = 'gene_name' -#%%############################################################################ -############################ -# MultModelsCl() -# Run Multiple Classifiers -############################ -# Multiple Classification - Model Pipeline -def MultModelsCl_logo(input_df - , target - , sel_cv - - , blind_test_df = pd.DataFrame() - , blind_test_target = pd.Series(dtype = int) - , tts_split_type = "none" - , group = 'none' - - , resampling_type = 'none' # default - , add_cm = True # adds confusion matrix based on cross_val_predict - , add_yn = True # adds target var class numbers - , var_type = ['numerical', 'categorical','mixed'] - , run_blind_test = True - , return_formatted_output = True): - - ''' - @ param input_df: input features - @ type: df with input features WITHOUT the target variable - - @param target: target (or output) feature - @type: df or np.array or Series - - @param skv_cv: stratifiedK fold int or object to allow shuffle and random state to pass - @type: int or StratifiedKfold() - - @var_type: numerical, categorical and mixed to determine what col_transform to apply (MinMaxScalar and/or one-ho t encoder) - @type: list - - returns - Dict containing multiple classification scores for each model and mean of each Stratified Kfold including training - ''' - - # if group == 'none': - # sel_cv = skf_cv - # else: - # group = 'none' - #====================================================== - # Determine categorical and numerical features - #====================================================== - numerical_ix = input_df.select_dtypes(include=['int64', 'float64']).columns - numerical_ix - categorical_ix = input_df.select_dtypes(include=['object', 'bool']).columns - categorical_ix - - #====================================================== - # Determine preprocessing steps ~ var_type - #====================================================== - if var_type == 'numerical': - t = [('num', MinMaxScaler(), numerical_ix)] - - if var_type == 'categorical': - t = [('cat', OneHotEncoder(), categorical_ix)] - - if var_type == 'mixed': - t = [('num', MinMaxScaler(), numerical_ix) - , ('cat', OneHotEncoder(), categorical_ix) ] - - col_transform = ColumnTransformer(transformers = t - , remainder='passthrough') - - #====================================================== - # Specify multiple Classification Models - #====================================================== - models = [('AdaBoost Classifier' , AdaBoostClassifier(**rs) ) - , ('Bagging Classifier' , BaggingClassifier(**rs, **njobs, bootstrap = True, oob_score = True) ) - , ('Decision Tree' , DecisionTreeClassifier(**rs) ) - , ('Extra Tree' , ExtraTreeClassifier(**rs) ) - , ('Extra Trees' , ExtraTreesClassifier(**rs) ) - , ('Gradient Boosting' , GradientBoostingClassifier(**rs) ) - , ('Gaussian NB' , GaussianNB() ) - , ('Gaussian Process' , GaussianProcessClassifier(**rs) ) - , ('K-Nearest Neighbors' , KNeighborsClassifier() ) - , ('LDA' , LinearDiscriminantAnalysis() ) - , ('Logistic Regression' , LogisticRegression(**rs) ) - , ('Logistic RegressionCV' , LogisticRegressionCV(cv = 3, **rs)) - , ('MLP' , MLPClassifier(max_iter = 500, **rs) ) - , ('Multinomial' , MultinomialNB() ) - , ('Naive Bayes' , BernoulliNB() ) - , ('Passive Aggresive' , PassiveAggressiveClassifier(**rs, **njobs) ) - , ('QDA' , QuadraticDiscriminantAnalysis() ) - , ('Random Forest' , RandomForestClassifier(**rs, n_estimators = 1000 ) ) - , ('Random Forest2' , RandomForestClassifier(min_samples_leaf = 5 - , n_estimators = 1000 - , bootstrap = True - , oob_score = True - , **njobs - , **rs - , max_features = 'auto') ) - , ('Ridge Classifier' , RidgeClassifier(**rs) ) - , ('Ridge ClassifierCV' , RidgeClassifierCV(cv = 3) ) - , ('SVC' , SVC(**rs) ) - , ('Stochastic GDescent' , SGDClassifier(**rs, **njobs) ) - , ('XGBoost' , XGBClassifier(**rs, verbosity = 0, use_label_encoder =False) ) - ] - - mm_skf_scoresD = {} - - print('\n==============================================================\n' - , '\nRunning several classification models (n):', len(models) - ,'\nList of models:') - for m in models: - print(m) - print('\n================================================================\n') - - index = 1 - for model_name, model_fn in models: - print('\nRunning classifier:', index - , '\nModel_name:' , model_name - , '\nModel func:' , model_fn) - index = index+1 - - model_pipeline = Pipeline([ - ('prep' , col_transform) - , ('model' , model_fn)]) - - print('\nRunning model pipeline:', model_pipeline) - cv_modD = cross_validate(model_pipeline - , input_df - , target - , cv = sel_cv - , groups = group - , scoring = scoring_fn - , return_train_score = True) - #============================== - # Extract mean values for CV - #============================== - mm_skf_scoresD[model_name] = {} - - for key, value in cv_modD.items(): - print('\nkey:', key, '\nvalue:', value) - print('\nmean value:', np.mean(value)) - mm_skf_scoresD[model_name][key] = round(np.mean(value),2) - - # ADD more info: meta data related to input df - mm_skf_scoresD[model_name]['resampling'] = resampling_type - mm_skf_scoresD[model_name]['n_training_size'] = len(input_df) - mm_skf_scoresD[model_name]['n_trainingY_ratio'] = round(Counter(target)[0]/Counter(target)[1], 2) - mm_skf_scoresD[model_name]['n_features'] = len(input_df.columns) - mm_skf_scoresD[model_name]['tts_split'] = tts_split_type - - ####################################################################### - #====================================================== - # Option: Add confusion matrix from cross_val_predict - # Understand and USE with caution - #====================================================== - if add_cm: - cmD = {} - - # Calculate cm - y_pred = cross_val_predict(model_pipeline, input_df, target, cv = sel_cv, groups = group, **njobs) - #_tn, _fp, _fn, _tp = confusion_matrix(y_pred, y).ravel() # internally - tn, fp, fn, tp = confusion_matrix(y_pred, target).ravel() - - # Build cm dict - cmD = {'TN' : tn - , 'FP': fp - , 'FN': fn - , 'TP': tp} - - # Update cv dict cmD - mm_skf_scoresD[model_name].update(cmD) - - #============================================= - # Option: Add targety numbers for data - #============================================= - if add_yn: - tnD = {} - - # Build tn numbers dict - tnD = {'n_trainingY_neg' : Counter(target)[0] - , 'n_trainingY_pos' : Counter(target)[1] } - - # Update cv dict with cmD and tnD - mm_skf_scoresD[model_name].update(tnD) - -#%% - #========================= - # Option: Blind test (bts) - #========================= - if run_blind_test: - btD = {} - - # Build bts numbers dict - btD = {'n_blindY_neg' : Counter(blind_test_target)[0] - , 'n_blindY_pos' : Counter(blind_test_target)[1] - , 'n_testY_ratio' : round(Counter(blind_test_target)[0]/Counter(blind_test_target)[1], 2) - , 'n_test_size' : len(blind_test_df) } - - # Update cmD+tnD dicts with btD - mm_skf_scoresD[model_name].update(btD) - - #-------------------------------------------------------- - # Build the final results with all scores for the model - #-------------------------------------------------------- - #bts_predict = gscv_fs.predict(blind_test_df) - model_pipeline.fit(input_df, target) - bts_predict = model_pipeline.predict(blind_test_df) - - bts_mcc_score = round(matthews_corrcoef(blind_test_target, bts_predict),2) - print('\nMCC on Blind test:' , bts_mcc_score) - print('\nAccuracy on Blind test:', round(accuracy_score(blind_test_target, bts_predict),2)) - - mm_skf_scoresD[model_name]['bts_mcc'] = bts_mcc_score - mm_skf_scoresD[model_name]['bts_fscore'] = round(f1_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_precision'] = round(precision_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_recall'] = round(recall_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_accuracy'] = round(accuracy_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_roc_auc'] = round(roc_auc_score(blind_test_target, bts_predict),2) - mm_skf_scoresD[model_name]['bts_jcc'] = round(jaccard_score(blind_test_target, bts_predict),2) - #mm_skf_scoresD[model_name]['diff_mcc'] = train_test_diff_MCC - - #return(mm_skf_scoresD) - #============================ - # Process the dict to have WF - #============================ - if return_formatted_output: - CV_BT_metaDF = ProcessMultModelsCl(mm_skf_scoresD) - return(CV_BT_metaDF) - else: - return(mm_skf_scoresD) - -#%% Process output function ################################################### -############################ -# ProcessMultModelsCl() -############################ -#Processes the dict from above if use_formatted_output = True - -def ProcessMultModelsCl(inputD = {} - , blind_test_data = True): - - scoresDF = pd.DataFrame(inputD) - - #------------------------ - # Extracting split_name - #----------------------- - tts_split_nameL = [] - for k,v in inputD.items(): - tts_split_nameL = tts_split_nameL + [v['tts_split']] - - if len(set(tts_split_nameL)) == 1: - tts_split_name = str(list(set(tts_split_nameL))[0]) - print('\nExtracting tts_split_name:', tts_split_name) - - #---------------------- - # WF: CV results - #---------------------- - scoresDFT = scoresDF.T - - scoresDF_CV = scoresDFT.filter(regex='^test_.*$', axis = 1); scoresDF_CV.columns - # map colnames for consistency to allow concatenting - scoresDF_CV.columns = scoresDF_CV.columns.map(scoreCV_mapD); scoresDF_CV.columns - scoresDF_CV['source_data'] = 'CV' - - #---------------------- - # WF: Meta data - #---------------------- - metaDF = scoresDFT.filter(regex='^(?!test_.*$|bts_.*$|train_.*$).*'); metaDF.columns - - print('\nTotal cols in each df:' - , '\nCV df:', len(scoresDF_CV.columns) - , '\nmetaDF:', len(metaDF.columns)) - - #------------------------------------- - # Combine WF: CV + Metadata - #------------------------------------- - - combDF = pd.merge(scoresDF_CV, metaDF, left_index = True, right_index = True) - print('\nAdding column: Model_name') - combDF['Model_name'] = combDF.index - - #---------------------- - # WF: BTS results - #---------------------- - if blind_test_data: - - scoresDF_BT = scoresDFT.filter(regex='^bts_.*$', axis = 1); scoresDF_BT.columns - # map colnames for consistency to allow concatenting - scoresDF_BT.columns = scoresDF_BT.columns.map(scoreBT_mapD); scoresDF_BT.columns - scoresDF_BT['source_data'] = 'BT' - - - print('\nTotal cols in bts df:' - , '\nBT_df:', len(scoresDF_BT.columns)) - - if len(scoresDF_CV.columns) == len(scoresDF_BT.columns): - print('\nFirst proceeding to rowbind CV and BT dfs:') - expected_ncols_out = len(scoresDF_BT.columns) + len(metaDF.columns) - print('\nFinal output should have:', expected_ncols_out, 'columns' ) - - #----------------- - # Combine WF - #----------------- - dfs_combine_wf = [scoresDF_CV, scoresDF_BT] - - print('\nCombinig', len(dfs_combine_wf), 'using pd.concat by row ~ rowbind' - , '\nChecking Dims of df to combine:' - , '\nDim of CV:', scoresDF_CV.shape - , '\nDim of BT:', scoresDF_BT.shape) - #print(scoresDF_CV) - #print(scoresDF_BT) - - dfs_nrows_wf = [] - for df in dfs_combine_wf: - dfs_nrows_wf = dfs_nrows_wf + [len(df)] - dfs_nrows_wf = max(dfs_nrows_wf) - - dfs_ncols_wf = [] - for df in dfs_combine_wf: - dfs_ncols_wf = dfs_ncols_wf + [len(df.columns)] - dfs_ncols_wf = max(dfs_ncols_wf) - print(dfs_ncols_wf) - - expected_nrows_wf = len(dfs_combine_wf) * dfs_nrows_wf - expected_ncols_wf = dfs_ncols_wf - - common_cols_wf = list(set.intersection(*(set(df.columns) for df in dfs_combine_wf))) - print('\nNumber of Common columns:', dfs_ncols_wf - , '\nThese are:', common_cols_wf) - - if len(common_cols_wf) == dfs_ncols_wf : - combined_baseline_wf = pd.concat([df[common_cols_wf] for df in dfs_combine_wf], ignore_index=False) - print('\nConcatenating dfs with different resampling methods [WF]:' - , '\nSplit type:', tts_split_name - , '\nNo. of dfs combining:', len(dfs_combine_wf)) - #print('\n================================================^^^^^^^^^^^^') - if len(combined_baseline_wf) == expected_nrows_wf and len(combined_baseline_wf.columns) == expected_ncols_wf: - #print('\n================================================^^^^^^^^^^^^') - - print('\nPASS:', len(dfs_combine_wf), 'dfs successfully combined' - , '\nnrows in combined_df_wf:', len(combined_baseline_wf) - , '\nncols in combined_df_wf:', len(combined_baseline_wf.columns)) - else: - print('\nFAIL: concatenating failed' - , '\nExpected nrows:', expected_nrows_wf - , '\nGot:', len(combined_baseline_wf) - , '\nExpected ncols:', expected_ncols_wf - , '\nGot:', len(combined_baseline_wf.columns)) - sys.exit('\nFIRST IF FAILS') - ## - c1L = list(set(combined_baseline_wf.index)) - c2L = list(metaDF.index) - - #if set(c1L) == set(c2L): - if set(c1L) == set(c2L) and all(x in c2L for x in c1L) and all(x in c1L for x in c2L): - print('\nPASS: proceeding to merge metadata with CV and BT dfs') - combDF = pd.merge(combined_baseline_wf, metaDF, left_index = True, right_index = True) - print('\nAdding column: Model_name') - combDF['Model_name'] = combDF.index - - else: - sys.exit('\nFAIL: Could not merge metadata with CV and BT dfs') - - else: - # print('\nConcatenting dfs not possible [WF],check numbers ') - print('\nOnly combining CV and metadata') - - #------------------------------------- - # Combine WF+Metadata: Final output - #------------------------------------- - - # if len(combDF.columns) == expected_ncols_out: - # print('\nPASS: Combined df has expected ncols') - # else: - # sys.exit('\nFAIL: Length mismatch for combined_df') - - # print('\nAdding column: Model_name') - # combDF['Model_name'] = combDF.index - - print('\n=========================================================' - , '\nSUCCESS: Ran multiple classifiers' - , '\n=======================================================') - - #resampling_methods_wf = combined_baseline_wf[['resampling']] - #resampling_methods_wf = resampling_methods_wf.drop_duplicates() - #, '\n', resampling_methods_wf) - - return combDF - -############################################################################### diff --git a/scripts/ml/ml_functions/MultClfs_logo_skf.py b/scripts/ml/ml_functions/MultClfs_logo_skf.py index 73e8b3a..c06ae0f 100755 --- a/scripts/ml/ml_functions/MultClfs_logo_skf.py +++ b/scripts/ml/ml_functions/MultClfs_logo_skf.py @@ -77,9 +77,11 @@ import re import itertools from sklearn.model_selection import LeaveOneGroupOut from sklearn.decomposition import PCA +from sklearn.naive_bayes import ComplementNB +from sklearn.dummy import DummyClassifier #%% GLOBALS -#rs = {'random_state': 42} +#rs = {'random_state': 42} # INSIDE FUNCTION CALL NOW #njobs = {'n_jobs': os.cpu_count() } # the number of jobs should equal the number of CPU cores scoring_fn = ({ 'mcc' : make_scorer(matthews_corrcoef) @@ -90,8 +92,7 @@ scoring_fn = ({ 'mcc' : make_scorer(matthews_corrcoef) , 'roc_auc' : make_scorer(roc_auc_score) , 'jcc' : make_scorer(jaccard_score) }) - -# for sel_cv +# for sel_cv INSIDE FUNCTION CALL NOW #skf_cv = StratifiedKFold(n_splits = 10 # #, shuffle = False, random_state= None) # , shuffle = True, **rs) @@ -149,25 +150,25 @@ scoreBT_mapD = {'bts_mcc' : 'MCC' ############################ # Multiple Classification - Model Pipeline def MultModelsCl_logo_skf(input_df - , target - , sel_cv - , tts_split_type - , resampling_type - #, group = None - - , add_cm = True # adds confusion matrix based on cross_val_predict - , add_yn = True # adds target var class numbers - , var_type = ['numerical', 'categorical','mixed'] - , scale_numeric = ['min_max', 'std', 'min_max_neg', 'none'] + , target + , sel_cv + , tts_split_type + , resampling_type + #, group = None + + , add_cm = True # adds confusion matrix based on cross_val_predict + , add_yn = True # adds target var class numbers + , var_type = ['numerical', 'categorical','mixed'] + , scale_numeric = ['min_max', 'std', 'min_max_neg', 'none'] - , run_blind_test = True - , blind_test_df = pd.DataFrame() - , blind_test_target = pd.Series(dtype = int) - , return_formatted_output = True + , run_blind_test = True + , blind_test_df = pd.DataFrame() + , blind_test_target = pd.Series(dtype = int) + , return_formatted_output = True - , random_state = 42 - , n_jobs = os.cpu_count() # the number of jobs should equal the number of CPU cores - ): + , random_state = 42 + , n_jobs = os.cpu_count() # the number of jobs should equal the number of CPU cores + ): ''' @ param input_df: input features @@ -189,7 +190,15 @@ def MultModelsCl_logo_skf(input_df #%% Func globals rs = {'random_state': random_state} njobs = {'n_jobs': n_jobs} + + skf_cv = StratifiedKFold(n_splits = 10 + #, shuffle = False, random_state= None) + , shuffle = True,**rs) + rskf_cv = RepeatedStratifiedKFold(n_splits = 10 + , n_repeats = 3 + , **rs) + logo = LeaveOneGroupOut() # select CV type: # if group == None: @@ -252,8 +261,10 @@ def MultModelsCl_logo_skf(input_df #====================================================== # Specify multiple Classification Models #====================================================== - models = [('AdaBoost Classifier' , AdaBoostClassifier(**rs) ) - , ('Bagging Classifier' , BaggingClassifier(**rs, **njobs, bootstrap = True, oob_score = True, verbose = 3, n_estimators = 100) ) + models = [('AdaBoost Classifier' , AdaBoostClassifier(**rs) ) + , ('Bagging Classifier' , BaggingClassifier(**rs, **njobs, bootstrap = True, oob_score = True, verbose = 3, n_estimators = 100) ) + #, ('Bernoulli NB' , BernoulliNB() ) # pks Naive Bayes, CAUTION + , ('Complement NB' , ComplementNB() ) , ('Decision Tree' , DecisionTreeClassifier(**rs) ) , ('Extra Tree' , ExtraTreeClassifier(**rs) ) , ('Extra Trees' , ExtraTreesClassifier(**rs) ) @@ -265,23 +276,23 @@ def MultModelsCl_logo_skf(input_df , ('Logistic Regression' , LogisticRegression(**rs) ) , ('Logistic RegressionCV' , LogisticRegressionCV(cv = 3, **rs)) , ('MLP' , MLPClassifier(max_iter = 500, **rs) ) - , ('Multinomial' , MultinomialNB() ) - , ('Naive Bayes' , BernoulliNB() ) + , ('Multinomial NB' , MultinomialNB() ) , ('Passive Aggresive' , PassiveAggressiveClassifier(**rs, **njobs) ) , ('QDA' , QuadraticDiscriminantAnalysis() ) , ('Random Forest' , RandomForestClassifier(**rs, n_estimators = 1000, **njobs ) ) , ('Random Forest2' , RandomForestClassifier(min_samples_leaf = 5 - , n_estimators = 1000 - , bootstrap = True - , oob_score = True - , **njobs - , **rs - , max_features = 'auto') ) - , ('Ridge Classifier' , RidgeClassifier(**rs) ) - , ('Ridge ClassifierCV' , RidgeClassifierCV(cv = 3) ) - , ('SVC' , SVC(**rs) ) - , ('Stochastic GDescent' , SGDClassifier(**rs, **njobs) ) - , ('XGBoost' , XGBClassifier(**rs, verbosity = 0, use_label_encoder = False, **njobs) ) + , n_estimators = 1000 + , bootstrap = True + , oob_score = True + , **njobs + , **rs + , max_features = 'auto') ) + , ('Ridge Classifier' , RidgeClassifier(**rs) ) + , ('Ridge ClassifierCV' , RidgeClassifierCV(cv = 3) ) + , ('SVC' , SVC(**rs) ) + , ('Stochastic GDescent' , SGDClassifier(**rs, **njobs) ) + , ('XGBoost' , XGBClassifier(**rs, verbosity = 0, use_label_encoder = False, **njobs) ) + , ('Dummy Classifier' , DummyClassifier(strategy = 'most_frequent') ) ] mm_skf_scoresD = {}