From b87f8d029523f0ac654fc44d7cfce231121684e4 Mon Sep 17 00:00:00 2001 From: Tanushree Tunstall Date: Thu, 28 Jul 2022 15:19:13 +0100 Subject: [PATCH] trying diff cv thresholds for single gene --- scripts/ml/ml_functions/MultClfs_noBTS.py | 453 ------------------ .../ml/ml_functions/test_func_singlegene.py | 62 ++- 2 files changed, 54 insertions(+), 461 deletions(-) delete mode 100755 scripts/ml/ml_functions/MultClfs_noBTS.py diff --git a/scripts/ml/ml_functions/MultClfs_noBTS.py b/scripts/ml/ml_functions/MultClfs_noBTS.py deleted file mode 100755 index 8e8848a..0000000 --- a/scripts/ml/ml_functions/MultClfs_noBTS.py +++ /dev/null @@ -1,453 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Fri Mar 4 15:25:33 2022 - -@author: tanu -""" -#%% -import os, sys -import pandas as pd -import numpy as np -import pprint as pp -from copy import deepcopy -from sklearn import linear_model -from sklearn import datasets -from collections import Counter - -from sklearn.linear_model import LogisticRegression, LogisticRegressionCV -from sklearn.linear_model import RidgeClassifier, RidgeClassifierCV, SGDClassifier, PassiveAggressiveClassifier - -from sklearn.naive_bayes import BernoulliNB -from sklearn.neighbors import KNeighborsClassifier -from sklearn.svm import SVC -from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier -from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier, BaggingClassifier -from sklearn.naive_bayes import GaussianNB -from sklearn.gaussian_process import GaussianProcessClassifier, kernels -from sklearn.gaussian_process.kernels import RBF, DotProduct, Matern, RationalQuadratic, WhiteKernel - -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis -from sklearn.neural_network import MLPClassifier - -from sklearn.svm import SVC -from xgboost import XGBClassifier -from sklearn.naive_bayes import MultinomialNB -from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder - -from sklearn.compose import ColumnTransformer -from sklearn.compose import make_column_transformer - -from sklearn.metrics import make_scorer, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score -from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score, classification_report - -# added -from sklearn.model_selection import train_test_split, cross_validate, cross_val_score, LeaveOneOut, KFold, RepeatedKFold, cross_val_predict - -from sklearn.model_selection import train_test_split, cross_validate, cross_val_score -from sklearn.model_selection import StratifiedKFold,RepeatedStratifiedKFold, RepeatedKFold - -from sklearn.pipeline import Pipeline, make_pipeline - -from sklearn.feature_selection import RFE, RFECV - -import itertools -import seaborn as sns -import matplotlib.pyplot as plt - -from statistics import mean, stdev, median, mode - -from imblearn.over_sampling import RandomOverSampler -from imblearn.under_sampling import RandomUnderSampler -from imblearn.over_sampling import SMOTE -from sklearn.datasets import make_classification -from imblearn.combine import SMOTEENN -from imblearn.combine import SMOTETomek - -from imblearn.over_sampling import SMOTENC -from imblearn.under_sampling import EditedNearestNeighbours -from imblearn.under_sampling import RepeatedEditedNearestNeighbours - -from sklearn.model_selection import GridSearchCV -from sklearn.base import BaseEstimator -from sklearn.impute import KNNImputer as KNN -import json -import argparse -import re -import itertools -from sklearn.model_selection import LeaveOneGroupOut -from sklearn.decomposition import PCA -from sklearn.naive_bayes import ComplementNB -from sklearn.dummy import DummyClassifier - -#%% GLOBALS -#rs = {'random_state': 42} # INSIDE FUNCTION CALL NOW -#njobs = {'n_jobs': os.cpu_count() } # the number of jobs should equal the number of CPU cores - -scoring_fn = ({ 'mcc' : make_scorer(matthews_corrcoef) - , 'fscore' : make_scorer(f1_score) - , 'precision' : make_scorer(precision_score) - , 'recall' : make_scorer(recall_score) - , 'accuracy' : make_scorer(accuracy_score) - , 'roc_auc' : make_scorer(roc_auc_score) - , 'jcc' : make_scorer(jaccard_score) - }) -# for sel_cv INSIDE FUNCTION CALL NOW -#skf_cv = StratifiedKFold(n_splits = 10 -# #, shuffle = False, random_state= None) -# , shuffle = True, **rs) - -#rskf_cv = RepeatedStratifiedKFold(n_splits = 10 -# , n_repeats = 3 -# , **rs) - -mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)} -jacc_score_fn = {'jcc': make_scorer(jaccard_score)} - -############################################################################### -score_type_ordermapD = { 'mcc' : 1 - , 'fscore' : 2 - , 'jcc' : 3 - , 'precision' : 4 - , 'recall' : 5 - , 'accuracy' : 6 - , 'roc_auc' : 7 - , 'TN' : 8 - , 'FP' : 9 - , 'FN' : 10 - , 'TP' : 11 - , 'trainingY_neg': 12 - , 'trainingY_pos': 13 - , 'blindY_neg' : 14 - , 'blindY_pos' : 15 - , 'fit_time' : 16 - , 'score_time' : 17 - } - -scoreCV_mapD = {'test_mcc' : 'MCC' - , 'test_fscore' : 'F1' - , 'test_precision' : 'Precision' - , 'test_recall' : 'Recall' - , 'test_accuracy' : 'Accuracy' - , 'test_roc_auc' : 'ROC_AUC' - , 'test_jcc' : 'JCC' - } - -scoreBT_mapD = {'bts_mcc' : 'MCC' - , 'bts_fscore' : 'F1' - , 'bts_precision' : 'Precision' - , 'bts_recall' : 'Recall' - , 'bts_accuracy' : 'Accuracy' - , 'bts_roc_auc' : 'ROC_AUC' - , 'bts_jcc' : 'JCC' - } - -#gene_group = 'gene_name' -#%%############################################################################ -############################ -# MultModelsCl() -# Run Multiple Classifiers -############################ -# Multiple Classification - Model Pipeline -def MultModelsCl_noBTS(input_df - , target - , tts_split_type - , resampling_type - #, group = None - , skf_cv_threshold = 10 #[None, 3, 5, 10] - - , add_cm = True # adds confusion matrix based on cross_val_predict - , add_yn = True # adds target var class numbers - , var_type = ['numerical', 'categorical','mixed'] - , scale_numeric = ['min_max', 'std', 'min_max_neg', 'none'] - - , return_formatted_output = True - - , random_state = 42 - , n_jobs = os.cpu_count() # the number of jobs should equal the number of CPU cores - ): - - ''' - @ param input_df: input features - @ type: df with input features WITHOUT the target variable - - @param target: target (or output) feature - @type: df or np.array or Series - - @param skv_cv: stratifiedK fold int or object to allow shuffle and random state to pass - @type: int or StratifiedKfold() - - @var_type: numerical, categorical and mixed to determine what col_transform to apply (MinMaxScalar and/or one-hot encoder) - @type: list - - returns - Dict containing multiple classification scores for each model and mean of each Stratified Kfold including training - ''' - -#%% Func globals - rs = {'random_state': random_state} - njobs = {'n_jobs': n_jobs} - - skf_cv = StratifiedKFold(n_splits = skf_cv_threshold - #, shuffle = False, random_state= None) - , shuffle = True,**rs) - - # rskf_cv = RepeatedStratifiedKFold(n_splits = skf_cv_threshold - # , n_repeats = 3 - # , **rs) - # logo = LeaveOneGroupOut() - - # select CV type: - # if group == None: - # sel_cv = skf_cv - # else: - # sel_cv = logo - #====================================================== - # Determine categorical and numerical features - #====================================================== - numerical_ix = input_df.select_dtypes(include=['int64', 'float64']).columns - numerical_ix - categorical_ix = input_df.select_dtypes(include=['object', 'bool']).columns - categorical_ix - - #====================================================== - # Determine preprocessing steps ~ var_type - #====================================================== - - if type(var_type) == list: - var_type = str(var_type[0]) - else: - var_type = var_type - - if var_type in ['numerical','mixed']: - if scale_numeric == ['none']: - t = [('cat', OneHotEncoder(), categorical_ix)] - if scale_numeric != ['none']: - if scale_numeric == ['min_max']: - scaler = MinMaxScaler() - if scale_numeric == ['min_max_neg']: - scaler = MinMaxScaler(feature_range=(-1, 1)) - if scale_numeric == ['std']: - scaler = StandardScaler() - - t = [('num', scaler, numerical_ix) - , ('cat', OneHotEncoder(), categorical_ix)] - - - if var_type == 'categorical': - t = [('cat', OneHotEncoder(), categorical_ix)] - - - col_transform = ColumnTransformer(transformers = t - , remainder='passthrough') - - - #====================================================== - # Specify multiple Classification Models - #====================================================== - models = [('AdaBoost Classifier' , AdaBoostClassifier(**rs) ) - # , ('Bagging Classifier' , BaggingClassifier(**rs, **njobs, bootstrap = True, oob_score = True, verbose = 3, n_estimators = 100) ) - # #, ('Bernoulli NB' , BernoulliNB() ) # pks Naive Bayes, CAUTION - # , ('Complement NB' , ComplementNB() ) - # , ('Decision Tree' , DecisionTreeClassifier(**rs) ) - # , ('Extra Tree' , ExtraTreeClassifier(**rs) ) - # , ('Extra Trees' , ExtraTreesClassifier(**rs) ) - # , ('Gradient Boosting' , GradientBoostingClassifier(**rs) ) - # , ('Gaussian NB' , GaussianNB() ) - # , ('Gaussian Process' , GaussianProcessClassifier(**rs) ) - # , ('K-Nearest Neighbors' , KNeighborsClassifier() ) - # , ('LDA' , LinearDiscriminantAnalysis() ) - # , ('Logistic Regression' , LogisticRegression(**rs) ) - # , ('Logistic RegressionCV' , LogisticRegressionCV(cv = 3, **rs)) - # , ('MLP' , MLPClassifier(max_iter = 500, **rs) ) - # , ('Multinomial NB' , MultinomialNB() ) - # , ('Passive Aggresive' , PassiveAggressiveClassifier(**rs, **njobs) ) - # , ('QDA' , QuadraticDiscriminantAnalysis() ) - # , ('Random Forest' , RandomForestClassifier(**rs, n_estimators = 1000, **njobs ) ) - # , ('Random Forest2' , RandomForestClassifier(min_samples_leaf = 5 - # , n_estimators = 1000 - # , bootstrap = True - # , oob_score = True - # , **njobs - # , **rs - # , max_features = 'auto') ) - # , ('Ridge Classifier' , RidgeClassifier(**rs) ) - # , ('Ridge ClassifierCV' , RidgeClassifierCV(cv = 3) ) - # , ('SVC' , SVC(**rs) ) - # , ('Stochastic GDescent' , SGDClassifier(**rs, **njobs) ) - , ('XGBoost' , XGBClassifier(**rs, verbosity = 0, use_label_encoder = False, **njobs) ) - , ('Dummy Classifier' , DummyClassifier(strategy = 'most_frequent') ) - ] - - mm_skf_scoresD = {} - - print('\n==============================================================\n' - , '\nRunning several classification models (n):', len(models) - ,'\nList of models:') - for m in models: - print(m) - print('\n================================================================\n') - - index = 1 - for model_name, model_fn in models: - print('\nRunning classifier:', index - , '\nModel_name:' , model_name - , '\nModel func:' , model_fn) - index = index+1 - - model_pipeline = Pipeline([ - ('prep' , col_transform) - , ('model' , model_fn)]) - - print('\nRunning model pipeline:', model_pipeline) - cv_modD = cross_validate(model_pipeline - , input_df - , target - , cv = skf_cv - #, groups = group - , scoring = scoring_fn - , return_train_score = True) - #============================== - # Extract mean values for CV - #============================== - mm_skf_scoresD[model_name] = {} - - for key, value in cv_modD.items(): - print('\nkey:', key, '\nvalue:', value) - print('\nmean value:', np.mean(value)) - mm_skf_scoresD[model_name][key] = round(np.mean(value),2) - - # ADD more info: meta data related to input df - mm_skf_scoresD[model_name]['resampling'] = resampling_type - mm_skf_scoresD[model_name]['n_training_size'] = len(input_df) - mm_skf_scoresD[model_name]['n_trainingY_ratio'] = round(Counter(target)[0]/Counter(target)[1], 2) - mm_skf_scoresD[model_name]['n_features'] = len(input_df.columns) - mm_skf_scoresD[model_name]['tts_split'] = tts_split_type - - ####################################################################### - #====================================================== - # Option: Add confusion matrix from cross_val_predict - # Understand and USE with caution - #====================================================== - if add_cm: - cmD = {} - - # Calculate cm - y_pred = cross_val_predict(model_pipeline - , input_df - , target - , cv = skf_cv - #, groups = group - , **njobs) - #_tn, _fp, _fn, _tp = confusion_matrix(y_pred, y).ravel() # internally - tn, fp, fn, tp = confusion_matrix(y_pred, target).ravel() - - # Build cm dict - cmD = {'TN' : tn - , 'FP': fp - , 'FN': fn - , 'TP': tp} - - # Update cv dict cmD - mm_skf_scoresD[model_name].update(cmD) - - #============================================= - # Option: Add targety numbers for data - #============================================= - if add_yn: - tnD = {} - - # Build tn numbers dict - tnD = {'n_trainingY_neg' : Counter(target)[0] - , 'n_trainingY_pos' : Counter(target)[1] } - - # Update cv dict with cmD and tnD - mm_skf_scoresD[model_name].update(tnD) - -#%% - #return(mm_skf_scoresD) - #============================ - # Process the dict to have WF - #============================ - if return_formatted_output: - CV_BT_metaDF = ProcessMultModelsCl(mm_skf_scoresD, cv_threshold_suffix = skf_cv_threshold) - return(CV_BT_metaDF) - else: - return(mm_skf_scoresD) - -#%% Process output function ################################################### -############################ -# ProcessMultModelsCl() -############################ -#Processes the dict from above if use_formatted_output = True - -def ProcessMultModelsCl(inputD = {} - , cv_threshold_suffix = 10 - #, blind_test_data = True - ): - - scoresDF = pd.DataFrame(inputD) - - #------------------------ - # Extracting split_name - #----------------------- - tts_split_nameL = [] - for k,v in inputD.items(): - tts_split_nameL = tts_split_nameL + [v['tts_split']] - - if len(set(tts_split_nameL)) == 1: - tts_split_name = str(list(set(tts_split_nameL))[0]) - print('\nExtracting tts_split_name:', tts_split_name) - - #---------------------- - # WF: CV results - #---------------------- - scoresDFT = scoresDF.T - - scoresDF_CV = scoresDFT.filter(regex='^test_.*$', axis = 1); scoresDF_CV.columns - # map colnames for consistency to allow concatenting - scoresDF_CV.columns = scoresDF_CV.columns.map(scoreCV_mapD); scoresDF_CV.columns - #scoresDF_CV['source_data'] = 'CV' - scoresDF_CV['source_data'] = 'CV_' + str(cv_threshold_suffix) - - - #---------------------- - # WF: Meta data - #---------------------- - metaDF = scoresDFT.filter(regex='^(?!test_.*$|bts_.*$|train_.*$).*'); metaDF.columns - - print('\nTotal cols in each df:' - , '\nCV df:', len(scoresDF_CV.columns) - , '\nmetaDF:', len(metaDF.columns)) - - #------------------------------------- - # Combine WF: CV + Metadata - #------------------------------------- - - combDF = pd.merge(scoresDF_CV, metaDF, left_index = True, right_index = True) - print('\nAdding column: Model_name') - combDF['Model_name'] = combDF.index - - #------------------------------------- - # Combine WF+Metadata: Final output - #------------------------------------- - - # if len(combDF.columns) == expected_ncols_out: - # print('\nPASS: Combined df has expected ncols') - # else: - # sys.exit('\nFAIL: Length mismatch for combined_df') - - # print('\nAdding column: Model_name') - # combDF['Model_name'] = combDF.index - - print('\n=========================================================' - , '\nSUCCESS: Ran multiple classifiers' - , '\n=======================================================') - - #resampling_methods_wf = combined_baseline_wf[['resampling']] - #resampling_methods_wf = resampling_methods_wf.drop_duplicates() - #, '\n', resampling_methods_wf) - - return combDF - -############################################################################### diff --git a/scripts/ml/ml_functions/test_func_singlegene.py b/scripts/ml/ml_functions/test_func_singlegene.py index ba2909b..e19128f 100644 --- a/scripts/ml/ml_functions/test_func_singlegene.py +++ b/scripts/ml/ml_functions/test_func_singlegene.py @@ -15,8 +15,7 @@ sys.path from GetMLData import * from SplitTTS import * from MultClfs import * -from MultClfs_noBTS import * - +from MultClfs_CVs import * #%% rs = {'random_state': 42} @@ -27,6 +26,7 @@ skf_cv = StratifiedKFold(n_splits = 10 # , n_repeats = 3 # , **rs) # param dict for getmldata() +#%% READ data gene_model_paramD = {'data_combined_model' : False , 'use_or' : False , 'omit_all_genomic_features': False @@ -40,7 +40,7 @@ df = getmldata('embB', 'ethambutol' , **gene_model_paramD) #df = getmldata('rpoB', 'rifampicin' , **gene_model_paramD) #df = getmldata('gid' , 'streptomycin' , **gene_model_paramD) #df = getmldata('alr' , 'cycloserine' , **gene_model_paramD) - +#%% SPLIT, Data and Resampling types all(df.columns.isin(['gene_name'])) # should be False spl_type = '70_30' #spl_type = '80_20' @@ -143,11 +143,13 @@ from sklearn.utils import all_estimators all_clfs = all_estimators(type_filter="classifier") df = pd.DataFrame (all_clfs, columns = ['classifier_name', 'classifier_fn']) df.to_csv("Model_names_ALL.csv") +################################################################ #%% TEST different CV Thresholds for split_type = NONE - +################################################################ Counter(df2['y']) Counter(df2['y_bts']) +# READ Data spl_type = 'none' data_type = "complete" @@ -160,13 +162,13 @@ df2 = split_tts(df , include_gene_name = True , random_state = 42 # default ) - -fooD = MultModelsCl_noBTS(input_df = df2['X'] +#%% Trying different CV thresholds for resampling 'none' ONLY +fooD = MultModelsCl_CVs(input_df = df2['X'] , target = df2['y'] , skf_cv_threshold = 10 # IMP to change , tts_split_type = spl_type - , resampling_type = 'XXXX' # default + , resampling_type = 'NONE' # default , add_cm = True # adds confusion matrix based on cross_val_predict , add_yn = True # adds target var class numbers @@ -185,7 +187,7 @@ for k, v in fooD.items(): ) # formatted df -foo_df3 = MultModelsCl_noBTS(input_df = df2['X'] +foo_df3 = MultModelsCl_CVs(input_df = df2['X'] , target = df2['y'] , skf_cv_threshold = 5 # IMP to change @@ -203,6 +205,7 @@ foo_df3 = MultModelsCl_noBTS(input_df = df2['X'] ) + dfs_combine_wf = [foo_df, foo_df2, foo_df3] common_cols_wf = list(set.intersection(*(set(df.columns) for df in dfs_combine_wf))) @@ -246,3 +249,46 @@ if len(common_cols_wf) == dfs_ncols_wf : , '\nGot:', len(combined_baseline_wf.columns)) sys.exit('\nFIRST IF FAILS') +#%% TRY with dict containing different Resampling types +paramD = { + 'baseline_paramD': { 'input_df' : df2['X'] + , 'target' : df2['y'] + , 'var_type' : 'mixed' + , 'resampling_type': 'none'} + + , 'smnc_paramD' : { 'input_df' : df2['X_smnc'] + , 'target' : df2['y_smnc'] + , 'var_type' : 'mixed' + , 'resampling_type' : 'smnc'} + } + +mmDD = {} +for k, v in paramD.items(): + print(k) + all_scoresDF = pd.DataFrame() + for skf_cv_threshold in [3,5]: + print('\nRunning CV threhhold:', skf_cv_threshold) + current_scoreDF = MultModelsCl_CVs(**paramD[k] + , skf_cv_threshold = skf_cv_threshold # IMP to change + , tts_split_type = spl_type + #, resampling_type = 'XXXX' # default + + , add_cm = True # adds confusion matrix based on cross_val_predict + , add_yn = True # adds target var class numbers + + #, var_type = ['mixed'] + , scale_numeric = ['min_max'] + , random_state = 42 + , n_jobs = os.cpu_count() + , return_formatted_output = True + ) + + all_scoresDF = pd.concat([all_scoresDF, current_scoreDF]) + mmDD[k] = all_scoresDF + +for k, v in mmDD.items(): + print(k, v) + out_wf= pd.concat(mmDD, ignore_index = True) + out_wf2= pd.concat(mmDD) + +