tanu/LSHTM_analysis
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

added MultClfs_fi to add FI scores for models, in development

Browse source
This commit is contained in:
Tanushree Tunstall 2022-07-05 14:19:35 +01:00
parent 53c229f480
commit 652cf4802e
1 changed files with 323 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

323
scripts/ml/ml_functions/MultClfs_fi.py Normal file
View file

@ -0,0 +1,323 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Mar 4 15:25:33 2022
@author: tanu
"""
#%%
import os, sys
import pandas as pd
import numpy as np
import pprint as pp
from copy import deepcopy
from sklearn import linear_model
from sklearn import datasets
from collections import Counter
from sklearn.linear_model import LogisticRegression, LogisticRegressionCV
from sklearn.linear_model import RidgeClassifier, RidgeClassifierCV, SGDClassifier, PassiveAggressiveClassifier
from sklearn.naive_bayes import BernoulliNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier, BaggingClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.gaussian_process import GaussianProcessClassifier, kernels
from sklearn.gaussian_process.kernels import RBF, DotProduct, Matern, RationalQuadratic, WhiteKernel
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
from sklearn.neural_network import MLPClassifier
from sklearn.svm import SVC
from xgboost import XGBClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.compose import make_column_transformer
from sklearn.metrics import make_scorer, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score
from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score, classification_report
# added
from sklearn.model_selection import train_test_split, cross_validate, cross_val_score, LeaveOneOut, KFold, RepeatedKFold, cross_val_predict
from sklearn.model_selection import train_test_split, cross_validate, cross_val_score
from sklearn.model_selection import StratifiedKFold,RepeatedStratifiedKFold, RepeatedKFold
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.feature_selection import RFE, RFECV
import itertools
import seaborn as sns
import matplotlib.pyplot as plt
from statistics import mean, stdev, median, mode
from imblearn.over_sampling import RandomOverSampler
from imblearn.under_sampling import RandomUnderSampler
from imblearn.over_sampling import SMOTE
from sklearn.datasets import make_classification
from imblearn.combine import SMOTEENN
from imblearn.combine import SMOTETomek
from imblearn.over_sampling import SMOTENC
from imblearn.under_sampling import EditedNearestNeighbours
from imblearn.under_sampling import RepeatedEditedNearestNeighbours
from sklearn.model_selection import GridSearchCV
from sklearn.base import BaseEstimator
from sklearn.impute import KNNImputer as KNN
import json
import argparse
import re
#%% GLOBALS
rs = {'random_state': 42}
njobs = {'n_jobs': os.cpu_count() } # the number of jobs should equal the number of CPU cores
scoring_fn = ({ 'mcc' : make_scorer(matthews_corrcoef)
, 'fscore' : make_scorer(f1_score)
, 'precision' : make_scorer(precision_score)
, 'recall' : make_scorer(recall_score)
, 'accuracy' : make_scorer(accuracy_score)
, 'roc_auc' : make_scorer(roc_auc_score)
, 'jcc' : make_scorer(jaccard_score)
})
skf_cv = StratifiedKFold(n_splits = 10
#, shuffle = False, random_state= None)
, shuffle = True,**rs)
rskf_cv = RepeatedStratifiedKFold(n_splits = 10
, n_repeats = 3
, **rs)
mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)}
jacc_score_fn = {'jcc': make_scorer(jaccard_score)}
###############################################################################
homedir = os.path.expanduser("~")
sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions')
sys.path
###############################################################################
outdir = homedir
from GetMLData import *
from SplitTTS import *
def remove(string):
return(string.replace(" ", ""))
#%%############################################################################
############################
# MultModelsCl()
# Run Multiple Classifiers
############################
# Multiple Classification - Model Pipeline
def XGBClf(input_df, target, sel_cv
, blind_test_df
, blind_test_target
, tts_split_type
, resampling_type = 'none' # default
#, add_cm = True # adds confusion matrix based on cross_val_predict
#, add_yn = True # adds target var class numbers
, var_type = ['numerical', 'categorical','mixed']
, run_blind_test = True
#, return_formatted_output = True
):
'''
@ param input_df: input features
@ type: df with input features WITHOUT the target variable
@param target: target (or output) feature
@type: df or np.array or Series
@param skv_cv: stratifiedK fold int or object to allow shuffle and random state to pass
@type: int or StratifiedKfold()
@var_type: numerical, categorical and mixed to determine what col_transform to apply (MinMaxScalar and/or one-ho t encoder)
@type: list
returns
Dict containing multiple classification scores for each model and mean of each Stratified Kfold including training
'''
#======================================================
# Determine categorical and numerical features
#======================================================
numerical_ix = input_df.select_dtypes(include=['int64', 'float64']).columns
numerical_ix
categorical_ix = input_df.select_dtypes(include=['object', 'bool']).columns
categorical_ix
#======================================================
# Determine preprocessing steps ~ var_type
#======================================================
if var_type == 'numerical':
t = [('num', MinMaxScaler(), numerical_ix)]
if var_type == 'categorical':
t = [('cat', OneHotEncoder(), categorical_ix)]
if var_type == 'mixed':
t = [('num', MinMaxScaler(), numerical_ix)
, ('cat', OneHotEncoder(), categorical_ix) ]
col_transform = ColumnTransformer(transformers = t
, remainder='passthrough')
#======================================================
# Specify multiple Classification Models
#======================================================
models = [ ('XGBoost' , XGBClassifier(**rs, verbosity = 3, use_label_encoder = False, **njobs) )
, ( 'Random Forest', RandomForestClassifier(**rs, **njobs, n_estimators = 1000))
, ('Logistic Regression', LogisticRegression(**rs))]
mm_skf_scoresD = {}
print('\n==============================================================\n'
, '\nRunning several classification models (n):', len(models)
,'\nList of models:')
for m in models:
print(m)
print('\n================================================================\n')
index = 1
for model_name, model_fn in models:
print('\nRunning classifier:', index
, '\nModel_name:' , model_name
, '\nModel func:' , model_fn)
index = index+1
model_pipeline = Pipeline([
('prep' , col_transform)
, ('model' , model_fn)])
print('\nRunning model pipeline:', model_pipeline)
skf_cv_modD = cross_validate(model_pipeline
, input_df
, target
, cv = sel_cv
, scoring = scoring_fn)
#==============================
# Extract mean values for CV
#==============================
mm_skf_scoresD[model_name] = {}
for key, value in skf_cv_modD.items():
print('\nkey:', key, '\nvalue:', value)
print('\nmean value:', np.mean(value))
mm_skf_scoresD[model_name][key] = round(np.mean(value),2)
# ADD more info: meta data related to input df
mm_skf_scoresD[model_name]['resampling'] = resampling_type
mm_skf_scoresD[model_name]['n_training_size'] = len(input_df)
mm_skf_scoresD[model_name]['n_trainingY_ratio'] = round(Counter(target)[0]/Counter(target)[1], 2)
mm_skf_scoresD[model_name]['n_features'] = len(input_df.columns)
mm_skf_scoresD[model_name]['tts_split'] = tts_split_type
# FS
#mnf = remove(model_name)
#model_pipeline.fit(input_df, target)
#print('\nFeature importance:', (model_pipeline.named_steps.model.feature_importances_))
#allf_xgboost = model_pipeline.feature_names_in_
#fsi_model = model_pipeline.named_steps.model.feature_importances_
#mm_skf_scoresD[model_name]['fs_importance'] = fsi_model
# TODO: add this as a key
#Add
#pyplot.bar(range(len(model_pipeline.named_steps.model.feature_importances_)), model_pipeline.named_steps.model.feature_importances_)
#pyplot.show()
#plot_importance(model_pipeline.named_steps.model.feature_importances_)
#pyplot.show()
if run_blind_test:
btD = {}
# Build bts numbers dict
btD = {'n_blindY_neg' : Counter(blind_test_target)[0]
, 'n_blindY_pos' : Counter(blind_test_target)[1]
, 'n_testY_ratio' : round(Counter(blind_test_target)[0]/Counter(blind_test_target)[1], 2)
, 'n_test_size' : len(blind_test_df) }
# Update cmD+tnD dicts with btD
mm_skf_scoresD[model_name].update(btD)
#--------------------------------------------------------
# Build the final results with all scores for the model
#--------------------------------------------------------
#bts_predict = gscv_fs.predict(blind_test_df)
model_pipeline.fit(input_df, target)
bts_predict = model_pipeline.predict(blind_test_df)
bts_mcc_score = round(matthews_corrcoef(blind_test_target, bts_predict),2)
print('\nMCC on Blind test:' , bts_mcc_score)
#print('\nAccuracy on Blind test:', round(accuracy_score(blind_test_target, bts_predict),2))
mm_skf_scoresD[model_name]['bts_mcc'] = bts_mcc_score
mm_skf_scoresD[model_name]['bts_fscore'] = round(f1_score(blind_test_target, bts_predict),2)
mm_skf_scoresD[model_name]['bts_precision'] = round(precision_score(blind_test_target, bts_predict),2)
mm_skf_scoresD[model_name]['bts_recall'] = round(recall_score(blind_test_target, bts_predict),2)
mm_skf_scoresD[model_name]['bts_accuracy'] = round(accuracy_score(blind_test_target, bts_predict),2)
mm_skf_scoresD[model_name]['bts_roc_auc'] = round(roc_auc_score(blind_test_target, bts_predict),2)
mm_skf_scoresD[model_name]['bts_jcc'] = round(jaccard_score(blind_test_target, bts_predict),2)
return(mm_skf_scoresD)
#%%
sel_cv = skf_cv
# param dict for getmldata()
combined_model_paramD = {'data_combined_model' : False
, 'use_or' : False
, 'omit_all_genomic_features': False
, 'write_maskfile' : False
, 'write_outfile' : False }
#df = getmldata(gene, drug, **combined_model_paramD)
df = getmldata('pncA', 'pyrazinamide', **combined_model_paramD)
df2 = split_tts(df
, data_type = 'actual'
, split_type = '80_20'
, oversampling = False
, dst_colname = 'dst'
, target_colname = 'dst_mode'
, include_gene_name = True
, random_state = 42 # default
)
all(df2['X'].columns.isin(['gene_name']))
fooD = XGBClf (input_df = df2['X']
, target = df2['y']
, sel_cv = skf_cv
, run_blind_test = True
, blind_test_df = df2['X_bts']
, blind_test_target = df2['y_bts']
, tts_split_type = '80_20'
, var_type = 'mixed'
, resampling_type = 'none' # default
)
for k, v in fooD.items():
print('\nK:', k
, '\nTRAIN MCC:', fooD[k]['test_mcc']
, '\nBTS MCC:' , fooD[k]['bts_mcc'] )
#%%
# # fit model no training data
# model = XGBClassifier()
# model.fit( df2['X'], df2['y'])
# # feature importance
# print(model.feature_importances_)
# # plot
# pyplot.bar(range(len(model.feature_importances_)), model.feature_importances_)
# pyplot.show()