diff --git a/scripts/ml/combined_model/cm_ml_iterator_TODO.py b/scripts/ml/combined_model/cm_ml_iterator_TODO.py deleted file mode 100755 index f899b68..0000000 --- a/scripts/ml/combined_model/cm_ml_iterator_TODO.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Wed Jun 29 20:29:36 2022 - -@author: tanu -""" -import sys, os -import pandas as pd -import numpy as np -import re - -############################################################################### -homedir = os.path.expanduser("~") -sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions') -sys.path -############################################################################### -outdir = homedir + '/git/LSHTM_ML/output/combined/' - -#==================== -# Import ML functions -#==================== -#from MultClfs import * -#from MultClfs_logo_skf import * -from MultClfs_logo_skf_split import * - -from GetMLData import * -from SplitTTS import * - -# Input data -from ml_data_combined import * - -############################################################################### -print('\nUsing data with 5 genes:', len(cm_input_df5)) - -############################################################################### - -split_types = ['70_30', '80_20', 'sl'] -split_data_types = ['actual', 'complete'] - -for split_type in split_types: - for data_type in split_data_types: - - out_filename = outdir + 'cm_' + split_type + '_' + data_type + '.csv' - print(out_filename) - tempD = split_tts(cm_input_df5 - , data_type = data_type - , split_type = split_type - , oversampling = True - , dst_colname = 'dst' - , target_colname = 'dst_mode' - , include_gene_name = True - ) - paramD = { - 'baseline_paramD': { 'input_df' : tempD['X'] - , 'target' : tempD['y'] - , 'var_type' : 'mixed' - , 'resampling_type' : 'none'} - , 'smnc_paramD' : { 'input_df' : tempD['X_smnc'] - , 'target' : tempD['y_smnc'] - , 'var_type' : 'mixed' - , 'resampling_type' : 'smnc'} - , 'ros_paramD' : { 'input_df' : tempD['X_ros'] - , 'target' : tempD['y_ros'] - , 'var_type' : 'mixed' - , 'resampling_type' : 'ros'} - , 'rus_paramD' : { 'input_df' : tempD['X_rus'] - , 'target' : tempD['y_rus'] - , 'var_type' : 'mixed' - , 'resampling_type' : 'rus'} - , 'rouC_paramD' : { 'input_df' : tempD['X_rouC'] - , 'target' : tempD['y_rouC'] - , 'var_type' : 'mixed' - , 'resampling_type' : 'rouC'} - } - - mmDD = {} - for k, v in paramD.items(): - scoresD = MultModelsCl_logo_skf(**paramD[k] - XXXXXXXXXXXXXXXXXXXXXXX - mmDD[k] = scoresD - - # Extracting the dfs from within the dict and concatenating to output as one df - for k, v in mmDD.items(): - out_wf= pd.concat(mmDD, ignore_index = True) - - out_wf_f = out_wf.sort_values(by = ['resampling', 'source_data', 'MCC'], ascending = [True, True, False], inplace = False) - out_wf_f.to_csv(('/home/tanu/git/Data/ml_combined/genes/'+out_filename), index = False) - diff --git a/scripts/ml/combined_model/combined_model_iterator.py b/scripts/ml/combined_model/combined_model_iterator.py new file mode 100644 index 0000000..7f8f07f --- /dev/null +++ b/scripts/ml/combined_model/combined_model_iterator.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Wed Jun 29 19:44:06 2022 + +@author: tanu +""" +import sys, os +import pandas as pd +import numpy as np +import re +from copy import deepcopy +from sklearn import linear_model +from sklearn import datasets +from collections import Counter + +from sklearn.linear_model import LogisticRegression, LogisticRegressionCV +from sklearn.linear_model import RidgeClassifier, RidgeClassifierCV, SGDClassifier, PassiveAggressiveClassifier + +from sklearn.naive_bayes import BernoulliNB +from sklearn.neighbors import KNeighborsClassifier +from sklearn.svm import SVC +from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier +from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier, BaggingClassifier +from sklearn.naive_bayes import GaussianNB +from sklearn.gaussian_process import GaussianProcessClassifier, kernels +from sklearn.gaussian_process.kernels import RBF, DotProduct, Matern, RationalQuadratic, WhiteKernel + +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis +from sklearn.neural_network import MLPClassifier + +from sklearn.svm import SVC +from xgboost import XGBClassifier +from sklearn.naive_bayes import MultinomialNB +from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder + +from sklearn.compose import ColumnTransformer +from sklearn.compose import make_column_transformer + +from sklearn.metrics import make_scorer, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score +from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score, classification_report + +# added +from sklearn.model_selection import train_test_split, cross_validate, cross_val_score, LeaveOneOut, KFold, RepeatedKFold, cross_val_predict + +from sklearn.model_selection import train_test_split, cross_validate, cross_val_score +from sklearn.model_selection import StratifiedKFold,RepeatedStratifiedKFold, RepeatedKFold + +from sklearn.pipeline import Pipeline, make_pipeline + +from sklearn.feature_selection import RFE, RFECV + +import itertools +import seaborn as sns +import matplotlib.pyplot as plt + +from statistics import mean, stdev, median, mode + +from imblearn.over_sampling import RandomOverSampler +from imblearn.under_sampling import RandomUnderSampler +from imblearn.over_sampling import SMOTE +from sklearn.datasets import make_classification +from imblearn.combine import SMOTEENN +from imblearn.combine import SMOTETomek + +from imblearn.over_sampling import SMOTENC +from imblearn.under_sampling import EditedNearestNeighbours +from imblearn.under_sampling import RepeatedEditedNearestNeighbours + +from sklearn.model_selection import GridSearchCV +from sklearn.base import BaseEstimator +from sklearn.impute import KNNImputer as KNN +import json +import argparse +import re +import itertools +from sklearn.model_selection import LeaveOneGroupOut +############################################################################### +homedir = os.path.expanduser("~") +sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions') +sys.path +############################################################################### +#outdir = homedir + '/git/LSHTM_ML/output/combined/' +outdir = homedir + '/git/LSHTM_ML/output/test/' + +#==================== +# Import ML functions +#==================== +from ml_data_combined import * +from MultClfs import * + +#from GetMLData import * +#from SplitTTS import * + +skf_cv = StratifiedKFold(n_splits = 10 , shuffle = True, random_state = 42) +#logo = LeaveOneGroupOut() +#random_state = 42 +#njobs = os.cpu_count() +sampling_names = {"": "none", "_ros": "Oversampling", "_rus": "Undersampling", "_rouC": "Over+Under", "_smnc": "SMOTE"} +######################################################################## +# COMPLETE data: No tts_split +######################################################################## +#%% +def CMLogoSkf(cm_input_df + , all_genes = ["embb", "katg", "rpob", "pnca", "gid", "alr"] + , bts_genes = ["embb", "katg", "rpob", "pnca", "gid"] + #, bts_genes = ["embb"] + , cols_to_drop = ['dst', 'dst_mode', 'gene_name'] + , target_var = 'dst_mode' + , gene_group = 'gene_name' + , std_gene_omit = [] + , output_dir = outdir + , file_suffix = "" + , random_state = 42 + , k_smote = 5 + , njobs = os.cpu_count() + ): + + + rs = {'random_state': random_state} + njobs = {'n_jobs': njobs } + + + for bts_gene in bts_genes: + outDict = {} + + print('\n BTS gene:', bts_gene) + if not std_gene_omit: + training_genesL = ['alr'] + else: + training_genesL = [] + + tr_gene_omit = std_gene_omit + [bts_gene] + n_tr_genes = (len(bts_genes) - (len(std_gene_omit))) + #n_total_genes = (len(bts_genes) - len(std_gene_omit)) + n_total_genes = len(all_genes) + + training_genesL = training_genesL + list(set(bts_genes) - set(tr_gene_omit)) + #training_genesL = [element for element in bts_genes if element not in tr_gene_omit] + + print('\nTotal genes: ', n_total_genes + ,'\nTraining on:', n_tr_genes + ,'\nTraining on genes:', training_genesL + , '\nOmitted genes:', tr_gene_omit + , '\nBlind test gene:', bts_gene) + + #print('\nDim of data:', cm_input_df.shape) + + tts_split_type = "logo_skf_BT_" + bts_gene + + # if len(file_suffix) > 0: + # file_suffix = '_' + file_suffix + # else: + # file_suffix = file_suffix + + #outFile = output_dir + str(n_tr_genes+1) + "genes_" + tts_split_type + '_' + file_suffix + ".csv" + + #print(outFile) + + #------- + # training + #------ + cm_training_df = cm_input_df[~cm_input_df['gene_name'].isin(tr_gene_omit)] + + cm_X = cm_training_df.drop(cols_to_drop, axis=1, inplace=False) + #cm_y = cm_training_df.loc[:,'dst_mode'] + cm_y = cm_training_df.loc[:, target_var] + + gene_group = cm_training_df.loc[:,'gene_name'] + + #print('\nTraining data dim:', cm_X.shape + # , '\nTraining Target dim:', cm_y.shape) + + if all(cm_X.columns.isin(cols_to_drop) == False): + print('\nChecked training df does NOT have Target var') + else: + sys.exit('\nFAIL: training data contains Target var') + + #--------------- + # BTS: genes + #--------------- + cm_test_df = cm_input_df[cm_input_df['gene_name'].isin([bts_gene])] + + cm_bts_X = cm_test_df.drop(cols_to_drop, axis = 1, inplace = False) + #cm_bts_y = cm_test_df.loc[:, 'dst_mode'] + cm_bts_y = cm_test_df.loc[:, target_var] + + #print('\nTEST data dim:' , cm_bts_X.shape + # , '\nTEST Target dim:' , cm_bts_y.shape) + + #print("Running Multiple models on LOGO with SKF") + # NULL, ros, rus, rouc, smnc + + outDict.update({'X_cm' : cm_X + , 'y_cm' : cm_y + , 'X_bts_cm' : cm_bts_X + , 'y_bts_cm' : cm_bts_y + }) + + ####################################################################### + # RESAMPLING + ####################################################################### + #------------------------------ + # Simple Random oversampling + # [Numerical + catgeorical] + #------------------------------ + oversample = RandomOverSampler(sampling_strategy='minority') + X_ros, y_ros = oversample.fit_resample(cm_X, cm_y) + # print('\nSimple Random OverSampling\n', Counter(y_ros)) + # print(X_ros.shape) + + #------------------------------ + # Simple Random Undersampling + # [Numerical + catgeorical] + #------------------------------ + undersample = RandomUnderSampler(sampling_strategy='majority') + X_rus, y_rus = undersample.fit_resample(cm_X, cm_y) + # print('\nSimple Random UnderSampling\n', Counter(y_rus)) + # print(X_rus.shape) + + #------------------------------ + # Simple combine ROS and RUS + # [Numerical + catgeorical] + #------------------------------ + oversample = RandomOverSampler(sampling_strategy='minority') + X_ros, y_ros = oversample.fit_resample(cm_X, cm_y) + + undersample = RandomUnderSampler(sampling_strategy='majority') + X_rouC, y_rouC = undersample.fit_resample(X_ros, y_ros) + # print('\nSimple Combined Over and UnderSampling\n', Counter(y_rouC)) + # print(X_rouC.shape) + + #------------------------------ + # SMOTE_NC: oversampling + # [numerical + categorical] + #https://stackoverflow.com/questions/47655813/oversampling-smote-for-binary-and-categorical-data-in-python + #------------------------------ + # Determine categorical and numerical features + numerical_ix = cm_X.select_dtypes(include=['int64', 'float64']).columns + num_featuresL = list(numerical_ix) + numerical_colind = cm_X.columns.get_indexer(list(numerical_ix) ) + + categorical_ix = cm_X.select_dtypes(include=['object', 'bool']).columns + categorical_colind = cm_X.columns.get_indexer(list(categorical_ix)) + + #k_sm = 5 # default + k_sm = k_smote + sm_nc = SMOTENC(categorical_features=categorical_colind, k_neighbors = k_sm + , **rs + , **njobs) + + X_smnc, y_smnc = sm_nc.fit_resample(cm_X, cm_y) + + outDict.update({'X_ros_cm' : X_ros + , 'y_ros_cm' : y_ros + + , 'X_rus_cm' : X_rus + , 'y_rus_cm' : y_rus + + , 'X_rouC_cm': X_rouC + , 'y_rouC_cm': y_rouC + + , 'X_smnc_cm': X_smnc + , 'y_smnc_cm': y_smnc}) + + #%%:Running Multiple models on LOGO with SKF +# cD3_v2 = MultModelsCl_logo_skf(input_df = cm_X # two func were identical excpet for name + for i in sampling_names.keys(): + print("thing:", "X"+i+"_cm", "y"+i+"_cm") + + current_X = "X" + i + "_cm" + current_y = "y" + i + "_cm" + + current_X_df = outDict[current_X] + current_y_df = outDict[current_y] + + + cD3_v2 = MultModelsCl(input_df = current_X_df + , target = current_y_df + , sel_cv = skf_cv + , tts_split_type = tts_split_type + , resampling_type = sampling_names[i] # 'none' # default + , add_cm = True + , add_yn = True + , var_type = 'mixed' + , scale_numeric = ['min_max'] + , run_blind_test = True + , blind_test_df = cm_bts_X + , blind_test_target = cm_bts_y + , return_formatted_output = True + , random_state = 42 + , n_jobs = os.cpu_count() # the number of jobs should equal the number of CPU cores + ) + outFile = output_dir + str(n_tr_genes+1) + "genes_" + tts_split_type + '_' + file_suffix + i + ".csv" + + + cD3_v2.to_csv(outFile) + + + # outDict.update({'X' : cm_X + # , 'y' : cm_y + # , 'X_bts' : cm_bts_X + # , 'y_bts' : cm_bts_y + # }) + + # return(outDict) + +#%% RUN +#=============== +# Complete Data +#============== +CMLogoSkf(cm_input_df = combined_df,file_suffix = "complete") +CMLogoSkf(cm_input_df = combined_df, std_gene_omit=['alr'], file_suffix = "complete") + +#=============== +# Actual Data +#=============== +#CMLogoSkf(cm_input_df = combined_df_actual, file_suffix = "actual") +#CMLogoSkf(cm_input_df = combined_df_actual, std_gene_omit=['alr'], file_suffix = "actual") + + diff --git a/scripts/ml/ml_functions/test_func_combined.py b/scripts/ml/ml_functions/test_func_combined.py index e868e3c..920a585 100644 --- a/scripts/ml/ml_functions/test_func_combined.py +++ b/scripts/ml/ml_functions/test_func_combined.py @@ -17,6 +17,11 @@ from SplitTTS import * from MultClfs import * from MultClfs_CVs import * +#==================== +# Import ML functions +#==================== +from ml_data_combined import * + #%% rs = {'random_state': 42} skf_cv = StratifiedKFold(n_splits = 10 @@ -35,7 +40,7 @@ gene_model_paramD = {'data_combined_model' : True #df = getmldata(gene, drug, **gene_model_paramD) #df = getmldata('pncA', 'pyrazinamide', **gene_model_paramD) -df = getmldata('embB', 'ethambutol' , **gene_model_paramD) +#df = getmldata('embB', 'ethambutol' , **gene_model_paramD) #df = getmldata('katG', 'isoniazid' , **gene_model_paramD) #df = getmldata('rpoB', 'rifampicin' , **gene_model_paramD) #df = getmldata('gid' , 'streptomycin' , **gene_model_paramD) @@ -43,9 +48,6 @@ df = getmldata('embB', 'ethambutol' , **gene_model_paramD) ########################## #%% TEST different CV Thresholds for split_type = NONE ################################################################ -Counter(df2['y']) -Counter(df2['y_bts']) - # READ Data spl_type = 'none' data_type = 'complete' @@ -59,6 +61,9 @@ df2 = split_tts(ml_input_data = combined_df , include_gene_name = True , random_state = 42 # default ) + +Counter(df2['y']) +Counter(df2['y_bts']) #%% Trying different CV thresholds for resampling 'none' ONLY fooD = MultModelsCl_CVs(input_df = df2['X'] , target = df2['y'] @@ -80,7 +85,8 @@ fooD = MultModelsCl_CVs(input_df = df2['X'] for k, v in fooD.items(): print('\nModel:', k - , '\nTRAIN MCC:', fooD[k]['test_mcc'] + , '\nTRAIN MCC:', fooD[k]['train_mcc'] + , '\nCV MCC:', fooD[k]['test_mcc'] )