#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Wed Jun 29 19:44:06 2022 @author: tanu """ import sys, os import pandas as pd import numpy as np import re from copy import deepcopy from sklearn import linear_model from sklearn import datasets from collections import Counter from sklearn.linear_model import LogisticRegression, LogisticRegressionCV from sklearn.linear_model import RidgeClassifier, RidgeClassifierCV, SGDClassifier, PassiveAggressiveClassifier from sklearn.naive_bayes import BernoulliNB from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier, BaggingClassifier from sklearn.naive_bayes import GaussianNB from sklearn.gaussian_process import GaussianProcessClassifier, kernels from sklearn.gaussian_process.kernels import RBF, DotProduct, Matern, RationalQuadratic, WhiteKernel from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis from sklearn.neural_network import MLPClassifier from sklearn.svm import SVC from xgboost import XGBClassifier from sklearn.naive_bayes import MultinomialNB from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder from sklearn.compose import ColumnTransformer from sklearn.compose import make_column_transformer from sklearn.metrics import make_scorer, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score, classification_report # added from sklearn.model_selection import train_test_split, cross_validate, cross_val_score, LeaveOneOut, KFold, RepeatedKFold, cross_val_predict from sklearn.model_selection import train_test_split, cross_validate, cross_val_score from sklearn.model_selection import StratifiedKFold,RepeatedStratifiedKFold, RepeatedKFold from sklearn.pipeline import Pipeline, make_pipeline from sklearn.feature_selection import RFE, RFECV import itertools import seaborn as sns import matplotlib.pyplot as plt from statistics import mean, stdev, median, mode from imblearn.over_sampling import RandomOverSampler from imblearn.under_sampling import RandomUnderSampler from imblearn.over_sampling import SMOTE from sklearn.datasets import make_classification from imblearn.combine import SMOTEENN from imblearn.combine import SMOTETomek from imblearn.over_sampling import SMOTENC from imblearn.under_sampling import EditedNearestNeighbours from imblearn.under_sampling import RepeatedEditedNearestNeighbours from sklearn.model_selection import GridSearchCV from sklearn.base import BaseEstimator from sklearn.impute import KNNImputer as KNN import json import argparse import re import itertools from sklearn.model_selection import LeaveOneGroupOut ############################################################################### homedir = os.path.expanduser("~") sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions') sys.path ############################################################################### outdir = homedir + '/git/LSHTM_ML/output/combined/' #==================== # Import ML functions #==================== from ml_data_combined import * #njobs = {'n_jobs': os.cpu_count() } # the number of jobs should equal the number of CPU cores ######################################################################## # COMPLETE data: No tts_split ######################################################################## #%% def combined_DF_OS(cm_input_df , all_genes = ["embb", "katg", "rpob", "pnca", "gid", "alr"] , bts_genes = ["embb", "katg", "rpob", "pnca", "gid"] , cols_to_drop = ['dst', 'dst_mode', 'gene_name'] , target_var = 'dst_mode' , gene_group = 'gene_name' , std_gene_omit = [] #, output_dir = outdir #, file_suffix = "" , oversampling = True , k_smote = 5 , random_state = 42 , njobs = os.cpu_count() # the number of jobs should equal the number of CPU cores ): outDict = {} rs = {'random_state': random_state} njobs = {'n_jobs': njobs } for bts_gene in bts_genes: print('\n BTS gene:', bts_gene) if not std_gene_omit: training_genesL = ['alr'] else: training_genesL = [] tr_gene_omit = std_gene_omit + [bts_gene] n_tr_genes = (len(bts_genes) - (len(std_gene_omit))) #n_total_genes = (len(bts_genes) - len(std_gene_omit)) n_total_genes = len(all_genes) training_genesL = training_genesL + list(set(bts_genes) - set(tr_gene_omit)) #training_genesL = [element for element in bts_genes if element not in tr_gene_omit] print('\nTotal genes: ', n_total_genes ,'\nTraining on:', n_tr_genes ,'\nTraining on genes:', training_genesL , '\nOmitted genes:', tr_gene_omit , '\nBlind test gene:', bts_gene) print('\nDim of data:', cm_input_df.shape) tts_split_type = "logo_skf_BT_" + bts_gene #------- # training #------ cm_training_df = cm_input_df[~cm_input_df['gene_name'].isin(tr_gene_omit)] cm_X = cm_training_df.drop(cols_to_drop, axis=1, inplace=False) #cm_y = cm_training_df.loc[:,'dst_mode'] cm_y = cm_training_df.loc[:, target_var] gene_group = cm_training_df.loc[:,'gene_name'] print('\nTraining data dim:', cm_X.shape , '\nTraining Target dim:', cm_y.shape) if all(cm_X.columns.isin(cols_to_drop) == False): print('\nChecked training df does NOT have Target var') else: sys.exit('\nFAIL: training data contains Target var') yc1 = Counter(cm_y) yc1_ratio = yc1[0]/yc1[1] #--------------- # BTS: genes #--------------- cm_test_df = cm_input_df[cm_input_df['gene_name'].isin([bts_gene])] cm_bts_X = cm_test_df.drop(cols_to_drop, axis = 1, inplace = False) #cm_bts_y = cm_test_df.loc[:, 'dst_mode'] cm_bts_y = cm_test_df.loc[:, target_var] print('\nTEST data dim:' , cm_bts_X.shape , '\nTEST Target dim:' , cm_bts_y.shape) print("Running Multiple models on LOGO with SKF") yc2 = Counter(cm_bts_y) yc2_ratio = yc2[0]/yc2[1] outDict.update({'X' : cm_X , 'y' : cm_y , 'X_bts' : cm_bts_X , 'y_bts' : cm_bts_y }) if oversampling: ####################################################################### # RESAMPLING ####################################################################### #------------------------------ # Simple Random oversampling # [Numerical + catgeorical] #------------------------------ oversample = RandomOverSampler(sampling_strategy='minority') X_ros, y_ros = oversample.fit_resample(cm_X, cm_y) print('\nSimple Random OverSampling\n', Counter(y_ros)) print(X_ros.shape) #------------------------------ # Simple Random Undersampling # [Numerical + catgeorical] #------------------------------ undersample = RandomUnderSampler(sampling_strategy='majority') X_rus, y_rus = undersample.fit_resample(cm_X, cm_y) print('\nSimple Random UnderSampling\n', Counter(y_rus)) print(X_rus.shape) #------------------------------ # Simple combine ROS and RUS # [Numerical + catgeorical] #------------------------------ oversample = RandomOverSampler(sampling_strategy='minority') X_ros, y_ros = oversample.fit_resample(cm_X, cm_y) undersample = RandomUnderSampler(sampling_strategy='majority') X_rouC, y_rouC = undersample.fit_resample(X_ros, y_ros) print('\nSimple Combined Over and UnderSampling\n', Counter(y_rouC)) print(X_rouC.shape) #------------------------------ # SMOTE_NC: oversampling # [numerical + categorical] #https://stackoverflow.com/questions/47655813/oversampling-smote-for-binary-and-categorical-data-in-python #------------------------------ # Determine categorical and numerical features numerical_ix = cm_X.select_dtypes(include=['int64', 'float64']).columns numerical_ix num_featuresL = list(numerical_ix) numerical_colind = cm_X.columns.get_indexer(list(numerical_ix) ) numerical_colind categorical_ix = cm_X.select_dtypes(include=['object', 'bool']).columns categorical_ix categorical_colind = cm_X.columns.get_indexer(list(categorical_ix)) categorical_colind #k_sm = 5 # default k_sm = k_smote sm_nc = SMOTENC(categorical_features=categorical_colind, k_neighbors = k_sm , **rs , **njobs) X_smnc, y_smnc = sm_nc.fit_resample(cm_X, cm_y) print('\nSMOTE_NC OverSampling\n', Counter(y_smnc)) print(X_smnc.shape) #====================== # vars #====================== #====================================================== # Determine categorical and numerical features #====================================================== numerical_cols = cm_X.select_dtypes(include=['int64', 'float64']).columns numerical_cols categorical_cols = cm_X.select_dtypes(include=['object', 'bool']).columns categorical_cols print('\n-------------------------------------------------------------' , '\nSuccessfully generated training and test data:' #, '\nData used:' , data_type #, '\nSplit type:', split_type , '\n\nTotal no. of input features:' , len(cm_X.columns) , '\n--------No. of numerical features:' , len(numerical_cols) , '\n--------No. of categorical features:', len(categorical_cols) , '\n===========================' , '\n Resampling: NONE' , '\n Baseline' , '\n===========================' , '\ninput data size:' , len(cm_input_df) , '\n\nTrain data size:' , cm_X.shape , '\ny_train numbers:' , yc1 , '\n\nTest data size:' , cm_bts_X.shape , '\ny_test_numbers:' , yc2 , '\n\ny_train ratio:' , yc1_ratio , '\ny_test ratio:' , yc2_ratio , '\n-------------------------------------------------------------') print('\nGenerated Resampled data as below:' , '\n=================================' , '\nResampling: Random oversampling' , '\n================================' , '\n\nTrain data size:', X_ros.shape , '\ny_train numbers:', len(y_ros) , '\n\ny_train ratio:', Counter(y_ros)[0]/Counter(y_ros)[1] , '\ny_test ratio:' , yc2_ratio ################################################################## , '\n================================' , '\nResampling: Random underampling' , '\n================================' , '\n\nTrain data size:', X_rus.shape , '\ny_train numbers:', len(y_rus) , '\n\ny_train ratio:', Counter(y_rus)[0]/Counter(y_rus)[1] , '\ny_test ratio:' , yc2_ratio ################################################################## , '\n================================' , '\nResampling:Combined (over+under)' , '\n================================' , '\n\nTrain data size:', X_rouC.shape , '\ny_train numbers:', len(y_rouC) , '\n\ny_train ratio:', Counter(y_rouC)[0]/Counter(y_rouC)[1] , '\ny_test ratio:' , yc2_ratio ################################################################## , '\n==============================' , '\nResampling: Smote NC' , '\n==============================' , '\n\nTrain data size:', X_smnc.shape , '\ny_train numbers:', len(y_smnc) , '\n\ny_train ratio:', Counter(y_smnc)[0]/Counter(y_smnc)[1] , '\ny_test ratio:' , yc2_ratio ################################################################## , '\n-------------------------------------------------------------') outDict.update({'X_ros' : X_ros , 'y_ros' : y_ros , 'X_rus' : X_rus , 'y_rus' : y_rus , 'X_rouC': X_rouC , 'y_rouC': y_rouC , 'X_smnc': X_smnc , 'y_smnc': y_smnc}) return(outDict) else: return(outDict)