diff --git a/MultClassPipe3.py b/MultClassPipe3.py index b7126fc..a795779 100644 --- a/MultClassPipe3.py +++ b/MultClassPipe3.py @@ -27,7 +27,7 @@ from sklearn.compose import ColumnTransformer from sklearn.compose import make_column_transformer from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score -from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef +from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score from sklearn.metrics import make_scorer from sklearn.metrics import classification_report @@ -70,7 +70,7 @@ scoring_fn = ({ 'fscore' : make_scorer(f1_score) , 'recall' : make_scorer(recall_score) , 'accuracy' : make_scorer(accuracy_score) , 'roc_auc' : make_scorer(roc_auc_score) - #, 'jaccard' : make_scorer(jaccard_score) + , 'jaccard' : make_scorer(jaccard_score) }) @@ -122,10 +122,11 @@ def MultClassPipeSKFCV(input_df, target, skf_cv, var_type = ['numerical', 'categ mlp = MLPClassifier(max_iter = 500, **rs) dt = DecisionTreeClassifier(**rs) et = ExtraTreesClassifier(**rs) - rf = RandomForestClassifier(**rs) + rf = RandomForestClassifier(**rs, + n_estimators = 1000 ) rf2 = RandomForestClassifier( - min_samples_leaf = 50 - , n_estimators = 150 + min_samples_leaf = 5 + , n_estimators = 1000 , bootstrap = True , oob_score = True , **njobs diff --git a/UQ_imports_pnca.py b/UQ_imports_pnca.py deleted file mode 100644 index 908dc15..0000000 --- a/UQ_imports_pnca.py +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Sun Mar 6 13:41:54 2022 - -@author: tanu -""" -import os, sys -import pandas as pd -import numpy as np -import pprint as pp -from copy import deepcopy -from sklearn import linear_model -from sklearn.linear_model import LogisticRegression, LinearRegression -from sklearn.naive_bayes import BernoulliNB -from sklearn.neighbors import KNeighborsClassifier -from sklearn.svm import SVC -from sklearn.tree import DecisionTreeClassifier -from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier -from sklearn.ensemble import AdaBoostClassifier -from sklearn.ensemble import GradientBoostingClassifier -from sklearn.neural_network import MLPClassifier -from xgboost import XGBClassifier -from sklearn.naive_bayes import MultinomialNB -from sklearn.linear_model import SGDClassifier -from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder - -from sklearn.compose import ColumnTransformer -from sklearn.compose import make_column_transformer - -from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score -from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef -from sklearn.metrics import jaccard_score - -from sklearn.metrics import make_scorer -from sklearn.metrics import classification_report - -from sklearn.metrics import average_precision_score - -from sklearn.model_selection import cross_validate -from sklearn.model_selection import train_test_split -from sklearn.model_selection import StratifiedKFold - -from sklearn.pipeline import Pipeline -from sklearn.pipeline import make_pipeline - -from sklearn.feature_selection import RFE -from sklearn.feature_selection import RFECV -import itertools -import seaborn as sns -import matplotlib.pyplot as plt -import numpy as np -print(np.__version__) -print(pd.__version__) -from statistics import mean, stdev, median, mode - -from imblearn.over_sampling import RandomOverSampler -from imblearn.over_sampling import SMOTE -from imblearn.pipeline import Pipeline -#from sklearn.datasets import make_classification -from sklearn.model_selection import cross_validate, cross_val_score -from sklearn.model_selection import RepeatedStratifiedKFold -from sklearn.ensemble import AdaBoostClassifier -from imblearn.combine import SMOTEENN -from imblearn.under_sampling import EditedNearestNeighbours - -from sklearn.model_selection import GridSearchCV -from sklearn.base import BaseEstimator - -scoring_fn = ({'accuracy' : make_scorer(accuracy_score) - , 'fscore' : make_scorer(f1_score) - , 'mcc' : make_scorer(matthews_corrcoef) - , 'precision' : make_scorer(precision_score) - , 'recall' : make_scorer(recall_score) - , 'roc_auc' : make_scorer(roc_auc_score) - , 'jcc' : make_scorer(jaccard_score) - }) - -rs = {'random_state': 42} -njobs = {'n_jobs': 10} -skf_cv = StratifiedKFold(n_splits = 10 - #, shuffle = False, random_state= None) - , shuffle = True,**rs) - -rskf_cv = RepeatedStratifiedKFold(n_splits = 10 - , n_repeats=3 - #, shuffle = False, random_state= None) - #, shuffle = True - ,**rs) -#my_mcc = make_scorer({'mcc':make_scorer(matthews_corrcoef}) -mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)} - -#%% -homedir = os.path.expanduser("~") -os.chdir(homedir + "/git/ML_AI_training/") - -# my function -#from MultClassPipe import MultClassPipeline -from MultClassPipe2 import MultClassPipeline2 -from loopity_loop import MultClassPipeSKFLoop -from MultClassPipe3 import MultClassPipeSKFCV - -gene = 'pncA' -drug = 'pyrazinamide' - -#============== -# directories -#============== -datadir = homedir + '/git/Data/' -indir = datadir + drug + '/input/' -outdir = datadir + drug + '/output/' - -#======= -# input -#======= -infile_ml1 = outdir + gene.lower() + '_merged_df3.csv' -#infile_ml2 = outdir + gene.lower() + '_merged_df2.csv' - -my_df = pd.read_csv(infile_ml1, index_col = 0) -my_df.dtypes -my_df_cols = my_df.columns - -geneL_basic = ['pnca'] - -# -- CHECK script -- imports.py -#%% get cols -mycols = my_df.columns -mycols - -# change from numberic to -num_type = ['int64', 'float64'] -cat_type = ['object', 'bool'] - -# TODO: -# Treat active site aa pos as category and not numerical: This needs to be part of merged_df3! -#if my_df['active_aa_pos'].dtype in num_type: -# my_df['active_aa_pos'] = my_df['active_aa_pos'].astype(object) -# my_df['active_aa_pos'].dtype - -# -- CHECK script -- imports.py -#%%============================================================================ -#%% IMPUTE values for OR - -#%% Combine mmCSM_lig Data - -#%% Combine PROVEAN data - -#%% Combine ED logo data - -#%% Masking columns (mCSM-lig, mCSM-NA, mCSM-ppi2) values for lig_dist >10 - -# get logic from upstream! -my_df_ml = my_df.copy() - -my_df_ml['mutationinformation'][my_df['ligand_distance']>10].value_counts() -my_df_ml.groupby('mutationinformation')['ligand_distance'].apply(lambda x: (x>10)).value_counts() -my_df_ml.groupby(['mutationinformation'])['ligand_distance'].apply(lambda x: (x>10)).value_counts() - -my_df_ml.loc[(my_df_ml['ligand_distance'] > 10), 'ligand_affinity_change'] = 0 -(my_df_ml['ligand_affinity_change'] == 0).sum() - -#%%============================================================================ -# Separate blind test set -my_df_ml[drug].isna().sum() - -blind_test_df = my_df_ml[my_df_ml[drug].isna()] - -training_df = my_df_ml[my_df_ml[drug].notna()] - -# Target1: dst -training_df[drug].value_counts() -training_df['dst_mode'].value_counts() - -#%% Build X -common_cols_stabiltyN = ['ligand_distance' - , 'ligand_affinity_change' - , 'duet_stability_change' - , 'ddg_foldx' - , 'deepddg' - , 'ddg_dynamut2'] - -foldX_cols = ['contacts' -#, 'electro_rr', 'electro_mm', 'electro_sm', 'electro_ss' -#, 'disulfide_rr', 'disulfide_mm', 'disulfide_sm', 'disulfide_ss' -#, 'hbonds_rr', 'hbonds_mm', 'hbonds_sm', 'hbonds_ss' -#, 'partcov_rr', 'partcov_mm', 'partcov_sm', 'partcov_ss' -#, 'vdwclashes_rr', 'vdwclashes_mm', 'vdwclashes_sm', 'vdwclashes_ss' -#, 'volumetric_rr', 'volumetric_mm', 'volumetric_ss' -] - -X_strFN = ['rsa' - #, 'asa' - , 'kd_values' - , 'rd_values'] - -X_evolFN = ['consurf_score' - , 'snap2_score'] - -# quick inspection which lineage to use: -#foo = my_df_ml[['lineage', 'lineage_count_all', 'lineage_count_unique']] - -X_genomicFN = ['maf' -# , 'or_mychisq' -# , 'or_logistic' -# , 'or_fisher' -# , 'pval_fisher' - #, 'lineage' - , 'lineage_count_all' - , 'lineage_count_unique' -] - -#%% Construct numerical and categorical column names - -# numerical feature names -numerical_FN = common_cols_stabiltyN + foldX_cols + X_strFN + X_evolFN + X_genomicFN - -#categorical feature names -categorical_FN = ['ss_class' - , 'wt_prop_water' - # , 'lineage_labels' # misleading if using merged_df3 - , 'mut_prop_water' - , 'wt_prop_polarity' - , 'mut_prop_polarity' - , 'wt_calcprop' - , 'mut_calcprop' - #, 'active_aa_pos' - ] - -#%% extracting dfs based on numerical, categorical column names -#---------------------------------- -# WITHOUT the target var included -#---------------------------------- -num_df = training_df[numerical_FN] -num_df.shape - -cat_df = training_df[categorical_FN] -cat_df.shape - -all_df = training_df[numerical_FN + categorical_FN] -all_df.shape - -#------------------------------ -# WITH the target var included: - #'wtgt': with target -#------------------------------ -# drug and dst_mode should be the same thing -num_df_wtgt = training_df[numerical_FN + ['dst_mode']] -num_df_wtgt.shape - -cat_df_wtgt = training_df[categorical_FN + ['dst_mode']] -cat_df_wtgt.shape - -all_df_wtgt = training_df[numerical_FN + categorical_FN + ['dst_mode']] -all_df_wtgt.shape -#%%================================================================ -#%% Apply ML - diff --git a/UQ_pnca_ml_CALL.py b/UQ_pnca_ml_CALL.py deleted file mode 100644 index 9032012..0000000 --- a/UQ_pnca_ml_CALL.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Mon May 16 05:59:12 2022 - -@author: tanu -""" -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Tue Mar 15 11:09:50 2022 - -@author: tanu -""" -#%% Data -X = all_df_wtgt[numerical_FN+categorical_FN] -X = all_df_wtgt[numerical_FN] - -y = all_df_wtgt['dst_mode'] -#%% variables - -#%% MultClassPipeSKFCV: function call() -mm_skf_scoresD = MultClassPipeSKFCV(input_df = X - , target = y - , var_type = 'numerical' - , skf_cv = skf_cv) - - -mm_skf_scores_df_all = pd.DataFrame(mm_skf_scoresD) -mm_skf_scores_df_all -mm_skf_scores_df_test = mm_skf_scores_df_all.filter(like='test_', axis=0) -mm_skf_scores_df_train = mm_skf_scores_df_all.filter(like='train_', axis=0) # helps to see if you trust the results - -#%% CHECK with BLIND test -#%% -import plotly.express as px - -corr = X.corr(method = 'spearman') -corr.head() - -#p = corr.style.background_gradient(cmap='coolwarm') -p = corr.style.background_gradient(cmap='coolwarm').set_precision(2) -p - -fig = px.imshow(corr) -fig.show() - - -#%%TODO: -# Add correlation plot -# Remove low variance features -# Add feature selection -# Then run your models on BLIND test WITHOUT CV - - - diff --git a/__pycache__/MultClassPipe3.cpython-37.pyc b/__pycache__/MultClassPipe3.cpython-37.pyc index 5acf839..20a2591 100644 Binary files a/__pycache__/MultClassPipe3.cpython-37.pyc and b/__pycache__/MultClassPipe3.cpython-37.pyc differ diff --git a/base_estimator3.py b/base_estimator3.py index 6627e78..4ca4002 100644 --- a/base_estimator3.py +++ b/base_estimator3.py @@ -17,16 +17,11 @@ from sklearn.model_selection import GridSearchCV from sklearn.linear_model import LogisticRegression from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder from xgboost import XGBClassifier -#%% Get train-test split and scoring functions -X_train, X_test, y_train, y_test = train_test_split(num_df_wtgt[numerical_FN] - , num_df_wtgt['mutation_class'] - , test_size = 0.33 - , random_state = 2 - , shuffle = True - , stratify = num_df_wtgt['mutation_class']) +####################################################### +y.to_frame().value_counts().plot(kind = 'bar') + +blind_test_df['dst_mode'].to_frame().value_counts().plot(kind = 'bar') -y_train.to_frame().value_counts().plot(kind = 'bar') -y_test.to_frame().value_counts().plot(kind = 'bar') scoring_fn = ({'accuracy' : make_scorer(accuracy_score) , 'fscore' : make_scorer(f1_score) , 'mcc' : make_scorer(matthews_corrcoef) diff --git a/classification_names_params.py b/classification_names_params.py index 905c152..09bda0d 100644 --- a/classification_names_params.py +++ b/classification_names_params.py @@ -32,7 +32,7 @@ names = [ ] classifiers = [ - KNeighborsClassifier(3), + KNeighborsClassifier(5), SVC(kernel="linear", C=0.025), SVC(gamma=2, C=1), GaussianProcessClassifier(1.0 * RBF(1.0)), @@ -97,7 +97,7 @@ classifiers = [ )) gs_knn_params = { - 'clf__n_neighbors': [3, 7, 10] + 'clf__n_neighbors': [5, 7, 11] #, 'clf__n_neighbors': range(1, 21, 2) ,'clf__metric' : ['euclidean', 'manhattan', 'minkowski'] , 'clf__weights' : ['uniform', 'distance'] @@ -120,7 +120,7 @@ classifiers = [ , 'clf__min_samples_leaf': [2, 4, 8, 50] , 'clf__min_samples_split': [10, 20] } -#%% XGBClassifier() +#%% XGBClassifier() # SPNT # https://stackoverflow.com/questions/34674797/xgboost-xgbclassifier-defaults-in-python # https://stackoverflow.com/questions/34674797/xgboost-xgbclassifier-defaults-in-python gs_xgb = Pipeline(( @@ -135,6 +135,7 @@ classifiers = [ , 'clf__min_samples_leaf': [4, 8, 12, 16, 20] , 'clf__max_features': ['auto', 'sqrt'] } + #%% MLPClassifier() # https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html gs_mlp = Pipeline(( @@ -190,6 +191,7 @@ classifiers = [ # If None, then the base estimator is a DecisionTreeClassifier. , 'clf__base_estimator' : ['None', 'SVC()', 'KNeighborsClassifier()']# if none, DT is used , 'clf__gamma': ['scale', 'auto'] } + #%% GradientBoostingClassifier() # https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html gs_gb = Pipeline(( @@ -198,7 +200,7 @@ classifiers = [ )) gs_bdt_params = { - 'clf__n_estimators' : [10, 100, 1000] + 'clf__n_estimators' : [10, 100, 200, 500, 1000] , 'clf__n_estimators' : [10, 100, 1000] , 'clf__learning_rate': [0.001, 0.01, 0.1] , 'clf__subsample' : [0.5, 0.7, 1.0] @@ -261,4 +263,4 @@ BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True) , 'clf__binarize':['None', 0] , 'clf__fit_prior': [True] , 'clf__class_prior': ['None'] - } \ No newline at end of file + }