various changes
This commit is contained in:
parent
f761dd4479
commit
5202be4adc
52 changed files with 1440 additions and 88 deletions
166
UQ_ML_data.py
Normal file → Executable file
166
UQ_ML_data.py
Normal file → Executable file
|
|
@ -27,6 +27,37 @@ def setvars(gene,drug):
|
|||
from imblearn.under_sampling import EditedNearestNeighbours
|
||||
from imblearn.under_sampling import RepeatedEditedNearestNeighbours
|
||||
|
||||
from sklearn.metrics import make_scorer, confusion_matrix, accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score
|
||||
from sklearn.metrics import roc_auc_score, roc_curve, f1_score, matthews_corrcoef, jaccard_score, classification_report
|
||||
|
||||
from sklearn.model_selection import train_test_split, cross_validate, cross_val_score
|
||||
from sklearn.model_selection import StratifiedKFold,RepeatedStratifiedKFold, RepeatedKFold
|
||||
|
||||
from sklearn.pipeline import Pipeline, make_pipeline
|
||||
#%% GLOBALS
|
||||
rs = {'random_state': 42}
|
||||
njobs = {'n_jobs': 10}
|
||||
|
||||
scoring_fn = ({ 'mcc' : make_scorer(matthews_corrcoef)
|
||||
, 'accuracy' : make_scorer(accuracy_score)
|
||||
, 'fscore' : make_scorer(f1_score)
|
||||
, 'precision' : make_scorer(precision_score)
|
||||
, 'recall' : make_scorer(recall_score)
|
||||
, 'roc_auc' : make_scorer(roc_auc_score)
|
||||
, 'jcc' : make_scorer(jaccard_score)
|
||||
})
|
||||
|
||||
skf_cv = StratifiedKFold(n_splits = 10
|
||||
#, shuffle = False, random_state= None)
|
||||
, shuffle = True,**rs)
|
||||
|
||||
rskf_cv = RepeatedStratifiedKFold(n_splits = 10
|
||||
, n_repeats = 3
|
||||
, **rs)
|
||||
|
||||
mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)}
|
||||
jacc_score_fn = {'jcc': make_scorer(jaccard_score)}
|
||||
|
||||
#%% FOR LATER: Combine ED logo data
|
||||
#%% FOR LARER: active aa site annotations
|
||||
###########################################################################
|
||||
|
|
@ -51,32 +82,42 @@ def setvars(gene,drug):
|
|||
my_df.dtypes
|
||||
my_df_cols = my_df.columns
|
||||
|
||||
geneL_basic = ['pnca']
|
||||
|
||||
# -- CHECK script -- imports.py
|
||||
geneL_basic = ['pnca']
|
||||
geneL_na = ['gid']
|
||||
geneL_na_ppi2 = ['rpob']
|
||||
geneL_ppi2 = ['alr', 'embb', 'katg']
|
||||
#%% get cols
|
||||
mycols = my_df.columns
|
||||
mycols
|
||||
|
||||
# change from numberic to
|
||||
num_type = ['int64', 'float64']
|
||||
cat_type = ['object', 'bool']
|
||||
# # change from numberic to
|
||||
# num_type = ['int64', 'float64']
|
||||
# cat_type = ['object', 'bool']
|
||||
|
||||
#TODO:
|
||||
# #Treat active site aa pos as category and not numerical: This needs to be part of merged_df3!
|
||||
# if my_df['active_aa_pos'].dtype in num_type:
|
||||
# my_df['active_aa_pos'] = my_df['active_aa_pos'].astype(object)
|
||||
# my_df['active_aa_pos'].dtype
|
||||
|
||||
# -- CHECK script -- imports.py
|
||||
# FIXME: if this is not structural, remove from source..
|
||||
# Drop NA where numerical cols have them
|
||||
if gene.lower() in geneL_na_ppi2:
|
||||
#D1148 get rid of
|
||||
na_index = my_df['mutationinformation'].index[my_df['mcsm_na_affinity'].apply(np.isnan)]
|
||||
my_df = my_df.drop(index=na_index)
|
||||
|
||||
# FIXME: either impute or remove!
|
||||
# for embb (L114M, F115L, V123L, V125I, V131M) delete for now
|
||||
if gene.lower() in ['embb']:
|
||||
na_index = my_df['mutationinformation'].index[my_df['ligand_distance'].apply(np.isnan)]
|
||||
my_df = my_df.drop(index=na_index)# RERUN embb with the 5 values now present
|
||||
|
||||
###########################################################################
|
||||
#%% Add lineage calculation columns
|
||||
#FIXME: Check if this can be imported from config?
|
||||
total_mtblineage_u = 8
|
||||
total_mtblineage_uc = 8
|
||||
lineage_colnames = ['lineage_list_all', 'lineage_count_all', 'lineage_count_unique', 'lineage_list_unique', 'lineage_multimode']
|
||||
#bar = my_df[lineage_colnames]
|
||||
my_df['lineage_proportion'] = my_df['lineage_count_unique']/my_df['lineage_count_all']
|
||||
my_df['dist_lineage_proportion'] = my_df['lineage_count_unique']/total_mtblineage_u
|
||||
my_df['dist_lineage_proportion'] = my_df['lineage_count_unique']/total_mtblineage_uc
|
||||
###########################################################################
|
||||
#%% AA property change
|
||||
#--------------------
|
||||
|
|
@ -219,15 +260,6 @@ def setvars(gene,drug):
|
|||
#==========================
|
||||
my_df_ml = my_df.copy()
|
||||
|
||||
#%% Masking columns (mCSM-lig, mCSM-NA, mCSM-ppi2) values for lig_dist >10
|
||||
my_df_ml['mutationinformation'][my_df['ligand_distance']>10].value_counts()
|
||||
my_df_ml.groupby('mutationinformation')['ligand_distance'].apply(lambda x: (x>10)).value_counts()
|
||||
my_df_ml.groupby(['mutationinformation'])['ligand_distance'].apply(lambda x: (x>10)).value_counts()
|
||||
|
||||
my_df_ml.loc[(my_df_ml['ligand_distance'] > 10), 'ligand_affinity_change'] = 0
|
||||
(my_df_ml['ligand_affinity_change'] == 0).sum()
|
||||
|
||||
#%%########################################################################
|
||||
#==========================
|
||||
# BLIND test set
|
||||
#==========================
|
||||
|
|
@ -253,8 +285,32 @@ def setvars(gene,drug):
|
|||
, 'ddg_dynamut2'
|
||||
, 'mmcsm_lig'
|
||||
, 'contacts']
|
||||
|
||||
# Build stability columns ~ gene
|
||||
if gene.lower() in geneL_basic:
|
||||
X_stabilityN = common_cols_stabiltyN
|
||||
cols_to_mask = ['ligand_affinity_change']
|
||||
|
||||
if gene.lower() in geneL_ppi2:
|
||||
# X_stabilityN = common_cols_stabiltyN + ['mcsm_ppi2_affinity' , 'interface_dist']
|
||||
geneL_ppi2_st_cols = ['mcsm_ppi2_affinity', 'interface_dist']
|
||||
X_stabilityN = common_cols_stabiltyN + geneL_ppi2_st_cols
|
||||
cols_to_mask = ['ligand_affinity_change', 'mcsm_ppi2_affinity']
|
||||
|
||||
if gene.lower() in geneL_na:
|
||||
# X_stabilityN = common_cols_stabiltyN + ['mcsm_na_affinity']
|
||||
geneL_na_st_cols = ['mcsm_na_affinity']
|
||||
X_stabilityN = common_cols_stabiltyN + geneL_na_st_cols
|
||||
cols_to_mask = ['ligand_affinity_change', 'mcsm_na_affinity']
|
||||
|
||||
if gene.lower() in geneL_na_ppi2:
|
||||
# X_stabilityN = common_cols_stabiltyN + ['mcsm_na_affinity'] + ['mcsm_ppi2_affinity', 'interface_dist']
|
||||
geneL_na_ppi2_st_cols = ['mcsm_na_affinity'] + ['mcsm_ppi2_affinity', 'interface_dist']
|
||||
X_stabilityN = common_cols_stabiltyN + geneL_na_ppi2_st_cols
|
||||
cols_to_mask = ['ligand_affinity_change', 'mcsm_na_affinity', 'mcsm_ppi2_affinity']
|
||||
|
||||
|
||||
foldX_cols = [ 'electro_rr', 'electro_mm', 'electro_sm', 'electro_ss'
|
||||
X_foldX_cols = [ 'electro_rr', 'electro_mm', 'electro_sm', 'electro_ss'
|
||||
, 'disulfide_rr', 'disulfide_mm', 'disulfide_sm', 'disulfide_ss'
|
||||
, 'hbonds_rr', 'hbonds_mm', 'hbonds_sm', 'hbonds_ss'
|
||||
, 'partcov_rr', 'partcov_mm', 'partcov_sm', 'partcov_ss'
|
||||
|
|
@ -262,11 +318,13 @@ def setvars(gene,drug):
|
|||
, 'volumetric_rr', 'volumetric_mm', 'volumetric_ss'
|
||||
]
|
||||
|
||||
X_strFN = ['rsa'
|
||||
X_str = ['rsa'
|
||||
#, 'asa'
|
||||
, 'kd_values'
|
||||
, 'rd_values']
|
||||
|
||||
X_ssFN = X_stabilityN + X_str + X_foldX_cols
|
||||
|
||||
X_evolFN = ['consurf_score'
|
||||
, 'snap2_score'
|
||||
, 'provean_score']
|
||||
|
|
@ -287,12 +345,14 @@ def setvars(gene,drug):
|
|||
, 'lineage_count_unique'
|
||||
]
|
||||
|
||||
X_genomicFN = X_genomic_mafor+X_genomic_linegae
|
||||
X_genomicFN = X_genomic_mafor + X_genomic_linegae
|
||||
|
||||
#%% Construct numerical and categorical column names
|
||||
# numerical feature names
|
||||
numerical_FN = common_cols_stabiltyN + foldX_cols + X_strFN + X_evolFN + X_genomicFN
|
||||
# numerical_FN = common_cols_stabiltyN + foldX_cols + X_strFN + X_evolFN + X_genomicFN
|
||||
|
||||
numerical_FN = X_ssFN + X_evolFN + X_genomicFN
|
||||
|
||||
#categorical feature names
|
||||
categorical_FN = ['ss_class'
|
||||
# , 'wt_prop_water'
|
||||
|
|
@ -306,9 +366,34 @@ def setvars(gene,drug):
|
|||
, 'polarity_change'
|
||||
, 'water_change'
|
||||
, 'drtype_mode_labels' # beware then you can use it to predict
|
||||
# , 'active_aa_pos' # TODO?
|
||||
#, 'active_aa_pos' # TODO?
|
||||
]
|
||||
###########################################################################
|
||||
#=======================
|
||||
# Masking columns:
|
||||
# (mCSM-lig, mCSM-NA, mCSM-ppi2) values for lig_dist >10
|
||||
#=======================
|
||||
#%% Masking columns
|
||||
# my_df_ml['mutationinformation'][my_df['ligand_distance']>10].value_counts()
|
||||
# my_df_ml.groupby('mutationinformation')['ligand_distance'].apply(lambda x: (x>10)).value_counts()
|
||||
|
||||
# my_df_ml.loc[(my_df_ml['ligand_distance'] > 10), 'ligand_affinity_change'] = 0
|
||||
# (my_df_ml['ligand_affinity_change'] == 0).sum()
|
||||
|
||||
my_df_ml['mutationinformation'][my_df_ml['ligand_distance']>10].value_counts()
|
||||
my_df_ml.groupby('mutationinformation')['ligand_distance'].apply(lambda x: (x>10)).value_counts()
|
||||
my_df_ml.loc[(my_df_ml['ligand_distance'] > 10), cols_to_mask].value_counts()
|
||||
|
||||
# mask the column ligand distance > 10
|
||||
my_df_ml.loc[(my_df_ml['ligand_distance'] > 10), cols_to_mask] = 0
|
||||
(my_df_ml['ligand_affinity_change'] == 0).sum()
|
||||
|
||||
mask_check = my_df_ml[['mutationinformation', 'ligand_distance'] + cols_to_mask]
|
||||
|
||||
# write file for check
|
||||
mask_check.sort_values(by = ['ligand_distance'], ascending = True, inplace = True)
|
||||
mask_check.to_csv(outdir + 'ml/' + gene.lower() + '_mask_check.csv')
|
||||
|
||||
#%% extracting dfs based on numerical, categorical column names
|
||||
#----------------------------------
|
||||
# WITHOUT the target var included
|
||||
|
|
@ -335,12 +420,12 @@ def setvars(gene,drug):
|
|||
|
||||
all_df_wtgt = training_df[numerical_FN + categorical_FN + ['dst_mode']]
|
||||
all_df_wtgt.shape
|
||||
#%%================================================================
|
||||
#%% Apply ML
|
||||
|
||||
#%% Data
|
||||
#%%########################################################################
|
||||
#============
|
||||
# ML data
|
||||
#============
|
||||
#------
|
||||
# X
|
||||
# X: Training and Blind test (BTS)
|
||||
#------
|
||||
X = all_df_wtgt[numerical_FN + categorical_FN] # training data ALL
|
||||
X_bts = blind_test_df[numerical_FN + categorical_FN] # blind test data ALL
|
||||
|
|
@ -352,17 +437,18 @@ def setvars(gene,drug):
|
|||
#------
|
||||
y = all_df_wtgt['dst_mode'] # training data y
|
||||
y_bts = blind_test_df['dst_mode'] # blind data test y
|
||||
|
||||
#Blind test data {same format}
|
||||
#X_bts = blind_test_df[numerical_FN]
|
||||
#X_bts = blind_test_df[numerical_FN + categorical_FN]
|
||||
#y_bts = blind_test_df['dst_mode']
|
||||
|
||||
X_bts_wt = blind_test_df[numerical_FN + ['dst_mode']]
|
||||
|
||||
#X_bts_wt = blind_test_df[numerical_FN + ['dst_mode']]
|
||||
|
||||
# Quick check
|
||||
(X['ligand_affinity_change']==0).sum() == (X['ligand_distance']>10).sum()
|
||||
##############################################################################
|
||||
#(X['ligand_affinity_change']==0).sum() == (X['ligand_distance']>10).sum()
|
||||
for i in range(len(cols_to_mask)):
|
||||
ind = i+1
|
||||
print('\nindex:', i, '\nind:', ind)
|
||||
print('\nMask count check:'
|
||||
, (my_df_ml[cols_to_mask[i]]==0).sum() == (my_df_ml['ligand_distance']>10).sum()
|
||||
)
|
||||
|
||||
print('Original Data\n', Counter(y)
|
||||
, 'Data dim:', X.shape)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue